From fc5784738a5ba6f2b8c49be52b925b7e80c4c0eb Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Tue, 12 Feb 2013 15:54:58 -0500 Subject: [PATCH 01/69] first commit --- .gitignore | 24 ++ README.textile | 78 ++++ config/elasticsearch.yml | 360 ++++++++++++++++++ config/logging.yml | 44 +++ example/primes.sh | 44 +++ pom.xml | 140 +++++++ src/main/assemblies/plugin.xml | 18 + .../plugin/NativeScriptExamplesPlugin.java | 40 ++ .../script/IsPrimeSearchScript.java | 83 ++++ src/main/resources/es-plugin.properties | 1 + .../script/AbstractSearchScriptTests.java | 32 ++ .../script/IsPrimeSearchScriptTests.java | 96 +++++ src/test/resources/log4j.properties | 5 + 13 files changed, 965 insertions(+) create mode 100644 .gitignore create mode 100644 README.textile create mode 100644 config/elasticsearch.yml create mode 100644 config/logging.yml create mode 100755 example/primes.sh create mode 100644 pom.xml create mode 100644 src/main/assemblies/plugin.xml create mode 100644 src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java create mode 100644 src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java create mode 100644 src/main/resources/es-plugin.properties create mode 100644 src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTests.java create mode 100644 src/test/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScriptTests.java create mode 100644 src/test/resources/log4j.properties diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000..7d2a3da4c8d6b --- /dev/null +++ b/.gitignore @@ -0,0 +1,24 @@ +# Temp and test files ignores +work/ +data/ +logs/ +build/ +target/ +.DS_Store + +# IDEA ignores +.idea/ +*.iml + +## eclipse ignores +.project +.classpath +.settings +*/.project +*/.classpath +*/.settings +*/eclipse-build + +## netbeans ignores +nb-configuration.xml +nbactions.xml diff --git a/README.textile b/README.textile new file mode 100644 index 0000000000000..0849ff3ece43f --- /dev/null +++ b/README.textile @@ -0,0 +1,78 @@ +h1. Example of Native Script Plugin for Elasticsearch + +h2. Introduction + +p. This plugin contains several examples of "native script":http://www.elasticsearch.org/guide/reference/modules/scripting.html for Elasticsearch. + +h2. Creating Elasticsearch Plugin + +p. The simplest way to deploy native script is by wrapping it into standard Elasticsearch plugin infrastructure. An Elasticsearch plugin can be written in java and built using maven. A typical plugin source directory looks like this: + +bc.. . +|- pom.xml +|- src + |- main + |- assemblies + | |- plugin.xml + |- java + | |- ... source code ... + |- resources + |- es-plugin.properties + +p. An Elasticsearch plugin can be created by following these six steps. + +* Create pom.xml file in the root directory of your plugin. The "pom.xml":https://github.com/imotov/elasticsearch-native-script-example/blob/master/pom.xml file in this project can be used as a starting point. +* Create source code directories: +** @mkdir -p src/main/assemblies@ +** @mkdir -p src/main/java@ +** @mkdir -p src/main/resources@ +* Download "plugin.xml":https://github.com/imotov/elasticsearch-native-script-example/blob/master/src/main/assemblies/plugin.xml to the @src/main/assemblies@ directory. This file specifies how plugin .zip package should be built. By default, the project artifact with all its dependencies is going to be included in the plugin package. For more information on how to configure the content of the package, see "Maven Assembly Plugin Documentation":http://maven.apache.org/plugins/maven-assembly-plugin/. +* Create main Plugin class in the @src/main/java@ directory. This project is using @org.elasticsearch.examples.nativescript.plugin.NativeScriptExamplesPlugin@ class as an example, so the it has to be saved as @src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java@ + +bc.. package org.elasticsearch.examples.nativescript.plugin; + +import org.elasticsearch.examples.nativescript.script.DividedBoostScoreSearchScript; +import org.elasticsearch.plugins.AbstractPlugin; +import org.elasticsearch.script.ScriptModule; + +public class NativeScriptExamplesPlugin extends AbstractPlugin { + @Override + public String name() { + return "native-script-examples"; + } + + @Override + public String description() { + return "Examples of native script"; + } +} + +p. + +* Create @es-plugin.properties@ file in the @src/main/resources@ directory to point to the Plugin class that was created in the previous step: + +bc. plugin=org.elasticsearch.examples.nativescript.plugin.NativeScriptExamplesPlugin + +* The plugin can be built using @mvn package@ command. The assembled .zip package can be found in the @target/releases/@ directory and deployed to elasticsearch installation using @plugin -install plugin-name -url path/to/plugin/zip/file@. + +h2. Adding Native Scripts + +p. Now that the plugin infrastructure is complete, it's possible to add a native script. + +h3. Is Prime Native Script + +p. One of the example scripts in this project is the "is_prime" script that can be used to check if a field contains a possible prime number. The script accepts two parameters @field@ and @certainty@. The @field@ parameter contains the name of the field that needs to be checked and the @certainty@ parameter specifies a measure of the uncertainty that the caller is willing to tolerate. The script returns @true@ if the field contains a probable prime number and @false@ otherwise. The probability that the number for which the script returned @true@ is prime exceeds (1 - 0.5^certainty). The script can be used in "Script Filter":http://www.elasticsearch.org/guide/reference/query-dsl/script-filter.html as well as a "Script Field":http://www.elasticsearch.org/guide/reference/api/search/script-fields.html. The implementation of the "is_prime" native script and it's factory can be found in the "IsPrimeSearchScript":https://github.com/imotov/elasticsearch-native-script-example/blob/master/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java class. + +p. In order to enable native script creation the plugin has to contain and register a class that implements "NativeScriptFactory":https://github.com/elasticsearch/elasticsearch/blob/master/src/main/java/org/elasticsearch/script/NativeScriptFactory.java. The NativeScriptFactory interface has only one method @newScript(Map params)@. This method accepts a list of parameters that are passed during script creation and returns an ExecutableScript object with an instance of the script. The factory has to be registered in the @onModule(ScriptModule module)@ method of the Plugin. + +bc.. public class NativeScriptExamplesPlugin extends AbstractPlugin { + /* ..... */ + public void onModule(ScriptModule module) { + // Register each script that we defined in this plugin + module.registerScript("is_prime", IsPrimeSearchScript.Factory.class); + } +} + +p. In general native scripts have to implement the interface "ExecutableScript":https://github.com/elasticsearch/elasticsearch/blob/master/src/main/java/org/elasticsearch/script/SearchScript.java, but if they are used in search, they have to also implement the "SearchScript":https://github.com/elasticsearch/elasticsearch/blob/master/src/main/java/org/elasticsearch/script/SearchScript.java interface. The SearchScript interface is quite complex, so in order to simplify implementation, the native script can simply extend the "AbstractSearchScript":https://github.com/elasticsearch/elasticsearch/blob/master/src/main/java/org/elasticsearch/script/AbstractSearchScript.java class instead. The AbstractSearchScript has only one abstract method @run()@. During search Elasticsearch calls this method for every single record in the search result. As in case of non-native script, the content of the current record can be accessed using DocLookup (@doc()@ method), FieldsLookup (@fields()@ method), SourceLookup (@source()@ method). + + diff --git a/config/elasticsearch.yml b/config/elasticsearch.yml new file mode 100644 index 0000000000000..14ab8ea13cc11 --- /dev/null +++ b/config/elasticsearch.yml @@ -0,0 +1,360 @@ +##################### ElasticSearch Configuration Example ##################### + +# This file contains an overview of various configuration settings, +# targeted at operations staff. Application developers should +# consult the guide at . +# +# The installation procedure is covered at +# . +# +# ElasticSearch comes with reasonable defaults for most settings, +# so you can try it out without bothering with configuration. +# +# Most of the time, these defaults are just fine for running a production +# cluster. If you're fine-tuning your cluster, or wondering about the +# effect of certain configuration option, please _do ask_ on the +# mailing list or IRC channel [http://elasticsearch.org/community]. + +# Any element in the configuration can be replaced with environment variables +# by placing them in ${...} notation. For example: +# +# node.rack: ${RACK_ENV_VAR} + +# See +# for information on supported formats and syntax for the configuration file. + + +################################### Cluster ################################### + +# Cluster name identifies your cluster for auto-discovery. If you're running +# multiple clusters on the same network, make sure you're using unique names. +# +# cluster.name: elasticsearch + + +#################################### Node ##################################### + +# Node names are generated dynamically on startup, so you're relieved +# from configuring them manually. You can tie this node to a specific name: +# +# node.name: "Franz Kafka" + +# Every node can be configured to allow or deny being eligible as the master, +# and to allow or deny to store the data. +# +# Allow this node to be eligible as a master node (enabled by default): +# +# node.master: true +# +# Allow this node to store data (enabled by default): +# +# node.data: true + +# You can exploit these settings to design advanced cluster topologies. +# +# 1. You want this node to never become a master node, only to hold data. +# This will be the "workhorse" of your cluster. +# +# node.master: false +# node.data: true +# +# 2. You want this node to only serve as a master: to not store any data and +# to have free resources. This will be the "coordinator" of your cluster. +# +# node.master: true +# node.data: false +# +# 3. You want this node to be neither master nor data node, but +# to act as a "search load balancer" (fetching data from nodes, +# aggregating results, etc.) +# +# node.master: false +# node.data: false + +# Use the Cluster Health API [http://localhost:9200/_cluster/health], the +# Node Info API [http://localhost:9200/_cluster/nodes] or GUI tools +# such as and +# to inspect the cluster state. + +# A node can have generic attributes associated with it, which can later be used +# for customized shard allocation filtering, or allocation awareness. An attribute +# is a simple key value pair, similar to node.key: value, here is an example: +# +# node.rack: rack314 + +# By default, multiple nodes are allowed to start from the same installation location +# to disable it, set the following: +# node.max_local_storage_nodes: 1 + + +#################################### Index #################################### + +# You can set a number of options (such as shard/replica options, mapping +# or analyzer definitions, translog settings, ...) for indices globally, +# in this file. +# +# Note, that it makes more sense to configure index settings specifically for +# a certain index, either when creating it or by using the index templates API. +# +# See and +# +# for more information. + +# Set the number of shards (splits) of an index (5 by default): +# +# index.number_of_shards: 5 + +# Set the number of replicas (additional copies) of an index (1 by default): +# +# index.number_of_replicas: 1 + +# Note, that for development on a local machine, with small indices, it usually +# makes sense to "disable" the distributed features: +# +# index.number_of_shards: 1 +# index.number_of_replicas: 0 + +# These settings directly affect the performance of index and search operations +# in your cluster. Assuming you have enough machines to hold shards and +# replicas, the rule of thumb is: +# +# 1. Having more *shards* enhances the _indexing_ performance and allows to +# _distribute_ a big index across machines. +# 2. Having more *replicas* enhances the _search_ performance and improves the +# cluster _availability_. +# +# The "number_of_shards" is a one-time setting for an index. +# +# The "number_of_replicas" can be increased or decreased anytime, +# by using the Index Update Settings API. +# +# ElasticSearch takes care about load balancing, relocating, gathering the +# results from nodes, etc. Experiment with different settings to fine-tune +# your setup. + +# Use the Index Status API () to inspect +# the index status. + + +#################################### Paths #################################### + +# Path to directory containing configuration (this file and logging.yml): +# +# path.conf: /path/to/conf + +# Path to directory where to store index data allocated for this node. +# +# path.data: /path/to/data +# +# Can optionally include more than one location, causing data to be striped across +# the locations (a la RAID 0) on a file level, favouring locations with most free +# space on creation. For example: +# +# path.data: /path/to/data1,/path/to/data2 + +# Path to temporary files: +# +# path.work: /path/to/work + +# Path to log files: +# +# path.logs: /path/to/logs + +# Path to where plugins are installed: +# +# path.plugins: /path/to/plugins + + +#################################### Plugin ################################### + +# If a plugin listed here is not installed for current node, the node will not start. +# +# plugin.mandatory: mapper-attachments,lang-groovy + + +################################### Memory #################################### + +# ElasticSearch performs poorly when JVM starts swapping: you should ensure that +# it _never_ swaps. +# +# Set this property to true to lock the memory: +# +# bootstrap.mlockall: true + +# Make sure that the ES_MIN_MEM and ES_MAX_MEM environment variables are set +# to the same value, and that the machine has enough memory to allocate +# for ElasticSearch, leaving enough memory for the operating system itself. +# +# You should also make sure that the ElasticSearch process is allowed to lock +# the memory, eg. by using `ulimit -l unlimited`. + + +############################## Network And HTTP ############################### + +# ElasticSearch, by default, binds itself to the 0.0.0.0 address, and listens +# on port [9200-9300] for HTTP traffic and on port [9300-9400] for node-to-node +# communication. (the range means that if the port is busy, it will automatically +# try the next port). + +# Set the bind address specifically (IPv4 or IPv6): +# +# network.bind_host: 192.168.0.1 + +# Set the address other nodes will use to communicate with this node. If not +# set, it is automatically derived. It must point to an actual IP address. +# +# network.publish_host: 192.168.0.1 + +# Set both 'bind_host' and 'publish_host': +# +# network.host: 192.168.0.1 + +# Set a custom port for the node to node communication (9300 by default): +# +# transport.tcp.port: 9300 + +# Enable compression for all communication between nodes (disabled by default): +# +# transport.tcp.compress: true + +# Set a custom port to listen for HTTP traffic: +# +# http.port: 9200 + +# Set a custom allowed content length: +# +# http.max_content_length: 100mb + +# Disable HTTP completely: +# +# http.enabled: false + + +################################### Gateway ################################### + +# The gateway allows for persisting the cluster state between full cluster +# restarts. Every change to the state (such as adding an index) will be stored +# in the gateway, and when the cluster starts up for the first time, +# it will read its state from the gateway. + +# There are several types of gateway implementations. For more information, +# see . + +# The default gateway type is the "local" gateway (recommended): +# +# gateway.type: local + +# Settings below control how and when to start the initial recovery process on +# a full cluster restart (to reuse as much local data as possible when using shared +# gateway). + +# Allow recovery process after N nodes in a cluster are up: +# +# gateway.recover_after_nodes: 1 + +# Set the timeout to initiate the recovery process, once the N nodes +# from previous setting are up (accepts time value): +# +# gateway.recover_after_time: 5m + +# Set how many nodes are expected in this cluster. Once these N nodes +# are up (and recover_after_nodes is met), begin recovery process immediately +# (without waiting for recover_after_time to expire): +# +# gateway.expected_nodes: 2 + + +############################# Recovery Throttling ############################# + +# These settings allow to control the process of shards allocation between +# nodes during initial recovery, replica allocation, rebalancing, +# or when adding and removing nodes. + +# Set the number of concurrent recoveries happening on a node: +# +# 1. During the initial recovery +# +# cluster.routing.allocation.node_initial_primaries_recoveries: 4 +# +# 2. During adding/removing nodes, rebalancing, etc +# +# cluster.routing.allocation.node_concurrent_recoveries: 2 + +# Set to throttle throughput when recovering (eg. 100mb, by default unlimited): +# +# indices.recovery.max_size_per_sec: 0 + +# Set to limit the number of open concurrent streams when +# recovering a shard from a peer: +# +# indices.recovery.concurrent_streams: 5 + + +################################## Discovery ################################## + +# Discovery infrastructure ensures nodes can be found within a cluster +# and master node is elected. Multicast discovery is the default. + +# Set to ensure a node sees N other master eligible nodes to be considered +# operational within the cluster. Set this option to a higher value (2-4) +# for large clusters (>3 nodes): +# +# discovery.zen.minimum_master_nodes: 1 + +# Set the time to wait for ping responses from other nodes when discovering. +# Set this option to a higher value on a slow or congested network +# to minimize discovery failures: +# +# discovery.zen.ping.timeout: 3s + +# See +# for more information. + +# Unicast discovery allows to explicitly control which nodes will be used +# to discover the cluster. It can be used when multicast is not present, +# or to restrict the cluster communication-wise. +# +# 1. Disable multicast discovery (enabled by default): +# +# discovery.zen.ping.multicast.enabled: false +# +# 2. Configure an initial list of master nodes in the cluster +# to perform discovery when new nodes (master or data) are started: +# +# discovery.zen.ping.unicast.hosts: ["host1", "host2:port", "host3[portX-portY]"] + +# EC2 discovery allows to use AWS EC2 API in order to perform discovery. +# +# You have to install the cloud-aws plugin for enabling the EC2 discovery. +# +# See +# for more information. +# +# See +# for a step-by-step tutorial. + + +################################## Slow Log ################################## + +# Shard level query and fetch threshold logging. + +#index.search.slowlog.level: TRACE +#index.search.slowlog.threshold.query.warn: 10s +#index.search.slowlog.threshold.query.info: 5s +#index.search.slowlog.threshold.query.debug: 2s +#index.search.slowlog.threshold.query.trace: 500ms + +#index.search.slowlog.threshold.fetch.warn: 1s +#index.search.slowlog.threshold.fetch.info: 800ms +#index.search.slowlog.threshold.fetch.debug: 500ms +#index.search.slowlog.threshold.fetch.trace: 200ms + +################################## GC Logging ################################ + +#monitor.jvm.gc.ParNew.warn: 1000ms +#monitor.jvm.gc.ParNew.info: 700ms +#monitor.jvm.gc.ParNew.debug: 400ms + +#monitor.jvm.gc.ConcurrentMarkSweep.warn: 10s +#monitor.jvm.gc.ConcurrentMarkSweep.info: 5s +#monitor.jvm.gc.ConcurrentMarkSweep.debug: 2s diff --git a/config/logging.yml b/config/logging.yml new file mode 100644 index 0000000000000..5f8c973d28214 --- /dev/null +++ b/config/logging.yml @@ -0,0 +1,44 @@ +rootLogger: INFO, console, file +logger: + # log action execution errors for easier debugging + action: DEBUG + # reduce the logging for aws, too much is logged under the default INFO + com.amazonaws: WARN + + # gateway + #gateway: DEBUG + #index.gateway: DEBUG + + # peer shard recovery + #indices.recovery: DEBUG + + # discovery + #discovery: TRACE + + index.search.slowlog: TRACE, index_search_slow_log_file + +additivity: + index.search.slowlog: false + +appender: + console: + type: console + layout: + type: consolePattern + conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n" + + file: + type: dailyRollingFile + file: ${path.logs}/${cluster.name}.log + datePattern: "'.'yyyy-MM-dd" + layout: + type: pattern + conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n" + + index_search_slow_log_file: + type: dailyRollingFile + file: ${path.logs}/${cluster.name}_index_search_slowlog.log + datePattern: "'.'yyyy-MM-dd" + layout: + type: pattern + conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n" diff --git a/example/primes.sh b/example/primes.sh new file mode 100755 index 0000000000000..eb878900e5803 --- /dev/null +++ b/example/primes.sh @@ -0,0 +1,44 @@ +curl -XDELETE "http://localhost:9200/test" +echo +curl -XPUT "http://localhost:9200/test/" -d '{ + "settings": { + "index.number_of_shards": 1, + "index.nubmer_of_replicas": 0 + }, + "mappings": { + "type1": { + "properties": { + "name": { + "type": "string" + }, + "number": { + "type": "integer" + } + } + } + } +}' +echo +for i in {0..100}; do curl -XPUT "localhost:9200/test/type1/$i" -d "{\"name\":\"rec $i\", \"number\":$i}"; done +curl -XPOST "http://localhost:9200/test/_refresh" +echo +curl "localhost:9200/test/type1/_search?pretty=true" -d '{ + "query": { + "filtered": { + "query": { + "match_all": {} + }, + "filter": { + "script": { + "script": "is_prime", + "lang": "native", + "params": { + "field": "number" + } + } + } + } + } +} +' + diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000000000..bd40655f9f4c0 --- /dev/null +++ b/pom.xml @@ -0,0 +1,140 @@ + + + + + + + elasticsearch-native-script-example + 4.0.0 + org.elasticsearch + elasticsearch-native-script-example + 1.0.0-SNAPSHOT + jar + ElasticSearch Plugin with Native Script Examples + 2012 + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + scm:git:git@github.com:imotov/elasticsearch-native-script-example.git + scm:git:git@github.com:imotov/elasticsearch-native-script-example.git + http://github.com/imotov/elasticsearch-native-script-example + + + + + + + 0.20.4 + + + + + + + + + org.elasticsearch + elasticsearch + ${elasticsearch.version} + compile + + + + log4j + log4j + 1.2.17 + compile + + + + org.slf4j + slf4j-api + 1.6.2 + compile + true + + + + org.testng + testng + 6.8 + test + + + org.hamcrest + hamcrest-core + + + junit + junit + + + + + + org.hamcrest + hamcrest-all + 1.3 + test + + + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 2.3.2 + + 1.6 + 1.6 + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.12.3 + + + **/*Tests.java + + + + + + + + maven-assembly-plugin + 2.3 + + ${project.build.directory}/releases/ + + ${basedir}/src/main/assemblies/plugin.xml + + + + + package + + single + + + + + + + \ No newline at end of file diff --git a/src/main/assemblies/plugin.xml b/src/main/assemblies/plugin.xml new file mode 100644 index 0000000000000..8e8e42e7c0544 --- /dev/null +++ b/src/main/assemblies/plugin.xml @@ -0,0 +1,18 @@ + + + plugin + + zip + + false + + + / + true + true + + org.elasticsearch:elasticsearch + + + + \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java b/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java new file mode 100644 index 0000000000000..254459c358af2 --- /dev/null +++ b/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java @@ -0,0 +1,40 @@ +package org.elasticsearch.examples.nativescript.plugin; + +import org.elasticsearch.examples.nativescript.script.IsPrimeSearchScript; +import org.elasticsearch.plugins.AbstractPlugin; +import org.elasticsearch.script.ScriptModule; + +/** + * This class is instantiated when Elasticsearch loads the plugin for the + * first time. If you change the name of this plugin, make sure to update + * src/main/resources/es-plugin.properties file that points to this class. + */ +public class NativeScriptExamplesPlugin extends AbstractPlugin { + + /** + * The name of the plugin. + *

+ * This name will be used by elasticsearch in the log file to refer to this plugin. + * + * @return plugin name. + */ + @Override + public String name() { + return "native-script-example"; + } + + /** + * The description of the plugin. + * + * @return plugin description + */ + @Override + public String description() { + return "Examples of native script"; + } + + public void onModule(ScriptModule module) { + // Register each script that we defined in this plugin + module.registerScript("is_prime", IsPrimeSearchScript.Factory.class); + } +} diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java new file mode 100644 index 0000000000000..e817f4d691c5a --- /dev/null +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java @@ -0,0 +1,83 @@ +package org.elasticsearch.examples.nativescript.script; + +import org.elasticsearch.ElasticSearchIllegalArgumentException; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.index.field.data.DocFieldData; +import org.elasticsearch.script.AbstractSearchScript; +import org.elasticsearch.script.ExecutableScript; +import org.elasticsearch.script.NativeScriptFactory; + +import java.math.BigInteger; +import java.util.Map; + +/** + * Implementation of the native script that checks that the field exists and contains a prime number. + *

+ * The native script has to implement {@link org.elasticsearch.script.SearchScript} interface. But the + * {@link org.elasticsearch.script.AbstractSearchScript} class can be used to simplify the implementation. + */ +public class IsPrimeSearchScript extends AbstractSearchScript { + + /** + * Native scripts are build using factories that are registered in the + * {@link org.elasticsearch.examples.nativescript.plugin.NativeScriptExamplesPlugin#onModule(org.elasticsearch.script.ScriptModule)} + * method when plugin is loaded. + */ + public static class Factory implements NativeScriptFactory { + + /** + * This method is called for every search on every shard. + * + * @param params list of script parameters passed with the query + * @return new native script + */ + @Override + public ExecutableScript newScript(@Nullable Map params) { + // Example of a mandatory string parameter + // The XContentMapValues helper class can be used to simplify parameter parsing + String fieldName = XContentMapValues.nodeStringValue(params.get("field"), null); + if (fieldName == null) { + throw new ElasticSearchIllegalArgumentException("Missing the field parameter"); + } + + // Example of an optional integer parameter + int certainty = params == null ? 10 : XContentMapValues.nodeIntegerValue(params.get("certainty"), 10); + return new IsPrimeSearchScript(fieldName, certainty); + } + } + + private final String fieldName; + + private final int certainty; + + /** + * Factory creates this script on every + * + * @param fieldName the name of the field that should be checked + * @param certainty the required certainty for the number to be prime + */ + private IsPrimeSearchScript(String fieldName, int certainty) { + this.fieldName = fieldName; + this.certainty = certainty; + } + + @Override + public Object run() { + // First we get field using doc lookup + DocFieldData field = doc().field(fieldName); + + // Check if field exists + if (field != null && !field.isEmpty()) { + try { + // Try to parse it as an integer + BigInteger bigInteger = new BigInteger(field.stringValue()); + // Check if it's prime + return bigInteger.isProbablePrime(certainty); + } catch (NumberFormatException ex) { + return false; + } + } + return false; + } +} diff --git a/src/main/resources/es-plugin.properties b/src/main/resources/es-plugin.properties new file mode 100644 index 0000000000000..b24fbc3db4d67 --- /dev/null +++ b/src/main/resources/es-plugin.properties @@ -0,0 +1 @@ +plugin=org.elasticsearch.examples.nativescript.plugin.NativeScriptExamplesPlugin diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTests.java new file mode 100644 index 0000000000000..9a59d0ce3ef8b --- /dev/null +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTests.java @@ -0,0 +1,32 @@ +package org.elasticsearch.examples.nativescript.script; + +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.node.Node; +import org.elasticsearch.node.NodeBuilder; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; + +/** + */ +public class AbstractSearchScriptTests { + protected Node node; + + @BeforeMethod + public void startNode() { + Settings settings = ImmutableSettings.settingsBuilder() + .put("gateway.type", "none") + .put("index.number_of_shards", 1) + .put("index.number_of_replicas", 0) + .build(); + + node = NodeBuilder.nodeBuilder().settings(settings).node(); + } + + @AfterMethod + public void closeNode() { + if (node != null) { + node.close(); + } + } +} diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScriptTests.java new file mode 100644 index 0000000000000..d0593ce7975de --- /dev/null +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScriptTests.java @@ -0,0 +1,96 @@ +package org.elasticsearch.examples.nativescript.script; + +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.indices.IndexMissingException; +import org.elasticsearch.search.sort.SortOrder; +import org.testng.annotations.Test; + +import java.util.Arrays; + +import static org.elasticsearch.index.query.FilterBuilders.scriptFilter; +import static org.elasticsearch.index.query.QueryBuilders.filteredQuery; +import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; + +/** + */ +public class IsPrimeSearchScriptTests extends AbstractSearchScriptTests { + + public static int[] PRIMES_10 = new int[]{2, 3, 5, 7, 11, 13, 17, 19, 23, 29}; + + @Test + public void testIsPrimeScript() throws Exception { + // Delete the old index + try { + node.client().admin().indices().prepareDelete("test").execute().actionGet(); + } catch (IndexMissingException ex) { + // Ignore + } + + // Create a new index + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties") + .startObject("name").field("type", "string").endObject() + .startObject("number").field("type", "integer").endObject() + .endObject().endObject().endObject() + .string(); + node.client().admin().indices().prepareCreate("test") + .addMapping("type", mapping) + .execute().actionGet(); + + // Index 100 records (0..99) + for (int i = 0; i < 100; i++) { + node.client().prepareIndex("test", "type", Integer.toString(i)) + .setSource(XContentFactory.jsonBuilder().startObject() + .field("name", "rec " + i) + .field("number", i) + .endObject()) + .execute().actionGet(); + } + // Index a few records with empty number + for (int i = 100; i < 105; i++) { + node.client().prepareIndex("test", "type", Integer.toString(i)) + .setSource(XContentFactory.jsonBuilder().startObject() + .field("name", "rec " + i) + .endObject()) + .execute().actionGet(); + } + node.client().admin().indices().prepareRefresh("test").execute().actionGet(); + + // Retrieve first 10 prime records + SearchResponse searchResponse = node.client().prepareSearch("test") + .setQuery(filteredQuery(matchAllQuery(), + scriptFilter("is_prime").lang("native").addParam("field", "number"))) + .addField("name") + .setSize(10) + .addSort("number", SortOrder.ASC) + .execute().actionGet(); + assertThat(Arrays.toString(searchResponse.shardFailures()), searchResponse.failedShards(), equalTo(0)); + + // There should be 25 prime numbers between 0 and 100 + assertThat(searchResponse.hits().totalHits(), equalTo(25l)); + + // Verify that they are indeed prime numbers + for (int i = 0; i < 10; i++) { + assertThat(searchResponse.hits().getAt(i).field("name").getValue().toString(), equalTo("rec " + PRIMES_10[i])); + } + + // Check certainty parameter - with certainty == 0, it should return all numbers, but only if numbers are present + searchResponse = node.client().prepareSearch("test") + .setQuery(filteredQuery(matchAllQuery(), + scriptFilter("is_prime").lang("native").addParam("field", "number").addParam("certainty", 0))) + .addField("name") + .setSize(10) + .addSort("number", SortOrder.ASC) + .execute().actionGet(); + assertThat(Arrays.toString(searchResponse.shardFailures()), searchResponse.failedShards(), equalTo(0)); + // With certainty 0 no check is done so it should return all numbers + assertThat(searchResponse.hits().totalHits(), equalTo(100l)); + + + } + + +} diff --git a/src/test/resources/log4j.properties b/src/test/resources/log4j.properties new file mode 100644 index 0000000000000..497c97f995974 --- /dev/null +++ b/src/test/resources/log4j.properties @@ -0,0 +1,5 @@ +log4j.rootLogger=INFO, out + +log4j.appender.out=org.apache.log4j.ConsoleAppender +log4j.appender.out.layout=org.apache.log4j.PatternLayout +log4j.appender.out.layout.conversionPattern=[%d{ISO8601}][%-5p][%-25c] %m%n From 4dc7b463650f66a3238e153b3fbdb3752a456ce4 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Tue, 12 Feb 2013 16:05:37 -0500 Subject: [PATCH 02/69] Fix link in README --- README.textile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.textile b/README.textile index 0849ff3ece43f..0713bad50aa46 100644 --- a/README.textile +++ b/README.textile @@ -73,6 +73,6 @@ bc.. public class NativeScriptExamplesPlugin extends AbstractPlugin { } } -p. In general native scripts have to implement the interface "ExecutableScript":https://github.com/elasticsearch/elasticsearch/blob/master/src/main/java/org/elasticsearch/script/SearchScript.java, but if they are used in search, they have to also implement the "SearchScript":https://github.com/elasticsearch/elasticsearch/blob/master/src/main/java/org/elasticsearch/script/SearchScript.java interface. The SearchScript interface is quite complex, so in order to simplify implementation, the native script can simply extend the "AbstractSearchScript":https://github.com/elasticsearch/elasticsearch/blob/master/src/main/java/org/elasticsearch/script/AbstractSearchScript.java class instead. The AbstractSearchScript has only one abstract method @run()@. During search Elasticsearch calls this method for every single record in the search result. As in case of non-native script, the content of the current record can be accessed using DocLookup (@doc()@ method), FieldsLookup (@fields()@ method), SourceLookup (@source()@ method). +p. In general native scripts have to implement the interface "ExecutableScript":https://github.com/elasticsearch/elasticsearch/blob/master/src/main/java/org/elasticsearch/script/ExecutableScript.java, but if they are used in search, they have to also implement the "SearchScript":https://github.com/elasticsearch/elasticsearch/blob/master/src/main/java/org/elasticsearch/script/SearchScript.java interface. The SearchScript interface is quite complex, so in order to simplify implementation, the native script can simply extend the "AbstractSearchScript":https://github.com/elasticsearch/elasticsearch/blob/master/src/main/java/org/elasticsearch/script/AbstractSearchScript.java class instead. The AbstractSearchScript has only one abstract method @run()@. During search Elasticsearch calls this method for every single record in the search result. As in case of non-native script, the content of the current record can be accessed using DocLookup (@doc()@ method), FieldsLookup (@fields()@ method), SourceLookup (@source()@ method). From 4e8d053dcbd4550a7c48c19c5b7303b5c039777e Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Sun, 17 Feb 2013 18:13:02 -0500 Subject: [PATCH 03/69] Upgrade to elasticsearch 0.20.5 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index bd40655f9f4c0..9f02929cc5054 100644 --- a/pom.xml +++ b/pom.xml @@ -31,7 +31,7 @@ - 0.20.4 + 0.20.5 From b15f57f3042c8fee9a957ef4e917a469ac66c7e3 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Sun, 17 Feb 2013 18:16:39 -0500 Subject: [PATCH 04/69] Add check for null parameters --- .../examples/nativescript/script/IsPrimeSearchScript.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java index e817f4d691c5a..38db8ec57f5b8 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java @@ -36,7 +36,7 @@ public static class Factory implements NativeScriptFactory { public ExecutableScript newScript(@Nullable Map params) { // Example of a mandatory string parameter // The XContentMapValues helper class can be used to simplify parameter parsing - String fieldName = XContentMapValues.nodeStringValue(params.get("field"), null); + String fieldName = params == null ? null : XContentMapValues.nodeStringValue(params.get("field"), null); if (fieldName == null) { throw new ElasticSearchIllegalArgumentException("Missing the field parameter"); } From ecb5cc5b63e6dc503231f9d98a161935cd43f82c Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Sun, 17 Feb 2013 18:18:24 -0500 Subject: [PATCH 05/69] Add an example of injecting client into native script --- .../plugin/NativeScriptExamplesPlugin.java | 2 + .../nativescript/script/LookupScript.java | 143 ++++++++++++++++++ .../script/LookupScriptTests.java | 111 ++++++++++++++ 3 files changed, 256 insertions(+) create mode 100644 src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java create mode 100644 src/test/java/org/elasticsearch/examples/nativescript/script/LookupScriptTests.java diff --git a/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java b/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java index 254459c358af2..1f304cc06b13d 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java @@ -1,6 +1,7 @@ package org.elasticsearch.examples.nativescript.plugin; import org.elasticsearch.examples.nativescript.script.IsPrimeSearchScript; +import org.elasticsearch.examples.nativescript.script.LookupScript; import org.elasticsearch.plugins.AbstractPlugin; import org.elasticsearch.script.ScriptModule; @@ -36,5 +37,6 @@ public String description() { public void onModule(ScriptModule module) { // Register each script that we defined in this plugin module.registerScript("is_prime", IsPrimeSearchScript.Factory.class); + module.registerScript("lookup", LookupScript.Factory.class); } } diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java new file mode 100644 index 0000000000000..9090eacb0049c --- /dev/null +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java @@ -0,0 +1,143 @@ +package org.elasticsearch.examples.nativescript.script; + +import org.elasticsearch.ElasticSearchIllegalArgumentException; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.cache.Cache; +import org.elasticsearch.common.cache.CacheBuilder; +import org.elasticsearch.common.collect.ImmutableMap; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.index.field.data.DocFieldData; +import org.elasticsearch.node.Node; +import org.elasticsearch.script.AbstractSearchScript; +import org.elasticsearch.script.ExecutableScript; +import org.elasticsearch.script.NativeScriptFactory; +import org.elasticsearch.script.ScriptException; + +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; + +/** + */ +public class LookupScript extends AbstractSearchScript { + + /** + * Native scripts are build using factories that are registered in the + * {@link org.elasticsearch.examples.nativescript.plugin.NativeScriptExamplesPlugin#onModule(org.elasticsearch.script.ScriptModule)} + * method when plugin is loaded. + */ + public static class Factory implements NativeScriptFactory { + + private final Node node; + + private final Cache, Map> cache; + + /** + * This constructor will be called by guice during initialization + * + * @param node injecting the reference to current node to get access to node's client + */ + @SuppressWarnings("unchecked") + @Inject + public Factory(Node node, Settings settings) { + // Node is not fully initialized here + // All we can do is save a reference to it for future use + this.node = node; + + // Setup lookup cache + ByteSizeValue size = settings.getAsBytesSize("examples.nativescript.lookup.size", null); + TimeValue expire = settings.getAsTime("expire", null); + CacheBuilder cacheBuilder = CacheBuilder.newBuilder(); + if (size != null) { + cacheBuilder.maximumSize(size.bytes()); + } + if (expire != null) { + cacheBuilder.expireAfterAccess(expire.nanos(), TimeUnit.NANOSECONDS); + } + cache = cacheBuilder.build(); + } + + /** + * This method is called for every search on every shard. + * + * @param params list of script parameters passed with the query + * @return new native script + */ + @Override + public ExecutableScript newScript(@Nullable Map params) { + if (params == null) { + throw new ElasticSearchIllegalArgumentException("Missing script parameters"); + } + String lookupIndex = XContentMapValues.nodeStringValue(params.get("lookup_index"), null); + if (lookupIndex == null) { + throw new ElasticSearchIllegalArgumentException("Missing the index parameter"); + } + String lookupType = XContentMapValues.nodeStringValue(params.get("lookup_type"), null); + if (lookupType == null) { + throw new ElasticSearchIllegalArgumentException("Missing the index parameter"); + } + String field = XContentMapValues.nodeStringValue(params.get("field"), null); + if (field == null) { + throw new ElasticSearchIllegalArgumentException("Missing the field parameter"); + } + return new LookupScript(node.client(), cache, lookupIndex, lookupType, field); + } + } + + private final String lookupIndex; + private final String lookupType; + private final String field; + + private final Client client; + private final Cache, Map> cache; + + private static final Map EMPTY_MAP = ImmutableMap.of(); + + private LookupScript(Client client, Cache, Map> cache, String lookupIndex, String lookupType, String field) { + this.client = client; + this.lookupIndex = lookupIndex; + this.lookupType = lookupType; + this.field = field; + this.cache = cache; + } + + @Override + public Object run() { + // First we get field using doc lookup + DocFieldData fieldData = doc().field(field); + + // This is not very efficient + // Check if field exists + if (fieldData != null && !fieldData.isEmpty()) { + final String fieldValue = fieldData.stringValue(); + if (fieldValue != null) { + try { + return cache.get(new Tuple(lookupIndex + "/" + lookupType, fieldValue), new Callable>() { + @Override + public Map call() throws Exception { + // This is not very efficient of doing this, but it demonstrates using injected client + // for record lookup + GetResponse response = client.prepareGet(lookupIndex, lookupType, fieldValue).execute().actionGet(); + if (response.exists()) { + return response.getSource(); + } + return EMPTY_MAP; + } + }); + + } catch (ExecutionException ex) { + throw new ScriptException("Lookup failure ", ex); + } + } + } + return null; + } +} diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/LookupScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/LookupScriptTests.java new file mode 100644 index 0000000000000..953cbbd273a10 --- /dev/null +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/LookupScriptTests.java @@ -0,0 +1,111 @@ +package org.elasticsearch.examples.nativescript.script; + +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.collect.MapBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.indices.IndexMissingException; +import org.elasticsearch.search.sort.SortOrder; +import org.testng.annotations.Test; + +import java.util.Arrays; +import java.util.Map; + +import static org.elasticsearch.index.query.QueryBuilders.matchQuery; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; + +/** + */ +public class LookupScriptTests extends AbstractSearchScriptTests { + + @SuppressWarnings("unchecked") + @Test + public void testLookup() throws Exception { + // Delete the old index + try { + node.client().admin().indices().prepareDelete("test").execute().actionGet(); + node.client().admin().indices().prepareDelete("lookup").execute().actionGet(); + } catch (IndexMissingException ex) { + // Ignore + } + + // Create a new lookup index + String lookup_mapping = XContentFactory.jsonBuilder().startObject().startObject("state") + .startObject("properties") + .startObject("name").field("type", "string").endObject() + .startObject("capital").field("type", "string").endObject() + .startObject("nickname").field("type", "string").endObject() + .endObject().endObject().endObject() + .string(); + node.client().admin().indices().prepareCreate("lookup") + .addMapping("type", lookup_mapping) + .execute().actionGet(); + + // Create a new test index + String test_mapping = XContentFactory.jsonBuilder().startObject().startObject("city") + .startObject("properties") + .startObject("city").field("type", "string").endObject() + .startObject("state").field("type", "string").field("index", "not_analyzed").endObject() + .startObject("population").field("type", "integer").endObject() + .endObject().endObject().endObject() + .string(); + node.client().admin().indices().prepareCreate("test") + .addMapping("city", test_mapping) + .execute().actionGet(); + + // Index Lookup records: + node.client().prepareIndex("lookup", "state", "CT").setSource("name", "Connecticut", "capital", "Hartford", "nickname", "Constitution State").execute().actionGet(); + node.client().prepareIndex("lookup", "state", "ME").setSource("name", "Maine", "capital", "Augusta", "nickname", "Lumber State").execute().actionGet(); + node.client().prepareIndex("lookup", "state", "MA").setSource("name", "Massachusetts", "capital", "Boston", "nickname", "Bay State").execute().actionGet(); + node.client().prepareIndex("lookup", "state", "NH").setSource("name", "New Hampshire", "capital", "Concord", "nickname", "Granite State").execute().actionGet(); + node.client().prepareIndex("lookup", "state", "RI").setSource("name", "Rhode Island", "capital", "Providence", "nickname", "Little Rhody").execute().actionGet(); + node.client().prepareIndex("lookup", "state", "VT").setSource("name", "Vermont", "capital", "Montpelier", "nickname", "Green Mountain State").execute().actionGet(); + + // Index main records + node.client().prepareIndex("test", "city", "1").setSource("city", "Cambridge", "state", "MA", "population", 105162).execute().actionGet(); + node.client().prepareIndex("test", "city", "2").setSource("city", "South Burlington", "state", "VT", "population", 17904).execute().actionGet(); + node.client().prepareIndex("test", "city", "3").setSource("city", "South Portland", "state", "ME", "population", 25002).execute().actionGet(); + node.client().prepareIndex("test", "city", "4").setSource("city", "Essex", "state", "VT", "population", 19587).execute().actionGet(); + node.client().prepareIndex("test", "city", "5").setSource("city", "Portland", "state", "ME", "population", 66194).execute().actionGet(); + node.client().prepareIndex("test", "city", "6").setSource("city", "Burlington", "state", "VT", "population", 42417).execute().actionGet(); + node.client().prepareIndex("test", "city", "7").setSource("city", "Stamford", "state", "CT", "population", 122643).execute().actionGet(); + node.client().prepareIndex("test", "city", "8").setSource("city", "Colchester", "state", "VT", "population", 17067).execute().actionGet(); + node.client().prepareIndex("test", "city", "9").setSource("city", "Concord", "state", "NH", "population", 42695).execute().actionGet(); + node.client().prepareIndex("test", "city", "10").setSource("city", "Boston", "state", "MA", "population", 617594).execute().actionGet(); + + node.client().admin().indices().prepareRefresh("lookup", "test").execute().actionGet(); + + // Script parameters + Map params = MapBuilder.newMapBuilder() + .put("lookup_index", "lookup") + .put("lookup_type", "state") + .put("field", "state") + .map(); + + + // Find smallest city with word + SearchResponse searchResponse = node.client().prepareSearch("test") + .setTypes("city") + .setQuery(matchQuery("city", "south burlington")) + .addField("city") + .addScriptField("state_info", "native", "lookup", params) + .setSize(10) + .addSort("population", SortOrder.DESC) + .execute().actionGet(); + assertThat(Arrays.toString(searchResponse.shardFailures()), searchResponse.failedShards(), equalTo(0)); + + // There should be 3 cities + assertThat(searchResponse.hits().totalHits(), equalTo(3L)); + + assertThat(searchResponse.hits().getAt(0).field("city").getValue().toString(), equalTo("Burlington")); + assertThat(((Map) searchResponse.hits().getAt(0).field("state_info").getValue()).get("name").toString(), equalTo("Vermont")); + + assertThat(searchResponse.hits().getAt(1).field("city").getValue().toString(), equalTo("South Portland")); + assertThat(((Map) searchResponse.hits().getAt(1).field("state_info").getValue()).get("name").toString(), equalTo("Maine")); + + assertThat(searchResponse.hits().getAt(2).field("city").getValue().toString(), equalTo("South Burlington")); + assertThat(((Map) searchResponse.hits().getAt(2).field("state_info").getValue()).get("name").toString(), equalTo("Vermont")); + } + + +} From 418c4c86235f30938a1f314000086d0c2a395fb0 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Sun, 17 Feb 2013 18:18:43 -0500 Subject: [PATCH 06/69] Add license --- LICENSE.txt | 202 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 202 insertions(+) create mode 100644 LICENSE.txt diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000000000..d645695673349 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. From 9525fa517f3d5201c88d87fe81d9305df05149d8 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Sat, 23 Feb 2013 21:51:14 -0500 Subject: [PATCH 07/69] Add Lookup Script description --- README.textile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.textile b/README.textile index 0713bad50aa46..2254654f9b318 100644 --- a/README.textile +++ b/README.textile @@ -75,4 +75,7 @@ bc.. public class NativeScriptExamplesPlugin extends AbstractPlugin { p. In general native scripts have to implement the interface "ExecutableScript":https://github.com/elasticsearch/elasticsearch/blob/master/src/main/java/org/elasticsearch/script/ExecutableScript.java, but if they are used in search, they have to also implement the "SearchScript":https://github.com/elasticsearch/elasticsearch/blob/master/src/main/java/org/elasticsearch/script/SearchScript.java interface. The SearchScript interface is quite complex, so in order to simplify implementation, the native script can simply extend the "AbstractSearchScript":https://github.com/elasticsearch/elasticsearch/blob/master/src/main/java/org/elasticsearch/script/AbstractSearchScript.java class instead. The AbstractSearchScript has only one abstract method @run()@. During search Elasticsearch calls this method for every single record in the search result. As in case of non-native script, the content of the current record can be accessed using DocLookup (@doc()@ method), FieldsLookup (@fields()@ method), SourceLookup (@source()@ method). +h3. Lookup Script + +p. The "lookup script":https://github.com/imotov/elasticsearch-native-script-example/blob/master/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java demonstrates how to make elsticsearch client available within the native script. When script factory is initially created, the node is not fully initialized yet. Because of this client cannot be directory injected into the factory. Instead, the reference to the node is injected and the client is obtained from the node during script creation. A same mechanism can be used to obtain other node components through node injector. From c9629a1198fe073bc3cf574b576eb37fed07ff74 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Sat, 23 Feb 2013 21:53:15 -0500 Subject: [PATCH 08/69] Make sure that the local shard will be used for the lookup operation if it's allocated on the local node --- .../examples/nativescript/script/LookupScript.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java index 9090eacb0049c..56756d03435eb 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java @@ -125,7 +125,7 @@ public Object run() { public Map call() throws Exception { // This is not very efficient of doing this, but it demonstrates using injected client // for record lookup - GetResponse response = client.prepareGet(lookupIndex, lookupType, fieldValue).execute().actionGet(); + GetResponse response = client.prepareGet(lookupIndex, lookupType, fieldValue).setPreference("_local").execute().actionGet(); if (response.exists()) { return response.getSource(); } From 25b15650e34d149b3915b7a7808ff97c9c88a65a Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Sun, 24 Feb 2013 22:58:42 -0500 Subject: [PATCH 09/69] Add random sort order native script --- .../plugin/NativeScriptExamplesPlugin.java | 2 + .../script/RandomSortScriptFactory.java | 76 +++++++++++++ .../script/RandomSortScriptTests.java | 103 ++++++++++++++++++ 3 files changed, 181 insertions(+) create mode 100644 src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java create mode 100644 src/test/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptTests.java diff --git a/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java b/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java index 1f304cc06b13d..28ea3516307dc 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java @@ -2,6 +2,7 @@ import org.elasticsearch.examples.nativescript.script.IsPrimeSearchScript; import org.elasticsearch.examples.nativescript.script.LookupScript; +import org.elasticsearch.examples.nativescript.script.RandomSortScriptFactory; import org.elasticsearch.plugins.AbstractPlugin; import org.elasticsearch.script.ScriptModule; @@ -38,5 +39,6 @@ public void onModule(ScriptModule module) { // Register each script that we defined in this plugin module.registerScript("is_prime", IsPrimeSearchScript.Factory.class); module.registerScript("lookup", LookupScript.Factory.class); + module.registerScript("random", RandomSortScriptFactory.class); } } diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java b/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java new file mode 100644 index 0000000000000..6315b02de470e --- /dev/null +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java @@ -0,0 +1,76 @@ +package org.elasticsearch.examples.nativescript.script; + +import org.elasticsearch.ElasticSearchIllegalArgumentException; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.index.field.data.DocFieldData; +import org.elasticsearch.index.mapper.internal.UidFieldMapper; +import org.elasticsearch.script.AbstractFloatSearchScript; +import org.elasticsearch.script.AbstractLongSearchScript; +import org.elasticsearch.script.ExecutableScript; +import org.elasticsearch.script.NativeScriptFactory; +import sun.security.provider.MD5; + +import java.util.Map; +import java.util.Random; + +/** + * This script demonstrates how native scripts can be used to create custom sort order. + * Since sort operation is expecting float parameter, the {@link AbstractFloatSearchScript} can be used. + * + * The script accepts one optional parameter salt. If parameter is specified, a pseudo random sort order is used. + * Otherwise, a random sort order is used. + */ +public class RandomSortScriptFactory implements NativeScriptFactory { + + /** + * This method is called for every search on every shard. + * + * @param params list of script parameters passed with the query + * @return new native script + */ + @Override + public ExecutableScript newScript(@Nullable Map params) { + String salt = params == null ? null : XContentMapValues.nodeStringValue(params.get("salt"), null); + if (salt == null) { + return new RandomSortScript(); + } else { + return new PseudoRandomSortScript(salt); + } + } + + private static class RandomSortScript extends AbstractLongSearchScript { + private final Random random; + + private RandomSortScript() { + random = new Random(); + } + + @Override + public long runAsLong() { + return random.nextLong(); + } + } + + private static class PseudoRandomSortScript extends AbstractLongSearchScript { + private final String salt; + + private PseudoRandomSortScript(String salt) { + this.salt = salt; + } + + @Override + public long runAsLong() { + DocFieldData fieldData = doc().field(UidFieldMapper.NAME); + byte[] sort = org.elasticsearch.common.Digest.md5(fieldData.stringValue() + salt); + return (sort[0] & 0xFFL) << 56 + | (sort[1] & 0xFFL) << 48 + | (sort[2] & 0xFFL) << 40 + | (sort[3] & 0xFFL) << 32 + | (sort[4] & 0xFFL) << 24 + | (sort[5] & 0xFFL) << 16 + | (sort[6] & 0xFFL) << 8 + | (sort[7] & 0xFFL); + } + } +} diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptTests.java new file mode 100644 index 0000000000000..2bbf8a15555b1 --- /dev/null +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptTests.java @@ -0,0 +1,103 @@ +package org.elasticsearch.examples.nativescript.script; + +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.collect.MapBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.indices.IndexMissingException; +import org.elasticsearch.search.sort.SortBuilders; +import org.testng.annotations.Test; + +import java.util.Arrays; + +import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; + +/** + */ +public class RandomSortScriptTests extends AbstractSearchScriptTests { + @Test + public void testPseudoRandomScript() throws Exception { + // Delete the old index + try { + node.client().admin().indices().prepareDelete("test").execute().actionGet(); + } catch (IndexMissingException ex) { + // Ignore + } + + // Create a new index + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties") + .startObject("name").field("type", "string").endObject() + .endObject().endObject().endObject() + .string(); + node.client().admin().indices().prepareCreate("test") + .addMapping("type", mapping) + .execute().actionGet(); + + // Index 100 records (0..99) + for (int i = 0; i < 100; i++) { + node.client().prepareIndex("test", "type", Integer.toString(i)) + .setSource(XContentFactory.jsonBuilder().startObject() + .field("name", "rec " + i) + .endObject()) + .execute().actionGet(); + } + node.client().admin().indices().prepareRefresh("test").execute().actionGet(); + + + // Retrieve first 10 records + SearchResponse searchResponse = node.client().prepareSearch("test") + .setQuery(matchAllQuery()) + .addField("name") + .setSize(10) + .addSort(SortBuilders.scriptSort("random", "number").lang("native").setParams(MapBuilder.newMapBuilder().put("salt", "1234").map())) + .execute().actionGet(); + assertThat(Arrays.toString(searchResponse.shardFailures()), searchResponse.failedShards(), equalTo(0)); + + // Check that random order was applied + assertThat(searchResponse.hits().getAt(0).field("name").getValue().toString(), not(equalTo("rec0"))); + + String[] records = new String[10]; + + // Store sort order + for (int i = 0; i < 10; i++) { + records[i] = searchResponse.hits().getAt(i).field("name").getValue().toString(); + } + + // Retrieve first 10 records again + searchResponse = node.client().prepareSearch("test") + .setQuery(matchAllQuery()) + .addField("name") + .setSize(10) + .addSort(SortBuilders.scriptSort("random", "number").lang("native").setParams(MapBuilder.newMapBuilder().put("salt", "1234").map())) + .execute().actionGet(); + assertThat(Arrays.toString(searchResponse.shardFailures()), searchResponse.failedShards(), equalTo(0)); + + // Verify the same sort order + for (int i = 0; i < 10; i++) { + assertThat(searchResponse.hits().getAt(i).field("name").getValue().toString(), equalTo(records[i])); + } + + // Retrieve first 10 records without salt + searchResponse = node.client().prepareSearch("test") + .setQuery(matchAllQuery()) + .addField("name") + .setSize(10) + .addSort(SortBuilders.scriptSort("random", "number").lang("native")) + .execute().actionGet(); + assertThat(Arrays.toString(searchResponse.shardFailures()), searchResponse.failedShards(), equalTo(0)); + + // Verify different sort order + boolean different = false; + for (int i = 0; i < 10; i++) { + if (!records[i].equals(searchResponse.hits().getAt(i).field("name").getValue().toString())) { + different = true; + break; + } + } + assertThat(different, equalTo(true)); + + } +} From 92c495033a8d79d7377c644fe0039d1c977e346f Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Mon, 25 Feb 2013 19:44:16 -0500 Subject: [PATCH 10/69] Add an example of using logger in a native script --- config/logging.yml | 1 + .../examples/nativescript/script/LookupScript.java | 14 +++++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/config/logging.yml b/config/logging.yml index 5f8c973d28214..d3550199ade95 100644 --- a/config/logging.yml +++ b/config/logging.yml @@ -14,6 +14,7 @@ logger: # discovery #discovery: TRACE + examples.nativescript.script: TRACE index.search.slowlog: TRACE, index_search_slow_log_file diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java index 56756d03435eb..17d956316938b 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java @@ -8,7 +8,9 @@ import org.elasticsearch.common.cache.CacheBuilder; import org.elasticsearch.common.collect.ImmutableMap; import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; @@ -34,7 +36,7 @@ public class LookupScript extends AbstractSearchScript { * {@link org.elasticsearch.examples.nativescript.plugin.NativeScriptExamplesPlugin#onModule(org.elasticsearch.script.ScriptModule)} * method when plugin is loaded. */ - public static class Factory implements NativeScriptFactory { + public static class Factory extends AbstractComponent implements NativeScriptFactory{ private final Node node; @@ -48,6 +50,7 @@ public static class Factory implements NativeScriptFactory { @SuppressWarnings("unchecked") @Inject public Factory(Node node, Settings settings) { + super(settings); // Node is not fully initialized here // All we can do is save a reference to it for future use this.node = node; @@ -88,21 +91,23 @@ public ExecutableScript newScript(@Nullable Map params) { if (field == null) { throw new ElasticSearchIllegalArgumentException("Missing the field parameter"); } - return new LookupScript(node.client(), cache, lookupIndex, lookupType, field); + return new LookupScript(node.client(), logger, cache, lookupIndex, lookupType, field); } } private final String lookupIndex; private final String lookupType; private final String field; + private final ESLogger logger; private final Client client; private final Cache, Map> cache; private static final Map EMPTY_MAP = ImmutableMap.of(); - private LookupScript(Client client, Cache, Map> cache, String lookupIndex, String lookupType, String field) { + private LookupScript(Client client, ESLogger logger, Cache, Map> cache, String lookupIndex, String lookupType, String field) { this.client = client; + this.logger = logger; this.lookupIndex = lookupIndex; this.lookupType = lookupType; this.field = field; @@ -126,6 +131,9 @@ public Map call() throws Exception { // This is not very efficient of doing this, but it demonstrates using injected client // for record lookup GetResponse response = client.prepareGet(lookupIndex, lookupType, fieldValue).setPreference("_local").execute().actionGet(); + if (logger.isTraceEnabled()) { + logger.trace("lookup [{}]/[{}]/[{}], found: [{}]", lookupIndex, lookupType, fieldValue, response.exists()); + } if (response.exists()) { return response.getSource(); } From f90c8b076adde12f6637fd7a8a35543a579037ea Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Mon, 25 Feb 2013 19:44:52 -0500 Subject: [PATCH 11/69] Add a curl example for lookup script --- example/lookup.sh | 81 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100755 example/lookup.sh diff --git a/example/lookup.sh b/example/lookup.sh new file mode 100755 index 0000000000000..a0a30bcdcbabc --- /dev/null +++ b/example/lookup.sh @@ -0,0 +1,81 @@ +curl -XDELETE "http://localhost:9200/test" +echo +curl -XPUT "http://localhost:9200/test/" -d '{ + "settings": { + "index.number_of_shards": 1, + "index.nubmer_of_replicas": 0 + }, + "mappings": { + "state": { + "properties": { + "name": { + "type": "string" + }, + "capital": { + "type": "string" + }, + "nickname": { + "type": "string" + } + } + }, + "city": { + "properties": { + "city": { + "type": "string" + }, + "state": { + "type": "string", + "index": "not_analyzed" + }, + "population": { + "type": "integer" + } + } + } + } +}' +echo +curl -XPUT "localhost:9200/test/state/CT" -d '{"name": "Connecticut", "capital": "Hartford", "nickname": "Constitution State"}' +curl -XPUT "localhost:9200/test/state/ME" -d '{"name": "Maine", "capital": "Augusta", "nickname": "Lumber State"}' +curl -XPUT "localhost:9200/test/state/MA" -d '{"name": "Massachusetts", "capital": "Boston", "nickname": "Bay State"}' +curl -XPUT "localhost:9200/test/state/NH" -d '{"name": "New Hampshire", "capital": "Concord", "nickname": "Granite State"}' +curl -XPUT "localhost:9200/test/state/RI" -d '{"name": "Rhode Island", "capital": "Providence", "nickname": "Little Rhody"}' +curl -XPUT "localhost:9200/test/state/VT" -d '{"name": "Vermont", "capital": "Montpelier", "nickname": "Green Mountain State"}' + +curl -XPUT "localhost:9200/test/city/1" -d '{"city": "Cambridge", "state": "MA", "population": 105162}' +curl -XPUT "localhost:9200/test/city/2" -d '{"city": "South Burlington", "state": "VT", "population": 17904}' +curl -XPUT "localhost:9200/test/city/3" -d '{"city": "South Portland", "state": "ME", "population": 25002}' +curl -XPUT "localhost:9200/test/city/4" -d '{"city": "Essex", "state": "VT", "population": 19587}' +curl -XPUT "localhost:9200/test/city/5" -d '{"city": "Portland", "state": "ME", "population": 66194}' +curl -XPUT "localhost:9200/test/city/6" -d '{"city": "Burlington", "state": "VT", "population": 42417}' +curl -XPUT "localhost:9200/test/city/7" -d '{"city": "Stamford", "state": "CT", "population": 122643}' +curl -XPUT "localhost:9200/test/city/8" -d '{"city": "Colchester", "state": "VT", "population": 17067}' +curl -XPUT "localhost:9200/test/city/9" -d '{"city": "Concord", "state": "NH", "population": 42695}' +curl -XPUT "localhost:9200/test/city/10" -d '{"city": "Boston", "state": "MA", "population": 617594}' + +curl -XPOST "http://localhost:9200/test/_refresh" +echo +curl "localhost:9200/test/city/_search?pretty=true" -d '{ + "query": { + "match_all": { } + }, + "script_fields": { + "state_info": { + "script": "lookup", + "lang": "native", + "params": { + "lookup_index": "test", + "lookup_type": "state", + "field": "state" + } + } + }, + "sort": [ + { + "population": "desc" + } + ] +} +' + From cb8fcc5968c8760c80ccb22ae2522b2def389964 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Wed, 27 Feb 2013 21:18:14 -0500 Subject: [PATCH 12/69] Add random sort script description --- README.textile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.textile b/README.textile index 2254654f9b318..6146673bdfdda 100644 --- a/README.textile +++ b/README.textile @@ -79,3 +79,8 @@ h3. Lookup Script p. The "lookup script":https://github.com/imotov/elasticsearch-native-script-example/blob/master/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java demonstrates how to make elsticsearch client available within the native script. When script factory is initially created, the node is not fully initialized yet. Because of this client cannot be directory injected into the factory. Instead, the reference to the node is injected and the client is obtained from the node during script creation. A same mechanism can be used to obtain other node components through node injector. +h3. Random Sort Script + +p. The "random sort script":https://github.com/imotov/elasticsearch-native-script-example/blob/master/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java demonstrates a slightly different approach to script/factory packaging. In this case the factory is the outer class which creates one inner script or another based on the input parameters. If the parameter @salt@ is present, the script is calculating hash value of @id + salt@ instead of generation random sort values. As a result, for any value of @salt@ the order of the records will appear random, but this order will be repeatable and therefore this approach would be more suitable for paging through result list than a completely random approach. + + From 9ab9fb1fe7723e7df06ee86a97a762bfa8055017 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Thu, 28 Feb 2013 22:10:59 -0500 Subject: [PATCH 13/69] Add a curl example for random sort script --- example/random.sh | 104 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100755 example/random.sh diff --git a/example/random.sh b/example/random.sh new file mode 100755 index 0000000000000..98a9107cd5306 --- /dev/null +++ b/example/random.sh @@ -0,0 +1,104 @@ +curl -XDELETE "http://localhost:9200/test" +echo +curl -XPUT "http://localhost:9200/test/" -d '{ + "settings": { + "index.number_of_shards": 1, + "index.nubmer_of_replicas": 0 + }, + "mappings": { + "type1": { + "properties": { + "name": { + "type": "string" + }, + "number": { + "type": "integer" + } + } + } + } +}' +echo +for i in {0..100}; do curl -XPUT "localhost:9200/test/type1/$i" -d "{\"name\":\"rec $i\", \"number\":$i}"; done +curl -XPOST "http://localhost:9200/test/_refresh" +echo +echo "Salt 123" +curl -s "localhost:9200/test/type1/_search?pretty=true" -d '{ + "query": { + "match_all": {} + }, + "sort": { + "_script": { + "script": "random", + "lang": "native", + "type": "number", + "params": { + "salt": "123" + } + } + } +} +' | grep \"_id\" +echo "Salt 123" +curl -s "localhost:9200/test/type1/_search?pretty=true" -d '{ + "query": { + "match_all": {} + }, + "sort": { + "_script": { + "script": "random", + "lang": "native", + "type": "number", + "params": { + "salt": "123" + } + } + } +} +' | grep \"_id\" +echo "Salt 124" +curl -s "localhost:9200/test/type1/_search?pretty=true" -d '{ + "query": { + "match_all": {} + }, + "sort": { + "_script": { + "script": "random", + "lang": "native", + "type": "number", + "params": { + "salt": "124" + } + } + } +} +' | grep \"_id\" +echo "No salt" +curl -s "localhost:9200/test/type1/_search?pretty=true" -d '{ + "query": { + "match_all": {} + }, + "sort": { + "_script": { + "script": "random", + "lang": "native", + "type": "number" + } + } +} +' | grep \"_id\" +echo "No salt" +curl -s "localhost:9200/test/type1/_search?pretty=true" -d '{ + "query": { + "match_all": {} + }, + "sort": { + "_script": { + "script": "random", + "lang": "native", + "type": "number" + } + } +} +' | grep \"_id\" +echo From 211f2675e2a1e5e2173b6aede1bd06b9d5b5c8c7 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Fri, 1 Mar 2013 19:54:00 -0500 Subject: [PATCH 14/69] Make sure that standard naming convention is used for plugin file --- pom.xml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9f02929cc5054..1e3d645744a94 100644 --- a/pom.xml +++ b/pom.xml @@ -13,7 +13,7 @@ 1.0.0-SNAPSHOT jar ElasticSearch Plugin with Native Script Examples - 2012 + 2013 The Apache Software License, Version 2.0 @@ -27,6 +27,13 @@ http://github.com/imotov/elasticsearch-native-script-example + + org.sonatype.oss + oss-parent + 7 + + + @@ -121,6 +128,7 @@ maven-assembly-plugin 2.3 + false ${project.build.directory}/releases/ ${basedir}/src/main/assemblies/plugin.xml From 2b184f3ba818a2d553f13c9340e4e33ded3dc06d Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Fri, 1 Mar 2013 20:02:17 -0500 Subject: [PATCH 15/69] Release v1.0.0 --- locator.json | 5 +++++ pom.xml | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 locator.json diff --git a/locator.json b/locator.json new file mode 100644 index 0000000000000..2478a62f9f7a2 --- /dev/null +++ b/locator.json @@ -0,0 +1,5 @@ +{ + "versions" : [ + {"version": "1.0.0", "from": "0.20.0", "to": "0.20.99", "location":"http://dl.bintray.com/content/imotov/elasticsearch-plugins/elasticsearch-native-script-example-1.0.0.zip" } + ] +} \ No newline at end of file diff --git a/pom.xml b/pom.xml index 1e3d645744a94..6bdfeaaa66cd7 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ 4.0.0 org.elasticsearch elasticsearch-native-script-example - 1.0.0-SNAPSHOT + 1.0.0 jar ElasticSearch Plugin with Native Script Examples 2013 From e5d10c7c4d3e70862ac64cc51906e6dc8937242a Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Fri, 1 Mar 2013 20:25:51 -0500 Subject: [PATCH 16/69] Move to 1.1.0-SNAPSHOT --- pom.xml | 4 ++-- .../nativescript/script/IsPrimeSearchScript.java | 6 +++--- .../nativescript/script/LookupScript.java | 10 +++++----- .../script/RandomSortScriptFactory.java | 6 +++--- .../script/IsPrimeSearchScriptTests.java | 10 +++++----- .../nativescript/script/LookupScriptTests.java | 16 ++++++++-------- .../script/RandomSortScriptTests.java | 14 +++++++------- 7 files changed, 33 insertions(+), 33 deletions(-) diff --git a/pom.xml b/pom.xml index 6bdfeaaa66cd7..3396668d6e55e 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ 4.0.0 org.elasticsearch elasticsearch-native-script-example - 1.0.0 + 1.1.0-SNAPSHOT jar ElasticSearch Plugin with Native Script Examples 2013 @@ -38,7 +38,7 @@ - 0.20.5 + 0.90.0.Beta1 diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java index 38db8ec57f5b8..22818faae8221 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java @@ -3,7 +3,7 @@ import org.elasticsearch.ElasticSearchIllegalArgumentException; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.xcontent.support.XContentMapValues; -import org.elasticsearch.index.field.data.DocFieldData; +import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.script.AbstractSearchScript; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.NativeScriptFactory; @@ -65,13 +65,13 @@ private IsPrimeSearchScript(String fieldName, int certainty) { @Override public Object run() { // First we get field using doc lookup - DocFieldData field = doc().field(fieldName); + ScriptDocValues.NumericLong field = (ScriptDocValues.NumericLong)doc().get(fieldName); // Check if field exists if (field != null && !field.isEmpty()) { try { // Try to parse it as an integer - BigInteger bigInteger = new BigInteger(field.stringValue()); + BigInteger bigInteger = new BigInteger(Long.toString(field.getValue())); // Check if it's prime return bigInteger.isProbablePrime(certainty); } catch (NumberFormatException ex) { diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java index 17d956316938b..caaa3c321aff9 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java @@ -15,7 +15,7 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.support.XContentMapValues; -import org.elasticsearch.index.field.data.DocFieldData; +import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.node.Node; import org.elasticsearch.script.AbstractSearchScript; import org.elasticsearch.script.ExecutableScript; @@ -117,12 +117,12 @@ private LookupScript(Client client, ESLogger logger, Cache @Override public Object run() { // First we get field using doc lookup - DocFieldData fieldData = doc().field(field); + ScriptDocValues.Strings fieldData = (ScriptDocValues.Strings )doc().get(field); // This is not very efficient // Check if field exists if (fieldData != null && !fieldData.isEmpty()) { - final String fieldValue = fieldData.stringValue(); + final String fieldValue = fieldData.getValue(); if (fieldValue != null) { try { return cache.get(new Tuple(lookupIndex + "/" + lookupType, fieldValue), new Callable>() { @@ -132,9 +132,9 @@ public Map call() throws Exception { // for record lookup GetResponse response = client.prepareGet(lookupIndex, lookupType, fieldValue).setPreference("_local").execute().actionGet(); if (logger.isTraceEnabled()) { - logger.trace("lookup [{}]/[{}]/[{}], found: [{}]", lookupIndex, lookupType, fieldValue, response.exists()); + logger.trace("lookup [{}]/[{}]/[{}], found: [{}]", lookupIndex, lookupType, fieldValue, response.isExists()); } - if (response.exists()) { + if (response.isExists()) { return response.getSource(); } return EMPTY_MAP; diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java b/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java index 6315b02de470e..42526f4541bf1 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java @@ -3,7 +3,7 @@ import org.elasticsearch.ElasticSearchIllegalArgumentException; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.xcontent.support.XContentMapValues; -import org.elasticsearch.index.field.data.DocFieldData; +import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.mapper.internal.UidFieldMapper; import org.elasticsearch.script.AbstractFloatSearchScript; import org.elasticsearch.script.AbstractLongSearchScript; @@ -61,8 +61,8 @@ private PseudoRandomSortScript(String salt) { @Override public long runAsLong() { - DocFieldData fieldData = doc().field(UidFieldMapper.NAME); - byte[] sort = org.elasticsearch.common.Digest.md5(fieldData.stringValue() + salt); + ScriptDocValues.Strings fieldData = (ScriptDocValues.Strings)doc().get(UidFieldMapper.NAME); + byte[] sort = org.elasticsearch.common.Digest.md5(fieldData.getValue() + salt); return (sort[0] & 0xFFL) << 56 | (sort[1] & 0xFFL) << 48 | (sort[2] & 0xFFL) << 40 diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScriptTests.java index d0593ce7975de..e38ced7b9e28e 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScriptTests.java @@ -67,14 +67,14 @@ public void testIsPrimeScript() throws Exception { .setSize(10) .addSort("number", SortOrder.ASC) .execute().actionGet(); - assertThat(Arrays.toString(searchResponse.shardFailures()), searchResponse.failedShards(), equalTo(0)); + assertThat(Arrays.toString(searchResponse.getShardFailures()), searchResponse.getFailedShards(), equalTo(0)); // There should be 25 prime numbers between 0 and 100 - assertThat(searchResponse.hits().totalHits(), equalTo(25l)); + assertThat(searchResponse.getHits().getTotalHits(), equalTo(25l)); // Verify that they are indeed prime numbers for (int i = 0; i < 10; i++) { - assertThat(searchResponse.hits().getAt(i).field("name").getValue().toString(), equalTo("rec " + PRIMES_10[i])); + assertThat(searchResponse.getHits().getAt(i).field("name").getValue().toString(), equalTo("rec " + PRIMES_10[i])); } // Check certainty parameter - with certainty == 0, it should return all numbers, but only if numbers are present @@ -85,9 +85,9 @@ public void testIsPrimeScript() throws Exception { .setSize(10) .addSort("number", SortOrder.ASC) .execute().actionGet(); - assertThat(Arrays.toString(searchResponse.shardFailures()), searchResponse.failedShards(), equalTo(0)); + assertThat(Arrays.toString(searchResponse.getShardFailures()), searchResponse.getFailedShards(), equalTo(0)); // With certainty 0 no check is done so it should return all numbers - assertThat(searchResponse.hits().totalHits(), equalTo(100l)); + assertThat(searchResponse.getHits().getTotalHits(), equalTo(100l)); } diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/LookupScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/LookupScriptTests.java index 953cbbd273a10..c9f001de535df 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/LookupScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/LookupScriptTests.java @@ -92,19 +92,19 @@ public void testLookup() throws Exception { .setSize(10) .addSort("population", SortOrder.DESC) .execute().actionGet(); - assertThat(Arrays.toString(searchResponse.shardFailures()), searchResponse.failedShards(), equalTo(0)); + assertThat(Arrays.toString(searchResponse.getShardFailures()), searchResponse.getFailedShards(), equalTo(0)); // There should be 3 cities - assertThat(searchResponse.hits().totalHits(), equalTo(3L)); + assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); - assertThat(searchResponse.hits().getAt(0).field("city").getValue().toString(), equalTo("Burlington")); - assertThat(((Map) searchResponse.hits().getAt(0).field("state_info").getValue()).get("name").toString(), equalTo("Vermont")); + assertThat(searchResponse.getHits().getAt(0).field("city").getValue().toString(), equalTo("Burlington")); + assertThat(((Map) searchResponse.getHits().getAt(0).field("state_info").getValue()).get("name").toString(), equalTo("Vermont")); - assertThat(searchResponse.hits().getAt(1).field("city").getValue().toString(), equalTo("South Portland")); - assertThat(((Map) searchResponse.hits().getAt(1).field("state_info").getValue()).get("name").toString(), equalTo("Maine")); + assertThat(searchResponse.getHits().getAt(1).field("city").getValue().toString(), equalTo("South Portland")); + assertThat(((Map) searchResponse.getHits().getAt(1).field("state_info").getValue()).get("name").toString(), equalTo("Maine")); - assertThat(searchResponse.hits().getAt(2).field("city").getValue().toString(), equalTo("South Burlington")); - assertThat(((Map) searchResponse.hits().getAt(2).field("state_info").getValue()).get("name").toString(), equalTo("Vermont")); + assertThat(searchResponse.getHits().getAt(2).field("city").getValue().toString(), equalTo("South Burlington")); + assertThat(((Map) searchResponse.getHits().getAt(2).field("state_info").getValue()).get("name").toString(), equalTo("Vermont")); } diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptTests.java index 2bbf8a15555b1..51ee3da071b34 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptTests.java @@ -54,16 +54,16 @@ public void testPseudoRandomScript() throws Exception { .setSize(10) .addSort(SortBuilders.scriptSort("random", "number").lang("native").setParams(MapBuilder.newMapBuilder().put("salt", "1234").map())) .execute().actionGet(); - assertThat(Arrays.toString(searchResponse.shardFailures()), searchResponse.failedShards(), equalTo(0)); + assertThat(Arrays.toString(searchResponse.getShardFailures()), searchResponse.getFailedShards(), equalTo(0)); // Check that random order was applied - assertThat(searchResponse.hits().getAt(0).field("name").getValue().toString(), not(equalTo("rec0"))); + assertThat(searchResponse.getHits().getAt(0).field("name").getValue().toString(), not(equalTo("rec0"))); String[] records = new String[10]; // Store sort order for (int i = 0; i < 10; i++) { - records[i] = searchResponse.hits().getAt(i).field("name").getValue().toString(); + records[i] = searchResponse.getHits().getAt(i).field("name").getValue().toString(); } // Retrieve first 10 records again @@ -73,11 +73,11 @@ public void testPseudoRandomScript() throws Exception { .setSize(10) .addSort(SortBuilders.scriptSort("random", "number").lang("native").setParams(MapBuilder.newMapBuilder().put("salt", "1234").map())) .execute().actionGet(); - assertThat(Arrays.toString(searchResponse.shardFailures()), searchResponse.failedShards(), equalTo(0)); + assertThat(Arrays.toString(searchResponse.getShardFailures()), searchResponse.getFailedShards(), equalTo(0)); // Verify the same sort order for (int i = 0; i < 10; i++) { - assertThat(searchResponse.hits().getAt(i).field("name").getValue().toString(), equalTo(records[i])); + assertThat(searchResponse.getHits().getAt(i).field("name").getValue().toString(), equalTo(records[i])); } // Retrieve first 10 records without salt @@ -87,12 +87,12 @@ public void testPseudoRandomScript() throws Exception { .setSize(10) .addSort(SortBuilders.scriptSort("random", "number").lang("native")) .execute().actionGet(); - assertThat(Arrays.toString(searchResponse.shardFailures()), searchResponse.failedShards(), equalTo(0)); + assertThat(Arrays.toString(searchResponse.getShardFailures()), searchResponse.getFailedShards(), equalTo(0)); // Verify different sort order boolean different = false; for (int i = 0; i < 10; i++) { - if (!records[i].equals(searchResponse.hits().getAt(i).field("name").getValue().toString())) { + if (!records[i].equals(searchResponse.getHits().getAt(i).field("name").getValue().toString())) { different = true; break; } From 210aab0ecc9fd04f8c4d720bb011cea46431e252 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Fri, 1 Mar 2013 20:30:12 -0500 Subject: [PATCH 17/69] Release v1.1.0 --- locator.json | 1 + pom.xml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/locator.json b/locator.json index 2478a62f9f7a2..d9f727f25249c 100644 --- a/locator.json +++ b/locator.json @@ -1,5 +1,6 @@ { "versions" : [ + {"version": "1.1.0", "from": "0.90.0.Beta1", "to": "0.90.99", "location":"http://dl.bintray.com/content/imotov/elasticsearch-plugins/elasticsearch-native-script-example-1.1.0.zip" }, {"version": "1.0.0", "from": "0.20.0", "to": "0.20.99", "location":"http://dl.bintray.com/content/imotov/elasticsearch-plugins/elasticsearch-native-script-example-1.0.0.zip" } ] } \ No newline at end of file diff --git a/pom.xml b/pom.xml index 3396668d6e55e..e85d307fa6076 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ 4.0.0 org.elasticsearch elasticsearch-native-script-example - 1.1.0-SNAPSHOT + 1.1.0 jar ElasticSearch Plugin with Native Script Examples 2013 From 574fd62d7cdfa3709ad2b72a3bd9bcc24b9bcfab Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Fri, 1 Mar 2013 20:38:56 -0500 Subject: [PATCH 18/69] Move to 1.1.0-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e85d307fa6076..8db9d96f18f6d 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ 4.0.0 org.elasticsearch elasticsearch-native-script-example - 1.1.0 + 1.1.1-SNAPSHOT jar ElasticSearch Plugin with Native Script Examples 2013 From 803337aa29a5550d4db8457553d307ab00ea7ef2 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Mon, 4 Mar 2013 22:27:32 -0500 Subject: [PATCH 19/69] Change node settings to avoid interference with other clusters --- config/elasticsearch.yml | 363 +-------------------------------------- 1 file changed, 3 insertions(+), 360 deletions(-) diff --git a/config/elasticsearch.yml b/config/elasticsearch.yml index 14ab8ea13cc11..f046d903aefa3 100644 --- a/config/elasticsearch.yml +++ b/config/elasticsearch.yml @@ -1,360 +1,3 @@ -##################### ElasticSearch Configuration Example ##################### - -# This file contains an overview of various configuration settings, -# targeted at operations staff. Application developers should -# consult the guide at . -# -# The installation procedure is covered at -# . -# -# ElasticSearch comes with reasonable defaults for most settings, -# so you can try it out without bothering with configuration. -# -# Most of the time, these defaults are just fine for running a production -# cluster. If you're fine-tuning your cluster, or wondering about the -# effect of certain configuration option, please _do ask_ on the -# mailing list or IRC channel [http://elasticsearch.org/community]. - -# Any element in the configuration can be replaced with environment variables -# by placing them in ${...} notation. For example: -# -# node.rack: ${RACK_ENV_VAR} - -# See -# for information on supported formats and syntax for the configuration file. - - -################################### Cluster ################################### - -# Cluster name identifies your cluster for auto-discovery. If you're running -# multiple clusters on the same network, make sure you're using unique names. -# -# cluster.name: elasticsearch - - -#################################### Node ##################################### - -# Node names are generated dynamically on startup, so you're relieved -# from configuring them manually. You can tie this node to a specific name: -# -# node.name: "Franz Kafka" - -# Every node can be configured to allow or deny being eligible as the master, -# and to allow or deny to store the data. -# -# Allow this node to be eligible as a master node (enabled by default): -# -# node.master: true -# -# Allow this node to store data (enabled by default): -# -# node.data: true - -# You can exploit these settings to design advanced cluster topologies. -# -# 1. You want this node to never become a master node, only to hold data. -# This will be the "workhorse" of your cluster. -# -# node.master: false -# node.data: true -# -# 2. You want this node to only serve as a master: to not store any data and -# to have free resources. This will be the "coordinator" of your cluster. -# -# node.master: true -# node.data: false -# -# 3. You want this node to be neither master nor data node, but -# to act as a "search load balancer" (fetching data from nodes, -# aggregating results, etc.) -# -# node.master: false -# node.data: false - -# Use the Cluster Health API [http://localhost:9200/_cluster/health], the -# Node Info API [http://localhost:9200/_cluster/nodes] or GUI tools -# such as and -# to inspect the cluster state. - -# A node can have generic attributes associated with it, which can later be used -# for customized shard allocation filtering, or allocation awareness. An attribute -# is a simple key value pair, similar to node.key: value, here is an example: -# -# node.rack: rack314 - -# By default, multiple nodes are allowed to start from the same installation location -# to disable it, set the following: -# node.max_local_storage_nodes: 1 - - -#################################### Index #################################### - -# You can set a number of options (such as shard/replica options, mapping -# or analyzer definitions, translog settings, ...) for indices globally, -# in this file. -# -# Note, that it makes more sense to configure index settings specifically for -# a certain index, either when creating it or by using the index templates API. -# -# See and -# -# for more information. - -# Set the number of shards (splits) of an index (5 by default): -# -# index.number_of_shards: 5 - -# Set the number of replicas (additional copies) of an index (1 by default): -# -# index.number_of_replicas: 1 - -# Note, that for development on a local machine, with small indices, it usually -# makes sense to "disable" the distributed features: -# -# index.number_of_shards: 1 -# index.number_of_replicas: 0 - -# These settings directly affect the performance of index and search operations -# in your cluster. Assuming you have enough machines to hold shards and -# replicas, the rule of thumb is: -# -# 1. Having more *shards* enhances the _indexing_ performance and allows to -# _distribute_ a big index across machines. -# 2. Having more *replicas* enhances the _search_ performance and improves the -# cluster _availability_. -# -# The "number_of_shards" is a one-time setting for an index. -# -# The "number_of_replicas" can be increased or decreased anytime, -# by using the Index Update Settings API. -# -# ElasticSearch takes care about load balancing, relocating, gathering the -# results from nodes, etc. Experiment with different settings to fine-tune -# your setup. - -# Use the Index Status API () to inspect -# the index status. - - -#################################### Paths #################################### - -# Path to directory containing configuration (this file and logging.yml): -# -# path.conf: /path/to/conf - -# Path to directory where to store index data allocated for this node. -# -# path.data: /path/to/data -# -# Can optionally include more than one location, causing data to be striped across -# the locations (a la RAID 0) on a file level, favouring locations with most free -# space on creation. For example: -# -# path.data: /path/to/data1,/path/to/data2 - -# Path to temporary files: -# -# path.work: /path/to/work - -# Path to log files: -# -# path.logs: /path/to/logs - -# Path to where plugins are installed: -# -# path.plugins: /path/to/plugins - - -#################################### Plugin ################################### - -# If a plugin listed here is not installed for current node, the node will not start. -# -# plugin.mandatory: mapper-attachments,lang-groovy - - -################################### Memory #################################### - -# ElasticSearch performs poorly when JVM starts swapping: you should ensure that -# it _never_ swaps. -# -# Set this property to true to lock the memory: -# -# bootstrap.mlockall: true - -# Make sure that the ES_MIN_MEM and ES_MAX_MEM environment variables are set -# to the same value, and that the machine has enough memory to allocate -# for ElasticSearch, leaving enough memory for the operating system itself. -# -# You should also make sure that the ElasticSearch process is allowed to lock -# the memory, eg. by using `ulimit -l unlimited`. - - -############################## Network And HTTP ############################### - -# ElasticSearch, by default, binds itself to the 0.0.0.0 address, and listens -# on port [9200-9300] for HTTP traffic and on port [9300-9400] for node-to-node -# communication. (the range means that if the port is busy, it will automatically -# try the next port). - -# Set the bind address specifically (IPv4 or IPv6): -# -# network.bind_host: 192.168.0.1 - -# Set the address other nodes will use to communicate with this node. If not -# set, it is automatically derived. It must point to an actual IP address. -# -# network.publish_host: 192.168.0.1 - -# Set both 'bind_host' and 'publish_host': -# -# network.host: 192.168.0.1 - -# Set a custom port for the node to node communication (9300 by default): -# -# transport.tcp.port: 9300 - -# Enable compression for all communication between nodes (disabled by default): -# -# transport.tcp.compress: true - -# Set a custom port to listen for HTTP traffic: -# -# http.port: 9200 - -# Set a custom allowed content length: -# -# http.max_content_length: 100mb - -# Disable HTTP completely: -# -# http.enabled: false - - -################################### Gateway ################################### - -# The gateway allows for persisting the cluster state between full cluster -# restarts. Every change to the state (such as adding an index) will be stored -# in the gateway, and when the cluster starts up for the first time, -# it will read its state from the gateway. - -# There are several types of gateway implementations. For more information, -# see . - -# The default gateway type is the "local" gateway (recommended): -# -# gateway.type: local - -# Settings below control how and when to start the initial recovery process on -# a full cluster restart (to reuse as much local data as possible when using shared -# gateway). - -# Allow recovery process after N nodes in a cluster are up: -# -# gateway.recover_after_nodes: 1 - -# Set the timeout to initiate the recovery process, once the N nodes -# from previous setting are up (accepts time value): -# -# gateway.recover_after_time: 5m - -# Set how many nodes are expected in this cluster. Once these N nodes -# are up (and recover_after_nodes is met), begin recovery process immediately -# (without waiting for recover_after_time to expire): -# -# gateway.expected_nodes: 2 - - -############################# Recovery Throttling ############################# - -# These settings allow to control the process of shards allocation between -# nodes during initial recovery, replica allocation, rebalancing, -# or when adding and removing nodes. - -# Set the number of concurrent recoveries happening on a node: -# -# 1. During the initial recovery -# -# cluster.routing.allocation.node_initial_primaries_recoveries: 4 -# -# 2. During adding/removing nodes, rebalancing, etc -# -# cluster.routing.allocation.node_concurrent_recoveries: 2 - -# Set to throttle throughput when recovering (eg. 100mb, by default unlimited): -# -# indices.recovery.max_size_per_sec: 0 - -# Set to limit the number of open concurrent streams when -# recovering a shard from a peer: -# -# indices.recovery.concurrent_streams: 5 - - -################################## Discovery ################################## - -# Discovery infrastructure ensures nodes can be found within a cluster -# and master node is elected. Multicast discovery is the default. - -# Set to ensure a node sees N other master eligible nodes to be considered -# operational within the cluster. Set this option to a higher value (2-4) -# for large clusters (>3 nodes): -# -# discovery.zen.minimum_master_nodes: 1 - -# Set the time to wait for ping responses from other nodes when discovering. -# Set this option to a higher value on a slow or congested network -# to minimize discovery failures: -# -# discovery.zen.ping.timeout: 3s - -# See -# for more information. - -# Unicast discovery allows to explicitly control which nodes will be used -# to discover the cluster. It can be used when multicast is not present, -# or to restrict the cluster communication-wise. -# -# 1. Disable multicast discovery (enabled by default): -# -# discovery.zen.ping.multicast.enabled: false -# -# 2. Configure an initial list of master nodes in the cluster -# to perform discovery when new nodes (master or data) are started: -# -# discovery.zen.ping.unicast.hosts: ["host1", "host2:port", "host3[portX-portY]"] - -# EC2 discovery allows to use AWS EC2 API in order to perform discovery. -# -# You have to install the cloud-aws plugin for enabling the EC2 discovery. -# -# See -# for more information. -# -# See -# for a step-by-step tutorial. - - -################################## Slow Log ################################## - -# Shard level query and fetch threshold logging. - -#index.search.slowlog.level: TRACE -#index.search.slowlog.threshold.query.warn: 10s -#index.search.slowlog.threshold.query.info: 5s -#index.search.slowlog.threshold.query.debug: 2s -#index.search.slowlog.threshold.query.trace: 500ms - -#index.search.slowlog.threshold.fetch.warn: 1s -#index.search.slowlog.threshold.fetch.info: 800ms -#index.search.slowlog.threshold.fetch.debug: 500ms -#index.search.slowlog.threshold.fetch.trace: 200ms - -################################## GC Logging ################################ - -#monitor.jvm.gc.ParNew.warn: 1000ms -#monitor.jvm.gc.ParNew.info: 700ms -#monitor.jvm.gc.ParNew.debug: 400ms - -#monitor.jvm.gc.ConcurrentMarkSweep.warn: 10s -#monitor.jvm.gc.ConcurrentMarkSweep.info: 5s -#monitor.jvm.gc.ConcurrentMarkSweep.debug: 2s +cluster.name: elasticsearch-native-script-tests +network.host: 127.0.0.1 +discovery.zen.ping.multicast.enabled: false From d12bfa7b2c11b544f74d542bb12adb87d3fad441 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Mon, 4 Mar 2013 22:28:52 -0500 Subject: [PATCH 20/69] Add example of a custom score script --- .../plugin/NativeScriptExamplesPlugin.java | 2 + .../script/PopularityScoreScriptFactory.java | 55 ++++++++++++ .../script/PopularityScoreScriptTests.java | 84 +++++++++++++++++++ 3 files changed, 141 insertions(+) create mode 100644 src/main/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptFactory.java create mode 100644 src/test/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptTests.java diff --git a/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java b/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java index 28ea3516307dc..6af82cf1f2fc7 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java @@ -2,6 +2,7 @@ import org.elasticsearch.examples.nativescript.script.IsPrimeSearchScript; import org.elasticsearch.examples.nativescript.script.LookupScript; +import org.elasticsearch.examples.nativescript.script.PopularityScoreScriptFactory; import org.elasticsearch.examples.nativescript.script.RandomSortScriptFactory; import org.elasticsearch.plugins.AbstractPlugin; import org.elasticsearch.script.ScriptModule; @@ -40,5 +41,6 @@ public void onModule(ScriptModule module) { module.registerScript("is_prime", IsPrimeSearchScript.Factory.class); module.registerScript("lookup", LookupScript.Factory.class); module.registerScript("random", RandomSortScriptFactory.class); + module.registerScript("popularity", PopularityScoreScriptFactory.class); } } diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptFactory.java b/src/main/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptFactory.java new file mode 100644 index 0000000000000..deb93130d0cc6 --- /dev/null +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptFactory.java @@ -0,0 +1,55 @@ +package org.elasticsearch.examples.nativescript.script; + +import org.elasticsearch.ElasticSearchIllegalArgumentException; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.script.AbstractFloatSearchScript; +import org.elasticsearch.script.ExecutableScript; +import org.elasticsearch.script.NativeScriptFactory; + +import java.util.Map; + +/** + * Factory for the script that boosts score of a record based on a value of the record's field. + *

+ * This native script demonstrates how to write native custom scores scripts. + */ +public class PopularityScoreScriptFactory implements NativeScriptFactory { + + @Override + public ExecutableScript newScript(@Nullable Map params) { + String fieldName = params == null ? null : XContentMapValues.nodeStringValue(params.get("field"), null); + if (fieldName == null) { + throw new ElasticSearchIllegalArgumentException("Missing the field parameter"); + } + return new PopularityScoreScript(fieldName); + } + + + /** + * This script takes a numeric value from the field specified in the parameter field. And calculates boost + * for the record using the following formula: 1 + log10(field_value + 1). So, records with value 0 in the field + * get no boost. Records with value 9 gets boost of 2.0, records with value 99, gets boost of 3, 999 - 4 and so on. + */ + private static class PopularityScoreScript extends AbstractFloatSearchScript { + + private final String field; + + public PopularityScoreScript(String field) { + this.field = field; + } + + @Override + public float runAsFloat() { + ScriptDocValues.NumericLong fieldData = (ScriptDocValues.NumericLong) doc().get(field); + if (fieldData != null && !fieldData.isEmpty()) { + double boost = 1 + Math.log10(fieldData.getValue() + 1); + // Because this script is used in custom_score script the value of score() is populated. + // In all other cases doc().getScore() should be used instead. + return (float) boost * score(); + } + return score(); + } + } +} diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptTests.java new file mode 100644 index 0000000000000..02a2178f99c8d --- /dev/null +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptTests.java @@ -0,0 +1,84 @@ +package org.elasticsearch.examples.nativescript.script; + +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.indices.IndexMissingException; +import org.testng.annotations.Test; + +import java.util.Arrays; + +import static org.elasticsearch.index.query.QueryBuilders.customScoreQuery; +import static org.elasticsearch.index.query.QueryBuilders.matchQuery; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; + +/** + */ +public class PopularityScoreScriptTests extends AbstractSearchScriptTests { + + @Test + public void testPopularityScoring() throws Exception { + // Delete the old index + try { + node.client().admin().indices().prepareDelete("test").execute().actionGet(); + } catch (IndexMissingException ex) { + // Ignore + } + + // Create a new index + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties") + .startObject("name").field("type", "string").endObject() + .startObject("number").field("type", "integer").endObject() + .endObject().endObject().endObject() + .string(); + node.client().admin().indices().prepareCreate("test") + .addMapping("type", mapping) + .execute().actionGet(); + + // Index 5 records with non-empty number field + for (int i = 0; i < 5; i++) { + node.client().prepareIndex("test", "type", Integer.toString(i)) + .setSource(XContentFactory.jsonBuilder().startObject() + .field("name", "rec " + i) + .field("number", i) + .endObject()) + .execute().actionGet(); + } + // Index a few records with empty number + for (int i = 5; i < 10; i++) { + node.client().prepareIndex("test", "type", Integer.toString(i)) + .setSource(XContentFactory.jsonBuilder().startObject() + .field("name", "rec " + i) + .endObject()) + .execute().actionGet(); + } + node.client().admin().indices().prepareRefresh("test").execute().actionGet(); + + // Retrieve first 10 hits + SearchResponse searchResponse = node.client().prepareSearch("test") + .setQuery(customScoreQuery(matchQuery("name", "rec")) + .script("popularity").lang("native").param("field", "number")) + .setSize(10) + .addField("name") + .execute().actionGet(); + assertThat(Arrays.toString(searchResponse.getShardFailures()), searchResponse.getFailedShards(), equalTo(0)); + + // There should be 10 hist + assertThat(searchResponse.getHits().getTotalHits(), equalTo(10l)); + + // Verify that first 5 hits are sorted from 4 to 0 + for (int i = 0; i < 5; i++) { + assertThat(searchResponse.getHits().getAt(i).field("name").getValue().toString(), equalTo("rec " + (4 - i))); + } + + // Verify that hit 5 has non-zero score + assertThat(searchResponse.getHits().getAt(5).score(), greaterThan(0.0f)); + + // Verify that the last 5 hits has the same score + for (int i = 6; i < 10; i++) { + assertThat(searchResponse.getHits().getAt(i).score(), equalTo(searchResponse.getHits().getAt(5).score())); + } + } +} From f280b76a32e8ab5658669d594307df8b55258dd8 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Wed, 13 Mar 2013 21:59:19 -0400 Subject: [PATCH 21/69] Small cosmetic changes --- .../nativescript/plugin/NativeScriptExamplesPlugin.java | 2 +- .../examples/nativescript/script/IsPrimeSearchScript.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java b/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java index 6af82cf1f2fc7..3b2288b5224ec 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java @@ -33,7 +33,7 @@ public String name() { */ @Override public String description() { - return "Examples of native script"; + return "Native script examples"; } public void onModule(ScriptModule module) { diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java index 22818faae8221..60b2e96673b09 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java @@ -3,7 +3,7 @@ import org.elasticsearch.ElasticSearchIllegalArgumentException; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.xcontent.support.XContentMapValues; -import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.index.fielddata.ScriptDocValues.NumericLong; import org.elasticsearch.script.AbstractSearchScript; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.NativeScriptFactory; @@ -65,7 +65,7 @@ private IsPrimeSearchScript(String fieldName, int certainty) { @Override public Object run() { // First we get field using doc lookup - ScriptDocValues.NumericLong field = (ScriptDocValues.NumericLong)doc().get(fieldName); + NumericLong field = (NumericLong) doc().get(fieldName); // Check if field exists if (field != null && !field.isEmpty()) { From a7f8b4da19aee85bf647cfea22c43519b2d21116 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Fri, 15 Mar 2013 13:09:07 -0400 Subject: [PATCH 22/69] Fix typo in index settings --- example/lookup.sh | 2 +- example/primes.sh | 2 +- example/random.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/example/lookup.sh b/example/lookup.sh index a0a30bcdcbabc..3b5a12646990a 100755 --- a/example/lookup.sh +++ b/example/lookup.sh @@ -3,7 +3,7 @@ echo curl -XPUT "http://localhost:9200/test/" -d '{ "settings": { "index.number_of_shards": 1, - "index.nubmer_of_replicas": 0 + "index.number_of_replicas": 0 }, "mappings": { "state": { diff --git a/example/primes.sh b/example/primes.sh index eb878900e5803..4d4f315998f67 100755 --- a/example/primes.sh +++ b/example/primes.sh @@ -3,7 +3,7 @@ echo curl -XPUT "http://localhost:9200/test/" -d '{ "settings": { "index.number_of_shards": 1, - "index.nubmer_of_replicas": 0 + "index.number_of_replicas": 0 }, "mappings": { "type1": { diff --git a/example/random.sh b/example/random.sh index 98a9107cd5306..b714e496c09bb 100755 --- a/example/random.sh +++ b/example/random.sh @@ -3,7 +3,7 @@ echo curl -XPUT "http://localhost:9200/test/" -d '{ "settings": { "index.number_of_shards": 1, - "index.nubmer_of_replicas": 0 + "index.number_of_replicas": 0 }, "mappings": { "type1": { From 1e2da9c3d03135297eab99508d98fd5532ccee17 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Fri, 15 Mar 2013 13:18:53 -0400 Subject: [PATCH 23/69] Add an example of popularity script --- example/popularity.sh | 60 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100755 example/popularity.sh diff --git a/example/popularity.sh b/example/popularity.sh new file mode 100755 index 0000000000000..ef6f9248e0f88 --- /dev/null +++ b/example/popularity.sh @@ -0,0 +1,60 @@ +curl -XDELETE "http://localhost:9200/test" +echo +curl -XPUT "http://localhost:9200/test/" -d '{ + "settings": { + "index.number_of_shards": 1, + "index.number_of_replicas": 0 + }, + "mappings": { + "type1": { + "properties": { + "name": { + "type": "string" + }, + "number": { + "type": "integer" + } + } + } + } +}' +echo +curl -XPUT "localhost:9200/test/type1/1" -d '{"name" : "foo bar baz", "number": 10000 }' +curl -XPUT "localhost:9200/test/type1/2" -d '{"name" : "foo foo foo", "number": 1 }' +curl -XPOST "http://localhost:9200/test/_refresh" +echo +curl -s "localhost:9200/test/type1/_search?pretty=true" -d '{ + "query": { + "custom_score" : { + "query": { + "match": { + "name" : "foo" + } + }, + "script": "popularity", + "lang": "native", + "params": { + "field": "number" + } + } + } +} +' +echo +curl -s "localhost:9200/test/type1/_search?pretty=true" -d '{ + "query": { + "custom_score" : { + "query": { + "match": { + "name" : "foo" + } + }, + "script": "_score * (1 + Math.log10(doc[field].value + 1))", + "params": { + "field": "number" + } + } + } +} +' +echo \ No newline at end of file From eed5d5352fbbdbbf7a0a6345d962100f142ac507 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Fri, 25 Oct 2013 19:27:43 -0400 Subject: [PATCH 24/69] Upgrade to elasticsearch v0.90.5 --- pom.xml | 2 +- .../script/IsPrimeSearchScript.java | 4 +-- .../script/PopularityScoreScriptFactory.java | 2 +- .../script/RandomSortScriptFactory.java | 31 ++++++++++++------- 4 files changed, 24 insertions(+), 15 deletions(-) diff --git a/pom.xml b/pom.xml index 8db9d96f18f6d..7b25502ca0dcc 100644 --- a/pom.xml +++ b/pom.xml @@ -38,7 +38,7 @@ - 0.90.0.Beta1 + 0.90.5 diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java index 60b2e96673b09..56a088b5c9b1c 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java @@ -3,7 +3,7 @@ import org.elasticsearch.ElasticSearchIllegalArgumentException; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.xcontent.support.XContentMapValues; -import org.elasticsearch.index.fielddata.ScriptDocValues.NumericLong; +import org.elasticsearch.index.fielddata.ScriptDocValues.Longs; import org.elasticsearch.script.AbstractSearchScript; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.NativeScriptFactory; @@ -65,7 +65,7 @@ private IsPrimeSearchScript(String fieldName, int certainty) { @Override public Object run() { // First we get field using doc lookup - NumericLong field = (NumericLong) doc().get(fieldName); + Longs field = (Longs) doc().get(fieldName); // Check if field exists if (field != null && !field.isEmpty()) { diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptFactory.java b/src/main/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptFactory.java index deb93130d0cc6..0e8f0eb978406 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptFactory.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptFactory.java @@ -42,7 +42,7 @@ public PopularityScoreScript(String field) { @Override public float runAsFloat() { - ScriptDocValues.NumericLong fieldData = (ScriptDocValues.NumericLong) doc().get(field); + ScriptDocValues.Longs fieldData = (ScriptDocValues.Longs) doc().get(field); if (fieldData != null && !fieldData.isEmpty()) { double boost = 1 + Math.log10(fieldData.getValue() + 1); // Because this script is used in custom_score script the value of score() is populated. diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java b/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java index 42526f4541bf1..5950fb722d343 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java @@ -11,13 +11,15 @@ import org.elasticsearch.script.NativeScriptFactory; import sun.security.provider.MD5; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.util.Map; import java.util.Random; /** * This script demonstrates how native scripts can be used to create custom sort order. * Since sort operation is expecting float parameter, the {@link AbstractFloatSearchScript} can be used. - * + *

* The script accepts one optional parameter salt. If parameter is specified, a pseudo random sort order is used. * Otherwise, a random sort order is used. */ @@ -61,16 +63,23 @@ private PseudoRandomSortScript(String salt) { @Override public long runAsLong() { - ScriptDocValues.Strings fieldData = (ScriptDocValues.Strings)doc().get(UidFieldMapper.NAME); - byte[] sort = org.elasticsearch.common.Digest.md5(fieldData.getValue() + salt); - return (sort[0] & 0xFFL) << 56 - | (sort[1] & 0xFFL) << 48 - | (sort[2] & 0xFFL) << 40 - | (sort[3] & 0xFFL) << 32 - | (sort[4] & 0xFFL) << 24 - | (sort[5] & 0xFFL) << 16 - | (sort[6] & 0xFFL) << 8 - | (sort[7] & 0xFFL); + ScriptDocValues.Strings fieldData = (ScriptDocValues.Strings) doc().get(UidFieldMapper.NAME); + try { + MessageDigest m = MessageDigest.getInstance("MD5"); + m.reset(); + m.update((fieldData.getValue() + salt).getBytes()); + byte[] sort = m.digest(); + return (sort[0] & 0xFFL) << 56 + | (sort[1] & 0xFFL) << 48 + | (sort[2] & 0xFFL) << 40 + | (sort[3] & 0xFFL) << 32 + | (sort[4] & 0xFFL) << 24 + | (sort[5] & 0xFFL) << 16 + | (sort[6] & 0xFFL) << 8 + | (sort[7] & 0xFFL); + } catch (NoSuchAlgorithmException ex) { + return -1; + } } } } From 1e3c3655c6814bbec48f6dda74d72d8d03af43ed Mon Sep 17 00:00:00 2001 From: Yann Date: Mon, 18 Nov 2013 16:43:06 +0100 Subject: [PATCH 25/69] Function Score Query Implemented Function Score Query model Switeched to 0.90.7 --- example/popularity.sh | 24 ++++++++++++------- pom.xml | 2 +- .../script/RandomSortScriptFactory.java | 2 -- .../script/PopularityScoreScriptTests.java | 14 ++++++++--- 4 files changed, 27 insertions(+), 15 deletions(-) diff --git a/example/popularity.sh b/example/popularity.sh index ef6f9248e0f88..9159cdf642f49 100755 --- a/example/popularity.sh +++ b/example/popularity.sh @@ -25,16 +25,19 @@ curl -XPOST "http://localhost:9200/test/_refresh" echo curl -s "localhost:9200/test/type1/_search?pretty=true" -d '{ "query": { - "custom_score" : { + "function_score" : { + "boost_mode": "replace", "query": { "match": { "name" : "foo" } }, - "script": "popularity", - "lang": "native", - "params": { - "field": "number" + "script_score": { + "script" : "popularity", + "lang": "native", + "params": { + "field": "number" + } } } } @@ -43,15 +46,18 @@ curl -s "localhost:9200/test/type1/_search?pretty=true" -d '{ echo curl -s "localhost:9200/test/type1/_search?pretty=true" -d '{ "query": { - "custom_score" : { + "function_score" : { + "boost_mode": "replace", "query": { "match": { "name" : "foo" } }, - "script": "_score * (1 + Math.log10(doc[field].value + 1))", - "params": { - "field": "number" + "script_score": { + "script": "_score * (1 + Math.log10(doc[field].value + 1))", + "params": { + "field": "number" + } } } } diff --git a/pom.xml b/pom.xml index 7b25502ca0dcc..24e1a2650d523 100644 --- a/pom.xml +++ b/pom.xml @@ -38,7 +38,7 @@ - 0.90.5 + 0.90.7 diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java b/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java index 5950fb722d343..17964f15c8fc3 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java @@ -1,6 +1,5 @@ package org.elasticsearch.examples.nativescript.script; -import org.elasticsearch.ElasticSearchIllegalArgumentException; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.fielddata.ScriptDocValues; @@ -9,7 +8,6 @@ import org.elasticsearch.script.AbstractLongSearchScript; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.NativeScriptFactory; -import sun.security.provider.MD5; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptTests.java index 02a2178f99c8d..ed923808d47dc 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptTests.java @@ -1,13 +1,17 @@ package org.elasticsearch.examples.nativescript.script; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.collect.MapBuilder; +import org.elasticsearch.common.lucene.search.function.CombineFunction; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders; import org.elasticsearch.indices.IndexMissingException; import org.testng.annotations.Test; import java.util.Arrays; +import java.util.Map; -import static org.elasticsearch.index.query.QueryBuilders.customScoreQuery; +import static org.elasticsearch.index.query.QueryBuilders.functionScoreQuery; import static org.elasticsearch.index.query.QueryBuilders.matchQuery; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; @@ -56,10 +60,14 @@ public void testPopularityScoring() throws Exception { } node.client().admin().indices().prepareRefresh("test").execute().actionGet(); + Map params = MapBuilder.newMapBuilder() + .put("field", "number") + .map(); // Retrieve first 10 hits SearchResponse searchResponse = node.client().prepareSearch("test") - .setQuery(customScoreQuery(matchQuery("name", "rec")) - .script("popularity").lang("native").param("field", "number")) + .setQuery(functionScoreQuery(matchQuery("name", "rec")) + .boostMode(CombineFunction.REPLACE) + .add(ScoreFunctionBuilders.scriptFunction("popularity", "native", params))) .setSize(10) .addField("name") .execute().actionGet(); From 0420b7a7188e399c71f417aa81377726a29853f4 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Mon, 18 Nov 2013 20:37:06 -0500 Subject: [PATCH 26/69] Fix elasticsearch.yml --- config/elasticsearch.yml | 1 + example/popularity.sh | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/config/elasticsearch.yml b/config/elasticsearch.yml index f046d903aefa3..263f6bdc6ec4e 100644 --- a/config/elasticsearch.yml +++ b/config/elasticsearch.yml @@ -1,3 +1,4 @@ cluster.name: elasticsearch-native-script-tests network.host: 127.0.0.1 discovery.zen.ping.multicast.enabled: false +discovery.zen.ping.unicast.hosts: ["localhost"] diff --git a/example/popularity.sh b/example/popularity.sh index 9159cdf642f49..5ad118fdfcd32 100755 --- a/example/popularity.sh +++ b/example/popularity.sh @@ -29,11 +29,11 @@ curl -s "localhost:9200/test/type1/_search?pretty=true" -d '{ "boost_mode": "replace", "query": { "match": { - "name" : "foo" + "name": "foo" } }, "script_score": { - "script" : "popularity", + "script": "popularity", "lang": "native", "params": { "field": "number" @@ -50,7 +50,7 @@ curl -s "localhost:9200/test/type1/_search?pretty=true" -d '{ "boost_mode": "replace", "query": { "match": { - "name" : "foo" + "name": "foo" } }, "script_score": { From d472879c705a87e494a859e3eec040a48013d809 Mon Sep 17 00:00:00 2001 From: Britta Weber Date: Thu, 12 Dec 2013 16:19:15 +0100 Subject: [PATCH 27/69] use elasticsearch version 0.90.8 and move from testNG to junit - use ElasticSearchIntegrationTest - use ElasticsearchAssertions - index random --- pom.xml | 39 ++++----- .../script/AbstractSearchScriptTests.java | 28 +++---- .../script/IsPrimeSearchScriptTests.java | 70 ++++++++-------- .../script/LookupScriptTests.java | 83 +++++++++---------- .../script/PopularityScoreScriptTests.java | 69 ++++++++------- .../script/RandomSortScriptTests.java | 59 ++++++------- 6 files changed, 168 insertions(+), 180 deletions(-) diff --git a/pom.xml b/pom.xml index 24e1a2650d523..d6bcc9a7a5700 100644 --- a/pom.xml +++ b/pom.xml @@ -38,7 +38,8 @@ - 0.90.7 + 0.90.8 + 4.6.0 @@ -46,12 +47,28 @@ + + + org.apache.lucene + lucene-test-framework + ${lucene.version} + test + + org.elasticsearch elasticsearch ${elasticsearch.version} compile + + + org.elasticsearch + elasticsearch + ${elasticsearch.version} + test + test-jar + log4j @@ -68,30 +85,14 @@ true - - org.testng - testng - 6.8 - test - - - org.hamcrest - hamcrest-core - - - junit - junit - - - - org.hamcrest hamcrest-all 1.3 test - + + diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTests.java index 9a59d0ce3ef8b..2055071b175ae 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTests.java @@ -2,31 +2,23 @@ import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.node.Node; -import org.elasticsearch.node.NodeBuilder; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; +import org.elasticsearch.test.ElasticsearchIntegrationTest; +import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope; +import org.elasticsearch.test.ElasticsearchIntegrationTest.Scope; /** */ -public class AbstractSearchScriptTests { - protected Node node; +@ClusterScope(scope = Scope.SUITE, numNodes = 1) +public class AbstractSearchScriptTests extends ElasticsearchIntegrationTest { - @BeforeMethod - public void startNode() { - Settings settings = ImmutableSettings.settingsBuilder() + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return ImmutableSettings.settingsBuilder() .put("gateway.type", "none") .put("index.number_of_shards", 1) .put("index.number_of_replicas", 0) + .put(super.nodeSettings(nodeOrdinal)) .build(); - - node = NodeBuilder.nodeBuilder().settings(settings).node(); - } - - @AfterMethod - public void closeNode() { - if (node != null) { - node.close(); - } } + } diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScriptTests.java index e38ced7b9e28e..69da5f35f3bea 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScriptTests.java @@ -1,34 +1,31 @@ package org.elasticsearch.examples.nativescript.script; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.indices.IndexMissingException; -import org.elasticsearch.search.sort.SortOrder; -import org.testng.annotations.Test; - -import java.util.Arrays; - import static org.elasticsearch.index.query.FilterBuilders.scriptFilter; import static org.elasticsearch.index.query.QueryBuilders.filteredQuery; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.hamcrest.MatcherAssert.assertThat; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; import static org.hamcrest.Matchers.equalTo; +import java.util.ArrayList; +import java.util.List; + +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.search.sort.SortOrder; +import org.junit.Test; + /** */ public class IsPrimeSearchScriptTests extends AbstractSearchScriptTests { - public static int[] PRIMES_10 = new int[]{2, 3, 5, 7, 11, 13, 17, 19, 23, 29}; + public static int[] PRIMES_10 = new int[] { 2, 3, 5, 7, 11, 13, 17, 19, 23, 29 }; @Test public void testIsPrimeScript() throws Exception { - // Delete the old index - try { - node.client().admin().indices().prepareDelete("test").execute().actionGet(); - } catch (IndexMissingException ex) { - // Ignore - } - + // Create a new index String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties") @@ -36,41 +33,44 @@ public void testIsPrimeScript() throws Exception { .startObject("number").field("type", "integer").endObject() .endObject().endObject().endObject() .string(); - node.client().admin().indices().prepareCreate("test") - .addMapping("type", mapping) - .execute().actionGet(); + + assertAcked(prepareCreate("test") + .addMapping("type", mapping)); + List indexBuilders = new ArrayList(); // Index 100 records (0..99) for (int i = 0; i < 100; i++) { - node.client().prepareIndex("test", "type", Integer.toString(i)) + indexBuilders.add( + client().prepareIndex("test", "type", Integer.toString(i)) .setSource(XContentFactory.jsonBuilder().startObject() .field("name", "rec " + i) .field("number", i) - .endObject()) - .execute().actionGet(); + .endObject())); } // Index a few records with empty number for (int i = 100; i < 105; i++) { - node.client().prepareIndex("test", "type", Integer.toString(i)) + indexBuilders.add( + client().prepareIndex("test", "type", Integer.toString(i)) .setSource(XContentFactory.jsonBuilder().startObject() .field("name", "rec " + i) - .endObject()) - .execute().actionGet(); + .endObject())); } - node.client().admin().indices().prepareRefresh("test").execute().actionGet(); + indexRandom(true, indexBuilders); + // Retrieve first 10 prime records - SearchResponse searchResponse = node.client().prepareSearch("test") - .setQuery(filteredQuery(matchAllQuery(), + SearchResponse searchResponse = client().prepareSearch("test") + .setQuery(filteredQuery(matchAllQuery(), scriptFilter("is_prime").lang("native").addParam("field", "number"))) .addField("name") .setSize(10) .addSort("number", SortOrder.ASC) .execute().actionGet(); - assertThat(Arrays.toString(searchResponse.getShardFailures()), searchResponse.getFailedShards(), equalTo(0)); + + assertNoFailures(searchResponse); // There should be 25 prime numbers between 0 and 100 - assertThat(searchResponse.getHits().getTotalHits(), equalTo(25l)); + assertHitCount(searchResponse, 25); // Verify that they are indeed prime numbers for (int i = 0; i < 10; i++) { @@ -78,19 +78,17 @@ public void testIsPrimeScript() throws Exception { } // Check certainty parameter - with certainty == 0, it should return all numbers, but only if numbers are present - searchResponse = node.client().prepareSearch("test") + searchResponse = client().prepareSearch("test") .setQuery(filteredQuery(matchAllQuery(), scriptFilter("is_prime").lang("native").addParam("field", "number").addParam("certainty", 0))) .addField("name") .setSize(10) .addSort("number", SortOrder.ASC) .execute().actionGet(); - assertThat(Arrays.toString(searchResponse.getShardFailures()), searchResponse.getFailedShards(), equalTo(0)); + assertNoFailures(searchResponse); // With certainty 0 no check is done so it should return all numbers - assertThat(searchResponse.getHits().getTotalHits(), equalTo(100l)); - + assertHitCount(searchResponse, 100); } - } diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/LookupScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/LookupScriptTests.java index c9f001de535df..1e8545bcdc57c 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/LookupScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/LookupScriptTests.java @@ -1,18 +1,20 @@ package org.elasticsearch.examples.nativescript.script; +import static org.elasticsearch.index.query.QueryBuilders.matchQuery; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; +import static org.hamcrest.Matchers.equalTo; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.indices.IndexMissingException; import org.elasticsearch.search.sort.SortOrder; -import org.testng.annotations.Test; - -import java.util.Arrays; -import java.util.Map; - -import static org.elasticsearch.index.query.QueryBuilders.matchQuery; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; +import org.junit.Test; /** */ @@ -21,13 +23,6 @@ public class LookupScriptTests extends AbstractSearchScriptTests { @SuppressWarnings("unchecked") @Test public void testLookup() throws Exception { - // Delete the old index - try { - node.client().admin().indices().prepareDelete("test").execute().actionGet(); - node.client().admin().indices().prepareDelete("lookup").execute().actionGet(); - } catch (IndexMissingException ex) { - // Ignore - } // Create a new lookup index String lookup_mapping = XContentFactory.jsonBuilder().startObject().startObject("state") @@ -37,9 +32,9 @@ public void testLookup() throws Exception { .startObject("nickname").field("type", "string").endObject() .endObject().endObject().endObject() .string(); - node.client().admin().indices().prepareCreate("lookup") - .addMapping("type", lookup_mapping) - .execute().actionGet(); + + assertAcked(prepareCreate("lookup") + .addMapping("type", lookup_mapping)); // Create a new test index String test_mapping = XContentFactory.jsonBuilder().startObject().startObject("city") @@ -49,31 +44,32 @@ public void testLookup() throws Exception { .startObject("population").field("type", "integer").endObject() .endObject().endObject().endObject() .string(); - node.client().admin().indices().prepareCreate("test") - .addMapping("city", test_mapping) - .execute().actionGet(); + + assertAcked(prepareCreate("test") + .addMapping("city", test_mapping)); + List indexBuilders = new ArrayList(); // Index Lookup records: - node.client().prepareIndex("lookup", "state", "CT").setSource("name", "Connecticut", "capital", "Hartford", "nickname", "Constitution State").execute().actionGet(); - node.client().prepareIndex("lookup", "state", "ME").setSource("name", "Maine", "capital", "Augusta", "nickname", "Lumber State").execute().actionGet(); - node.client().prepareIndex("lookup", "state", "MA").setSource("name", "Massachusetts", "capital", "Boston", "nickname", "Bay State").execute().actionGet(); - node.client().prepareIndex("lookup", "state", "NH").setSource("name", "New Hampshire", "capital", "Concord", "nickname", "Granite State").execute().actionGet(); - node.client().prepareIndex("lookup", "state", "RI").setSource("name", "Rhode Island", "capital", "Providence", "nickname", "Little Rhody").execute().actionGet(); - node.client().prepareIndex("lookup", "state", "VT").setSource("name", "Vermont", "capital", "Montpelier", "nickname", "Green Mountain State").execute().actionGet(); + indexBuilders.add(client().prepareIndex("lookup", "state", "CT").setSource("name", "Connecticut", "capital", "Hartford", "nickname", "Constitution State")); + indexBuilders.add(client().prepareIndex("lookup", "state", "ME").setSource("name", "Maine", "capital", "Augusta", "nickname", "Lumber State")); + indexBuilders.add(client().prepareIndex("lookup", "state", "MA").setSource("name", "Massachusetts", "capital", "Boston", "nickname", "Bay State")); + indexBuilders.add(client().prepareIndex("lookup", "state", "NH").setSource("name", "New Hampshire", "capital", "Concord", "nickname", "Granite State")); + indexBuilders.add(client().prepareIndex("lookup", "state", "RI").setSource("name", "Rhode Island", "capital", "Providence", "nickname", "Little Rhody")); + indexBuilders.add(client().prepareIndex("lookup", "state", "VT").setSource("name", "Vermont", "capital", "Montpelier", "nickname", "Green Mountain State")); // Index main records - node.client().prepareIndex("test", "city", "1").setSource("city", "Cambridge", "state", "MA", "population", 105162).execute().actionGet(); - node.client().prepareIndex("test", "city", "2").setSource("city", "South Burlington", "state", "VT", "population", 17904).execute().actionGet(); - node.client().prepareIndex("test", "city", "3").setSource("city", "South Portland", "state", "ME", "population", 25002).execute().actionGet(); - node.client().prepareIndex("test", "city", "4").setSource("city", "Essex", "state", "VT", "population", 19587).execute().actionGet(); - node.client().prepareIndex("test", "city", "5").setSource("city", "Portland", "state", "ME", "population", 66194).execute().actionGet(); - node.client().prepareIndex("test", "city", "6").setSource("city", "Burlington", "state", "VT", "population", 42417).execute().actionGet(); - node.client().prepareIndex("test", "city", "7").setSource("city", "Stamford", "state", "CT", "population", 122643).execute().actionGet(); - node.client().prepareIndex("test", "city", "8").setSource("city", "Colchester", "state", "VT", "population", 17067).execute().actionGet(); - node.client().prepareIndex("test", "city", "9").setSource("city", "Concord", "state", "NH", "population", 42695).execute().actionGet(); - node.client().prepareIndex("test", "city", "10").setSource("city", "Boston", "state", "MA", "population", 617594).execute().actionGet(); - - node.client().admin().indices().prepareRefresh("lookup", "test").execute().actionGet(); + indexBuilders.add(client().prepareIndex("test", "city", "1").setSource("city", "Cambridge", "state", "MA", "population", 105162)); + indexBuilders.add(client().prepareIndex("test", "city", "2").setSource("city", "South Burlington", "state", "VT", "population", 17904)); + indexBuilders.add(client().prepareIndex("test", "city", "3").setSource("city", "South Portland", "state", "ME", "population", 25002)); + indexBuilders.add(client().prepareIndex("test", "city", "4").setSource("city", "Essex", "state", "VT", "population", 19587)); + indexBuilders.add(client().prepareIndex("test", "city", "5").setSource("city", "Portland", "state", "ME", "population", 66194)); + indexBuilders.add(client().prepareIndex("test", "city", "6").setSource("city", "Burlington", "state", "VT", "population", 42417)); + indexBuilders.add(client().prepareIndex("test", "city", "7").setSource("city", "Stamford", "state", "CT", "population", 122643)); + indexBuilders.add(client().prepareIndex("test", "city", "8").setSource("city", "Colchester", "state", "VT", "population", 17067)); + indexBuilders.add(client().prepareIndex("test", "city", "9").setSource("city", "Concord", "state", "NH", "population", 42695)); + indexBuilders.add(client().prepareIndex("test", "city", "10").setSource("city", "Boston", "state", "MA", "population", 617594)); + + indexRandom(true, indexBuilders); // Script parameters Map params = MapBuilder.newMapBuilder() @@ -84,7 +80,7 @@ public void testLookup() throws Exception { // Find smallest city with word - SearchResponse searchResponse = node.client().prepareSearch("test") + SearchResponse searchResponse = client().prepareSearch("test") .setTypes("city") .setQuery(matchQuery("city", "south burlington")) .addField("city") @@ -92,10 +88,11 @@ public void testLookup() throws Exception { .setSize(10) .addSort("population", SortOrder.DESC) .execute().actionGet(); - assertThat(Arrays.toString(searchResponse.getShardFailures()), searchResponse.getFailedShards(), equalTo(0)); + + assertNoFailures(searchResponse); // There should be 3 cities - assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); + assertHitCount(searchResponse, 3); assertThat(searchResponse.getHits().getAt(0).field("city").getValue().toString(), equalTo("Burlington")); assertThat(((Map) searchResponse.getHits().getAt(0).field("state_info").getValue()).get("name").toString(), equalTo("Vermont")); diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptTests.java index ed923808d47dc..7ed0596753cf4 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptTests.java @@ -1,21 +1,24 @@ package org.elasticsearch.examples.nativescript.script; +import static org.elasticsearch.index.query.QueryBuilders.functionScoreQuery; +import static org.elasticsearch.index.query.QueryBuilders.matchQuery; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.lucene.search.function.CombineFunction; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders; -import org.elasticsearch.indices.IndexMissingException; -import org.testng.annotations.Test; - -import java.util.Arrays; -import java.util.Map; - -import static org.elasticsearch.index.query.QueryBuilders.functionScoreQuery; -import static org.elasticsearch.index.query.QueryBuilders.matchQuery; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; +import org.junit.Test; /** */ @@ -23,13 +26,7 @@ public class PopularityScoreScriptTests extends AbstractSearchScriptTests { @Test public void testPopularityScoring() throws Exception { - // Delete the old index - try { - node.client().admin().indices().prepareDelete("test").execute().actionGet(); - } catch (IndexMissingException ex) { - // Ignore - } - + // Create a new index String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties") @@ -37,44 +34,46 @@ public void testPopularityScoring() throws Exception { .startObject("number").field("type", "integer").endObject() .endObject().endObject().endObject() .string(); - node.client().admin().indices().prepareCreate("test") - .addMapping("type", mapping) - .execute().actionGet(); + + assertAcked(prepareCreate("test") + .addMapping("type", mapping)); + List indexBuilders = new ArrayList(); + // Index 5 records with non-empty number field for (int i = 0; i < 5; i++) { - node.client().prepareIndex("test", "type", Integer.toString(i)) + indexBuilders.add( + client().prepareIndex("test", "type", Integer.toString(i)) .setSource(XContentFactory.jsonBuilder().startObject() .field("name", "rec " + i) .field("number", i) - .endObject()) - .execute().actionGet(); + .endObject())); } // Index a few records with empty number for (int i = 5; i < 10; i++) { - node.client().prepareIndex("test", "type", Integer.toString(i)) + indexBuilders.add( + client().prepareIndex("test", "type", Integer.toString(i)) .setSource(XContentFactory.jsonBuilder().startObject() .field("name", "rec " + i) - .endObject()) - .execute().actionGet(); + .endObject())); } - node.client().admin().indices().prepareRefresh("test").execute().actionGet(); - - Map params = MapBuilder.newMapBuilder() - .put("field", "number") - .map(); + + indexRandom(true, indexBuilders); + + Map params = MapBuilder. newMapBuilder().put("field", "number").map(); // Retrieve first 10 hits - SearchResponse searchResponse = node.client().prepareSearch("test") + SearchResponse searchResponse = client().prepareSearch("test") .setQuery(functionScoreQuery(matchQuery("name", "rec")) .boostMode(CombineFunction.REPLACE) .add(ScoreFunctionBuilders.scriptFunction("popularity", "native", params))) .setSize(10) .addField("name") .execute().actionGet(); - assertThat(Arrays.toString(searchResponse.getShardFailures()), searchResponse.getFailedShards(), equalTo(0)); + + assertNoFailures(searchResponse); // There should be 10 hist - assertThat(searchResponse.getHits().getTotalHits(), equalTo(10l)); + assertHitCount(searchResponse, 10); // Verify that first 5 hits are sorted from 4 to 0 for (int i = 0; i < 5; i++) { diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptTests.java index 51ee3da071b34..8185e0ec22171 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptTests.java @@ -1,60 +1,59 @@ package org.elasticsearch.examples.nativescript.script; +import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; + +import java.util.ArrayList; +import java.util.List; + +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.indices.IndexMissingException; import org.elasticsearch.search.sort.SortBuilders; -import org.testng.annotations.Test; - -import java.util.Arrays; - -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.not; +import org.junit.Test; /** */ public class RandomSortScriptTests extends AbstractSearchScriptTests { @Test public void testPseudoRandomScript() throws Exception { - // Delete the old index - try { - node.client().admin().indices().prepareDelete("test").execute().actionGet(); - } catch (IndexMissingException ex) { - // Ignore - } - + // Create a new index String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties") .startObject("name").field("type", "string").endObject() .endObject().endObject().endObject() .string(); - node.client().admin().indices().prepareCreate("test") - .addMapping("type", mapping) - .execute().actionGet(); + + assertAcked(prepareCreate("test") + .addMapping("type", mapping)); + List indexBuilders = new ArrayList(); + // Index 100 records (0..99) for (int i = 0; i < 100; i++) { - node.client().prepareIndex("test", "type", Integer.toString(i)) + indexBuilders.add( + client().prepareIndex("test", "type", Integer.toString(i)) .setSource(XContentFactory.jsonBuilder().startObject() .field("name", "rec " + i) - .endObject()) - .execute().actionGet(); + .endObject())); } - node.client().admin().indices().prepareRefresh("test").execute().actionGet(); + indexRandom(true, indexBuilders); // Retrieve first 10 records - SearchResponse searchResponse = node.client().prepareSearch("test") + SearchResponse searchResponse = client().prepareSearch("test") .setQuery(matchAllQuery()) .addField("name") .setSize(10) .addSort(SortBuilders.scriptSort("random", "number").lang("native").setParams(MapBuilder.newMapBuilder().put("salt", "1234").map())) .execute().actionGet(); - assertThat(Arrays.toString(searchResponse.getShardFailures()), searchResponse.getFailedShards(), equalTo(0)); + + assertNoFailures(searchResponse); // Check that random order was applied assertThat(searchResponse.getHits().getAt(0).field("name").getValue().toString(), not(equalTo("rec0"))); @@ -67,13 +66,14 @@ public void testPseudoRandomScript() throws Exception { } // Retrieve first 10 records again - searchResponse = node.client().prepareSearch("test") + searchResponse = client().prepareSearch("test") .setQuery(matchAllQuery()) .addField("name") .setSize(10) .addSort(SortBuilders.scriptSort("random", "number").lang("native").setParams(MapBuilder.newMapBuilder().put("salt", "1234").map())) .execute().actionGet(); - assertThat(Arrays.toString(searchResponse.getShardFailures()), searchResponse.getFailedShards(), equalTo(0)); + + assertNoFailures(searchResponse); // Verify the same sort order for (int i = 0; i < 10; i++) { @@ -81,13 +81,14 @@ public void testPseudoRandomScript() throws Exception { } // Retrieve first 10 records without salt - searchResponse = node.client().prepareSearch("test") + searchResponse = client().prepareSearch("test") .setQuery(matchAllQuery()) .addField("name") .setSize(10) .addSort(SortBuilders.scriptSort("random", "number").lang("native")) .execute().actionGet(); - assertThat(Arrays.toString(searchResponse.getShardFailures()), searchResponse.getFailedShards(), equalTo(0)); + + assertNoFailures(searchResponse); // Verify different sort order boolean different = false; From 035dc8f5793db23cc939aaf31484910bc0546f7d Mon Sep 17 00:00:00 2001 From: Britta Weber Date: Wed, 18 Dec 2013 14:59:23 +0100 Subject: [PATCH 28/69] fix doc values lookup Check isEmpty() before casting to Longs or Strings. If a segment does not conatain any document with the looked up field, the lookup will return an instance of Empty instead of Longs/Strings that contains no value. The search will then fail with a ClassCastException. --- .../nativescript/script/IsPrimeSearchScript.java | 14 +++++++------- .../examples/nativescript/script/LookupScript.java | 7 +++---- .../script/PopularityScoreScriptFactory.java | 5 +++-- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java index 56a088b5c9b1c..5df776ea24a72 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java @@ -1,16 +1,17 @@ package org.elasticsearch.examples.nativescript.script; +import java.math.BigInteger; +import java.util.Map; + import org.elasticsearch.ElasticSearchIllegalArgumentException; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.fielddata.ScriptDocValues.Longs; import org.elasticsearch.script.AbstractSearchScript; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.NativeScriptFactory; -import java.math.BigInteger; -import java.util.Map; - /** * Implementation of the native script that checks that the field exists and contains a prime number. *

@@ -65,13 +66,12 @@ private IsPrimeSearchScript(String fieldName, int certainty) { @Override public Object run() { // First we get field using doc lookup - Longs field = (Longs) doc().get(fieldName); - + ScriptDocValues docValue = (ScriptDocValues) doc().get(fieldName); // Check if field exists - if (field != null && !field.isEmpty()) { + if (docValue != null && !docValue.isEmpty()) { try { // Try to parse it as an integer - BigInteger bigInteger = new BigInteger(Long.toString(field.getValue())); + BigInteger bigInteger = new BigInteger(Long.toString(((Longs) docValue).getValue())); // Check if it's prime return bigInteger.isProbablePrime(certainty); } catch (NumberFormatException ex) { diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java index caaa3c321aff9..d3593871f4c9a 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java @@ -117,12 +117,11 @@ private LookupScript(Client client, ESLogger logger, Cache @Override public Object run() { // First we get field using doc lookup - ScriptDocValues.Strings fieldData = (ScriptDocValues.Strings )doc().get(field); - + ScriptDocValues docValue = (ScriptDocValues)doc().get(field); // This is not very efficient // Check if field exists - if (fieldData != null && !fieldData.isEmpty()) { - final String fieldValue = fieldData.getValue(); + if (docValue != null && !docValue.isEmpty()) { + final String fieldValue = ((ScriptDocValues.Strings) docValue).getValue(); if (fieldValue != null) { try { return cache.get(new Tuple(lookupIndex + "/" + lookupType, fieldValue), new Callable>() { diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptFactory.java b/src/main/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptFactory.java index 0e8f0eb978406..198c093766621 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptFactory.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptFactory.java @@ -42,8 +42,9 @@ public PopularityScoreScript(String field) { @Override public float runAsFloat() { - ScriptDocValues.Longs fieldData = (ScriptDocValues.Longs) doc().get(field); - if (fieldData != null && !fieldData.isEmpty()) { + ScriptDocValues docValue = (ScriptDocValues) doc().get(field); + if (docValue != null && !docValue.isEmpty()) { + ScriptDocValues.Longs fieldData = (ScriptDocValues.Longs) docValue; double boost = 1 + Math.log10(fieldData.getValue() + 1); // Because this script is used in custom_score script the value of score() is populated. // In all other cases doc().getScore() should be used instead. From 4d774aa65b0c081b6966977a24a945e17746ed33 Mon Sep 17 00:00:00 2001 From: Britta Weber Date: Wed, 18 Dec 2013 15:42:23 +0100 Subject: [PATCH 29/69] fix popularity test The doc rec-0 got the same score as rec5-9 so rec0 cannot be assumed to be at osition 5 when retrieving result. Adding 1 to the "number" field fixes that. --- .../nativescript/script/PopularityScoreScriptTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptTests.java index 7ed0596753cf4..75ef9ff30f16c 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptTests.java @@ -46,7 +46,7 @@ public void testPopularityScoring() throws Exception { client().prepareIndex("test", "type", Integer.toString(i)) .setSource(XContentFactory.jsonBuilder().startObject() .field("name", "rec " + i) - .field("number", i) + .field("number", i + 1) .endObject())); } // Index a few records with empty number From a6db7f9ea42c2742b7e72c3f7a09dae2d12fcfc6 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Wed, 18 Dec 2013 18:09:09 -0500 Subject: [PATCH 30/69] Add -s flag to curl requests in the sample scripts --- example/lookup.sh | 42 ++++++++++++++++++++++-------------------- example/popularity.sh | 11 ++++++----- example/primes.sh | 11 ++++++----- example/random.sh | 9 +++++---- 4 files changed, 39 insertions(+), 34 deletions(-) diff --git a/example/lookup.sh b/example/lookup.sh index 3b5a12646990a..8383ee9c88415 100755 --- a/example/lookup.sh +++ b/example/lookup.sh @@ -1,6 +1,7 @@ -curl -XDELETE "http://localhost:9200/test" +#!/bin/sh +curl -s -XDELETE "http://localhost:9200/test" echo -curl -XPUT "http://localhost:9200/test/" -d '{ +curl -s -XPUT "http://localhost:9200/test/" -d '{ "settings": { "index.number_of_shards": 1, "index.number_of_replicas": 0 @@ -36,30 +37,31 @@ curl -XPUT "http://localhost:9200/test/" -d '{ } }' echo -curl -XPUT "localhost:9200/test/state/CT" -d '{"name": "Connecticut", "capital": "Hartford", "nickname": "Constitution State"}' -curl -XPUT "localhost:9200/test/state/ME" -d '{"name": "Maine", "capital": "Augusta", "nickname": "Lumber State"}' -curl -XPUT "localhost:9200/test/state/MA" -d '{"name": "Massachusetts", "capital": "Boston", "nickname": "Bay State"}' -curl -XPUT "localhost:9200/test/state/NH" -d '{"name": "New Hampshire", "capital": "Concord", "nickname": "Granite State"}' -curl -XPUT "localhost:9200/test/state/RI" -d '{"name": "Rhode Island", "capital": "Providence", "nickname": "Little Rhody"}' -curl -XPUT "localhost:9200/test/state/VT" -d '{"name": "Vermont", "capital": "Montpelier", "nickname": "Green Mountain State"}' +curl -s -XPUT "localhost:9200/test/state/CT" -d '{"name": "Connecticut", "capital": "Hartford", "nickname": "Constitution State"}' +curl -s -XPUT "localhost:9200/test/state/ME" -d '{"name": "Maine", "capital": "Augusta", "nickname": "Lumber State"}' +curl -s -XPUT "localhost:9200/test/state/MA" -d '{"name": "Massachusetts", "capital": "Boston", "nickname": "Bay State"}' +curl -s -XPUT "localhost:9200/test/state/NH" -d '{"name": "New Hampshire", "capital": "Concord", "nickname": "Granite State"}' +curl -s -XPUT "localhost:9200/test/state/RI" -d '{"name": "Rhode Island", "capital": "Providence", "nickname": "Little Rhody"}' +curl -s -XPUT "localhost:9200/test/state/VT" -d '{"name": "Vermont", "capital": "Montpelier", "nickname": "Green Mountain State"}' -curl -XPUT "localhost:9200/test/city/1" -d '{"city": "Cambridge", "state": "MA", "population": 105162}' -curl -XPUT "localhost:9200/test/city/2" -d '{"city": "South Burlington", "state": "VT", "population": 17904}' -curl -XPUT "localhost:9200/test/city/3" -d '{"city": "South Portland", "state": "ME", "population": 25002}' -curl -XPUT "localhost:9200/test/city/4" -d '{"city": "Essex", "state": "VT", "population": 19587}' -curl -XPUT "localhost:9200/test/city/5" -d '{"city": "Portland", "state": "ME", "population": 66194}' -curl -XPUT "localhost:9200/test/city/6" -d '{"city": "Burlington", "state": "VT", "population": 42417}' -curl -XPUT "localhost:9200/test/city/7" -d '{"city": "Stamford", "state": "CT", "population": 122643}' -curl -XPUT "localhost:9200/test/city/8" -d '{"city": "Colchester", "state": "VT", "population": 17067}' -curl -XPUT "localhost:9200/test/city/9" -d '{"city": "Concord", "state": "NH", "population": 42695}' -curl -XPUT "localhost:9200/test/city/10" -d '{"city": "Boston", "state": "MA", "population": 617594}' +curl -s -XPUT "localhost:9200/test/city/1" -d '{"city": "Cambridge", "state": "MA", "population": 105162}' +curl -s -XPUT "localhost:9200/test/city/2" -d '{"city": "South Burlington", "state": "VT", "population": 17904}' +curl -s -XPUT "localhost:9200/test/city/3" -d '{"city": "South Portland", "state": "ME", "population": 25002}' +curl -s -XPUT "localhost:9200/test/city/4" -d '{"city": "Essex", "state": "VT", "population": 19587}' +curl -s -XPUT "localhost:9200/test/city/5" -d '{"city": "Portland", "state": "ME", "population": 66194}' +curl -s -XPUT "localhost:9200/test/city/6" -d '{"city": "Burlington", "state": "VT", "population": 42417}' +curl -s -XPUT "localhost:9200/test/city/7" -d '{"city": "Stamford", "state": "CT", "population": 122643}' +curl -s -XPUT "localhost:9200/test/city/8" -d '{"city": "Colchester", "state": "VT", "population": 17067}' +curl -s -XPUT "localhost:9200/test/city/9" -d '{"city": "Concord", "state": "NH", "population": 42695}' +curl -s -XPUT "localhost:9200/test/city/10" -d '{"city": "Boston", "state": "MA", "population": 617594}' -curl -XPOST "http://localhost:9200/test/_refresh" +curl -s -XPOST "http://localhost:9200/test/_refresh" echo -curl "localhost:9200/test/city/_search?pretty=true" -d '{ +curl -s -XGET "localhost:9200/test/city/_search?pretty=true" -d '{ "query": { "match_all": { } }, + "fields": ["city", "state"], "script_fields": { "state_info": { "script": "lookup", diff --git a/example/popularity.sh b/example/popularity.sh index 5ad118fdfcd32..286431986a0ea 100755 --- a/example/popularity.sh +++ b/example/popularity.sh @@ -1,6 +1,7 @@ -curl -XDELETE "http://localhost:9200/test" +#!/bin/sh +curl -s -XDELETE "http://localhost:9200/test" echo -curl -XPUT "http://localhost:9200/test/" -d '{ +curl -s -XPUT "http://localhost:9200/test/" -d '{ "settings": { "index.number_of_shards": 1, "index.number_of_replicas": 0 @@ -19,9 +20,9 @@ curl -XPUT "http://localhost:9200/test/" -d '{ } }' echo -curl -XPUT "localhost:9200/test/type1/1" -d '{"name" : "foo bar baz", "number": 10000 }' -curl -XPUT "localhost:9200/test/type1/2" -d '{"name" : "foo foo foo", "number": 1 }' -curl -XPOST "http://localhost:9200/test/_refresh" +curl -s -XPUT "localhost:9200/test/type1/1" -d '{"name" : "foo bar baz", "number": 10000 }' +curl -s -XPUT "localhost:9200/test/type1/2" -d '{"name" : "foo foo foo", "number": 1 }' +curl -s -XPOST "http://localhost:9200/test/_refresh" echo curl -s "localhost:9200/test/type1/_search?pretty=true" -d '{ "query": { diff --git a/example/primes.sh b/example/primes.sh index 4d4f315998f67..daf8e6fed5e47 100755 --- a/example/primes.sh +++ b/example/primes.sh @@ -1,6 +1,7 @@ -curl -XDELETE "http://localhost:9200/test" +#!/bin/sh +curl -s -XDELETE "http://localhost:9200/test" echo -curl -XPUT "http://localhost:9200/test/" -d '{ +curl -s -XPUT "http://localhost:9200/test/" -d '{ "settings": { "index.number_of_shards": 1, "index.number_of_replicas": 0 @@ -19,10 +20,10 @@ curl -XPUT "http://localhost:9200/test/" -d '{ } }' echo -for i in {0..100}; do curl -XPUT "localhost:9200/test/type1/$i" -d "{\"name\":\"rec $i\", \"number\":$i}"; done -curl -XPOST "http://localhost:9200/test/_refresh" +for i in {0..100}; do curl -s -XPUT "localhost:9200/test/type1/$i" -d "{\"name\":\"rec $i\", \"number\":$i}"; done +curl -s -XPOST "http://localhost:9200/test/_refresh" echo -curl "localhost:9200/test/type1/_search?pretty=true" -d '{ +curl -s "localhost:9200/test/type1/_search?pretty=true" -d '{ "query": { "filtered": { "query": { diff --git a/example/random.sh b/example/random.sh index b714e496c09bb..ac07dbec5777d 100755 --- a/example/random.sh +++ b/example/random.sh @@ -1,6 +1,7 @@ -curl -XDELETE "http://localhost:9200/test" +#!/bin/sh +curl -s -XDELETE "http://localhost:9200/test" echo -curl -XPUT "http://localhost:9200/test/" -d '{ +curl -s -XPUT "http://localhost:9200/test/" -d '{ "settings": { "index.number_of_shards": 1, "index.number_of_replicas": 0 @@ -19,8 +20,8 @@ curl -XPUT "http://localhost:9200/test/" -d '{ } }' echo -for i in {0..100}; do curl -XPUT "localhost:9200/test/type1/$i" -d "{\"name\":\"rec $i\", \"number\":$i}"; done -curl -XPOST "http://localhost:9200/test/_refresh" +for i in {0..100}; do curl -s -XPUT "localhost:9200/test/type1/$i" -d "{\"name\":\"rec $i\", \"number\":$i}"; done +curl -s -XPOST "http://localhost:9200/test/_refresh" echo echo "Salt 123" curl -s "localhost:9200/test/type1/_search?pretty=true" -d '{ From 5604d3dc5a6fc5ff6752874a65a6b5452279c6a5 Mon Sep 17 00:00:00 2001 From: Britta Weber Date: Thu, 19 Dec 2013 12:11:14 +0100 Subject: [PATCH 31/69] add terms based scoring scripts Examples for term based script scoring in a native script. See also http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/modules-advanced-scripting.html This adds scripts for - cosime similarity - tfidf - language model scoring - simple phrase scorer --- example/termscoring.sh | 262 ++++++++++++++++++ pom.xml | 4 +- .../plugin/NativeScriptExamplesPlugin.java | 8 + .../script/CosineSimilarityScoreScript.java | 102 +++++++ .../script/LanguageModelScoreScript.java | 131 +++++++++ .../script/PhraseScoreScript.java | 88 ++++++ .../nativescript/script/TFIDFScoreScript.java | 90 ++++++ .../script/TermScoringScriptTests.java | 173 ++++++++++++ 8 files changed, 856 insertions(+), 2 deletions(-) create mode 100755 example/termscoring.sh create mode 100644 src/main/java/org/elasticsearch/examples/nativescript/script/CosineSimilarityScoreScript.java create mode 100644 src/main/java/org/elasticsearch/examples/nativescript/script/LanguageModelScoreScript.java create mode 100644 src/main/java/org/elasticsearch/examples/nativescript/script/PhraseScoreScript.java create mode 100644 src/main/java/org/elasticsearch/examples/nativescript/script/TFIDFScoreScript.java create mode 100644 src/test/java/org/elasticsearch/examples/nativescript/script/TermScoringScriptTests.java diff --git a/example/termscoring.sh b/example/termscoring.sh new file mode 100755 index 0000000000000..45f2f4b4ba3ad --- /dev/null +++ b/example/termscoring.sh @@ -0,0 +1,262 @@ +#!/bin/sh + +# Init an index that includes wordcount for the field that we will be searching on: + +curl -XPUT "http://localhost:9200/termscore" -d' +{ + "settings": { + "number_of_shards": 1, + "number_of_replicas": 0 + }, + "mappings": { + "doc": { + "properties": { + "text": { + "type": "multi_field", + "fields": { + "text": { + "type": "string" + }, + "word_count": { + "type": "token_count", + "store": "yes", + "analyzer": "standard" + } + } + } + } + } + } +}' + +curl -XPOST "http://localhost:9200/termscore/doc" -d' +{ + "text": "John Smith is the main villain in the Matrix." +}' + +curl -XPOST "http://localhost:9200/termscore/doc" -d' +{ + "text": "John \"Hannibal\" Smith is the leader of the A-team." +}' + +curl -XPOST "http://localhost:9200/termscore/_refresh" + +# search for people that are named John Smith: + +curl -XPOST "http://localhost:9200/termscore/doc/_search?pretty" -d' +{ + "query": { + "function_score": { + "query": { + "bool": { + "must": [ + { + "match": { + "text": { + "query": "john smith", + "operator": "and" + } + } + } + ] + } + }, + "functions": [ + { + "script_score": { + "params": { + "field": "text", + "terms": [ + "john", + "smith" + ] + }, + "script": "phrase_script_score", + "lang": "native" + } + } + ], + "boost_mode": "replace" + } + } +}' + +curl -XPOST "http://localhost:9200/termscore/doc" -d' +{ + "text": "I am Sam. I am Sam. Sam I am. That Sam-I-am. That Sam-I-am! I do not like that Sam-I-am. Do you like green eggs and ham? I do not like them, Sam-I-am.I do not like green eggs and ham." +}' + +curl -XPOST "http://localhost:9200/termscore/doc" -d' +{ + "text": "Would you like them Here or there? I would not like them here or there. I would not like them anywhere. I do not like green eggs and ham. I do not like them, Sam-I-am." +}' + +curl -XPOST "http://localhost:9200/termscore/doc" -d' +{ + "text": "Would you like them in a house? Would you like them with a mouse? I do not like them in a house. I do not like them with a mouse. I do not like them here or there. I do not like them anywhere. I do not like green eggs and ham. I do not like them, Sam-I-am." +}' + +curl -XPOST "http://localhost:9200/termscore/_refresh" + +# Search for "I am Sam" with cosine similarity: + +curl -XPOST "http://localhost:9200/termscore/doc/_search?pretty" -d' +{ + "script_fields": { + "i-tf": { + "script": "_index[\"text\"][\"i\"].tf()" + }, + "am-tf": { + "script": "_index[\"text\"][\"am\"].tf()" + }, + "sam-tf": { + "script": "_index[\"text\"][\"sam\"].tf()" + } + }, + "query": { + "function_score": { + "query": { + "bool": { + "must": [ + { + "match": { + "text": { + "query": "sam i am", + "operator": "and" + } + } + } + ] + } + }, + "functions": [ + { + "script_score": { + "params": { + "field": "text", + "terms": [ + "sam", + "i", + "am" + ], + "weights": [ + 1.0, + 1.0, + 1.0 + ] + }, + "script": "cosine_sim_script_score", + "lang": "native" + } + } + ], + "boost_mode": "replace" + } + } +}' + +# Search for "I am Sam" with naive tf-ifd score: + +curl -XPOST "http://localhost:9200/termscore/doc/_search?pretty" -d' +{ + "script_fields": { + "i-tf": { + "script": "_index[\"text\"][\"i\"].tf()" + }, + "am-tf": { + "script": "_index[\"text\"][\"am\"].tf()" + }, + "sam-tf": { + "script": "_index[\"text\"][\"sam\"].tf()" + } + }, + "query": { + "function_score": { + "query": { + "bool": { + "must": [ + { + "match": { + "text": { + "query": "sam i am", + "operator": "and" + } + } + } + ] + } + }, + "functions": [ + { + "script_score": { + "params": { + "field": "text", + "terms": [ + "sam", + "i", + "am" + ] + }, + "script": "tfidf_script_score", + "lang": "native" + } + } + ], + "boost_mode": "replace" + } + } +}' + +# Search for "I am Sam" with language model scoring: + +curl -XPOST "http://localhost:9200/termscore/doc/_search?pretty" -d' +{ + "script_fields": { + "i-tf": { + "script": "_index[\"text\"][\"i\"].tf()" + }, + "am-tf": { + "script": "_index[\"text\"][\"am\"].tf()" + }, + "sam-tf": { + "script": "_index[\"text\"][\"sam\"].tf()" + } + }, + "query": { + "function_score": { + "query": { + "bool": { + "must": [ + { + "match": { + "text": { + "query": "sam i am", + "operator": "and" + } + } + } + ] + } + }, + "functions": [ + { + "script_score": { + "params": { + "field": "text", + "terms": [ + "sam", + "i", + "am" + ], + "lambda": 0.5, + "word_count_field": "text.word_count" + }, + "script": "language_model_script_score", + "lang": "native" + } + } + ], + "boost_mode": "replace" + } + } +}' \ No newline at end of file diff --git a/pom.xml b/pom.xml index d6bcc9a7a5700..a504dec6e1b0c 100644 --- a/pom.xml +++ b/pom.xml @@ -38,7 +38,7 @@ - 0.90.8 + 0.90.10 4.6.0 @@ -146,4 +146,4 @@ - \ No newline at end of file + diff --git a/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java b/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java index 3b2288b5224ec..141ade74b3209 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java @@ -1,7 +1,11 @@ package org.elasticsearch.examples.nativescript.plugin; import org.elasticsearch.examples.nativescript.script.IsPrimeSearchScript; +import org.elasticsearch.examples.nativescript.script.LanguageModelScoreScript; import org.elasticsearch.examples.nativescript.script.LookupScript; +import org.elasticsearch.examples.nativescript.script.CosineSimilarityScoreScript; +import org.elasticsearch.examples.nativescript.script.PhraseScoreScript; +import org.elasticsearch.examples.nativescript.script.TFIDFScoreScript; import org.elasticsearch.examples.nativescript.script.PopularityScoreScriptFactory; import org.elasticsearch.examples.nativescript.script.RandomSortScriptFactory; import org.elasticsearch.plugins.AbstractPlugin; @@ -42,5 +46,9 @@ public void onModule(ScriptModule module) { module.registerScript("lookup", LookupScript.Factory.class); module.registerScript("random", RandomSortScriptFactory.class); module.registerScript("popularity", PopularityScoreScriptFactory.class); + module.registerScript(TFIDFScoreScript.SCRIPT_NAME, TFIDFScoreScript.Factory.class); + module.registerScript(CosineSimilarityScoreScript.SCRIPT_NAME, CosineSimilarityScoreScript.Factory.class); + module.registerScript(PhraseScoreScript.SCRIPT_NAME, PhraseScoreScript.Factory.class); + module.registerScript(LanguageModelScoreScript.SCRIPT_NAME, LanguageModelScoreScript.Factory.class); } } diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/CosineSimilarityScoreScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/CosineSimilarityScoreScript.java new file mode 100644 index 0000000000000..e769d8b194a82 --- /dev/null +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/CosineSimilarityScoreScript.java @@ -0,0 +1,102 @@ +package org.elasticsearch.examples.nativescript.script; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Map; + +import org.elasticsearch.ElasticSearchException; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.script.AbstractSearchScript; +import org.elasticsearch.script.ExecutableScript; +import org.elasticsearch.script.NativeScriptFactory; +import org.elasticsearch.search.lookup.IndexField; +import org.elasticsearch.search.lookup.IndexFieldTerm; + +/** + * Script that scores documents with cosine similarity, see Manning et al., + * "Information Retrieval", Chapter 6, Eq. 6.12 (link: + * http://nlp.stanford.edu/IR-book/). This implementation only scores a list of + * terms on one field. + */ +public class CosineSimilarityScoreScript extends AbstractSearchScript { + + // the field containing the terms that should be scored, must be initialized + // in constructor from parameters. + String field = null; + // terms that are used for scoring, must be unique + ArrayList terms = null; + // weights, in case we want to put emphasis on a specific term. In the most + // simple case, 1.0 for every term. + ArrayList weights = null; + + final static public String SCRIPT_NAME = "cosine_sim_script_score"; + + /** + * Factory that is registered in + * {@link org.elasticsearch.examples.nativescript.plugin.NativeScriptExamplesPlugin#onModule(org.elasticsearch.script.ScriptModule)} + * method when the plugin is loaded. + */ + public static class Factory implements NativeScriptFactory { + + /** + * This method is called for every search on every shard. + * + * @param params + * list of script parameters passed with the query + * @return new native script + */ + @Override + public ExecutableScript newScript(@Nullable Map params) { + return new CosineSimilarityScoreScript(params); + } + } + + /** + * @param params + * terms that a scored are placed in this parameter. Initialize + * them here. + */ + private CosineSimilarityScoreScript(Map params) { + params.entrySet(); + // get the terms + terms = (ArrayList) params.get("terms"); + weights = (ArrayList) params.get("weights"); + // get the field + field = (String) params.get("field"); + if (field == null || terms == null || weights == null) { + throw new ElasticSearchException("cannot initialize " + SCRIPT_NAME + ": field, terms or weights parameter missing!"); + } + if (weights.size() != terms.size()) { + throw new ElasticSearchException("cannot initialize " + SCRIPT_NAME + ": terms and weights array must have same length!"); + } + } + + @Override + public Object run() { + try { + float score = 0; + // first, get the ShardTerms object for the field. + IndexField indexField = this.indexLookup().get(field); + double queryWeightSum = 0.0f; + double docWeightSum = 0.0f; + for (int i = 0; i < terms.size(); i++) { + // Now, get the ShardTerm object that can be used to access all + // the term statistics + IndexFieldTerm indexTermField = indexField.get(terms.get(i)); + // compute the most naive tfidf and add to current score + int df = (int) indexTermField.df(); + int tf = indexTermField.tf(); + if (df != 0 && tf != 0) { + double termscore = (double) tf * weights.get(i); + score += termscore; + docWeightSum += Math.pow(tf, 2.0); + } + queryWeightSum += Math.pow(weights.get(i), 2.0); + } + return score / (Math.sqrt(docWeightSum) * Math.sqrt(queryWeightSum)); + } catch (IOException ex) { + throw new ElasticSearchException("Could not compute cosine similarity: ", ex); + } + } + +} diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/LanguageModelScoreScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/LanguageModelScoreScript.java new file mode 100644 index 0000000000000..42aed66950bbb --- /dev/null +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/LanguageModelScoreScript.java @@ -0,0 +1,131 @@ +package org.elasticsearch.examples.nativescript.script; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Map; + +import org.elasticsearch.ElasticSearchException; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.script.AbstractSearchScript; +import org.elasticsearch.script.ExecutableScript; +import org.elasticsearch.script.NativeScriptFactory; +import org.elasticsearch.search.lookup.IndexField; +import org.elasticsearch.search.lookup.IndexFieldTerm; + +/** + * Script that scores documents with a language model similarity with linear + * interpolation, see Manning et al., "Information Retrieval", Chapter 12, + * Equation 12.12 (link: http://nlp.stanford.edu/IR-book/) This implementation + * only scores a list of terms on one field. + */ +public class LanguageModelScoreScript extends AbstractSearchScript { + + // the field containing the terms that should be scored, must be initialized + // in constructor from parameters. + String field; + // name of the field that holds the word count of a field, see + // http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/mapping-core-types.html) + String docLengthField; + // terms that are used for scoring + ArrayList terms; + // lambda parameter + float lambda; + + final static public String SCRIPT_NAME = "language_model_script_score"; + + /** + * Factory that is registered in + * {@link org.elasticsearch.examples.nativescript.plugin.NativeScriptExamplesPlugin#onModule(org.elasticsearch.script.ScriptModule)} + * method when the plugin is loaded. + */ + public static class Factory implements NativeScriptFactory { + + /** + * This method is called for every search on every shard. + * + * @param params + * list of script parameters passed with the query + * @return new native script + */ + @Override + public ExecutableScript newScript(@Nullable Map params) { + return new LanguageModelScoreScript(params); + } + } + + /** + * @param params + * terms that a scored are placed in this parameter. Initialize + * them here. + */ + private LanguageModelScoreScript(Map params) { + params.entrySet(); + // get the terms + terms = (ArrayList) params.get("terms"); + // get the field + field = (String) params.get("field"); + // get the field holding the document length + docLengthField = (String) params.get("word_count_field"); + // get lambda + lambda = ((Double) params.get("lambda")).floatValue(); + if (field == null || terms == null || docLengthField == null) { + throw new ElasticSearchException("cannot initialize " + SCRIPT_NAME + ": field, terms or length field parameter missing!"); + } + } + + @Override + public Object run() { + try { + double score = 0.0; + // first, get the ShardTerms object for the field. + IndexField indexField = indexLookup().get(field); + long T = indexField.sumttf(); + /* + * document length cannot be obtained by the shardTerms, we use the + * word_count field instead (link: + * http://www.elasticsearch.org/guide + * /en/elasticsearch/reference/current/mapping-core-types.html) + */ + ScriptDocValues docValues = (ScriptDocValues) doc().get(docLengthField); + if (docValues == null || !docValues.isEmpty()) { + long L_d = ((ScriptDocValues.Longs) docValues).getValue(); + for (int i = 0; i < terms.size(); i++) { + // Now, get the ShardTerm object that can be used to access + // all + // the term statistics + IndexFieldTerm indexFieldTerm = indexField.get(terms.get(i)); + + /* + * compute M_c as ttf/T, see Manning et al., + * "Information Retrieval", Chapter 12, Equation just before + * Equation 12.10 (link: http://nlp.stanford.edu/IR-book/) + */ + long cf_t = indexFieldTerm.ttf(); + double M_c = (double) cf_t / (double) T; + /* + * Compute M_d, see Manning et al., "Information Retrieval", + * Chapter 12, Equation just before Equation 12.9 (link: + * http://nlp.stanford.edu/IR-book/) + */ + double M_d = (double) indexFieldTerm.tf() / (double) L_d; + /* + * compute score contribution for this term, but sum the log + * to avoid underflow, see Manning et al., + * "Information Retrieval", Chapter 12, Equation 12.12 + * (link: http://nlp.stanford.edu/IR-book/) + */ + score += Math.log((1.0 - lambda) * M_c + lambda * M_d); + + } + return score; + } else { + throw new ElasticSearchException("Could not compute language model score, word count field missing."); + } + + } catch (IOException ex) { + throw new ElasticSearchException("Could not compute language model score: ", ex); + } + } + +} diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/PhraseScoreScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/PhraseScoreScript.java new file mode 100644 index 0000000000000..fafbb62dd937e --- /dev/null +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/PhraseScoreScript.java @@ -0,0 +1,88 @@ +package org.elasticsearch.examples.nativescript.script; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.Map; + +import org.elasticsearch.ElasticSearchException; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.script.AbstractSearchScript; +import org.elasticsearch.script.ExecutableScript; +import org.elasticsearch.script.NativeScriptFactory; +import org.elasticsearch.search.lookup.IndexField; +import org.elasticsearch.search.lookup.IndexLookup; +import org.elasticsearch.search.lookup.TermPosition; + +/** + * Script that scores documents by distance of two given terms in a text, see + * Manning et al., "Information Retrieval", Chapter 2.4 (link: + * http://nlp.stanford.edu/IR-book/) for more information on positional indexes. + * Might be useful if you search for names and know first and last name. + */ +public class PhraseScoreScript extends AbstractSearchScript { + + // the field containing the terms that should be scored, must be initialized + // in constructor from parameters. + String field = null; + // terms that are used for scoring + ArrayList terms = null; + + final static public String SCRIPT_NAME = "phrase_script_score"; + + /** + * Factory that is registered in + * {@link org.elasticsearch.examples.nativescript.plugin.NativeScriptExamplesPlugin#onModule(org.elasticsearch.script.ScriptModule)} + * method when the plugin is loaded. + */ + public static class Factory implements NativeScriptFactory { + + /** + * This method is called for every search on every shard. + * + * @param params + * list of script parameters passed with the query + * @return new native script + */ + @Override + public ExecutableScript newScript(@Nullable Map params) { + return new PhraseScoreScript(params); + } + } + + /** + * @param params + * terms that a scored are placed in this parameter. Initialize + * them here. + */ + private PhraseScoreScript(Map params) { + params.entrySet(); + // get the terms + terms = (ArrayList) params.get("terms"); + // get the field + field = (String) params.get("field"); + if (field == null || terms == null) { + throw new ElasticSearchException("cannot initialize " + SCRIPT_NAME + ": field or terms parameter missing!"); + } + assert (terms.size() == 2); + } + + @Override + public Object run() { + double score = 1.e10; + // first, get the ShardTerms object for the field. + IndexField indexField = this.indexLookup().get(field); + // get the positions iterators + Iterator firstNameIter = indexField.get(terms.get(0), IndexLookup.FLAG_POSITIONS).iterator(); + Iterator lastNameIter = indexField.get(terms.get(1), IndexLookup.FLAG_POSITIONS).iterator(); + int lastNamePos = -1; + while (firstNameIter.hasNext()) { + int firstNamePos = firstNameIter.next().position; + while (lastNameIter.hasNext() && lastNamePos < firstNamePos) { + lastNamePos = lastNameIter.next().position; + } + score = Math.min(score, lastNamePos - firstNamePos); + } + return 1.0 / (float) score; + } + +} diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/TFIDFScoreScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/TFIDFScoreScript.java new file mode 100644 index 0000000000000..ce79837ebae67 --- /dev/null +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/TFIDFScoreScript.java @@ -0,0 +1,90 @@ +package org.elasticsearch.examples.nativescript.script; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Map; + +import org.elasticsearch.ElasticSearchException; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.script.AbstractSearchScript; +import org.elasticsearch.script.ExecutableScript; +import org.elasticsearch.script.NativeScriptFactory; +import org.elasticsearch.search.lookup.IndexField; +import org.elasticsearch.search.lookup.IndexFieldTerm; + +/** + * Script that scores documents as sum_t(tf_t * (#docs+2)/(df_t+1)), which + * equals ntn in SMART notation, see Manning et al., "Information Retrieval", + * Chapter 6, Figure 6.15 (link: http://nlp.stanford.edu/IR-book/) This + * implementation only scores a list of terms on one field. + */ +public class TFIDFScoreScript extends AbstractSearchScript { + + // the field containing the terms that should be scored, must be initialized + // in constructor from parameters. + String field = null; + // terms that are used for scoring + ArrayList terms = null; + + final static public String SCRIPT_NAME = "tfidf_script_score"; + + /** + * Factory that is registered in + * {@link org.elasticsearch.examples.nativescript.plugin.NativeScriptExamplesPlugin#onModule(org.elasticsearch.script.ScriptModule)} + * method when the plugin is loaded. + */ + public static class Factory implements NativeScriptFactory { + + /** + * This method is called for every search on every shard. + * + * @param params + * list of script parameters passed with the query + * @return new native script + */ + @Override + public ExecutableScript newScript(@Nullable Map params) { + return new TFIDFScoreScript(params); + } + } + + /** + * @param params + * terms that a scored are placed in this parameter. Initialize + * them here. + */ + private TFIDFScoreScript(Map params) { + params.entrySet(); + // get the terms + terms = (ArrayList) params.get("terms"); + // get the field + field = (String) params.get("field"); + if (field == null || terms == null) { + throw new ElasticSearchException("cannot initialize " + SCRIPT_NAME + ": field or terms parameter missing!"); + } + } + + @Override + public Object run() { + try { + float score = 0; + // first, get the IndexField object for the field. + IndexField indexField = indexLookup().get(field); + for (int i = 0; i < terms.size(); i++) { + // Now, get the IndexFieldTerm object that can be used to access all + // the term statistics + IndexFieldTerm indexFieldTerm = indexField.get(terms.get(i)); + // compute the most naive tfidf and add to current score + int df = (int) indexFieldTerm.df(); + int tf = indexFieldTerm.tf(); + if (df != 0 && tf != 0) { + score += (float) indexFieldTerm.tf() * Math.log(((float) indexField.docCount() + 2.0) / ((float) df + 1.0)); + } + } + return score; + } catch (IOException ex) { + throw new ElasticSearchException("Could not compute tfidf: ", ex); + } + } + +} diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/TermScoringScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/TermScoringScriptTests.java new file mode 100644 index 0000000000000..055e054c4db67 --- /dev/null +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/TermScoringScriptTests.java @@ -0,0 +1,173 @@ +package org.elasticsearch.examples.nativescript.script; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; +import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.equalTo; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; + +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.lucene.search.function.CombineFunction; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders; +import org.elasticsearch.search.SearchHit; +import org.junit.Test; + +/** + * Test if the computed tfidf in NaiveTFIDFScoreScript equals 0.0 for each + * document as would be expected if each document in the index contains only one + * and always the same term. + * + */ +public class TermScoringScriptTests extends AbstractSearchScriptTests { + + final static String[] searchTerms = { "foo", "bar" }; + final static String field = "field"; + final static String wordCountField = field + ".word_count"; + final static String placeholder = "placeholder"; + final static Double[] weights = { 1.0, 1.0 }; + final static int numDocs = 100; + + @Test + public void testTFIDF() throws Exception { + + initData(); + + // initialize parameters for script + Map params = new HashMap(); + params.put("field", field); + params.put("terms", searchTerms); + + // Retrieve records and see if they scored 0.0 + SearchResponse searchResponse = client() + .prepareSearch("test") + .setQuery( + QueryBuilders.functionScoreQuery() + .add(ScoreFunctionBuilders.scriptFunction(TFIDFScoreScript.SCRIPT_NAME, "native", params)) + .boostMode(CombineFunction.REPLACE.getName())).setSize(numDocs).execute().actionGet(); + assertNoFailures(searchResponse); + assertHitCount(searchResponse, numDocs); + SearchHit[] hits = searchResponse.getHits().hits(); + for (int i = 0; i < numDocs; i++) { + assertThat(hits[i].getId(), equalTo(Integer.toString(numDocs - i - 1))); + } + + } + + @Test + public void testCosineSimilarity() throws Exception { + + initData(); + + // initialize parameters for script + Map params = new HashMap(); + params.put("field", field); + params.put("terms", searchTerms); + params.put("weights", weights); + + // Retrieve records and see if they scored 0.0 + SearchResponse searchResponse = client() + .prepareSearch("test") + .setQuery( + QueryBuilders.functionScoreQuery() + .add(ScoreFunctionBuilders.scriptFunction(CosineSimilarityScoreScript.SCRIPT_NAME, "native", params)) + .boostMode(CombineFunction.REPLACE.getName())).setSize(numDocs).execute().actionGet(); + assertNoFailures(searchResponse); + assertHitCount(searchResponse, numDocs); + SearchHit[] hits = searchResponse.getHits().hits(); + for (int i = 0; i < numDocs; i++) { + assertThat(hits[i].getId(), equalTo(Integer.toString(i))); + } + } + + @Test + public void testPhraseScorer() throws Exception { + + initData(); + + // initialize parameters for script + Map params = new HashMap(); + params.put("field", field); + params.put("terms", searchTerms); + + // Retrieve records and see if they scored 0.0 + SearchResponse searchResponse = client() + .prepareSearch("test") + .setQuery( + QueryBuilders.functionScoreQuery() + .add(ScoreFunctionBuilders.scriptFunction(PhraseScoreScript.SCRIPT_NAME, "native", params)) + .boostMode(CombineFunction.REPLACE.getName())).setSize(numDocs).execute().actionGet(); + assertNoFailures(searchResponse); + assertHitCount(searchResponse, numDocs); + SearchHit[] hits = searchResponse.getHits().hits(); + for (int i = 0; i < numDocs; i++) { + assertThat(hits[i].getId(), equalTo(Integer.toString(i))); + assertThat((double) hits[i].score(), closeTo(1.0 / (float) (i + 2), 1.e-6)); + } + } + + @Test + public void testLanguageModelScorer() throws Exception { + + initData(); + + // initialize parameters for script + Map params = new HashMap(); + params.put("field", field); + params.put("terms", searchTerms); + params.put("word_count_field", wordCountField); + params.put("lambda", 0.9); + + // Retrieve records and see if they scored 0.0 + SearchResponse searchResponse = client() + .prepareSearch("test") + .setQuery( + QueryBuilders.functionScoreQuery() + .add(ScoreFunctionBuilders.scriptFunction(LanguageModelScoreScript.SCRIPT_NAME, "native", params)) + .boostMode(CombineFunction.REPLACE.getName())).setSize(numDocs).execute().actionGet(); + assertNoFailures(searchResponse); + assertHitCount(searchResponse, numDocs); + SearchHit[] hits = searchResponse.getHits().hits(); + for (int i = 0; i < numDocs; i++) { + assertThat(hits[i].getId(), equalTo(Integer.toString(i))); + } + } + + private void initData() throws IOException, InterruptedException, ExecutionException { + // Create a new index + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties").startObject(field) + .field("type", "multi_field").startObject("fields").startObject(field).field("type", "String").endObject() + .startObject("word_count").field("analyzer", "standard").field("type", "token_count").startObject("fielddata") + .field("format", "doc_values").endObject().endObject().endObject().endObject().endObject().endObject().endObject().string(); + assertAcked(client().admin().indices().prepareCreate("test").addMapping("type", mapping)); + + List indexBuilders = new ArrayList(); + // Index numDocs records (0..99) + for (int i = 0; i < numDocs; i++) { + indexBuilders.add(client().prepareIndex("test", "type", Integer.toString(i)) + .setSource(XContentFactory.jsonBuilder().startObject().field(field, createText(i + 1)).endObject()) + .setId(Integer.toString(i))); + } + indexRandom(true, indexBuilders); + } + + private String createText(int numFoo) { + String text = ""; + for (int i = 0; i < numFoo; i++) { + text = text + " foo "; + } + for (int i = 0; i < numFoo; i++) { + text = text + " " + placeholder + " "; + } + return text + " bar"; + } +} From 3e22997eccd43eb7fc02ebb5a2e8cdc93bc5eba0 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Thu, 16 Jan 2014 13:13:24 -0500 Subject: [PATCH 32/69] Add -s flag to curl requests in the termscoring script --- example/termscoring.sh | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/example/termscoring.sh b/example/termscoring.sh index 45f2f4b4ba3ad..482336324a23c 100755 --- a/example/termscoring.sh +++ b/example/termscoring.sh @@ -2,7 +2,7 @@ # Init an index that includes wordcount for the field that we will be searching on: -curl -XPUT "http://localhost:9200/termscore" -d' +curl -s -XPUT "http://localhost:9200/termscore" -d' { "settings": { "number_of_shards": 1, @@ -29,21 +29,21 @@ curl -XPUT "http://localhost:9200/termscore" -d' } }' -curl -XPOST "http://localhost:9200/termscore/doc" -d' +curl -s -XPOST "http://localhost:9200/termscore/doc" -d' { "text": "John Smith is the main villain in the Matrix." }' -curl -XPOST "http://localhost:9200/termscore/doc" -d' +curl -s -XPOST "http://localhost:9200/termscore/doc" -d' { "text": "John \"Hannibal\" Smith is the leader of the A-team." }' -curl -XPOST "http://localhost:9200/termscore/_refresh" +curl -s -XPOST "http://localhost:9200/termscore/_refresh" # search for people that are named John Smith: -curl -XPOST "http://localhost:9200/termscore/doc/_search?pretty" -d' +curl -s -XPOST "http://localhost:9200/termscore/doc/_search?pretty" -d' { "query": { "function_score": { @@ -81,26 +81,26 @@ curl -XPOST "http://localhost:9200/termscore/doc/_search?pretty" -d' } }' -curl -XPOST "http://localhost:9200/termscore/doc" -d' +curl -s -XPOST "http://localhost:9200/termscore/doc" -d' { "text": "I am Sam. I am Sam. Sam I am. That Sam-I-am. That Sam-I-am! I do not like that Sam-I-am. Do you like green eggs and ham? I do not like them, Sam-I-am.I do not like green eggs and ham." }' -curl -XPOST "http://localhost:9200/termscore/doc" -d' +curl -s -XPOST "http://localhost:9200/termscore/doc" -d' { "text": "Would you like them Here or there? I would not like them here or there. I would not like them anywhere. I do not like green eggs and ham. I do not like them, Sam-I-am." }' -curl -XPOST "http://localhost:9200/termscore/doc" -d' +curl -s -XPOST "http://localhost:9200/termscore/doc" -d' { "text": "Would you like them in a house? Would you like them with a mouse? I do not like them in a house. I do not like them with a mouse. I do not like them here or there. I do not like them anywhere. I do not like green eggs and ham. I do not like them, Sam-I-am." }' -curl -XPOST "http://localhost:9200/termscore/_refresh" +curl -s -XPOST "http://localhost:9200/termscore/_refresh" # Search for "I am Sam" with cosine similarity: -curl -XPOST "http://localhost:9200/termscore/doc/_search?pretty" -d' +curl -s -XPOST "http://localhost:9200/termscore/doc/_search?pretty" -d' { "script_fields": { "i-tf": { @@ -157,7 +157,7 @@ curl -XPOST "http://localhost:9200/termscore/doc/_search?pretty" -d' # Search for "I am Sam" with naive tf-ifd score: -curl -XPOST "http://localhost:9200/termscore/doc/_search?pretty" -d' +curl -s -XPOST "http://localhost:9200/termscore/doc/_search?pretty" -d' { "script_fields": { "i-tf": { @@ -209,7 +209,7 @@ curl -XPOST "http://localhost:9200/termscore/doc/_search?pretty" -d' # Search for "I am Sam" with language model scoring: -curl -XPOST "http://localhost:9200/termscore/doc/_search?pretty" -d' +curl -s -XPOST "http://localhost:9200/termscore/doc/_search?pretty" -d' { "script_fields": { "i-tf": { From 29975b742edfaad7589082a7710622217cd6c77e Mon Sep 17 00:00:00 2001 From: Hugues Malphettes Date: Sat, 24 May 2014 12:20:04 +0800 Subject: [PATCH 33/69] Fix LookupScriptTests mappings --- .../examples/nativescript/script/LookupScriptTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/LookupScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/LookupScriptTests.java index 1e8545bcdc57c..a9e098a6ce7fe 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/LookupScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/LookupScriptTests.java @@ -34,7 +34,7 @@ public void testLookup() throws Exception { .string(); assertAcked(prepareCreate("lookup") - .addMapping("type", lookup_mapping)); + .addMapping("state", lookup_mapping)); // Create a new test index String test_mapping = XContentFactory.jsonBuilder().startObject().startObject("city") From f3659989ca76628c318965125406263ea8d5f96c Mon Sep 17 00:00:00 2001 From: Hugues Malphettes Date: Sat, 24 May 2014 12:20:35 +0800 Subject: [PATCH 34/69] Upgrade to ES-1.2.0 --- pom.xml | 10 +++++----- .../script/CosineSimilarityScoreScript.java | 14 ++++++++------ .../nativescript/script/IsPrimeSearchScript.java | 4 ++-- .../script/LanguageModelScoreScript.java | 8 ++++---- .../examples/nativescript/script/LookupScript.java | 9 ++++----- .../nativescript/script/PhraseScoreScript.java | 4 ++-- .../script/PopularityScoreScriptFactory.java | 4 ++-- .../nativescript/script/TFIDFScoreScript.java | 6 +++--- .../script/AbstractSearchScriptTests.java | 2 +- 9 files changed, 31 insertions(+), 30 deletions(-) diff --git a/pom.xml b/pom.xml index a504dec6e1b0c..280616faa5a62 100644 --- a/pom.xml +++ b/pom.xml @@ -38,8 +38,8 @@ - 0.90.10 - 4.6.0 + 1.2.0 + 4.8.1 @@ -98,15 +98,15 @@ - + org.apache.maven.plugins maven-compiler-plugin 2.3.2 - 1.6 - 1.6 + 1.7 + 1.7 diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/CosineSimilarityScoreScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/CosineSimilarityScoreScript.java index e769d8b194a82..111959c2243da 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/CosineSimilarityScoreScript.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/CosineSimilarityScoreScript.java @@ -4,7 +4,8 @@ import java.util.ArrayList; import java.util.Map; -import org.elasticsearch.ElasticSearchException; +import org.elasticsearch.script.ScriptException; + import org.elasticsearch.common.Nullable; import org.elasticsearch.script.AbstractSearchScript; import org.elasticsearch.script.ExecutableScript; @@ -46,7 +47,7 @@ public static class Factory implements NativeScriptFactory { * @return new native script */ @Override - public ExecutableScript newScript(@Nullable Map params) { + public ExecutableScript newScript(@Nullable Map params) throws ScriptException { return new CosineSimilarityScoreScript(params); } } @@ -55,8 +56,9 @@ public ExecutableScript newScript(@Nullable Map params) { * @param params * terms that a scored are placed in this parameter. Initialize * them here. + * @throws ScriptException */ - private CosineSimilarityScoreScript(Map params) { + private CosineSimilarityScoreScript(Map params) throws ScriptException { params.entrySet(); // get the terms terms = (ArrayList) params.get("terms"); @@ -64,10 +66,10 @@ private CosineSimilarityScoreScript(Map params) { // get the field field = (String) params.get("field"); if (field == null || terms == null || weights == null) { - throw new ElasticSearchException("cannot initialize " + SCRIPT_NAME + ": field, terms or weights parameter missing!"); + throw new ScriptException("cannot initialize " + SCRIPT_NAME + ": field, terms or weights parameter missing!"); } if (weights.size() != terms.size()) { - throw new ElasticSearchException("cannot initialize " + SCRIPT_NAME + ": terms and weights array must have same length!"); + throw new ScriptException("cannot initialize " + SCRIPT_NAME + ": terms and weights array must have same length!"); } } @@ -95,7 +97,7 @@ public Object run() { } return score / (Math.sqrt(docWeightSum) * Math.sqrt(queryWeightSum)); } catch (IOException ex) { - throw new ElasticSearchException("Could not compute cosine similarity: ", ex); + throw new ScriptException("Could not compute cosine similarity: ", ex); } } diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java index 5df776ea24a72..0924dbcb7d12e 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java @@ -3,7 +3,7 @@ import java.math.BigInteger; import java.util.Map; -import org.elasticsearch.ElasticSearchIllegalArgumentException; +import org.elasticsearch.script.ScriptException; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.fielddata.ScriptDocValues; @@ -39,7 +39,7 @@ public ExecutableScript newScript(@Nullable Map params) { // The XContentMapValues helper class can be used to simplify parameter parsing String fieldName = params == null ? null : XContentMapValues.nodeStringValue(params.get("field"), null); if (fieldName == null) { - throw new ElasticSearchIllegalArgumentException("Missing the field parameter"); + throw new ScriptException("Missing the field parameter"); } // Example of an optional integer parameter diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/LanguageModelScoreScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/LanguageModelScoreScript.java index 42aed66950bbb..2211ae3bb4c35 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/LanguageModelScoreScript.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/LanguageModelScoreScript.java @@ -4,12 +4,12 @@ import java.util.ArrayList; import java.util.Map; -import org.elasticsearch.ElasticSearchException; import org.elasticsearch.common.Nullable; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.script.AbstractSearchScript; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.NativeScriptFactory; +import org.elasticsearch.script.ScriptException; import org.elasticsearch.search.lookup.IndexField; import org.elasticsearch.search.lookup.IndexFieldTerm; @@ -70,7 +70,7 @@ private LanguageModelScoreScript(Map params) { // get lambda lambda = ((Double) params.get("lambda")).floatValue(); if (field == null || terms == null || docLengthField == null) { - throw new ElasticSearchException("cannot initialize " + SCRIPT_NAME + ": field, terms or length field parameter missing!"); + throw new ScriptException("cannot initialize " + SCRIPT_NAME + ": field, terms or length field parameter missing!"); } } @@ -120,11 +120,11 @@ public Object run() { } return score; } else { - throw new ElasticSearchException("Could not compute language model score, word count field missing."); + throw new ScriptException("Could not compute language model score, word count field missing."); } } catch (IOException ex) { - throw new ElasticSearchException("Could not compute language model score: ", ex); + throw new ScriptException("Could not compute language model score: ", ex); } } diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java index d3593871f4c9a..3143475b95d16 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java @@ -1,6 +1,5 @@ package org.elasticsearch.examples.nativescript.script; -import org.elasticsearch.ElasticSearchIllegalArgumentException; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.client.Client; import org.elasticsearch.common.Nullable; @@ -77,19 +76,19 @@ public Factory(Node node, Settings settings) { @Override public ExecutableScript newScript(@Nullable Map params) { if (params == null) { - throw new ElasticSearchIllegalArgumentException("Missing script parameters"); + throw new ScriptException("Missing script parameters"); } String lookupIndex = XContentMapValues.nodeStringValue(params.get("lookup_index"), null); if (lookupIndex == null) { - throw new ElasticSearchIllegalArgumentException("Missing the index parameter"); + throw new ScriptException("Missing the index parameter"); } String lookupType = XContentMapValues.nodeStringValue(params.get("lookup_type"), null); if (lookupType == null) { - throw new ElasticSearchIllegalArgumentException("Missing the index parameter"); + throw new ScriptException("Missing the index parameter"); } String field = XContentMapValues.nodeStringValue(params.get("field"), null); if (field == null) { - throw new ElasticSearchIllegalArgumentException("Missing the field parameter"); + throw new ScriptException("Missing the field parameter"); } return new LookupScript(node.client(), logger, cache, lookupIndex, lookupType, field); } diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/PhraseScoreScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/PhraseScoreScript.java index fafbb62dd937e..cbf48dd2b683b 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/PhraseScoreScript.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/PhraseScoreScript.java @@ -4,11 +4,11 @@ import java.util.Iterator; import java.util.Map; -import org.elasticsearch.ElasticSearchException; import org.elasticsearch.common.Nullable; import org.elasticsearch.script.AbstractSearchScript; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.NativeScriptFactory; +import org.elasticsearch.script.ScriptException; import org.elasticsearch.search.lookup.IndexField; import org.elasticsearch.search.lookup.IndexLookup; import org.elasticsearch.search.lookup.TermPosition; @@ -61,7 +61,7 @@ private PhraseScoreScript(Map params) { // get the field field = (String) params.get("field"); if (field == null || terms == null) { - throw new ElasticSearchException("cannot initialize " + SCRIPT_NAME + ": field or terms parameter missing!"); + throw new ScriptException("cannot initialize " + SCRIPT_NAME + ": field or terms parameter missing!"); } assert (terms.size() == 2); } diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptFactory.java b/src/main/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptFactory.java index 198c093766621..efa840eed1fd7 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptFactory.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptFactory.java @@ -1,12 +1,12 @@ package org.elasticsearch.examples.nativescript.script; -import org.elasticsearch.ElasticSearchIllegalArgumentException; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.script.AbstractFloatSearchScript; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.NativeScriptFactory; +import org.elasticsearch.script.ScriptException; import java.util.Map; @@ -21,7 +21,7 @@ public class PopularityScoreScriptFactory implements NativeScriptFactory { public ExecutableScript newScript(@Nullable Map params) { String fieldName = params == null ? null : XContentMapValues.nodeStringValue(params.get("field"), null); if (fieldName == null) { - throw new ElasticSearchIllegalArgumentException("Missing the field parameter"); + throw new ScriptException("Missing the field parameter"); } return new PopularityScoreScript(fieldName); } diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/TFIDFScoreScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/TFIDFScoreScript.java index ce79837ebae67..0c3f73145e326 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/TFIDFScoreScript.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/TFIDFScoreScript.java @@ -4,11 +4,11 @@ import java.util.ArrayList; import java.util.Map; -import org.elasticsearch.ElasticSearchException; import org.elasticsearch.common.Nullable; import org.elasticsearch.script.AbstractSearchScript; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.NativeScriptFactory; +import org.elasticsearch.script.ScriptException; import org.elasticsearch.search.lookup.IndexField; import org.elasticsearch.search.lookup.IndexFieldTerm; @@ -60,7 +60,7 @@ private TFIDFScoreScript(Map params) { // get the field field = (String) params.get("field"); if (field == null || terms == null) { - throw new ElasticSearchException("cannot initialize " + SCRIPT_NAME + ": field or terms parameter missing!"); + throw new ScriptException("cannot initialize " + SCRIPT_NAME + ": field or terms parameter missing!"); } } @@ -83,7 +83,7 @@ public Object run() { } return score; } catch (IOException ex) { - throw new ElasticSearchException("Could not compute tfidf: ", ex); + throw new ScriptException("Could not compute tfidf: ", ex); } } diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTests.java index 2055071b175ae..4b8399dde272f 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTests.java @@ -8,7 +8,7 @@ /** */ -@ClusterScope(scope = Scope.SUITE, numNodes = 1) +@ClusterScope(scope = Scope.SUITE, numDataNodes = 1) public class AbstractSearchScriptTests extends ElasticsearchIntegrationTest { @Override From 93d8b8110735ed5c6b1543b9bc1b3c43c5b2838d Mon Sep 17 00:00:00 2001 From: Britta Weber Date: Fri, 30 May 2014 12:09:33 +0200 Subject: [PATCH 35/69] fix number of shards settings indexSettings() must be changed to make sure no of shards is 1. These index settings are ignored when set via nodeSettings() method. See https://github.com/elasticsearch/elasticsearch/commit/d5aaa90f3466872d2e45a4df4db79ce182100f07 --- .../script/AbstractSearchScriptTests.java | 17 ++++++++++++++--- .../script/TermScoringScriptTests.java | 8 +++++++- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTests.java index 4b8399dde272f..dfcc3a0b7c185 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTests.java @@ -5,18 +5,29 @@ import org.elasticsearch.test.ElasticsearchIntegrationTest; import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope; import org.elasticsearch.test.ElasticsearchIntegrationTest.Scope; +import org.junit.Test; + +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; +import static org.hamcrest.Matchers.equalTo; /** */ @ClusterScope(scope = Scope.SUITE, numDataNodes = 1) public class AbstractSearchScriptTests extends ElasticsearchIntegrationTest { + @Override + public Settings indexSettings() { + ImmutableSettings.Builder builder = ImmutableSettings.builder(); + builder.put(SETTING_NUMBER_OF_SHARDS, 1); + builder.put(SETTING_NUMBER_OF_REPLICAS, 0); + return builder.build(); + } + @Override protected Settings nodeSettings(int nodeOrdinal) { - return ImmutableSettings.settingsBuilder() + return ImmutableSettings.settingsBuilder() .put("gateway.type", "none") - .put("index.number_of_shards", 1) - .put("index.number_of_replicas", 0) .put(super.nodeSettings(nodeOrdinal)) .build(); } diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/TermScoringScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/TermScoringScriptTests.java index 055e054c4db67..b0156baa81bdb 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/TermScoringScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/TermScoringScriptTests.java @@ -37,6 +37,12 @@ public class TermScoringScriptTests extends AbstractSearchScriptTests { final static Double[] weights = { 1.0, 1.0 }; final static int numDocs = 100; + @Test + public void testNoOfShardsIs1() { + assertAcked(prepareCreate("test").get()); + assertThat(client().admin().indices().prepareGetSettings("test").get().getSetting("test", "index.number_of_shards"), equalTo("1")); + } + @Test public void testTFIDF() throws Exception { @@ -148,7 +154,7 @@ private void initData() throws IOException, InterruptedException, ExecutionExcep .field("type", "multi_field").startObject("fields").startObject(field).field("type", "String").endObject() .startObject("word_count").field("analyzer", "standard").field("type", "token_count").startObject("fielddata") .field("format", "doc_values").endObject().endObject().endObject().endObject().endObject().endObject().endObject().string(); - assertAcked(client().admin().indices().prepareCreate("test").addMapping("type", mapping)); + assertAcked(prepareCreate("test").addMapping("type", mapping)); List indexBuilders = new ArrayList(); // Index numDocs records (0..99) From c5ed8ee067aac77ddc9f01785d9f6f9ba6b5de38 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Thu, 16 Oct 2014 16:57:59 -0400 Subject: [PATCH 36/69] Upgrade to elasticsearch 1.3.4 --- pom.xml | 4 ++-- .../nativescript/script/AbstractSearchScriptTests.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index 280616faa5a62..8117c5acf3ce7 100644 --- a/pom.xml +++ b/pom.xml @@ -38,8 +38,8 @@ - 1.2.0 - 4.8.1 + 1.3.4 + 4.9.1 diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTests.java index dfcc3a0b7c185..5e0b2a7237064 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTests.java @@ -2,14 +2,13 @@ import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.test.ElasticsearchIntegrationTest; import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope; import org.elasticsearch.test.ElasticsearchIntegrationTest.Scope; -import org.junit.Test; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; -import static org.hamcrest.Matchers.equalTo; /** */ @@ -28,6 +27,7 @@ public Settings indexSettings() { protected Settings nodeSettings(int nodeOrdinal) { return ImmutableSettings.settingsBuilder() .put("gateway.type", "none") + .put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true) .put(super.nodeSettings(nodeOrdinal)) .build(); } From 1254c4837de635a42e43b2bdbbd5ef70b621b010 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Sat, 8 Nov 2014 10:40:12 -0800 Subject: [PATCH 37/69] Upgrade to elasticsearch 1.4.0 Fixes #8 --- pom.xml | 12 ++++++-- .../script/CosineSimilarityScoreScript.java | 6 ++++ .../script/LanguageModelScoreScript.java | 6 ++++ .../script/PhraseScoreScript.java | 6 ++++ .../script/PopularityScoreScriptFactory.java | 30 ++++++++++++++----- .../script/RandomSortScriptFactory.java | 11 +++++++ .../nativescript/script/TFIDFScoreScript.java | 6 ++++ .../script/AbstractSearchScriptTests.java | 2 +- 8 files changed, 67 insertions(+), 12 deletions(-) diff --git a/pom.xml b/pom.xml index 8117c5acf3ce7..f6dcc4d5f9398 100644 --- a/pom.xml +++ b/pom.xml @@ -38,8 +38,8 @@ - 1.3.4 - 4.9.1 + 1.4.0 + 4.10.2 @@ -91,7 +91,13 @@ 1.3 test - + + + com.carrotsearch.randomizedtesting + randomizedtesting-runner + 2.1.10 + test + diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/CosineSimilarityScoreScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/CosineSimilarityScoreScript.java index 111959c2243da..f447c12799b4e 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/CosineSimilarityScoreScript.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/CosineSimilarityScoreScript.java @@ -4,6 +4,7 @@ import java.util.ArrayList; import java.util.Map; +import org.apache.lucene.search.Scorer; import org.elasticsearch.script.ScriptException; import org.elasticsearch.common.Nullable; @@ -32,6 +33,11 @@ public class CosineSimilarityScoreScript extends AbstractSearchScript { final static public String SCRIPT_NAME = "cosine_sim_script_score"; + @Override + public void setScorer(Scorer scorer) { + // ignore + } + /** * Factory that is registered in * {@link org.elasticsearch.examples.nativescript.plugin.NativeScriptExamplesPlugin#onModule(org.elasticsearch.script.ScriptModule)} diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/LanguageModelScoreScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/LanguageModelScoreScript.java index 2211ae3bb4c35..3a8ded9131eec 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/LanguageModelScoreScript.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/LanguageModelScoreScript.java @@ -4,6 +4,7 @@ import java.util.ArrayList; import java.util.Map; +import org.apache.lucene.search.Scorer; import org.elasticsearch.common.Nullable; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.script.AbstractSearchScript; @@ -34,6 +35,11 @@ public class LanguageModelScoreScript extends AbstractSearchScript { final static public String SCRIPT_NAME = "language_model_script_score"; + @Override + public void setScorer(Scorer scorer) { + // ignore + } + /** * Factory that is registered in * {@link org.elasticsearch.examples.nativescript.plugin.NativeScriptExamplesPlugin#onModule(org.elasticsearch.script.ScriptModule)} diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/PhraseScoreScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/PhraseScoreScript.java index cbf48dd2b683b..f33f218a2cbf1 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/PhraseScoreScript.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/PhraseScoreScript.java @@ -4,6 +4,7 @@ import java.util.Iterator; import java.util.Map; +import org.apache.lucene.search.Scorer; import org.elasticsearch.common.Nullable; import org.elasticsearch.script.AbstractSearchScript; import org.elasticsearch.script.ExecutableScript; @@ -29,6 +30,11 @@ public class PhraseScoreScript extends AbstractSearchScript { final static public String SCRIPT_NAME = "phrase_script_score"; + @Override + public void setScorer(Scorer scorer) { + // ignore + } + /** * Factory that is registered in * {@link org.elasticsearch.examples.nativescript.plugin.NativeScriptExamplesPlugin#onModule(org.elasticsearch.script.ScriptModule)} diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptFactory.java b/src/main/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptFactory.java index efa840eed1fd7..84ed966120d32 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptFactory.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptFactory.java @@ -1,5 +1,6 @@ package org.elasticsearch.examples.nativescript.script; +import org.apache.lucene.search.Scorer; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.fielddata.ScriptDocValues; @@ -8,6 +9,7 @@ import org.elasticsearch.script.NativeScriptFactory; import org.elasticsearch.script.ScriptException; +import java.io.IOException; import java.util.Map; /** @@ -36,21 +38,33 @@ private static class PopularityScoreScript extends AbstractFloatSearchScript { private final String field; + private Scorer scorer; + public PopularityScoreScript(String field) { this.field = field; } + @Override + public void setScorer(Scorer scorer) { + this.scorer = scorer; + } + @Override public float runAsFloat() { - ScriptDocValues docValue = (ScriptDocValues) doc().get(field); - if (docValue != null && !docValue.isEmpty()) { - ScriptDocValues.Longs fieldData = (ScriptDocValues.Longs) docValue; - double boost = 1 + Math.log10(fieldData.getValue() + 1); - // Because this script is used in custom_score script the value of score() is populated. - // In all other cases doc().getScore() should be used instead. - return (float) boost * score(); + try { + ScriptDocValues docValue = (ScriptDocValues) doc().get(field); + if (docValue != null && !docValue.isEmpty()) { + ScriptDocValues.Longs fieldData = (ScriptDocValues.Longs) docValue; + double boost = 1 + Math.log10(fieldData.getValue() + 1); + // Because this script is used in custom_score script the value of score() is populated. + // In all other cases doc().getScore() should be used instead. + return (float) boost * scorer.score(); + + } + return scorer.score(); + } catch (IOException ex) { + return 0.0f; } - return score(); } } } diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java b/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java index 17964f15c8fc3..c70a5561b5f32 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java @@ -1,5 +1,6 @@ package org.elasticsearch.examples.nativescript.script; +import org.apache.lucene.search.Scorer; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.fielddata.ScriptDocValues; @@ -50,6 +51,11 @@ private RandomSortScript() { public long runAsLong() { return random.nextLong(); } + + @Override + public void setScorer(Scorer scorer) { + // we are not using it - ignore + } } private static class PseudoRandomSortScript extends AbstractLongSearchScript { @@ -79,5 +85,10 @@ public long runAsLong() { return -1; } } + + @Override + public void setScorer(Scorer scorer) { + // we are not using it - ignore + } } } diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/TFIDFScoreScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/TFIDFScoreScript.java index 0c3f73145e326..37a1a63e1da74 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/TFIDFScoreScript.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/TFIDFScoreScript.java @@ -4,6 +4,7 @@ import java.util.ArrayList; import java.util.Map; +import org.apache.lucene.search.Scorer; import org.elasticsearch.common.Nullable; import org.elasticsearch.script.AbstractSearchScript; import org.elasticsearch.script.ExecutableScript; @@ -28,6 +29,11 @@ public class TFIDFScoreScript extends AbstractSearchScript { final static public String SCRIPT_NAME = "tfidf_script_score"; + @Override + public void setScorer(Scorer scorer) { + // ignore + } + /** * Factory that is registered in * {@link org.elasticsearch.examples.nativescript.plugin.NativeScriptExamplesPlugin#onModule(org.elasticsearch.script.ScriptModule)} diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTests.java index 5e0b2a7237064..16f505daf7731 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTests.java @@ -13,7 +13,7 @@ /** */ @ClusterScope(scope = Scope.SUITE, numDataNodes = 1) -public class AbstractSearchScriptTests extends ElasticsearchIntegrationTest { +public abstract class AbstractSearchScriptTests extends ElasticsearchIntegrationTest { @Override public Settings indexSettings() { From 9406941fb8d7c72c584c6186007e464686d35bd2 Mon Sep 17 00:00:00 2001 From: Britta Weber Date: Thu, 26 Mar 2015 15:21:13 +0100 Subject: [PATCH 38/69] fix links in README --- README.textile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.textile b/README.textile index 6146673bdfdda..25ab51e9c2b27 100644 --- a/README.textile +++ b/README.textile @@ -2,7 +2,7 @@ h1. Example of Native Script Plugin for Elasticsearch h2. Introduction -p. This plugin contains several examples of "native script":http://www.elasticsearch.org/guide/reference/modules/scripting.html for Elasticsearch. +p. This plugin contains several examples of "native script":http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-scripting.html for Elasticsearch. h2. Creating Elasticsearch Plugin @@ -61,7 +61,7 @@ p. Now that the plugin infrastructure is complete, it's possible to add a native h3. Is Prime Native Script -p. One of the example scripts in this project is the "is_prime" script that can be used to check if a field contains a possible prime number. The script accepts two parameters @field@ and @certainty@. The @field@ parameter contains the name of the field that needs to be checked and the @certainty@ parameter specifies a measure of the uncertainty that the caller is willing to tolerate. The script returns @true@ if the field contains a probable prime number and @false@ otherwise. The probability that the number for which the script returned @true@ is prime exceeds (1 - 0.5^certainty). The script can be used in "Script Filter":http://www.elasticsearch.org/guide/reference/query-dsl/script-filter.html as well as a "Script Field":http://www.elasticsearch.org/guide/reference/api/search/script-fields.html. The implementation of the "is_prime" native script and it's factory can be found in the "IsPrimeSearchScript":https://github.com/imotov/elasticsearch-native-script-example/blob/master/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java class. +p. One of the example scripts in this project is the "is_prime" script that can be used to check if a field contains a possible prime number. The script accepts two parameters @field@ and @certainty@. The @field@ parameter contains the name of the field that needs to be checked and the @certainty@ parameter specifies a measure of the uncertainty that the caller is willing to tolerate. The script returns @true@ if the field contains a probable prime number and @false@ otherwise. The probability that the number for which the script returned @true@ is prime exceeds (1 - 0.5^certainty). The script can be used in "Script Filter":http://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-script-filter.html as well as a "Script Field":http://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-script-fields.html. The implementation of the "is_prime" native script and it's factory can be found in the "IsPrimeSearchScript":https://github.com/imotov/elasticsearch-native-script-example/blob/master/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java class. p. In order to enable native script creation the plugin has to contain and register a class that implements "NativeScriptFactory":https://github.com/elasticsearch/elasticsearch/blob/master/src/main/java/org/elasticsearch/script/NativeScriptFactory.java. The NativeScriptFactory interface has only one method @newScript(Map params)@. This method accepts a list of parameters that are passed during script creation and returns an ExecutableScript object with an instance of the script. The factory has to be registered in the @onModule(ScriptModule module)@ method of the Plugin. From d6a8b87c94acb5ff6d51d38f44a27de406f54a7b Mon Sep 17 00:00:00 2001 From: navins Date: Mon, 27 Apr 2015 20:32:56 +0800 Subject: [PATCH 39/69] doc: remove not exist import. --- README.textile | 1 - 1 file changed, 1 deletion(-) diff --git a/README.textile b/README.textile index 25ab51e9c2b27..c36786cee2294 100644 --- a/README.textile +++ b/README.textile @@ -31,7 +31,6 @@ p. An Elasticsearch plugin can be created by following these six steps. bc.. package org.elasticsearch.examples.nativescript.plugin; -import org.elasticsearch.examples.nativescript.script.DividedBoostScoreSearchScript; import org.elasticsearch.plugins.AbstractPlugin; import org.elasticsearch.script.ScriptModule; From 6ccdf8cd0e4caaa46fdb8b87ae5a18e7eb6e196d Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Sun, 14 Jun 2015 19:41:36 -0400 Subject: [PATCH 40/69] Update to 2.0.0-SNAPSHOT --- .gitignore | 1 + .../license_header_definition.xml | 13 +++++ .../native_script_example_license_header.txt | 11 +++++ pom.xml | 45 +++++++---------- .../plugin/NativeScriptExamplesPlugin.java | 14 ++++++ .../script/CosineSimilarityScoreScript.java | 14 ++++++ .../script/IsPrimeSearchScript.java | 14 ++++++ .../script/LanguageModelScoreScript.java | 14 ++++++ .../nativescript/script/LookupScript.java | 26 +++++++--- .../script/PhraseScoreScript.java | 14 ++++++ .../script/PopularityScoreScriptFactory.java | 14 ++++++ .../script/RandomSortScriptFactory.java | 17 ++++++- .../nativescript/script/TFIDFScoreScript.java | 14 ++++++ .../script/AbstractSearchScriptTests.java | 18 +++++-- .../script/IsPrimeSearchScriptTests.java | 49 +++++++++++++------ .../script/LookupScriptTests.java | 18 ++++++- .../script/PopularityScoreScriptTests.java | 46 +++++++++++------ .../script/RandomSortScriptTests.java | 14 ++++++ .../script/TermScoringScriptTests.java | 31 +++++++++--- 19 files changed, 311 insertions(+), 76 deletions(-) create mode 100644 dev-tools/src/main/resources/license-check/license_header_definition.xml create mode 100644 dev-tools/src/main/resources/license-check/native_script_example_license_header.txt diff --git a/.gitignore b/.gitignore index 7d2a3da4c8d6b..7c6339c28a5fc 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ logs/ build/ target/ .DS_Store +*-execution-hints.log # IDEA ignores .idea/ diff --git a/dev-tools/src/main/resources/license-check/license_header_definition.xml b/dev-tools/src/main/resources/license-check/license_header_definition.xml new file mode 100644 index 0000000000000..1cc53bda0018a --- /dev/null +++ b/dev-tools/src/main/resources/license-check/license_header_definition.xml @@ -0,0 +1,13 @@ + + + + /* + * + */ + + (\s|\t)*/\*.*$ + .*\*/(\s|\t)*$ + false + true + + diff --git a/dev-tools/src/main/resources/license-check/native_script_example_license_header.txt b/dev-tools/src/main/resources/license-check/native_script_example_license_header.txt new file mode 100644 index 0000000000000..86a8beecb2733 --- /dev/null +++ b/dev-tools/src/main/resources/license-check/native_script_example_license_header.txt @@ -0,0 +1,11 @@ +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file diff --git a/pom.xml b/pom.xml index f6dcc4d5f9398..b00304b925a53 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ 4.0.0 org.elasticsearch elasticsearch-native-script-example - 1.1.1-SNAPSHOT + 2.0.0-SNAPSHOT jar ElasticSearch Plugin with Native Script Examples 2013 @@ -28,19 +28,18 @@ - org.sonatype.oss - oss-parent - 7 + org.elasticsearch + elasticsearch-parent + 2.0.0-SNAPSHOT - - - - - - 1.4.0 - 4.10.2 - + + + oss-snapshots + Sonatype OSS Snapshots + https://oss.sonatype.org/content/repositories/snapshots/ + + @@ -51,21 +50,16 @@ org.apache.lucene lucene-test-framework - ${lucene.version} - test org.elasticsearch elasticsearch - ${elasticsearch.version} - compile org.elasticsearch elasticsearch - ${elasticsearch.version} test test-jar @@ -73,30 +67,21 @@ log4j log4j - 1.2.17 - compile org.slf4j slf4j-api - 1.6.2 - compile - true org.hamcrest hamcrest-all - 1.3 - test com.carrotsearch.randomizedtesting randomizedtesting-runner - 2.1.10 - test @@ -150,6 +135,14 @@ + + + com.mycila + license-maven-plugin + +

${project.basedir}/dev-tools/src/main/resources/license-check/native_script_example_license_header.txt
+ + diff --git a/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java b/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java index 141ade74b3209..633adfc2d0541 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java @@ -1,3 +1,17 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.elasticsearch.examples.nativescript.plugin; import org.elasticsearch.examples.nativescript.script.IsPrimeSearchScript; diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/CosineSimilarityScoreScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/CosineSimilarityScoreScript.java index f447c12799b4e..abcfc4162d245 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/CosineSimilarityScoreScript.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/CosineSimilarityScoreScript.java @@ -1,3 +1,17 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.elasticsearch.examples.nativescript.script; import java.io.IOException; diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java index 0924dbcb7d12e..1c0c934467354 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java @@ -1,3 +1,17 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.elasticsearch.examples.nativescript.script; import java.math.BigInteger; diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/LanguageModelScoreScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/LanguageModelScoreScript.java index 3a8ded9131eec..0a3a08809288e 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/LanguageModelScoreScript.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/LanguageModelScoreScript.java @@ -1,3 +1,17 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.elasticsearch.examples.nativescript.script; import java.io.IOException; diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java index 3143475b95d16..bf245cbcfce9d 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java @@ -1,11 +1,24 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.elasticsearch.examples.nativescript.script; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.client.Client; import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.cache.Cache; -import org.elasticsearch.common.cache.CacheBuilder; -import org.elasticsearch.common.collect.ImmutableMap; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; @@ -21,6 +34,7 @@ import org.elasticsearch.script.NativeScriptFactory; import org.elasticsearch.script.ScriptException; +import java.util.Collections; import java.util.Map; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; @@ -35,7 +49,7 @@ public class LookupScript extends AbstractSearchScript { * {@link org.elasticsearch.examples.nativescript.plugin.NativeScriptExamplesPlugin#onModule(org.elasticsearch.script.ScriptModule)} * method when plugin is loaded. */ - public static class Factory extends AbstractComponent implements NativeScriptFactory{ + public static class Factory extends AbstractComponent implements NativeScriptFactory { private final Node node; @@ -102,7 +116,7 @@ public ExecutableScript newScript(@Nullable Map params) { private final Client client; private final Cache, Map> cache; - private static final Map EMPTY_MAP = ImmutableMap.of(); + private static final Map EMPTY_MAP = Collections.emptyMap(); private LookupScript(Client client, ESLogger logger, Cache, Map> cache, String lookupIndex, String lookupType, String field) { this.client = client; @@ -116,7 +130,7 @@ private LookupScript(Client client, ESLogger logger, Cache @Override public Object run() { // First we get field using doc lookup - ScriptDocValues docValue = (ScriptDocValues)doc().get(field); + ScriptDocValues docValue = (ScriptDocValues) doc().get(field); // This is not very efficient // Check if field exists if (docValue != null && !docValue.isEmpty()) { diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/PhraseScoreScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/PhraseScoreScript.java index f33f218a2cbf1..ecd8b09cc11a8 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/PhraseScoreScript.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/PhraseScoreScript.java @@ -1,3 +1,17 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.elasticsearch.examples.nativescript.script; import java.util.ArrayList; diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptFactory.java b/src/main/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptFactory.java index 84ed966120d32..4df16a554bbbc 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptFactory.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptFactory.java @@ -1,3 +1,17 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.elasticsearch.examples.nativescript.script; import org.apache.lucene.search.Scorer; diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java b/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java index c70a5561b5f32..17ba7c07e41ed 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java @@ -1,5 +1,20 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.elasticsearch.examples.nativescript.script; +import com.google.common.base.Charsets; import org.apache.lucene.search.Scorer; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.xcontent.support.XContentMapValues; @@ -71,7 +86,7 @@ public long runAsLong() { try { MessageDigest m = MessageDigest.getInstance("MD5"); m.reset(); - m.update((fieldData.getValue() + salt).getBytes()); + m.update((fieldData.getValue() + salt).getBytes(Charsets.UTF_8)); byte[] sort = m.digest(); return (sort[0] & 0xFFL) << 56 | (sort[1] & 0xFFL) << 48 diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/TFIDFScoreScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/TFIDFScoreScript.java index 37a1a63e1da74..a7109686120f8 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/TFIDFScoreScript.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/TFIDFScoreScript.java @@ -1,3 +1,17 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.elasticsearch.examples.nativescript.script; import java.io.IOException; diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTests.java index 16f505daf7731..bd79d4f1d0623 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTests.java @@ -1,6 +1,18 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.elasticsearch.examples.nativescript.script; -import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.test.ElasticsearchIntegrationTest; @@ -17,7 +29,7 @@ public abstract class AbstractSearchScriptTests extends ElasticsearchIntegration @Override public Settings indexSettings() { - ImmutableSettings.Builder builder = ImmutableSettings.builder(); + Settings.Builder builder = Settings.builder(); builder.put(SETTING_NUMBER_OF_SHARDS, 1); builder.put(SETTING_NUMBER_OF_REPLICAS, 0); return builder.build(); @@ -25,7 +37,7 @@ public Settings indexSettings() { @Override protected Settings nodeSettings(int nodeOrdinal) { - return ImmutableSettings.settingsBuilder() + return Settings.settingsBuilder() .put("gateway.type", "none") .put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true) .put(super.nodeSettings(nodeOrdinal)) diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScriptTests.java index 69da5f35f3bea..a0c50b6a506f4 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScriptTests.java @@ -1,22 +1,36 @@ -package org.elasticsearch.examples.nativescript.script; - -import static org.elasticsearch.index.query.FilterBuilders.scriptFilter; -import static org.elasticsearch.index.query.QueryBuilders.filteredQuery; -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; -import static org.hamcrest.Matchers.equalTo; +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ -import java.util.ArrayList; -import java.util.List; +package org.elasticsearch.examples.nativescript.script; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.sort.SortOrder; import org.junit.Test; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static com.google.common.collect.Maps.newHashMap; +import static org.elasticsearch.index.query.QueryBuilders.scriptQuery; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; +import static org.hamcrest.Matchers.equalTo; + /** */ public class IsPrimeSearchScriptTests extends AbstractSearchScriptTests { @@ -57,11 +71,12 @@ public void testIsPrimeScript() throws Exception { } indexRandom(true, indexBuilders); - + + Map params = newHashMap(); + params.put("field", "number"); // Retrieve first 10 prime records SearchResponse searchResponse = client().prepareSearch("test") - .setQuery(filteredQuery(matchAllQuery(), - scriptFilter("is_prime").lang("native").addParam("field", "number"))) + .setQuery(scriptQuery(new Script("is_prime", ScriptService.ScriptType.INLINE, "native", params))) .addField("name") .setSize(10) .addSort("number", SortOrder.ASC) @@ -77,10 +92,12 @@ public void testIsPrimeScript() throws Exception { assertThat(searchResponse.getHits().getAt(i).field("name").getValue().toString(), equalTo("rec " + PRIMES_10[i])); } + params = newHashMap(); + params.put("field", "number"); + params.put("certainty", 0); // Check certainty parameter - with certainty == 0, it should return all numbers, but only if numbers are present searchResponse = client().prepareSearch("test") - .setQuery(filteredQuery(matchAllQuery(), - scriptFilter("is_prime").lang("native").addParam("field", "number").addParam("certainty", 0))) + .setQuery(scriptQuery(new Script("is_prime", ScriptService.ScriptType.INLINE, "native", params))) .addField("name") .setSize(10) .addSort("number", SortOrder.ASC) diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/LookupScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/LookupScriptTests.java index a9e098a6ce7fe..c349c65ac3b0e 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/LookupScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/LookupScriptTests.java @@ -1,3 +1,17 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.elasticsearch.examples.nativescript.script; import static org.elasticsearch.index.query.QueryBuilders.matchQuery; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; @@ -13,6 +27,8 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.sort.SortOrder; import org.junit.Test; @@ -84,7 +100,7 @@ public void testLookup() throws Exception { .setTypes("city") .setQuery(matchQuery("city", "south burlington")) .addField("city") - .addScriptField("state_info", "native", "lookup", params) + .addScriptField("state_info", new Script("lookup", ScriptService.ScriptType.INLINE, "native", params)) .setSize(10) .addSort("population", SortOrder.DESC) .execute().actionGet(); diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptTests.java index 75ef9ff30f16c..68f4b4451fd8f 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptTests.java @@ -1,3 +1,17 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.elasticsearch.examples.nativescript.script; import static org.elasticsearch.index.query.QueryBuilders.functionScoreQuery; @@ -18,6 +32,8 @@ import org.elasticsearch.common.lucene.search.function.CombineFunction; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptService; import org.junit.Test; /** @@ -26,7 +42,7 @@ public class PopularityScoreScriptTests extends AbstractSearchScriptTests { @Test public void testPopularityScoring() throws Exception { - + // Create a new index String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties") @@ -34,42 +50,42 @@ public void testPopularityScoring() throws Exception { .startObject("number").field("type", "integer").endObject() .endObject().endObject().endObject() .string(); - + assertAcked(prepareCreate("test") .addMapping("type", mapping)); List indexBuilders = new ArrayList(); - + // Index 5 records with non-empty number field for (int i = 0; i < 5; i++) { indexBuilders.add( client().prepareIndex("test", "type", Integer.toString(i)) - .setSource(XContentFactory.jsonBuilder().startObject() - .field("name", "rec " + i) - .field("number", i + 1) - .endObject())); + .setSource(XContentFactory.jsonBuilder().startObject() + .field("name", "rec " + i) + .field("number", i + 1) + .endObject())); } // Index a few records with empty number for (int i = 5; i < 10; i++) { indexBuilders.add( client().prepareIndex("test", "type", Integer.toString(i)) - .setSource(XContentFactory.jsonBuilder().startObject() - .field("name", "rec " + i) - .endObject())); + .setSource(XContentFactory.jsonBuilder().startObject() + .field("name", "rec " + i) + .endObject())); } - + indexRandom(true, indexBuilders); - - Map params = MapBuilder. newMapBuilder().put("field", "number").map(); + + Map params = MapBuilder.newMapBuilder().put("field", "number").map(); // Retrieve first 10 hits SearchResponse searchResponse = client().prepareSearch("test") .setQuery(functionScoreQuery(matchQuery("name", "rec")) .boostMode(CombineFunction.REPLACE) - .add(ScoreFunctionBuilders.scriptFunction("popularity", "native", params))) + .add(ScoreFunctionBuilders.scriptFunction(new Script("popularity", ScriptService.ScriptType.INLINE, "native", params)))) .setSize(10) .addField("name") .execute().actionGet(); - + assertNoFailures(searchResponse); // There should be 10 hist diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptTests.java index 8185e0ec22171..ab839e23ca948 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptTests.java @@ -1,3 +1,17 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.elasticsearch.examples.nativescript.script; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/TermScoringScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/TermScoringScriptTests.java index b0156baa81bdb..11aab6f685a0b 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/TermScoringScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/TermScoringScriptTests.java @@ -1,3 +1,17 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.elasticsearch.examples.nativescript.script; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; @@ -19,6 +33,8 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.SearchHit; import org.junit.Test; @@ -26,15 +42,14 @@ * Test if the computed tfidf in NaiveTFIDFScoreScript equals 0.0 for each * document as would be expected if each document in the index contains only one * and always the same term. - * */ public class TermScoringScriptTests extends AbstractSearchScriptTests { - final static String[] searchTerms = { "foo", "bar" }; + final static String[] searchTerms = {"foo", "bar"}; final static String field = "field"; final static String wordCountField = field + ".word_count"; - final static String placeholder = "placeholder"; - final static Double[] weights = { 1.0, 1.0 }; + final static String placeholder = "placeholder"; + final static Double[] weights = {1.0, 1.0}; final static int numDocs = 100; @Test @@ -58,7 +73,7 @@ public void testTFIDF() throws Exception { .prepareSearch("test") .setQuery( QueryBuilders.functionScoreQuery() - .add(ScoreFunctionBuilders.scriptFunction(TFIDFScoreScript.SCRIPT_NAME, "native", params)) + .add(ScoreFunctionBuilders.scriptFunction(new Script(TFIDFScoreScript.SCRIPT_NAME, ScriptService.ScriptType.INLINE, "native", params))) .boostMode(CombineFunction.REPLACE.getName())).setSize(numDocs).execute().actionGet(); assertNoFailures(searchResponse); assertHitCount(searchResponse, numDocs); @@ -85,7 +100,7 @@ public void testCosineSimilarity() throws Exception { .prepareSearch("test") .setQuery( QueryBuilders.functionScoreQuery() - .add(ScoreFunctionBuilders.scriptFunction(CosineSimilarityScoreScript.SCRIPT_NAME, "native", params)) + .add(ScoreFunctionBuilders.scriptFunction(new Script(CosineSimilarityScoreScript.SCRIPT_NAME, ScriptService.ScriptType.INLINE, "native", params))) .boostMode(CombineFunction.REPLACE.getName())).setSize(numDocs).execute().actionGet(); assertNoFailures(searchResponse); assertHitCount(searchResponse, numDocs); @@ -110,7 +125,7 @@ public void testPhraseScorer() throws Exception { .prepareSearch("test") .setQuery( QueryBuilders.functionScoreQuery() - .add(ScoreFunctionBuilders.scriptFunction(PhraseScoreScript.SCRIPT_NAME, "native", params)) + .add(ScoreFunctionBuilders.scriptFunction(new Script(PhraseScoreScript.SCRIPT_NAME, ScriptService.ScriptType.INLINE, "native", params))) .boostMode(CombineFunction.REPLACE.getName())).setSize(numDocs).execute().actionGet(); assertNoFailures(searchResponse); assertHitCount(searchResponse, numDocs); @@ -138,7 +153,7 @@ public void testLanguageModelScorer() throws Exception { .prepareSearch("test") .setQuery( QueryBuilders.functionScoreQuery() - .add(ScoreFunctionBuilders.scriptFunction(LanguageModelScoreScript.SCRIPT_NAME, "native", params)) + .add(ScoreFunctionBuilders.scriptFunction(new Script(LanguageModelScoreScript.SCRIPT_NAME, ScriptService.ScriptType.INLINE, "native", params))) .boostMode(CombineFunction.REPLACE.getName())).setSize(numDocs).execute().actionGet(); assertNoFailures(searchResponse); assertHitCount(searchResponse, numDocs); From 54e30809ca85161fcca998eb007ed06c05e5a525 Mon Sep 17 00:00:00 2001 From: Chris Earle Date: Thu, 18 Jun 2015 23:54:35 -0400 Subject: [PATCH 41/69] Use BigInteger.valueOf, not constructor Prefer the use of [`BigInteger.valueOf`](http://docs.oracle.com/javase/7/docs/api/java/math/BigInteger.html#valueOf%28long%29) over the constructor to allow the JVM to possibly reuse cached values (and otherwise just simplifies it in my opinion). --- .../examples/nativescript/script/IsPrimeSearchScript.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java index 1c0c934467354..406404f6ab571 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java @@ -85,7 +85,7 @@ public Object run() { if (docValue != null && !docValue.isEmpty()) { try { // Try to parse it as an integer - BigInteger bigInteger = new BigInteger(Long.toString(((Longs) docValue).getValue())); + BigInteger bigInteger = BigInteger.valueOf(((Longs) docValue).getValue()); // Check if it's prime return bigInteger.isProbablePrime(certainty); } catch (NumberFormatException ex) { From 089f4c92382790443d4284568dc1eadda5baa626 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Fri, 17 Jul 2015 13:17:02 -0400 Subject: [PATCH 42/69] Update to 2.0.0-beta1-SNAPSHOT --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index b00304b925a53..82d5006d9ae60 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ 4.0.0 org.elasticsearch elasticsearch-native-script-example - 2.0.0-SNAPSHOT + 2.0.0-beta1-SNAPSHOT jar ElasticSearch Plugin with Native Script Examples 2013 @@ -30,7 +30,7 @@ org.elasticsearch elasticsearch-parent - 2.0.0-SNAPSHOT + 2.0.0-beta1-SNAPSHOT From 96f8b93f27f80346503dfa91e564b87c9334fbdf Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Thu, 23 Jul 2015 13:09:33 -0400 Subject: [PATCH 43/69] Update the way plugin is loaded in the test Changes related to elastic/elasticsearch#12367 --- .../nativescript/script/AbstractSearchScriptTests.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTests.java index bd79d4f1d0623..3c4aeb6b0f4ac 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTests.java @@ -14,7 +14,7 @@ package org.elasticsearch.examples.nativescript.script; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.plugins.PluginsService; +import org.elasticsearch.examples.nativescript.plugin.NativeScriptExamplesPlugin; import org.elasticsearch.test.ElasticsearchIntegrationTest; import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope; import org.elasticsearch.test.ElasticsearchIntegrationTest.Scope; @@ -39,7 +39,7 @@ public Settings indexSettings() { protected Settings nodeSettings(int nodeOrdinal) { return Settings.settingsBuilder() .put("gateway.type", "none") - .put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true) + .put("plugin.types", NativeScriptExamplesPlugin.class) .put(super.nodeSettings(nodeOrdinal)) .build(); } From d12279d8525b23bd323506164b969219c9302e73 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Tue, 28 Jul 2015 12:22:58 -0400 Subject: [PATCH 44/69] Switch to elasticsearch-plugin as a parent project Changes related to elastic/elasticsearch#12367 --- pom.xml | 138 ++++-------------- .../test/nativescript/10_primes.yaml | 48 ++++++ src/main/assemblies/plugin.xml | 18 --- src/main/resources/es-plugin.properties | 1 - .../plugin/NativeScriptExamplesRestIT.java | 35 +++++ 5 files changed, 113 insertions(+), 127 deletions(-) create mode 100644 rest-api-spec/test/nativescript/10_primes.yaml delete mode 100644 src/main/assemblies/plugin.xml delete mode 100644 src/main/resources/es-plugin.properties create mode 100644 src/test/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesRestIT.java diff --git a/pom.xml b/pom.xml index 82d5006d9ae60..0c7f14e188502 100644 --- a/pom.xml +++ b/pom.xml @@ -2,18 +2,21 @@ - - - - - elasticsearch-native-script-example 4.0.0 - org.elasticsearch + + + org.elasticsearch.plugin + elasticsearch-plugin + 2.0.0-beta1-SNAPSHOT + + + elasticsearch-native-script-example + org.elasticsearch.plugin elasticsearch-native-script-example 2.0.0-beta1-SNAPSHOT - jar ElasticSearch Plugin with Native Script Examples 2013 + The Apache Software License, Version 2.0 @@ -21,18 +24,33 @@ repo + scm:git:git@github.com:imotov/elasticsearch-native-script-example.git scm:git:git@github.com:imotov/elasticsearch-native-script-example.git http://github.com/imotov/elasticsearch-native-script-example - - org.elasticsearch - elasticsearch-parent - 2.0.0-beta1-SNAPSHOT - + + + + + + + + org.elasticsearch.examples.nativescript.plugin.NativeScriptExamplesPlugin + + ${project.basedir}/dev-tools/src/main/resources/license-check/native_script_example_license_header.txt + ${project.basedir}/dev-tools/src/main/resources/license-check/license_header_definition.xml + + + warn + nativescript + false + + + oss-snapshots @@ -41,107 +59,11 @@ - - - - - - - - org.apache.lucene - lucene-test-framework - - - - org.elasticsearch - elasticsearch - - - - org.elasticsearch - elasticsearch - test - test-jar - - - - log4j - log4j - - - - org.slf4j - slf4j-api - - - - org.hamcrest - hamcrest-all - - - - com.carrotsearch.randomizedtesting - randomizedtesting-runner - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 2.3.2 - - 1.7 - 1.7 - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - 2.12.3 - - - **/*Tests.java - - - - - - - maven-assembly-plugin - 2.3 - - false - ${project.build.directory}/releases/ - - ${basedir}/src/main/assemblies/plugin.xml - - - - - package - - single - - - - - - - com.mycila - license-maven-plugin - -
${project.basedir}/dev-tools/src/main/resources/license-check/native_script_example_license_header.txt
- diff --git a/rest-api-spec/test/nativescript/10_primes.yaml b/rest-api-spec/test/nativescript/10_primes.yaml new file mode 100644 index 0000000000000..c43b748fbbec4 --- /dev/null +++ b/rest-api-spec/test/nativescript/10_primes.yaml @@ -0,0 +1,48 @@ +# Integration tests for the lookup script +# +setup: + - do: + indices.create: + index: test_index + body: + settings: + index.number_of_shards: 1 + index.number_of_replicas: 0 + mappings.type1: + properties: + name.type: string + number.type: integer + + - do: + index: {index: test_index, type: type1, id: 1, body: {name: "rec 1", number: 1}} + + - do: + index: {index: test_index, type: type1, id: 2, body: {name: "rec 2", number: 2}} + + - do: + index: {index: test_index, type: type1, id: 3, body: {name: "rec 3", number: 3}} + + - do: + index: {index: test_index, type: type1, id: 4, body: {name: "rec 4", number: 4}} + + - do: + index: {index: test_index, type: type1, id: 5, body: {name: "rec 5", number: 5}} + - do: + indices.refresh: {} + +--- + +"Primes Test": + - do: + search: + body: + query: + filtered: + filter: + script: + script: is_prime + lang: native + params: + field: number + + - match: { hits.total: 3} diff --git a/src/main/assemblies/plugin.xml b/src/main/assemblies/plugin.xml deleted file mode 100644 index 8e8e42e7c0544..0000000000000 --- a/src/main/assemblies/plugin.xml +++ /dev/null @@ -1,18 +0,0 @@ - - - plugin - - zip - - false - - - / - true - true - - org.elasticsearch:elasticsearch - - - - \ No newline at end of file diff --git a/src/main/resources/es-plugin.properties b/src/main/resources/es-plugin.properties deleted file mode 100644 index b24fbc3db4d67..0000000000000 --- a/src/main/resources/es-plugin.properties +++ /dev/null @@ -1 +0,0 @@ -plugin=org.elasticsearch.examples.nativescript.plugin.NativeScriptExamplesPlugin diff --git a/src/test/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesRestIT.java b/src/test/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesRestIT.java new file mode 100644 index 0000000000000..4e2e1c1752b8e --- /dev/null +++ b/src/test/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesRestIT.java @@ -0,0 +1,35 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.elasticsearch.examples.nativescript.plugin; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.test.rest.ElasticsearchRestTestCase; +import org.elasticsearch.test.rest.RestTestCandidate; +import org.elasticsearch.test.rest.parser.RestTestParseException; + +import java.io.IOException; + +public class NativeScriptExamplesRestIT extends ElasticsearchRestTestCase { + + public NativeScriptExamplesRestIT(@Name("yaml") RestTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws IOException, RestTestParseException { + return ElasticsearchRestTestCase.createParameters(0, 1); + } +} + From 6c4ba4ab967b938fd06200b51e78dcf523861c40 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Tue, 4 Aug 2015 10:47:28 -0400 Subject: [PATCH 45/69] Rename test classes Changes related to elastic/elasticsearch#10659 --- .../nativescript/plugin/NativeScriptExamplesRestIT.java | 6 +++--- .../nativescript/script/AbstractSearchScriptTests.java | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/test/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesRestIT.java b/src/test/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesRestIT.java index 4e2e1c1752b8e..f9a751ac9ad51 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesRestIT.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesRestIT.java @@ -15,13 +15,13 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.elasticsearch.test.rest.ElasticsearchRestTestCase; +import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; -public class NativeScriptExamplesRestIT extends ElasticsearchRestTestCase { +public class NativeScriptExamplesRestIT extends ESRestTestCase { public NativeScriptExamplesRestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); @@ -29,7 +29,7 @@ public NativeScriptExamplesRestIT(@Name("yaml") RestTestCandidate testCandidate) @ParametersFactory public static Iterable parameters() throws IOException, RestTestParseException { - return ElasticsearchRestTestCase.createParameters(0, 1); + return ESRestTestCase.createParameters(0, 1); } } diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTests.java index 3c4aeb6b0f4ac..435e0a9f38c3e 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTests.java @@ -15,9 +15,9 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.examples.nativescript.plugin.NativeScriptExamplesPlugin; -import org.elasticsearch.test.ElasticsearchIntegrationTest; -import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope; -import org.elasticsearch.test.ElasticsearchIntegrationTest.Scope; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.ESIntegTestCase.Scope; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; @@ -25,7 +25,7 @@ /** */ @ClusterScope(scope = Scope.SUITE, numDataNodes = 1) -public abstract class AbstractSearchScriptTests extends ElasticsearchIntegrationTest { +public abstract class AbstractSearchScriptTests extends ESIntegTestCase { @Override public Settings indexSettings() { From 4cda2ccdaa8094c5de3e86ccfeea0fe56b26d353 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Thu, 13 Aug 2015 15:20:07 -0400 Subject: [PATCH 46/69] Scripts should expose whether they use the `_score` or not Changes related to elastic/elasticsearch#12695 --- .../script/CosineSimilarityScoreScript.java | 15 +++++++++----- .../script/IsPrimeSearchScript.java | 11 ++++++++++ .../script/LanguageModelScoreScript.java | 12 ++++++----- .../nativescript/script/LookupScript.java | 11 ++++++++++ .../script/PhraseScoreScript.java | 15 +++++++++----- .../script/PopularityScoreScriptFactory.java | 9 +++++++++ .../script/RandomSortScriptFactory.java | 20 +++++++++---------- .../nativescript/script/TFIDFScoreScript.java | 15 +++++++++----- 8 files changed, 78 insertions(+), 30 deletions(-) diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/CosineSimilarityScoreScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/CosineSimilarityScoreScript.java index abcfc4162d245..0d6a4750f3305 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/CosineSimilarityScoreScript.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/CosineSimilarityScoreScript.java @@ -47,11 +47,6 @@ public class CosineSimilarityScoreScript extends AbstractSearchScript { final static public String SCRIPT_NAME = "cosine_sim_script_score"; - @Override - public void setScorer(Scorer scorer) { - // ignore - } - /** * Factory that is registered in * {@link org.elasticsearch.examples.nativescript.plugin.NativeScriptExamplesPlugin#onModule(org.elasticsearch.script.ScriptModule)} @@ -70,6 +65,16 @@ public static class Factory implements NativeScriptFactory { public ExecutableScript newScript(@Nullable Map params) throws ScriptException { return new CosineSimilarityScoreScript(params); } + + /** + * Indicates if document scores may be needed by the produced scripts. + * + * @return {@code true} if scores are needed. + */ + @Override + public boolean needsScores() { + return false; + } } /** diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java index 406404f6ab571..8dde581b6c707 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java @@ -60,6 +60,17 @@ public ExecutableScript newScript(@Nullable Map params) { int certainty = params == null ? 10 : XContentMapValues.nodeIntegerValue(params.get("certainty"), 10); return new IsPrimeSearchScript(fieldName, certainty); } + + /** + * Indicates if document scores may be needed by the produced scripts. + * + * @return {@code true} if scores are needed. + */ + @Override + public boolean needsScores() { + return false; + } + } private final String fieldName; diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/LanguageModelScoreScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/LanguageModelScoreScript.java index 0a3a08809288e..4eadbe47e202d 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/LanguageModelScoreScript.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/LanguageModelScoreScript.java @@ -49,11 +49,6 @@ public class LanguageModelScoreScript extends AbstractSearchScript { final static public String SCRIPT_NAME = "language_model_script_score"; - @Override - public void setScorer(Scorer scorer) { - // ignore - } - /** * Factory that is registered in * {@link org.elasticsearch.examples.nativescript.plugin.NativeScriptExamplesPlugin#onModule(org.elasticsearch.script.ScriptModule)} @@ -72,6 +67,13 @@ public static class Factory implements NativeScriptFactory { public ExecutableScript newScript(@Nullable Map params) { return new LanguageModelScoreScript(params); } + + + @Override + public boolean needsScores() { + return false; + } + } /** diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java index bf245cbcfce9d..435c3b61c97db 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java @@ -106,6 +106,17 @@ public ExecutableScript newScript(@Nullable Map params) { } return new LookupScript(node.client(), logger, cache, lookupIndex, lookupType, field); } + + /** + * Indicates if document scores may be needed by the produced scripts. + * + * @return {@code true} if scores are needed. + */ + @Override + public boolean needsScores() { + return false; + } + } private final String lookupIndex; diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/PhraseScoreScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/PhraseScoreScript.java index ecd8b09cc11a8..ad7437f186251 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/PhraseScoreScript.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/PhraseScoreScript.java @@ -44,11 +44,6 @@ public class PhraseScoreScript extends AbstractSearchScript { final static public String SCRIPT_NAME = "phrase_script_score"; - @Override - public void setScorer(Scorer scorer) { - // ignore - } - /** * Factory that is registered in * {@link org.elasticsearch.examples.nativescript.plugin.NativeScriptExamplesPlugin#onModule(org.elasticsearch.script.ScriptModule)} @@ -67,6 +62,16 @@ public static class Factory implements NativeScriptFactory { public ExecutableScript newScript(@Nullable Map params) { return new PhraseScoreScript(params); } + + /** + * Indicates if document scores may be needed by the produced scripts. + * + * @return {@code true} if scores are needed. + */ + @Override + public boolean needsScores() { + return false; + } } /** diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptFactory.java b/src/main/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptFactory.java index 4df16a554bbbc..fdf690536bbc3 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptFactory.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptFactory.java @@ -42,6 +42,15 @@ public ExecutableScript newScript(@Nullable Map params) { return new PopularityScoreScript(fieldName); } + /** + * Indicates if document scores may be needed by the produced scripts. + * + * @return {@code true} if scores are needed. + */ + @Override + public boolean needsScores() { + return true; + } /** * This script takes a numeric value from the field specified in the parameter field. And calculates boost diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java b/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java index 17ba7c07e41ed..269f77ea8ef2a 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java @@ -55,6 +55,16 @@ public ExecutableScript newScript(@Nullable Map params) { } } + /** + * Indicates if document scores may be needed by the produced scripts. + * + * @return {@code true} if scores are needed. + */ + @Override + public boolean needsScores() { + return false; + } + private static class RandomSortScript extends AbstractLongSearchScript { private final Random random; @@ -66,11 +76,6 @@ private RandomSortScript() { public long runAsLong() { return random.nextLong(); } - - @Override - public void setScorer(Scorer scorer) { - // we are not using it - ignore - } } private static class PseudoRandomSortScript extends AbstractLongSearchScript { @@ -100,10 +105,5 @@ public long runAsLong() { return -1; } } - - @Override - public void setScorer(Scorer scorer) { - // we are not using it - ignore - } } } diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/TFIDFScoreScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/TFIDFScoreScript.java index a7109686120f8..22c0dbb47e2d7 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/TFIDFScoreScript.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/TFIDFScoreScript.java @@ -43,11 +43,6 @@ public class TFIDFScoreScript extends AbstractSearchScript { final static public String SCRIPT_NAME = "tfidf_script_score"; - @Override - public void setScorer(Scorer scorer) { - // ignore - } - /** * Factory that is registered in * {@link org.elasticsearch.examples.nativescript.plugin.NativeScriptExamplesPlugin#onModule(org.elasticsearch.script.ScriptModule)} @@ -66,6 +61,16 @@ public static class Factory implements NativeScriptFactory { public ExecutableScript newScript(@Nullable Map params) { return new TFIDFScoreScript(params); } + + /** + * Indicates if document scores may be needed by the produced scripts. + * + * @return {@code true} if scores are needed. + */ + @Override + public boolean needsScores() { + return false; + } } /** From f5feb9e839fc3a0e0f35e338c1ed68de60a53122 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Thu, 13 Aug 2015 15:20:38 -0400 Subject: [PATCH 47/69] Add licenses directory to make license checker happy --- licenses/no_deps.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 licenses/no_deps.txt diff --git a/licenses/no_deps.txt b/licenses/no_deps.txt new file mode 100644 index 0000000000000..cfffc5f993b1c --- /dev/null +++ b/licenses/no_deps.txt @@ -0,0 +1 @@ +This plugin has no third party dependencies \ No newline at end of file From b95943ec18eeacb5c417d8389466cbd60d69fb4e Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Thu, 13 Aug 2015 21:18:26 -0400 Subject: [PATCH 48/69] Add an example of using scripts in the scripted metrics aggs Closes #12 --- example/stockaggs.sh | 47 +++++++++++ rest-api-spec/test/nativescript/20_aggs.yaml | 42 ++++++++++ .../plugin/NativeScriptExamplesPlugin.java | 9 ++ .../stockaggs/CombineScriptFactory.java | 61 ++++++++++++++ .../script/stockaggs/InitScriptFactory.java | 50 +++++++++++ .../script/stockaggs/MapScriptFactory.java | 66 +++++++++++++++ .../script/stockaggs/ReduceScriptFactory.java | 60 ++++++++++++++ .../script/ScriptedMetricAggsScriptTests.java | 82 +++++++++++++++++++ 8 files changed, 417 insertions(+) create mode 100755 example/stockaggs.sh create mode 100644 rest-api-spec/test/nativescript/20_aggs.yaml create mode 100644 src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/CombineScriptFactory.java create mode 100644 src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/InitScriptFactory.java create mode 100644 src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/MapScriptFactory.java create mode 100644 src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/ReduceScriptFactory.java create mode 100644 src/test/java/org/elasticsearch/examples/nativescript/script/ScriptedMetricAggsScriptTests.java diff --git a/example/stockaggs.sh b/example/stockaggs.sh new file mode 100755 index 0000000000000..8ade779df580e --- /dev/null +++ b/example/stockaggs.sh @@ -0,0 +1,47 @@ +#!/bin/sh +curl -s -XDELETE "http://localhost:9200/transactions" +echo +curl -s -XPUT "http://localhost:9200/transactions/" -d '{ + "settings": { + "index.number_of_shards": 1, + "index.number_of_replicas": 0 + }, + "mappings": { + "stock": { + "properties": { + "type": { + "type": "string", + "index": "not_analyzed" + }, + "amount": { + "type": "long" + } + } + } + } +}' +echo +curl -s -XPUT 'http://localhost:9200/transactions/stock/1' -d '{"type": "sale", "amount": 80}' +curl -s -XPUT 'http://localhost:9200/transactions/stock/2' -d '{"type": "cost", "amount": 10}' +curl -s -XPUT 'http://localhost:9200/transactions/stock/3' -d '{"type": "cost", "amount": 30}' +curl -s -XPUT 'http://localhost:9200/transactions/stock/4' -d '{"type": "sale", "amount": 130}' + +curl -s -XPOST "http://localhost:9200/transactions/_refresh" +echo +curl -s -XGET "localhost:9200/transactions/stock/_search?pretty=true" -d '{ + "query" : { + "match_all" : {} + }, + "aggs": { + "profit": { + "scripted_metric": { + "init_script" : "stockaggs_init", + "map_script" : "stockaggs_map", + "combine_script" : "stockaggs_combine", + "reduce_script" : "stockaggs_reduce", + "lang": "native" + } + } + }, + "size": 0 +}' diff --git a/rest-api-spec/test/nativescript/20_aggs.yaml b/rest-api-spec/test/nativescript/20_aggs.yaml new file mode 100644 index 0000000000000..2a3e947fa6a1b --- /dev/null +++ b/rest-api-spec/test/nativescript/20_aggs.yaml @@ -0,0 +1,42 @@ +# Integration tests for aggregation scripts +# +setup: + - do: + indices.create: + index: test_index + body: + settings: + index.number_of_shards: 1 + index.number_of_replicas: 0 + mappings.type1: + properties: + type: {type: string, index: not_analyzed} + amount.type: long + + - do: + index: {index: transactions, type: stock, id: 1, body: {type: "sale", amount: 80}} + + - do: + index: {index: transactions, type: stock, id: 2, body: {type: "cost", amount: 10}} + + - do: + index: {index: transactions, type: stock, id: 3, body: {type: "cost", amount: 30}} + + - do: + index: {index: transactions, type: stock, id: 4, body: {type: "sale", amount: 130}} + + - do: + indices.refresh: {} + +--- + +"Primes Test": + - do: + search: + body: + aggs: + profit: + scripted_metric: { "init_script" : "stockaggs_init", "map_script" : "stockaggs_map", "combine_script" : "stockaggs_combine", "reduce_script" : "stockaggs_reduce", "lang": "native"} + + - match: { hits.total: 4} + - match: { aggregations.profit.value: 170} diff --git a/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java b/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java index 633adfc2d0541..a245626ae6797 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java @@ -22,6 +22,10 @@ import org.elasticsearch.examples.nativescript.script.TFIDFScoreScript; import org.elasticsearch.examples.nativescript.script.PopularityScoreScriptFactory; import org.elasticsearch.examples.nativescript.script.RandomSortScriptFactory; +import org.elasticsearch.examples.nativescript.script.stockaggs.CombineScriptFactory; +import org.elasticsearch.examples.nativescript.script.stockaggs.InitScriptFactory; +import org.elasticsearch.examples.nativescript.script.stockaggs.MapScriptFactory; +import org.elasticsearch.examples.nativescript.script.stockaggs.ReduceScriptFactory; import org.elasticsearch.plugins.AbstractPlugin; import org.elasticsearch.script.ScriptModule; @@ -64,5 +68,10 @@ public void onModule(ScriptModule module) { module.registerScript(CosineSimilarityScoreScript.SCRIPT_NAME, CosineSimilarityScoreScript.Factory.class); module.registerScript(PhraseScoreScript.SCRIPT_NAME, PhraseScoreScript.Factory.class); module.registerScript(LanguageModelScoreScript.SCRIPT_NAME, LanguageModelScoreScript.Factory.class); + // Scripted Metric Aggregation Scripts + module.registerScript("stockaggs_init", InitScriptFactory.class); + module.registerScript("stockaggs_map", MapScriptFactory.class); + module.registerScript("stockaggs_combine", CombineScriptFactory.class); + module.registerScript("stockaggs_reduce", ReduceScriptFactory.class); } } diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/CombineScriptFactory.java b/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/CombineScriptFactory.java new file mode 100644 index 0000000000000..8a92f16131fc6 --- /dev/null +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/CombineScriptFactory.java @@ -0,0 +1,61 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.elasticsearch.examples.nativescript.script.stockaggs; + +import org.elasticsearch.common.Nullable; +import org.elasticsearch.script.AbstractExecutableScript; +import org.elasticsearch.script.ExecutableScript; +import org.elasticsearch.script.NativeScriptFactory; + +import java.util.ArrayList; +import java.util.Map; + +/** + * Combine script from https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-scripted-metric-aggregation.html + *

+ * profit = 0; for (t in _agg.transactions) { profit += t }; return profit + */ +public class CombineScriptFactory implements NativeScriptFactory { + + @Override + public ExecutableScript newScript(final @Nullable Map params) { + Map agg = (Map) params.get("_agg"); + final ArrayList transactions = (ArrayList) agg.get(InitScriptFactory.TRANSACTIONS_FIELD); + return new CombineScript(transactions); + } + + @Override + public boolean needsScores() { + return false; + } + + private static class CombineScript extends AbstractExecutableScript { + + private final ArrayList transactions; + + public CombineScript(ArrayList transactions) { + this.transactions = transactions; + } + + @Override + public Object run() { + long profit = 0; + for (long t : transactions) { + profit += t; + } + return profit; + } + } +} diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/InitScriptFactory.java b/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/InitScriptFactory.java new file mode 100644 index 0000000000000..66ed6b9d4c8f6 --- /dev/null +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/InitScriptFactory.java @@ -0,0 +1,50 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.elasticsearch.examples.nativescript.script.stockaggs; + +import org.elasticsearch.common.Nullable; +import org.elasticsearch.script.AbstractExecutableScript; +import org.elasticsearch.script.ExecutableScript; +import org.elasticsearch.script.NativeScriptFactory; + +import java.util.Map; + +import static com.google.common.collect.Lists.newArrayList; + +/** + * Init script from https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-scripted-metric-aggregation.html + * + * _agg['transactions'] = [] + */ +public class InitScriptFactory implements NativeScriptFactory { + + public static final String TRANSACTIONS_FIELD = "transactions"; + + @Override + public ExecutableScript newScript(final @Nullable Map params) { + return new AbstractExecutableScript() { + @Override + public Object run() { + ((Map)params.get("_agg")).put(TRANSACTIONS_FIELD, newArrayList()); + return null; + } + }; + } + + @Override + public boolean needsScores() { + return false; + } +} diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/MapScriptFactory.java b/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/MapScriptFactory.java new file mode 100644 index 0000000000000..f73784f6840e8 --- /dev/null +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/MapScriptFactory.java @@ -0,0 +1,66 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.elasticsearch.examples.nativescript.script.stockaggs; + +import org.elasticsearch.common.Nullable; +import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.script.AbstractSearchScript; +import org.elasticsearch.script.ExecutableScript; +import org.elasticsearch.script.NativeScriptFactory; + +import java.util.ArrayList; +import java.util.Map; + +/** + * Map script from https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-scripted-metric-aggregation.html + *

+ * if (doc['type'].value == \"sale\") { _agg.transactions.add(doc['amount'].value) } else {_agg.transactions.add(-1 * doc['amount'].value)} + */ +public class MapScriptFactory implements NativeScriptFactory { + + @Override + public ExecutableScript newScript(final @Nullable Map params) { + Map agg = (Map) params.get("_agg"); + ArrayList transactions = (ArrayList) agg.get(InitScriptFactory.TRANSACTIONS_FIELD); + return new MapScript(transactions); + } + + @Override + public boolean needsScores() { + return false; + } + + private static class MapScript extends AbstractSearchScript { + + private final ArrayList transactions; + + public MapScript(ArrayList transactions) { + this.transactions = transactions; + + } + + @Override + public Object run() { + ScriptDocValues.Longs amount = (ScriptDocValues.Longs) doc().get("amount"); + ScriptDocValues.Strings type = (ScriptDocValues.Strings) doc().get("type"); + if ("sale".equals(type.getValue())) { + transactions.add(amount.getValue()); + } else { + transactions.add(-amount.getValue()); + } + return null; + } + } +} diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/ReduceScriptFactory.java b/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/ReduceScriptFactory.java new file mode 100644 index 0000000000000..98a558b47a4d2 --- /dev/null +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/ReduceScriptFactory.java @@ -0,0 +1,60 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.elasticsearch.examples.nativescript.script.stockaggs; + +import org.elasticsearch.common.Nullable; +import org.elasticsearch.script.AbstractExecutableScript; +import org.elasticsearch.script.ExecutableScript; +import org.elasticsearch.script.NativeScriptFactory; + +import java.util.ArrayList; +import java.util.Map; + +/** + * Combine script from https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-scripted-metric-aggregation.html + *

+ * profit = 0; for (t in _agg.transactions) { profit += t }; return profit + */ +public class ReduceScriptFactory implements NativeScriptFactory { + + @Override + public ExecutableScript newScript(final @Nullable Map params) { + final ArrayList aggs = (ArrayList) params.get("_aggs"); + return new ReduceScript(aggs); + } + + @Override + public boolean needsScores() { + return false; + } + + private static class ReduceScript extends AbstractExecutableScript { + + private final ArrayList aggs; + + public ReduceScript(ArrayList aggs) { + this.aggs = aggs; + } + + @Override + public Object run() { + long profit = 0; + for (long t : aggs) { + profit += t; + } + return profit; + } + } +} diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/ScriptedMetricAggsScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/ScriptedMetricAggsScriptTests.java new file mode 100644 index 0000000000000..fc2a12c384586 --- /dev/null +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/ScriptedMetricAggsScriptTests.java @@ -0,0 +1,82 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.elasticsearch.examples.nativescript.script; + +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptService; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.elasticsearch.search.aggregations.AggregationBuilders.scriptedMetric; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; +import static org.hamcrest.Matchers.equalTo; + +/** + */ +public class ScriptedMetricAggsScriptTests extends AbstractSearchScriptTests { + + @SuppressWarnings("unchecked") + @Test + public void testScriptedMetricAggs() throws Exception { + + // Create a new lookup index + String stockMapping = XContentFactory.jsonBuilder().startObject().startObject("stock") + .startObject("properties") + .startObject("type").field("type", "string").field("index", "not_analyzed").endObject() + .startObject("amount").field("type", "long").endObject() + .endObject().endObject().endObject() + .string(); + + assertAcked(prepareCreate("transactions") + .addMapping("stock", stockMapping)); + + List indexBuilders = new ArrayList(); + // Index stock records: + indexBuilders.add(client().prepareIndex("transactions", "stock", "1").setSource("type", "sale", "amount", 80)); + indexBuilders.add(client().prepareIndex("transactions", "stock", "2").setSource("type", "cost", "amount", 10)); + indexBuilders.add(client().prepareIndex("transactions", "stock", "3").setSource("type", "cost", "amount", 30)); + indexBuilders.add(client().prepareIndex("transactions", "stock", "4").setSource("type", "sale", "amount", 130)); + + indexRandom(true, indexBuilders); + + // Find profit from all transaction + SearchResponse searchResponse = client().prepareSearch("transactions") + .setTypes("stock") + .setQuery(matchAllQuery()) + .setSize(0) + .addAggregation(scriptedMetric("profit") + .initScript(new Script("stockaggs_init", ScriptService.ScriptType.INLINE, "native", null)) + .mapScript(new Script("stockaggs_map", ScriptService.ScriptType.INLINE, "native", null)) + .combineScript(new Script("stockaggs_combine", ScriptService.ScriptType.INLINE, "native", null)) + .reduceScript(new Script("stockaggs_reduce", ScriptService.ScriptType.INLINE, "native", null))) + .execute().actionGet(); + + assertNoFailures(searchResponse); + + // There should be 4 hits - we are running aggs on everything + assertHitCount(searchResponse, 4); + + // The profit should be 170 + assertThat((long) searchResponse.getAggregations().get("profit").getProperty("value"), equalTo(170L)); + } + + +} From 2b87eb9e21a554f954ba63b1349d6df66877ab31 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Thu, 20 Aug 2015 12:59:35 -0400 Subject: [PATCH 49/69] Changes related to renaming of AbstractPlugin and parent pom --- pom.xml | 2 +- .../nativescript/plugin/NativeScriptExamplesPlugin.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 0c7f14e188502..5903b4e7e4f84 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.elasticsearch.plugin - elasticsearch-plugin + plugins 2.0.0-beta1-SNAPSHOT diff --git a/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java b/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java index a245626ae6797..2e728b19d95bd 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java @@ -26,7 +26,7 @@ import org.elasticsearch.examples.nativescript.script.stockaggs.InitScriptFactory; import org.elasticsearch.examples.nativescript.script.stockaggs.MapScriptFactory; import org.elasticsearch.examples.nativescript.script.stockaggs.ReduceScriptFactory; -import org.elasticsearch.plugins.AbstractPlugin; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.ScriptModule; /** @@ -34,7 +34,7 @@ * first time. If you change the name of this plugin, make sure to update * src/main/resources/es-plugin.properties file that points to this class. */ -public class NativeScriptExamplesPlugin extends AbstractPlugin { +public class NativeScriptExamplesPlugin extends Plugin { /** * The name of the plugin. From 8881606d503c2c385df04131a5b510c4deba42c0 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Fri, 21 Aug 2015 17:14:48 -0400 Subject: [PATCH 50/69] Update README to reflect changes in 2.0 Closes #13 --- README.textile | 51 ++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 39 insertions(+), 12 deletions(-) diff --git a/README.textile b/README.textile index c36786cee2294..c3d2a2cfa1d6d 100644 --- a/README.textile +++ b/README.textile @@ -12,29 +12,46 @@ bc.. . |- pom.xml |- src |- main - |- assemblies - | |- plugin.xml + | |- java + | | |- ... source code ... + |- test |- java | |- ... source code ... |- resources - |- es-plugin.properties + |- ... test resources ... + p. An Elasticsearch plugin can be created by following these six steps. * Create pom.xml file in the root directory of your plugin. The "pom.xml":https://github.com/imotov/elasticsearch-native-script-example/blob/master/pom.xml file in this project can be used as a starting point. * Create source code directories: -** @mkdir -p src/main/assemblies@ ** @mkdir -p src/main/java@ -** @mkdir -p src/main/resources@ -* Download "plugin.xml":https://github.com/imotov/elasticsearch-native-script-example/blob/master/src/main/assemblies/plugin.xml to the @src/main/assemblies@ directory. This file specifies how plugin .zip package should be built. By default, the project artifact with all its dependencies is going to be included in the plugin package. For more information on how to configure the content of the package, see "Maven Assembly Plugin Documentation":http://maven.apache.org/plugins/maven-assembly-plugin/. +** @mkdir -p src/test/java@ +** @mkdir -p src/test/resources@ +* The parent project @org.elasticsearch.plugin:plugins@ provides all needed tasks to assemble the plugin. It is using the following properties that should be modified to match the project's plugin class name and license file definition. + +bc.. + + org.elasticsearch.examples.nativescript.plugin.NativeScriptExamplesPlugin + + + ${project.basedir}/dev-tools/src/main/resources/license-check/native_script_example_license_header.txt + ${project.basedir}/dev-tools/src/main/resources/license-check/license_header_definition.xml + + ... other properties ...... + + + +p. + * Create main Plugin class in the @src/main/java@ directory. This project is using @org.elasticsearch.examples.nativescript.plugin.NativeScriptExamplesPlugin@ class as an example, so the it has to be saved as @src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java@ bc.. package org.elasticsearch.examples.nativescript.plugin; -import org.elasticsearch.plugins.AbstractPlugin; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.ScriptModule; -public class NativeScriptExamplesPlugin extends AbstractPlugin { +public class NativeScriptExamplesPlugin extends Plugin { @Override public String name() { return "native-script-examples"; @@ -46,14 +63,24 @@ public class NativeScriptExamplesPlugin extends AbstractPlugin { } } -p. - -* Create @es-plugin.properties@ file in the @src/main/resources@ directory to point to the Plugin class that was created in the previous step: +p. -bc. plugin=org.elasticsearch.examples.nativescript.plugin.NativeScriptExamplesPlugin +* The parent project will automatically create @plugin-descriptor.properties@ for you. +* If you are not using the standard parent project, you can create this file manually by using this "plugin-descriptor.properties":https://github.com/elastic/elasticsearch/blob/master/dev-tools/src/main/resources/plugin-metadata/plugin-descriptor.properties as a template. You will also need to package the plugin into .zip file, which can be done using maven assemble task and "plugin-assembly.xml":https://github.com/elastic/elasticsearch/blob/master/dev-tools/src/main/resources/plugin-metadata/plugin-assembly.xml assembly definition. * The plugin can be built using @mvn package@ command. The assembled .zip package can be found in the @target/releases/@ directory and deployed to elasticsearch installation using @plugin -install plugin-name -url path/to/plugin/zip/file@. +h2. Migration from 1.x + +p. The plugin infrastructure significantly changed in 2.0. So, the plugin project will need to be modified in order to be used with elasticsearch 2.0: + +* Instead of using @es-plugin.properties@ file that in 1.x was places in the plugin jar, the plugin infrastructure is now using the @plugin-descriptor.properties@ file that describes not only the main plugin class, version and description but also plugin type (_site and/or jvm), required minimal java and Elasticsearch versions. The @plugin-descriptor.properties@ file should be placed into root directory of the .zip file that the plugin is packaged into. The simplest way to deal with this change is by switching the plugin project to @org.elasticsearch.plugin:plugins@ as a parent. +* Elasticsearch 2.0 is also stricter when it comes to plugin classes. For example, the "jar hell" prevention mechanism will not allow the plugin to contain classes that are already defined in the Elasticsearch classpath. Make sure that your project doesn't have any dependencies that are conflicting with existing elasticsearch classes. +* In 2.0 the base class for the plugin was renamed from @AbstractPlugin@ to @Plugin@ +* Plugins are no longer loaded from the classpath, so they have to be "explicitly loaded":https://github.com/imotov/elasticsearch-native-script-example/commit/96f8b93f27f80346503dfa91e564b87c9334fbdf in the tests. +* Some base test classes need to be "renamed":https://github.com/imotov/elasticsearch-native-script-example/commit/6c4ba4ab967b938fd06200b51e78dcf523861c40. +* Native scripts now have to "indicate":https://github.com/imotov/elasticsearch-native-script-example/commit/4cda2ccdaa8094c5de3e86ccfeea0fe56b26d353 whether they use the `_score` or not. + h2. Adding Native Scripts p. Now that the plugin infrastructure is complete, it's possible to add a native script. From a19cdcdf99106581d4bfd896abcd9a38bee0ffbf Mon Sep 17 00:00:00 2001 From: Yann Barraud Date: Sat, 22 Aug 2015 14:48:25 +0200 Subject: [PATCH 51/69] Fixing glitches in readme --- README.textile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.textile b/README.textile index c3d2a2cfa1d6d..4c8c7e888a225 100644 --- a/README.textile +++ b/README.textile @@ -42,7 +42,8 @@ bc.. -p. + +p. * Create main Plugin class in the @src/main/java@ directory. This project is using @org.elasticsearch.examples.nativescript.plugin.NativeScriptExamplesPlugin@ class as an example, so the it has to be saved as @src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java@ @@ -63,7 +64,7 @@ public class NativeScriptExamplesPlugin extends Plugin { } } -p. +p. * The parent project will automatically create @plugin-descriptor.properties@ for you. * If you are not using the standard parent project, you can create this file manually by using this "plugin-descriptor.properties":https://github.com/elastic/elasticsearch/blob/master/dev-tools/src/main/resources/plugin-metadata/plugin-descriptor.properties as a template. You will also need to package the plugin into .zip file, which can be done using maven assemble task and "plugin-assembly.xml":https://github.com/elastic/elasticsearch/blob/master/dev-tools/src/main/resources/plugin-metadata/plugin-assembly.xml assembly definition. From e41024a3eae263522270e973cb3385e37587f8a2 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Fri, 28 Aug 2015 13:04:45 -0400 Subject: [PATCH 52/69] Update to 2.1.0-SNAPSHOT --- pom.xml | 4 ++-- .../nativescript/script/stockaggs/InitScriptFactory.java | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index 5903b4e7e4f84..915e24e9e50ec 100644 --- a/pom.xml +++ b/pom.xml @@ -7,13 +7,13 @@ org.elasticsearch.plugin plugins - 2.0.0-beta1-SNAPSHOT + 2.1.0-SNAPSHOT elasticsearch-native-script-example org.elasticsearch.plugin elasticsearch-native-script-example - 2.0.0-beta1-SNAPSHOT + 2.1.0-SNAPSHOT ElasticSearch Plugin with Native Script Examples 2013 diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/InitScriptFactory.java b/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/InitScriptFactory.java index 66ed6b9d4c8f6..c80878289bfa3 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/InitScriptFactory.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/InitScriptFactory.java @@ -19,10 +19,9 @@ import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.NativeScriptFactory; +import java.util.ArrayList; import java.util.Map; -import static com.google.common.collect.Lists.newArrayList; - /** * Init script from https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-scripted-metric-aggregation.html * @@ -37,7 +36,7 @@ public ExecutableScript newScript(final @Nullable Map params) { return new AbstractExecutableScript() { @Override public Object run() { - ((Map)params.get("_agg")).put(TRANSACTIONS_FIELD, newArrayList()); + ((Map)params.get("_agg")).put(TRANSACTIONS_FIELD, new ArrayList<>()); return null; } }; From 089b01cbe4d1cdb253b85b1cc95ff187ec06359b Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Mon, 31 Aug 2015 10:25:46 -0400 Subject: [PATCH 53/69] Switch to the direct way of injecting plugins into test nodes Changes related to elastic/elasticsearch#13055 --- .../script/AbstractSearchScriptTests.java | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTests.java index 435e0a9f38c3e..0626e4befb3a5 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTests.java @@ -15,10 +15,13 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.examples.nativescript.plugin.NativeScriptExamplesPlugin; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; +import java.util.Collection; + import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; @@ -36,12 +39,7 @@ public Settings indexSettings() { } @Override - protected Settings nodeSettings(int nodeOrdinal) { - return Settings.settingsBuilder() - .put("gateway.type", "none") - .put("plugin.types", NativeScriptExamplesPlugin.class) - .put(super.nodeSettings(nodeOrdinal)) - .build(); + protected Collection> nodePlugins() { + return pluginList(NativeScriptExamplesPlugin.class); } - } From 4a2debec7b253df60e48d950229b11ecab5f3e7a Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Mon, 31 Aug 2015 10:45:04 -0400 Subject: [PATCH 54/69] Update README to include branches that should be used for different version of elasticsearch. --- README.textile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.textile b/README.textile index 4c8c7e888a225..d054151693c7a 100644 --- a/README.textile +++ b/README.textile @@ -4,6 +4,13 @@ h2. Introduction p. This plugin contains several examples of "native script":http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-scripting.html for Elasticsearch. +p. Please make sure to use the correct branch of this repository that corresponds to the version of elasticsearch that you are developing the plugin for. + +|_. Example Plugin Branch |_. Elasticsearch | +| "master":https://github.com/imotov/elasticsearch-native-script-example | 2.1.0 and above | +| "2.0":https://github.com/imotov/elasticsearch-native-script-example/tree/2.0 | 2.0.x | +| "1.x":https://github.com/imotov/elasticsearch-native-script-example/tree/1.x | 1.x.x | + h2. Creating Elasticsearch Plugin p. The simplest way to deploy native script is by wrapping it into standard Elasticsearch plugin infrastructure. An Elasticsearch plugin can be written in java and built using maven. A typical plugin source directory looks like this: From 2ba2cd61bec2f73952d5f16b005b7331d7c4bed6 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Thu, 3 Sep 2015 08:49:31 -0400 Subject: [PATCH 55/69] Update the test case name to follow the new test naming convention See elastic/elasticsearch#13282 --- ...SearchScriptTests.java => AbstractSearchScriptTestCase.java} | 2 +- .../examples/nativescript/script/IsPrimeSearchScriptTests.java | 2 +- .../examples/nativescript/script/LookupScriptTests.java | 2 +- .../nativescript/script/PopularityScoreScriptTests.java | 2 +- .../examples/nativescript/script/RandomSortScriptTests.java | 2 +- .../nativescript/script/ScriptedMetricAggsScriptTests.java | 2 +- .../examples/nativescript/script/TermScoringScriptTests.java | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) rename src/test/java/org/elasticsearch/examples/nativescript/script/{AbstractSearchScriptTests.java => AbstractSearchScriptTestCase.java} (95%) diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTestCase.java similarity index 95% rename from src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTests.java rename to src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTestCase.java index 0626e4befb3a5..21e10e46e4ad5 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTestCase.java @@ -28,7 +28,7 @@ /** */ @ClusterScope(scope = Scope.SUITE, numDataNodes = 1) -public abstract class AbstractSearchScriptTests extends ESIntegTestCase { +public abstract class AbstractSearchScriptTestCase extends ESIntegTestCase { @Override public Settings indexSettings() { diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScriptTests.java index a0c50b6a506f4..7464a61e5124c 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScriptTests.java @@ -33,7 +33,7 @@ /** */ -public class IsPrimeSearchScriptTests extends AbstractSearchScriptTests { +public class IsPrimeSearchScriptTests extends AbstractSearchScriptTestCase { public static int[] PRIMES_10 = new int[] { 2, 3, 5, 7, 11, 13, 17, 19, 23, 29 }; diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/LookupScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/LookupScriptTests.java index c349c65ac3b0e..9745e74f4b026 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/LookupScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/LookupScriptTests.java @@ -34,7 +34,7 @@ /** */ -public class LookupScriptTests extends AbstractSearchScriptTests { +public class LookupScriptTests extends AbstractSearchScriptTestCase { @SuppressWarnings("unchecked") @Test diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptTests.java index 68f4b4451fd8f..dc998d68c91e6 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptTests.java @@ -38,7 +38,7 @@ /** */ -public class PopularityScoreScriptTests extends AbstractSearchScriptTests { +public class PopularityScoreScriptTests extends AbstractSearchScriptTestCase { @Test public void testPopularityScoring() throws Exception { diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptTests.java index ab839e23ca948..5b0e88c959041 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptTests.java @@ -32,7 +32,7 @@ /** */ -public class RandomSortScriptTests extends AbstractSearchScriptTests { +public class RandomSortScriptTests extends AbstractSearchScriptTestCase { @Test public void testPseudoRandomScript() throws Exception { diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/ScriptedMetricAggsScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/ScriptedMetricAggsScriptTests.java index fc2a12c384586..01fffbc144854 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/ScriptedMetricAggsScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/ScriptedMetricAggsScriptTests.java @@ -31,7 +31,7 @@ /** */ -public class ScriptedMetricAggsScriptTests extends AbstractSearchScriptTests { +public class ScriptedMetricAggsScriptTests extends AbstractSearchScriptTestCase { @SuppressWarnings("unchecked") @Test diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/TermScoringScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/TermScoringScriptTests.java index 11aab6f685a0b..704c3174e24f8 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/TermScoringScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/TermScoringScriptTests.java @@ -43,7 +43,7 @@ * document as would be expected if each document in the index contains only one * and always the same term. */ -public class TermScoringScriptTests extends AbstractSearchScriptTests { +public class TermScoringScriptTests extends AbstractSearchScriptTestCase { final static String[] searchTerms = {"foo", "bar"}; final static String field = "field"; From 89f2009afef7c7214bd29138db1716ddfa5080df Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Fri, 11 Sep 2015 16:54:11 -0400 Subject: [PATCH 56/69] Update to 3.0.0-SNAPSHOT --- pom.xml | 4 ++-- rest-api-spec/test/nativescript/10_primes.yaml | 2 +- .../nativescript/script/IsPrimeSearchScriptTests.java | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pom.xml b/pom.xml index 915e24e9e50ec..717d07559bbc1 100644 --- a/pom.xml +++ b/pom.xml @@ -7,13 +7,13 @@ org.elasticsearch.plugin plugins - 2.1.0-SNAPSHOT + 3.0.0-SNAPSHOT elasticsearch-native-script-example org.elasticsearch.plugin elasticsearch-native-script-example - 2.1.0-SNAPSHOT + 3.0.0-SNAPSHOT ElasticSearch Plugin with Native Script Examples 2013 diff --git a/rest-api-spec/test/nativescript/10_primes.yaml b/rest-api-spec/test/nativescript/10_primes.yaml index c43b748fbbec4..61944a1f0223a 100644 --- a/rest-api-spec/test/nativescript/10_primes.yaml +++ b/rest-api-spec/test/nativescript/10_primes.yaml @@ -37,7 +37,7 @@ setup: search: body: query: - filtered: + constant_score: filter: script: script: is_prime diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScriptTests.java index 7464a61e5124c..57832a5bafc0f 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScriptTests.java @@ -23,10 +23,10 @@ import org.junit.Test; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; -import static com.google.common.collect.Maps.newHashMap; import static org.elasticsearch.index.query.QueryBuilders.scriptQuery; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; import static org.hamcrest.Matchers.equalTo; @@ -72,7 +72,7 @@ public void testIsPrimeScript() throws Exception { indexRandom(true, indexBuilders); - Map params = newHashMap(); + Map params = new HashMap(); params.put("field", "number"); // Retrieve first 10 prime records SearchResponse searchResponse = client().prepareSearch("test") @@ -92,7 +92,7 @@ public void testIsPrimeScript() throws Exception { assertThat(searchResponse.getHits().getAt(i).field("name").getValue().toString(), equalTo("rec " + PRIMES_10[i])); } - params = newHashMap(); + params = new HashMap(); params.put("field", "number"); params.put("certainty", 0); // Check certainty parameter - with certainty == 0, it should return all numbers, but only if numbers are present From b0efbbb11c5148e07bb532f628553ba987a249a3 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Wed, 16 Sep 2015 10:57:30 -0400 Subject: [PATCH 57/69] Remove reference to Guava Charsets See elastic/elasticsearch#13224 --- .../nativescript/script/RandomSortScriptFactory.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java b/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java index 269f77ea8ef2a..f8a250ad402c9 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java @@ -14,8 +14,6 @@ package org.elasticsearch.examples.nativescript.script; -import com.google.common.base.Charsets; -import org.apache.lucene.search.Scorer; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.fielddata.ScriptDocValues; @@ -25,6 +23,7 @@ import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.NativeScriptFactory; +import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.Map; @@ -91,7 +90,7 @@ public long runAsLong() { try { MessageDigest m = MessageDigest.getInstance("MD5"); m.reset(); - m.update((fieldData.getValue() + salt).getBytes(Charsets.UTF_8)); + m.update((fieldData.getValue() + salt).getBytes(StandardCharsets.UTF_8)); byte[] sort = m.digest(); return (sort[0] & 0xFFL) << 56 | (sort[1] & 0xFFL) << 48 From db6c5686d84eb00a05ee760905ac293fb3219ef1 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Thu, 17 Sep 2015 09:21:54 -0400 Subject: [PATCH 58/69] Move rest-api-spec tests to resources dirs See elastic/elasticsearch#13611 --- .../resources/rest-api-spec}/test/nativescript/10_primes.yaml | 0 .../test/resources/rest-api-spec}/test/nativescript/20_aggs.yaml | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename {rest-api-spec => src/test/resources/rest-api-spec}/test/nativescript/10_primes.yaml (100%) rename {rest-api-spec => src/test/resources/rest-api-spec}/test/nativescript/20_aggs.yaml (100%) diff --git a/rest-api-spec/test/nativescript/10_primes.yaml b/src/test/resources/rest-api-spec/test/nativescript/10_primes.yaml similarity index 100% rename from rest-api-spec/test/nativescript/10_primes.yaml rename to src/test/resources/rest-api-spec/test/nativescript/10_primes.yaml diff --git a/rest-api-spec/test/nativescript/20_aggs.yaml b/src/test/resources/rest-api-spec/test/nativescript/20_aggs.yaml similarity index 100% rename from rest-api-spec/test/nativescript/20_aggs.yaml rename to src/test/resources/rest-api-spec/test/nativescript/20_aggs.yaml From 548c09028f2ff7091ceed71ff83a32ee578701c9 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Tue, 29 Sep 2015 14:48:03 +0200 Subject: [PATCH 59/69] Update code to work with the latest 3.0 snapshot --- .../plugin/NativeScriptExamplesPlugin.java | 2 +- .../script/CosineSimilarityScoreScript.java | 3 +- .../script/IsPrimeSearchScript.java | 5 ++- .../script/LanguageModelScoreScript.java | 4 +- .../nativescript/script/LookupScript.java | 5 ++- .../script/PhraseScoreScript.java | 1 + .../script/PopularityScoreScriptFactory.java | 5 ++- .../script/RandomSortScriptFactory.java | 2 +- .../nativescript/script/TFIDFScoreScript.java | 1 + .../stockaggs/CombineScriptFactory.java | 3 +- .../script/stockaggs/InitScriptFactory.java | 1 + .../script/stockaggs/MapScriptFactory.java | 3 +- .../script/stockaggs/ReduceScriptFactory.java | 3 +- .../script/IsPrimeSearchScriptTests.java | 4 +- .../script/PopularityScoreScriptTests.java | 8 ++-- .../script/RandomSortScriptTests.java | 10 +++-- .../script/TermScoringScriptTests.java | 42 +++++++++---------- 17 files changed, 57 insertions(+), 45 deletions(-) diff --git a/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java b/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java index 2e728b19d95bd..077ba11a29d70 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java @@ -38,7 +38,7 @@ public class NativeScriptExamplesPlugin extends Plugin { /** * The name of the plugin. - *

+ *

* This name will be used by elasticsearch in the log file to refer to this plugin. * * @return plugin name. diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/CosineSimilarityScoreScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/CosineSimilarityScoreScript.java index 0d6a4750f3305..889f2ffe526a1 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/CosineSimilarityScoreScript.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/CosineSimilarityScoreScript.java @@ -18,7 +18,6 @@ import java.util.ArrayList; import java.util.Map; -import org.apache.lucene.search.Scorer; import org.elasticsearch.script.ScriptException; import org.elasticsearch.common.Nullable; @@ -81,8 +80,8 @@ public boolean needsScores() { * @param params * terms that a scored are placed in this parameter. Initialize * them here. - * @throws ScriptException */ + @SuppressWarnings("unchecked") private CosineSimilarityScoreScript(Map params) throws ScriptException { params.entrySet(); // get the terms diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java index 8dde581b6c707..62e74383870d6 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java @@ -28,7 +28,7 @@ /** * Implementation of the native script that checks that the field exists and contains a prime number. - *

+ *

* The native script has to implement {@link org.elasticsearch.script.SearchScript} interface. But the * {@link org.elasticsearch.script.AbstractSearchScript} class can be used to simplify the implementation. */ @@ -89,9 +89,10 @@ private IsPrimeSearchScript(String fieldName, int certainty) { } @Override + @SuppressWarnings("unchecked") public Object run() { // First we get field using doc lookup - ScriptDocValues docValue = (ScriptDocValues) doc().get(fieldName); + ScriptDocValues docValue = (ScriptDocValues) doc().get(fieldName); // Check if field exists if (docValue != null && !docValue.isEmpty()) { try { diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/LanguageModelScoreScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/LanguageModelScoreScript.java index 4eadbe47e202d..fd609368f6ee1 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/LanguageModelScoreScript.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/LanguageModelScoreScript.java @@ -81,6 +81,7 @@ public boolean needsScores() { * terms that a scored are placed in this parameter. Initialize * them here. */ + @SuppressWarnings("unchecked") private LanguageModelScoreScript(Map params) { params.entrySet(); // get the terms @@ -97,6 +98,7 @@ private LanguageModelScoreScript(Map params) { } @Override + @SuppressWarnings("unchecked") public Object run() { try { double score = 0.0; @@ -109,7 +111,7 @@ public Object run() { * http://www.elasticsearch.org/guide * /en/elasticsearch/reference/current/mapping-core-types.html) */ - ScriptDocValues docValues = (ScriptDocValues) doc().get(docLengthField); + ScriptDocValues docValues = (ScriptDocValues) doc().get(docLengthField); if (docValues == null || !docValues.isEmpty()) { long L_d = ((ScriptDocValues.Longs) docValues).getValue(); for (int i = 0; i < terms.size(); i++) { diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java index 435c3b61c97db..c18dbd44f6eb8 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java @@ -71,7 +71,7 @@ public Factory(Node node, Settings settings) { // Setup lookup cache ByteSizeValue size = settings.getAsBytesSize("examples.nativescript.lookup.size", null); TimeValue expire = settings.getAsTime("expire", null); - CacheBuilder cacheBuilder = CacheBuilder.newBuilder(); + CacheBuilder cacheBuilder = CacheBuilder.newBuilder(); if (size != null) { cacheBuilder.maximumSize(size.bytes()); } @@ -139,9 +139,10 @@ private LookupScript(Client client, ESLogger logger, Cache } @Override + @SuppressWarnings("unchecked") public Object run() { // First we get field using doc lookup - ScriptDocValues docValue = (ScriptDocValues) doc().get(field); + ScriptDocValues docValue = (ScriptDocValues) doc().get(field); // This is not very efficient // Check if field exists if (docValue != null && !docValue.isEmpty()) { diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/PhraseScoreScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/PhraseScoreScript.java index ad7437f186251..843530948dfa1 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/PhraseScoreScript.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/PhraseScoreScript.java @@ -79,6 +79,7 @@ public boolean needsScores() { * terms that a scored are placed in this parameter. Initialize * them here. */ + @SuppressWarnings("unchecked") private PhraseScoreScript(Map params) { params.entrySet(); // get the terms diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptFactory.java b/src/main/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptFactory.java index fdf690536bbc3..26cd8bd344e1b 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptFactory.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptFactory.java @@ -28,7 +28,7 @@ /** * Factory for the script that boosts score of a record based on a value of the record's field. - *

+ *

* This native script demonstrates how to write native custom scores scripts. */ public class PopularityScoreScriptFactory implements NativeScriptFactory { @@ -73,9 +73,10 @@ public void setScorer(Scorer scorer) { } @Override + @SuppressWarnings("unchecked") public float runAsFloat() { try { - ScriptDocValues docValue = (ScriptDocValues) doc().get(field); + ScriptDocValues docValue = (ScriptDocValues) doc().get(field); if (docValue != null && !docValue.isEmpty()) { ScriptDocValues.Longs fieldData = (ScriptDocValues.Longs) docValue; double boost = 1 + Math.log10(fieldData.getValue() + 1); diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java b/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java index f8a250ad402c9..f3a754810c6f1 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java @@ -32,7 +32,7 @@ /** * This script demonstrates how native scripts can be used to create custom sort order. * Since sort operation is expecting float parameter, the {@link AbstractFloatSearchScript} can be used. - *

+ *

* The script accepts one optional parameter salt. If parameter is specified, a pseudo random sort order is used. * Otherwise, a random sort order is used. */ diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/TFIDFScoreScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/TFIDFScoreScript.java index 22c0dbb47e2d7..b1c14204f8af7 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/TFIDFScoreScript.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/TFIDFScoreScript.java @@ -78,6 +78,7 @@ public boolean needsScores() { * terms that a scored are placed in this parameter. Initialize * them here. */ + @SuppressWarnings("unchecked") private TFIDFScoreScript(Map params) { params.entrySet(); // get the terms diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/CombineScriptFactory.java b/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/CombineScriptFactory.java index 8a92f16131fc6..7e588ffb816b5 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/CombineScriptFactory.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/CombineScriptFactory.java @@ -24,12 +24,13 @@ /** * Combine script from https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-scripted-metric-aggregation.html - *

+ *

* profit = 0; for (t in _agg.transactions) { profit += t }; return profit */ public class CombineScriptFactory implements NativeScriptFactory { @Override + @SuppressWarnings("unchecked") public ExecutableScript newScript(final @Nullable Map params) { Map agg = (Map) params.get("_agg"); final ArrayList transactions = (ArrayList) agg.get(InitScriptFactory.TRANSACTIONS_FIELD); diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/InitScriptFactory.java b/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/InitScriptFactory.java index c80878289bfa3..dbca240a2fcab 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/InitScriptFactory.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/InitScriptFactory.java @@ -32,6 +32,7 @@ public class InitScriptFactory implements NativeScriptFactory { public static final String TRANSACTIONS_FIELD = "transactions"; @Override + @SuppressWarnings("unchecked") public ExecutableScript newScript(final @Nullable Map params) { return new AbstractExecutableScript() { @Override diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/MapScriptFactory.java b/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/MapScriptFactory.java index f73784f6840e8..fee4f402f6baa 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/MapScriptFactory.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/MapScriptFactory.java @@ -25,12 +25,13 @@ /** * Map script from https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-scripted-metric-aggregation.html - *

+ *

* if (doc['type'].value == \"sale\") { _agg.transactions.add(doc['amount'].value) } else {_agg.transactions.add(-1 * doc['amount'].value)} */ public class MapScriptFactory implements NativeScriptFactory { @Override + @SuppressWarnings("unchecked") public ExecutableScript newScript(final @Nullable Map params) { Map agg = (Map) params.get("_agg"); ArrayList transactions = (ArrayList) agg.get(InitScriptFactory.TRANSACTIONS_FIELD); diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/ReduceScriptFactory.java b/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/ReduceScriptFactory.java index 98a558b47a4d2..186b3a09ce411 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/ReduceScriptFactory.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/ReduceScriptFactory.java @@ -24,12 +24,13 @@ /** * Combine script from https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-scripted-metric-aggregation.html - *

+ *

* profit = 0; for (t in _agg.transactions) { profit += t }; return profit */ public class ReduceScriptFactory implements NativeScriptFactory { @Override + @SuppressWarnings("unchecked") public ExecutableScript newScript(final @Nullable Map params) { final ArrayList aggs = (ArrayList) params.get("_aggs"); return new ReduceScript(aggs); diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScriptTests.java index 57832a5bafc0f..492e3b1ec8841 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScriptTests.java @@ -72,7 +72,7 @@ public void testIsPrimeScript() throws Exception { indexRandom(true, indexBuilders); - Map params = new HashMap(); + Map params = new HashMap<>(); params.put("field", "number"); // Retrieve first 10 prime records SearchResponse searchResponse = client().prepareSearch("test") @@ -92,7 +92,7 @@ public void testIsPrimeScript() throws Exception { assertThat(searchResponse.getHits().getAt(i).field("name").getValue().toString(), equalTo("rec " + PRIMES_10[i])); } - params = new HashMap(); + params = new HashMap<>(); params.put("field", "number"); params.put("certainty", 0); // Check certainty parameter - with certainty == 0, it should return all numbers, but only if numbers are present diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptTests.java index dc998d68c91e6..bac839d0683ae 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptTests.java @@ -31,6 +31,8 @@ import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.lucene.search.function.CombineFunction; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder; import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService; @@ -79,9 +81,9 @@ public void testPopularityScoring() throws Exception { Map params = MapBuilder.newMapBuilder().put("field", "number").map(); // Retrieve first 10 hits SearchResponse searchResponse = client().prepareSearch("test") - .setQuery(functionScoreQuery(matchQuery("name", "rec")) - .boostMode(CombineFunction.REPLACE) - .add(ScoreFunctionBuilders.scriptFunction(new Script("popularity", ScriptService.ScriptType.INLINE, "native", params)))) + .setQuery(QueryBuilders.functionScoreQuery(matchQuery("name", "rec"), new FunctionScoreQueryBuilder.FilterFunctionBuilder[]{ + new FunctionScoreQueryBuilder.FilterFunctionBuilder(ScoreFunctionBuilders.scriptFunction(new Script("popularity", ScriptService.ScriptType.INLINE, "native", params)))}) + .boostMode(CombineFunction.REPLACE)) .setSize(10) .addField("name") .execute().actionGet(); diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptTests.java index 5b0e88c959041..a7cf8983c7014 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptTests.java @@ -27,6 +27,8 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.sort.SortBuilders; import org.junit.Test; @@ -46,7 +48,7 @@ public void testPseudoRandomScript() throws Exception { assertAcked(prepareCreate("test") .addMapping("type", mapping)); - List indexBuilders = new ArrayList(); + List indexBuilders = new ArrayList<>(); // Index 100 records (0..99) for (int i = 0; i < 100; i++) { @@ -64,7 +66,7 @@ public void testPseudoRandomScript() throws Exception { .setQuery(matchAllQuery()) .addField("name") .setSize(10) - .addSort(SortBuilders.scriptSort("random", "number").lang("native").setParams(MapBuilder.newMapBuilder().put("salt", "1234").map())) + .addSort(SortBuilders.scriptSort(new Script("random", ScriptService.ScriptType.INLINE, "native", MapBuilder.newMapBuilder().put("salt", "1234").map()), "number")) .execute().actionGet(); assertNoFailures(searchResponse); @@ -84,7 +86,7 @@ public void testPseudoRandomScript() throws Exception { .setQuery(matchAllQuery()) .addField("name") .setSize(10) - .addSort(SortBuilders.scriptSort("random", "number").lang("native").setParams(MapBuilder.newMapBuilder().put("salt", "1234").map())) + .addSort(SortBuilders.scriptSort(new Script("random", ScriptService.ScriptType.INLINE, "native", MapBuilder.newMapBuilder().put("salt", "1234").map()), "number")) .execute().actionGet(); assertNoFailures(searchResponse); @@ -99,7 +101,7 @@ public void testPseudoRandomScript() throws Exception { .setQuery(matchAllQuery()) .addField("name") .setSize(10) - .addSort(SortBuilders.scriptSort("random", "number").lang("native")) + .addSort(SortBuilders.scriptSort(new Script("random", ScriptService.ScriptType.INLINE, "native", null), "number")) .execute().actionGet(); assertNoFailures(searchResponse); diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/TermScoringScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/TermScoringScriptTests.java index 704c3174e24f8..f9d827e1b439b 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/TermScoringScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/TermScoringScriptTests.java @@ -14,6 +14,7 @@ package org.elasticsearch.examples.nativescript.script; +import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.scriptFunction; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; @@ -32,6 +33,7 @@ import org.elasticsearch.common.lucene.search.function.CombineFunction; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder; import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService; @@ -64,24 +66,22 @@ public void testTFIDF() throws Exception { initData(); // initialize parameters for script - Map params = new HashMap(); + Map params = new HashMap<>(); params.put("field", field); params.put("terms", searchTerms); // Retrieve records and see if they scored 0.0 SearchResponse searchResponse = client() .prepareSearch("test") - .setQuery( - QueryBuilders.functionScoreQuery() - .add(ScoreFunctionBuilders.scriptFunction(new Script(TFIDFScoreScript.SCRIPT_NAME, ScriptService.ScriptType.INLINE, "native", params))) - .boostMode(CombineFunction.REPLACE.getName())).setSize(numDocs).execute().actionGet(); + .setQuery(QueryBuilders.functionScoreQuery(QueryBuilders.matchAllQuery(), new FunctionScoreQueryBuilder.FilterFunctionBuilder[]{ + new FunctionScoreQueryBuilder.FilterFunctionBuilder(ScoreFunctionBuilders.scriptFunction(new Script(TFIDFScoreScript.SCRIPT_NAME, ScriptService.ScriptType.INLINE, "native", params)))}) + .boostMode(CombineFunction.REPLACE)).setSize(numDocs).execute().actionGet(); assertNoFailures(searchResponse); assertHitCount(searchResponse, numDocs); SearchHit[] hits = searchResponse.getHits().hits(); for (int i = 0; i < numDocs; i++) { assertThat(hits[i].getId(), equalTo(Integer.toString(numDocs - i - 1))); } - } @Test @@ -90,7 +90,7 @@ public void testCosineSimilarity() throws Exception { initData(); // initialize parameters for script - Map params = new HashMap(); + Map params = new HashMap<>(); params.put("field", field); params.put("terms", searchTerms); params.put("weights", weights); @@ -98,10 +98,9 @@ public void testCosineSimilarity() throws Exception { // Retrieve records and see if they scored 0.0 SearchResponse searchResponse = client() .prepareSearch("test") - .setQuery( - QueryBuilders.functionScoreQuery() - .add(ScoreFunctionBuilders.scriptFunction(new Script(CosineSimilarityScoreScript.SCRIPT_NAME, ScriptService.ScriptType.INLINE, "native", params))) - .boostMode(CombineFunction.REPLACE.getName())).setSize(numDocs).execute().actionGet(); + .setQuery(QueryBuilders.functionScoreQuery(QueryBuilders.matchAllQuery(), new FunctionScoreQueryBuilder.FilterFunctionBuilder[]{ + new FunctionScoreQueryBuilder.FilterFunctionBuilder(ScoreFunctionBuilders.scriptFunction(new Script(CosineSimilarityScoreScript.SCRIPT_NAME, ScriptService.ScriptType.INLINE, "native", params)))}) + .boostMode(CombineFunction.REPLACE)).setSize(numDocs).execute().actionGet(); assertNoFailures(searchResponse); assertHitCount(searchResponse, numDocs); SearchHit[] hits = searchResponse.getHits().hits(); @@ -116,17 +115,17 @@ public void testPhraseScorer() throws Exception { initData(); // initialize parameters for script - Map params = new HashMap(); + Map params = new HashMap<>(); params.put("field", field); params.put("terms", searchTerms); // Retrieve records and see if they scored 0.0 SearchResponse searchResponse = client() .prepareSearch("test") - .setQuery( - QueryBuilders.functionScoreQuery() - .add(ScoreFunctionBuilders.scriptFunction(new Script(PhraseScoreScript.SCRIPT_NAME, ScriptService.ScriptType.INLINE, "native", params))) - .boostMode(CombineFunction.REPLACE.getName())).setSize(numDocs).execute().actionGet(); + .setQuery(QueryBuilders.functionScoreQuery(QueryBuilders.matchAllQuery(), new FunctionScoreQueryBuilder.FilterFunctionBuilder[]{ + new FunctionScoreQueryBuilder.FilterFunctionBuilder(ScoreFunctionBuilders.scriptFunction( + new Script(PhraseScoreScript.SCRIPT_NAME, ScriptService.ScriptType.INLINE, "native", params)))}) + .boostMode(CombineFunction.REPLACE)).setSize(numDocs).execute().actionGet(); assertNoFailures(searchResponse); assertHitCount(searchResponse, numDocs); SearchHit[] hits = searchResponse.getHits().hits(); @@ -142,7 +141,7 @@ public void testLanguageModelScorer() throws Exception { initData(); // initialize parameters for script - Map params = new HashMap(); + Map params = new HashMap<>(); params.put("field", field); params.put("terms", searchTerms); params.put("word_count_field", wordCountField); @@ -151,10 +150,9 @@ public void testLanguageModelScorer() throws Exception { // Retrieve records and see if they scored 0.0 SearchResponse searchResponse = client() .prepareSearch("test") - .setQuery( - QueryBuilders.functionScoreQuery() - .add(ScoreFunctionBuilders.scriptFunction(new Script(LanguageModelScoreScript.SCRIPT_NAME, ScriptService.ScriptType.INLINE, "native", params))) - .boostMode(CombineFunction.REPLACE.getName())).setSize(numDocs).execute().actionGet(); + .setQuery(QueryBuilders.functionScoreQuery(QueryBuilders.matchAllQuery(), new FunctionScoreQueryBuilder.FilterFunctionBuilder[]{ + new FunctionScoreQueryBuilder.FilterFunctionBuilder(ScoreFunctionBuilders.scriptFunction(new Script(LanguageModelScoreScript.SCRIPT_NAME, ScriptService.ScriptType.INLINE, "native", params)))}) + .boostMode(CombineFunction.REPLACE)).setSize(numDocs).execute().actionGet(); assertNoFailures(searchResponse); assertHitCount(searchResponse, numDocs); SearchHit[] hits = searchResponse.getHits().hits(); @@ -171,7 +169,7 @@ private void initData() throws IOException, InterruptedException, ExecutionExcep .field("format", "doc_values").endObject().endObject().endObject().endObject().endObject().endObject().endObject().string(); assertAcked(prepareCreate("test").addMapping("type", mapping)); - List indexBuilders = new ArrayList(); + List indexBuilders = new ArrayList<>(); // Index numDocs records (0..99) for (int i = 0; i < numDocs; i++) { indexBuilders.add(client().prepareIndex("test", "type", Integer.toString(i)) From 375b7f7e3340681f68f5ebc001bd936ebc3eb33f Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Sat, 10 Oct 2015 17:33:56 -0400 Subject: [PATCH 60/69] Remove guava cache --- .../nativescript/script/LookupScript.java | 35 ++++++++----------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java index c18dbd44f6eb8..5d187f7cde74d 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java @@ -14,11 +14,11 @@ package org.elasticsearch.examples.nativescript.script; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.client.Client; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.cache.Cache; +import org.elasticsearch.common.cache.CacheBuilder; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; @@ -36,9 +36,7 @@ import java.util.Collections; import java.util.Map; -import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; /** */ @@ -71,12 +69,12 @@ public Factory(Node node, Settings settings) { // Setup lookup cache ByteSizeValue size = settings.getAsBytesSize("examples.nativescript.lookup.size", null); TimeValue expire = settings.getAsTime("expire", null); - CacheBuilder cacheBuilder = CacheBuilder.newBuilder(); + CacheBuilder, Map> cacheBuilder = CacheBuilder.builder(); if (size != null) { - cacheBuilder.maximumSize(size.bytes()); + cacheBuilder.setMaximumWeight(size.bytes()); } if (expire != null) { - cacheBuilder.expireAfterAccess(expire.nanos(), TimeUnit.NANOSECONDS); + cacheBuilder.setExpireAfterAccess(expire.nanos()); } cache = cacheBuilder.build(); } @@ -149,20 +147,17 @@ public Object run() { final String fieldValue = ((ScriptDocValues.Strings) docValue).getValue(); if (fieldValue != null) { try { - return cache.get(new Tuple(lookupIndex + "/" + lookupType, fieldValue), new Callable>() { - @Override - public Map call() throws Exception { - // This is not very efficient of doing this, but it demonstrates using injected client - // for record lookup - GetResponse response = client.prepareGet(lookupIndex, lookupType, fieldValue).setPreference("_local").execute().actionGet(); - if (logger.isTraceEnabled()) { - logger.trace("lookup [{}]/[{}]/[{}], found: [{}]", lookupIndex, lookupType, fieldValue, response.isExists()); - } - if (response.isExists()) { - return response.getSource(); - } - return EMPTY_MAP; + return cache.computeIfAbsent(new Tuple<>(lookupIndex + "/" + lookupType, fieldValue), key -> { + // This is not very efficient of doing this, but it demonstrates using injected client + // for record lookup + GetResponse response = client.prepareGet(lookupIndex, lookupType, fieldValue).setPreference("_local").execute().actionGet(); + if (logger.isTraceEnabled()) { + logger.trace("lookup [{}]/[{}]/[{}], found: [{}]", lookupIndex, lookupType, fieldValue, response.isExists()); } + if (response.isExists()) { + return response.getSource(); + } + return EMPTY_MAP; }); } catch (ExecutionException ex) { From db8281b7a08c244f8c8ef49a4223099d302394a0 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Fri, 16 Oct 2015 15:22:20 -0400 Subject: [PATCH 61/69] Switch from array to arraylist since the params are no longer re-parsed in the test --- .../nativescript/script/TermScoringScriptTests.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/TermScoringScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/TermScoringScriptTests.java index f9d827e1b439b..ae283b3c3fa4e 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/TermScoringScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/TermScoringScriptTests.java @@ -22,10 +22,7 @@ import static org.hamcrest.Matchers.equalTo; import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.ExecutionException; import org.elasticsearch.action.index.IndexRequestBuilder; @@ -47,11 +44,11 @@ */ public class TermScoringScriptTests extends AbstractSearchScriptTestCase { - final static String[] searchTerms = {"foo", "bar"}; + final static List searchTerms = Arrays.asList("foo", "bar"); final static String field = "field"; final static String wordCountField = field + ".word_count"; final static String placeholder = "placeholder"; - final static Double[] weights = {1.0, 1.0}; + final static List weights = Arrays.asList(1.0, 1.0); final static int numDocs = 100; @Test From 6ae94c7dcb05548716d231080344737daa8c763e Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Fri, 16 Oct 2015 15:24:06 -0400 Subject: [PATCH 62/69] Add a simple way to start elasticsearch with plugin installed from IDE --- config/elasticsearch.yml | 3 - example/lookup.sh | 14 +++-- example/popularity.sh | 20 ------- example/primes.sh | 5 +- .../ElasticsearchWithPluginLauncher.java | 60 +++++++++++++++++++ 5 files changed, 69 insertions(+), 33 deletions(-) create mode 100644 src/test/java/org/elasticsearch/examples/nativescript/ElasticsearchWithPluginLauncher.java diff --git a/config/elasticsearch.yml b/config/elasticsearch.yml index 263f6bdc6ec4e..63bc3986b9f8f 100644 --- a/config/elasticsearch.yml +++ b/config/elasticsearch.yml @@ -1,4 +1 @@ cluster.name: elasticsearch-native-script-tests -network.host: 127.0.0.1 -discovery.zen.ping.multicast.enabled: false -discovery.zen.ping.unicast.hosts: ["localhost"] diff --git a/example/lookup.sh b/example/lookup.sh index 8383ee9c88415..918f33bc7c534 100755 --- a/example/lookup.sh +++ b/example/lookup.sh @@ -64,12 +64,14 @@ curl -s -XGET "localhost:9200/test/city/_search?pretty=true" -d '{ "fields": ["city", "state"], "script_fields": { "state_info": { - "script": "lookup", - "lang": "native", - "params": { - "lookup_index": "test", - "lookup_type": "state", - "field": "state" + "script": { + "inline": "lookup", + "lang": "native", + "params": { + "lookup_index": "test", + "lookup_type": "state", + "field": "state" + } } } }, diff --git a/example/popularity.sh b/example/popularity.sh index 286431986a0ea..5faee142f6b79 100755 --- a/example/popularity.sh +++ b/example/popularity.sh @@ -44,24 +44,4 @@ curl -s "localhost:9200/test/type1/_search?pretty=true" -d '{ } } ' -echo -curl -s "localhost:9200/test/type1/_search?pretty=true" -d '{ - "query": { - "function_score" : { - "boost_mode": "replace", - "query": { - "match": { - "name": "foo" - } - }, - "script_score": { - "script": "_score * (1 + Math.log10(doc[field].value + 1))", - "params": { - "field": "number" - } - } - } - } -} -' echo \ No newline at end of file diff --git a/example/primes.sh b/example/primes.sh index daf8e6fed5e47..edfecc592c223 100755 --- a/example/primes.sh +++ b/example/primes.sh @@ -25,10 +25,7 @@ curl -s -XPOST "http://localhost:9200/test/_refresh" echo curl -s "localhost:9200/test/type1/_search?pretty=true" -d '{ "query": { - "filtered": { - "query": { - "match_all": {} - }, + "bool": { "filter": { "script": { "script": "is_prime", diff --git a/src/test/java/org/elasticsearch/examples/nativescript/ElasticsearchWithPluginLauncher.java b/src/test/java/org/elasticsearch/examples/nativescript/ElasticsearchWithPluginLauncher.java new file mode 100644 index 0000000000000..d01be4e14b639 --- /dev/null +++ b/src/test/java/org/elasticsearch/examples/nativescript/ElasticsearchWithPluginLauncher.java @@ -0,0 +1,60 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.elasticsearch.examples.nativescript; + +import org.elasticsearch.Version; +import org.elasticsearch.common.cli.Terminal; +import org.elasticsearch.common.logging.log4j.LogConfigurator; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.examples.nativescript.plugin.NativeScriptExamplesPlugin; +import org.elasticsearch.node.MockNode; +import org.elasticsearch.node.Node; +import org.elasticsearch.node.internal.InternalSettingsPreparer; + +import java.util.Collections; +import java.util.concurrent.CountDownLatch; + +/** + * Main class to easily run the the plugin from a IDE. + */ +public class ElasticsearchWithPluginLauncher { + public static void main(String[] args) throws Throwable { + System.setProperty("es.logger.prefix", ""); + Settings settings = Settings.builder() + .put("security.manager.enabled", "false") + .put("plugins.load_classpath_plugins", "false") + .put("path.home", System.getProperty("es.path.home", System.getProperty("user.dir"))) + .build(); + + // Setup logging using config/logging.yml + Environment environment = InternalSettingsPreparer.prepareEnvironment(settings, Terminal.DEFAULT); + Class.forName("org.apache.log4j.Logger"); + LogConfigurator.configure(environment.settings(), true); + + final CountDownLatch latch = new CountDownLatch(1); + final Node node = new MockNode(settings, Version.CURRENT, Collections.singletonList(NativeScriptExamplesPlugin.class)); + Runtime.getRuntime().addShutdownHook(new Thread() { + + @Override + public void run() { + node.close(); + latch.countDown(); + } + }); + node.start(); + latch.await(); + } +} From ff3c1aaace492858f147ade9435a7cf126caecda Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Fri, 16 Oct 2015 15:38:41 -0400 Subject: [PATCH 63/69] Get rid of ArrayList conversion --- .../script/CosineSimilarityScoreScript.java | 10 +++++----- .../nativescript/script/LanguageModelScoreScript.java | 6 +++--- .../examples/nativescript/script/TFIDFScoreScript.java | 7 ++++--- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/CosineSimilarityScoreScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/CosineSimilarityScoreScript.java index 889f2ffe526a1..643c0d39501e1 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/CosineSimilarityScoreScript.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/CosineSimilarityScoreScript.java @@ -15,7 +15,7 @@ package org.elasticsearch.examples.nativescript.script; import java.io.IOException; -import java.util.ArrayList; +import java.util.List; import java.util.Map; import org.elasticsearch.script.ScriptException; @@ -39,10 +39,10 @@ public class CosineSimilarityScoreScript extends AbstractSearchScript { // in constructor from parameters. String field = null; // terms that are used for scoring, must be unique - ArrayList terms = null; + List terms = null; // weights, in case we want to put emphasis on a specific term. In the most // simple case, 1.0 for every term. - ArrayList weights = null; + List weights = null; final static public String SCRIPT_NAME = "cosine_sim_script_score"; @@ -85,8 +85,8 @@ public boolean needsScores() { private CosineSimilarityScoreScript(Map params) throws ScriptException { params.entrySet(); // get the terms - terms = (ArrayList) params.get("terms"); - weights = (ArrayList) params.get("weights"); + terms = (List) params.get("terms"); + weights = (List) params.get("weights"); // get the field field = (String) params.get("field"); if (field == null || terms == null || weights == null) { diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/LanguageModelScoreScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/LanguageModelScoreScript.java index fd609368f6ee1..0ece06bd2909f 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/LanguageModelScoreScript.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/LanguageModelScoreScript.java @@ -15,7 +15,7 @@ package org.elasticsearch.examples.nativescript.script; import java.io.IOException; -import java.util.ArrayList; +import java.util.List; import java.util.Map; import org.apache.lucene.search.Scorer; @@ -43,7 +43,7 @@ public class LanguageModelScoreScript extends AbstractSearchScript { // http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/mapping-core-types.html) String docLengthField; // terms that are used for scoring - ArrayList terms; + List terms; // lambda parameter float lambda; @@ -85,7 +85,7 @@ public boolean needsScores() { private LanguageModelScoreScript(Map params) { params.entrySet(); // get the terms - terms = (ArrayList) params.get("terms"); + terms = (List) params.get("terms"); // get the field field = (String) params.get("field"); // get the field holding the document length diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/TFIDFScoreScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/TFIDFScoreScript.java index b1c14204f8af7..de16ffa37ccf7 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/TFIDFScoreScript.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/TFIDFScoreScript.java @@ -15,7 +15,8 @@ package org.elasticsearch.examples.nativescript.script; import java.io.IOException; -import java.util.ArrayList; +import java.util.List; +import java.util.List; import java.util.Map; import org.apache.lucene.search.Scorer; @@ -39,7 +40,7 @@ public class TFIDFScoreScript extends AbstractSearchScript { // in constructor from parameters. String field = null; // terms that are used for scoring - ArrayList terms = null; + List terms = null; final static public String SCRIPT_NAME = "tfidf_script_score"; @@ -82,7 +83,7 @@ public boolean needsScores() { private TFIDFScoreScript(Map params) { params.entrySet(); // get the terms - terms = (ArrayList) params.get("terms"); + terms = (List) params.get("terms"); // get the field field = (String) params.get("field"); if (field == null || terms == null) { From 5b22950a8d681aa726d47684c4d27687061d9253 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Fri, 16 Oct 2015 21:03:32 -0400 Subject: [PATCH 64/69] Get rid of ArrayList conversion in PhraseScoreScript --- .../examples/nativescript/script/PhraseScoreScript.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/PhraseScoreScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/PhraseScoreScript.java index 843530948dfa1..c821b83f26338 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/PhraseScoreScript.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/PhraseScoreScript.java @@ -14,11 +14,10 @@ package org.elasticsearch.examples.nativescript.script; -import java.util.ArrayList; +import java.util.List; import java.util.Iterator; import java.util.Map; -import org.apache.lucene.search.Scorer; import org.elasticsearch.common.Nullable; import org.elasticsearch.script.AbstractSearchScript; import org.elasticsearch.script.ExecutableScript; @@ -40,7 +39,7 @@ public class PhraseScoreScript extends AbstractSearchScript { // in constructor from parameters. String field = null; // terms that are used for scoring - ArrayList terms = null; + List terms = null; final static public String SCRIPT_NAME = "phrase_script_score"; @@ -83,7 +82,7 @@ public boolean needsScores() { private PhraseScoreScript(Map params) { params.entrySet(); // get the terms - terms = (ArrayList) params.get("terms"); + terms = (List) params.get("terms"); // get the field field = (String) params.get("field"); if (field == null || terms == null) { From c4187329262e599b231502f9ea3129a94bb99bd8 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Wed, 21 Oct 2015 17:43:33 -0400 Subject: [PATCH 65/69] Remove Test annotation --- .../nativescript/script/IsPrimeSearchScriptTests.java | 2 -- .../examples/nativescript/script/LookupScriptTests.java | 2 -- .../nativescript/script/PopularityScoreScriptTests.java | 2 -- .../examples/nativescript/script/RandomSortScriptTests.java | 3 +-- .../nativescript/script/ScriptedMetricAggsScriptTests.java | 2 -- .../nativescript/script/TermScoringScriptTests.java | 6 ------ 6 files changed, 1 insertion(+), 16 deletions(-) diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScriptTests.java index 492e3b1ec8841..0e4323d0fe2f3 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScriptTests.java @@ -20,7 +20,6 @@ import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.sort.SortOrder; -import org.junit.Test; import java.util.ArrayList; import java.util.HashMap; @@ -37,7 +36,6 @@ public class IsPrimeSearchScriptTests extends AbstractSearchScriptTestCase { public static int[] PRIMES_10 = new int[] { 2, 3, 5, 7, 11, 13, 17, 19, 23, 29 }; - @Test public void testIsPrimeScript() throws Exception { // Create a new index diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/LookupScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/LookupScriptTests.java index 9745e74f4b026..ba5a61b327549 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/LookupScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/LookupScriptTests.java @@ -30,14 +30,12 @@ import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.sort.SortOrder; -import org.junit.Test; /** */ public class LookupScriptTests extends AbstractSearchScriptTestCase { @SuppressWarnings("unchecked") - @Test public void testLookup() throws Exception { // Create a new lookup index diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptTests.java index bac839d0683ae..9c0d56e0b7bc7 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptTests.java @@ -36,13 +36,11 @@ import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService; -import org.junit.Test; /** */ public class PopularityScoreScriptTests extends AbstractSearchScriptTestCase { - @Test public void testPopularityScoring() throws Exception { // Create a new index diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptTests.java index a7cf8983c7014..982c288a1737e 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptTests.java @@ -30,12 +30,11 @@ import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.sort.SortBuilders; -import org.junit.Test; /** */ public class RandomSortScriptTests extends AbstractSearchScriptTestCase { - @Test + public void testPseudoRandomScript() throws Exception { // Create a new index diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/ScriptedMetricAggsScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/ScriptedMetricAggsScriptTests.java index 01fffbc144854..04e12ffd852de 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/ScriptedMetricAggsScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/ScriptedMetricAggsScriptTests.java @@ -19,7 +19,6 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService; -import org.junit.Test; import java.util.ArrayList; import java.util.List; @@ -34,7 +33,6 @@ public class ScriptedMetricAggsScriptTests extends AbstractSearchScriptTestCase { @SuppressWarnings("unchecked") - @Test public void testScriptedMetricAggs() throws Exception { // Create a new lookup index diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/TermScoringScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/TermScoringScriptTests.java index ae283b3c3fa4e..dc9c960c1e102 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/TermScoringScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/TermScoringScriptTests.java @@ -35,7 +35,6 @@ import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.SearchHit; -import org.junit.Test; /** * Test if the computed tfidf in NaiveTFIDFScoreScript equals 0.0 for each @@ -51,13 +50,11 @@ public class TermScoringScriptTests extends AbstractSearchScriptTestCase { final static List weights = Arrays.asList(1.0, 1.0); final static int numDocs = 100; - @Test public void testNoOfShardsIs1() { assertAcked(prepareCreate("test").get()); assertThat(client().admin().indices().prepareGetSettings("test").get().getSetting("test", "index.number_of_shards"), equalTo("1")); } - @Test public void testTFIDF() throws Exception { initData(); @@ -81,7 +78,6 @@ public void testTFIDF() throws Exception { } } - @Test public void testCosineSimilarity() throws Exception { initData(); @@ -106,7 +102,6 @@ public void testCosineSimilarity() throws Exception { } } - @Test public void testPhraseScorer() throws Exception { initData(); @@ -132,7 +127,6 @@ public void testPhraseScorer() throws Exception { } } - @Test public void testLanguageModelScorer() throws Exception { initData(); From 8aa2eb4cdd0cb22f9d95e99e8dc14adca16b35ad Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Mon, 4 Jan 2016 22:38:15 -0500 Subject: [PATCH 66/69] Move to 3.0.0-SNAPSHOT --- .editorconfig | 10 + .gitignore | 3 + README.textile | 31 ++- build.gradle | 42 ++++ .../license_header_definition.xml | 13 -- .../native_script_example_license_header.txt | 11 - gradle.properties | 1 + licenses/commons-math3-3.5.jar.sha1 | 1 + licenses/commons-math3-LICENSE.txt | 202 ++++++++++++++++++ licenses/commons-math3-NOTICE.txt | 9 + licenses/no_deps.txt | 1 - pom.xml | 70 ------ .../nativescript/script/LookupScript.java | 1 + .../script/RandomSortScriptFactory.java | 3 +- .../ElasticsearchWithPluginLauncher.java | 8 +- .../script/IsPrimeSearchScriptTests.java | 60 +++--- .../script/LookupScriptTests.java | 63 +++--- .../script/PopularityScoreScriptTests.java | 4 +- .../script/RandomSortScriptTests.java | 66 +++--- .../script/ScriptedMetricAggsScriptTests.java | 4 +- .../script/TermScoringScriptTests.java | 73 ++++--- 21 files changed, 430 insertions(+), 246 deletions(-) create mode 100644 .editorconfig create mode 100644 build.gradle delete mode 100644 dev-tools/src/main/resources/license-check/license_header_definition.xml delete mode 100644 dev-tools/src/main/resources/license-check/native_script_example_license_header.txt create mode 100644 gradle.properties create mode 100644 licenses/commons-math3-3.5.jar.sha1 create mode 100644 licenses/commons-math3-LICENSE.txt create mode 100644 licenses/commons-math3-NOTICE.txt delete mode 100644 licenses/no_deps.txt delete mode 100644 pom.xml diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000000000..9d4bfbf55d3a1 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,10 @@ +# EditorConfig: http://editorconfig.org/ + +root = true + +[*.java] +charset = utf-8 +indent_style = space +indent_size = 4 +trim_trailing_whitespace = true +insert_final_newline = true diff --git a/.gitignore b/.gitignore index 7c6339c28a5fc..aadcbf9b02ce3 100644 --- a/.gitignore +++ b/.gitignore @@ -4,8 +4,11 @@ data/ logs/ build/ target/ +generated-resources/ .DS_Store *-execution-hints.log +*-execution-times.log +.gradle/ # IDEA ignores .idea/ diff --git a/README.textile b/README.textile index d054151693c7a..56f71d0376cda 100644 --- a/README.textile +++ b/README.textile @@ -7,16 +7,17 @@ p. This plugin contains several examples of "native script":http://www.elastic.c p. Please make sure to use the correct branch of this repository that corresponds to the version of elasticsearch that you are developing the plugin for. |_. Example Plugin Branch |_. Elasticsearch | -| "master":https://github.com/imotov/elasticsearch-native-script-example | 2.1.0 and above | +| "master":https://github.com/imotov/elasticsearch-native-script-example | master | +| "2.x":https://github.com/imotov/elasticsearch-native-script-example/tree/2.x | 2.x.x | | "2.0":https://github.com/imotov/elasticsearch-native-script-example/tree/2.0 | 2.0.x | | "1.x":https://github.com/imotov/elasticsearch-native-script-example/tree/1.x | 1.x.x | h2. Creating Elasticsearch Plugin -p. The simplest way to deploy native script is by wrapping it into standard Elasticsearch plugin infrastructure. An Elasticsearch plugin can be written in java and built using maven. A typical plugin source directory looks like this: +p. The simplest way to deploy native script is by wrapping it into standard Elasticsearch plugin infrastructure. An Elasticsearch plugin can be written in java and built using gradle. A typical plugin source directory looks like this: bc.. . -|- pom.xml +|- build.gradle |- src |- main | |- java @@ -30,25 +31,17 @@ bc.. . p. An Elasticsearch plugin can be created by following these six steps. -* Create pom.xml file in the root directory of your plugin. The "pom.xml":https://github.com/imotov/elasticsearch-native-script-example/blob/master/pom.xml file in this project can be used as a starting point. +* Create build.gradle file in the root directory of your plugin. The "pom.xml":https://github.com/imotov/elasticsearch-native-script-example/blob/master/pom.xml file in this project can be used as a starting point. * Create source code directories: ** @mkdir -p src/main/java@ ** @mkdir -p src/test/java@ ** @mkdir -p src/test/resources@ -* The parent project @org.elasticsearch.plugin:plugins@ provides all needed tasks to assemble the plugin. It is using the following properties that should be modified to match the project's plugin class name and license file definition. - -bc.. - - org.elasticsearch.examples.nativescript.plugin.NativeScriptExamplesPlugin - - - ${project.basedir}/dev-tools/src/main/resources/license-check/native_script_example_license_header.txt - ${project.basedir}/dev-tools/src/main/resources/license-check/license_header_definition.xml - - ... other properties ...... - - +* The gradle plugin @esplugin@ provides all needed tasks to assemble the plugin. It is using the following settings that should be modified to match the project's plugin class name and license file definition. +bc.. esplugin { + description 'ElasticSearch Plugin with Native Script Examples.' + classname 'org.elasticsearch.examples.nativescript.plugin.NativeScriptExamplesPlugin' + } p. @@ -73,14 +66,14 @@ public class NativeScriptExamplesPlugin extends Plugin { p. -* The parent project will automatically create @plugin-descriptor.properties@ for you. +* The gradle plugin will automatically create @plugin-descriptor.properties@ for you. * If you are not using the standard parent project, you can create this file manually by using this "plugin-descriptor.properties":https://github.com/elastic/elasticsearch/blob/master/dev-tools/src/main/resources/plugin-metadata/plugin-descriptor.properties as a template. You will also need to package the plugin into .zip file, which can be done using maven assemble task and "plugin-assembly.xml":https://github.com/elastic/elasticsearch/blob/master/dev-tools/src/main/resources/plugin-metadata/plugin-assembly.xml assembly definition. * The plugin can be built using @mvn package@ command. The assembled .zip package can be found in the @target/releases/@ directory and deployed to elasticsearch installation using @plugin -install plugin-name -url path/to/plugin/zip/file@. h2. Migration from 1.x -p. The plugin infrastructure significantly changed in 2.0. So, the plugin project will need to be modified in order to be used with elasticsearch 2.0: +p. The plugin infrastructure significantly changed in 2.0 and above. So, the plugin project will need to be modified in order to be used with elasticsearch 2.0: * Instead of using @es-plugin.properties@ file that in 1.x was places in the plugin jar, the plugin infrastructure is now using the @plugin-descriptor.properties@ file that describes not only the main plugin class, version and description but also plugin type (_site and/or jvm), required minimal java and Elasticsearch versions. The @plugin-descriptor.properties@ file should be placed into root directory of the .zip file that the plugin is packaged into. The simplest way to deal with this change is by switching the plugin project to @org.elasticsearch.plugin:plugins@ as a parent. * Elasticsearch 2.0 is also stricter when it comes to plugin classes. For example, the "jar hell" prevention mechanism will not allow the plugin to contain classes that are already defined in the Elasticsearch classpath. Make sure that your project doesn't have any dependencies that are conflicting with existing elasticsearch classes. diff --git a/build.gradle b/build.gradle new file mode 100644 index 0000000000000..2d23579dede0d --- /dev/null +++ b/build.gradle @@ -0,0 +1,42 @@ +buildscript { + repositories { + mavenCentral() + maven { + name 'sonatype-snapshots' + url "https://oss.sonatype.org/content/repositories/snapshots/" + } + jcenter() + } + dependencies { + // Version of elasticsearch + classpath "org.elasticsearch.gradle:build-tools:3.0.0-SNAPSHOT" + } +} + +apply plugin: 'idea' +apply plugin: 'eclipse' +apply plugin: 'elasticsearch.esplugin' + +// Version of the plugin +version = '3.0.0-SNAPSHOT' + +esplugin { + description 'ElasticSearch Plugin with Native Script Examples.' + classname 'org.elasticsearch.examples.nativescript.plugin.NativeScriptExamplesPlugin' +} + +dependencies { + // Not really used at the moment - just to show how to include dependencies + compile "org.apache.commons:commons-math3:3.5" +} + +compileJava.options.compilerArgs << "-Xlint:-deprecation" + +// uncomment to disable 3rd party audit - TODO: temporary workaround until it's fixed in elasticsearch +thirdPartyAudit.enabled = false + +// uncomment to disable forbidden patterns checks +// forbiddenPatterns.enabled = false + +// uncomment to disable license headers checks +// licenseHeaders.enabled = false diff --git a/dev-tools/src/main/resources/license-check/license_header_definition.xml b/dev-tools/src/main/resources/license-check/license_header_definition.xml deleted file mode 100644 index 1cc53bda0018a..0000000000000 --- a/dev-tools/src/main/resources/license-check/license_header_definition.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - /* - * - */ - - (\s|\t)*/\*.*$ - .*\*/(\s|\t)*$ - false - true - - diff --git a/dev-tools/src/main/resources/license-check/native_script_example_license_header.txt b/dev-tools/src/main/resources/license-check/native_script_example_license_header.txt deleted file mode 100644 index 86a8beecb2733..0000000000000 --- a/dev-tools/src/main/resources/license-check/native_script_example_license_header.txt +++ /dev/null @@ -1,11 +0,0 @@ -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. \ No newline at end of file diff --git a/gradle.properties b/gradle.properties new file mode 100644 index 0000000000000..6b1823d86a69a --- /dev/null +++ b/gradle.properties @@ -0,0 +1 @@ +org.gradle.daemon=false diff --git a/licenses/commons-math3-3.5.jar.sha1 b/licenses/commons-math3-3.5.jar.sha1 new file mode 100644 index 0000000000000..c3c6cfc20e61f --- /dev/null +++ b/licenses/commons-math3-3.5.jar.sha1 @@ -0,0 +1 @@ +13af68e9a11576953f5c4b09436f8207be9d3a27 \ No newline at end of file diff --git a/licenses/commons-math3-LICENSE.txt b/licenses/commons-math3-LICENSE.txt new file mode 100644 index 0000000000000..d645695673349 --- /dev/null +++ b/licenses/commons-math3-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/licenses/commons-math3-NOTICE.txt b/licenses/commons-math3-NOTICE.txt new file mode 100644 index 0000000000000..ce791e4f32652 --- /dev/null +++ b/licenses/commons-math3-NOTICE.txt @@ -0,0 +1,9 @@ +Apache Commons Math +Copyright 2001-2015 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This product includes software developed for Orekit by +CS Systèmes d'Information (http://www.c-s.fr/) +Copyright 2010-2012 CS Systèmes d'Information diff --git a/licenses/no_deps.txt b/licenses/no_deps.txt deleted file mode 100644 index cfffc5f993b1c..0000000000000 --- a/licenses/no_deps.txt +++ /dev/null @@ -1 +0,0 @@ -This plugin has no third party dependencies \ No newline at end of file diff --git a/pom.xml b/pom.xml deleted file mode 100644 index 717d07559bbc1..0000000000000 --- a/pom.xml +++ /dev/null @@ -1,70 +0,0 @@ - - - 4.0.0 - - - org.elasticsearch.plugin - plugins - 3.0.0-SNAPSHOT - - - elasticsearch-native-script-example - org.elasticsearch.plugin - elasticsearch-native-script-example - 3.0.0-SNAPSHOT - ElasticSearch Plugin with Native Script Examples - 2013 - - - - The Apache Software License, Version 2.0 - http://www.apache.org/licenses/LICENSE-2.0.txt - repo - - - - - scm:git:git@github.com:imotov/elasticsearch-native-script-example.git - scm:git:git@github.com:imotov/elasticsearch-native-script-example.git - http://github.com/imotov/elasticsearch-native-script-example - - - - - - - - - - org.elasticsearch.examples.nativescript.plugin.NativeScriptExamplesPlugin - - - ${project.basedir}/dev-tools/src/main/resources/license-check/native_script_example_license_header.txt - ${project.basedir}/dev-tools/src/main/resources/license-check/license_header_definition.xml - - - warn - nativescript - false - - - - - - oss-snapshots - Sonatype OSS Snapshots - https://oss.sonatype.org/content/repositories/snapshots/ - - - - - - - org.apache.maven.plugins - maven-assembly-plugin - - - - diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java b/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java index 5d187f7cde74d..6549678e2a1ff 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java @@ -57,6 +57,7 @@ public static class Factory extends AbstractComponent implements NativeScriptFac * This constructor will be called by guice during initialization * * @param node injecting the reference to current node to get access to node's client + * @param settings current node settings */ @SuppressWarnings("unchecked") @Inject diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java b/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java index f3a754810c6f1..9f2c628e9c33a 100644 --- a/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java +++ b/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java @@ -15,6 +15,7 @@ package org.elasticsearch.examples.nativescript.script; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Randomness; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.mapper.internal.UidFieldMapper; @@ -68,7 +69,7 @@ private static class RandomSortScript extends AbstractLongSearchScript { private final Random random; private RandomSortScript() { - random = new Random(); + random = Randomness.get(); } @Override diff --git a/src/test/java/org/elasticsearch/examples/nativescript/ElasticsearchWithPluginLauncher.java b/src/test/java/org/elasticsearch/examples/nativescript/ElasticsearchWithPluginLauncher.java index d01be4e14b639..711412f10bb0f 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/ElasticsearchWithPluginLauncher.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/ElasticsearchWithPluginLauncher.java @@ -15,6 +15,7 @@ package org.elasticsearch.examples.nativescript; import org.elasticsearch.Version; +import org.elasticsearch.common.SuppressForbidden; import org.elasticsearch.common.cli.Terminal; import org.elasticsearch.common.logging.log4j.LogConfigurator; import org.elasticsearch.common.settings.Settings; @@ -23,7 +24,10 @@ import org.elasticsearch.node.MockNode; import org.elasticsearch.node.Node; import org.elasticsearch.node.internal.InternalSettingsPreparer; +import org.elasticsearch.plugins.Plugin; +import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.concurrent.CountDownLatch; @@ -31,6 +35,8 @@ * Main class to easily run the the plugin from a IDE. */ public class ElasticsearchWithPluginLauncher { + + @SuppressForbidden(reason = "not really code or a test") public static void main(String[] args) throws Throwable { System.setProperty("es.logger.prefix", ""); Settings settings = Settings.builder() @@ -45,7 +51,7 @@ public static void main(String[] args) throws Throwable { LogConfigurator.configure(environment.settings(), true); final CountDownLatch latch = new CountDownLatch(1); - final Node node = new MockNode(settings, Version.CURRENT, Collections.singletonList(NativeScriptExamplesPlugin.class)); + final Node node = new MockNode(settings, Version.CURRENT, Arrays.asList(NativeScriptExamplesPlugin.class)); Runtime.getRuntime().addShutdownHook(new Thread() { @Override diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScriptTests.java index 0e4323d0fe2f3..4b899eaa31564 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScriptTests.java @@ -27,45 +27,47 @@ import java.util.Map; import static org.elasticsearch.index.query.QueryBuilders.scriptQuery; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; import static org.hamcrest.Matchers.equalTo; /** */ public class IsPrimeSearchScriptTests extends AbstractSearchScriptTestCase { - public static int[] PRIMES_10 = new int[] { 2, 3, 5, 7, 11, 13, 17, 19, 23, 29 }; + public static int[] PRIMES_10 = new int[]{2, 3, 5, 7, 11, 13, 17, 19, 23, 29}; public void testIsPrimeScript() throws Exception { - + // Create a new index String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("properties") - .startObject("name").field("type", "string").endObject() - .startObject("number").field("type", "integer").endObject() - .endObject().endObject().endObject() - .string(); - + .startObject("properties") + .startObject("name").field("type", "string").endObject() + .startObject("number").field("type", "integer").endObject() + .endObject().endObject().endObject() + .string(); + assertAcked(prepareCreate("test") - .addMapping("type", mapping)); + .addMapping("type", mapping)); List indexBuilders = new ArrayList(); // Index 100 records (0..99) for (int i = 0; i < 100; i++) { indexBuilders.add( - client().prepareIndex("test", "type", Integer.toString(i)) + client().prepareIndex("test", "type", Integer.toString(i)) .setSource(XContentFactory.jsonBuilder().startObject() - .field("name", "rec " + i) - .field("number", i) - .endObject())); + .field("name", "rec " + i) + .field("number", i) + .endObject())); } // Index a few records with empty number for (int i = 100; i < 105; i++) { indexBuilders.add( - client().prepareIndex("test", "type", Integer.toString(i)) + client().prepareIndex("test", "type", Integer.toString(i)) .setSource(XContentFactory.jsonBuilder().startObject() - .field("name", "rec " + i) - .endObject())); + .field("name", "rec " + i) + .endObject())); } indexRandom(true, indexBuilders); @@ -74,12 +76,12 @@ public void testIsPrimeScript() throws Exception { params.put("field", "number"); // Retrieve first 10 prime records SearchResponse searchResponse = client().prepareSearch("test") - .setQuery(scriptQuery(new Script("is_prime", ScriptService.ScriptType.INLINE, "native", params))) - .addField("name") - .setSize(10) - .addSort("number", SortOrder.ASC) - .execute().actionGet(); - + .setQuery(scriptQuery(new Script("is_prime", ScriptService.ScriptType.INLINE, "native", params))) + .setFetchSource("name", null) + .setSize(10) + .addSort("number", SortOrder.ASC) + .execute().actionGet(); + assertNoFailures(searchResponse); // There should be 25 prime numbers between 0 and 100 @@ -87,7 +89,7 @@ public void testIsPrimeScript() throws Exception { // Verify that they are indeed prime numbers for (int i = 0; i < 10; i++) { - assertThat(searchResponse.getHits().getAt(i).field("name").getValue().toString(), equalTo("rec " + PRIMES_10[i])); + assertThat(searchResponse.getHits().getAt(i).getSource().get("name"), equalTo("rec " + PRIMES_10[i])); } params = new HashMap<>(); @@ -95,11 +97,11 @@ public void testIsPrimeScript() throws Exception { params.put("certainty", 0); // Check certainty parameter - with certainty == 0, it should return all numbers, but only if numbers are present searchResponse = client().prepareSearch("test") - .setQuery(scriptQuery(new Script("is_prime", ScriptService.ScriptType.INLINE, "native", params))) - .addField("name") - .setSize(10) - .addSort("number", SortOrder.ASC) - .execute().actionGet(); + .setQuery(scriptQuery(new Script("is_prime", ScriptService.ScriptType.INLINE, "native", params))) + .setFetchSource("name", null) + .setSize(10) + .addSort("number", SortOrder.ASC) + .execute().actionGet(); assertNoFailures(searchResponse); // With certainty 0 no check is done so it should return all numbers assertHitCount(searchResponse, 100); diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/LookupScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/LookupScriptTests.java index ba5a61b327549..2ca31c11071cf 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/LookupScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/LookupScriptTests.java @@ -13,6 +13,7 @@ */ package org.elasticsearch.examples.nativescript.script; + import static org.elasticsearch.index.query.QueryBuilders.matchQuery; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; @@ -40,27 +41,27 @@ public void testLookup() throws Exception { // Create a new lookup index String lookup_mapping = XContentFactory.jsonBuilder().startObject().startObject("state") - .startObject("properties") - .startObject("name").field("type", "string").endObject() - .startObject("capital").field("type", "string").endObject() - .startObject("nickname").field("type", "string").endObject() - .endObject().endObject().endObject() - .string(); - + .startObject("properties") + .startObject("name").field("type", "string").endObject() + .startObject("capital").field("type", "string").endObject() + .startObject("nickname").field("type", "string").endObject() + .endObject().endObject().endObject() + .string(); + assertAcked(prepareCreate("lookup") - .addMapping("state", lookup_mapping)); + .addMapping("state", lookup_mapping)); // Create a new test index String test_mapping = XContentFactory.jsonBuilder().startObject().startObject("city") - .startObject("properties") - .startObject("city").field("type", "string").endObject() - .startObject("state").field("type", "string").field("index", "not_analyzed").endObject() - .startObject("population").field("type", "integer").endObject() - .endObject().endObject().endObject() - .string(); - + .startObject("properties") + .startObject("city").field("type", "string").endObject() + .startObject("state").field("type", "string").field("index", "not_analyzed").endObject() + .startObject("population").field("type", "integer").endObject() + .endObject().endObject().endObject() + .string(); + assertAcked(prepareCreate("test") - .addMapping("city", test_mapping)); + .addMapping("city", test_mapping)); List indexBuilders = new ArrayList(); // Index Lookup records: @@ -87,34 +88,34 @@ public void testLookup() throws Exception { // Script parameters Map params = MapBuilder.newMapBuilder() - .put("lookup_index", "lookup") - .put("lookup_type", "state") - .put("field", "state") - .map(); + .put("lookup_index", "lookup") + .put("lookup_type", "state") + .put("field", "state") + .map(); // Find smallest city with word SearchResponse searchResponse = client().prepareSearch("test") - .setTypes("city") - .setQuery(matchQuery("city", "south burlington")) - .addField("city") - .addScriptField("state_info", new Script("lookup", ScriptService.ScriptType.INLINE, "native", params)) - .setSize(10) - .addSort("population", SortOrder.DESC) - .execute().actionGet(); - + .setTypes("city") + .setQuery(matchQuery("city", "south burlington")) + .setFetchSource(true) + .addScriptField("state_info", new Script("lookup", ScriptService.ScriptType.INLINE, "native", params)) + .setSize(10) + .addSort("population", SortOrder.DESC) + .execute().actionGet(); + assertNoFailures(searchResponse); // There should be 3 cities assertHitCount(searchResponse, 3); - assertThat(searchResponse.getHits().getAt(0).field("city").getValue().toString(), equalTo("Burlington")); + assertThat(searchResponse.getHits().getAt(0).getSource().get("city"), equalTo("Burlington")); assertThat(((Map) searchResponse.getHits().getAt(0).field("state_info").getValue()).get("name").toString(), equalTo("Vermont")); - assertThat(searchResponse.getHits().getAt(1).field("city").getValue().toString(), equalTo("South Portland")); + assertThat(searchResponse.getHits().getAt(1).getSource().get("city"), equalTo("South Portland")); assertThat(((Map) searchResponse.getHits().getAt(1).field("state_info").getValue()).get("name").toString(), equalTo("Maine")); - assertThat(searchResponse.getHits().getAt(2).field("city").getValue().toString(), equalTo("South Burlington")); + assertThat(searchResponse.getHits().getAt(2).getSource().get("city"), equalTo("South Burlington")); assertThat(((Map) searchResponse.getHits().getAt(2).field("state_info").getValue()).get("name").toString(), equalTo("Vermont")); } diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptTests.java index 9c0d56e0b7bc7..5926e5b2e46ba 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptTests.java @@ -83,7 +83,7 @@ public void testPopularityScoring() throws Exception { new FunctionScoreQueryBuilder.FilterFunctionBuilder(ScoreFunctionBuilders.scriptFunction(new Script("popularity", ScriptService.ScriptType.INLINE, "native", params)))}) .boostMode(CombineFunction.REPLACE)) .setSize(10) - .addField("name") + .setFetchSource("name", null) .execute().actionGet(); assertNoFailures(searchResponse); @@ -93,7 +93,7 @@ public void testPopularityScoring() throws Exception { // Verify that first 5 hits are sorted from 4 to 0 for (int i = 0; i < 5; i++) { - assertThat(searchResponse.getHits().getAt(i).field("name").getValue().toString(), equalTo("rec " + (4 - i))); + assertThat(searchResponse.getHits().getAt(i).sourceAsMap().get("name"), equalTo("rec " + (4 - i))); } // Verify that hit 5 has non-zero score diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptTests.java index 982c288a1737e..8ff0b2ce7cfec 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptTests.java @@ -36,79 +36,79 @@ public class RandomSortScriptTests extends AbstractSearchScriptTestCase { public void testPseudoRandomScript() throws Exception { - + // Create a new index String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("properties") - .startObject("name").field("type", "string").endObject() - .endObject().endObject().endObject() - .string(); - + .startObject("properties") + .startObject("name").field("type", "string").endObject() + .endObject().endObject().endObject() + .string(); + assertAcked(prepareCreate("test") - .addMapping("type", mapping)); + .addMapping("type", mapping)); List indexBuilders = new ArrayList<>(); - + // Index 100 records (0..99) for (int i = 0; i < 100; i++) { indexBuilders.add( - client().prepareIndex("test", "type", Integer.toString(i)) + client().prepareIndex("test", "type", Integer.toString(i)) .setSource(XContentFactory.jsonBuilder().startObject() - .field("name", "rec " + i) - .endObject())); + .field("name", "rec " + i) + .endObject())); } indexRandom(true, indexBuilders); // Retrieve first 10 records SearchResponse searchResponse = client().prepareSearch("test") - .setQuery(matchAllQuery()) - .addField("name") - .setSize(10) - .addSort(SortBuilders.scriptSort(new Script("random", ScriptService.ScriptType.INLINE, "native", MapBuilder.newMapBuilder().put("salt", "1234").map()), "number")) - .execute().actionGet(); - + .setQuery(matchAllQuery()) + .setFetchSource("name", null) + .setSize(10) + .addSort(SortBuilders.scriptSort(new Script("random", ScriptService.ScriptType.INLINE, "native", MapBuilder.newMapBuilder().put("salt", "1234").map()), "number")) + .execute().actionGet(); + assertNoFailures(searchResponse); // Check that random order was applied - assertThat(searchResponse.getHits().getAt(0).field("name").getValue().toString(), not(equalTo("rec0"))); + assertThat(searchResponse.getHits().getAt(0).getSource().get("name"), not(equalTo("rec0"))); String[] records = new String[10]; // Store sort order for (int i = 0; i < 10; i++) { - records[i] = searchResponse.getHits().getAt(i).field("name").getValue().toString(); + records[i] = searchResponse.getHits().getAt(i).getSource().get("name").toString(); } // Retrieve first 10 records again searchResponse = client().prepareSearch("test") - .setQuery(matchAllQuery()) - .addField("name") - .setSize(10) - .addSort(SortBuilders.scriptSort(new Script("random", ScriptService.ScriptType.INLINE, "native", MapBuilder.newMapBuilder().put("salt", "1234").map()), "number")) - .execute().actionGet(); - + .setQuery(matchAllQuery()) + .setFetchSource("name", null) + .setSize(10) + .addSort(SortBuilders.scriptSort(new Script("random", ScriptService.ScriptType.INLINE, "native", MapBuilder.newMapBuilder().put("salt", "1234").map()), "number")) + .execute().actionGet(); + assertNoFailures(searchResponse); // Verify the same sort order for (int i = 0; i < 10; i++) { - assertThat(searchResponse.getHits().getAt(i).field("name").getValue().toString(), equalTo(records[i])); + assertThat(searchResponse.getHits().getAt(i).getSource().get("name"), equalTo(records[i])); } // Retrieve first 10 records without salt searchResponse = client().prepareSearch("test") - .setQuery(matchAllQuery()) - .addField("name") - .setSize(10) - .addSort(SortBuilders.scriptSort(new Script("random", ScriptService.ScriptType.INLINE, "native", null), "number")) - .execute().actionGet(); - + .setQuery(matchAllQuery()) + .setFetchSource("name", null) + .setSize(10) + .addSort(SortBuilders.scriptSort(new Script("random", ScriptService.ScriptType.INLINE, "native", null), "number")) + .execute().actionGet(); + assertNoFailures(searchResponse); // Verify different sort order boolean different = false; for (int i = 0; i < 10; i++) { - if (!records[i].equals(searchResponse.getHits().getAt(i).field("name").getValue().toString())) { + if (!records[i].equals(searchResponse.getHits().getAt(i).getSource().get("name"))) { different = true; break; } diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/ScriptedMetricAggsScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/ScriptedMetricAggsScriptTests.java index 04e12ffd852de..9905c69abbe6a 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/ScriptedMetricAggsScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/ScriptedMetricAggsScriptTests.java @@ -25,7 +25,9 @@ import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.search.aggregations.AggregationBuilders.scriptedMetric; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; import static org.hamcrest.Matchers.equalTo; /** diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/TermScoringScriptTests.java b/src/test/java/org/elasticsearch/examples/nativescript/script/TermScoringScriptTests.java index dc9c960c1e102..c0192862b3dd9 100644 --- a/src/test/java/org/elasticsearch/examples/nativescript/script/TermScoringScriptTests.java +++ b/src/test/java/org/elasticsearch/examples/nativescript/script/TermScoringScriptTests.java @@ -14,17 +14,6 @@ package org.elasticsearch.examples.nativescript.script; -import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.scriptFunction; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; -import static org.hamcrest.Matchers.closeTo; -import static org.hamcrest.Matchers.equalTo; - -import java.io.IOException; -import java.util.*; -import java.util.concurrent.ExecutionException; - import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.lucene.search.function.CombineFunction; @@ -36,6 +25,20 @@ import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.SearchHit; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; +import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.equalTo; + /** * Test if the computed tfidf in NaiveTFIDFScoreScript equals 0.0 for each * document as would be expected if each document in the index contains only one @@ -66,10 +69,10 @@ public void testTFIDF() throws Exception { // Retrieve records and see if they scored 0.0 SearchResponse searchResponse = client() - .prepareSearch("test") - .setQuery(QueryBuilders.functionScoreQuery(QueryBuilders.matchAllQuery(), new FunctionScoreQueryBuilder.FilterFunctionBuilder[]{ - new FunctionScoreQueryBuilder.FilterFunctionBuilder(ScoreFunctionBuilders.scriptFunction(new Script(TFIDFScoreScript.SCRIPT_NAME, ScriptService.ScriptType.INLINE, "native", params)))}) - .boostMode(CombineFunction.REPLACE)).setSize(numDocs).execute().actionGet(); + .prepareSearch("test") + .setQuery(QueryBuilders.functionScoreQuery(QueryBuilders.matchAllQuery(), new FunctionScoreQueryBuilder.FilterFunctionBuilder[]{ + new FunctionScoreQueryBuilder.FilterFunctionBuilder(ScoreFunctionBuilders.scriptFunction(new Script(TFIDFScoreScript.SCRIPT_NAME, ScriptService.ScriptType.INLINE, "native", params)))}) + .boostMode(CombineFunction.REPLACE)).setSize(numDocs).execute().actionGet(); assertNoFailures(searchResponse); assertHitCount(searchResponse, numDocs); SearchHit[] hits = searchResponse.getHits().hits(); @@ -90,10 +93,10 @@ public void testCosineSimilarity() throws Exception { // Retrieve records and see if they scored 0.0 SearchResponse searchResponse = client() - .prepareSearch("test") - .setQuery(QueryBuilders.functionScoreQuery(QueryBuilders.matchAllQuery(), new FunctionScoreQueryBuilder.FilterFunctionBuilder[]{ - new FunctionScoreQueryBuilder.FilterFunctionBuilder(ScoreFunctionBuilders.scriptFunction(new Script(CosineSimilarityScoreScript.SCRIPT_NAME, ScriptService.ScriptType.INLINE, "native", params)))}) - .boostMode(CombineFunction.REPLACE)).setSize(numDocs).execute().actionGet(); + .prepareSearch("test") + .setQuery(QueryBuilders.functionScoreQuery(QueryBuilders.matchAllQuery(), new FunctionScoreQueryBuilder.FilterFunctionBuilder[]{ + new FunctionScoreQueryBuilder.FilterFunctionBuilder(ScoreFunctionBuilders.scriptFunction(new Script(CosineSimilarityScoreScript.SCRIPT_NAME, ScriptService.ScriptType.INLINE, "native", params)))}) + .boostMode(CombineFunction.REPLACE)).setSize(numDocs).execute().actionGet(); assertNoFailures(searchResponse); assertHitCount(searchResponse, numDocs); SearchHit[] hits = searchResponse.getHits().hits(); @@ -113,11 +116,11 @@ public void testPhraseScorer() throws Exception { // Retrieve records and see if they scored 0.0 SearchResponse searchResponse = client() - .prepareSearch("test") - .setQuery(QueryBuilders.functionScoreQuery(QueryBuilders.matchAllQuery(), new FunctionScoreQueryBuilder.FilterFunctionBuilder[]{ - new FunctionScoreQueryBuilder.FilterFunctionBuilder(ScoreFunctionBuilders.scriptFunction( - new Script(PhraseScoreScript.SCRIPT_NAME, ScriptService.ScriptType.INLINE, "native", params)))}) - .boostMode(CombineFunction.REPLACE)).setSize(numDocs).execute().actionGet(); + .prepareSearch("test") + .setQuery(QueryBuilders.functionScoreQuery(QueryBuilders.matchAllQuery(), new FunctionScoreQueryBuilder.FilterFunctionBuilder[]{ + new FunctionScoreQueryBuilder.FilterFunctionBuilder(ScoreFunctionBuilders.scriptFunction( + new Script(PhraseScoreScript.SCRIPT_NAME, ScriptService.ScriptType.INLINE, "native", params)))}) + .boostMode(CombineFunction.REPLACE)).setSize(numDocs).execute().actionGet(); assertNoFailures(searchResponse); assertHitCount(searchResponse, numDocs); SearchHit[] hits = searchResponse.getHits().hits(); @@ -140,10 +143,10 @@ public void testLanguageModelScorer() throws Exception { // Retrieve records and see if they scored 0.0 SearchResponse searchResponse = client() - .prepareSearch("test") - .setQuery(QueryBuilders.functionScoreQuery(QueryBuilders.matchAllQuery(), new FunctionScoreQueryBuilder.FilterFunctionBuilder[]{ - new FunctionScoreQueryBuilder.FilterFunctionBuilder(ScoreFunctionBuilders.scriptFunction(new Script(LanguageModelScoreScript.SCRIPT_NAME, ScriptService.ScriptType.INLINE, "native", params)))}) - .boostMode(CombineFunction.REPLACE)).setSize(numDocs).execute().actionGet(); + .prepareSearch("test") + .setQuery(QueryBuilders.functionScoreQuery(QueryBuilders.matchAllQuery(), new FunctionScoreQueryBuilder.FilterFunctionBuilder[]{ + new FunctionScoreQueryBuilder.FilterFunctionBuilder(ScoreFunctionBuilders.scriptFunction(new Script(LanguageModelScoreScript.SCRIPT_NAME, ScriptService.ScriptType.INLINE, "native", params)))}) + .boostMode(CombineFunction.REPLACE)).setSize(numDocs).execute().actionGet(); assertNoFailures(searchResponse); assertHitCount(searchResponse, numDocs); SearchHit[] hits = searchResponse.getHits().hits(); @@ -154,18 +157,20 @@ public void testLanguageModelScorer() throws Exception { private void initData() throws IOException, InterruptedException, ExecutionException { // Create a new index - String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties").startObject(field) - .field("type", "multi_field").startObject("fields").startObject(field).field("type", "String").endObject() - .startObject("word_count").field("analyzer", "standard").field("type", "token_count").startObject("fielddata") - .field("format", "doc_values").endObject().endObject().endObject().endObject().endObject().endObject().endObject().string(); + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") + .startObject(field).field("type", "string") + .startObject("fields") + .startObject("word_count").field("type", "string").field("analyzer", "standard").field("type", "token_count").startObject("fielddata").field("format", "doc_values").endObject().endObject() + .endObject() + .endObject().endObject().endObject().endObject().string(); assertAcked(prepareCreate("test").addMapping("type", mapping)); List indexBuilders = new ArrayList<>(); // Index numDocs records (0..99) for (int i = 0; i < numDocs; i++) { indexBuilders.add(client().prepareIndex("test", "type", Integer.toString(i)) - .setSource(XContentFactory.jsonBuilder().startObject().field(field, createText(i + 1)).endObject()) - .setId(Integer.toString(i))); + .setSource(XContentFactory.jsonBuilder().startObject().field(field, createText(i + 1)).endObject()) + .setId(Integer.toString(i))); } indexRandom(true, indexBuilders); } From 3a42192be040d22b3cc84e27ca8e615c096db42a Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Thu, 7 Jul 2016 12:46:46 -0400 Subject: [PATCH 67/69] Move native script examples to the plugins directory --- LICENSE.txt => plugins/native-script-example/LICENSE.txt | 0 README.textile => plugins/native-script-example/README.textile | 0 build.gradle => plugins/native-script-example/build.gradle | 0 .../native-script-example/config}/elasticsearch.yml | 0 {config => plugins/native-script-example/config}/logging.yml | 0 {example => plugins/native-script-example/example}/lookup.sh | 0 {example => plugins/native-script-example/example}/popularity.sh | 0 {example => plugins/native-script-example/example}/primes.sh | 0 {example => plugins/native-script-example/example}/random.sh | 0 {example => plugins/native-script-example/example}/stockaggs.sh | 0 {example => plugins/native-script-example/example}/termscoring.sh | 0 .../native-script-example/gradle.properties | 0 .../native-script-example/licenses}/commons-math3-3.5.jar.sha1 | 0 .../native-script-example/licenses}/commons-math3-LICENSE.txt | 0 .../native-script-example/licenses}/commons-math3-NOTICE.txt | 0 locator.json => plugins/native-script-example/locator.json | 0 .../examples/nativescript/plugin/NativeScriptExamplesPlugin.java | 0 .../examples/nativescript/script/CosineSimilarityScoreScript.java | 0 .../examples/nativescript/script/IsPrimeSearchScript.java | 0 .../examples/nativescript/script/LanguageModelScoreScript.java | 0 .../elasticsearch/examples/nativescript/script/LookupScript.java | 0 .../examples/nativescript/script/PhraseScoreScript.java | 0 .../nativescript/script/PopularityScoreScriptFactory.java | 0 .../examples/nativescript/script/RandomSortScriptFactory.java | 0 .../examples/nativescript/script/TFIDFScoreScript.java | 0 .../nativescript/script/stockaggs/CombineScriptFactory.java | 0 .../examples/nativescript/script/stockaggs/InitScriptFactory.java | 0 .../examples/nativescript/script/stockaggs/MapScriptFactory.java | 0 .../nativescript/script/stockaggs/ReduceScriptFactory.java | 0 .../examples/nativescript/ElasticsearchWithPluginLauncher.java | 0 .../examples/nativescript/plugin/NativeScriptExamplesRestIT.java | 0 .../nativescript/script/AbstractSearchScriptTestCase.java | 0 .../examples/nativescript/script/IsPrimeSearchScriptTests.java | 0 .../examples/nativescript/script/LookupScriptTests.java | 0 .../examples/nativescript/script/PopularityScoreScriptTests.java | 0 .../examples/nativescript/script/RandomSortScriptTests.java | 0 .../nativescript/script/ScriptedMetricAggsScriptTests.java | 0 .../examples/nativescript/script/TermScoringScriptTests.java | 0 .../native-script-example/src}/test/resources/log4j.properties | 0 .../test/resources/rest-api-spec/test/nativescript/10_primes.yaml | 0 .../test/resources/rest-api-spec/test/nativescript/20_aggs.yaml | 0 41 files changed, 0 insertions(+), 0 deletions(-) rename LICENSE.txt => plugins/native-script-example/LICENSE.txt (100%) rename README.textile => plugins/native-script-example/README.textile (100%) rename build.gradle => plugins/native-script-example/build.gradle (100%) rename {config => plugins/native-script-example/config}/elasticsearch.yml (100%) rename {config => plugins/native-script-example/config}/logging.yml (100%) rename {example => plugins/native-script-example/example}/lookup.sh (100%) rename {example => plugins/native-script-example/example}/popularity.sh (100%) rename {example => plugins/native-script-example/example}/primes.sh (100%) rename {example => plugins/native-script-example/example}/random.sh (100%) rename {example => plugins/native-script-example/example}/stockaggs.sh (100%) rename {example => plugins/native-script-example/example}/termscoring.sh (100%) rename gradle.properties => plugins/native-script-example/gradle.properties (100%) rename {licenses => plugins/native-script-example/licenses}/commons-math3-3.5.jar.sha1 (100%) rename {licenses => plugins/native-script-example/licenses}/commons-math3-LICENSE.txt (100%) rename {licenses => plugins/native-script-example/licenses}/commons-math3-NOTICE.txt (100%) rename locator.json => plugins/native-script-example/locator.json (100%) rename {src => plugins/native-script-example/src}/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java (100%) rename {src => plugins/native-script-example/src}/main/java/org/elasticsearch/examples/nativescript/script/CosineSimilarityScoreScript.java (100%) rename {src => plugins/native-script-example/src}/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java (100%) rename {src => plugins/native-script-example/src}/main/java/org/elasticsearch/examples/nativescript/script/LanguageModelScoreScript.java (100%) rename {src => plugins/native-script-example/src}/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java (100%) rename {src => plugins/native-script-example/src}/main/java/org/elasticsearch/examples/nativescript/script/PhraseScoreScript.java (100%) rename {src => plugins/native-script-example/src}/main/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptFactory.java (100%) rename {src => plugins/native-script-example/src}/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java (100%) rename {src => plugins/native-script-example/src}/main/java/org/elasticsearch/examples/nativescript/script/TFIDFScoreScript.java (100%) rename {src => plugins/native-script-example/src}/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/CombineScriptFactory.java (100%) rename {src => plugins/native-script-example/src}/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/InitScriptFactory.java (100%) rename {src => plugins/native-script-example/src}/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/MapScriptFactory.java (100%) rename {src => plugins/native-script-example/src}/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/ReduceScriptFactory.java (100%) rename {src => plugins/native-script-example/src}/test/java/org/elasticsearch/examples/nativescript/ElasticsearchWithPluginLauncher.java (100%) rename {src => plugins/native-script-example/src}/test/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesRestIT.java (100%) rename {src => plugins/native-script-example/src}/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTestCase.java (100%) rename {src => plugins/native-script-example/src}/test/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScriptTests.java (100%) rename {src => plugins/native-script-example/src}/test/java/org/elasticsearch/examples/nativescript/script/LookupScriptTests.java (100%) rename {src => plugins/native-script-example/src}/test/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptTests.java (100%) rename {src => plugins/native-script-example/src}/test/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptTests.java (100%) rename {src => plugins/native-script-example/src}/test/java/org/elasticsearch/examples/nativescript/script/ScriptedMetricAggsScriptTests.java (100%) rename {src => plugins/native-script-example/src}/test/java/org/elasticsearch/examples/nativescript/script/TermScoringScriptTests.java (100%) rename {src => plugins/native-script-example/src}/test/resources/log4j.properties (100%) rename {src => plugins/native-script-example/src}/test/resources/rest-api-spec/test/nativescript/10_primes.yaml (100%) rename {src => plugins/native-script-example/src}/test/resources/rest-api-spec/test/nativescript/20_aggs.yaml (100%) diff --git a/LICENSE.txt b/plugins/native-script-example/LICENSE.txt similarity index 100% rename from LICENSE.txt rename to plugins/native-script-example/LICENSE.txt diff --git a/README.textile b/plugins/native-script-example/README.textile similarity index 100% rename from README.textile rename to plugins/native-script-example/README.textile diff --git a/build.gradle b/plugins/native-script-example/build.gradle similarity index 100% rename from build.gradle rename to plugins/native-script-example/build.gradle diff --git a/config/elasticsearch.yml b/plugins/native-script-example/config/elasticsearch.yml similarity index 100% rename from config/elasticsearch.yml rename to plugins/native-script-example/config/elasticsearch.yml diff --git a/config/logging.yml b/plugins/native-script-example/config/logging.yml similarity index 100% rename from config/logging.yml rename to plugins/native-script-example/config/logging.yml diff --git a/example/lookup.sh b/plugins/native-script-example/example/lookup.sh similarity index 100% rename from example/lookup.sh rename to plugins/native-script-example/example/lookup.sh diff --git a/example/popularity.sh b/plugins/native-script-example/example/popularity.sh similarity index 100% rename from example/popularity.sh rename to plugins/native-script-example/example/popularity.sh diff --git a/example/primes.sh b/plugins/native-script-example/example/primes.sh similarity index 100% rename from example/primes.sh rename to plugins/native-script-example/example/primes.sh diff --git a/example/random.sh b/plugins/native-script-example/example/random.sh similarity index 100% rename from example/random.sh rename to plugins/native-script-example/example/random.sh diff --git a/example/stockaggs.sh b/plugins/native-script-example/example/stockaggs.sh similarity index 100% rename from example/stockaggs.sh rename to plugins/native-script-example/example/stockaggs.sh diff --git a/example/termscoring.sh b/plugins/native-script-example/example/termscoring.sh similarity index 100% rename from example/termscoring.sh rename to plugins/native-script-example/example/termscoring.sh diff --git a/gradle.properties b/plugins/native-script-example/gradle.properties similarity index 100% rename from gradle.properties rename to plugins/native-script-example/gradle.properties diff --git a/licenses/commons-math3-3.5.jar.sha1 b/plugins/native-script-example/licenses/commons-math3-3.5.jar.sha1 similarity index 100% rename from licenses/commons-math3-3.5.jar.sha1 rename to plugins/native-script-example/licenses/commons-math3-3.5.jar.sha1 diff --git a/licenses/commons-math3-LICENSE.txt b/plugins/native-script-example/licenses/commons-math3-LICENSE.txt similarity index 100% rename from licenses/commons-math3-LICENSE.txt rename to plugins/native-script-example/licenses/commons-math3-LICENSE.txt diff --git a/licenses/commons-math3-NOTICE.txt b/plugins/native-script-example/licenses/commons-math3-NOTICE.txt similarity index 100% rename from licenses/commons-math3-NOTICE.txt rename to plugins/native-script-example/licenses/commons-math3-NOTICE.txt diff --git a/locator.json b/plugins/native-script-example/locator.json similarity index 100% rename from locator.json rename to plugins/native-script-example/locator.json diff --git a/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java b/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java similarity index 100% rename from src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java rename to plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/CosineSimilarityScoreScript.java b/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/CosineSimilarityScoreScript.java similarity index 100% rename from src/main/java/org/elasticsearch/examples/nativescript/script/CosineSimilarityScoreScript.java rename to plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/CosineSimilarityScoreScript.java diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java b/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java similarity index 100% rename from src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java rename to plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/LanguageModelScoreScript.java b/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/LanguageModelScoreScript.java similarity index 100% rename from src/main/java/org/elasticsearch/examples/nativescript/script/LanguageModelScoreScript.java rename to plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/LanguageModelScoreScript.java diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java b/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java similarity index 100% rename from src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java rename to plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/PhraseScoreScript.java b/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/PhraseScoreScript.java similarity index 100% rename from src/main/java/org/elasticsearch/examples/nativescript/script/PhraseScoreScript.java rename to plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/PhraseScoreScript.java diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptFactory.java b/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptFactory.java similarity index 100% rename from src/main/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptFactory.java rename to plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptFactory.java diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java b/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java similarity index 100% rename from src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java rename to plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/TFIDFScoreScript.java b/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/TFIDFScoreScript.java similarity index 100% rename from src/main/java/org/elasticsearch/examples/nativescript/script/TFIDFScoreScript.java rename to plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/TFIDFScoreScript.java diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/CombineScriptFactory.java b/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/CombineScriptFactory.java similarity index 100% rename from src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/CombineScriptFactory.java rename to plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/CombineScriptFactory.java diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/InitScriptFactory.java b/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/InitScriptFactory.java similarity index 100% rename from src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/InitScriptFactory.java rename to plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/InitScriptFactory.java diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/MapScriptFactory.java b/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/MapScriptFactory.java similarity index 100% rename from src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/MapScriptFactory.java rename to plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/MapScriptFactory.java diff --git a/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/ReduceScriptFactory.java b/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/ReduceScriptFactory.java similarity index 100% rename from src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/ReduceScriptFactory.java rename to plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/ReduceScriptFactory.java diff --git a/src/test/java/org/elasticsearch/examples/nativescript/ElasticsearchWithPluginLauncher.java b/plugins/native-script-example/src/test/java/org/elasticsearch/examples/nativescript/ElasticsearchWithPluginLauncher.java similarity index 100% rename from src/test/java/org/elasticsearch/examples/nativescript/ElasticsearchWithPluginLauncher.java rename to plugins/native-script-example/src/test/java/org/elasticsearch/examples/nativescript/ElasticsearchWithPluginLauncher.java diff --git a/src/test/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesRestIT.java b/plugins/native-script-example/src/test/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesRestIT.java similarity index 100% rename from src/test/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesRestIT.java rename to plugins/native-script-example/src/test/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesRestIT.java diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTestCase.java b/plugins/native-script-example/src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTestCase.java similarity index 100% rename from src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTestCase.java rename to plugins/native-script-example/src/test/java/org/elasticsearch/examples/nativescript/script/AbstractSearchScriptTestCase.java diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScriptTests.java b/plugins/native-script-example/src/test/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScriptTests.java similarity index 100% rename from src/test/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScriptTests.java rename to plugins/native-script-example/src/test/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScriptTests.java diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/LookupScriptTests.java b/plugins/native-script-example/src/test/java/org/elasticsearch/examples/nativescript/script/LookupScriptTests.java similarity index 100% rename from src/test/java/org/elasticsearch/examples/nativescript/script/LookupScriptTests.java rename to plugins/native-script-example/src/test/java/org/elasticsearch/examples/nativescript/script/LookupScriptTests.java diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptTests.java b/plugins/native-script-example/src/test/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptTests.java similarity index 100% rename from src/test/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptTests.java rename to plugins/native-script-example/src/test/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptTests.java diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptTests.java b/plugins/native-script-example/src/test/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptTests.java similarity index 100% rename from src/test/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptTests.java rename to plugins/native-script-example/src/test/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptTests.java diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/ScriptedMetricAggsScriptTests.java b/plugins/native-script-example/src/test/java/org/elasticsearch/examples/nativescript/script/ScriptedMetricAggsScriptTests.java similarity index 100% rename from src/test/java/org/elasticsearch/examples/nativescript/script/ScriptedMetricAggsScriptTests.java rename to plugins/native-script-example/src/test/java/org/elasticsearch/examples/nativescript/script/ScriptedMetricAggsScriptTests.java diff --git a/src/test/java/org/elasticsearch/examples/nativescript/script/TermScoringScriptTests.java b/plugins/native-script-example/src/test/java/org/elasticsearch/examples/nativescript/script/TermScoringScriptTests.java similarity index 100% rename from src/test/java/org/elasticsearch/examples/nativescript/script/TermScoringScriptTests.java rename to plugins/native-script-example/src/test/java/org/elasticsearch/examples/nativescript/script/TermScoringScriptTests.java diff --git a/src/test/resources/log4j.properties b/plugins/native-script-example/src/test/resources/log4j.properties similarity index 100% rename from src/test/resources/log4j.properties rename to plugins/native-script-example/src/test/resources/log4j.properties diff --git a/src/test/resources/rest-api-spec/test/nativescript/10_primes.yaml b/plugins/native-script-example/src/test/resources/rest-api-spec/test/nativescript/10_primes.yaml similarity index 100% rename from src/test/resources/rest-api-spec/test/nativescript/10_primes.yaml rename to plugins/native-script-example/src/test/resources/rest-api-spec/test/nativescript/10_primes.yaml diff --git a/src/test/resources/rest-api-spec/test/nativescript/20_aggs.yaml b/plugins/native-script-example/src/test/resources/rest-api-spec/test/nativescript/20_aggs.yaml similarity index 100% rename from src/test/resources/rest-api-spec/test/nativescript/20_aggs.yaml rename to plugins/native-script-example/src/test/resources/rest-api-spec/test/nativescript/20_aggs.yaml From 7df2d06eb62dbbc5cc365153088993d70801c286 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Thu, 7 Jul 2016 19:11:52 -0400 Subject: [PATCH 68/69] Update to work with the latest master --- plugins/native-script-example/LICENSE.txt | 202 ------------------ plugins/native-script-example/README.textile | 58 ++--- plugins/native-script-example/build.gradle | 33 +-- .../config/elasticsearch.yml | 1 - .../native-script-example/config/logging.yml | 45 ---- .../native-script-example/example/lookup.sh | 85 -------- .../native-script-example/gradle.properties | 1 - .../licenses/commons-math3-3.5.jar.sha1 | 1 - .../licenses/commons-math3-LICENSE.txt | 202 ------------------ .../licenses/commons-math3-NOTICE.txt | 9 - plugins/native-script-example/locator.json | 6 - .../plugin/NativeScriptExamplesPlugin.java | 54 ++--- .../script/CosineSimilarityScoreScript.java | 21 +- .../script/IsPrimeSearchScript.java | 8 +- .../script/LanguageModelScoreScript.java | 17 +- .../nativescript/script/LookupScript.java | 171 --------------- .../script/PhraseScoreScript.java | 12 +- .../script/PopularityScoreScriptFactory.java | 23 +- .../script/RandomSortScriptFactory.java | 8 +- .../nativescript/script/TFIDFScoreScript.java | 15 +- .../stockaggs/CombineScriptFactory.java | 10 +- .../script/stockaggs/InitScriptFactory.java | 10 +- .../script/stockaggs/MapScriptFactory.java | 10 +- .../script/stockaggs/ReduceScriptFactory.java | 10 +- .../ElasticsearchWithPluginLauncher.java | 66 ------ .../script/LookupScriptTests.java | 123 ----------- .../script/PopularityScoreScriptTests.java | 3 +- .../script/RandomSortScriptTests.java | 12 +- .../script/TermScoringScriptTests.java | 25 ++- .../src/test/resources/log4j.properties | 5 - .../test/nativescript/10_primes.yaml | 10 +- .../test/nativescript/20_aggs.yaml | 13 +- settings.gradle | 1 + 33 files changed, 170 insertions(+), 1100 deletions(-) delete mode 100644 plugins/native-script-example/LICENSE.txt delete mode 100644 plugins/native-script-example/config/elasticsearch.yml delete mode 100644 plugins/native-script-example/config/logging.yml delete mode 100755 plugins/native-script-example/example/lookup.sh delete mode 100644 plugins/native-script-example/gradle.properties delete mode 100644 plugins/native-script-example/licenses/commons-math3-3.5.jar.sha1 delete mode 100644 plugins/native-script-example/licenses/commons-math3-LICENSE.txt delete mode 100644 plugins/native-script-example/licenses/commons-math3-NOTICE.txt delete mode 100644 plugins/native-script-example/locator.json delete mode 100644 plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java delete mode 100644 plugins/native-script-example/src/test/java/org/elasticsearch/examples/nativescript/ElasticsearchWithPluginLauncher.java delete mode 100644 plugins/native-script-example/src/test/java/org/elasticsearch/examples/nativescript/script/LookupScriptTests.java delete mode 100644 plugins/native-script-example/src/test/resources/log4j.properties diff --git a/plugins/native-script-example/LICENSE.txt b/plugins/native-script-example/LICENSE.txt deleted file mode 100644 index d645695673349..0000000000000 --- a/plugins/native-script-example/LICENSE.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/plugins/native-script-example/README.textile b/plugins/native-script-example/README.textile index 56f71d0376cda..c119c46012b57 100644 --- a/plugins/native-script-example/README.textile +++ b/plugins/native-script-example/README.textile @@ -4,14 +4,6 @@ h2. Introduction p. This plugin contains several examples of "native script":http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-scripting.html for Elasticsearch. -p. Please make sure to use the correct branch of this repository that corresponds to the version of elasticsearch that you are developing the plugin for. - -|_. Example Plugin Branch |_. Elasticsearch | -| "master":https://github.com/imotov/elasticsearch-native-script-example | master | -| "2.x":https://github.com/imotov/elasticsearch-native-script-example/tree/2.x | 2.x.x | -| "2.0":https://github.com/imotov/elasticsearch-native-script-example/tree/2.0 | 2.0.x | -| "1.x":https://github.com/imotov/elasticsearch-native-script-example/tree/1.x | 1.x.x | - h2. Creating Elasticsearch Plugin p. The simplest way to deploy native script is by wrapping it into standard Elasticsearch plugin infrastructure. An Elasticsearch plugin can be written in java and built using gradle. A typical plugin source directory looks like this: @@ -31,7 +23,7 @@ bc.. . p. An Elasticsearch plugin can be created by following these six steps. -* Create build.gradle file in the root directory of your plugin. The "pom.xml":https://github.com/imotov/elasticsearch-native-script-example/blob/master/pom.xml file in this project can be used as a starting point. +* Create build.gradle file in the root directory of your plugin. * Create source code directories: ** @mkdir -p src/main/java@ ** @mkdir -p src/test/java@ @@ -52,35 +44,13 @@ bc.. package org.elasticsearch.examples.nativescript.plugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.ScriptModule; -public class NativeScriptExamplesPlugin extends Plugin { - @Override - public String name() { - return "native-script-examples"; - } - - @Override - public String description() { - return "Examples of native script"; - } +public class NativeScriptExamplesPlugin extends Plugin implements ScriptPlugin { + /* .... */ } p. -* The gradle plugin will automatically create @plugin-descriptor.properties@ for you. -* If you are not using the standard parent project, you can create this file manually by using this "plugin-descriptor.properties":https://github.com/elastic/elasticsearch/blob/master/dev-tools/src/main/resources/plugin-metadata/plugin-descriptor.properties as a template. You will also need to package the plugin into .zip file, which can be done using maven assemble task and "plugin-assembly.xml":https://github.com/elastic/elasticsearch/blob/master/dev-tools/src/main/resources/plugin-metadata/plugin-assembly.xml assembly definition. - -* The plugin can be built using @mvn package@ command. The assembled .zip package can be found in the @target/releases/@ directory and deployed to elasticsearch installation using @plugin -install plugin-name -url path/to/plugin/zip/file@. - -h2. Migration from 1.x - -p. The plugin infrastructure significantly changed in 2.0 and above. So, the plugin project will need to be modified in order to be used with elasticsearch 2.0: - -* Instead of using @es-plugin.properties@ file that in 1.x was places in the plugin jar, the plugin infrastructure is now using the @plugin-descriptor.properties@ file that describes not only the main plugin class, version and description but also plugin type (_site and/or jvm), required minimal java and Elasticsearch versions. The @plugin-descriptor.properties@ file should be placed into root directory of the .zip file that the plugin is packaged into. The simplest way to deal with this change is by switching the plugin project to @org.elasticsearch.plugin:plugins@ as a parent. -* Elasticsearch 2.0 is also stricter when it comes to plugin classes. For example, the "jar hell" prevention mechanism will not allow the plugin to contain classes that are already defined in the Elasticsearch classpath. Make sure that your project doesn't have any dependencies that are conflicting with existing elasticsearch classes. -* In 2.0 the base class for the plugin was renamed from @AbstractPlugin@ to @Plugin@ -* Plugins are no longer loaded from the classpath, so they have to be "explicitly loaded":https://github.com/imotov/elasticsearch-native-script-example/commit/96f8b93f27f80346503dfa91e564b87c9334fbdf in the tests. -* Some base test classes need to be "renamed":https://github.com/imotov/elasticsearch-native-script-example/commit/6c4ba4ab967b938fd06200b51e78dcf523861c40. -* Native scripts now have to "indicate":https://github.com/imotov/elasticsearch-native-script-example/commit/4cda2ccdaa8094c5de3e86ccfeea0fe56b26d353 whether they use the `_score` or not. +* The plugin can be built using @gradle build@ command and executed within elasticsearch by using @gradle run@. h2. Adding Native Scripts @@ -88,26 +58,26 @@ p. Now that the plugin infrastructure is complete, it's possible to add a native h3. Is Prime Native Script -p. One of the example scripts in this project is the "is_prime" script that can be used to check if a field contains a possible prime number. The script accepts two parameters @field@ and @certainty@. The @field@ parameter contains the name of the field that needs to be checked and the @certainty@ parameter specifies a measure of the uncertainty that the caller is willing to tolerate. The script returns @true@ if the field contains a probable prime number and @false@ otherwise. The probability that the number for which the script returned @true@ is prime exceeds (1 - 0.5^certainty). The script can be used in "Script Filter":http://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-script-filter.html as well as a "Script Field":http://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-script-fields.html. The implementation of the "is_prime" native script and it's factory can be found in the "IsPrimeSearchScript":https://github.com/imotov/elasticsearch-native-script-example/blob/master/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java class. +p. One of the example scripts in this project is the "is_prime" script that can be used to check if a field contains a possible prime number. The script accepts two parameters @field@ and @certainty@. The @field@ parameter contains the name of the field that needs to be checked and the @certainty@ parameter specifies a measure of the uncertainty that the caller is willing to tolerate. The script returns @true@ if the field contains a probable prime number and @false@ otherwise. The probability that the number for which the script returned @true@ is prime exceeds (1 - 0.5^certainty). The script can be used in "Script Filter":http://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-script-filter.html as well as a "Script Field":http://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-script-fields.html. The implementation of the "is_prime" native script and it's factory can be found in the "IsPrimeSearchScript":https://github.com/elastic/elasticsearch/blob/master/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java class. + +p. In order to enable native script creation the plugin has to contain and register a class that implements "NativeScriptFactory":https://github.com/elastic/elasticsearch/blob/master/core/src/main/java/org/elasticsearch/script/NativeScriptFactory.java. The NativeScriptFactory interface has only one method @newScript(Map params)@. This method accepts a list of parameters that are passed during script creation and returns an ExecutableScript object with an instance of the script. The factory has to be returned in the @onModule(ScriptModule module)@ method of the Plugin. -p. In order to enable native script creation the plugin has to contain and register a class that implements "NativeScriptFactory":https://github.com/elasticsearch/elasticsearch/blob/master/src/main/java/org/elasticsearch/script/NativeScriptFactory.java. The NativeScriptFactory interface has only one method @newScript(Map params)@. This method accepts a list of parameters that are passed during script creation and returns an ExecutableScript object with an instance of the script. The factory has to be registered in the @onModule(ScriptModule module)@ method of the Plugin. +bc.. public class NativeScriptExamplesPlugin extends Plugin implements ScriptPlugin { -bc.. public class NativeScriptExamplesPlugin extends AbstractPlugin { - /* ..... */ - public void onModule(ScriptModule module) { - // Register each script that we defined in this plugin - module.registerScript("is_prime", IsPrimeSearchScript.Factory.class); + @Override + public List getNativeScripts() { + return Arrays.asList(new IsPrimeSearchScript.Factory()); } } -p. In general native scripts have to implement the interface "ExecutableScript":https://github.com/elasticsearch/elasticsearch/blob/master/src/main/java/org/elasticsearch/script/ExecutableScript.java, but if they are used in search, they have to also implement the "SearchScript":https://github.com/elasticsearch/elasticsearch/blob/master/src/main/java/org/elasticsearch/script/SearchScript.java interface. The SearchScript interface is quite complex, so in order to simplify implementation, the native script can simply extend the "AbstractSearchScript":https://github.com/elasticsearch/elasticsearch/blob/master/src/main/java/org/elasticsearch/script/AbstractSearchScript.java class instead. The AbstractSearchScript has only one abstract method @run()@. During search Elasticsearch calls this method for every single record in the search result. As in case of non-native script, the content of the current record can be accessed using DocLookup (@doc()@ method), FieldsLookup (@fields()@ method), SourceLookup (@source()@ method). +p. In general native scripts have to implement the interface "ExecutableScript":https://github.com/elasticsearch/elasticsearch/blob/master/core/src/main/java/org/elasticsearch/script/ExecutableScript.java, but if they are used in search, they have to also implement the "SearchScript":https://github.com/elasticsearch/elasticsearch/blob/master/core/src/main/java/org/elasticsearch/script/SearchScript.java interface. The SearchScript interface is quite complex, so in order to simplify implementation, the native script can simply extend the "AbstractSearchScript":https://github.com/elasticsearch/elasticsearch/blob/master/core/src/main/java/org/elasticsearch/script/AbstractSearchScript.java class instead. The AbstractSearchScript has only one abstract method @run()@. During search Elasticsearch calls this method for every single record in the search result. As in case of non-native script, the content of the current record can be accessed using DocLookup (@doc()@ method), FieldsLookup (@fields()@ method), SourceLookup (@source()@ method). h3. Lookup Script -p. The "lookup script":https://github.com/imotov/elasticsearch-native-script-example/blob/master/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java demonstrates how to make elsticsearch client available within the native script. When script factory is initially created, the node is not fully initialized yet. Because of this client cannot be directory injected into the factory. Instead, the reference to the node is injected and the client is obtained from the node during script creation. A same mechanism can be used to obtain other node components through node injector. +p. The "lookup script":https://github.com/elastic/elasticsearch/blob/master/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java demonstrates how to make elsticsearch client available within the native script. When script factory is initially created, the node is not fully initialized yet. Because of this client cannot be directory injected into the factory. Instead, the reference to the node is injected and the client is obtained from the node during script creation. A same mechanism can be used to obtain other node components through node injector. h3. Random Sort Script -p. The "random sort script":https://github.com/imotov/elasticsearch-native-script-example/blob/master/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java demonstrates a slightly different approach to script/factory packaging. In this case the factory is the outer class which creates one inner script or another based on the input parameters. If the parameter @salt@ is present, the script is calculating hash value of @id + salt@ instead of generation random sort values. As a result, for any value of @salt@ the order of the records will appear random, but this order will be repeatable and therefore this approach would be more suitable for paging through result list than a completely random approach. +p. The "random sort script":https://github.com/elastic/elasticsearch/blob/master/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java demonstrates a slightly different approach to script/factory packaging. In this case the factory is the outer class which creates one inner script or another based on the input parameters. If the parameter @salt@ is present, the script is calculating hash value of @id + salt@ instead of generation random sort values. As a result, for any value of @salt@ the order of the records will appear random, but this order will be repeatable and therefore this approach would be more suitable for paging through result list than a completely random approach. diff --git a/plugins/native-script-example/build.gradle b/plugins/native-script-example/build.gradle index 2d23579dede0d..e129154efcb55 100644 --- a/plugins/native-script-example/build.gradle +++ b/plugins/native-script-example/build.gradle @@ -1,39 +1,12 @@ -buildscript { - repositories { - mavenCentral() - maven { - name 'sonatype-snapshots' - url "https://oss.sonatype.org/content/repositories/snapshots/" - } - jcenter() - } - dependencies { - // Version of elasticsearch - classpath "org.elasticsearch.gradle:build-tools:3.0.0-SNAPSHOT" - } -} - -apply plugin: 'idea' -apply plugin: 'eclipse' -apply plugin: 'elasticsearch.esplugin' - -// Version of the plugin -version = '3.0.0-SNAPSHOT' - esplugin { - description 'ElasticSearch Plugin with Native Script Examples.' + description 'Demonstrates writtting elasticsearch plugins with native scripts' classname 'org.elasticsearch.examples.nativescript.plugin.NativeScriptExamplesPlugin' } -dependencies { - // Not really used at the moment - just to show how to include dependencies - compile "org.apache.commons:commons-math3:3.5" -} - compileJava.options.compilerArgs << "-Xlint:-deprecation" -// uncomment to disable 3rd party audit - TODO: temporary workaround until it's fixed in elasticsearch -thirdPartyAudit.enabled = false +// uncomment to disable 3rd party audit +// thirdPartyAudit.enabled = false // uncomment to disable forbidden patterns checks // forbiddenPatterns.enabled = false diff --git a/plugins/native-script-example/config/elasticsearch.yml b/plugins/native-script-example/config/elasticsearch.yml deleted file mode 100644 index 63bc3986b9f8f..0000000000000 --- a/plugins/native-script-example/config/elasticsearch.yml +++ /dev/null @@ -1 +0,0 @@ -cluster.name: elasticsearch-native-script-tests diff --git a/plugins/native-script-example/config/logging.yml b/plugins/native-script-example/config/logging.yml deleted file mode 100644 index d3550199ade95..0000000000000 --- a/plugins/native-script-example/config/logging.yml +++ /dev/null @@ -1,45 +0,0 @@ -rootLogger: INFO, console, file -logger: - # log action execution errors for easier debugging - action: DEBUG - # reduce the logging for aws, too much is logged under the default INFO - com.amazonaws: WARN - - # gateway - #gateway: DEBUG - #index.gateway: DEBUG - - # peer shard recovery - #indices.recovery: DEBUG - - # discovery - #discovery: TRACE - examples.nativescript.script: TRACE - - index.search.slowlog: TRACE, index_search_slow_log_file - -additivity: - index.search.slowlog: false - -appender: - console: - type: console - layout: - type: consolePattern - conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n" - - file: - type: dailyRollingFile - file: ${path.logs}/${cluster.name}.log - datePattern: "'.'yyyy-MM-dd" - layout: - type: pattern - conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n" - - index_search_slow_log_file: - type: dailyRollingFile - file: ${path.logs}/${cluster.name}_index_search_slowlog.log - datePattern: "'.'yyyy-MM-dd" - layout: - type: pattern - conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n" diff --git a/plugins/native-script-example/example/lookup.sh b/plugins/native-script-example/example/lookup.sh deleted file mode 100755 index 918f33bc7c534..0000000000000 --- a/plugins/native-script-example/example/lookup.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/bin/sh -curl -s -XDELETE "http://localhost:9200/test" -echo -curl -s -XPUT "http://localhost:9200/test/" -d '{ - "settings": { - "index.number_of_shards": 1, - "index.number_of_replicas": 0 - }, - "mappings": { - "state": { - "properties": { - "name": { - "type": "string" - }, - "capital": { - "type": "string" - }, - "nickname": { - "type": "string" - } - } - }, - "city": { - "properties": { - "city": { - "type": "string" - }, - "state": { - "type": "string", - "index": "not_analyzed" - }, - "population": { - "type": "integer" - } - } - } - } -}' -echo -curl -s -XPUT "localhost:9200/test/state/CT" -d '{"name": "Connecticut", "capital": "Hartford", "nickname": "Constitution State"}' -curl -s -XPUT "localhost:9200/test/state/ME" -d '{"name": "Maine", "capital": "Augusta", "nickname": "Lumber State"}' -curl -s -XPUT "localhost:9200/test/state/MA" -d '{"name": "Massachusetts", "capital": "Boston", "nickname": "Bay State"}' -curl -s -XPUT "localhost:9200/test/state/NH" -d '{"name": "New Hampshire", "capital": "Concord", "nickname": "Granite State"}' -curl -s -XPUT "localhost:9200/test/state/RI" -d '{"name": "Rhode Island", "capital": "Providence", "nickname": "Little Rhody"}' -curl -s -XPUT "localhost:9200/test/state/VT" -d '{"name": "Vermont", "capital": "Montpelier", "nickname": "Green Mountain State"}' - -curl -s -XPUT "localhost:9200/test/city/1" -d '{"city": "Cambridge", "state": "MA", "population": 105162}' -curl -s -XPUT "localhost:9200/test/city/2" -d '{"city": "South Burlington", "state": "VT", "population": 17904}' -curl -s -XPUT "localhost:9200/test/city/3" -d '{"city": "South Portland", "state": "ME", "population": 25002}' -curl -s -XPUT "localhost:9200/test/city/4" -d '{"city": "Essex", "state": "VT", "population": 19587}' -curl -s -XPUT "localhost:9200/test/city/5" -d '{"city": "Portland", "state": "ME", "population": 66194}' -curl -s -XPUT "localhost:9200/test/city/6" -d '{"city": "Burlington", "state": "VT", "population": 42417}' -curl -s -XPUT "localhost:9200/test/city/7" -d '{"city": "Stamford", "state": "CT", "population": 122643}' -curl -s -XPUT "localhost:9200/test/city/8" -d '{"city": "Colchester", "state": "VT", "population": 17067}' -curl -s -XPUT "localhost:9200/test/city/9" -d '{"city": "Concord", "state": "NH", "population": 42695}' -curl -s -XPUT "localhost:9200/test/city/10" -d '{"city": "Boston", "state": "MA", "population": 617594}' - -curl -s -XPOST "http://localhost:9200/test/_refresh" -echo -curl -s -XGET "localhost:9200/test/city/_search?pretty=true" -d '{ - "query": { - "match_all": { } - }, - "fields": ["city", "state"], - "script_fields": { - "state_info": { - "script": { - "inline": "lookup", - "lang": "native", - "params": { - "lookup_index": "test", - "lookup_type": "state", - "field": "state" - } - } - } - }, - "sort": [ - { - "population": "desc" - } - ] -} -' - diff --git a/plugins/native-script-example/gradle.properties b/plugins/native-script-example/gradle.properties deleted file mode 100644 index 6b1823d86a69a..0000000000000 --- a/plugins/native-script-example/gradle.properties +++ /dev/null @@ -1 +0,0 @@ -org.gradle.daemon=false diff --git a/plugins/native-script-example/licenses/commons-math3-3.5.jar.sha1 b/plugins/native-script-example/licenses/commons-math3-3.5.jar.sha1 deleted file mode 100644 index c3c6cfc20e61f..0000000000000 --- a/plugins/native-script-example/licenses/commons-math3-3.5.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -13af68e9a11576953f5c4b09436f8207be9d3a27 \ No newline at end of file diff --git a/plugins/native-script-example/licenses/commons-math3-LICENSE.txt b/plugins/native-script-example/licenses/commons-math3-LICENSE.txt deleted file mode 100644 index d645695673349..0000000000000 --- a/plugins/native-script-example/licenses/commons-math3-LICENSE.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/plugins/native-script-example/licenses/commons-math3-NOTICE.txt b/plugins/native-script-example/licenses/commons-math3-NOTICE.txt deleted file mode 100644 index ce791e4f32652..0000000000000 --- a/plugins/native-script-example/licenses/commons-math3-NOTICE.txt +++ /dev/null @@ -1,9 +0,0 @@ -Apache Commons Math -Copyright 2001-2015 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -This product includes software developed for Orekit by -CS Systèmes d'Information (http://www.c-s.fr/) -Copyright 2010-2012 CS Systèmes d'Information diff --git a/plugins/native-script-example/locator.json b/plugins/native-script-example/locator.json deleted file mode 100644 index d9f727f25249c..0000000000000 --- a/plugins/native-script-example/locator.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "versions" : [ - {"version": "1.1.0", "from": "0.90.0.Beta1", "to": "0.90.99", "location":"http://dl.bintray.com/content/imotov/elasticsearch-plugins/elasticsearch-native-script-example-1.1.0.zip" }, - {"version": "1.0.0", "from": "0.20.0", "to": "0.20.99", "location":"http://dl.bintray.com/content/imotov/elasticsearch-plugins/elasticsearch-native-script-example-1.0.0.zip" } - ] -} \ No newline at end of file diff --git a/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java b/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java index 077ba11a29d70..c03cb28f595b8 100644 --- a/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java +++ b/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java @@ -16,7 +16,6 @@ import org.elasticsearch.examples.nativescript.script.IsPrimeSearchScript; import org.elasticsearch.examples.nativescript.script.LanguageModelScoreScript; -import org.elasticsearch.examples.nativescript.script.LookupScript; import org.elasticsearch.examples.nativescript.script.CosineSimilarityScoreScript; import org.elasticsearch.examples.nativescript.script.PhraseScoreScript; import org.elasticsearch.examples.nativescript.script.TFIDFScoreScript; @@ -27,51 +26,24 @@ import org.elasticsearch.examples.nativescript.script.stockaggs.MapScriptFactory; import org.elasticsearch.examples.nativescript.script.stockaggs.ReduceScriptFactory; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.script.ScriptModule; +import org.elasticsearch.plugins.ScriptPlugin; +import org.elasticsearch.script.NativeScriptFactory; + +import java.util.Arrays; +import java.util.List; /** * This class is instantiated when Elasticsearch loads the plugin for the - * first time. If you change the name of this plugin, make sure to update - * src/main/resources/es-plugin.properties file that points to this class. + * first time. */ -public class NativeScriptExamplesPlugin extends Plugin { - - /** - * The name of the plugin. - *

- * This name will be used by elasticsearch in the log file to refer to this plugin. - * - * @return plugin name. - */ - @Override - public String name() { - return "native-script-example"; - } +public class NativeScriptExamplesPlugin extends Plugin implements ScriptPlugin { - /** - * The description of the plugin. - * - * @return plugin description - */ @Override - public String description() { - return "Native script examples"; - } - - public void onModule(ScriptModule module) { - // Register each script that we defined in this plugin - module.registerScript("is_prime", IsPrimeSearchScript.Factory.class); - module.registerScript("lookup", LookupScript.Factory.class); - module.registerScript("random", RandomSortScriptFactory.class); - module.registerScript("popularity", PopularityScoreScriptFactory.class); - module.registerScript(TFIDFScoreScript.SCRIPT_NAME, TFIDFScoreScript.Factory.class); - module.registerScript(CosineSimilarityScoreScript.SCRIPT_NAME, CosineSimilarityScoreScript.Factory.class); - module.registerScript(PhraseScoreScript.SCRIPT_NAME, PhraseScoreScript.Factory.class); - module.registerScript(LanguageModelScoreScript.SCRIPT_NAME, LanguageModelScoreScript.Factory.class); - // Scripted Metric Aggregation Scripts - module.registerScript("stockaggs_init", InitScriptFactory.class); - module.registerScript("stockaggs_map", MapScriptFactory.class); - module.registerScript("stockaggs_combine", CombineScriptFactory.class); - module.registerScript("stockaggs_reduce", ReduceScriptFactory.class); + public List getNativeScripts() { + return Arrays.asList(new IsPrimeSearchScript.Factory(), new RandomSortScriptFactory(), + new PopularityScoreScriptFactory(), new TFIDFScoreScript.Factory(), new CosineSimilarityScoreScript.Factory(), + new PhraseScoreScript.Factory(), new LanguageModelScoreScript.Factory(), + // Scripted Metric Aggregations Scripts + new InitScriptFactory(), new MapScriptFactory(), new CombineScriptFactory(), new ReduceScriptFactory()); } } diff --git a/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/CosineSimilarityScoreScript.java b/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/CosineSimilarityScoreScript.java index 643c0d39501e1..831bc3d916628 100644 --- a/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/CosineSimilarityScoreScript.java +++ b/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/CosineSimilarityScoreScript.java @@ -18,8 +18,6 @@ import java.util.List; import java.util.Map; -import org.elasticsearch.script.ScriptException; - import org.elasticsearch.common.Nullable; import org.elasticsearch.script.AbstractSearchScript; import org.elasticsearch.script.ExecutableScript; @@ -44,7 +42,7 @@ public class CosineSimilarityScoreScript extends AbstractSearchScript { // simple case, 1.0 for every term. List weights = null; - final static public String SCRIPT_NAME = "cosine_sim_script_score"; + public static final String SCRIPT_NAME = "cosine_sim_script_score"; /** * Factory that is registered in @@ -55,13 +53,13 @@ public static class Factory implements NativeScriptFactory { /** * This method is called for every search on every shard. - * + * * @param params * list of script parameters passed with the query * @return new native script */ @Override - public ExecutableScript newScript(@Nullable Map params) throws ScriptException { + public ExecutableScript newScript(@Nullable Map params) { return new CosineSimilarityScoreScript(params); } @@ -74,6 +72,11 @@ public ExecutableScript newScript(@Nullable Map params) throws S public boolean needsScores() { return false; } + + @Override + public String getName() { + return SCRIPT_NAME; + } } /** @@ -82,7 +85,7 @@ public boolean needsScores() { * them here. */ @SuppressWarnings("unchecked") - private CosineSimilarityScoreScript(Map params) throws ScriptException { + private CosineSimilarityScoreScript(Map params) { params.entrySet(); // get the terms terms = (List) params.get("terms"); @@ -90,10 +93,10 @@ private CosineSimilarityScoreScript(Map params) throws ScriptExc // get the field field = (String) params.get("field"); if (field == null || terms == null || weights == null) { - throw new ScriptException("cannot initialize " + SCRIPT_NAME + ": field, terms or weights parameter missing!"); + throw new IllegalArgumentException("cannot initialize " + SCRIPT_NAME + ": field, terms or weights parameter missing!"); } if (weights.size() != terms.size()) { - throw new ScriptException("cannot initialize " + SCRIPT_NAME + ": terms and weights array must have same length!"); + throw new IllegalArgumentException("cannot initialize " + SCRIPT_NAME + ": terms and weights array must have same length!"); } } @@ -121,7 +124,7 @@ public Object run() { } return score / (Math.sqrt(docWeightSum) * Math.sqrt(queryWeightSum)); } catch (IOException ex) { - throw new ScriptException("Could not compute cosine similarity: ", ex); + throw new IllegalStateException("Could not compute cosine similarity: ", ex); } } diff --git a/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java b/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java index 62e74383870d6..7c1573f3d5e7d 100644 --- a/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java +++ b/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/IsPrimeSearchScript.java @@ -17,7 +17,6 @@ import java.math.BigInteger; import java.util.Map; -import org.elasticsearch.script.ScriptException; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.fielddata.ScriptDocValues; @@ -53,7 +52,7 @@ public ExecutableScript newScript(@Nullable Map params) { // The XContentMapValues helper class can be used to simplify parameter parsing String fieldName = params == null ? null : XContentMapValues.nodeStringValue(params.get("field"), null); if (fieldName == null) { - throw new ScriptException("Missing the field parameter"); + throw new IllegalArgumentException("Missing the field parameter"); } // Example of an optional integer parameter @@ -71,6 +70,11 @@ public boolean needsScores() { return false; } + @Override + public String getName() { + return "is_prime"; + } + } private final String fieldName; diff --git a/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/LanguageModelScoreScript.java b/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/LanguageModelScoreScript.java index 0ece06bd2909f..1846039eb3ac8 100644 --- a/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/LanguageModelScoreScript.java +++ b/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/LanguageModelScoreScript.java @@ -18,13 +18,11 @@ import java.util.List; import java.util.Map; -import org.apache.lucene.search.Scorer; import org.elasticsearch.common.Nullable; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.script.AbstractSearchScript; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.NativeScriptFactory; -import org.elasticsearch.script.ScriptException; import org.elasticsearch.search.lookup.IndexField; import org.elasticsearch.search.lookup.IndexFieldTerm; @@ -47,7 +45,7 @@ public class LanguageModelScoreScript extends AbstractSearchScript { // lambda parameter float lambda; - final static public String SCRIPT_NAME = "language_model_script_score"; + public static final String SCRIPT_NAME = "language_model_script_score"; /** * Factory that is registered in @@ -58,7 +56,7 @@ public static class Factory implements NativeScriptFactory { /** * This method is called for every search on every shard. - * + * * @param params * list of script parameters passed with the query * @return new native script @@ -74,6 +72,11 @@ public boolean needsScores() { return false; } + @Override + public String getName() { + return SCRIPT_NAME; + } + } /** @@ -93,7 +96,7 @@ private LanguageModelScoreScript(Map params) { // get lambda lambda = ((Double) params.get("lambda")).floatValue(); if (field == null || terms == null || docLengthField == null) { - throw new ScriptException("cannot initialize " + SCRIPT_NAME + ": field, terms or length field parameter missing!"); + throw new IllegalArgumentException("cannot initialize " + SCRIPT_NAME + ": field, terms or length field parameter missing!"); } } @@ -144,11 +147,11 @@ public Object run() { } return score; } else { - throw new ScriptException("Could not compute language model score, word count field missing."); + throw new IllegalStateException("Could not compute language model score, word count field missing."); } } catch (IOException ex) { - throw new ScriptException("Could not compute language model score: ", ex); + throw new IllegalStateException("Could not compute language model score: ", ex); } } diff --git a/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java b/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java deleted file mode 100644 index 6549678e2a1ff..0000000000000 --- a/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/LookupScript.java +++ /dev/null @@ -1,171 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.elasticsearch.examples.nativescript.script; - -import org.elasticsearch.action.get.GetResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.cache.Cache; -import org.elasticsearch.common.cache.CacheBuilder; -import org.elasticsearch.common.collect.Tuple; -import org.elasticsearch.common.component.AbstractComponent; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.support.XContentMapValues; -import org.elasticsearch.index.fielddata.ScriptDocValues; -import org.elasticsearch.node.Node; -import org.elasticsearch.script.AbstractSearchScript; -import org.elasticsearch.script.ExecutableScript; -import org.elasticsearch.script.NativeScriptFactory; -import org.elasticsearch.script.ScriptException; - -import java.util.Collections; -import java.util.Map; -import java.util.concurrent.ExecutionException; - -/** - */ -public class LookupScript extends AbstractSearchScript { - - /** - * Native scripts are build using factories that are registered in the - * {@link org.elasticsearch.examples.nativescript.plugin.NativeScriptExamplesPlugin#onModule(org.elasticsearch.script.ScriptModule)} - * method when plugin is loaded. - */ - public static class Factory extends AbstractComponent implements NativeScriptFactory { - - private final Node node; - - private final Cache, Map> cache; - - /** - * This constructor will be called by guice during initialization - * - * @param node injecting the reference to current node to get access to node's client - * @param settings current node settings - */ - @SuppressWarnings("unchecked") - @Inject - public Factory(Node node, Settings settings) { - super(settings); - // Node is not fully initialized here - // All we can do is save a reference to it for future use - this.node = node; - - // Setup lookup cache - ByteSizeValue size = settings.getAsBytesSize("examples.nativescript.lookup.size", null); - TimeValue expire = settings.getAsTime("expire", null); - CacheBuilder, Map> cacheBuilder = CacheBuilder.builder(); - if (size != null) { - cacheBuilder.setMaximumWeight(size.bytes()); - } - if (expire != null) { - cacheBuilder.setExpireAfterAccess(expire.nanos()); - } - cache = cacheBuilder.build(); - } - - /** - * This method is called for every search on every shard. - * - * @param params list of script parameters passed with the query - * @return new native script - */ - @Override - public ExecutableScript newScript(@Nullable Map params) { - if (params == null) { - throw new ScriptException("Missing script parameters"); - } - String lookupIndex = XContentMapValues.nodeStringValue(params.get("lookup_index"), null); - if (lookupIndex == null) { - throw new ScriptException("Missing the index parameter"); - } - String lookupType = XContentMapValues.nodeStringValue(params.get("lookup_type"), null); - if (lookupType == null) { - throw new ScriptException("Missing the index parameter"); - } - String field = XContentMapValues.nodeStringValue(params.get("field"), null); - if (field == null) { - throw new ScriptException("Missing the field parameter"); - } - return new LookupScript(node.client(), logger, cache, lookupIndex, lookupType, field); - } - - /** - * Indicates if document scores may be needed by the produced scripts. - * - * @return {@code true} if scores are needed. - */ - @Override - public boolean needsScores() { - return false; - } - - } - - private final String lookupIndex; - private final String lookupType; - private final String field; - private final ESLogger logger; - - private final Client client; - private final Cache, Map> cache; - - private static final Map EMPTY_MAP = Collections.emptyMap(); - - private LookupScript(Client client, ESLogger logger, Cache, Map> cache, String lookupIndex, String lookupType, String field) { - this.client = client; - this.logger = logger; - this.lookupIndex = lookupIndex; - this.lookupType = lookupType; - this.field = field; - this.cache = cache; - } - - @Override - @SuppressWarnings("unchecked") - public Object run() { - // First we get field using doc lookup - ScriptDocValues docValue = (ScriptDocValues) doc().get(field); - // This is not very efficient - // Check if field exists - if (docValue != null && !docValue.isEmpty()) { - final String fieldValue = ((ScriptDocValues.Strings) docValue).getValue(); - if (fieldValue != null) { - try { - return cache.computeIfAbsent(new Tuple<>(lookupIndex + "/" + lookupType, fieldValue), key -> { - // This is not very efficient of doing this, but it demonstrates using injected client - // for record lookup - GetResponse response = client.prepareGet(lookupIndex, lookupType, fieldValue).setPreference("_local").execute().actionGet(); - if (logger.isTraceEnabled()) { - logger.trace("lookup [{}]/[{}]/[{}], found: [{}]", lookupIndex, lookupType, fieldValue, response.isExists()); - } - if (response.isExists()) { - return response.getSource(); - } - return EMPTY_MAP; - }); - - } catch (ExecutionException ex) { - throw new ScriptException("Lookup failure ", ex); - } - } - } - return null; - } -} diff --git a/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/PhraseScoreScript.java b/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/PhraseScoreScript.java index c821b83f26338..140d1a88dc3a6 100644 --- a/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/PhraseScoreScript.java +++ b/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/PhraseScoreScript.java @@ -22,7 +22,6 @@ import org.elasticsearch.script.AbstractSearchScript; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.NativeScriptFactory; -import org.elasticsearch.script.ScriptException; import org.elasticsearch.search.lookup.IndexField; import org.elasticsearch.search.lookup.IndexLookup; import org.elasticsearch.search.lookup.TermPosition; @@ -41,7 +40,7 @@ public class PhraseScoreScript extends AbstractSearchScript { // terms that are used for scoring List terms = null; - final static public String SCRIPT_NAME = "phrase_script_score"; + public static final String SCRIPT_NAME = "phrase_script_score"; /** * Factory that is registered in @@ -52,7 +51,7 @@ public static class Factory implements NativeScriptFactory { /** * This method is called for every search on every shard. - * + * * @param params * list of script parameters passed with the query * @return new native script @@ -71,6 +70,11 @@ public ExecutableScript newScript(@Nullable Map params) { public boolean needsScores() { return false; } + + @Override + public String getName() { + return SCRIPT_NAME; + } } /** @@ -86,7 +90,7 @@ private PhraseScoreScript(Map params) { // get the field field = (String) params.get("field"); if (field == null || terms == null) { - throw new ScriptException("cannot initialize " + SCRIPT_NAME + ": field or terms parameter missing!"); + throw new IllegalArgumentException("cannot initialize " + SCRIPT_NAME + ": field or terms parameter missing!"); } assert (terms.size() == 2); } diff --git a/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptFactory.java b/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptFactory.java index 26cd8bd344e1b..712cdb88263a1 100644 --- a/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptFactory.java +++ b/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptFactory.java @@ -18,10 +18,9 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.fielddata.ScriptDocValues; -import org.elasticsearch.script.AbstractFloatSearchScript; +import org.elasticsearch.script.AbstractDoubleSearchScript; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.NativeScriptFactory; -import org.elasticsearch.script.ScriptException; import java.io.IOException; import java.util.Map; @@ -37,7 +36,7 @@ public class PopularityScoreScriptFactory implements NativeScriptFactory { public ExecutableScript newScript(@Nullable Map params) { String fieldName = params == null ? null : XContentMapValues.nodeStringValue(params.get("field"), null); if (fieldName == null) { - throw new ScriptException("Missing the field parameter"); + throw new IllegalArgumentException("Missing the field parameter"); } return new PopularityScoreScript(fieldName); } @@ -52,12 +51,17 @@ public boolean needsScores() { return true; } + @Override + public String getName() { + return "popularity"; + } + /** * This script takes a numeric value from the field specified in the parameter field. And calculates boost * for the record using the following formula: 1 + log10(field_value + 1). So, records with value 0 in the field * get no boost. Records with value 9 gets boost of 2.0, records with value 99, gets boost of 3, 999 - 4 and so on. */ - private static class PopularityScoreScript extends AbstractFloatSearchScript { + private static class PopularityScoreScript extends AbstractDoubleSearchScript { private final String field; @@ -74,7 +78,7 @@ public void setScorer(Scorer scorer) { @Override @SuppressWarnings("unchecked") - public float runAsFloat() { + public double runAsDouble() { try { ScriptDocValues docValue = (ScriptDocValues) doc().get(field); if (docValue != null && !docValue.isEmpty()) { @@ -82,13 +86,18 @@ public float runAsFloat() { double boost = 1 + Math.log10(fieldData.getValue() + 1); // Because this script is used in custom_score script the value of score() is populated. // In all other cases doc().getScore() should be used instead. - return (float) boost * scorer.score(); + return boost * scorer.score(); } return scorer.score(); } catch (IOException ex) { - return 0.0f; + return 0.0; } } + + @Override + public Object run() { + return null; + } } } diff --git a/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java b/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java index 9f2c628e9c33a..c4d8b8aa3780f 100644 --- a/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java +++ b/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptFactory.java @@ -19,7 +19,6 @@ import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.mapper.internal.UidFieldMapper; -import org.elasticsearch.script.AbstractFloatSearchScript; import org.elasticsearch.script.AbstractLongSearchScript; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.NativeScriptFactory; @@ -32,7 +31,7 @@ /** * This script demonstrates how native scripts can be used to create custom sort order. - * Since sort operation is expecting float parameter, the {@link AbstractFloatSearchScript} can be used. + * Since sort operation is expecting float parameter, the {@link AbstractLongSearchScript} can be used. *

* The script accepts one optional parameter salt. If parameter is specified, a pseudo random sort order is used. * Otherwise, a random sort order is used. @@ -65,6 +64,11 @@ public boolean needsScores() { return false; } + @Override + public String getName() { + return "random"; + } + private static class RandomSortScript extends AbstractLongSearchScript { private final Random random; diff --git a/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/TFIDFScoreScript.java b/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/TFIDFScoreScript.java index de16ffa37ccf7..4f1c575fbd543 100644 --- a/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/TFIDFScoreScript.java +++ b/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/TFIDFScoreScript.java @@ -16,10 +16,8 @@ import java.io.IOException; import java.util.List; -import java.util.List; import java.util.Map; -import org.apache.lucene.search.Scorer; import org.elasticsearch.common.Nullable; import org.elasticsearch.script.AbstractSearchScript; import org.elasticsearch.script.ExecutableScript; @@ -42,7 +40,7 @@ public class TFIDFScoreScript extends AbstractSearchScript { // terms that are used for scoring List terms = null; - final static public String SCRIPT_NAME = "tfidf_script_score"; + public static final String SCRIPT_NAME = "tfidf_script_score"; /** * Factory that is registered in @@ -53,7 +51,7 @@ public static class Factory implements NativeScriptFactory { /** * This method is called for every search on every shard. - * + * * @param params * list of script parameters passed with the query * @return new native script @@ -72,6 +70,11 @@ public ExecutableScript newScript(@Nullable Map params) { public boolean needsScores() { return false; } + + @Override + public String getName() { + return SCRIPT_NAME; + } } /** @@ -87,7 +90,7 @@ private TFIDFScoreScript(Map params) { // get the field field = (String) params.get("field"); if (field == null || terms == null) { - throw new ScriptException("cannot initialize " + SCRIPT_NAME + ": field or terms parameter missing!"); + throw new IllegalArgumentException("cannot initialize " + SCRIPT_NAME + ": field or terms parameter missing!"); } } @@ -110,7 +113,7 @@ public Object run() { } return score; } catch (IOException ex) { - throw new ScriptException("Could not compute tfidf: ", ex); + throw new IllegalStateException("Could not compute tfidf: ", ex); } } diff --git a/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/CombineScriptFactory.java b/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/CombineScriptFactory.java index 7e588ffb816b5..8d15b4795bfa7 100644 --- a/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/CombineScriptFactory.java +++ b/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/CombineScriptFactory.java @@ -23,7 +23,8 @@ import java.util.Map; /** - * Combine script from https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-scripted-metric-aggregation.html + * Combine script from + * https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-scripted-metric-aggregation.html *

* profit = 0; for (t in _agg.transactions) { profit += t }; return profit */ @@ -31,7 +32,7 @@ public class CombineScriptFactory implements NativeScriptFactory { @Override @SuppressWarnings("unchecked") - public ExecutableScript newScript(final @Nullable Map params) { + public ExecutableScript newScript(@Nullable final Map params) { Map agg = (Map) params.get("_agg"); final ArrayList transactions = (ArrayList) agg.get(InitScriptFactory.TRANSACTIONS_FIELD); return new CombineScript(transactions); @@ -42,6 +43,11 @@ public boolean needsScores() { return false; } + @Override + public String getName() { + return "stockaggs_combine"; + } + private static class CombineScript extends AbstractExecutableScript { private final ArrayList transactions; diff --git a/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/InitScriptFactory.java b/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/InitScriptFactory.java index dbca240a2fcab..9754bc2096ac2 100644 --- a/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/InitScriptFactory.java +++ b/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/InitScriptFactory.java @@ -23,7 +23,8 @@ import java.util.Map; /** - * Init script from https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-scripted-metric-aggregation.html + * Init script from + * https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-scripted-metric-aggregation.html * * _agg['transactions'] = [] */ @@ -33,7 +34,7 @@ public class InitScriptFactory implements NativeScriptFactory { @Override @SuppressWarnings("unchecked") - public ExecutableScript newScript(final @Nullable Map params) { + public ExecutableScript newScript(@Nullable final Map params) { return new AbstractExecutableScript() { @Override public Object run() { @@ -47,4 +48,9 @@ public Object run() { public boolean needsScores() { return false; } + + @Override + public String getName() { + return "stockaggs_init"; + } } diff --git a/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/MapScriptFactory.java b/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/MapScriptFactory.java index fee4f402f6baa..9887a9899c203 100644 --- a/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/MapScriptFactory.java +++ b/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/MapScriptFactory.java @@ -24,7 +24,8 @@ import java.util.Map; /** - * Map script from https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-scripted-metric-aggregation.html + * Map script from + * https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-scripted-metric-aggregation.html *

* if (doc['type'].value == \"sale\") { _agg.transactions.add(doc['amount'].value) } else {_agg.transactions.add(-1 * doc['amount'].value)} */ @@ -32,7 +33,7 @@ public class MapScriptFactory implements NativeScriptFactory { @Override @SuppressWarnings("unchecked") - public ExecutableScript newScript(final @Nullable Map params) { + public ExecutableScript newScript(@Nullable final Map params) { Map agg = (Map) params.get("_agg"); ArrayList transactions = (ArrayList) agg.get(InitScriptFactory.TRANSACTIONS_FIELD); return new MapScript(transactions); @@ -43,6 +44,11 @@ public boolean needsScores() { return false; } + @Override + public String getName() { + return "stockaggs_map"; + } + private static class MapScript extends AbstractSearchScript { private final ArrayList transactions; diff --git a/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/ReduceScriptFactory.java b/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/ReduceScriptFactory.java index 186b3a09ce411..37ef7111d03af 100644 --- a/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/ReduceScriptFactory.java +++ b/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/stockaggs/ReduceScriptFactory.java @@ -23,7 +23,8 @@ import java.util.Map; /** - * Combine script from https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-scripted-metric-aggregation.html + * Combine script from + * https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-scripted-metric-aggregation.html *

* profit = 0; for (t in _agg.transactions) { profit += t }; return profit */ @@ -31,7 +32,7 @@ public class ReduceScriptFactory implements NativeScriptFactory { @Override @SuppressWarnings("unchecked") - public ExecutableScript newScript(final @Nullable Map params) { + public ExecutableScript newScript(@Nullable final Map params) { final ArrayList aggs = (ArrayList) params.get("_aggs"); return new ReduceScript(aggs); } @@ -41,6 +42,11 @@ public boolean needsScores() { return false; } + @Override + public String getName() { + return "stockaggs_reduce"; + } + private static class ReduceScript extends AbstractExecutableScript { private final ArrayList aggs; diff --git a/plugins/native-script-example/src/test/java/org/elasticsearch/examples/nativescript/ElasticsearchWithPluginLauncher.java b/plugins/native-script-example/src/test/java/org/elasticsearch/examples/nativescript/ElasticsearchWithPluginLauncher.java deleted file mode 100644 index 711412f10bb0f..0000000000000 --- a/plugins/native-script-example/src/test/java/org/elasticsearch/examples/nativescript/ElasticsearchWithPluginLauncher.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.elasticsearch.examples.nativescript; - -import org.elasticsearch.Version; -import org.elasticsearch.common.SuppressForbidden; -import org.elasticsearch.common.cli.Terminal; -import org.elasticsearch.common.logging.log4j.LogConfigurator; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.env.Environment; -import org.elasticsearch.examples.nativescript.plugin.NativeScriptExamplesPlugin; -import org.elasticsearch.node.MockNode; -import org.elasticsearch.node.Node; -import org.elasticsearch.node.internal.InternalSettingsPreparer; -import org.elasticsearch.plugins.Plugin; - -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.concurrent.CountDownLatch; - -/** - * Main class to easily run the the plugin from a IDE. - */ -public class ElasticsearchWithPluginLauncher { - - @SuppressForbidden(reason = "not really code or a test") - public static void main(String[] args) throws Throwable { - System.setProperty("es.logger.prefix", ""); - Settings settings = Settings.builder() - .put("security.manager.enabled", "false") - .put("plugins.load_classpath_plugins", "false") - .put("path.home", System.getProperty("es.path.home", System.getProperty("user.dir"))) - .build(); - - // Setup logging using config/logging.yml - Environment environment = InternalSettingsPreparer.prepareEnvironment(settings, Terminal.DEFAULT); - Class.forName("org.apache.log4j.Logger"); - LogConfigurator.configure(environment.settings(), true); - - final CountDownLatch latch = new CountDownLatch(1); - final Node node = new MockNode(settings, Version.CURRENT, Arrays.asList(NativeScriptExamplesPlugin.class)); - Runtime.getRuntime().addShutdownHook(new Thread() { - - @Override - public void run() { - node.close(); - latch.countDown(); - } - }); - node.start(); - latch.await(); - } -} diff --git a/plugins/native-script-example/src/test/java/org/elasticsearch/examples/nativescript/script/LookupScriptTests.java b/plugins/native-script-example/src/test/java/org/elasticsearch/examples/nativescript/script/LookupScriptTests.java deleted file mode 100644 index 2ca31c11071cf..0000000000000 --- a/plugins/native-script-example/src/test/java/org/elasticsearch/examples/nativescript/script/LookupScriptTests.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.elasticsearch.examples.nativescript.script; - -import static org.elasticsearch.index.query.QueryBuilders.matchQuery; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; -import static org.hamcrest.Matchers.equalTo; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -import org.elasticsearch.action.index.IndexRequestBuilder; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.common.collect.MapBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.script.Script; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.search.sort.SortOrder; - -/** - */ -public class LookupScriptTests extends AbstractSearchScriptTestCase { - - @SuppressWarnings("unchecked") - public void testLookup() throws Exception { - - // Create a new lookup index - String lookup_mapping = XContentFactory.jsonBuilder().startObject().startObject("state") - .startObject("properties") - .startObject("name").field("type", "string").endObject() - .startObject("capital").field("type", "string").endObject() - .startObject("nickname").field("type", "string").endObject() - .endObject().endObject().endObject() - .string(); - - assertAcked(prepareCreate("lookup") - .addMapping("state", lookup_mapping)); - - // Create a new test index - String test_mapping = XContentFactory.jsonBuilder().startObject().startObject("city") - .startObject("properties") - .startObject("city").field("type", "string").endObject() - .startObject("state").field("type", "string").field("index", "not_analyzed").endObject() - .startObject("population").field("type", "integer").endObject() - .endObject().endObject().endObject() - .string(); - - assertAcked(prepareCreate("test") - .addMapping("city", test_mapping)); - - List indexBuilders = new ArrayList(); - // Index Lookup records: - indexBuilders.add(client().prepareIndex("lookup", "state", "CT").setSource("name", "Connecticut", "capital", "Hartford", "nickname", "Constitution State")); - indexBuilders.add(client().prepareIndex("lookup", "state", "ME").setSource("name", "Maine", "capital", "Augusta", "nickname", "Lumber State")); - indexBuilders.add(client().prepareIndex("lookup", "state", "MA").setSource("name", "Massachusetts", "capital", "Boston", "nickname", "Bay State")); - indexBuilders.add(client().prepareIndex("lookup", "state", "NH").setSource("name", "New Hampshire", "capital", "Concord", "nickname", "Granite State")); - indexBuilders.add(client().prepareIndex("lookup", "state", "RI").setSource("name", "Rhode Island", "capital", "Providence", "nickname", "Little Rhody")); - indexBuilders.add(client().prepareIndex("lookup", "state", "VT").setSource("name", "Vermont", "capital", "Montpelier", "nickname", "Green Mountain State")); - - // Index main records - indexBuilders.add(client().prepareIndex("test", "city", "1").setSource("city", "Cambridge", "state", "MA", "population", 105162)); - indexBuilders.add(client().prepareIndex("test", "city", "2").setSource("city", "South Burlington", "state", "VT", "population", 17904)); - indexBuilders.add(client().prepareIndex("test", "city", "3").setSource("city", "South Portland", "state", "ME", "population", 25002)); - indexBuilders.add(client().prepareIndex("test", "city", "4").setSource("city", "Essex", "state", "VT", "population", 19587)); - indexBuilders.add(client().prepareIndex("test", "city", "5").setSource("city", "Portland", "state", "ME", "population", 66194)); - indexBuilders.add(client().prepareIndex("test", "city", "6").setSource("city", "Burlington", "state", "VT", "population", 42417)); - indexBuilders.add(client().prepareIndex("test", "city", "7").setSource("city", "Stamford", "state", "CT", "population", 122643)); - indexBuilders.add(client().prepareIndex("test", "city", "8").setSource("city", "Colchester", "state", "VT", "population", 17067)); - indexBuilders.add(client().prepareIndex("test", "city", "9").setSource("city", "Concord", "state", "NH", "population", 42695)); - indexBuilders.add(client().prepareIndex("test", "city", "10").setSource("city", "Boston", "state", "MA", "population", 617594)); - - indexRandom(true, indexBuilders); - - // Script parameters - Map params = MapBuilder.newMapBuilder() - .put("lookup_index", "lookup") - .put("lookup_type", "state") - .put("field", "state") - .map(); - - - // Find smallest city with word - SearchResponse searchResponse = client().prepareSearch("test") - .setTypes("city") - .setQuery(matchQuery("city", "south burlington")) - .setFetchSource(true) - .addScriptField("state_info", new Script("lookup", ScriptService.ScriptType.INLINE, "native", params)) - .setSize(10) - .addSort("population", SortOrder.DESC) - .execute().actionGet(); - - assertNoFailures(searchResponse); - - // There should be 3 cities - assertHitCount(searchResponse, 3); - - assertThat(searchResponse.getHits().getAt(0).getSource().get("city"), equalTo("Burlington")); - assertThat(((Map) searchResponse.getHits().getAt(0).field("state_info").getValue()).get("name").toString(), equalTo("Vermont")); - - assertThat(searchResponse.getHits().getAt(1).getSource().get("city"), equalTo("South Portland")); - assertThat(((Map) searchResponse.getHits().getAt(1).field("state_info").getValue()).get("name").toString(), equalTo("Maine")); - - assertThat(searchResponse.getHits().getAt(2).getSource().get("city"), equalTo("South Burlington")); - assertThat(((Map) searchResponse.getHits().getAt(2).field("state_info").getValue()).get("name").toString(), equalTo("Vermont")); - } - - -} diff --git a/plugins/native-script-example/src/test/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptTests.java b/plugins/native-script-example/src/test/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptTests.java index 5926e5b2e46ba..1a2d8d45c68dd 100644 --- a/plugins/native-script-example/src/test/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptTests.java +++ b/plugins/native-script-example/src/test/java/org/elasticsearch/examples/nativescript/script/PopularityScoreScriptTests.java @@ -80,7 +80,8 @@ public void testPopularityScoring() throws Exception { // Retrieve first 10 hits SearchResponse searchResponse = client().prepareSearch("test") .setQuery(QueryBuilders.functionScoreQuery(matchQuery("name", "rec"), new FunctionScoreQueryBuilder.FilterFunctionBuilder[]{ - new FunctionScoreQueryBuilder.FilterFunctionBuilder(ScoreFunctionBuilders.scriptFunction(new Script("popularity", ScriptService.ScriptType.INLINE, "native", params)))}) + new FunctionScoreQueryBuilder.FilterFunctionBuilder(ScoreFunctionBuilders.scriptFunction( + new Script("popularity", ScriptService.ScriptType.INLINE, "native", params)))}) .boostMode(CombineFunction.REPLACE)) .setSize(10) .setFetchSource("name", null) diff --git a/plugins/native-script-example/src/test/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptTests.java b/plugins/native-script-example/src/test/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptTests.java index 8ff0b2ce7cfec..76aae74f6fb97 100644 --- a/plugins/native-script-example/src/test/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptTests.java +++ b/plugins/native-script-example/src/test/java/org/elasticsearch/examples/nativescript/script/RandomSortScriptTests.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService; +import org.elasticsearch.search.sort.ScriptSortBuilder; import org.elasticsearch.search.sort.SortBuilders; /** @@ -65,7 +66,9 @@ public void testPseudoRandomScript() throws Exception { .setQuery(matchAllQuery()) .setFetchSource("name", null) .setSize(10) - .addSort(SortBuilders.scriptSort(new Script("random", ScriptService.ScriptType.INLINE, "native", MapBuilder.newMapBuilder().put("salt", "1234").map()), "number")) + .addSort(SortBuilders.scriptSort( + new Script("random", ScriptService.ScriptType.INLINE, "native", MapBuilder.newMapBuilder() + .put("salt", "1234").map()), ScriptSortBuilder.ScriptSortType.NUMBER)) .execute().actionGet(); assertNoFailures(searchResponse); @@ -85,7 +88,9 @@ public void testPseudoRandomScript() throws Exception { .setQuery(matchAllQuery()) .setFetchSource("name", null) .setSize(10) - .addSort(SortBuilders.scriptSort(new Script("random", ScriptService.ScriptType.INLINE, "native", MapBuilder.newMapBuilder().put("salt", "1234").map()), "number")) + .addSort(SortBuilders.scriptSort(new Script("random", ScriptService.ScriptType.INLINE, "native", MapBuilder.newMapBuilder() + .put("salt", "1234").map()), ScriptSortBuilder.ScriptSortType.NUMBER)) .execute().actionGet(); assertNoFailures(searchResponse); @@ -100,7 +105,8 @@ public void testPseudoRandomScript() throws Exception { .setQuery(matchAllQuery()) .setFetchSource("name", null) .setSize(10) - .addSort(SortBuilders.scriptSort(new Script("random", ScriptService.ScriptType.INLINE, "native", null), "number")) + .addSort(SortBuilders.scriptSort(new Script("random", ScriptService.ScriptType.INLINE, "native", null), + ScriptSortBuilder.ScriptSortType.NUMBER)) .execute().actionGet(); assertNoFailures(searchResponse); diff --git a/plugins/native-script-example/src/test/java/org/elasticsearch/examples/nativescript/script/TermScoringScriptTests.java b/plugins/native-script-example/src/test/java/org/elasticsearch/examples/nativescript/script/TermScoringScriptTests.java index c0192862b3dd9..25688eb27e96d 100644 --- a/plugins/native-script-example/src/test/java/org/elasticsearch/examples/nativescript/script/TermScoringScriptTests.java +++ b/plugins/native-script-example/src/test/java/org/elasticsearch/examples/nativescript/script/TermScoringScriptTests.java @@ -46,12 +46,12 @@ */ public class TermScoringScriptTests extends AbstractSearchScriptTestCase { - final static List searchTerms = Arrays.asList("foo", "bar"); - final static String field = "field"; - final static String wordCountField = field + ".word_count"; - final static String placeholder = "placeholder"; - final static List weights = Arrays.asList(1.0, 1.0); - final static int numDocs = 100; + static final List searchTerms = Arrays.asList("foo", "bar"); + static final String field = "field"; + static final String wordCountField = field + ".word_count"; + static final String placeholder = "placeholder"; + static final List weights = Arrays.asList(1.0, 1.0); + static final int numDocs = 100; public void testNoOfShardsIs1() { assertAcked(prepareCreate("test").get()); @@ -71,7 +71,8 @@ public void testTFIDF() throws Exception { SearchResponse searchResponse = client() .prepareSearch("test") .setQuery(QueryBuilders.functionScoreQuery(QueryBuilders.matchAllQuery(), new FunctionScoreQueryBuilder.FilterFunctionBuilder[]{ - new FunctionScoreQueryBuilder.FilterFunctionBuilder(ScoreFunctionBuilders.scriptFunction(new Script(TFIDFScoreScript.SCRIPT_NAME, ScriptService.ScriptType.INLINE, "native", params)))}) + new FunctionScoreQueryBuilder.FilterFunctionBuilder(ScoreFunctionBuilders.scriptFunction( + new Script(TFIDFScoreScript.SCRIPT_NAME, ScriptService.ScriptType.INLINE, "native", params)))}) .boostMode(CombineFunction.REPLACE)).setSize(numDocs).execute().actionGet(); assertNoFailures(searchResponse); assertHitCount(searchResponse, numDocs); @@ -95,7 +96,8 @@ public void testCosineSimilarity() throws Exception { SearchResponse searchResponse = client() .prepareSearch("test") .setQuery(QueryBuilders.functionScoreQuery(QueryBuilders.matchAllQuery(), new FunctionScoreQueryBuilder.FilterFunctionBuilder[]{ - new FunctionScoreQueryBuilder.FilterFunctionBuilder(ScoreFunctionBuilders.scriptFunction(new Script(CosineSimilarityScoreScript.SCRIPT_NAME, ScriptService.ScriptType.INLINE, "native", params)))}) + new FunctionScoreQueryBuilder.FilterFunctionBuilder(ScoreFunctionBuilders.scriptFunction( + new Script(CosineSimilarityScoreScript.SCRIPT_NAME, ScriptService.ScriptType.INLINE, "native", params)))}) .boostMode(CombineFunction.REPLACE)).setSize(numDocs).execute().actionGet(); assertNoFailures(searchResponse); assertHitCount(searchResponse, numDocs); @@ -145,7 +147,8 @@ public void testLanguageModelScorer() throws Exception { SearchResponse searchResponse = client() .prepareSearch("test") .setQuery(QueryBuilders.functionScoreQuery(QueryBuilders.matchAllQuery(), new FunctionScoreQueryBuilder.FilterFunctionBuilder[]{ - new FunctionScoreQueryBuilder.FilterFunctionBuilder(ScoreFunctionBuilders.scriptFunction(new Script(LanguageModelScoreScript.SCRIPT_NAME, ScriptService.ScriptType.INLINE, "native", params)))}) + new FunctionScoreQueryBuilder.FilterFunctionBuilder(ScoreFunctionBuilders.scriptFunction( + new Script(LanguageModelScoreScript.SCRIPT_NAME, ScriptService.ScriptType.INLINE, "native", params)))}) .boostMode(CombineFunction.REPLACE)).setSize(numDocs).execute().actionGet(); assertNoFailures(searchResponse); assertHitCount(searchResponse, numDocs); @@ -160,7 +163,9 @@ private void initData() throws IOException, InterruptedException, ExecutionExcep String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") .startObject(field).field("type", "string") .startObject("fields") - .startObject("word_count").field("type", "string").field("analyzer", "standard").field("type", "token_count").startObject("fielddata").field("format", "doc_values").endObject().endObject() + .startObject("word_count") + .field("type", "string").field("analyzer", "standard").field("type", "token_count") + .endObject() .endObject() .endObject().endObject().endObject().endObject().string(); assertAcked(prepareCreate("test").addMapping("type", mapping)); diff --git a/plugins/native-script-example/src/test/resources/log4j.properties b/plugins/native-script-example/src/test/resources/log4j.properties deleted file mode 100644 index 497c97f995974..0000000000000 --- a/plugins/native-script-example/src/test/resources/log4j.properties +++ /dev/null @@ -1,5 +0,0 @@ -log4j.rootLogger=INFO, out - -log4j.appender.out=org.apache.log4j.ConsoleAppender -log4j.appender.out.layout=org.apache.log4j.PatternLayout -log4j.appender.out.layout.conversionPattern=[%d{ISO8601}][%-5p][%-25c] %m%n diff --git a/plugins/native-script-example/src/test/resources/rest-api-spec/test/nativescript/10_primes.yaml b/plugins/native-script-example/src/test/resources/rest-api-spec/test/nativescript/10_primes.yaml index 61944a1f0223a..858d11f111f22 100644 --- a/plugins/native-script-example/src/test/resources/rest-api-spec/test/nativescript/10_primes.yaml +++ b/plugins/native-script-example/src/test/resources/rest-api-spec/test/nativescript/10_primes.yaml @@ -8,10 +8,12 @@ setup: settings: index.number_of_shards: 1 index.number_of_replicas: 0 - mappings.type1: - properties: - name.type: string - number.type: integer + mappings: + type1: + properties: + name: {type: text} + number: {type: integer} + - do: index: {index: test_index, type: type1, id: 1, body: {name: "rec 1", number: 1}} diff --git a/plugins/native-script-example/src/test/resources/rest-api-spec/test/nativescript/20_aggs.yaml b/plugins/native-script-example/src/test/resources/rest-api-spec/test/nativescript/20_aggs.yaml index 2a3e947fa6a1b..a2b79101af73e 100644 --- a/plugins/native-script-example/src/test/resources/rest-api-spec/test/nativescript/20_aggs.yaml +++ b/plugins/native-script-example/src/test/resources/rest-api-spec/test/nativescript/20_aggs.yaml @@ -3,15 +3,17 @@ setup: - do: indices.create: - index: test_index + index: transactions body: settings: index.number_of_shards: 1 index.number_of_replicas: 0 - mappings.type1: - properties: - type: {type: string, index: not_analyzed} - amount.type: long + mappings: + stock: + properties: + type: {type: keyword} + amount: {type: long} + - do: index: {index: transactions, type: stock, id: 1, body: {type: "sale", amount: 80}} @@ -33,6 +35,7 @@ setup: "Primes Test": - do: search: + index: transactions body: aggs: profit: diff --git a/settings.gradle b/settings.gradle index 3896082d9d8b4..64cb4551f0323 100644 --- a/settings.gradle +++ b/settings.gradle @@ -42,6 +42,7 @@ List projects = [ 'plugins:mapper-attachments', 'plugins:mapper-murmur3', 'plugins:mapper-size', + 'plugins:native-script-example', 'plugins:repository-azure', 'plugins:repository-gcs', 'plugins:repository-hdfs', From 3c7ea4c6c6092c4483122c58ec0749676c8dda87 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Mon, 18 Jul 2016 11:10:42 -0400 Subject: [PATCH 69/69] Remove all scripts that demonstrate use of indexLookup() --- .../example/termscoring.sh | 262 ------------------ .../plugin/NativeScriptExamplesPlugin.java | 8 +- .../script/CosineSimilarityScoreScript.java | 131 --------- .../script/LanguageModelScoreScript.java | 158 ----------- .../script/PhraseScoreScript.java | 117 -------- .../nativescript/script/TFIDFScoreScript.java | 120 -------- .../script/TermScoringScriptTests.java | 193 ------------- 7 files changed, 1 insertion(+), 988 deletions(-) delete mode 100755 plugins/native-script-example/example/termscoring.sh delete mode 100644 plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/CosineSimilarityScoreScript.java delete mode 100644 plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/LanguageModelScoreScript.java delete mode 100644 plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/PhraseScoreScript.java delete mode 100644 plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/TFIDFScoreScript.java delete mode 100644 plugins/native-script-example/src/test/java/org/elasticsearch/examples/nativescript/script/TermScoringScriptTests.java diff --git a/plugins/native-script-example/example/termscoring.sh b/plugins/native-script-example/example/termscoring.sh deleted file mode 100755 index 482336324a23c..0000000000000 --- a/plugins/native-script-example/example/termscoring.sh +++ /dev/null @@ -1,262 +0,0 @@ -#!/bin/sh - -# Init an index that includes wordcount for the field that we will be searching on: - -curl -s -XPUT "http://localhost:9200/termscore" -d' -{ - "settings": { - "number_of_shards": 1, - "number_of_replicas": 0 - }, - "mappings": { - "doc": { - "properties": { - "text": { - "type": "multi_field", - "fields": { - "text": { - "type": "string" - }, - "word_count": { - "type": "token_count", - "store": "yes", - "analyzer": "standard" - } - } - } - } - } - } -}' - -curl -s -XPOST "http://localhost:9200/termscore/doc" -d' -{ - "text": "John Smith is the main villain in the Matrix." -}' - -curl -s -XPOST "http://localhost:9200/termscore/doc" -d' -{ - "text": "John \"Hannibal\" Smith is the leader of the A-team." -}' - -curl -s -XPOST "http://localhost:9200/termscore/_refresh" - -# search for people that are named John Smith: - -curl -s -XPOST "http://localhost:9200/termscore/doc/_search?pretty" -d' -{ - "query": { - "function_score": { - "query": { - "bool": { - "must": [ - { - "match": { - "text": { - "query": "john smith", - "operator": "and" - } - } - } - ] - } - }, - "functions": [ - { - "script_score": { - "params": { - "field": "text", - "terms": [ - "john", - "smith" - ] - }, - "script": "phrase_script_score", - "lang": "native" - } - } - ], - "boost_mode": "replace" - } - } -}' - -curl -s -XPOST "http://localhost:9200/termscore/doc" -d' -{ - "text": "I am Sam. I am Sam. Sam I am. That Sam-I-am. That Sam-I-am! I do not like that Sam-I-am. Do you like green eggs and ham? I do not like them, Sam-I-am.I do not like green eggs and ham." -}' - -curl -s -XPOST "http://localhost:9200/termscore/doc" -d' -{ - "text": "Would you like them Here or there? I would not like them here or there. I would not like them anywhere. I do not like green eggs and ham. I do not like them, Sam-I-am." -}' - -curl -s -XPOST "http://localhost:9200/termscore/doc" -d' -{ - "text": "Would you like them in a house? Would you like them with a mouse? I do not like them in a house. I do not like them with a mouse. I do not like them here or there. I do not like them anywhere. I do not like green eggs and ham. I do not like them, Sam-I-am." -}' - -curl -s -XPOST "http://localhost:9200/termscore/_refresh" - -# Search for "I am Sam" with cosine similarity: - -curl -s -XPOST "http://localhost:9200/termscore/doc/_search?pretty" -d' -{ - "script_fields": { - "i-tf": { - "script": "_index[\"text\"][\"i\"].tf()" - }, - "am-tf": { - "script": "_index[\"text\"][\"am\"].tf()" - }, - "sam-tf": { - "script": "_index[\"text\"][\"sam\"].tf()" - } - }, - "query": { - "function_score": { - "query": { - "bool": { - "must": [ - { - "match": { - "text": { - "query": "sam i am", - "operator": "and" - } - } - } - ] - } - }, - "functions": [ - { - "script_score": { - "params": { - "field": "text", - "terms": [ - "sam", - "i", - "am" - ], - "weights": [ - 1.0, - 1.0, - 1.0 - ] - }, - "script": "cosine_sim_script_score", - "lang": "native" - } - } - ], - "boost_mode": "replace" - } - } -}' - -# Search for "I am Sam" with naive tf-ifd score: - -curl -s -XPOST "http://localhost:9200/termscore/doc/_search?pretty" -d' -{ - "script_fields": { - "i-tf": { - "script": "_index[\"text\"][\"i\"].tf()" - }, - "am-tf": { - "script": "_index[\"text\"][\"am\"].tf()" - }, - "sam-tf": { - "script": "_index[\"text\"][\"sam\"].tf()" - } - }, - "query": { - "function_score": { - "query": { - "bool": { - "must": [ - { - "match": { - "text": { - "query": "sam i am", - "operator": "and" - } - } - } - ] - } - }, - "functions": [ - { - "script_score": { - "params": { - "field": "text", - "terms": [ - "sam", - "i", - "am" - ] - }, - "script": "tfidf_script_score", - "lang": "native" - } - } - ], - "boost_mode": "replace" - } - } -}' - -# Search for "I am Sam" with language model scoring: - -curl -s -XPOST "http://localhost:9200/termscore/doc/_search?pretty" -d' -{ - "script_fields": { - "i-tf": { - "script": "_index[\"text\"][\"i\"].tf()" - }, - "am-tf": { - "script": "_index[\"text\"][\"am\"].tf()" - }, - "sam-tf": { - "script": "_index[\"text\"][\"sam\"].tf()" - } - }, - "query": { - "function_score": { - "query": { - "bool": { - "must": [ - { - "match": { - "text": { - "query": "sam i am", - "operator": "and" - } - } - } - ] - } - }, - "functions": [ - { - "script_score": { - "params": { - "field": "text", - "terms": [ - "sam", - "i", - "am" - ], - "lambda": 0.5, - "word_count_field": "text.word_count" - }, - "script": "language_model_script_score", - "lang": "native" - } - } - ], - "boost_mode": "replace" - } - } -}' \ No newline at end of file diff --git a/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java b/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java index c03cb28f595b8..6bc29dfea4734 100644 --- a/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java +++ b/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/plugin/NativeScriptExamplesPlugin.java @@ -15,10 +15,6 @@ package org.elasticsearch.examples.nativescript.plugin; import org.elasticsearch.examples.nativescript.script.IsPrimeSearchScript; -import org.elasticsearch.examples.nativescript.script.LanguageModelScoreScript; -import org.elasticsearch.examples.nativescript.script.CosineSimilarityScoreScript; -import org.elasticsearch.examples.nativescript.script.PhraseScoreScript; -import org.elasticsearch.examples.nativescript.script.TFIDFScoreScript; import org.elasticsearch.examples.nativescript.script.PopularityScoreScriptFactory; import org.elasticsearch.examples.nativescript.script.RandomSortScriptFactory; import org.elasticsearch.examples.nativescript.script.stockaggs.CombineScriptFactory; @@ -40,9 +36,7 @@ public class NativeScriptExamplesPlugin extends Plugin implements ScriptPlugin { @Override public List getNativeScripts() { - return Arrays.asList(new IsPrimeSearchScript.Factory(), new RandomSortScriptFactory(), - new PopularityScoreScriptFactory(), new TFIDFScoreScript.Factory(), new CosineSimilarityScoreScript.Factory(), - new PhraseScoreScript.Factory(), new LanguageModelScoreScript.Factory(), + return Arrays.asList(new IsPrimeSearchScript.Factory(), new RandomSortScriptFactory(), new PopularityScoreScriptFactory(), // Scripted Metric Aggregations Scripts new InitScriptFactory(), new MapScriptFactory(), new CombineScriptFactory(), new ReduceScriptFactory()); } diff --git a/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/CosineSimilarityScoreScript.java b/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/CosineSimilarityScoreScript.java deleted file mode 100644 index 831bc3d916628..0000000000000 --- a/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/CosineSimilarityScoreScript.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.elasticsearch.examples.nativescript.script; - -import java.io.IOException; -import java.util.List; -import java.util.Map; - -import org.elasticsearch.common.Nullable; -import org.elasticsearch.script.AbstractSearchScript; -import org.elasticsearch.script.ExecutableScript; -import org.elasticsearch.script.NativeScriptFactory; -import org.elasticsearch.search.lookup.IndexField; -import org.elasticsearch.search.lookup.IndexFieldTerm; - -/** - * Script that scores documents with cosine similarity, see Manning et al., - * "Information Retrieval", Chapter 6, Eq. 6.12 (link: - * http://nlp.stanford.edu/IR-book/). This implementation only scores a list of - * terms on one field. - */ -public class CosineSimilarityScoreScript extends AbstractSearchScript { - - // the field containing the terms that should be scored, must be initialized - // in constructor from parameters. - String field = null; - // terms that are used for scoring, must be unique - List terms = null; - // weights, in case we want to put emphasis on a specific term. In the most - // simple case, 1.0 for every term. - List weights = null; - - public static final String SCRIPT_NAME = "cosine_sim_script_score"; - - /** - * Factory that is registered in - * {@link org.elasticsearch.examples.nativescript.plugin.NativeScriptExamplesPlugin#onModule(org.elasticsearch.script.ScriptModule)} - * method when the plugin is loaded. - */ - public static class Factory implements NativeScriptFactory { - - /** - * This method is called for every search on every shard. - * - * @param params - * list of script parameters passed with the query - * @return new native script - */ - @Override - public ExecutableScript newScript(@Nullable Map params) { - return new CosineSimilarityScoreScript(params); - } - - /** - * Indicates if document scores may be needed by the produced scripts. - * - * @return {@code true} if scores are needed. - */ - @Override - public boolean needsScores() { - return false; - } - - @Override - public String getName() { - return SCRIPT_NAME; - } - } - - /** - * @param params - * terms that a scored are placed in this parameter. Initialize - * them here. - */ - @SuppressWarnings("unchecked") - private CosineSimilarityScoreScript(Map params) { - params.entrySet(); - // get the terms - terms = (List) params.get("terms"); - weights = (List) params.get("weights"); - // get the field - field = (String) params.get("field"); - if (field == null || terms == null || weights == null) { - throw new IllegalArgumentException("cannot initialize " + SCRIPT_NAME + ": field, terms or weights parameter missing!"); - } - if (weights.size() != terms.size()) { - throw new IllegalArgumentException("cannot initialize " + SCRIPT_NAME + ": terms and weights array must have same length!"); - } - } - - @Override - public Object run() { - try { - float score = 0; - // first, get the ShardTerms object for the field. - IndexField indexField = this.indexLookup().get(field); - double queryWeightSum = 0.0f; - double docWeightSum = 0.0f; - for (int i = 0; i < terms.size(); i++) { - // Now, get the ShardTerm object that can be used to access all - // the term statistics - IndexFieldTerm indexTermField = indexField.get(terms.get(i)); - // compute the most naive tfidf and add to current score - int df = (int) indexTermField.df(); - int tf = indexTermField.tf(); - if (df != 0 && tf != 0) { - double termscore = (double) tf * weights.get(i); - score += termscore; - docWeightSum += Math.pow(tf, 2.0); - } - queryWeightSum += Math.pow(weights.get(i), 2.0); - } - return score / (Math.sqrt(docWeightSum) * Math.sqrt(queryWeightSum)); - } catch (IOException ex) { - throw new IllegalStateException("Could not compute cosine similarity: ", ex); - } - } - -} diff --git a/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/LanguageModelScoreScript.java b/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/LanguageModelScoreScript.java deleted file mode 100644 index 1846039eb3ac8..0000000000000 --- a/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/LanguageModelScoreScript.java +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.elasticsearch.examples.nativescript.script; - -import java.io.IOException; -import java.util.List; -import java.util.Map; - -import org.elasticsearch.common.Nullable; -import org.elasticsearch.index.fielddata.ScriptDocValues; -import org.elasticsearch.script.AbstractSearchScript; -import org.elasticsearch.script.ExecutableScript; -import org.elasticsearch.script.NativeScriptFactory; -import org.elasticsearch.search.lookup.IndexField; -import org.elasticsearch.search.lookup.IndexFieldTerm; - -/** - * Script that scores documents with a language model similarity with linear - * interpolation, see Manning et al., "Information Retrieval", Chapter 12, - * Equation 12.12 (link: http://nlp.stanford.edu/IR-book/) This implementation - * only scores a list of terms on one field. - */ -public class LanguageModelScoreScript extends AbstractSearchScript { - - // the field containing the terms that should be scored, must be initialized - // in constructor from parameters. - String field; - // name of the field that holds the word count of a field, see - // http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/mapping-core-types.html) - String docLengthField; - // terms that are used for scoring - List terms; - // lambda parameter - float lambda; - - public static final String SCRIPT_NAME = "language_model_script_score"; - - /** - * Factory that is registered in - * {@link org.elasticsearch.examples.nativescript.plugin.NativeScriptExamplesPlugin#onModule(org.elasticsearch.script.ScriptModule)} - * method when the plugin is loaded. - */ - public static class Factory implements NativeScriptFactory { - - /** - * This method is called for every search on every shard. - * - * @param params - * list of script parameters passed with the query - * @return new native script - */ - @Override - public ExecutableScript newScript(@Nullable Map params) { - return new LanguageModelScoreScript(params); - } - - - @Override - public boolean needsScores() { - return false; - } - - @Override - public String getName() { - return SCRIPT_NAME; - } - - } - - /** - * @param params - * terms that a scored are placed in this parameter. Initialize - * them here. - */ - @SuppressWarnings("unchecked") - private LanguageModelScoreScript(Map params) { - params.entrySet(); - // get the terms - terms = (List) params.get("terms"); - // get the field - field = (String) params.get("field"); - // get the field holding the document length - docLengthField = (String) params.get("word_count_field"); - // get lambda - lambda = ((Double) params.get("lambda")).floatValue(); - if (field == null || terms == null || docLengthField == null) { - throw new IllegalArgumentException("cannot initialize " + SCRIPT_NAME + ": field, terms or length field parameter missing!"); - } - } - - @Override - @SuppressWarnings("unchecked") - public Object run() { - try { - double score = 0.0; - // first, get the ShardTerms object for the field. - IndexField indexField = indexLookup().get(field); - long T = indexField.sumttf(); - /* - * document length cannot be obtained by the shardTerms, we use the - * word_count field instead (link: - * http://www.elasticsearch.org/guide - * /en/elasticsearch/reference/current/mapping-core-types.html) - */ - ScriptDocValues docValues = (ScriptDocValues) doc().get(docLengthField); - if (docValues == null || !docValues.isEmpty()) { - long L_d = ((ScriptDocValues.Longs) docValues).getValue(); - for (int i = 0; i < terms.size(); i++) { - // Now, get the ShardTerm object that can be used to access - // all - // the term statistics - IndexFieldTerm indexFieldTerm = indexField.get(terms.get(i)); - - /* - * compute M_c as ttf/T, see Manning et al., - * "Information Retrieval", Chapter 12, Equation just before - * Equation 12.10 (link: http://nlp.stanford.edu/IR-book/) - */ - long cf_t = indexFieldTerm.ttf(); - double M_c = (double) cf_t / (double) T; - /* - * Compute M_d, see Manning et al., "Information Retrieval", - * Chapter 12, Equation just before Equation 12.9 (link: - * http://nlp.stanford.edu/IR-book/) - */ - double M_d = (double) indexFieldTerm.tf() / (double) L_d; - /* - * compute score contribution for this term, but sum the log - * to avoid underflow, see Manning et al., - * "Information Retrieval", Chapter 12, Equation 12.12 - * (link: http://nlp.stanford.edu/IR-book/) - */ - score += Math.log((1.0 - lambda) * M_c + lambda * M_d); - - } - return score; - } else { - throw new IllegalStateException("Could not compute language model score, word count field missing."); - } - - } catch (IOException ex) { - throw new IllegalStateException("Could not compute language model score: ", ex); - } - } - -} diff --git a/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/PhraseScoreScript.java b/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/PhraseScoreScript.java deleted file mode 100644 index 140d1a88dc3a6..0000000000000 --- a/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/PhraseScoreScript.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.elasticsearch.examples.nativescript.script; - -import java.util.List; -import java.util.Iterator; -import java.util.Map; - -import org.elasticsearch.common.Nullable; -import org.elasticsearch.script.AbstractSearchScript; -import org.elasticsearch.script.ExecutableScript; -import org.elasticsearch.script.NativeScriptFactory; -import org.elasticsearch.search.lookup.IndexField; -import org.elasticsearch.search.lookup.IndexLookup; -import org.elasticsearch.search.lookup.TermPosition; - -/** - * Script that scores documents by distance of two given terms in a text, see - * Manning et al., "Information Retrieval", Chapter 2.4 (link: - * http://nlp.stanford.edu/IR-book/) for more information on positional indexes. - * Might be useful if you search for names and know first and last name. - */ -public class PhraseScoreScript extends AbstractSearchScript { - - // the field containing the terms that should be scored, must be initialized - // in constructor from parameters. - String field = null; - // terms that are used for scoring - List terms = null; - - public static final String SCRIPT_NAME = "phrase_script_score"; - - /** - * Factory that is registered in - * {@link org.elasticsearch.examples.nativescript.plugin.NativeScriptExamplesPlugin#onModule(org.elasticsearch.script.ScriptModule)} - * method when the plugin is loaded. - */ - public static class Factory implements NativeScriptFactory { - - /** - * This method is called for every search on every shard. - * - * @param params - * list of script parameters passed with the query - * @return new native script - */ - @Override - public ExecutableScript newScript(@Nullable Map params) { - return new PhraseScoreScript(params); - } - - /** - * Indicates if document scores may be needed by the produced scripts. - * - * @return {@code true} if scores are needed. - */ - @Override - public boolean needsScores() { - return false; - } - - @Override - public String getName() { - return SCRIPT_NAME; - } - } - - /** - * @param params - * terms that a scored are placed in this parameter. Initialize - * them here. - */ - @SuppressWarnings("unchecked") - private PhraseScoreScript(Map params) { - params.entrySet(); - // get the terms - terms = (List) params.get("terms"); - // get the field - field = (String) params.get("field"); - if (field == null || terms == null) { - throw new IllegalArgumentException("cannot initialize " + SCRIPT_NAME + ": field or terms parameter missing!"); - } - assert (terms.size() == 2); - } - - @Override - public Object run() { - double score = 1.e10; - // first, get the ShardTerms object for the field. - IndexField indexField = this.indexLookup().get(field); - // get the positions iterators - Iterator firstNameIter = indexField.get(terms.get(0), IndexLookup.FLAG_POSITIONS).iterator(); - Iterator lastNameIter = indexField.get(terms.get(1), IndexLookup.FLAG_POSITIONS).iterator(); - int lastNamePos = -1; - while (firstNameIter.hasNext()) { - int firstNamePos = firstNameIter.next().position; - while (lastNameIter.hasNext() && lastNamePos < firstNamePos) { - lastNamePos = lastNameIter.next().position; - } - score = Math.min(score, lastNamePos - firstNamePos); - } - return 1.0 / (float) score; - } - -} diff --git a/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/TFIDFScoreScript.java b/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/TFIDFScoreScript.java deleted file mode 100644 index 4f1c575fbd543..0000000000000 --- a/plugins/native-script-example/src/main/java/org/elasticsearch/examples/nativescript/script/TFIDFScoreScript.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.elasticsearch.examples.nativescript.script; - -import java.io.IOException; -import java.util.List; -import java.util.Map; - -import org.elasticsearch.common.Nullable; -import org.elasticsearch.script.AbstractSearchScript; -import org.elasticsearch.script.ExecutableScript; -import org.elasticsearch.script.NativeScriptFactory; -import org.elasticsearch.script.ScriptException; -import org.elasticsearch.search.lookup.IndexField; -import org.elasticsearch.search.lookup.IndexFieldTerm; - -/** - * Script that scores documents as sum_t(tf_t * (#docs+2)/(df_t+1)), which - * equals ntn in SMART notation, see Manning et al., "Information Retrieval", - * Chapter 6, Figure 6.15 (link: http://nlp.stanford.edu/IR-book/) This - * implementation only scores a list of terms on one field. - */ -public class TFIDFScoreScript extends AbstractSearchScript { - - // the field containing the terms that should be scored, must be initialized - // in constructor from parameters. - String field = null; - // terms that are used for scoring - List terms = null; - - public static final String SCRIPT_NAME = "tfidf_script_score"; - - /** - * Factory that is registered in - * {@link org.elasticsearch.examples.nativescript.plugin.NativeScriptExamplesPlugin#onModule(org.elasticsearch.script.ScriptModule)} - * method when the plugin is loaded. - */ - public static class Factory implements NativeScriptFactory { - - /** - * This method is called for every search on every shard. - * - * @param params - * list of script parameters passed with the query - * @return new native script - */ - @Override - public ExecutableScript newScript(@Nullable Map params) { - return new TFIDFScoreScript(params); - } - - /** - * Indicates if document scores may be needed by the produced scripts. - * - * @return {@code true} if scores are needed. - */ - @Override - public boolean needsScores() { - return false; - } - - @Override - public String getName() { - return SCRIPT_NAME; - } - } - - /** - * @param params - * terms that a scored are placed in this parameter. Initialize - * them here. - */ - @SuppressWarnings("unchecked") - private TFIDFScoreScript(Map params) { - params.entrySet(); - // get the terms - terms = (List) params.get("terms"); - // get the field - field = (String) params.get("field"); - if (field == null || terms == null) { - throw new IllegalArgumentException("cannot initialize " + SCRIPT_NAME + ": field or terms parameter missing!"); - } - } - - @Override - public Object run() { - try { - float score = 0; - // first, get the IndexField object for the field. - IndexField indexField = indexLookup().get(field); - for (int i = 0; i < terms.size(); i++) { - // Now, get the IndexFieldTerm object that can be used to access all - // the term statistics - IndexFieldTerm indexFieldTerm = indexField.get(terms.get(i)); - // compute the most naive tfidf and add to current score - int df = (int) indexFieldTerm.df(); - int tf = indexFieldTerm.tf(); - if (df != 0 && tf != 0) { - score += (float) indexFieldTerm.tf() * Math.log(((float) indexField.docCount() + 2.0) / ((float) df + 1.0)); - } - } - return score; - } catch (IOException ex) { - throw new IllegalStateException("Could not compute tfidf: ", ex); - } - } - -} diff --git a/plugins/native-script-example/src/test/java/org/elasticsearch/examples/nativescript/script/TermScoringScriptTests.java b/plugins/native-script-example/src/test/java/org/elasticsearch/examples/nativescript/script/TermScoringScriptTests.java deleted file mode 100644 index 25688eb27e96d..0000000000000 --- a/plugins/native-script-example/src/test/java/org/elasticsearch/examples/nativescript/script/TermScoringScriptTests.java +++ /dev/null @@ -1,193 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.elasticsearch.examples.nativescript.script; - -import org.elasticsearch.action.index.IndexRequestBuilder; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.common.lucene.search.function.CombineFunction; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder; -import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders; -import org.elasticsearch.script.Script; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.search.SearchHit; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ExecutionException; - -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; -import static org.hamcrest.Matchers.closeTo; -import static org.hamcrest.Matchers.equalTo; - -/** - * Test if the computed tfidf in NaiveTFIDFScoreScript equals 0.0 for each - * document as would be expected if each document in the index contains only one - * and always the same term. - */ -public class TermScoringScriptTests extends AbstractSearchScriptTestCase { - - static final List searchTerms = Arrays.asList("foo", "bar"); - static final String field = "field"; - static final String wordCountField = field + ".word_count"; - static final String placeholder = "placeholder"; - static final List weights = Arrays.asList(1.0, 1.0); - static final int numDocs = 100; - - public void testNoOfShardsIs1() { - assertAcked(prepareCreate("test").get()); - assertThat(client().admin().indices().prepareGetSettings("test").get().getSetting("test", "index.number_of_shards"), equalTo("1")); - } - - public void testTFIDF() throws Exception { - - initData(); - - // initialize parameters for script - Map params = new HashMap<>(); - params.put("field", field); - params.put("terms", searchTerms); - - // Retrieve records and see if they scored 0.0 - SearchResponse searchResponse = client() - .prepareSearch("test") - .setQuery(QueryBuilders.functionScoreQuery(QueryBuilders.matchAllQuery(), new FunctionScoreQueryBuilder.FilterFunctionBuilder[]{ - new FunctionScoreQueryBuilder.FilterFunctionBuilder(ScoreFunctionBuilders.scriptFunction( - new Script(TFIDFScoreScript.SCRIPT_NAME, ScriptService.ScriptType.INLINE, "native", params)))}) - .boostMode(CombineFunction.REPLACE)).setSize(numDocs).execute().actionGet(); - assertNoFailures(searchResponse); - assertHitCount(searchResponse, numDocs); - SearchHit[] hits = searchResponse.getHits().hits(); - for (int i = 0; i < numDocs; i++) { - assertThat(hits[i].getId(), equalTo(Integer.toString(numDocs - i - 1))); - } - } - - public void testCosineSimilarity() throws Exception { - - initData(); - - // initialize parameters for script - Map params = new HashMap<>(); - params.put("field", field); - params.put("terms", searchTerms); - params.put("weights", weights); - - // Retrieve records and see if they scored 0.0 - SearchResponse searchResponse = client() - .prepareSearch("test") - .setQuery(QueryBuilders.functionScoreQuery(QueryBuilders.matchAllQuery(), new FunctionScoreQueryBuilder.FilterFunctionBuilder[]{ - new FunctionScoreQueryBuilder.FilterFunctionBuilder(ScoreFunctionBuilders.scriptFunction( - new Script(CosineSimilarityScoreScript.SCRIPT_NAME, ScriptService.ScriptType.INLINE, "native", params)))}) - .boostMode(CombineFunction.REPLACE)).setSize(numDocs).execute().actionGet(); - assertNoFailures(searchResponse); - assertHitCount(searchResponse, numDocs); - SearchHit[] hits = searchResponse.getHits().hits(); - for (int i = 0; i < numDocs; i++) { - assertThat(hits[i].getId(), equalTo(Integer.toString(i))); - } - } - - public void testPhraseScorer() throws Exception { - - initData(); - - // initialize parameters for script - Map params = new HashMap<>(); - params.put("field", field); - params.put("terms", searchTerms); - - // Retrieve records and see if they scored 0.0 - SearchResponse searchResponse = client() - .prepareSearch("test") - .setQuery(QueryBuilders.functionScoreQuery(QueryBuilders.matchAllQuery(), new FunctionScoreQueryBuilder.FilterFunctionBuilder[]{ - new FunctionScoreQueryBuilder.FilterFunctionBuilder(ScoreFunctionBuilders.scriptFunction( - new Script(PhraseScoreScript.SCRIPT_NAME, ScriptService.ScriptType.INLINE, "native", params)))}) - .boostMode(CombineFunction.REPLACE)).setSize(numDocs).execute().actionGet(); - assertNoFailures(searchResponse); - assertHitCount(searchResponse, numDocs); - SearchHit[] hits = searchResponse.getHits().hits(); - for (int i = 0; i < numDocs; i++) { - assertThat(hits[i].getId(), equalTo(Integer.toString(i))); - assertThat((double) hits[i].score(), closeTo(1.0 / (float) (i + 2), 1.e-6)); - } - } - - public void testLanguageModelScorer() throws Exception { - - initData(); - - // initialize parameters for script - Map params = new HashMap<>(); - params.put("field", field); - params.put("terms", searchTerms); - params.put("word_count_field", wordCountField); - params.put("lambda", 0.9); - - // Retrieve records and see if they scored 0.0 - SearchResponse searchResponse = client() - .prepareSearch("test") - .setQuery(QueryBuilders.functionScoreQuery(QueryBuilders.matchAllQuery(), new FunctionScoreQueryBuilder.FilterFunctionBuilder[]{ - new FunctionScoreQueryBuilder.FilterFunctionBuilder(ScoreFunctionBuilders.scriptFunction( - new Script(LanguageModelScoreScript.SCRIPT_NAME, ScriptService.ScriptType.INLINE, "native", params)))}) - .boostMode(CombineFunction.REPLACE)).setSize(numDocs).execute().actionGet(); - assertNoFailures(searchResponse); - assertHitCount(searchResponse, numDocs); - SearchHit[] hits = searchResponse.getHits().hits(); - for (int i = 0; i < numDocs; i++) { - assertThat(hits[i].getId(), equalTo(Integer.toString(i))); - } - } - - private void initData() throws IOException, InterruptedException, ExecutionException { - // Create a new index - String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") - .startObject(field).field("type", "string") - .startObject("fields") - .startObject("word_count") - .field("type", "string").field("analyzer", "standard").field("type", "token_count") - .endObject() - .endObject() - .endObject().endObject().endObject().endObject().string(); - assertAcked(prepareCreate("test").addMapping("type", mapping)); - - List indexBuilders = new ArrayList<>(); - // Index numDocs records (0..99) - for (int i = 0; i < numDocs; i++) { - indexBuilders.add(client().prepareIndex("test", "type", Integer.toString(i)) - .setSource(XContentFactory.jsonBuilder().startObject().field(field, createText(i + 1)).endObject()) - .setId(Integer.toString(i))); - } - indexRandom(true, indexBuilders); - } - - private String createText(int numFoo) { - String text = ""; - for (int i = 0; i < numFoo; i++) { - text = text + " foo "; - } - for (int i = 0; i < numFoo; i++) { - text = text + " " + placeholder + " "; - } - return text + " bar"; - } -}