Add delete-by-query plugin

The delete by query plugin adds support for deleting all of the documents (from one or more indices) which match the specified query. It is a replacement for the problematic delete-by-query functionality which has been removed from Elasticsearch core in 2.0. Internally, it uses the Scan/Scroll and Bulk APIs to delete documents in an efficient and safe manner. It is slower than the old delete-by-query functionality, but fixes the problems with the previous implementation. Closes #7052
elastic · Jun 17, 2015 · ba35406 · ba35406
1 parent 0434ecf
commit ba35406
Show file tree

Hide file tree

Showing 19 changed files with 2,651 additions and 0 deletions.
diff --git a/plugins/delete-by-query/pom.xml b/plugins/delete-by-query/pom.xml
@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Licensed to Elasticsearch under one or more contributor
+license agreements. See the NOTICE file distributed with this work for additional
+information regarding copyright ownership. ElasticSearch licenses this file to you
+under the Apache License, Version 2.0 (the "License"); you may not use this
+file except in compliance with the License. You may obtain a copy of the
+License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by
+applicable law or agreed to in writing, software distributed under the License
+is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the specific language
+governing permissions and limitations under the License. -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>org.elasticsearch.plugin</groupId>
+    <artifactId>elasticsearch-delete-by-query</artifactId>
+
+    <packaging>jar</packaging>
+    <name>Elasticsearch Delete By Query plugin</name>
+    <description>The Delete By Query plugin allows to delete documents in Elasticsearch with a single query.</description>
+
+    <parent>
+        <groupId>org.elasticsearch</groupId>
+        <artifactId>elasticsearch-plugin</artifactId>
+        <version>2.0.0-SNAPSHOT</version>
+    </parent>
+
+    <properties>
+        <tests.ifNoTests>warn</tests.ifNoTests>
+        <tests.rest.suite>delete_by_query</tests.rest.suite>
+    </properties>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>
diff --git a/plugins/delete-by-query/rest-api-spec/api/delete_by_query.json b/plugins/delete-by-query/rest-api-spec/api/delete_by_query.json
@@ -0,0 +1,70 @@
+{
+  "delete_by_query": {
+    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/docs-delete-by-query.html",
+    "methods": ["DELETE"],
+    "url": {
+      "path": "/{index}/_query",
+      "paths": ["/{index}/_query", "/{index}/{type}/_query"],
+      "parts": {
+        "index": {
+          "type" : "list",
+          "required": true,
+          "description" : "A comma-separated list of indices to restrict the operation; use `_all` to perform the operation on all indices"
+        },
+        "type": {
+          "type" : "list",
+          "description" : "A comma-separated list of types to restrict the operation"
+        }
+      },
+      "params": {
+        "analyzer": {
+          "type" : "string",
+          "description" : "The analyzer to use for the query string"
+        },
+        "default_operator": {
+          "type" : "enum",
+          "options" : ["AND","OR"],
+          "default" : "OR",
+          "description" : "The default operator for query string query (AND or OR)"
+        },
+        "df": {
+          "type" : "string",
+          "description" : "The field to use as default where no field prefix is given in the query string"
+        },
+        "ignore_unavailable": {
+          "type" : "boolean",
+          "description" : "Whether specified concrete indices should be ignored when unavailable (missing or closed)"
+        },
+        "allow_no_indices": {
+          "type" : "boolean",
+          "description" : "Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified)"
+        },
+        "expand_wildcards": {
+          "type" : "enum",
+          "options" : ["open","closed","none","all"],
+          "default" : "open",
+          "description" : "Whether to expand wildcard expression to concrete indices that are open, closed or both."
+        },
+        "q": {
+          "type" : "string",
+          "description" : "Query in the Lucene query string syntax"
+        },
+        "routing": {
+          "type" : "string",
+          "description" : "Specific routing value"
+        },
+        "source": {
+          "type" : "string",
+          "description" : "The URL-encoded query definition (instead of using the request body)"
+        },
+        "timeout": {
+          "type" : "time",
+          "description" : "Explicit operation timeout"
+        }
+      }
+    },
+    "body": {
+      "description" : "A query to restrict the operation specified with the Query DSL"
+    }
+  }
+}
diff --git a/plugins/delete-by-query/rest-api-spec/test/delete_by_query/10_basic.yaml b/plugins/delete-by-query/rest-api-spec/test/delete_by_query/10_basic.yaml
@@ -0,0 +1,42 @@
+---
+"Basic delete_by_query":
+  - do:
+      index:
+          index:  test_1
+          type:   test
+          id:     1
+          body:   { foo: bar }
+
+  - do:
+      index:
+          index:  test_1
+          type:   test
+          id:     2
+          body:   { foo: baz }
+
+  - do:
+      index:
+          index:  test_1
+          type:   test
+          id:     3
+          body:   { foo: foo }
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      delete_by_query:
+        index:  test_1
+        body:
+          query:
+            match:
+              foo: bar
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      count:
+        index: test_1
+
+  - match:  { count:     2  }
diff --git a/plugins/delete-by-query/src/main/assemblies/plugin.xml b/plugins/delete-by-query/src/main/assemblies/plugin.xml
@@ -0,0 +1,34 @@
+<?xml version="1.0"?>
+<!-- Licensed to ElasticSearch under one or more contributor
+license agreements. See the NOTICE file distributed with this work for additional
+information regarding copyright ownership. ElasticSearch licenses this file to you
+under the Apache License, Version 2.0 (the "License"); you may not use this
+file except in compliance with the License. You may obtain a copy of the
+License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by
+applicable law or agreed to in writing, software distributed under the License
+is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the specific language
+governing permissions and limitations under the License. -->
+
+<assembly>
+    <id>plugin</id>
+    <formats>
+        <format>zip</format>
+    </formats>
+    <includeBaseDirectory>false</includeBaseDirectory>
+    <dependencySets>
+        <dependencySet>
+            <outputDirectory>/</outputDirectory>
+            <useProjectArtifact>true</useProjectArtifact>
+            <useTransitiveFiltering>true</useTransitiveFiltering>
+            <excludes>
+                <exclude>org.elasticsearch:elasticsearch</exclude>
+            </excludes>
+        </dependencySet>
+        <dependencySet>
+            <outputDirectory>/</outputDirectory>
+            <useProjectArtifact>true</useProjectArtifact>
+            <useTransitiveFiltering>true</useTransitiveFiltering>
+        </dependencySet>
+    </dependencySets>
+</assembly>
diff --git a/...te-by-query/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryAction.java b/...te-by-query/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryAction.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.deletebyquery;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.client.ElasticsearchClient;
+
+public class DeleteByQueryAction extends Action<DeleteByQueryRequest, DeleteByQueryResponse, DeleteByQueryRequestBuilder> {
+
+    public static final DeleteByQueryAction INSTANCE = new DeleteByQueryAction();
+    public static final String NAME = "indices:data/write/delete/by_query";
+
+    private DeleteByQueryAction() {
+        super(NAME);
+    }
+
+    @Override
+    public DeleteByQueryResponse newResponse() {
+        return new DeleteByQueryResponse();
+    }
+
+    @Override
+    public DeleteByQueryRequestBuilder newRequestBuilder(ElasticsearchClient client) {
+        return new DeleteByQueryRequestBuilder(client, this);
+    }
+}