Initial implementation of Snapshot/Restore API

Closes #3826

docs/reference/modules.asciidoc

@@ -27,5 +27,7 @@ include::modules/thrift.asciidoc[]
 
 include::modules/transport.asciidoc[]
 
+include::modules/snapshots.asciidoc[]
+
 
 

docs/reference/modules/snapshots.asciidoc

@@ -0,0 +1,183 @@
+[[modules-snapshots]]
+== Snapshot And Restore
+
+The snapshot and restore module allows to create snapshots of individual indices or an entire cluster into a remote
+repository. At the time of the initial release only shared file system repository is supported.
+
+[float]
+=== Repositories
+
+Before any snapshot or restore operation can be performed a snapshot repository should be registered in
+Elasticsearch. The following command registers a shared file system repository with the name `my_backup` that
+will use location `/mount/backups/my_backup` to store snapshots.
+
+[source,js]
+-----------------------------------
+$ curl -XPUT 'http://localhost:9200/_snapshot/my_backup' -d '{
+    "type": "fs",
+    "settings": {
+        "location": "/mount/backups/my_backup",
+        "compress": true
+    }
+}'
+-----------------------------------
+
+Once repository is registered, its information can be obtained using the following command:
+
+[source,js]
+-----------------------------------
+$ curl -XGET 'http://localhost:9200/_snapshot/my_backup?pretty'
+-----------------------------------
+[source,js]
+-----------------------------------
+{
+  "my_backup" : {
+    "type" : "fs",
+    "settings" : {
+      "compress" : "false",
+      "location" : "/mount/backups/my_backup"
+    }
+  }
+}
+-----------------------------------
+
+If a repository name is not specified, or `_all` is used as repository name Elasticsearch will return information about
+all repositories currently registered in the cluster:
+
+[source,js]
+-----------------------------------
+$ curl -XGET 'http://localhost:9200/_snapshot'
+-----------------------------------
+
+or
+
+[source,js]
+-----------------------------------
+$ curl -XGET 'http://localhost:9200/_snapshot/all'
+-----------------------------------
+
+[float]
+===== Shared File System Repository
+
+The shared file system repository (`"type": "fs"`) is using shared file system to store snapshot. The path
+specified in the `location` parameter should point to the same location in the shared filesystem and be accessible
+on all data and master nodes. The following settings are supported:
+
+[horizontal]
+`location`:: Location of the snapshots. Mandatory.
+`compress`:: Turns on compression of the snapshot files. Defaults to `true`.
+`concurrent_streams`:: Throttles the number of streams (per node) preforming snapshot operation. Defaults to `5`
+`chunk_size`:: Big files can be broken down into chunks during snapshotting if needed. The chunk size can be specified in bytes or by
+ using size value notation, i.e. 1g, 10m, 5k. Defaults to `null` (unlimited chunk size).
+
+
+[float]
+=== Snapshot
+
+A repository can contain multiple snapshots of the same cluster. Snapshot are identified by unique names within the
+cluster. A snapshot with the name `snapshot_1` in the repository `my_backup` can be created by executing the following
+command:
+
+[source,js]
+-----------------------------------
+$ curl -XPUT "localhost:9200/_snapshot/my_backup/snapshot_1?wait_for_completion=true"
+-----------------------------------
+
+The `wait_for_completion` parameter specifies whether or not the request should return immediately or wait for snapshot
+completion. By default snapshot of all open and started indices in the cluster is created. This behavior can be changed
+by specifying the list of indices in the body of the snapshot request.
+
+[source,js]
+-----------------------------------
+$ curl -XPUT "localhost:9200/_snapshot/my_backup/snapshot_1" -d '{
+    "indices": "index_1,index_2",
+    "ignore_indices": "missing",
+    "include_global_state": false
+}'
+-----------------------------------
+
+The list of indices that should be included into the snapshot can be specified using the `indices` parameter that
+supports <<search-multi-index-type,multi index syntax>>. The snapshot request also supports the
+`ignore_indices` option. Setting it to `missing` will cause indices that do not exists to be ignored during snapshot
+creation. By default, when `ignore_indices` option is not set and an index is missing the snapshot request will fail.
+By setting `include_global_state` to false it's possible to prevent the cluster global state to be stored as part of
+the snapshot.
+
+The index snapshot process is incremental. In the process of making the index snapshot Elasticsearch analyses
+the list of the index files that are already stored in the repository and copies only files that were created or
+changed since the last snapshot. That allows multiple snapshots to be preserved in the repository in a compact form.
+Snapshotting process is executed in non-blocking fashion. All indexing and searching operation can continue to be
+executed against the index that is being snapshotted. However, a snapshot represents the point-in-time view of the index
+at the moment when snapshot was created, so no records that were added to the index after snapshot process had started
+will be present in the snapshot.
+
+Besides creating a copy of each index the snapshot process can also store global cluster metadata, which includes persistent
+cluster settings and templates. The transient settings and registered snapshot repositories are not stored as part of
+the snapshot.
+
+Only one snapshot process can be executed in the cluster at any time. While snapshot of a particular shard is being
+created this shard cannot be moved to another node, which can interfere with rebalancing process and allocation
+filtering. Once snapshot of the shard is finished Elasticsearch will be able to move shard to another node according
+to the current allocation filtering settings and rebalancing algorithm.
+
+Once a snapshot is created information about this snapshot can be obtained using the following command:
+
+[source,shell]
+-----------------------------------
+$ curl -XGET "localhost:9200/_snapshot/my_backup/snapshot_1"
+-----------------------------------
+
+All snapshots currently stored in the repository can be listed using the following command:
+
+[source,shell]
+-----------------------------------
+$ curl -XGET "localhost:9200/_snapshot/my_backup/_all"
+-----------------------------------
+
+A snapshot can be deleted from the repository using the following command:
+
+[source,shell]
+-----------------------------------
+$ curl -XDELETE "localhost:9200/_snapshot/my_backup/snapshot_1"
+-----------------------------------
+
+When a snapshot is deleted from a repository, Elasticsearch deletes all files that are associated with the deleted
+snapshot and not used by any other snapshots. If the deleted snapshot operation is executed while the snapshot is being
+created the snapshotting process will be aborted and all files created as part of the snapshotting process will be
+cleaned. Therefore, the delete snapshot operation can be used to cancel long running snapshot operations that were
+started by mistake.
+
+
+[float]
+=== Restore
+
+A snapshot can be restored using this following command:
+
+[source,shell]
+-----------------------------------
+$ curl -XPOST "localhost:9200/_snapshot/my_backup/snapshot_1/_restore"
+-----------------------------------
+
+By default, all indices in the snapshot as well as cluster state are restored. It's possible to select indices that
+should be restored as well as prevent global cluster state from being restored by using `indices` and
+`include_global_state` options in the restore request body. The list of indices supports
+<<search-multi-index-type,multi index syntax>>. The `rename_pattern` and `rename_replacement` options can be also used to
+rename index on restore using regular expression that supports referencing the original text as explained
+http://docs.oracle.com/javase/6/docs/api/java/util/regex/Matcher.html#appendReplacement(java.lang.StringBuffer,%20java.lang.String)[here].
+
+[source,js]
+-----------------------------------
+$ curl -XPOST "localhost:9200/_snapshot/my_backup/snapshot_1/_restore" -d '{
+    "indices": "index_1,index_2",
+    "ignore_indices": "missing",
+    "include_global_state": false,
+    "rename_pattern": "index_(.)+",
+    "rename_replacement": "restored_index_$1"
+}'
+-----------------------------------
+
+The restore operation can be performed on a functioning cluster. However, an existing index can be only restored if it's
+closed. The restore operation automatically opens restored indices if they were closed and creates new indices if they
+didn't exist in the cluster. If cluster state is restored, the restored templates that don't currently exist in the
+cluster are added and existing templates with the same name are replaced by the restored templates. The restored
+persistent settings are added to the existing persistent settings.

src/main/java/org/elasticsearch/action/ActionModule.java

@@ -32,12 +32,26 @@
 import org.elasticsearch.action.admin.cluster.node.shutdown.TransportNodesShutdownAction;
 import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsAction;
 import org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction;
+import org.elasticsearch.action.admin.cluster.repositories.delete.DeleteRepositoryAction;
+import org.elasticsearch.action.admin.cluster.repositories.delete.TransportDeleteRepositoryAction;
+import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesAction;
+import org.elasticsearch.action.admin.cluster.repositories.get.TransportGetRepositoriesAction;
+import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryAction;
+import org.elasticsearch.action.admin.cluster.repositories.put.TransportPutRepositoryAction;
 import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteAction;
 import org.elasticsearch.action.admin.cluster.reroute.TransportClusterRerouteAction;
 import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsAction;
 import org.elasticsearch.action.admin.cluster.settings.TransportClusterUpdateSettingsAction;
 import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsAction;
 import org.elasticsearch.action.admin.cluster.shards.TransportClusterSearchShardsAction;
+import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotAction;
+import org.elasticsearch.action.admin.cluster.snapshots.create.TransportCreateSnapshotAction;
+import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotAction;
+import org.elasticsearch.action.admin.cluster.snapshots.delete.TransportDeleteSnapshotAction;
+import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsAction;
+import org.elasticsearch.action.admin.cluster.snapshots.get.TransportGetSnapshotsAction;
+import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotAction;
+import org.elasticsearch.action.admin.cluster.snapshots.restore.TransportRestoreSnapshotAction;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
 import org.elasticsearch.action.admin.cluster.state.TransportClusterStateAction;
 import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksAction;
@@ -191,6 +205,13 @@ protected void configure() {
         registerAction(ClusterRerouteAction.INSTANCE, TransportClusterRerouteAction.class);
         registerAction(ClusterSearchShardsAction.INSTANCE, TransportClusterSearchShardsAction.class);
         registerAction(PendingClusterTasksAction.INSTANCE, TransportPendingClusterTasksAction.class);
+        registerAction(PutRepositoryAction.INSTANCE, TransportPutRepositoryAction.class);
+        registerAction(GetRepositoriesAction.INSTANCE, TransportGetRepositoriesAction.class);
+        registerAction(DeleteRepositoryAction.INSTANCE, TransportDeleteRepositoryAction.class);
+        registerAction(GetSnapshotsAction.INSTANCE, TransportGetSnapshotsAction.class);
+        registerAction(DeleteSnapshotAction.INSTANCE, TransportDeleteSnapshotAction.class);
+        registerAction(CreateSnapshotAction.INSTANCE, TransportCreateSnapshotAction.class);
+        registerAction(RestoreSnapshotAction.INSTANCE, TransportRestoreSnapshotAction.class);
 
         registerAction(IndicesStatsAction.INSTANCE, TransportIndicesStatsAction.class);
         registerAction(IndicesStatusAction.INSTANCE, TransportIndicesStatusAction.class);

src/main/java/org/elasticsearch/action/admin/cluster/repositories/delete/DeleteRepositoryAction.java

@@ -0,0 +1,47 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.cluster.repositories.delete;
+
+import org.elasticsearch.action.admin.cluster.ClusterAction;
+import org.elasticsearch.client.ClusterAdminClient;
+
+/**
+ * Unregister repository action
+ */
+public class DeleteRepositoryAction extends ClusterAction<DeleteRepositoryRequest, DeleteRepositoryResponse, DeleteRepositoryRequestBuilder> {
+
+    public static final DeleteRepositoryAction INSTANCE = new DeleteRepositoryAction();
+    public static final String NAME = "cluster/repository/delete";
+
+    private DeleteRepositoryAction() {
+        super(NAME);
+    }
+
+    @Override
+    public DeleteRepositoryResponse newResponse() {
+        return new DeleteRepositoryResponse();
+    }
+
+    @Override
+    public DeleteRepositoryRequestBuilder newRequestBuilder(ClusterAdminClient client) {
+        return new DeleteRepositoryRequestBuilder(client);
+    }
+}
+

src/main/java/org/elasticsearch/action/admin/cluster/repositories/delete/DeleteRepositoryRequest.java

@@ -0,0 +1,93 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.cluster.repositories.delete;
+
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.support.master.AcknowledgedRequest;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
+/**
+ * Unregister repository request.
+ * <p/>
+ * The unregister repository command just unregisters the repository. No data is getting deleted from the repository.
+ */
+public class DeleteRepositoryRequest extends AcknowledgedRequest<DeleteRepositoryRequest> {
+
+    private String name;
+
+    DeleteRepositoryRequest() {
+    }
+
+    /**
+     * Constructs a new unregister repository request with the provided name.
+     *
+     * @param name name of the repository
+     */
+    public DeleteRepositoryRequest(String name) {
+        this.name = name;
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        ActionRequestValidationException validationException = null;
+        if (name == null) {
+            validationException = addValidationError("name is missing", validationException);
+        }
+        return validationException;
+    }
+
+    /**
+     * Sets the name of the repository to unregister.
+     *
+     * @param name name of the repository
+     */
+    public DeleteRepositoryRequest name(String name) {
+        this.name = name;
+        return this;
+    }
+
+    /**
+     * The name of the repository.
+     *
+     * @return the name of the repository
+     */
+    public String name() {
+        return this.name;
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        name = in.readString();
+        readTimeout(in, null);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeString(name);
+        writeTimeout(out, null);
+    }
+}