Merge pull request #9231 from mmedenjak/hot-backup

Implement cluster backup/snapshot of hot restart store

hazelcast/src/main/java/com/hazelcast/config/HotRestartPersistenceConfig.java

@@ -55,6 +55,7 @@ public class HotRestartPersistenceConfig {
 
     private boolean enabled;
     private File baseDir = new File(HOT_RESTART_BASE_DIR_DEFAULT);
+    private File backupDir;
     private int parallelism = DEFAULT_PARALLELISM;
     private int validationTimeoutSeconds = DEFAULT_VALIDATION_TIMEOUT;
     private int dataLoadTimeoutSeconds = DEFAULT_DATA_LOAD_TIMEOUT;
@@ -121,6 +122,24 @@ public HotRestartPersistenceConfig setBaseDir(File baseDir) {
         return this;
     }
 
+    /**
+     * Base directory for all Hot Restart stores.
+     */
+    public File getBackupDir() {
+        return backupDir;
+    }
+
+    /**
+     * Sets base directory for all Hot Restart stores.
+     *
+     * @param backupDir home directory
+     * @return HotRestartConfig
+     */
+    public HotRestartPersistenceConfig setBackupDir(File backupDir) {
+        this.backupDir = backupDir;
+        return this;
+    }
+
     /**
      * Gets the configured number of Hot Restart store instance to create for one Hazelcast instance.
      */

hazelcast/src/main/java/com/hazelcast/hotrestart/BackupTaskState.java

@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2008-2016, Hazelcast, Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.hazelcast.hotrestart;
+
+/**
+ * The state of the hot restart backup task
+ */
+public enum BackupTaskState {
+    /** The backup task has not yet started. */
+    NOT_STARTED,
+    /** The backup task is currently in progress */
+    IN_PROGRESS,
+    /** The backup task has failed */
+    FAILURE,
+    /** The backup task completed successfully */
+    SUCCESS
+}

hazelcast/src/main/java/com/hazelcast/hotrestart/BackupTaskStatus.java

@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) 2008-2016, Hazelcast, Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.hazelcast.hotrestart;
+
+/**
+ * The status of the hot restart backup task, including progress and state.
+ */
+public class BackupTaskStatus {
+    private final BackupTaskState state;
+    private final int completed;
+    private final int total;
+
+    public BackupTaskStatus(BackupTaskState state, int completed, int total) {
+        this.state = state;
+        this.completed = completed;
+        this.total = total;
+    }
+
+    public BackupTaskState getState() {
+        return state;
+    }
+
+    public int getCompleted() {
+        return completed;
+    }
+
+    public int getTotal() {
+        return total;
+    }
+
+    public float getProgress() {
+        return total > 0 ? (float) completed / total : 0;
+    }
+
+    @Override
+    public String toString() {
+        return "BackupTaskStatus{state=" + state + ", completed=" + completed + ", total=" + total + '}';
+    }
+
+    @Override
+    @SuppressWarnings("checkstyle:innerassignment")
+    public boolean equals(Object obj) {
+        final BackupTaskStatus that;
+        return obj instanceof BackupTaskStatus
+                && this.completed == (that = (BackupTaskStatus) obj).completed
+                && this.total == that.total
+                && this.state == that.state;
+    }
+
+    @Override
+    public int hashCode() {
+        int result = state != null ? state.hashCode() : 0;
+        result = 31 * result + completed;
+        result = 31 * result + total;
+        return result;
+    }
+}

hazelcast/src/main/java/com/hazelcast/hotrestart/HotRestartBackupService.java

@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2008-2016, Hazelcast, Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.hazelcast.hotrestart;
+
+/**
+ * Service for starting cluster-wide hot restart data backups, determining the local backup state and interrupting a currently
+ * running local hot restart backup.
+ */
+public interface HotRestartBackupService {
+    /** The prefix for each hot restart backup directory. The backup sequence is appended to this prefix. */
+    String BACKUP_DIR_PREFIX = "backup-";
+
+    /**
+     * Attempts to perform a cluster hot restart data backup. Each node will create a directory under the defined backup dir
+     * with the name {@link #BACKUP_DIR_PREFIX} followed by the cluster time defined by this node.
+     * The backup request is performed transactionally. This method will throw an exception if an another request (transaction)
+     * is already in progress. If a node is already performing a backup (there is a file indicating a backup is in progress),
+     * the node will only log a warning and ignore the backup request.
+     */
+    void backup();
+
+    /**
+     * Attempts to perform a cluster hot restart data backup. Each node will create a directory under the defined backup dir
+     * with the name {@link #BACKUP_DIR_PREFIX} followed by the {@code backupSeq}.
+     * The backup request is performed transactionally. This method will throw an exception if an another request (transaction)
+     * is already in progress. If a node is already performing a backup (there is a file indicating a backup is in progress),
+     * the node will only log a warning and ignore the backup request.
+     *
+     * @param backupSeq the suffix of the backup directory for this cluster hot restart backup
+     */
+    void backup(long backupSeq);
+
+    /**
+     * Returns the local hot restart backup task status (not the cluster backup status).
+     */
+    BackupTaskStatus getBackupTaskStatus();
+
+    /**
+     * Interrupts the local backup task if one is currently running. The contents of the target backup directory will be left
+     * as-is.
+     */
+    void interruptLocalBackupTask();
+
+    /**
+     * Interrupts the backup tasks on each cluster member if one is currently running. The contents of the target backup
+     * directories will be left as-is.
+     */
+    void interruptBackupTask();
+}

hazelcast/src/main/java/com/hazelcast/hotrestart/package-info.java

@@ -0,0 +1,20 @@
+/*
+ * Copyright (c) 2008-2016, Hazelcast, Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <p>This package contains Hot Restart classes needed for open source Hazelcast code<br/>
+ */
+package com.hazelcast.hotrestart;

hazelcast/src/main/java/com/hazelcast/instance/DefaultNodeExtension.java

@@ -23,6 +23,7 @@
 import com.hazelcast.config.SerializationConfig;
 import com.hazelcast.core.HazelcastInstanceAware;
 import com.hazelcast.core.PartitioningStrategy;
+import com.hazelcast.hotrestart.HotRestartBackupService;
 import com.hazelcast.internal.cluster.ClusterStateListener;
 import com.hazelcast.internal.cluster.ClusterVersionListener;
 import com.hazelcast.internal.cluster.impl.JoinMessage;
@@ -298,6 +299,12 @@ public boolean triggerPartialStart() {
         return false;
     }
 
+    @Override
+    public HotRestartBackupService getHotRestartBackupService() {
+        logger.warning("Hot restart data backup features are only available on Hazelcast Enterprise!");
+        return null;
+    }
+
     @Override
     public String createMemberUuid(Address address) {
         return UuidUtil.createMemberUuid(address);

hazelcast/src/main/java/com/hazelcast/instance/NodeExtension.java

@@ -18,6 +18,7 @@
 
 import com.hazelcast.cluster.ClusterState;
 import com.hazelcast.config.HotRestartPersistenceConfig;
+import com.hazelcast.hotrestart.HotRestartBackupService;
 import com.hazelcast.internal.cluster.impl.JoinMessage;
 import com.hazelcast.internal.cluster.impl.JoinRequest;
 import com.hazelcast.internal.management.dto.ClusterHotRestartStatusDTO;
@@ -230,6 +231,14 @@ public interface NodeExtension {
      */
     boolean triggerPartialStart();
 
+
+    /**
+     * Returns the hot restart data backup service or null if Hot Restart backup is not available (not EE) or not enabled
+     *
+     * @return the hot restart data backup service
+     */
+    HotRestartBackupService getHotRestartBackupService();
+
     /**
      * Creates a UUID for local member
      * @param address address of local member

hazelcast/src/main/java/com/hazelcast/internal/ascii/rest/HttpCommandProcessor.java

@@ -31,7 +31,10 @@ public abstract class HttpCommandProcessor<T> extends AbstractTextCommandProcess
     public static final String URI_SHUTDOWN_CLUSTER_URL =  URI_CLUSTER_MANAGEMENT_BASE_URL + "/clusterShutdown";
     public static final String URI_FORCESTART_CLUSTER_URL =  URI_CLUSTER_MANAGEMENT_BASE_URL + "/forceStart";
     public static final String URI_PARTIALSTART_CLUSTER_URL =  URI_CLUSTER_MANAGEMENT_BASE_URL + "/partialStart";
-    public static final String URI_SHUTDOWN_NODE_CLUSTER_URL =  URI_CLUSTER_MANAGEMENT_BASE_URL + "/memberShutdown";
+    public static final String URI_HOT_RESTART_BACKUP_CLUSTER_URL = URI_CLUSTER_MANAGEMENT_BASE_URL + "/hotBackup";
+    public static final String URI_HOT_RESTART_BACKUP_INTERRUPT_CLUSTER_URL = URI_CLUSTER_MANAGEMENT_BASE_URL
+            + "/hotBackupInterrupt";
+    public static final String URI_SHUTDOWN_NODE_CLUSTER_URL = URI_CLUSTER_MANAGEMENT_BASE_URL + "/memberShutdown";
     public static final String URI_CLUSTER_NODES_URL = URI_CLUSTER_MANAGEMENT_BASE_URL + "/nodes";
     public static final String URI_MANCENTER_CHANGE_URL = "/hazelcast/rest/mancenter/changeurl";
     public static final String URI_WAN_SYNC_MAP = "/hazelcast/rest/wan/sync/map";

hazelcast/src/main/java/com/hazelcast/internal/ascii/rest/HttpPostCommandProcessor.java

@@ -21,6 +21,7 @@
 import com.hazelcast.config.GroupConfig;
 import com.hazelcast.config.WanReplicationConfig;
 import com.hazelcast.core.Member;
+import com.hazelcast.hotrestart.HotRestartBackupService;
 import com.hazelcast.instance.BuildInfoProvider;
 import com.hazelcast.instance.Node;
 import com.hazelcast.internal.ascii.TextCommandService;
@@ -70,6 +71,10 @@ public void handle(HttpPostCommand command) {
                 return;
             } else if (uri.startsWith(URI_FORCESTART_CLUSTER_URL)) {
                 handleForceStart(command);
+            } else if (uri.startsWith(URI_HOT_RESTART_BACKUP_CLUSTER_URL)) {
+                handleHotRestartBackup(command);
+            } else if (uri.startsWith(URI_HOT_RESTART_BACKUP_INTERRUPT_CLUSTER_URL)) {
+                handleHotRestartBackupInterrupt(command);
             } else if (uri.startsWith(URI_PARTIALSTART_CLUSTER_URL)) {
                 handlePartialStart(command);
             } else if (uri.startsWith(URI_CLUSTER_NODES_URL)) {
@@ -260,6 +265,44 @@ private void handlePartialStart(HttpPostCommand command) throws UnsupportedEncod
         textCommandService.sendResponse(command);
     }
 
+    private void handleHotRestartBackup(HttpPostCommand command) throws UnsupportedEncodingException {
+        String res;
+        try {
+            if (checkCredentials(command)) {
+                final HotRestartBackupService backupService = textCommandService.getNode().getNodeExtension()
+                                                                                .getHotRestartBackupService();
+                if (backupService != null) {
+                    backupService.backup();
+                }
+                res = "success";
+            } else {
+                res = "forbidden";
+            }
+        } catch (Throwable throwable) {
+            res = "fail";
+        }
+        sendResponse(command, String.format("{\"status\":\"%s\"}", res));
+    }
+
+    private void handleHotRestartBackupInterrupt(HttpPostCommand command) throws UnsupportedEncodingException {
+        String res;
+        try {
+            if (checkCredentials(command)) {
+                final HotRestartBackupService backupService = textCommandService.getNode().getNodeExtension()
+                                                                                .getHotRestartBackupService();
+                if (backupService != null) {
+                    backupService.interruptBackupTask();
+                }
+                res = "success";
+            } else {
+                res = "forbidden";
+            }
+        } catch (Throwable throwable) {
+            res = "fail";
+        }
+        sendResponse(command, String.format("{\"status\":\"%s\"}", res));
+    }
+
     private void handleClusterShutdown(HttpPostCommand command) throws UnsupportedEncodingException {
         byte[] data = command.getData();
         String[] strList = bytesToString(data).split("&");
@@ -486,6 +529,20 @@ private void handleAddWanConfig(HttpPostCommand command) throws UnsupportedEncod
         command.setResponse(HttpCommand.CONTENT_TYPE_JSON, stringToBytes(res));
     }
 
+    private boolean checkCredentials(HttpPostCommand command) throws UnsupportedEncodingException {
+        byte[] data = command.getData();
+        final String[] strList = bytesToString(data).split("&");
+        final String groupName = URLDecoder.decode(strList[0], "UTF-8");
+        final String groupPass = URLDecoder.decode(strList[1], "UTF-8");
+        final GroupConfig groupConfig = textCommandService.getNode().getConfig().getGroupConfig();
+        return groupConfig.getName().equals(groupName) && groupConfig.getPassword().equals(groupPass);
+    }
+
+    private void sendResponse(HttpPostCommand command, String value) {
+        command.setResponse(HttpCommand.CONTENT_TYPE_JSON, stringToBytes(value));
+        textCommandService.sendResponse(command);
+    }
+
     @Override
     public void handleRejection(HttpPostCommand command) {
         handle(command);