Refactor backup code and add removal of old backups (#393)

app/scheduler/backup.js

@@ -1,14 +1,21 @@
+var fs = require("fs-extra");
 var config = require("config");
+var Moment = require("moment");
 var redis = require("redis").createClient();
 var helper = require("helper");
-var upload = helper.upload;
+var tmp = helper.tempDir();
+var ensure = helper.ensure;
 var encrypt = helper.encrypter.encrypt;
-var fs = require("fs-extra");
+var joinpath = require("path").join;
 var DB_PATH = require("path").resolve(__dirname + "/../../db/dump.rdb");
+var BUCKET = config.backup.bucket;
+var AWS = require("aws-sdk");
+var DAYS_OF_BACKUPS_TO_KEEP = 7;
 
-var tmp = helper.tempDir();
-var Moment = require("moment");
-var joinpath = require("path").join;
+AWS.config.update({
+  accessKeyId: config.aws.key,
+  secretAccessKey: config.aws.secret,
+});
 
 // called from command line
 if (require.main === module) {
@@ -26,25 +33,26 @@ function backup(callback) {
     var suffix = "-encrypted-dump.rdb";
     var filename = datestring + suffix;
     var encrypted_clone_path = joinpath(tmp, filename);
-    var options = {
-      bucket: config.backup.bucket,
-      remote: filename,
-    };
 
     console.log("Backup: Encrypting database dump");
     encrypt(DB_PATH, encrypted_clone_path, function (err) {
       if (err) return callback(err);
 
       console.log("Backup: Uploading encrypted copy of dump");
-      upload(encrypted_clone_path, options, function (err) {
+      upload(encrypted_clone_path, filename, function (err) {
         if (err) return callback(err);
 
         console.log("Backup: Removing local encrypted copy of dump");
         fs.remove(encrypted_clone_path, function (err) {
           if (err) return callback(err);
 
-          console.log("Backup: Complete");
-          callback();
+          console.log("Backup: Looking up old backup to remove");
+          wipeOldBackups(function (err) {
+            if (err) return callback(err);
+
+            console.log("Backup: Complete");
+            callback();
+          });
         });
       });
     });
@@ -73,4 +81,86 @@ function saveDBToDisk(callback) {
   });
 }
 
+function upload(path, filename, callback) {
+  ensure(path, "string").and(filename, "string").and(callback, "function");
+
+  var root = config.environment === "development" ? "_dev/" : "";
+  var remote = joinpath(root, filename);
+  var body = fs.createReadStream(path);
+
+  var params = {
+    Bucket: BUCKET,
+    Key: remote,
+  };
+
+  var s3Client = new AWS.S3({ params: params });
+
+  s3Client.upload({ Body: body }).send(function (err) {
+    if (err) return callback(err);
+    callback(null);
+  });
+}
+
+function wipeOldBackups(callback) {
+  var datestring = Moment.utc()
+    .subtract(DAYS_OF_BACKUPS_TO_KEEP, "days")
+    .format("YYYY-MM-DD-");
+
+  var root = config.environment === "development" ? "_dev/" : "";
+  var remote = joinpath(root, datestring);
+
+  console.log(
+    "Backup: Looking up backups made " +
+      DAYS_OF_BACKUPS_TO_KEEP +
+      " days ago on " +
+      datestring
+  );
+
+  var s3Client = new AWS.S3();
+  s3Client.listObjects(
+    {
+      Bucket: BUCKET,
+      Prefix: remote,
+    },
+    function (err, data) {
+      if (err) return callback(err);
+
+      if (!data || !data.Contents.length) {
+        console.log("Backup: No old backups to remove");
+        return callback();
+      }
+
+      var Objects = data.Contents.map((i) => {
+        return { Key: i.Key };
+      });
+
+      s3Client.deleteObjects(
+        {
+          Bucket: BUCKET,
+          Delete: {
+            Objects: Objects,
+            Quiet: false,
+          },
+        },
+        function (err, data) {
+          if (err) return callback(err);
+
+          if (data && data.Deleted && data.Deleted.length) {
+            console.log(
+              "Backup: Deleted " +
+                data.Deleted.length +
+                " dump made " +
+                DAYS_OF_BACKUPS_TO_KEEP +
+                " days ago: " +
+                data.Deleted.map((i) => i.Key).join(", ")
+            );
+          }
+
+          callback();
+        }
+      );
+    }
+  );
+}
+
 module.exports = backup;

todo.txt

@@ -1,5 +1,3 @@
-Write script to safely delete backups on a rolling basis
-
 Write script to identify and safely delete long overdue users
 
 Work out how to cache 404s at the NGINX level
@@ -19,12 +17,10 @@ Improvements to templates
 ---
 
 Server changes
+- Write useful benchmark tool for Blot
 - Add memory monitor
 - Migrate to lower-cost r6gd.xlarge instance, which is not possible using the AWS console due to some difference in architecture (intel vs. amazon chip?). 
 - Work out how to detect stray build processes. There were a bunch of zombies consuming memory on Blot's server. Work out how to kill them safely
-- Try and cache 404s in some efficient way
-- Create plan for AWS account suspension
-  - backup blogs folder + db somewhere offsite at least once per week
 - Experiment with different EBS types, perhaps try to rely on ephemeral disk more?
 
 Add Q and A question section to site where users could post and tell [Taylan](https://mail.google.com/mail/u/0/#inbox/FMfcgxwKkHkkKQrhrRPRnNNtzNGqtCQW) and [Adil](https://mail.google.com/mail/u/0/#inbox/QgrcJHsTfRRbxdszSvlzvFQCMxgLcRvXcvG)
@@ -77,6 +73,11 @@ Improve the automatic title generated for screenshots with default MacOs filenam
 
 Add support for posts generated from Excel files, would generate a table. Use [this library](https://github.com/joshboyan/excel-to-html-table)?
 
+Improvements to backup system
+- Add hourly local redis rdb dumps to /cache
+- Test maintenance mode
+- Add script to roll back into hourly local db dumps and remote backups
+
 Use [blurhash](https://github.com/woltapp/blurhash) to generate placeholders when lazy loading images and tell [David](https://mail.google.com/mail/u/0/#inbox/FMfcgxwJXfqzZvHgrtSxzLBCwcvTsFFB)
 
 Add status page information to news page:
@@ -362,10 +363,17 @@ Reply to [Jamie](https://mail.google.com/mail/u/0/#inbox/FMfcgxwChSCwMxNdtdvMHnM
 
 Improve scheduler
 - Adds a command to restore database from backup.
-- Adds a scheduled task to flush old backups using app/scheduler/backup/remove.js in branch 'scheduler' which was ported from the old remove method of the upload module
-- Adds offsite (i.e. non AWS backups)
 - Rotates log files
 
+Remove dependence on AWS
+- Adds offsite (i.e. non AWS backups) backups
+- Write out plan to move off AWS
+  - DNS
+  - EC2
+  - S3 for backups
+  - Cloudfront for CDN
+  - backup blogs folder + db somewhere offsite at least once per week
+
 Metadata parser
 - Make custom metadata more strict, follow up with Chris](https://mail.google.com/mail/u/0/#inbox/FMfcgxwCgpdRvrccbzfckkZRtPXHzqjs)
 - Add support for [Pelican-style metadata](http://docs.getpelican.com/en/stable/content.html) in HTML posts using <meta name="date" content="2012-07-09 22:28" />
@@ -512,12 +520,6 @@ Add nice import step under 'Services' and as part of sign up
 - Look into how Hexo has written imports for [blogger](https://github.com/hexojs/hexo-migrator-blogger), [joomla](https://github.com/welksonramos/hexo-migrator-joomla), [rss](https://github.com/hexojs/hexo-migrator-rss), [wordpress](https://github.com/hexojs/hexo-migrator-wordpress).
 - Add importer for Twitter and sell [Sawyer](https://mail.google.com/mail/u/0/#inbox/FMfcgxwBVMhRNdQhfDwlXsqqFNgmKfTk)
 
-Improvements to backup system
-- Add hourly local redis rdb dumps
-- Add script to roll back into hourly local db dumps and remote backups
-- Delete backups older than a month from s3 automatically
-- Archive logfiles with backups and wipe on a rolling basis
-
 Create script to export Blot sites to common formats like Wordpress and let Roberto know
 
 Move to Linode?