feat: restore archived volumes (#20)

README.md

@@ -58,9 +58,9 @@ $ boxes start steambox
 Options:
 
 - `--wait`: wait for instance to complete startup
-- `--archive-volumes`: [experimental] detach, snapshot and delete instance volumes
+- `--yes`: [experimental] confirm restoration of archived volumes
 
-Note that the `--archive-volumes` option is experimental and may cause data loss.
+Note that the restoration of archived volumes option is experimental and may cause data loss.
 
 ### `boxes stop`
 
@@ -74,6 +74,9 @@ $ boxes stop steambox
 Options:
 
 - `--wait`: wait for instance to complete shutdown
+- `--archive-volumes`: [experimental] detach, snapshot and delete instance volumes
+
+Note that the `--archive-volumes` option is experimental and may cause data loss.
 
 ### `boxes info`
 
@@ -354,11 +357,11 @@ Quick and dirty task-list.
 
 ### Alpha
 
-- [ ] bug: stat/stop show 'pending' rather than 'stopped' due to order of logging
+- [ ] feat: 'import' option to tag a box and update local config
+- [x] bug: stat/stop show 'pending' rather than 'stopped' due to order of logging
 - [x] feat: document copy password in connect, maybe better default off
-- [ ] refactor: suck it up and use TS
-- [ ] feat: read AWS region from config file, using node-configuration
-- [ ] feat: save EBS costs by snapshot/detach/delete/replace (optional) - would save me $40 per month :) (see https://repost.aws/knowledge-center/ebs-charge-stopped-instance https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-restoring-volume.html https://cloudfix.com/blog/reduce-aws-costs-deleting-unnecessary-ebs-volumes/)
+- [x] refactor: suck it up and use TS
+- [x] feat: read AWS region from config file, using node-configuration
 - [x] npm badge download link
 - [x] bug: package.json path
 - [x] build / lint / test / deploy pipeline
@@ -403,15 +406,16 @@ This would be demo-able.
 ### Epic - Volume Management
 
 - [x] test '-wait' on start/stop and doc
-- [ ] propagate tags w/test
-- [ ] 'start' can now check for 'has archived volumes' and restore if available, this is the next big one
-- [ ] delete tag on volume restore...
-- [ ] ...so that we can auto restore volumes when calling 'start' - which will need to wait for the volumes to be ready
-- [ ] auto-restore on start, this is a good incremental point to check-in, even
+- [x] propagate tags w/test
+- [x] 'start' can now check for 'has archived volumes' and restore if available, this is the next big one
+- [x] delete tag on volume restore...
+- [x] ...so that we can auto restore volumes when calling 'start' - which will need to wait for the volumes to be ready
+- [x] auto-restore on start, this is a good incremental point to check-in, even
       if backup is only via 'debug' and comes later
-- [ ] data loss warning and generalise the 'yes' flag
+- [x] rename functions to 'archive' and 'restore' syntax
+- [x] data loss warning and generalise the 'yes' flag
 - [x] delete snapshot on successful restore
-- [ ] better logging for non-debug mode (warn user can take time)
-- [ ] new task list - docs, function, parameters, cost saving info, etc
+- [x] better logging for non-debug mode (warn user can take time)
 - [ ] calling 'detach/etc' fails if instance is not stopped or stopping as it doesn't try to stop the instance - must fail if state is not stopping or stopped
-- [ ] complete stop/start unit tests
+- [ ] new task list - docs, function, parameters, cost saving info, etc
+- [x] complete stop/start unit tests

src/cli.ts

@@ -83,10 +83,12 @@ program
   .description("Start a box")
   .argument("<boxId>", 'id of the box, e.g: "steambox"')
   .option("-w, --wait", "wait for box to complete startup", false)
+  .option("-y, --yes", "[experimental] confirm restore archived volumes", false)
   .action(async (boxId, options) => {
     const { instanceId, currentState, previousState } = await start({
       boxId,
       wait: options.wait,
+      restoreArchivedVolumes: options.yes,
     });
     console.log(
       `  ${theme.boxId(boxId)} (${instanceId}): ${theme.state(
@@ -109,7 +111,7 @@ program
         options,
         "yes",
         `The '--archive-volumes' feature is experimental and may cause data loss.
-  To accept this risk, re-run with the '--yes' parameter.`,
+To accept this risk, re-run with the '--yes' parameter.`,
       );
     }
     const { instanceId, currentState, previousState } = await stop({

src/commands/debug.ts

@@ -1,8 +1,8 @@
 import { TerminatingWarning } from "../lib/errors";
 import {
   getDetachableVolumes,
-  recreateVolumesFromSnapshotTag,
-  snapshotTagDeleteVolumes,
+  restoreArchivedVolumes,
+  archiveVolumes,
 } from "../lib/volumes";
 
 // eslint-disable-next-line  @typescript-eslint/no-explicit-any
@@ -36,11 +36,7 @@ export async function debug(command: string, parameters: string[]) {
     const detachableVolumes = await getDetachableVolumes(instanceId);
     logJson(detachableVolumes);
     console.log("Snapshotting / tagging...");
-    const result = await snapshotTagDeleteVolumes(
-      instanceId,
-      detachableVolumes,
-      tags,
-    );
+    const result = await archiveVolumes(instanceId, detachableVolumes, tags);
     logJson(result);
 
     return result;
@@ -51,7 +47,7 @@ export async function debug(command: string, parameters: string[]) {
       console.error("instanceid is required as the first parameter");
       return;
     }
-    const result = await recreateVolumesFromSnapshotTag(instanceId);
+    const result = await restoreArchivedVolumes(instanceId);
     console.log(result);
   } else {
     throw new TerminatingWarning(`unknown debug command ${command}`);

src/commands/start.test.ts

@@ -37,7 +37,11 @@ describe("start", () => {
       .on(StartInstancesCommand)
       .resolves(instancesStartSteambox);
 
-    await start({ boxId: "steambox", wait: false });
+    await start({
+      boxId: "steambox",
+      wait: false,
+      restoreArchivedVolumes: false,
+    });
 
     expect(ec2Mock).toHaveReceivedCommand(DescribeInstancesCommand);
     expect(ec2Mock).toHaveReceivedCommandWith(StartInstancesCommand, {

src/commands/start.ts

@@ -5,6 +5,7 @@ import { getBoxes } from "../lib/get-boxes";
 import { BoxState, awsStateToBoxState } from "../box";
 import { getConfiguration } from "../configuration";
 import { waitForInstanceState } from "../lib/aws-helpers";
+import { restoreArchivedVolumes } from "../lib/volumes";
 
 const debug = dbg("boxes:start");
 
@@ -18,10 +19,11 @@ export interface BoxTransition {
 export interface StartOptions {
   boxId: string;
   wait: boolean;
+  restoreArchivedVolumes: boolean;
 }
 
 export async function start(options: StartOptions): Promise<BoxTransition> {
-  const { boxId, wait } = options;
+  const { boxId, wait, restoreArchivedVolumes: enableRestore } = options;
 
   //  Get the box, fail with a warning if it is not found.
   const boxes = await getBoxes();
@@ -37,11 +39,30 @@ export async function start(options: StartOptions): Promise<BoxTransition> {
     );
   }
 
+  //  If the box has archived volumes, but we have not confirmed we will restore
+  //  them, fail.
+  if (box.hasArchivedVolumes && !enableRestore) {
+    throw new TerminatingWarning(
+      `This box has archived volumes which must be restored.
+This feature is experimental and may cause data loss.
+To accept this risk, re-run with the '--yes' parameter.`,
+    );
+  }
+
   //  Create an EC2 client.
   const { aws: awsConfig } = await getConfiguration();
   const client = new EC2Client(awsConfig);
 
-  //  Send the 'stop instances' command. Find the status of the stopping
+  //  If we must restore volumes, do so now.
+  if (box.hasArchivedVolumes) {
+    debug(`preparing to restore archived volumes for ${box.instanceId}...`);
+    console.log(
+      `  restoring archived volume(s)} for ${boxId}, this may take some time...`,
+    );
+    await restoreArchivedVolumes(box.instanceId);
+  }
+
+  //  Send the 'start instances' command. Find the status of the starting
   //  instance in the respose.
   debug(`preparing to start instance ${box.instanceId}...`);
   const response = await client.send(

src/commands/stop.ts

@@ -6,7 +6,7 @@ import { BoxState, awsStateToBoxState } from "../box";
 import { getConfiguration } from "../configuration";
 import { BoxTransition } from "./start";
 import { waitForInstanceState } from "../lib/aws-helpers";
-import { getDetachableVolumes, snapshotTagDeleteVolumes } from "../lib/volumes";
+import { getDetachableVolumes, archiveVolumes } from "../lib/volumes";
 import { tagNames } from "../lib/constants";
 
 const debug = dbg("boxes:stop");
@@ -18,7 +18,7 @@ export interface StopOptions {
 }
 
 export async function stop(options: StopOptions): Promise<BoxTransition> {
-  const { boxId, wait, archiveVolumes } = options;
+  const { boxId, wait, archiveVolumes: enableArchive } = options;
 
   //  Get the box, fail with a warning if it is not found.
   const boxes = await getBoxes();
@@ -57,7 +57,7 @@ export async function stop(options: StopOptions): Promise<BoxTransition> {
 
   //  If the wait flag has been specified, wait for the instance to enter
   //  the 'started' state. We also must wait if we are archiving.
-  if (wait || archiveVolumes) {
+  if (wait || enableArchive) {
     console.log(
       `  waiting for ${boxId} to shutdown - this may take some time...`,
     );
@@ -68,12 +68,12 @@ export async function stop(options: StopOptions): Promise<BoxTransition> {
   //  If we are archiving the volumes, do so now before we try and stop the box.
   //  Make sure to tag the snapshots with the box id so that we track its costs
   //  and can restore the tag to the volume later.
-  if (archiveVolumes) {
+  if (enableArchive) {
     const volumes = await getDetachableVolumes(box.instanceId);
     console.log(
       `  archiving ${volumes.length} volume(s), this may take some time...`,
     );
-    await snapshotTagDeleteVolumes(box.instanceId, volumes, [
+    await archiveVolumes(box.instanceId, volumes, [
       {
         Key: tagNames.boxId,
         Value: boxId,

src/lib/volumes.test.ts

@@ -9,15 +9,16 @@ import {
   DescribeInstancesCommand,
   AttachVolumeCommand,
   DeleteSnapshotCommand,
+  DeleteTagsCommand,
 } from "@aws-sdk/client-ec2";
 import { mockClient } from "aws-sdk-client-mock";
 import "aws-sdk-client-mock-jest";
 import mock from "mock-fs";
 import {
   DetachableVolume,
   getDetachableVolumes,
-  recreateVolumesFromSnapshotTag,
-  snapshotTagDeleteVolumes,
+  restoreArchivedVolumes,
+  archiveVolumes,
 } from "./volumes";
 import { tagNames } from "./constants";
 
@@ -30,6 +31,7 @@ import createVolume1Response from "../fixtures/volumes-create-volume1.json";
 import createVolume2Response from "../fixtures/volumes-create-volume2.json";
 import attachVolume1Response from "../fixtures/volumes-attach-volume1.json";
 import attachVolume2Response from "../fixtures/volumes-attach-volume2.json";
+import { TerminatingWarning } from "./errors";
 
 describe("volumes", () => {
   //  Mock the config file.
@@ -81,7 +83,7 @@ describe("volumes", () => {
     });
   });
 
-  describe("snapshot-and-delete-volumes", () => {
+  describe("archiveVolumes", () => {
     test("can snapshot with tags and delete volumes", async () => {
       //  Note: to record the fixtures and update this test process, check the
       //  ./fixtures/volumes-test-script.md file for the AWS commands to use.
@@ -116,11 +118,7 @@ describe("volumes", () => {
       ];
 
       const tags = [{ Key: tagNames.boxId, Value: "torrentbox" }];
-      const result = await snapshotTagDeleteVolumes(
-        instanceId,
-        detachableVolumes,
-        tags,
-      );
+      const result = await archiveVolumes(instanceId, detachableVolumes, tags);
 
       //  Assert we have detached the two volumes.
       expect(ec2Mock).toHaveReceivedCommandWith(DetachVolumeCommand, {
@@ -188,7 +186,7 @@ describe("volumes", () => {
     });
   });
 
-  describe("recreateVolumesFromSnapshotTag", () => {
+  describe("restoreArchivedVolumes", () => {
     test("throws a TerminatingWarning if the required tags are not present", async () => {
       const instanceId = "i-08fec1692931e31e7"; // fixture 'torrentbox' id
       const ec2Mock = mockClient(EC2Client)
@@ -198,12 +196,16 @@ describe("volumes", () => {
       //  Recreate the volumes from the snapshot tag. This response is missing
       //  the required tag so should throw. Jest doesn't let us check the
       //  message/type of error particularly well so it's a bit janky here.
-      await expect(
-        recreateVolumesFromSnapshotTag(instanceId),
-      ).rejects.toMatchObject({
-        message:
+      try {
+        await restoreArchivedVolumes(instanceId);
+        fail("recreate volumes did not throw");
+      } catch (err) {
+        const error = err as TerminatingWarning;
+        expect(error).toBeInstanceOf(TerminatingWarning);
+        expect(error.message).toMatch(
           "unable to restore volume snapshots - required tags are missing",
-      });
+        );
+      }
 
       //  Expect the call to the mock.
       expect(ec2Mock).toHaveReceivedCommandWith(DescribeInstancesCommand, {
@@ -240,7 +242,7 @@ describe("volumes", () => {
         .resolves(attachVolume2Response);
 
       //  Recreate the volumes from the snapshot tag.
-      const recreatedVolumes = await recreateVolumesFromSnapshotTag(instanceId);
+      const recreatedVolumes = await restoreArchivedVolumes(instanceId);
 
       expect(recreatedVolumes).toEqual([
         {
@@ -265,10 +267,32 @@ describe("volumes", () => {
       expect(ec2Mock).toHaveReceivedCommandWith(CreateVolumeCommand, {
         SnapshotId: snapshotId1,
         AvailabilityZone: "us-west-2a", // the az of our fixture instance
+        TagSpecifications: [
+          {
+            ResourceType: "volume",
+            Tags: [
+              {
+                Key: tagNames.boxId,
+                Value: "torrentbox",
+              },
+            ],
+          },
+        ],
       });
       expect(ec2Mock).toHaveReceivedCommandWith(CreateVolumeCommand, {
         SnapshotId: snapshotId2,
         AvailabilityZone: "us-west-2a", // the az of our fixture instance
+        TagSpecifications: [
+          {
+            ResourceType: "volume",
+            Tags: [
+              {
+                Key: tagNames.boxId,
+                Value: "torrentbox",
+              },
+            ],
+          },
+        ],
       });
 
       //  Now the newly created volumes should be attached.
@@ -279,13 +303,23 @@ describe("volumes", () => {
         Device: device2,
       });
 
-      //  Finally, the two snapshots should have been deleted.
+      //  The two snapshots should have been deleted.
       expect(ec2Mock).toHaveReceivedCommandWith(DeleteSnapshotCommand, {
         SnapshotId: snapshotId1,
       });
       expect(ec2Mock).toHaveReceivedCommandWith(DeleteSnapshotCommand, {
         SnapshotId: snapshotId2,
       });
+
+      //  The 'archived volumes' tag on the instance should have been removed.
+      expect(ec2Mock).toHaveReceivedCommandWith(DeleteTagsCommand, {
+        Resources: [instanceId],
+        Tags: [
+          {
+            Key: tagNames.volumeArchives,
+          },
+        ],
+      });
     });
   });
 });