diff --git a/.gitignore b/.gitignore index 798b9f1..cc9739c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,10 @@ -node_modules -jspm_packages -package-lock.json -boxes.json -artifacts/ .DS_Store +artifacts/ build/ +node_modules/ +jspm_packages/ +package-lock.json + +# Ignore any local boxes config - but track the test fixture version. +boxes.json +!src/fixtures/boxes.json diff --git a/README.md b/README.md index aaf97c9..b3ea463 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,3 @@ -# todo - -test ssh, new torrent script with logging, keep existing script for testing -detach this eve -region in config - -test a burner box for michelle's book - # boxes [![main](https://github.com/dwmkerr/boxes/actions/workflows/main.yml/badge.svg)](https://github.com/dwmkerr/boxes/actions/workflows/main.yml) ![npm (scoped)](https://img.shields.io/npm/v/%40dwmkerr/boxes) [![codecov](https://codecov.io/gh/dwmkerr/boxes/graph/badge.svg?token=uGVpjGFbDf)](https://codecov.io/gh/dwmkerr/boxes) @@ -63,6 +55,10 @@ $ boxes start steambox steambox (i-098e8d30d5e399b03): stopped -> pending ``` +Options: + +- `--wait`: wait for instance to complete startup + ### `boxes stop` Run `boxes start ` to stop a box: @@ -72,6 +68,10 @@ $ boxes stop steambox steambox (i-098e8d30d5e399b03): running -> stopping ``` +Options: + +- `--wait`: wait for instance to complete shutdown + ### `boxes info` Run `boxes info ` to show detailed info on a box: @@ -287,7 +287,15 @@ If you are developing and would like to run the `boxes` command without relinkin npm run build:watch ``` -This will keep the `./build` folder up-to-date and the `boxes` command will use the latest compiled code. +This will keep the `./build` folder up-to-date and the `boxes` command will use the latest compiled code. This will *sometimes* work but it might miss certain changes, so `relink` is the safer option. `build:watch` works well if you are making small changes to existing files, but not if you are adding new files (it seems). + +### Debugging + +The [`debug`](https://github.com/debug-js/debug) library is used to make it easy to provide debug level output. Debug logging to the console can be enabled with: + +```bash +DEBUG='boxes*' boxes list +``` ### Error Handling @@ -341,7 +349,7 @@ Typically occurs if AWS SDK packages are not at the exact same number as the `@a Quick and dirty task-list. -## Alpha +### Alpha - [x] feat: document copy password in connect, maybe better default off - [ ] refactor: suck it up and use TS @@ -362,10 +370,13 @@ Quick and dirty task-list. - [ ] feat: boxes aws-console opens link eg (https://us-west-2.console.aws.amazon.com/ec2/home?region=us-west-2#InstanceDetails:instanceId=i-043a3c1ce6c9ea6ad) - [ ] bug: EBS devices not tagged -I've tagged two (manually) in jan - check w/ feb bill -## Beta +### Beta + +- [ ] 'wait' flag for start/stop to wait until operation complete - default to 1hr and document the timeout info -## Later +### Later +- [ ] refactor: make 'debug' command local/debug build only? - [ ] feat: 'import' command to take an instance ID and create local box config for it and tag the instance - [ ] docs: cost allocation tags blog post - [ ] docs: create and share blogpost @@ -374,3 +385,28 @@ Quick and dirty task-list. - [ ] feat: autocomplete - [ ] feat: aws profile in config file - [ ] epic: 'boxes create' to create from a template +- [ ] refactor: find a better way to mock / inject config (rather than importing arbitrarily) + +### Epic - Interactive Setup + +Run `boxes init` - lets you choose a region, select instances, give a name. +Will add the tags - but will also add the tags to the volumes and will notify if the cost explorer tag is not setup. +Creates the local config. + +This would be demo-able. + +### Epic - Volume Management + +- [x] test '-wait' on start/stop and doc +- [ ] 'start' can now check for 'has archived volumes' and restore if available, this is the next big one +- [ ] propagate tags w/test +- [ ] delete tag on volume restore... +- [ ] ...so that we can auto restore volumes when calling 'start' - which will need to wait for the volumes to be ready +- [ ] auto-restore on start, this is a good incremental point to check-in, even + if backup is only via 'debug' and comes later +- [ ] data loss warning and generalise the 'yes' flag +- [x] delete snapshot on successful restore +- [ ] better logging for non-debug mode (warn user can take time) +- [ ] new task list - docs, function, parameters, cost saving info, etc +- [ ] calling 'detach/etc' fails if instance is not stopped or stopping as it doesn't try to stop the instance - must fail if state is not stopping or stopped +- [ ] complete stop/start unit tests diff --git a/jest.config.ts b/jest.config.ts index 27fc523..b4f4701 100644 --- a/jest.config.ts +++ b/jest.config.ts @@ -1,4 +1,5 @@ import type { Config } from "jest"; + /* * For a detailed explanation regarding each configuration property, visit: * https://jestjs.io/docs/configuration @@ -28,8 +29,8 @@ const config: Config = { // Only look for tests in the src folder, i.e. excluded build. roots: ["./src/"], - // Initial config/setup function for jest - globalSetup: "./src/jest-global-setup.ts", + // Jest setup function. + setupFilesAfterEnv: ["./src/jest.setup.ts"], // Automatically clear mock calls, instances, contexts and results before every test clearMocks: true, diff --git a/package.json b/package.json index 34092df..33013f3 100644 --- a/package.json +++ b/package.json @@ -11,11 +11,12 @@ "build": "tsc", "build:watch": "tsc -w", "start": "ts-node ./src/cli.ts", + "start:debug": "NODE_OPTIONS='--experimental-vm-modules' node --inspect-brk node_modules/.bin/ts-node ./src/cli.ts", "test": "NODE_OPTIONS='--experimental-vm-modules' jest", "lint": "eslint .", "lint:fix": "eslint --fix .", "test:debug": "NODE_OPTIONS='--experimental-vm-modules' node --inspect-brk node_modules/.bin/jest --runInBand", - "test:watch": "NODE_OPTIONS='--experimental-vm-modules' node node_modules/.bin/jest --runInBand --watch", + "test:watch": "NODE_OPTIONS='--experimental-vm-modules' node node_modules/.bin/jest --runInBand --watch --no-coverage", "test:cov": "NODE_OPTIONS='--experimental-vm-modules' jest --coverage", "tsc": "tsc", "relink": "npm run build && npm unlink boxes && npm link boxes" @@ -36,17 +37,22 @@ "clipboardy": "^4.0.0", "colors": "^1.4.0", "commander": "^11.1.0", + "debug": "^4.3.4", "open": "^9.1.0" }, "devDependencies": { "@aws-sdk/types": "3.10.0", + "@types/debug": "^4.1.12", "@types/jest": "^29.5.11", + "@types/mock-fs": "^4.13.4", + "@types/node": "^20.11.5", "@typescript-eslint/eslint-plugin": "^6.19.0", "aws-sdk-client-mock-jest": "^3.0.0", "eslint": "^8.53.0", "eslint-config-prettier": "^9.0.0", "eslint-plugin-prettier": "^5.0.1", "jest": "^29.7.0", + "mock-fs": "^5.2.0", "ts-jest": "^29.1.1", "ts-node": "^10.9.2", "typescript": "^5.3.3" diff --git a/src/box.ts b/src/box.ts index 4f52ce3..ae1e143 100644 --- a/src/box.ts +++ b/src/box.ts @@ -34,5 +34,6 @@ export interface Box { name: string; state: BoxState; instanceId: string | undefined; + hasArchivedVolumes: boolean; instance: Instance | undefined; } diff --git a/src/cli.ts b/src/cli.ts index f5661fb..34a5fe9 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -1,7 +1,9 @@ #!/usr/bin/env node +import dbg from "debug"; import { Command } from "commander"; import { list, info } from "./commands"; +import { debug } from "./commands/debug"; import { start } from "./commands/start"; import { stop } from "./commands/stop"; import { ssh } from "./commands/ssh"; @@ -12,6 +14,10 @@ import theme from "./theme"; import { TerminatingWarning } from "./lib/errors"; import packageJson from "../package.json"; import { BoxState } from "./box"; +import { assertConfirmation } from "./lib/cli-helpers"; + +// While we're developing, debug output is always enabled. +dbg.enable("boxes*"); const ERROR_CODE_WARNING = 1; const ERROR_CODE_CONNECTION = 2; @@ -30,12 +36,15 @@ program const boxes = await list(); boxes.forEach((box) => { theme.printBoxHeading(box.boxId, box.state); - theme.printBoxDetail("Name", box.name); + theme.printBoxDetail("Name", box.name || ""); // Only show DNS details if they exist (i.e. if the box is running). if (box.instance?.PublicDnsName && box.instance?.PublicIpAddress) { theme.printBoxDetail("DNS", box.instance.PublicDnsName); theme.printBoxDetail("IP", box.instance.PublicIpAddress); } + if (box.hasArchivedVolumes) { + theme.printBoxDetail("Archived Volumes", "true"); + } }); }); @@ -75,8 +84,12 @@ program .command("start") .description("Start a box") .argument("", 'id of the box, e.g: "steambox"') - .action(async (boxId) => { - const { instanceId, currentState, previousState } = await start(boxId); + .option("-w, --wait", "wait for box to complete startup", false) + .action(async (boxId, options) => { + const { instanceId, currentState, previousState } = await start({ + boxId, + wait: options.wait, + }); console.log( ` ${theme.boxId(boxId)} (${instanceId}): ${theme.state( previousState, @@ -88,12 +101,12 @@ program .command("stop") .description("Stop a box") .argument("", 'id of the box, e.g: "steambox"') - .option("--detach-volumes", "detach EBS volumes (experimental)", false) + .option("-w, --wait", "wait for box to complete startup", false) .action(async (boxId, options) => { - const { instanceId, currentState, previousState } = await stop( + const { instanceId, currentState, previousState } = await stop({ boxId, - options.detachVolumes, - ); + wait: options.wait, + }); console.log( ` ${theme.boxId(boxId)} (${instanceId}): ${theme.state( previousState, @@ -108,6 +121,14 @@ program .option("-y, --year ", "month of year", undefined) .option("-m, --month ", "month of year", undefined) .action(async (options) => { + // Demand confirmation. + await assertConfirmation( + options, + "yes", + `The AWS cost explorer charges $0.01 per call. +To accept charges, re-run with the '--yes' parameter.`, + ); + const boxes = await list(); const costs = await getCosts({ yes: options.yes, @@ -146,6 +167,16 @@ program console.log(JSON.stringify(configuration, null, 2)); }); +program + .command("debug") + .description("Additional commands used for debugging") + .argument("", 'debug command to use, e.g. "test-detach"') + .argument("", 'parameters for the command, e.g. "one two"') + .action(async (command, parameters) => { + const result = await debug(command, parameters); + console.log(JSON.stringify(result)); + }); + async function run() { try { await program.parseAsync(); diff --git a/src/commands/connect.ts b/src/commands/connect.ts index 0f96d59..a495169 100644 --- a/src/commands/connect.ts +++ b/src/commands/connect.ts @@ -1,9 +1,10 @@ -// used in the 'connect' function below. -// import clipboard from "clipboardy"; +import dbg from "debug"; import { getBoxes } from "../lib/get-boxes"; import { getConfiguration } from "../configuration"; import { TerminatingWarning } from "../lib/errors"; +const debug = dbg("boxes:connect"); + export async function connect( boxId: string, openConnection: boolean, @@ -36,9 +37,11 @@ export async function connect( // Expand the url string, which'll look something like this: // http://${host}:9091/transmission/web/ + debug(`expanding from: ${boxConfig.connectUrl}`); const expandedUrl = boxConfig.connectUrl .replace("${host}", box.instance.PublicDnsName) .replace("${username}", boxConfig.username); + debug(`expanded result: ${boxConfig.connectUrl}`); // If the user has asked for the password to be copied, put it on the // clipboard. diff --git a/src/commands/debug.ts b/src/commands/debug.ts new file mode 100644 index 0000000..dd2ef99 --- /dev/null +++ b/src/commands/debug.ts @@ -0,0 +1,61 @@ +import { TerminatingWarning } from "../lib/errors"; +import { + getDetachableVolumes, + recreateVolumesFromSnapshotTag, + snapshotTagDeleteVolumes, +} from "../lib/volumes"; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +function logJson(val: any) { + console.log(JSON.stringify(val, null, 2)); +} + +export async function debug(command: string, parameters: string[]) { + console.log(`debug: command - ${command} with parameters ${parameters}`); + if (command === "test-detach") { + console.log("debug: test-detach"); + const instanceId = parameters[0]; + if (!instanceId) { + console.error("instanceid is required as the first parameter"); + return; + } + + const detachableVolumes = getDetachableVolumes(instanceId); + + return detachableVolumes; + } else if (command === "test-detach-snapshot-tag") { + console.log("debug: test-detach"); + const instanceId = parameters[0]; + if (!instanceId) { + console.error("instanceid is required as the first parameter"); + return; + } + + // TODO fix tag settings + const tags = [{ key: "boxes.boxid", value: "torrentbox" }]; + console.log("Getting detachable volumes..."); + const detachableVolumes = await getDetachableVolumes(instanceId); + logJson(detachableVolumes); + console.log("Snapshotting / tagging..."); + const result = await snapshotTagDeleteVolumes( + instanceId, + detachableVolumes, + tags, + ); + logJson(result); + + return result; + } else if (command === "test-restore-volumes") { + console.log("debug: test-restore-volumes"); + const instanceId = parameters[0]; + if (!instanceId) { + console.error("instanceid is required as the first parameter"); + return; + } + const result = await recreateVolumesFromSnapshotTag(instanceId); + console.log(result); + } else { + throw new TerminatingWarning(`unknown debug command ${command}`); + } + return {}; +} diff --git a/src/commands/getCosts.ts b/src/commands/getCosts.ts index fcb2570..2700658 100644 --- a/src/commands/getCosts.ts +++ b/src/commands/getCosts.ts @@ -1,10 +1,8 @@ import { getBoxesCosts } from "../lib/get-boxes-costs"; -import { TerminatingWarning } from "../lib/errors"; type BoxCosts = Record; export async function getCosts({ - yes, year, month, }: { @@ -12,13 +10,6 @@ export async function getCosts({ year: string; month: string; }): Promise { - // If the user hasn't passed the 'yes' parameter to confirm, ask now. - if (yes !== true) { - const message = `The AWS cost explorer charges $0.01 per call. -To accept charges, re-run with the '--yes' parameter.`; - throw new TerminatingWarning(message); - } - // Parse the year/month number if provided. const yearNumber = year ? parseInt(year, 10) : undefined; const monthNumber = month ? parseInt(month, 10) : undefined; diff --git a/src/commands/start.test.ts b/src/commands/start.test.ts new file mode 100644 index 0000000..7445710 --- /dev/null +++ b/src/commands/start.test.ts @@ -0,0 +1,47 @@ +import { + EC2Client, + DescribeInstancesCommand, + StartInstancesCommand, +} from "@aws-sdk/client-ec2"; +import { mockClient } from "aws-sdk-client-mock"; +import "aws-sdk-client-mock-jest"; +import path from "path"; +import mock from "mock-fs"; +import { start } from "./start"; + +import describeInstancesResponse from "../fixtures/get-boxes-describe-instances.json"; +import instancesStartSteambox from "../fixtures/instances-start-steambox.json"; + +describe("start", () => { + // Mock the config file. + beforeEach(() => { + const boxesPath = path.join(path.resolve(), "./boxes.json"); + mock({ + [boxesPath]: mock.load( + path.join(path.resolve(), "./src/fixtures/boxes.json"), + ), + }); + }); + + afterEach(() => { + mock.restore(); + }); + + test("can start box", async () => { + // Record fixtures with: + // AWS_PROFILE=dwmkerr aws ec2 describe-instances --filters "Name=tag:boxes.boxid,Values=*" > ./src/fixtures/aws-ec2-describe-instances.json + // aws ec2 start-instances --instance-ids i-043a3c1ce6c9ea6ad | tee ./src/fixtures/instances-start-steambox.json + const ec2Mock = mockClient(EC2Client) + .on(DescribeInstancesCommand) + .resolves(describeInstancesResponse) + .on(StartInstancesCommand) + .resolves(instancesStartSteambox); + + await start({ boxId: "steambox", wait: false }); + + expect(ec2Mock).toHaveReceivedCommand(DescribeInstancesCommand); + expect(ec2Mock).toHaveReceivedCommandWith(StartInstancesCommand, { + InstanceIds: ["i-043a3c1ce6c9ea6ad"], + }); + }); +}); diff --git a/src/commands/start.ts b/src/commands/start.ts index 2729840..0b70141 100644 --- a/src/commands/start.ts +++ b/src/commands/start.ts @@ -1,8 +1,12 @@ +import dbg from "debug"; import { EC2Client, StartInstancesCommand } from "@aws-sdk/client-ec2"; import { TerminatingWarning } from "../lib/errors"; import { getBoxes } from "../lib/get-boxes"; import { BoxState, awsStateToBoxState } from "../box"; import { getConfiguration } from "../configuration"; +import { waitForInstanceState } from "../lib/aws-helpers"; + +const debug = dbg("boxes:start"); export interface BoxTransition { boxId: string; @@ -11,7 +15,14 @@ export interface BoxTransition { previousState: BoxState; } -export async function start(boxId: string): Promise { +export interface StartOptions { + boxId: string; + wait: boolean; +} + +export async function start(options: StartOptions): Promise { + const { boxId, wait } = options; + // Get the box, fail with a warning if it is not found. const boxes = await getBoxes(); const box = boxes.find((b) => b.boxId === boxId); @@ -32,19 +43,30 @@ export async function start(boxId: string): Promise { // Send the 'stop instances' command. Find the status of the stopping // instance in the respose. + debug(`preparing to start instance ${box.instanceId}...`); const response = await client.send( new StartInstancesCommand({ InstanceIds: [box.instanceId], }), ); - const stoppingInstance = response.StartingInstances?.find( + debug(`...complete, ${response.StartingInstances?.length} instances started`); + const startingInstances = response.StartingInstances?.find( (si) => si.InstanceId === box.instanceId, ); + // If the wait flag has been specified, wait for the instance to enter + // the 'started' state. + if (wait) { + console.log( + ` waiting for ${boxId} to startup - this may take some time...`, + ); + waitForInstanceState(client, box.instanceId, "running"); + } + return { boxId, instanceId: box.instanceId, - currentState: awsStateToBoxState(stoppingInstance?.CurrentState?.Name), - previousState: awsStateToBoxState(stoppingInstance?.PreviousState?.Name), + currentState: awsStateToBoxState(startingInstances?.CurrentState?.Name), + previousState: awsStateToBoxState(startingInstances?.PreviousState?.Name), }; } diff --git a/src/commands/stop.test.ts b/src/commands/stop.test.ts new file mode 100644 index 0000000..3f0c421 --- /dev/null +++ b/src/commands/stop.test.ts @@ -0,0 +1,47 @@ +import { + EC2Client, + DescribeInstancesCommand, + StopInstancesCommand, +} from "@aws-sdk/client-ec2"; +import { mockClient } from "aws-sdk-client-mock"; +import "aws-sdk-client-mock-jest"; +import path from "path"; +import mock from "mock-fs"; +import { stop } from "./stop"; + +import describeInstancesResponse from "../fixtures/get-boxes-describe-instances.json"; +import instancesStopSteambox from "../fixtures/instances-stop-steambox.json"; + +describe("stop", () => { + // Mock the config file. + beforeEach(() => { + const boxesPath = path.join(path.resolve(), "./boxes.json"); + mock({ + [boxesPath]: mock.load( + path.join(path.resolve(), "./src/fixtures/boxes.json"), + ), + }); + }); + + afterEach(() => { + mock.restore(); + }); + + test.skip("can stop boxes", async () => { + // Record fixtures with: + // AWS_PROFILE=dwmkerr aws ec2 describe-instances --filters "Name=tag:boxes.boxid,Values=*" > ./src/fixtures/aws-ec2-describe-instances.json + // aws ec2 stop-instances --instance-ids i-043a3c1ce6c9ea6ad | tee ./src/fixtures/instances-stop-steambox.json + const ec2Mock = mockClient(EC2Client) + .on(DescribeInstancesCommand) + .resolves(describeInstancesResponse) + .on(StopInstancesCommand) + .resolves(instancesStopSteambox); + + await stop({ boxId: "steambox", wait: false }); + + expect(ec2Mock).toHaveReceivedCommand(DescribeInstancesCommand); + expect(ec2Mock).toHaveReceivedCommandWith(StopInstancesCommand, { + InstanceIds: ["i-043a3c1ce6c9ea6ad"], + }); + }); +}); diff --git a/src/commands/stop.ts b/src/commands/stop.ts index 94d2293..8eca9b3 100644 --- a/src/commands/stop.ts +++ b/src/commands/stop.ts @@ -1,10 +1,22 @@ +import dbg from "debug"; import { EC2Client, StopInstancesCommand } from "@aws-sdk/client-ec2"; import { TerminatingWarning } from "../lib/errors"; import { getBoxes } from "../lib/get-boxes"; import { awsStateToBoxState } from "../box"; import { getConfiguration } from "../configuration"; +import { BoxTransition } from "./start"; +import { waitForInstanceState } from "../lib/aws-helpers"; + +const debug = dbg("boxes:stop"); + +export interface StopOptions { + boxId: string; + wait: boolean; +} + +export async function stop(options: StopOptions): Promise { + const { boxId, wait } = options; -export async function stop(boxId: string, detach: boolean) { // Get the box, fail with a warning if it is not found. const boxes = await getBoxes(); const box = boxes.find((b) => b.boxId === boxId); @@ -25,20 +37,26 @@ export async function stop(boxId: string, detach: boolean) { // Send the 'stop instances' command. Find the status of the stopping // instance in the respose. + debug(`preparing to stop instance ${box.instanceId}...`); const response = await client.send( new StopInstancesCommand({ InstanceIds: [box.instanceId], }), ); + debug(`...complete, ${response.StoppingInstances?.length} instances stopped`); const stoppingInstance = response.StoppingInstances?.find( (si) => si.InstanceId === box.instanceId, ); - if (detach) { - throw new TerminatingWarning( - `'detach' parameter currently not implemented`, + // If the wait flag has been specified, wait for the instance to enter + // the 'started' state. + if (wait) { + console.log( + ` waiting for ${boxId} to shutdown - this may take some time...`, ); + waitForInstanceState(client, box.instanceId, "stopped"); } + return { boxId, instanceId: box.instanceId, diff --git a/src/fixtures/boxes.json b/src/fixtures/boxes.json new file mode 100644 index 0000000..b66dd61 --- /dev/null +++ b/src/fixtures/boxes.json @@ -0,0 +1,20 @@ +{ + "boxes": { + "steambox": { + "connectUrl": "dcv://${host}:8443", + "username": "Administrator", + "password": "", + "sshCommand": "open rdp://${host}" + }, + "torrentbox": { + "connectUrl": "http://${username}@${host}:9091/transmission/web/", + "username": "admin", + "password": "", + "sshCommand": "ssh -i ~/.ssh/mykey.pem ec2-user@${host}" + } + }, + "aws": { + "region": "us-west-2" + } +} + diff --git a/src/fixtures/get-boxes-describe-instances-with-archived-volumes.json b/src/fixtures/get-boxes-describe-instances-with-archived-volumes.json new file mode 100644 index 0000000..cb769a8 --- /dev/null +++ b/src/fixtures/get-boxes-describe-instances-with-archived-volumes.json @@ -0,0 +1,324 @@ +{ + "Reservations": [ + { + "Groups": [], + "Instances": [ + { + "AmiLaunchIndex": 0, + "ImageId": "ami-0872c164f38dcc49f", + "InstanceId": "i-08fec1692931e31e7", + "InstanceType": "t2.micro", + "KeyName": "dwmkerr_aws_key", + "LaunchTime": "2024-01-22T07:34:32+00:00", + "Monitoring": { + "State": "disabled" + }, + "Placement": { + "AvailabilityZone": "us-west-2a", + "GroupName": "", + "Tenancy": "default" + }, + "PrivateDnsName": "ip-10-0-1-11.us-west-2.compute.internal", + "PrivateIpAddress": "10.0.1.11", + "ProductCodes": [], + "PublicDnsName": "", + "State": { + "Code": 80, + "Name": "stopped" + }, + "StateTransitionReason": "User initiated (2024-01-22 07:36:40 GMT)", + "SubnetId": "subnet-0218aa127fc710ca2", + "VpcId": "vpc-000512d60555e3586", + "Architecture": "x86_64", + "BlockDeviceMappings": [ + { + "DeviceName": "/dev/xvda", + "Ebs": { + "AttachTime": "2024-01-22T07:33:59+00:00", + "DeleteOnTermination": false, + "Status": "attached", + "VolumeId": "vol-0c3940cade857692b" + } + }, + { + "DeviceName": "/dev/xvdf", + "Ebs": { + "AttachTime": "2024-01-22T07:34:18+00:00", + "DeleteOnTermination": false, + "Status": "attached", + "VolumeId": "vol-059b4ea55caf83199" + } + } + ], + "ClientToken": "terraform-20231110021418137600000002", + "EbsOptimized": false, + "EnaSupport": true, + "Hypervisor": "xen", + "IamInstanceProfile": { + "Arn": "arn:aws:iam::705383350627:instance-profile/torrent-box", + "Id": "AIPA2IPA7YFR7R52P3QTA" + }, + "NetworkInterfaces": [ + { + "Attachment": { + "AttachTime": "2023-11-10T02:14:24+00:00", + "AttachmentId": "eni-attach-0aadacdaeea6ee30d", + "DeleteOnTermination": true, + "DeviceIndex": 0, + "Status": "attached", + "NetworkCardIndex": 0 + }, + "Description": "", + "Groups": [ + { + "GroupName": "ssh-ingress", + "GroupId": "sg-0197adf4dda7c939e" + } + ], + "Ipv6Addresses": [], + "MacAddress": "06:3a:34:27:1e:3f", + "NetworkInterfaceId": "eni-04b6899b732026480", + "OwnerId": "705383350627", + "PrivateDnsName": "ip-10-0-1-11.us-west-2.compute.internal", + "PrivateIpAddress": "10.0.1.11", + "PrivateIpAddresses": [ + { + "Primary": true, + "PrivateDnsName": "ip-10-0-1-11.us-west-2.compute.internal", + "PrivateIpAddress": "10.0.1.11" + } + ], + "SourceDestCheck": false, + "Status": "in-use", + "SubnetId": "subnet-0218aa127fc710ca2", + "VpcId": "vpc-000512d60555e3586", + "InterfaceType": "interface" + } + ], + "RootDeviceName": "/dev/xvda", + "RootDeviceType": "ebs", + "SecurityGroups": [ + { + "GroupName": "ssh-ingress", + "GroupId": "sg-0197adf4dda7c939e" + } + ], + "SourceDestCheck": false, + "StateReason": { + "Code": "Client.UserInitiatedShutdown", + "Message": "Client.UserInitiatedShutdown: User initiated shutdown" + }, + "Tags": [ + { + "Key": "Owner", + "Value": "dwmkerr" + }, + { + "Key": "boxes.volumesnapshots", + "Value": "[{\"snapshotId\":\"snap-03c3efc7e9254ab0a\",\"device\":\"/dev/xvda\"},{\"snapshotId\":\"snap-056afd3da4b3b003b\",\"device\":\"/dev/xvdf\"}]" + }, + { + "Key": "boxes.boxid", + "Value": "torrentbox" + }, + { + "Key": "Project", + "Value": "github.com/dwmkerr/dwmkerr" + }, + { + "Key": "Name", + "Value": "Torrent Box" + } + ], + "VirtualizationType": "hvm", + "CpuOptions": { + "CoreCount": 1, + "ThreadsPerCore": 1 + }, + "CapacityReservationSpecification": { + "CapacityReservationPreference": "open" + }, + "HibernationOptions": { + "Configured": false + }, + "MetadataOptions": { + "State": "applied", + "HttpTokens": "optional", + "HttpPutResponseHopLimit": 1, + "HttpEndpoint": "enabled", + "HttpProtocolIpv6": "disabled", + "InstanceMetadataTags": "disabled" + }, + "EnclaveOptions": { + "Enabled": false + }, + "PlatformDetails": "Linux/UNIX", + "UsageOperation": "RunInstances", + "UsageOperationUpdateTime": "2023-11-10T02:14:24+00:00", + "PrivateDnsNameOptions": { + "HostnameType": "ip-name", + "EnableResourceNameDnsARecord": false, + "EnableResourceNameDnsAAAARecord": false + }, + "MaintenanceOptions": { + "AutoRecovery": "default" + }, + "CurrentInstanceBootMode": "legacy-bios" + } + ], + "OwnerId": "705383350627", + "ReservationId": "r-00ed8dde4b0ca365d" + }, + { + "Groups": [], + "Instances": [ + { + "AmiLaunchIndex": 0, + "ImageId": "ami-0fae5ac34f36d5963", + "InstanceId": "i-043a3c1ce6c9ea6ad", + "InstanceType": "g4ad.xlarge", + "KeyName": "dwmkerr_aws_key", + "LaunchTime": "2024-01-23T20:15:05+00:00", + "Monitoring": { + "State": "disabled" + }, + "Placement": { + "AvailabilityZone": "us-west-2a", + "GroupName": "", + "Tenancy": "default" + }, + "Platform": "windows", + "PrivateDnsName": "ip-10-0-1-196.us-west-2.compute.internal", + "PrivateIpAddress": "10.0.1.196", + "ProductCodes": [], + "PublicDnsName": "", + "State": { + "Code": 80, + "Name": "stopped" + }, + "StateTransitionReason": "User initiated (2024-01-23 21:13:28 GMT)", + "SubnetId": "subnet-0218aa127fc710ca2", + "VpcId": "vpc-000512d60555e3586", + "Architecture": "x86_64", + "BlockDeviceMappings": [], + "ClientToken": "terraform-20231114025128673500000001", + "EbsOptimized": false, + "EnaSupport": true, + "Hypervisor": "xen", + "IamInstanceProfile": { + "Arn": "arn:aws:iam::705383350627:instance-profile/steam_box", + "Id": "AIPA2IPA7YFRXUHWENJ7X" + }, + "NetworkInterfaces": [ + { + "Attachment": { + "AttachTime": "2023-11-14T02:51:29+00:00", + "AttachmentId": "eni-attach-0453b01fe301d0ff3", + "DeleteOnTermination": true, + "DeviceIndex": 0, + "Status": "attached", + "NetworkCardIndex": 0 + }, + "Description": "", + "Groups": [ + { + "GroupName": "steam_box", + "GroupId": "sg-0645d8e0c59a84f6c" + } + ], + "Ipv6Addresses": [], + "MacAddress": "06:e1:61:85:bc:15", + "NetworkInterfaceId": "eni-053f5be39cc1fe3c1", + "OwnerId": "705383350627", + "PrivateDnsName": "ip-10-0-1-196.us-west-2.compute.internal", + "PrivateIpAddress": "10.0.1.196", + "PrivateIpAddresses": [ + { + "Primary": true, + "PrivateDnsName": "ip-10-0-1-196.us-west-2.compute.internal", + "PrivateIpAddress": "10.0.1.196" + } + ], + "SourceDestCheck": false, + "Status": "in-use", + "SubnetId": "subnet-0218aa127fc710ca2", + "VpcId": "vpc-000512d60555e3586", + "InterfaceType": "interface" + } + ], + "RootDeviceName": "/dev/sda1", + "RootDeviceType": "ebs", + "SecurityGroups": [ + { + "GroupName": "steam_box", + "GroupId": "sg-0645d8e0c59a84f6c" + } + ], + "SourceDestCheck": false, + "StateReason": { + "Code": "Client.UserInitiatedShutdown", + "Message": "Client.UserInitiatedShutdown: User initiated shutdown" + }, + "Tags": [ + { + "Key": "Project", + "Value": "github.com/dwmkerr/dwmkerr" + }, + { + "Key": "boxes.boxid", + "Value": "steambox" + }, + { + "Key": "Name", + "Value": "Steam Box" + }, + { + "Key": "boxes.volumearchives", + "Value": "[{\"device\":\"/dev/sda1\",\"snapshotId\":\"snap-09c3d75ee1f3c7b42\"}]" + }, + { + "Key": "Owner", + "Value": "dwmkerr" + } + ], + "VirtualizationType": "hvm", + "CpuOptions": { + "CoreCount": 2, + "ThreadsPerCore": 2 + }, + "CapacityReservationSpecification": { + "CapacityReservationPreference": "open" + }, + "HibernationOptions": { + "Configured": false + }, + "MetadataOptions": { + "State": "applied", + "HttpTokens": "optional", + "HttpPutResponseHopLimit": 1, + "HttpEndpoint": "enabled", + "HttpProtocolIpv6": "disabled", + "InstanceMetadataTags": "disabled" + }, + "EnclaveOptions": { + "Enabled": false + }, + "PlatformDetails": "Windows", + "UsageOperation": "RunInstances:0002", + "UsageOperationUpdateTime": "2023-11-14T02:51:29+00:00", + "PrivateDnsNameOptions": { + "HostnameType": "ip-name", + "EnableResourceNameDnsARecord": false, + "EnableResourceNameDnsAAAARecord": false + }, + "MaintenanceOptions": { + "AutoRecovery": "default" + }, + "CurrentInstanceBootMode": "legacy-bios" + } + ], + "OwnerId": "705383350627", + "ReservationId": "r-031e24d6b39110a56" + } + ] +} diff --git a/src/fixtures/aws-ec2-describe-instances.json b/src/fixtures/get-boxes-describe-instances.json similarity index 100% rename from src/fixtures/aws-ec2-describe-instances.json rename to src/fixtures/get-boxes-describe-instances.json diff --git a/src/fixtures/instances-start-steambox.json b/src/fixtures/instances-start-steambox.json new file mode 100644 index 0000000..03cadfb --- /dev/null +++ b/src/fixtures/instances-start-steambox.json @@ -0,0 +1,15 @@ +{ + "StartingInstances": [ + { + "CurrentState": { + "Code": 0, + "Name": "pending" + }, + "InstanceId": "i-043a3c1ce6c9ea6ad", + "PreviousState": { + "Code": 80, + "Name": "stopped" + } + } + ] +} diff --git a/src/fixtures/instances-stop-steambox.json b/src/fixtures/instances-stop-steambox.json new file mode 100644 index 0000000..fb93a83 --- /dev/null +++ b/src/fixtures/instances-stop-steambox.json @@ -0,0 +1,15 @@ +{ + "StoppingInstances": [ + { + "CurrentState": { + "Code": 64, + "Name": "stopping" + }, + "InstanceId": "i-043a3c1ce6c9ea6ad", + "PreviousState": { + "Code": 16, + "Name": "running" + } + } + ] +} diff --git a/src/fixtures/volumes-attach-volume1.json b/src/fixtures/volumes-attach-volume1.json new file mode 100644 index 0000000..094d353 --- /dev/null +++ b/src/fixtures/volumes-attach-volume1.json @@ -0,0 +1,7 @@ +{ + "AttachTime": "2024-01-21T21:33:32.447000+00:00", + "Device": "/dev/xvdf", + "InstanceId": "i-08fec1692931e31e7", + "State": "attaching", + "VolumeId": "vol-0c3940cade857692b" +} diff --git a/src/fixtures/volumes-attach-volume2.json b/src/fixtures/volumes-attach-volume2.json new file mode 100644 index 0000000..74cd060 --- /dev/null +++ b/src/fixtures/volumes-attach-volume2.json @@ -0,0 +1,7 @@ +{ + "AttachTime": "2024-01-21T21:34:12.469000+00:00", + "Device": "/dev/xvda", + "InstanceId": "i-08fec1692931e31e7", + "State": "attaching", + "VolumeId": "vol-059b4ea55caf83199" +} diff --git a/src/fixtures/volumes-create-snapshot-volume1.json b/src/fixtures/volumes-create-snapshot-volume1.json new file mode 100644 index 0000000..700ea39 --- /dev/null +++ b/src/fixtures/volumes-create-snapshot-volume1.json @@ -0,0 +1,17 @@ +{ + "Description": "", + "Encrypted": true, + "OwnerId": "705383350627", + "Progress": "", + "SnapshotId": "snap-03c3efc7e9254ab0a", + "StartTime": "2024-01-21T17:22:59.660000+00:00", + "State": "pending", + "VolumeId": "vol-0582d7fc0f3d797fc", + "VolumeSize": 20, + "Tags": [ + { + "Key": "boxes.boxid", + "Value": "torrentbox" + } + ] +} diff --git a/src/fixtures/volumes-create-snapshot-volume2.json b/src/fixtures/volumes-create-snapshot-volume2.json new file mode 100644 index 0000000..6486938 --- /dev/null +++ b/src/fixtures/volumes-create-snapshot-volume2.json @@ -0,0 +1,17 @@ +{ + "Description": "", + "Encrypted": true, + "OwnerId": "705383350627", + "Progress": "", + "SnapshotId": "snap-056afd3da4b3b003b", + "StartTime": "2024-01-21T17:23:02.989000+00:00", + "State": "pending", + "VolumeId": "vol-0987a9ce9bb4c7b1d", + "VolumeSize": 200, + "Tags": [ + { + "Key": "boxes.boxid", + "Value": "torrentbox" + } + ] +} diff --git a/src/fixtures/volumes-create-volume1.json b/src/fixtures/volumes-create-volume1.json new file mode 100644 index 0000000..336167b --- /dev/null +++ b/src/fixtures/volumes-create-volume1.json @@ -0,0 +1,14 @@ +{ + "AvailabilityZone": "us-west-2a", + "CreateTime": "2024-01-21T21:16:47+00:00", + "Encrypted": true, + "KmsKeyId": "arn:aws:kms:us-west-2:705383350627:key/1043b4a7-5766-4242-ab25-e6dc43ab41bd", + "Size": 20, + "SnapshotId": "snap-0f87e8940e82b46ce", + "State": "creating", + "VolumeId": "vol-0c3940cade857692b", + "Iops": 100, + "Tags": [], + "VolumeType": "gp2", + "MultiAttachEnabled": false +} diff --git a/src/fixtures/volumes-create-volume2.json b/src/fixtures/volumes-create-volume2.json new file mode 100644 index 0000000..4d255d4 --- /dev/null +++ b/src/fixtures/volumes-create-volume2.json @@ -0,0 +1,14 @@ +{ + "AvailabilityZone": "us-west-2a", + "CreateTime": "2024-01-21T21:16:51+00:00", + "Encrypted": true, + "KmsKeyId": "arn:aws:kms:us-west-2:705383350627:key/1043b4a7-5766-4242-ab25-e6dc43ab41bd", + "Size": 200, + "SnapshotId": "snap-0d4180edffe7ecdbc", + "State": "creating", + "VolumeId": "vol-059b4ea55caf83199", + "Iops": 600, + "Tags": [], + "VolumeType": "gp2", + "MultiAttachEnabled": false +} diff --git a/src/fixtures/volumes-describe-instances-missing-tags.json b/src/fixtures/volumes-describe-instances-missing-tags.json new file mode 100644 index 0000000..2a20c0e --- /dev/null +++ b/src/fixtures/volumes-describe-instances-missing-tags.json @@ -0,0 +1,150 @@ +{ + "Reservations": [ + { + "Groups": [], + "Instances": [ + { + "AmiLaunchIndex": 0, + "ImageId": "ami-0872c164f38dcc49f", + "InstanceId": "i-08fec1692931e31e7", + "InstanceType": "t2.micro", + "KeyName": "dwmkerr_aws_key", + "LaunchTime": "2024-01-19T15:15:08+00:00", + "Monitoring": { + "State": "disabled" + }, + "Placement": { + "AvailabilityZone": "us-west-2a", + "GroupName": "", + "Tenancy": "default" + }, + "PrivateDnsName": "ip-10-0-1-11.us-west-2.compute.internal", + "PrivateIpAddress": "10.0.1.11", + "ProductCodes": [], + "PublicDnsName": "", + "State": { + "Code": 80, + "Name": "stopped" + }, + "StateTransitionReason": "User initiated (2024-01-21 16:16:20 GMT)", + "SubnetId": "subnet-0218aa127fc710ca2", + "VpcId": "vpc-000512d60555e3586", + "Architecture": "x86_64", + "BlockDeviceMappings": [], + "ClientToken": "terraform-20231110021418137600000002", + "EbsOptimized": false, + "EnaSupport": true, + "Hypervisor": "xen", + "IamInstanceProfile": { + "Arn": "arn:aws:iam::705383350627:instance-profile/torrent-box", + "Id": "AIPA2IPA7YFR7R52P3QTA" + }, + "NetworkInterfaces": [ + { + "Attachment": { + "AttachTime": "2023-11-10T02:14:24+00:00", + "AttachmentId": "eni-attach-0aadacdaeea6ee30d", + "DeleteOnTermination": true, + "DeviceIndex": 0, + "Status": "attached", + "NetworkCardIndex": 0 + }, + "Description": "", + "Groups": [ + { + "GroupName": "ssh-ingress", + "GroupId": "sg-0197adf4dda7c939e" + } + ], + "Ipv6Addresses": [], + "MacAddress": "06:3a:34:27:1e:3f", + "NetworkInterfaceId": "eni-04b6899b732026480", + "OwnerId": "705383350627", + "PrivateDnsName": "ip-10-0-1-11.us-west-2.compute.internal", + "PrivateIpAddress": "10.0.1.11", + "PrivateIpAddresses": [ + { + "Primary": true, + "PrivateDnsName": "ip-10-0-1-11.us-west-2.compute.internal", + "PrivateIpAddress": "10.0.1.11" + } + ], + "SourceDestCheck": false, + "Status": "in-use", + "SubnetId": "subnet-0218aa127fc710ca2", + "VpcId": "vpc-000512d60555e3586", + "InterfaceType": "interface" + } + ], + "RootDeviceName": "/dev/xvda", + "RootDeviceType": "ebs", + "SecurityGroups": [ + { + "GroupName": "ssh-ingress", + "GroupId": "sg-0197adf4dda7c939e" + } + ], + "SourceDestCheck": false, + "StateReason": { + "Code": "Client.UserInitiatedShutdown", + "Message": "Client.UserInitiatedShutdown: User initiated shutdown" + }, + "Tags": [ + { + "Key": "Owner", + "Value": "dwmkerr" + }, + { + "Key": "boxes.boxid", + "Value": "torrentbox" + }, + { + "Key": "Project", + "Value": "github.com/dwmkerr/dwmkerr" + }, + { + "Key": "Name", + "Value": "Torrent Box" + } + ], + "VirtualizationType": "hvm", + "CpuOptions": { + "CoreCount": 1, + "ThreadsPerCore": 1 + }, + "CapacityReservationSpecification": { + "CapacityReservationPreference": "open" + }, + "HibernationOptions": { + "Configured": false + }, + "MetadataOptions": { + "State": "applied", + "HttpTokens": "optional", + "HttpPutResponseHopLimit": 1, + "HttpEndpoint": "enabled", + "HttpProtocolIpv6": "disabled", + "InstanceMetadataTags": "disabled" + }, + "EnclaveOptions": { + "Enabled": false + }, + "PlatformDetails": "Linux/UNIX", + "UsageOperation": "RunInstances", + "UsageOperationUpdateTime": "2023-11-10T02:14:24+00:00", + "PrivateDnsNameOptions": { + "HostnameType": "ip-name", + "EnableResourceNameDnsARecord": false, + "EnableResourceNameDnsAAAARecord": false + }, + "MaintenanceOptions": { + "AutoRecovery": "default" + }, + "CurrentInstanceBootMode": "legacy-bios" + } + ], + "OwnerId": "705383350627", + "ReservationId": "r-00ed8dde4b0ca365d" + } + ] +} diff --git a/src/fixtures/volumes-describe-instances.json b/src/fixtures/volumes-describe-instances.json new file mode 100644 index 0000000..bb0fbb9 --- /dev/null +++ b/src/fixtures/volumes-describe-instances.json @@ -0,0 +1,154 @@ +{ + "Reservations": [ + { + "Groups": [], + "Instances": [ + { + "AmiLaunchIndex": 0, + "ImageId": "ami-0872c164f38dcc49f", + "InstanceId": "i-08fec1692931e31e7", + "InstanceType": "t2.micro", + "KeyName": "dwmkerr_aws_key", + "LaunchTime": "2024-01-19T15:15:08+00:00", + "Monitoring": { + "State": "disabled" + }, + "Placement": { + "AvailabilityZone": "us-west-2a", + "GroupName": "", + "Tenancy": "default" + }, + "PrivateDnsName": "ip-10-0-1-11.us-west-2.compute.internal", + "PrivateIpAddress": "10.0.1.11", + "ProductCodes": [], + "PublicDnsName": "", + "State": { + "Code": 80, + "Name": "stopped" + }, + "StateTransitionReason": "User initiated (2024-01-21 16:16:20 GMT)", + "SubnetId": "subnet-0218aa127fc710ca2", + "VpcId": "vpc-000512d60555e3586", + "Architecture": "x86_64", + "BlockDeviceMappings": [], + "ClientToken": "terraform-20231110021418137600000002", + "EbsOptimized": false, + "EnaSupport": true, + "Hypervisor": "xen", + "IamInstanceProfile": { + "Arn": "arn:aws:iam::705383350627:instance-profile/torrent-box", + "Id": "AIPA2IPA7YFR7R52P3QTA" + }, + "NetworkInterfaces": [ + { + "Attachment": { + "AttachTime": "2023-11-10T02:14:24+00:00", + "AttachmentId": "eni-attach-0aadacdaeea6ee30d", + "DeleteOnTermination": true, + "DeviceIndex": 0, + "Status": "attached", + "NetworkCardIndex": 0 + }, + "Description": "", + "Groups": [ + { + "GroupName": "ssh-ingress", + "GroupId": "sg-0197adf4dda7c939e" + } + ], + "Ipv6Addresses": [], + "MacAddress": "06:3a:34:27:1e:3f", + "NetworkInterfaceId": "eni-04b6899b732026480", + "OwnerId": "705383350627", + "PrivateDnsName": "ip-10-0-1-11.us-west-2.compute.internal", + "PrivateIpAddress": "10.0.1.11", + "PrivateIpAddresses": [ + { + "Primary": true, + "PrivateDnsName": "ip-10-0-1-11.us-west-2.compute.internal", + "PrivateIpAddress": "10.0.1.11" + } + ], + "SourceDestCheck": false, + "Status": "in-use", + "SubnetId": "subnet-0218aa127fc710ca2", + "VpcId": "vpc-000512d60555e3586", + "InterfaceType": "interface" + } + ], + "RootDeviceName": "/dev/xvda", + "RootDeviceType": "ebs", + "SecurityGroups": [ + { + "GroupName": "ssh-ingress", + "GroupId": "sg-0197adf4dda7c939e" + } + ], + "SourceDestCheck": false, + "StateReason": { + "Code": "Client.UserInitiatedShutdown", + "Message": "Client.UserInitiatedShutdown: User initiated shutdown" + }, + "Tags": [ + { + "Key": "Owner", + "Value": "dwmkerr" + }, + { + "Key": "boxes.volumearchives", + "Value": "[{\"snapshotId\":\"snap-03c3efc7e9254ab0a\",\"device\":\"/dev/xvda\"},{\"snapshotId\":\"snap-056afd3da4b3b003b\",\"device\":\"/dev/xvdf\"}]" + }, + { + "Key": "boxes.boxid", + "Value": "torrentbox" + }, + { + "Key": "Project", + "Value": "github.com/dwmkerr/dwmkerr" + }, + { + "Key": "Name", + "Value": "Torrent Box" + } + ], + "VirtualizationType": "hvm", + "CpuOptions": { + "CoreCount": 1, + "ThreadsPerCore": 1 + }, + "CapacityReservationSpecification": { + "CapacityReservationPreference": "open" + }, + "HibernationOptions": { + "Configured": false + }, + "MetadataOptions": { + "State": "applied", + "HttpTokens": "optional", + "HttpPutResponseHopLimit": 1, + "HttpEndpoint": "enabled", + "HttpProtocolIpv6": "disabled", + "InstanceMetadataTags": "disabled" + }, + "EnclaveOptions": { + "Enabled": false + }, + "PlatformDetails": "Linux/UNIX", + "UsageOperation": "RunInstances", + "UsageOperationUpdateTime": "2023-11-10T02:14:24+00:00", + "PrivateDnsNameOptions": { + "HostnameType": "ip-name", + "EnableResourceNameDnsARecord": false, + "EnableResourceNameDnsAAAARecord": false + }, + "MaintenanceOptions": { + "AutoRecovery": "default" + }, + "CurrentInstanceBootMode": "legacy-bios" + } + ], + "OwnerId": "705383350627", + "ReservationId": "r-00ed8dde4b0ca365d" + } + ] +} diff --git a/src/fixtures/volumes-describe-volumes-torrent-box.json b/src/fixtures/volumes-describe-volumes-torrent-box.json new file mode 100644 index 0000000..3ed482f --- /dev/null +++ b/src/fixtures/volumes-describe-volumes-torrent-box.json @@ -0,0 +1,56 @@ +{ + "Volumes": [ + { + "Attachments": [ + { + "AttachTime": "2023-11-10T02:14:25+00:00", + "Device": "/dev/xvda", + "InstanceId": "i-08fec1692931e31e7", + "State": "attached", + "VolumeId": "vol-0582d7fc0f3d797fc", + "DeleteOnTermination": true + } + ], + "AvailabilityZone": "us-west-2a", + "CreateTime": "2023-11-10T02:14:25.545000+00:00", + "Encrypted": true, + "KmsKeyId": "arn:aws:kms:us-west-2:705383350627:key/1043b4a7-5766-4242-ab25-e6dc43ab41bd", + "Size": 20, + "SnapshotId": "snap-001c614f9d26384b4", + "State": "in-use", + "VolumeId": "vol-0582d7fc0f3d797fc", + "Iops": 100, + "VolumeType": "gp2", + "MultiAttachEnabled": false + }, + { + "Attachments": [ + { + "AttachTime": "2023-11-10T02:14:25+00:00", + "Device": "/dev/xvdf", + "InstanceId": "i-08fec1692931e31e7", + "State": "attached", + "VolumeId": "vol-0987a9ce9bb4c7b1d", + "DeleteOnTermination": true + } + ], + "AvailabilityZone": "us-west-2a", + "CreateTime": "2023-11-10T02:14:25.657000+00:00", + "Encrypted": true, + "KmsKeyId": "arn:aws:kms:us-west-2:705383350627:key/1043b4a7-5766-4242-ab25-e6dc43ab41bd", + "Size": 200, + "SnapshotId": "", + "State": "in-use", + "VolumeId": "vol-0987a9ce9bb4c7b1d", + "Iops": 600, + "Tags": [ + { + "Key": "boxes.boxid", + "Value": "torrentbox" + } + ], + "VolumeType": "gp2", + "MultiAttachEnabled": false + } + ] +} diff --git a/src/fixtures/volumes-detach-volume1.json b/src/fixtures/volumes-detach-volume1.json new file mode 100644 index 0000000..3222e39 --- /dev/null +++ b/src/fixtures/volumes-detach-volume1.json @@ -0,0 +1,7 @@ +{ + "AttachTime": "2023-11-10T02:14:25+00:00", + "Device": "/dev/xvda", + "InstanceId": "i-08fec1692931e31e7", + "State": "detaching", + "VolumeId": "vol-0582d7fc0f3d797fc" +} diff --git a/src/fixtures/volumes-detach-volume2.json b/src/fixtures/volumes-detach-volume2.json new file mode 100644 index 0000000..18be0fd --- /dev/null +++ b/src/fixtures/volumes-detach-volume2.json @@ -0,0 +1,7 @@ +{ + "AttachTime": "2023-11-10T02:14:25+00:00", + "Device": "/dev/xvdf", + "InstanceId": "i-08fec1692931e31e7", + "State": "detaching", + "VolumeId": "vol-0987a9ce9bb4c7b1d" +} diff --git a/src/fixtures/volumes-test-script.md b/src/fixtures/volumes-test-script.md new file mode 100644 index 0000000..28d5916 --- /dev/null +++ b/src/fixtures/volumes-test-script.md @@ -0,0 +1,41 @@ +# Volumes Test Script + +Using: + +``` +torrentbox (i-08fec1692931e31e7) +{"volumeId":"vol-0582d7fc0f3d797fc","device":"/dev/xvda"}, +{"volumeId":"vol-0987a9ce9bb4c7b1d","device":"/dev/xvdf"} +``` + +These are the commands, roughly: + +```bash +// detach... +aws ec2 detach-volume --volume-id vol-0582d7fc0f3d797fc >> ./src/fixtures/volumes-detach-volume1.json +aws ec2 detach-volume --volume-id vol-0987a9ce9bb4c7b1d >> ./src/fixtures/volumes-detach-volume2.json + +// snapshot... +aws ec2 create-snapshot --volume-id vol-0582d7fc0f3d797fc >> ./src/fixtures/create-snaphshot-volume1.json +aws ec2 create-snapshot --volume-id vol-0987a9ce9bb4c7b1d >> ./src/fixtures/create-snaphshot-volume2.json + +# create tags from snapshot details: +aws ec2 create-tags --resources i-08fec1692931e31e7 --tags 'Key="boxes.volumesnapshots",Value="[{\"snapshotId\":\"snap-03c3efc7e9254ab0a\",\"device\":\"/dev/xvda\"},{\"snapshotId\":\"snap-056afd3da4b3b003b\",\"device\":\"/dev/xvdf\"}]"' + + +// delete - no response needs to be recorded for fixtures. +aws ec2 delete-volume --volume-id vol-0582d7fc0f3d797fc +aws ec2 delete-volume --volume-id vol-0987a9ce9bb4c7b1d + +// Create volumes from snapshots. + +# Get the EC2 instance details - we need the tags and its availability zone. +aws ec2 describe-instances --instance-ids i-08fec1692931e31e7 >> ./src/fixtures/volumes-describe-instances.json + +aws ec2 create-volume --snapshot-id "snap-0f87e8940e82b46ce" --availability-zone us-west-2a > ./src/fixtures/volumes-create-volume1.json +aws ec2 create-volume --snapshot-id "snap-0d4180edffe7ecdbc" --availability-zone us-west-2a > ./src/fixtures/volumes-create-volume2.json + +aws ec2 attach-volume --volume-id vol-0c3940cade857692b --instance-id i-08fec1692931e31e7 --device '/dev/xvdf' > ./src/fixtures/volumes-attach-volume1.json +aws ec2 attach-volume --volume-id vol-059b4ea55caf83199 --instance-id i-08fec1692931e31e7 --device '/dev/xvda' > ./src/fixtures/volumes-attach-volume2.json + +``` diff --git a/src/jest-global-setup.ts b/src/jest-global-setup.ts deleted file mode 100644 index 805b0bd..0000000 --- a/src/jest-global-setup.ts +++ /dev/null @@ -1,5 +0,0 @@ -export default async () => { - // It is essential we explicitly set the timezone so that our tests that - // check around edge cases for dates run deterministically in all environments. - process.env.TZ = "America/Los_Angeles"; -}; diff --git a/src/jest.setup.ts b/src/jest.setup.ts new file mode 100644 index 0000000..e69de29 diff --git a/src/lib/aws-helpers.test.ts b/src/lib/aws-helpers.test.ts new file mode 100644 index 0000000..22eb0d1 --- /dev/null +++ b/src/lib/aws-helpers.test.ts @@ -0,0 +1,71 @@ +import { + tagsAsObject, + SnapshotDetails, + snapshotDetailsToTag, + snapshotDetailsFromTag, +} from "./aws-helpers"; + +describe("aws-helpers", () => { + describe("tagsAsObject", () => { + it("should correctly convert an array of tags into an object", () => { + // Test case with a sample array of Tag objects + const tags = [ + { + Key: "Name", + Value: "Torrent Box", + }, + { + Key: "Owner", + Value: "dwmkerr", + }, + { + Key: "Project", + Value: "github.com/dwmkerr/dwmkerr", + }, + { + Key: "boxes.boxid", + Value: "torrentbox", + }, + { + Key: "boxes.volumesnapshots", + Value: "[]", + }, + { + Key: "missing value", + Value: "", + }, + ]; + + const expectedObject = { + Name: "Torrent Box", + Owner: "dwmkerr", + Project: "github.com/dwmkerr/dwmkerr", + "boxes.boxid": "torrentbox", + "boxes.volumesnapshots": "[]", + "missing value": "", + }; + + // Call the function and assert the result + const result = tagsAsObject(tags); + expect(result).toEqual(expectedObject); + }); + }); + + describe("snapshotDetails serialization and deserialization", () => { + it("should serialize and deserialize snapshot details correctly", () => { + // Assert we can go to/from the JSON form of the snapshot details + // which are stored in an AWS tag. + const originalSnapshotDetails: SnapshotDetails[] = [ + { snapshotId: "snap-03c3efc7e9254ab0a", device: "/dev/xvda" }, + { snapshotId: "snap-056afd3da4b3b003b", device: "/dev/xvdf" }, + ]; + + const serializedString = snapshotDetailsToTag(originalSnapshotDetails); + const deserializedSnapshotDetails = + snapshotDetailsFromTag(serializedString); + + // Then: The deserialized result should match the original + expect(deserializedSnapshotDetails).toEqual(originalSnapshotDetails); + }); + }); +}); diff --git a/src/lib/aws-helpers.ts b/src/lib/aws-helpers.ts new file mode 100644 index 0000000..5cf0bee --- /dev/null +++ b/src/lib/aws-helpers.ts @@ -0,0 +1,157 @@ +import dbg from "debug"; +import { + EC2Client, + DescribeVolumesCommand, + DescribeInstancesCommand, + Tag, +} from "@aws-sdk/client-ec2"; + +const debug = dbg("boxes:aws"); + +export function tagsAsObject(tags: Tag[] | undefined): Record { + return ( + tags?.reduce((result: Record, tag) => { + return { + ...result, + ...(tag?.Key && { [tag.Key]: tag.Value || "" }), + }; + }, {}) || {} + ); +} + +export interface SnapshotDetails { + snapshotId: string; + device: string; +} + +export function snapshotDetailsToTag( + snapshotDetails: SnapshotDetails[], +): string { + return JSON.stringify( + snapshotDetails.map((snapshot) => ({ + device: snapshot.device, + snapshotId: snapshot.snapshotId, + })), + ); +} + +export function snapshotDetailsFromTag(tagValue: string) { + const rawDetails = JSON.parse(tagValue) as Record[]; + const snapshots = rawDetails.map((raw) => { + if (!raw["device"] || !raw["snapshotId"]) { + throw new Error("snapshot details tag missing device/volume data"); + } + return { + device: raw["device"], + snapshotId: raw["snapshotId"], + }; + }); + return snapshots; +} + +export async function waitForVolumeReady( + client: EC2Client, + volumeId: string, + interval = 5000, + maxAttempts = 60, +) { + // When running unit tests in Jest we can return immediately as + // all service calls are mocked to correct values. This function + // can be tested independently in the future. + if (process.env.JEST_WORKER_ID !== undefined) { + return true; + } + + let attempts = 0; + let volumeState = ""; + + do { + try { + const { Volumes } = await client.send( + new DescribeVolumesCommand({ VolumeIds: [volumeId] }), + ); + if (Volumes && Volumes.length > 0 && Volumes?.[0].State) { + volumeState = Volumes[0].State; + } else { + throw new Error("Volume not found"); + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (error: any) { + throw new Error(`Error describing volume ${volumeId}: ${error.message}`); + } + + if (volumeState === "available") { + debug(`volume ${volumeId} is now in a ready state.`); + return true; + } + + attempts++; + debug( + `waiting for volume ${volumeId} to be in a ready state (attempt ${attempts}/${maxAttempts})...`, + ); + + await new Promise((resolve) => setTimeout(resolve, interval)); + } while (attempts < maxAttempts); + + return false; +} + +type EC2InstanceState = + | "pending" + | "running" + | "shutting-down" + | "terminated" + | "stopping" + | "stopped"; + +export async function waitForInstanceState( + client: EC2Client, + instanceId: string, + targetState: EC2InstanceState, + interval = 5000, + maxAttempts = 60, +) { + // When running unit tests in Jest we can return immediately as + // all service calls are mocked to correct values. This function + // can be tested independently in the future. + if (process.env.JEST_WORKER_ID !== undefined) { + return true; + } + + let attempts = 0; + + do { + try { + const { Reservations } = await client.send( + new DescribeInstancesCommand({ + InstanceIds: [instanceId], + }), + ); + const currentState = Reservations?.[0].Instances?.[0].State?.Name; + if (!currentState) { + throw new Error("Instance or instance state not found"); + } + if (currentState === (targetState as string)) { + debug(`instance ${instanceId} is now in target state '${targetState}'`); + return true; + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (error: any) { + throw new Error( + `Error describing instance ${instanceId}: ${error.message}`, + ); + } + + attempts++; + debug( + `waiting for instance ${instanceId} to be in target state '${targetState}' (attempt ${attempts}/${maxAttempts})...`, + ); + + await new Promise((resolve) => setTimeout(resolve, interval)); + } while (attempts < maxAttempts); + + debug( + `timeout waiting for instance ${instanceId} to be in the target state '${targetState}'`, + ); + return false; +} diff --git a/src/lib/boxes-tags.ts b/src/lib/boxes-tags.ts new file mode 100644 index 0000000..e69de29 diff --git a/src/lib/cli-helpers.ts b/src/lib/cli-helpers.ts new file mode 100644 index 0000000..e2d8fe8 --- /dev/null +++ b/src/lib/cli-helpers.ts @@ -0,0 +1,19 @@ +import { TerminatingWarning } from "./errors"; + +export async function assertConfirmation( + // Commander JS uses 'any' for options, so disable the warning. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + options: any, + confirmationFlag: string, + message: string, +) { + // If the user has provided the required confirmation option, we can return + // safely. + if (options[confirmationFlag] === true) { + return; + } + + // The user has not provided the required confirmation flag, so we must warn + // and fail. + throw new TerminatingWarning(message); +} diff --git a/src/lib/constants.ts b/src/lib/constants.ts new file mode 100644 index 0000000..e1ef2b9 --- /dev/null +++ b/src/lib/constants.ts @@ -0,0 +1,4 @@ +export const tagNames = { + boxId: "boxes.boxid", + volumeArchives: "boxes.volumearchives", +}; diff --git a/src/lib/get-boxes-costs.test.ts b/src/lib/get-boxes-costs.test.ts index 0e1b878..99d1376 100644 --- a/src/lib/get-boxes-costs.test.ts +++ b/src/lib/get-boxes-costs.test.ts @@ -1,49 +1,44 @@ -import { jest } from "@jest/globals"; +import path from "path"; +import mock from "mock-fs"; import { CostExplorerClient, GetCostAndUsageCommand, } from "@aws-sdk/client-cost-explorer"; // ES Modules import import { mockClient } from "aws-sdk-client-mock"; import "aws-sdk-client-mock-jest"; -import { getBoxesCosts } from "./get-boxes-costs"; +import { getBoxesCosts, dateToLocalDateString } from "./get-boxes-costs"; import getMonthlyCostsResponse from "../fixtures/aws-ce-get-costs.json"; describe("get-boxes-costs", () => { - beforeAll(() => { - jest.useFakeTimers(); - }); - afterAll(() => { - jest.useRealTimers(); + // Mock the config file. + beforeEach(() => { + const boxesPath = path.join(path.resolve(), "./boxes.json"); + mock({ + [boxesPath]: mock.load( + path.join(path.resolve(), "./src/fixtures/boxes.json"), + ), + }); }); - - test("time zone is set properly for tests", () => { - expect(process.env.TZ).toEqual("America/Los_Angeles"); - expect(new Date().toString()).toMatch(/Pacific Standard Time/); + afterEach(() => { + mock.restore(); }); - // Note this test is skipped as it seems that the 'setSystemTime' - // call is not working properly, so the curent time is being used. - test.skip("can get boxes costs with default options", async () => { + test("can get boxes costs for a given month", async () => { const ecMock = mockClient(CostExplorerClient) .on(GetCostAndUsageCommand) .resolves(getMonthlyCostsResponse); - const boxCosts = await getBoxesCosts(); - - // Set the time to the end of the month UTC - this means that if we are - // running in any locale which is > 0 from UTC the localisation must be - // working properly (otherwise this'll show as december, not november). - // We explicitly run our tests in Los Angeles (UTC+8 or UTC+7) to allow - // us to test edge cases like this. - const mockedCurrentDate = new Date("2023-11-30T23:59:59.000Z"); - jest.setSystemTime(mockedCurrentDate.getTime()); + const boxCosts = await getBoxesCosts({ + yearNumber: 2023, + monthNumber: 11, + }); // Assert that we've hit the mocked current month from the first date // to the last. expect(ecMock).toHaveReceivedCommandWith(GetCostAndUsageCommand, { TimePeriod: { - Start: `${mockedCurrentDate.getFullYear()}-${mockedCurrentDate.getMonth()}-01`, - End: `${mockedCurrentDate.getFullYear()}-${mockedCurrentDate.getMonth()}-30`, + Start: `2023-11-01`, + End: `2023-11-30`, }, Metrics: ["UNBLENDED_COST"], Granularity: "MONTHLY", @@ -57,26 +52,24 @@ describe("get-boxes-costs", () => { ]); }); - test("correctly sets the month number if provided", async () => { + test("defaults to the current year and month if no options provided", async () => { + // Get the first and last date of the current month. + const now = new Date(); + const firstDay = new Date(now.getFullYear(), now.getMonth(), 1); + const lastDay = new Date(now.getFullYear(), now.getMonth() + 1, 0); + + // Mock, then call the get costs function. const ecMock = mockClient(CostExplorerClient) .on(GetCostAndUsageCommand) .resolves(getMonthlyCostsResponse); - const mockedCurrentDate = new Date("2023-11-13T23:59:59.000Z"); - jest.setSystemTime(mockedCurrentDate.getTime()); - - // Explicitly look for the previous month from the mocked current date. - const monthNumber = 10; - debugger; - await getBoxesCosts({ - yearNumber: 2023, - monthNumber, - }); + await getBoxesCosts(); - // Assert that we've hit the mocked current year with the specified month. + // ...we don't care about the result of the call, just that the dates + // specified matched the current month. expect(ecMock).toHaveReceivedCommandWith(GetCostAndUsageCommand, { TimePeriod: { - Start: `${mockedCurrentDate.getFullYear()}-${monthNumber}-01`, - End: `${mockedCurrentDate.getFullYear()}-${monthNumber}-31`, + Start: `${dateToLocalDateString(firstDay)}`, + End: `${dateToLocalDateString(lastDay)}`, }, Metrics: ["UNBLENDED_COST"], Granularity: "MONTHLY", diff --git a/src/lib/get-boxes-costs.ts b/src/lib/get-boxes-costs.ts index 487eea8..5a54b35 100644 --- a/src/lib/get-boxes-costs.ts +++ b/src/lib/get-boxes-costs.ts @@ -5,7 +5,7 @@ import { import { TerminatingWarning } from "./errors"; import { getConfiguration } from "../configuration"; -function dateToLocalDateString(date: Date): string { +export function dateToLocalDateString(date: Date): string { const year = `${date.getFullYear()}`.padStart(4, "0"); const month = `${date.getMonth() + 1}`.padStart(2, "0"); const day = `${date.getDate()}`.padStart(2, "0"); diff --git a/src/lib/get-boxes.test.ts b/src/lib/get-boxes.test.ts index c3789ab..b094f0c 100644 --- a/src/lib/get-boxes.test.ts +++ b/src/lib/get-boxes.test.ts @@ -1,12 +1,28 @@ +import path from "path"; +import mock from "mock-fs"; import { EC2Client, DescribeInstancesCommand } from "@aws-sdk/client-ec2"; import { mockClient } from "aws-sdk-client-mock"; import "aws-sdk-client-mock-jest"; import { getBoxes } from "./get-boxes"; -import describeInstancesResponse from "../fixtures/aws-ec2-describe-instances.json"; +import describeInstancesResponse from "../fixtures/get-boxes-describe-instances.json"; +import describeInstancesWithArchivedVolumesResponse from "../fixtures/get-boxes-describe-instances-with-archived-volumes.json"; import { BoxState } from "../box"; describe("get-boxes", () => { + // Mock the config file. + beforeEach(() => { + const boxesPath = path.join(path.resolve(), "./boxes.json"); + mock({ + [boxesPath]: mock.load( + path.join(path.resolve(), "./src/fixtures/boxes.json"), + ), + }); + }); + afterEach(() => { + mock.restore(); + }); + test("can get boxes", async () => { // Record fixture with: // AWS_PROFILE=dwmkerr aws ec2 describe-instances --filters "Name=tag:boxes.boxid,Values=*" > ./src/fixtures/aws-ec2-describe-instances.json @@ -44,4 +60,31 @@ describe("get-boxes", () => { // we don't care too much about the 'instance' object.. }); }); + + test("can correctly identify if a box has archived volumes", async () => { + // Record fixture with: + // aws ec2 describe-instances --filters "Name=tag:boxes.boxid,Values=*" > ./src/fixtures/get-boxes-describe-instances-with-archived-volumes.json + const ec2Mock = mockClient(EC2Client) + .on(DescribeInstancesCommand) + .resolves(describeInstancesWithArchivedVolumesResponse); + + const boxes = await getBoxes(); + + expect(ec2Mock).toHaveReceivedCommand(DescribeInstancesCommand); + expect(boxes[0]).toMatchObject({ + boxId: "torrentbox", + instanceId: "i-08fec1692931e31e7", + name: "Torrent Box", + state: BoxState.Stopped, + // we don't care too much about the 'instance' object.. + }); + expect(boxes[1]).toMatchObject({ + boxId: "steambox", + instanceId: "i-043a3c1ce6c9ea6ad", + name: "Steam Box", + state: BoxState.Stopped, + hasArchivedVolumes: true, + // we don't care too much about the 'instance' object.. + }); + }); }); diff --git a/src/lib/get-boxes.ts b/src/lib/get-boxes.ts index 54362d4..7ad2b3a 100644 --- a/src/lib/get-boxes.ts +++ b/src/lib/get-boxes.ts @@ -1,19 +1,24 @@ -import { EC2Client, DescribeInstancesCommand, Tag } from "@aws-sdk/client-ec2"; +import dbg from "debug"; +import { EC2Client, DescribeInstancesCommand } from "@aws-sdk/client-ec2"; import { Box, awsStateToBoxState } from "../box"; import { TerminatingWarning } from "./errors"; import { getConfiguration } from "../configuration"; +import { tagsAsObject } from "./aws-helpers"; +import { tagNames } from "./constants"; +const debug = dbg("boxes"); export async function getBoxes(): Promise { const { aws: awsConfig } = await getConfiguration(); const client = new EC2Client(awsConfig); + debug("preparing to describe instances..."); const instancesResponse = await client.send( new DescribeInstancesCommand({ // TODO typescript this seems to not be found... // IncludeAllInstances: true, Filters: [ { - Name: "tag:boxes.boxid", + Name: tagNames.boxId, Values: ["*"], }, ], @@ -23,6 +28,7 @@ export async function getBoxes(): Promise { if (!instancesResponse || !instancesResponse.Reservations) { throw new TerminatingWarning("Failed to query AWS for boxes/reservations"); } + debug("...described successfully"); // Filter down to instances which have a state. const instances = instancesResponse.Reservations.flatMap((r) => { @@ -35,22 +41,18 @@ export async function getBoxes(): Promise { (i) => i?.State?.Name !== "terminated", ); - const boxes = validInstances.map((i) => ({ - boxId: getTagValOr(i?.Tags || [], "boxes.boxid", ""), - instanceId: i?.InstanceId, - name: nameFromTags(i?.Tags || []) || "", - state: awsStateToBoxState(i?.State?.Name), - instance: i, - })); + const boxes = validInstances.map((i): Box => { + const tags = tagsAsObject(i?.Tags); + return { + boxId: tags[tagNames.boxId], + instanceId: i?.InstanceId, + name: tags?.["Name"], + state: awsStateToBoxState(i?.State?.Name), + hasArchivedVolumes: tags.hasOwnProperty(tagNames.volumeArchives), + instance: i, + }; + }); + debug(`found ${boxes.length} boxes`); return boxes; } - -const getTagValOr = (tags: Tag[], tagName: string, fallback: string) => { - return tags.reduce((acc, val) => { - return val.Key == tagName && val.Value !== undefined ? val.Value : acc; - }, fallback); -}; -const nameFromTags = (tags: Tag[]): string => { - return getTagValOr(tags, "Name", ""); -}; diff --git a/src/lib/volumes.test.ts b/src/lib/volumes.test.ts new file mode 100644 index 0000000..0f147ba --- /dev/null +++ b/src/lib/volumes.test.ts @@ -0,0 +1,291 @@ +import path from "path"; +import { + EC2Client, + DescribeVolumesCommand, + CreateSnapshotCommand, + CreateTagsCommand, + DetachVolumeCommand, + CreateVolumeCommand, + DescribeInstancesCommand, + AttachVolumeCommand, + DeleteSnapshotCommand, +} from "@aws-sdk/client-ec2"; +import { mockClient } from "aws-sdk-client-mock"; +import "aws-sdk-client-mock-jest"; +import mock from "mock-fs"; +import { + DetachableVolume, + getDetachableVolumes, + recreateVolumesFromSnapshotTag, + snapshotTagDeleteVolumes, +} from "./volumes"; +import { tagNames } from "./constants"; + +import describeVolumesTorrentBoxResponse from "../fixtures/volumes-describe-volumes-torrent-box.json"; +import createSnapshot1Response from "../fixtures/volumes-create-snapshot-volume1.json"; +import createSnapshot2Response from "../fixtures/volumes-create-snapshot-volume2.json"; +import describeInstancesResponse from "../fixtures/volumes-describe-instances.json"; +import describeInstancesMissingTagsResponse from "../fixtures/volumes-describe-instances-missing-tags.json"; +import createVolume1Response from "../fixtures/volumes-create-volume1.json"; +import createVolume2Response from "../fixtures/volumes-create-volume2.json"; +import attachVolume1Response from "../fixtures/volumes-attach-volume1.json"; +import attachVolume2Response from "../fixtures/volumes-attach-volume2.json"; + +describe("volumes", () => { + // Mock the config file. + beforeEach(() => { + const boxesPath = path.join(path.resolve(), "./boxes.json"); + mock({ + [boxesPath]: mock.load( + path.join(path.resolve(), "./src/fixtures/boxes.json"), + ), + }); + }); + + afterEach(() => { + mock.restore(); + }); + + describe("get-detachable-volumes", () => { + test("can get detachable volumes from 'torrentbox'", async () => { + // Record fixtures with: + // aws ec2 describe-volumes --filters Name=attachment.instance-id,Values=i-08fec1692931e31e7 > ./src/fixtures/aws-ec2-describe-volumes-torrent-box.json + const ec2Mock = mockClient(EC2Client) + .on(DescribeVolumesCommand) + .resolves(describeVolumesTorrentBoxResponse); + + // Get the detachable volumes, assert the command was called with the + // correct instance id. + const instanceId = "i-08fec1692931e31e7"; // fixture 'torrentbox' id + const detachableVolumes = await getDetachableVolumes(instanceId); + + expect(ec2Mock).toHaveReceivedCommandWith(DescribeVolumesCommand, { + Filters: [ + { + Name: "attachment.instance-id", + Values: [instanceId], + }, + ], + }); + + expect(detachableVolumes).toEqual([ + { + volumeId: "vol-0582d7fc0f3d797fc", + device: "/dev/xvda", + }, + { + volumeId: "vol-0987a9ce9bb4c7b1d", + device: "/dev/xvdf", + }, + ]); + }); + }); + + describe("snapshot-and-delete-volumes", () => { + test("can snapshot with tags and delete volumes", async () => { + // Note: to record the fixtures and update this test process, check the + // ./fixtures/volumes-test-script.md file for the AWS commands to use. + const ec2Mock = mockClient(EC2Client) + .on(DetachVolumeCommand, { + VolumeId: "vol-0582d7fc0f3d797fc", + }) + .on(DetachVolumeCommand, { + VolumeId: "vol-0582d7fc0f3d797fc", + }) + .on(CreateSnapshotCommand, { + VolumeId: "vol-0582d7fc0f3d797fc", + }) + .resolves(createSnapshot1Response) + .on(CreateSnapshotCommand, { + VolumeId: "vol-0987a9ce9bb4c7b1d", + }) + .resolves(createSnapshot2Response); + + // Get the detachable volumes, assert the command was called with the + // correct instance id. + const instanceId = "i-08fec1692931e31e7"; // fixture 'torrentbox' id + const detachableVolumes: DetachableVolume[] = [ + { + volumeId: "vol-0582d7fc0f3d797fc", + device: "/dev/xvda", + }, + { + volumeId: "vol-0987a9ce9bb4c7b1d", + device: "/dev/xvdf", + }, + ]; + + const tags = [{ key: tagNames.boxId, value: "torrentbox" }]; + const result = await snapshotTagDeleteVolumes( + instanceId, + detachableVolumes, + tags, + ); + + // Assert we have detached the two volumes. + expect(ec2Mock).toHaveReceivedCommandWith(DetachVolumeCommand, { + VolumeId: "vol-0582d7fc0f3d797fc", + }); + expect(ec2Mock).toHaveReceivedCommandWith(DetachVolumeCommand, { + VolumeId: "vol-0987a9ce9bb4c7b1d", + }); + + // Assert we have created the two snapshots. + expect(ec2Mock).toHaveReceivedCommandWith(CreateSnapshotCommand, { + VolumeId: "vol-0582d7fc0f3d797fc", + TagSpecifications: [ + { + ResourceType: "snapshot", + Tags: [ + { + Key: tagNames.boxId, + Value: "torrentbox", + }, + ], + }, + ], + }); + expect(ec2Mock).toHaveReceivedCommandWith(CreateSnapshotCommand, { + VolumeId: "vol-0987a9ce9bb4c7b1d", + TagSpecifications: [ + { + ResourceType: "snapshot", + Tags: [ + { + Key: tagNames.boxId, + Value: "torrentbox", + }, + ], + }, + ], + }); + + expect(result).toEqual([ + { + volumeId: "vol-0582d7fc0f3d797fc", + device: "/dev/xvda", + snapshotId: "snap-03c3efc7e9254ab0a", + }, + { + volumeId: "vol-0987a9ce9bb4c7b1d", + device: "/dev/xvdf", + snapshotId: "snap-056afd3da4b3b003b", + }, + ]); + + // Now we need to assert that the instance is updated with a set of tags + // that idenfify the snapshots. + expect(ec2Mock).toHaveReceivedCommandWith(CreateTagsCommand, { + Resources: [instanceId], + Tags: [ + { + Key: tagNames.volumeArchives, + Value: + '[{"device":"/dev/xvda","snapshotId":"snap-03c3efc7e9254ab0a"},{"device":"/dev/xvdf","snapshotId":"snap-056afd3da4b3b003b"}]', + }, + ], + }); + }); + }); + + describe("recreateVolumesFromSnapshotTag", () => { + test("throws a TerminatingWarning if the required tags are not present", async () => { + const instanceId = "i-08fec1692931e31e7"; // fixture 'torrentbox' id + const ec2Mock = mockClient(EC2Client) + .on(DescribeInstancesCommand) + .resolves(describeInstancesMissingTagsResponse); + + // Recreate the volumes from the snapshot tag. This response is missing + // the required tag so should throw. Jest doesn't let us check the + // message/type of error particularly well so it's a bit janky here. + await expect( + recreateVolumesFromSnapshotTag(instanceId), + ).rejects.toMatchObject({ + message: + "unable to restore volume snapshots - required tags are missing", + }); + + // Expect the call to the mock. + expect(ec2Mock).toHaveReceivedCommandWith(DescribeInstancesCommand, { + InstanceIds: [instanceId], + }); + }); + + test("can recreate volumes from snapshot tag", async () => { + // Note: to record the fixtures and update this test process, check the + // ./fixtures/volumes-test-script.md file for the AWS commands to use. + const instanceId = "i-08fec1692931e31e7"; // fixture 'torrentbox' id + const device1 = "/dev/xvda"; + const snapshotId1 = "snap-03c3efc7e9254ab0a"; + const device2 = "/dev/xvdf"; + const snapshotId2 = "snap-056afd3da4b3b003b"; + const ec2Mock = mockClient(EC2Client) + .on(DescribeInstancesCommand) + .resolves(describeInstancesResponse) + .on(CreateVolumeCommand, { + SnapshotId: snapshotId1, + }) + .resolves(createVolume1Response) + .on(CreateVolumeCommand, { + SnapshotId: snapshotId2, + }) + .resolves(createVolume2Response) + .on(AttachVolumeCommand, { + Device: device1, + }) + .resolves(attachVolume1Response) + .on(AttachVolumeCommand, { + Device: device2, + }) + .resolves(attachVolume2Response); + + // Recreate the volumes from the snapshot tag. + const recreatedVolumes = await recreateVolumesFromSnapshotTag(instanceId); + + expect(recreatedVolumes).toEqual([ + { + snapshotId: snapshotId1, + device: device1, + volumeId: "vol-0c3940cade857692b", + }, + { + snapshotId: snapshotId2, + device: device2, + volumeId: "vol-059b4ea55caf83199", + }, + ]); + + // First, instance is queried to get snapshot and AZ data. + expect(ec2Mock).toHaveReceivedCommandWith(DescribeInstancesCommand, { + InstanceIds: [instanceId], + }); + + // With the tags loaded and the snapshot data available, expect the + // snapshots to be restored. + expect(ec2Mock).toHaveReceivedCommandWith(CreateVolumeCommand, { + SnapshotId: snapshotId1, + AvailabilityZone: "us-west-2a", // the az of our fixture instance + }); + expect(ec2Mock).toHaveReceivedCommandWith(CreateVolumeCommand, { + SnapshotId: snapshotId2, + AvailabilityZone: "us-west-2a", // the az of our fixture instance + }); + + // Now the newly created volumes should be attached. + expect(ec2Mock).toHaveReceivedCommandWith(AttachVolumeCommand, { + Device: device1, + }); + expect(ec2Mock).toHaveReceivedCommandWith(AttachVolumeCommand, { + Device: device2, + }); + + // Finally, the two snapshots should have been deleted. + expect(ec2Mock).toHaveReceivedCommandWith(DeleteSnapshotCommand, { + SnapshotId: snapshotId1, + }); + expect(ec2Mock).toHaveReceivedCommandWith(DeleteSnapshotCommand, { + SnapshotId: snapshotId2, + }); + }); + }); +}); diff --git a/src/lib/volumes.ts b/src/lib/volumes.ts new file mode 100644 index 0000000..d33de1f --- /dev/null +++ b/src/lib/volumes.ts @@ -0,0 +1,291 @@ +import dbg from "debug"; +import { + DescribeVolumesCommand, + CreateSnapshotCommand, + EC2Client, + VolumeAttachment, + CreateTagsCommand, + DetachVolumeCommand, + DeleteVolumeCommand, + CreateVolumeCommand, + DescribeInstancesCommand, + AttachVolumeCommand, + DeleteSnapshotCommand, +} from "@aws-sdk/client-ec2"; +import { getConfiguration } from "../configuration"; +import { TerminatingWarning } from "./errors"; +import * as aws from "./aws-helpers"; +import { tagNames } from "./constants"; + +const debug = dbg("boxes:volumes"); + +export interface DetachableVolume { + volumeId: string; + device: string; +} + +export interface SnapshottedAndDeletedVolume extends DetachableVolume { + snapshotId: string; +} + +export interface RecreatedVolume extends DetachableVolume { + snapshotId: string; +} + +export async function getDetachableVolumes( + instanceId: string, +): Promise { + // Create an EC2 client. + const { aws: awsConfig } = await getConfiguration(); + const client = new EC2Client(awsConfig); + + // Get the volumes for the box. + debug(`getting detachable volumes for ${instanceId}...`); + const response = await client.send( + new DescribeVolumesCommand({ + Filters: [ + { + Name: "attachment.instance-id", + Values: [instanceId], + }, + ], + }), + ); + + // If there are no volumes, we're done. + if (!response.Volumes) { + debug("no volumes found"); + return []; + } + + // Filter down the volumes to ones which are attached. + const volumeAttachments = response.Volumes.flatMap((volume) => { + return volume?.Attachments?.[0]; + }).filter((va): va is VolumeAttachment => !!va); + + // Grab the detachable volumes from the response. + const detachableVolumes = volumeAttachments.reduce( + (result: DetachableVolume[], attachment) => { + debug( + `found volume ${attachment.VolumeId} on device ${attachment.Device}`, + ); + if (!attachment.VolumeId || !attachment.Device) { + debug(`volumeid or device missing, skipping this volume`); + return result; + } + result.push({ + volumeId: attachment.VolumeId, + device: attachment.Device, + }); + return result; + }, + [], + ); + + debug(`successfully found ${detachableVolumes.length} detachable volumes`); + return detachableVolumes; +} + +export async function snapshotTagDeleteVolumes( + instanceId: string, + volumes: DetachableVolume[], + tags: Record[], +): Promise { + // Create an EC2 client. + const { aws: awsConfig } = await getConfiguration(); + const client = new EC2Client(awsConfig); + const awsTags = tags.map((tag) => ({ + Key: tag.key, + Value: tag.value, + })); + debug(`preparing to snapshot/tag/delete volumes for instance ${instanceId}`); + + debug(`waiting for instance ${instanceId} to be in 'stopped' state...`); + const instanceStopped = await aws.waitForInstanceState( + client, + instanceId, + "stopped", + 5000, + 720, // 5s*720 = 1hr + ); + if (!instanceStopped) { + throw new TerminatingWarning( + `timed out waiting for instance ${instanceId} to enter 'stopped' state`, + ); + } + + // Detach each volume. No useful results are returned, but the client will + // throw on an error. + // TODO we may need to 'wait' as well, no built in parameter for this. + await Promise.all( + volumes.map(async (volume) => { + debug(`detaching ${volume.volumeId}...`); + return await client.send( + new DetachVolumeCommand({ VolumeId: volume.volumeId }), + ); + }), + ); + + // Snapshot each volume. + // TODO we may need to 'wait' as well, no built in parameter for this. + const snapshots = await Promise.all( + volumes.map(async (volume) => { + debug(`snapshotting ${volume.volumeId}...`); + const response = await client.send( + new CreateSnapshotCommand({ + VolumeId: volume.volumeId, + TagSpecifications: [ + { + ResourceType: "snapshot", + Tags: awsTags, + }, + ], + }), + ); + + if (!response.SnapshotId) { + throw new TerminatingWarning( + `Failed to get a snapshot ID when snaphotting volume ${volume}, aborting to prevent data loss`, + ); + } + + return { + ...volume, + snapshotId: response.SnapshotId, + }; + }), + ); + + // Now we must tag the instance with the details of the snapshots, so that + // we can later restore them. Note that there is no response for this call, + // it will just throw for errors. + const snapshotDetailsTag = { + Key: tagNames.volumeArchives, + Value: aws.snapshotDetailsToTag(snapshots), + }; + debug("creating snapshot details tag", snapshotDetailsTag); + await client.send( + new CreateTagsCommand({ + Resources: [instanceId], + Tags: [snapshotDetailsTag], + }), + ); + + // We've created the snapshots, now we can delete the volumes. + await Promise.all( + volumes.map(async (volume) => { + debug(`deleting ${volume.volumeId}...`); + await client.send(new DeleteVolumeCommand({ VolumeId: volume.volumeId })); + }), + ); + + debug( + `successfully snapshotted/tagged/deleted ${snapshots.length} snapshots`, + ); + return snapshots; +} + +export async function recreateVolumesFromSnapshotTag( + instanceId: string, +): Promise { + // Create an EC2 client. + const { aws: awsConfig } = await getConfiguration(); + const client = new EC2Client(awsConfig); + debug(`preparing to recreate volumes for instance ${instanceId}`); + + // Get the details of the instance, we'll need the tags and AZ. + debug(`getting instance details...`); + const result = await client.send( + new DescribeInstancesCommand({ + InstanceIds: [instanceId], + }), + ); + const instance = result?.Reservations?.[0].Instances?.[0]; + if (!instance) { + throw new TerminatingWarning( + `Cannot restore volumes - unable to get instance details for instance '${instanceId}'`, + ); + } + const availabilityZone = instance?.Placement?.AvailabilityZone; + if (!availabilityZone) { + throw new TerminatingWarning( + `Cannot restore volumes - unable to find availability zone for instance instance '${instanceId}'`, + ); + } + + // Get the tags. + const tags = aws.tagsAsObject(instance.Tags); + + // If we don't have the required snapshots tag, we must fail. + const snapshotDetailsTag = tags[tagNames.volumeArchives]; + if (!snapshotDetailsTag) { + throw new TerminatingWarning( + "unable to restore volume snapshots - required tags are missing", + ); + } + + // From the snapshot details tag, load the actual snapshot details. + const snapshotDetails = aws.snapshotDetailsFromTag(snapshotDetailsTag); + debug("loaded snapshot details from instance tag", snapshotDetails); + + // We're now going to go through each snapshot, create a volume, attach, + // and then delete the snapshot. + const recreatedVolumes = await Promise.all( + snapshotDetails.map( + async ({ snapshotId, device }): Promise => { + // First, create the volume from the snapshot. + debug( + `creating volume from snapshot ${snapshotId} in AZ ${availabilityZone}...`, + ); + const { VolumeId: volumeId } = await client.send( + new CreateVolumeCommand({ + SnapshotId: snapshotId, + AvailabilityZone: availabilityZone, + }), + ); + if (!volumeId) { + throw new Error( + `create volume from snapshot ${snapshotId} has no returned volume id`, + ); + } + + // Wait for the volume to become ready. + debug(`waiting for volume ${volumeId} to be ready...`); + const ready = await aws.waitForVolumeReady(client, volumeId); + if (!ready) { + throw new TerminatingWarning( + `timed out waiting for volumes to restore from snapshots`, + ); + } + + // Then attach the snapshot to the instance. + debug(`attaching volume ${volumeId} to ${instanceId} on ${device}...`); + await client.send( + new AttachVolumeCommand({ + InstanceId: instanceId, + VolumeId: volumeId, + Device: device, + }), + ); + + // Finally, delete the snapshot. + debug(`deleting snapshot ${snapshotId}...`); + await client.send( + new DeleteSnapshotCommand({ + SnapshotId: snapshotId, + }), + ); + + // We can return the details of the newly created volume. + return { + volumeId, + device, + snapshotId, + }; + }, + ), + ); + + debug(`successfully recreated ${recreatedVolumes.length} volumes`); + return recreatedVolumes; +}