Permalink
Find file
Fetching contributors…
Cannot retrieve contributors at this time
225 lines (173 sloc) 8.3 KB
// Copyright 2016 Gareth Watts
// Licensed under an MIT license
// See the LICENSE file for details
/*
Command dyndump dumps or restores a single DynamoDB table to/from a JSON
formatted file or collection of S3 objects.
It supports parallel connections for increased throughput, and rate
limiting to a specified read or write capacity.
JSON is emitted as a stream of objects, one per item in the canonical format
used by the DynamoDB API. Each object has a key for each field name with a
value object holding the type and field value. Eg
{
"string-field": {"S": "string value"},
"number-field": {"N": "123"}
}
The following types are defined by the DynamoDB API:
* S - String
* N - Number (encoded in JSON as a string)
* B - Binary (a base64 encoded string)
* BOOL - Boolean
* NULL - Null
* SS - String set
* NS - Number set
* BS - Binary set
* L - List
* M - Map
AWS credentials will be read from EC2 metadata, ~/.aws/credentials or from
the following environment variables:
* AWS_REGION
* AWS_ACCESS_KEY_ID
* AWS_SECRET_ACCESS_KEY
Usage:
dyndump supports four commands:
DUMP
Usage: dyndump dump [--silent] [--no-progress] [-cmpr] [--filename | --stdout] [(--s3-bucket --s3-prefix)] TABLENAME
Dump a table to file or S3
Arguments:
TABLENAME="" Table name to dump from Dynamo
Options:
-c, --consistent-read=false Enable consistent reads (at 2x capacity use)
-f, --filename="" Filename to write data to.
--stdout=false If true then send the output to stdout
-m, --maxitems=0 Maximum number of items to dump. Set to 0 to process all items
-p, --parallel=5 Number of concurrent channels to open to DynamoDB
-r, --read-capacity=5 Average aggregate read capacity to use for scan (set to 0 for unlimited)
--s3-bucket="" S3 bucket name to upload to
--s3-prefix="" Path prefix to use to store data in S3 (eg. "backups/2016-04-01-12:25-")
--silent=false Set to true to disable all non-error output
--no-progress=false Set to true to disable the progress bar
LOAD
Usage: dyndump load [--silent] [--no-progress] [-mpw] (--filename | --stdin | (--s3-bucket --s3-prefix)) TABLENAME
Load a table dump from S3 or file to a DynamoDB table
Arguments:
TABLENAME="" Table name to load into
Options:
--allow-overwrite=false Set to true to overwrite any existing rows
-f, --filename="" Filename to read data from. Set to "-" for stdin
--stdin=false If true then read the dump data from stdin
-m, --maxitems=0 Maximum number of items to load. Set to 0 to process all items
-p, --parallel=4 Number of concurrent channels to open to DynamoDB
-w, --write-capacity=5 Average aggregate write capacity to use for load (set to 0 for unlimited)
--s3-bucket="" S3 bucket name to read from
--s3-prefix="" Path prefix to use to read data from S3 (eg. "backups/2016-04-01-12:25-")
--silent=false Set to true to disable all non-error output
--no-progress=false Set to true to disable the progress bar
INFO
Usage: dyndump info --s3-bucket --s3-prefix
Display backup metadata from an S3 backup
Options:
--s3-bucket="" S3 bucket name to read from
--s3-prefix="" Path prefix to use to read data from S3 (eg. "backups/2016-04-01-12:25-")
DELETE
Usage: dyndump delete [--silent] [--no-progress] --s3-bucket --s3-prefix [--force]
Delete a backup from S3
Options:
--s3-bucket="" S3 bucket name to delete from
--s3-prefix="" Path prefix to use to delete from S3 (eg. "backups/2016-04-01-12:25-")
--force=false Set to true to disable the delete prompt
--silent=false Set to true to disable all non-error output
--no-progress=false Set to true to disable the progress bar
*/
package main
import (
"fmt"
"os"
"time"
"github.com/jawher/mow.cli"
)
const (
maxParallel = 1000
statsFrequency = 2 * time.Second
)
func fail(format string, a ...interface{}) {
fmt.Fprintf(os.Stderr, format+"\n", a...)
cli.Exit(100)
}
func checkGTE(value, min int, flag string) {
if value < min {
fail("%s must be %d or greater", flag, min)
}
}
func checkLTE(value, max int, flag string) {
if value > max {
fail("%s must be %d or less", flag, max)
}
}
func main() {
app := cli.App("dyndump", "Dump and restore DynamoDB database tables")
app.LongDesc = "long desc goes here"
app.Command("dump", "Dump a table to file or S3", func(cmd *cli.Cmd) {
cmd.Spec = "[-cmpr] [--filename | --stdout] [(--s3-bucket --s3-prefix)] TABLENAME"
action := &dumper{
tableName: cmd.StringArg("TABLENAME", "", "Table name to dump from Dynamo"),
consistentRead: cmd.BoolOpt("c consistent-read", false, "Enable consistent reads (at 2x capacity use)"),
filename: cmd.StringOpt("f filename", "", "Filename to write data to."),
stdout: cmd.BoolOpt("stdout", false, "If true then send the output to stdout"),
maxItems: cmd.IntOpt("m maxitems", 0, "Maximum number of items to dump. Set to 0 to process all items"),
parallel: cmd.IntOpt("p parallel", 5, "Number of concurrent channels to open to DynamoDB"),
readCapacity: cmd.IntOpt("r read-capacity", 5, "Average aggregate read capacity to use for scan (set to 0 for unlimited)"),
s3BucketName: cmd.StringOpt("s3-bucket", "", "S3 bucket name to upload to"),
s3Prefix: cmd.StringOpt("s3-prefix", "", `Path prefix to use to store data in S3 (eg. "backups/2016-04-01-12:25-")`),
}
cmd.Before = func() {
checkGTE(*action.parallel, 1, "--parallel")
checkLTE(*action.parallel, maxParallel, "--parallel")
checkGTE(*action.maxItems, 0, "--max-items")
checkGTE(*action.readCapacity, 0, "--read-capacity")
if *action.filename == "" && !*action.stdout && *action.s3BucketName == "" {
fail("Either --filename/--stdout and/or --s3-bucket and --s3-prefix must be set")
}
}
cmd.Action = actionRunner(cmd, action)
})
app.Command("load", "Load a table dump from S3 or file to a DynamoDB table", func(cmd *cli.Cmd) {
cmd.Spec = "[-mpw] [--allow-overwrite] (--filename | --stdin | (--s3-bucket --s3-prefix)) TABLENAME"
action := &loader{
tableName: cmd.StringArg("TABLENAME", "", "Table name to load into"),
allowOverwrite: cmd.BoolOpt("allow-overwrite", false, "Set to true to overwrite any existing rows"),
filename: cmd.StringOpt("f filename", "", "Filename to read data from. Set to \"-\" for stdin"),
stdin: cmd.BoolOpt("stdin", false, "If true then read the dump data from stdin"),
maxItems: cmd.IntOpt("m maxitems", 0, "Maximum number of items to load. Set to 0 to process all items"),
parallel: cmd.IntOpt("p parallel", 4, "Number of concurrent channels to open to DynamoDB"),
writeCapacity: cmd.IntOpt("w write-capacity", 5, "Average aggregate write capacity to use for load (set to 0 for unlimited)"),
s3BucketName: cmd.StringOpt("s3-bucket", "", "S3 bucket name to read from"),
s3Prefix: cmd.StringOpt("s3-prefix", "", `Path prefix to use to read data from S3 (eg. "backups/2016-04-01-12:25-")`),
}
cmd.Before = func() {
checkGTE(*action.parallel, 1, "--parallel")
checkLTE(*action.parallel, maxParallel, "--parallel")
checkGTE(*action.maxItems, 0, "--max-items")
checkGTE(*action.writeCapacity, 0, "--write-capacity")
}
cmd.Action = actionRunner(cmd, action)
})
app.Command("info", "Display backup metadata from an S3 backup", func(cmd *cli.Cmd) {
cmd.Spec = "--s3-bucket --s3-prefix"
action := &metadataDumper{
s3BucketName: cmd.StringOpt("s3-bucket", "", "S3 bucket name to read from"),
s3Prefix: cmd.StringOpt("s3-prefix", "", `Path prefix to use to read data from S3 (eg. "backups/2016-04-01-12:25-")`),
}
cmd.Action = action.run
})
app.Command("delete", "Delete a backup from S3", func(cmd *cli.Cmd) {
cmd.Spec = "--s3-bucket --s3-prefix [--force]"
action := &deleter{
s3BucketName: cmd.StringOpt("s3-bucket", "", "S3 bucket name to delete from"),
s3Prefix: cmd.StringOpt("s3-prefix", "", `Path prefix to use to delete from S3 (eg. "backups/2016-04-01-12:25-")`),
force: cmd.BoolOpt("force", false, "Set to true to disable the delete prompt"),
}
cmd.Action = actionRunner(cmd, action)
})
app.Run(os.Args)
}