From c0f60c8ec51e2b07aac7a93fe8795d1675c0cd46 Mon Sep 17 00:00:00 2001 From: James Hunt Date: Wed, 1 Aug 2018 17:30:51 -0400 Subject: [PATCH] Make Compression Optional (#437) Compression is now optional, and can be set on a per-target basis. --- .gitignore | 5 + Makefile | 13 ++- agent/command.go | 3 + agent/test/bin/shield-report | 2 + bin/shield-pipe | 82 +++++++++++-- client/v2/shield/archives.go | 1 + client/v2/shield/targets.go | 11 +- cmd/shield-report/main.go | 56 +++++++++ cmd/shield/main.go | 49 +++++--- core/agent.go | 1 + core/compression.go | 7 ++ core/core.go | 13 ++- core/v2.go | 76 +++++++++---- db/archives.go | 17 +-- db/archives_test.go | 1 + db/jobs.go | 33 +++--- db/schema.go | 1 + db/schema_test.go | 2 +- db/schema_v5.go | 44 +++++++ db/targets.go | 40 +++---- db/tasks.go | 44 ++++--- db/tasks_test.go | 4 +- docs/API.md | 214 ++++++++++++++++++---------------- docs/API.yml | 215 +++++++++++++++++++---------------- t/travis | 1 + web2/htdocs/index.html | 62 ++++++---- web2/htdocs/shield.css | 2 +- web2/htdocs/shield.js | 1 + web2/src/js/shield.js | 1 + 29 files changed, 656 insertions(+), 345 deletions(-) create mode 100755 agent/test/bin/shield-report create mode 100644 cmd/shield-report/main.go create mode 100644 core/compression.go create mode 100644 db/schema_v5.go diff --git a/.gitignore b/.gitignore index 647172e16..ddbbde3a5 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ /shield-schema /buckler /shield-crypt +/shield-report # stuff we use in t/* /bin/vault @@ -48,6 +49,10 @@ # ignore tmp stuff from testdev /tmp +# ignore travis test bins +/t/travis-bin +/vault.zip + # ignore testing files cmd/shield-umc/shield-umc cmd/shield-umc/main diff --git a/Makefile b/Makefile index 60857d4f8..5e8c49b5f 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ BUILD_TYPE?=build # Everything; this is the default behavior -all: format shieldd shield shield-agent shield-schema shield-migrate shield-crypt plugins test +all: format shieldd shield shield-agent shield-schema shield-migrate shield-crypt shield-report plugins test # go fmt ftw format: @@ -18,7 +18,7 @@ plugin-tests: plugins @rm -f mock go-tests: export PATH=$$PATH:test/bin; go list ./... | grep -v vendor | xargs go test -api-tests: +api-tests: shieldd shield-schema shield-crypt shield-agent shield-report ./t/api # Running Tests for race conditions @@ -26,7 +26,7 @@ race: ginkgo -race * # Building Shield -shield: shieldd shield-agent shield-schema shield-crypt +shield: shieldd shield-agent shield-schema shield-crypt shield-report shield-crypt: go $(BUILD_TYPE) ./cmd/shield-crypt @@ -38,6 +38,8 @@ shield-schema: go $(BUILD_TYPE) ./cmd/shield-schema shield-migrate: go $(BUILD_TYPE) ./cmd/shield-migrate +shield-report: + go $(BUILD_TYPE) ./cmd/shield-report shield: cmd/shield/help.go go $(BUILD_TYPE) ./cmd/shield @@ -63,7 +65,7 @@ docs/API.md: docs/API.yml mv $@~ $@ clean: - rm -f shield shieldd shield-agent shield-schema shield-crypt shield-migrate + rm -f shield shieldd shield-agent shield-schema shield-crypt shield-migrate shield-report rm -f $$(cat plugins) dummy @@ -110,6 +112,7 @@ release: done go build -ldflags="$(LDFLAGS)" -o "$(ARTIFACTS)/crypter/shield-crypt" ./cmd/shield-crypt go build -ldflags="$(LDFLAGS)" -o "$(ARTIFACTS)/agent/shield-agent" ./cmd/shield-agent + go build -ldflags="$(LDFLAGS)" -o "$(ARTIFACTS)/agent/shield-report" ./cmd/shield-report go build -ldflags="$(LDFLAGS)" -o "$(ARTIFACTS)/cli/shield" ./cmd/shield CGO_ENABLED=1 go build -ldflags="$(LDFLAGS)" -o "$(ARTIFACTS)/daemon/shield-schema" ./cmd/shield-schema CGO_ENABLED=1 go build -ldflags="$(LDFLAGS)" -o "$(ARTIFACTS)/daemon/shield-migrate" ./cmd/shield-migrate @@ -128,4 +131,4 @@ web2/htdocs/shield.js: $(JAVASCRIPTS) web2: web2/htdocs/shield.js -.PHONY: plugins dev web2 shield shieldd shield-schema shield-agent shield-crypt demo +.PHONY: plugins dev web2 shield shieldd shield-schema shield-agent shield-crypt shield-report demo diff --git a/agent/command.go b/agent/command.go index 1f412e95a..83782cb1f 100644 --- a/agent/command.go +++ b/agent/command.go @@ -25,6 +25,7 @@ type Command struct { EncryptType string `json:"encrypt_type,omitempty"` EncryptKey string `json:"encrypt_key,omitempty"` EncryptIV string `json:"encrypt_iv,omitempty"` + Compression string `json:"compression,omitempty"` } func ParseCommand(b []byte) (*Command, error) { @@ -80,6 +81,7 @@ func ParseCommand(b []byte) (*Command, error) { if cmd.RestoreKey == "" { return nil, fmt.Errorf("missing required 'restore_key' value in payload (for purge operation)") } + case "test-store": if cmd.StorePlugin == "" { return nil, fmt.Errorf("missing required 'store_plugin' value in payload") @@ -153,6 +155,7 @@ func (agent *Agent) Execute(c *Command, out chan string) error { fmt.Sprintf("SHIELD_ENCRYPT_TYPE=%s", c.EncryptType), fmt.Sprintf("SHIELD_ENCRYPT_KEY=%s", c.EncryptKey), fmt.Sprintf("SHIELD_ENCRYPT_IV=%s", c.EncryptIV), + fmt.Sprintf("SHIELD_COMPRESSION=%s", c.Compression), } if log.LogLevel() == syslog.LOG_DEBUG { diff --git a/agent/test/bin/shield-report b/agent/test/bin/shield-report new file mode 100755 index 000000000..00990993b --- /dev/null +++ b/agent/test/bin/shield-report @@ -0,0 +1,2 @@ +#!/bin/sh +exec cat \ No newline at end of file diff --git a/bin/shield-pipe b/bin/shield-pipe index 540394e41..573de6a78 100755 --- a/bin/shield-pipe +++ b/bin/shield-pipe @@ -15,6 +15,7 @@ # SHIELD_STORE_PLUGIN Path to the store plugin to use # SHIELD_STORE_ENDPOINT The store endpoint config (probably JSON) # SHIELD_RESTORE_KEY Archive key for 'restore' operations +# SHIELD_COMPRESSION What type of compression to perform # # Temporary Environment Variables (Unset before call to shield plugin) # --------------------- @@ -106,6 +107,8 @@ if [ -n "$SHIELD_ENCRYPT_IV" ]; then unset SHIELD_ENCRYPT_IV fi +SHIELD_COMPRESSION=${SHIELD_COMPRESSION:-bzip2} + case ${SHIELD_OP} in (status) needenv SHIELD_OP \ @@ -210,16 +213,47 @@ exit 0 validate TARGET ${SHIELD_TARGET_PLUGIN} "${SHIELD_TARGET_ENDPOINT}" validate STORE ${SHIELD_STORE_PLUGIN} "${SHIELD_STORE_ENDPOINT}" - header "Running backup task (using bzip2 compression)" + case $SHIELD_COMPRESSION in + bzip2) header "Running backup task (using bzip2 compression)" ;; + none) header "Running backup task (without compression)" ;; + *) + fail "Unrecognized compression scheme '$SHIELD_COMPRESSION'" + exit 145 + ;; + esac PULSE=$(mktemp -t shield-pipe.XXXXX) trap "rm -f ${PULSE}" QUIT TERM INT set -o pipefail - #The use of 3<<< shown below is to write the encryption config as json to FD 3 thus allowing us to drop it from the environment - # and prevent further propogation - ${SHIELD_TARGET_PLUGIN} backup -e "${SHIELD_TARGET_ENDPOINT}" | \ - tee >(tail -c1 >$PULSE) | bzip2 | shield-crypt --encrypt 3<<<"{\"enc_key\":\"$enc_key\",\"enc_iv\":\"$enc_iv\",\"enc_type\":\"$enc_type\"}" | ${SHIELD_STORE_PLUGIN} store -e "${SHIELD_STORE_ENDPOINT}" + + # The use of 3<<< shown below is to write the encryption + # config as JSON to fd3, allowing us to drop it from the + # environment and prevent further propogation + + case $SHIELD_COMPRESSION in + bzip2) + ${SHIELD_TARGET_PLUGIN} backup -e "${SHIELD_TARGET_ENDPOINT}" | \ + tee >(tail -c1 >$PULSE) | \ + bzip2 | \ + shield-crypt --encrypt 3<<<"{\"enc_key\":\"$enc_key\",\"enc_iv\":\"$enc_iv\",\"enc_type\":\"$enc_type\"}" | \ + ${SHIELD_STORE_PLUGIN} store -e "${SHIELD_STORE_ENDPOINT}" | \ + shield-report --compression bzip2 + ;; + + none) + ${SHIELD_TARGET_PLUGIN} backup -e "${SHIELD_TARGET_ENDPOINT}" | \ + tee >(tail -c1 >$PULSE) | \ + shield-crypt --encrypt 3<<<"{\"enc_key\":\"$enc_key\",\"enc_iv\":\"$enc_iv\",\"enc_type\":\"$enc_type\"}" | \ + ${SHIELD_STORE_PLUGIN} store -e "${SHIELD_STORE_ENDPOINT}" | \ + shield-report --compression none + ;; + + *) + fail "Unrecognized compression scheme '$SHIELD_COMPRESSION'" + exit 145 + ;; + esac if [[ ! -s ${PULSE} ]]; then rm -f ${PULSE} @@ -243,12 +277,40 @@ exit 0 validate TARGET ${SHIELD_TARGET_PLUGIN} "${SHIELD_TARGET_ENDPOINT}" validate STORE ${SHIELD_STORE_PLUGIN} "${SHIELD_STORE_ENDPOINT}" - header "Running restore task (using bzip2 compression)" + case $SHIELD_COMPRESSION in + bzip2) header "Running restore task (using bzip2 compression)" ;; + none) header "Running restore task (without compression)" ;; + *) + fail "Unrecognized compression scheme '$SHIELD_COMPRESSION'" + exit 145 + ;; + esac + set -o pipefail - #The use of 3<<< shown below is to write the encryption config as json to FD 3 thus allowing us to drop it from the environment - # and prevent further propogation - ${SHIELD_STORE_PLUGIN} retrieve -k "${SHIELD_RESTORE_KEY}" -e "${SHIELD_STORE_ENDPOINT}" | shield-crypt --decrypt 3<<<"{\"enc_key\":\"$enc_key\",\"enc_iv\":\"$enc_iv\",\"enc_type\":\"$enc_type\"}" | bunzip2 | \ - ${SHIELD_TARGET_PLUGIN} restore -e "${SHIELD_TARGET_ENDPOINT}" + + # The use of 3<<< shown below is to write the encryption + # config as JSON to fd3, allowing us to drop it from the + # environment and prevent further propogation + + case $SHIELD_COMPRESSION in + bzip2) + ${SHIELD_STORE_PLUGIN} retrieve -k "${SHIELD_RESTORE_KEY}" -e "${SHIELD_STORE_ENDPOINT}" | \ + shield-crypt --decrypt 3<<<"{\"enc_key\":\"$enc_key\",\"enc_iv\":\"$enc_iv\",\"enc_type\":\"$enc_type\"}" | \ + bunzip2 | \ + ${SHIELD_TARGET_PLUGIN} restore -e "${SHIELD_TARGET_ENDPOINT}" + ;; + + none) + ${SHIELD_STORE_PLUGIN} retrieve -k "${SHIELD_RESTORE_KEY}" -e "${SHIELD_STORE_ENDPOINT}" | \ + shield-crypt --decrypt 3<<<"{\"enc_key\":\"$enc_key\",\"enc_iv\":\"$enc_iv\",\"enc_type\":\"$enc_type\"}" | \ + ${SHIELD_TARGET_PLUGIN} restore -e "${SHIELD_TARGET_ENDPOINT}" + ;; + + *) + fail "Unrecognized compression scheme '$SHIELD_COMPRESSION'" + exit 145 + ;; + esac exit 0 ;; diff --git a/client/v2/shield/archives.go b/client/v2/shield/archives.go index cc4599e88..94c5e6cf7 100644 --- a/client/v2/shield/archives.go +++ b/client/v2/shield/archives.go @@ -16,6 +16,7 @@ type Archive struct { Store *Store `json:"store,omitempty"` Policy *Policy `json:"policy,omitempty"` + Compression string `json:"compression"` EncryptionType string `json:"encryption_type"` Size int64 `json:"size"` } diff --git a/client/v2/shield/targets.go b/client/v2/shield/targets.go index da4b7b786..5b0ca140f 100644 --- a/client/v2/shield/targets.go +++ b/client/v2/shield/targets.go @@ -8,11 +8,12 @@ import ( ) type Target struct { - UUID string `json:"uuid,omitempty"` - Name string `json:"name"` - Summary string `json:"summary"` - Plugin string `json:"plugin"` - Agent string `json:"agent"` + UUID string `json:"uuid,omitempty"` + Name string `json:"name"` + Summary string `json:"summary"` + Plugin string `json:"plugin"` + Agent string `json:"agent"` + Compression string `json:"compression"` Config map[string]interface{} `json:"config"` } diff --git a/cmd/shield-report/main.go b/cmd/shield-report/main.go new file mode 100644 index 000000000..79760b0f4 --- /dev/null +++ b/cmd/shield-report/main.go @@ -0,0 +1,56 @@ +package main + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + + "github.com/jhunt/go-cli" +) + +var opt struct { + Help bool `cli:"-h, --help"` + + Compression string `cli:"-c, --compression"` +} + +func main() { + if _, _, err := cli.Parse(&opt); err != nil { + fmt.Fprintf(os.Stderr, "!!! shield-report utility failed to parse command-line flags: %s\n", err) + os.Exit(2) + } + + if opt.Help { + fmt.Printf("echo '{\"some\":\"json\"}' | shield-report [OPTIONS]\n\n") + fmt.Printf("OPTIONS\n\n") + fmt.Printf(" -h, --help Show this help screen.\n") + fmt.Printf(" -c, --compression ... Set the \"compression\" key in the output JSON.\n") + fmt.Printf("\n") + os.Exit(0) + } + + b, err := ioutil.ReadAll(os.Stdin) + if err != nil { + fmt.Fprintf(os.Stderr, "!!! shield-report utility failed to read standard input: %s\n", err) + os.Exit(3) + } + + var data map[string]interface{} + if err := json.Unmarshal(b, &data); err != nil { + fmt.Fprintf(os.Stderr, "!!! shield-report utility failed to parse JSON from standard input: %s\n", err) + os.Exit(3) + } + + if opt.Compression != "" { + data["compression"] = opt.Compression + } + + b, err = json.Marshal(data) + if err != nil { + fmt.Fprintf(os.Stderr, "!!! shield-report utility failed to encode output JSON: %s\n", err) + os.Exit(4) + } + os.Stdout.Write(b) + os.Exit(0) +} diff --git a/cmd/shield/main.go b/cmd/shield/main.go index 8f8029bfa..9d1bb0fc0 100644 --- a/cmd/shield/main.go +++ b/cmd/shield/main.go @@ -126,19 +126,21 @@ var opts struct { Target struct{} `cli:"target"` DeletTarget struct{} `cli:"delete-target"` CreateTarget struct { - Name string `cli:"-n, --name"` - Summary string `cli:"-s, --summary"` - Agent string `cli:"-a, --agent"` - Plugin string `cli:"-p, --plugin"` - Data []string `cli:"-d, --data"` + Name string `cli:"-n, --name"` + Summary string `cli:"-s, --summary"` + Agent string `cli:"-a, --agent"` + Plugin string `cli:"-p, --plugin"` + Data []string `cli:"-d, --data"` + Compression string `cli:"-C, --compression"` } `cli:"create-target"` UpdateTarget struct { - Name string `cli:"-n, --name"` - Summary string `cli:"-s, --summary"` - Agent string `cli:"-a, --agent"` - Plugin string `cli:"-p, --plugin"` - ClearData bool `cli:"--clear-data"` - Data []string `cli:"-d, --data"` + Name string `cli:"-n, --name"` + Summary string `cli:"-s, --summary"` + Agent string `cli:"-a, --agent"` + Plugin string `cli:"-p, --plugin"` + Compression string `cli:"-C, --compression"` + ClearData bool `cli:"--clear-data"` + Data []string `cli:"-d, --data"` } `cli:"update-target"` /* }}} */ @@ -1315,12 +1317,17 @@ func main() { } } + if opts.CreateTarget.Compression == "" { + opts.CreateTarget.Compression = "bzip2" + } + t, err := c.CreateTarget(tenant, &shield.Target{ - Name: opts.CreateTarget.Name, - Summary: opts.CreateTarget.Summary, - Agent: opts.CreateTarget.Agent, - Plugin: opts.CreateTarget.Plugin, - Config: conf, + Name: opts.CreateTarget.Name, + Summary: opts.CreateTarget.Summary, + Agent: opts.CreateTarget.Agent, + Plugin: opts.CreateTarget.Plugin, + Compression: opts.CreateTarget.Compression, + Config: conf, }) bail(err) @@ -1333,6 +1340,7 @@ func main() { r.Add("UUID", t.UUID) r.Add("Name", t.Name) r.Add("Summary", t.Summary) + r.Add("Compression", t.Compression) r.Add("SHIELD Agent", t.Agent) r.Add("Backup Plugin", t.Plugin) r.Output(os.Stdout) @@ -1365,6 +1373,9 @@ func main() { opts.UpdateTarget.ClearData = true t.Plugin = opts.UpdateTarget.Plugin } + if opts.UpdateTarget.Compression != "" { + t.Compression = opts.UpdateTarget.Compression + } if t.Config == nil { t.Config = make(map[string]interface{}) @@ -1389,6 +1400,7 @@ func main() { r.Add("UUID", t.UUID) r.Add("Name", t.Name) r.Add("Summary", t.Summary) + r.Add("Compression", t.Compression) r.Add("SHIELD Agent", t.Agent) r.Add("Backup Plugin", t.Plugin) r.Output(os.Stdout) @@ -2580,9 +2592,9 @@ func main() { ) */ /* FIXME: support --long / -l and maybe --output / -o "fmt-str" */ - tbl := tui.NewTable("UUID", "Key", "Status") + tbl := tui.NewTable("UUID", "Key", "Compression", "Status") for _, archive := range archives { - tbl.Row(archive, archive.UUID, archive.Key, archive.Status) + tbl.Row(archive, archive.UUID, archive.Key, archive.Compression, archive.Status) } tbl.Output(os.Stdout) @@ -2607,6 +2619,7 @@ func main() { r := tui.NewReport() r.Add("UUID", archive.UUID) r.Add("Key", archive.Key) + r.Add("Compression", archive.Compression) r.Add("Status", archive.Status) r.Output(os.Stdout) diff --git a/core/agent.go b/core/agent.go index c2e442274..aa05385b1 100644 --- a/core/agent.go +++ b/core/agent.go @@ -27,6 +27,7 @@ type AgentCommand struct { EncryptType string `json:"encrypt_type,omitempty"` EncryptKey string `json:"encrypt_key,omitempty"` EncryptIV string `json:"encrypt_iv,omitempty"` + Compression string `json:"compression,omitempty"` } type AgentClient struct { diff --git a/core/compression.go b/core/compression.go new file mode 100644 index 000000000..ee6c139f7 --- /dev/null +++ b/core/compression.go @@ -0,0 +1,7 @@ +package core + +var DefaultCompressionType = "bzip2" + +func ValidCompressionType(t string) bool { + return t == "bzip2" || t == "none" +} diff --git a/core/core.go b/core/core.go index 83e8cf764..b1cb89cb3 100644 --- a/core/core.go +++ b/core/core.go @@ -646,6 +646,7 @@ func (core *Core) worker(id int) { Op: task.Op, TargetPlugin: task.TargetPlugin, TargetEndpoint: task.TargetEndpoint, + Compression: task.Compression, StorePlugin: task.StorePlugin, StoreEndpoint: task.StoreEndpoint, RestoreKey: task.RestoreKey, @@ -681,17 +682,23 @@ func (core *Core) worker(id int) { response := <-stdout if task.Op == db.BackupOperation { var v struct { - Key string `json:"key"` - Size int64 `json:"archive_size"` + Key string `json:"key"` + Size int64 `json:"archive_size"` + Compression string `json:"compression"` } if err := json.Unmarshal([]byte(response), &v); err != nil { core.failTask(task, "shield worker %d failed to parse JSON response from remote agent %s: %s\n", id, task.Agent, err) continue } else { + if v.Compression == "" { + /* older shield-pipes will always bzip2; and if they aren't + reporting their compression type, it's gotta be bzip2 */ + v.Compression = "bzip2" + } if v.Key != "" { log.Infof(" %s: restore key is %s", task.UUID, v.Key) - if _, err := core.DB.CreateTaskArchive(task.UUID, task.ArchiveUUID, v.Key, time.Now(), core.encryptionType, v.Size, task.TenantUUID); err != nil { + if _, err := core.DB.CreateTaskArchive(task.UUID, task.ArchiveUUID, v.Key, time.Now(), core.encryptionType, v.Compression, v.Size, task.TenantUUID); err != nil { log.Errorf(" %s: !! failed to update database: %s", task.UUID, err) } diff --git a/core/v2.go b/core/v2.go index 1da47879a..dcb8e1f58 100644 --- a/core/v2.go +++ b/core/v2.go @@ -56,11 +56,12 @@ type v2SystemTask struct { Log string `json:"log"` } type v2SystemJob struct { - UUID uuid.UUID `json:"uuid"` - Schedule string `json:"schedule"` - From string `json:"from"` - To string `json:"to"` - OK bool `json:"ok"` + UUID uuid.UUID `json:"uuid"` + Schedule string `json:"schedule"` + Compression string `json:"compression"` + From string `json:"from"` + To string `json:"to"` + OK bool `json:"ok"` Store struct { UUID uuid.UUID `json:"uuid"` @@ -83,10 +84,11 @@ type v2SystemJob struct { } `json:"retention"` } type v2System struct { - UUID uuid.UUID `json:"uuid"` - Name string `json:"name"` - Notes string `json:"notes"` - OK bool `json:"ok"` + UUID uuid.UUID `json:"uuid"` + Name string `json:"name"` + Notes string `json:"notes"` + OK bool `json:"ok"` + Compression string `json:"compression"` Jobs []v2SystemJob `json:"jobs"` Tasks []v2SystemTask `json:"tasks"` @@ -1558,10 +1560,11 @@ func (core *Core) v2API() *route.Router { } var in struct { - Name string `json:"name"` - Summary string `json:"summary"` - Plugin string `json:"plugin"` - Agent string `json:"agent"` + Name string `json:"name"` + Summary string `json:"summary"` + Compression string `json:"compression"` + Plugin string `json:"plugin"` + Agent string `json:"agent"` Config map[string]interface{} `json:"config"` endpoint string @@ -1574,6 +1577,7 @@ func (core *Core) v2API() *route.Router { b, err := json.Marshal(in.Config) if err != nil { r.Fail(route.Oops(err, "Unable to create target")) + return } in.endpoint = string(b) } else { @@ -1583,18 +1587,33 @@ func (core *Core) v2API() *route.Router { return } + if in.Compression == "" { + in.Compression = DefaultCompressionType + } + + if !ValidCompressionType(in.Compression) { + r.Fail(route.Bad(err, "Invalid compression type '%s'", in.Compression)) + return + } + if r.ParamIs("test", "t") { r.Success("validation suceeded (request made in ?test=t mode)") return } + if !ValidCompressionType(in.Compression) { + r.Fail(route.Bad(err, "Invalid compression type '%s'", in.Compression)) + return + } + target, err := core.DB.CreateTarget(&db.Target{ - TenantUUID: uuid.Parse(r.Args[1]), - Name: in.Name, - Summary: in.Summary, - Plugin: in.Plugin, - Config: in.Config, - Agent: in.Agent, + TenantUUID: uuid.Parse(r.Args[1]), + Name: in.Name, + Summary: in.Summary, + Plugin: in.Plugin, + Config: in.Config, + Agent: in.Agent, + Compression: in.Compression, }) if target == nil || err != nil { r.Fail(route.Oops(err, "Unable to create new data target")) @@ -1640,11 +1659,12 @@ func (core *Core) v2API() *route.Router { } var in struct { - Name string `json:"name"` - Summary string `json:"summary"` - Plugin string `json:"plugin"` - Endpoint string `json:"endpoint"` - Agent string `json:"agent"` + Name string `json:"name"` + Summary string `json:"summary"` + Compression string `json:"compression"` + Plugin string `json:"plugin"` + Endpoint string `json:"endpoint"` + Agent string `json:"agent"` Config map[string]interface{} `json:"config"` } @@ -1674,6 +1694,13 @@ func (core *Core) v2API() *route.Router { if in.Agent != "" { target.Agent = in.Agent } + if in.Compression != "" { + if !ValidCompressionType(in.Compression) { + r.Fail(route.Bad(err, "Invalid compression type '%s'", in.Compression)) + return + } + target.Compression = in.Compression + } if err := core.DB.UpdateTarget(target); err != nil { r.Fail(route.Oops(err, "Unable to update target")) @@ -3245,6 +3272,7 @@ func (core *Core) v2copyTarget(dst *v2System, target *db.Target) error { dst.Name = target.Name dst.Notes = target.Summary dst.OK = true /* FIXME */ + dst.Compression = target.Compression jobs, err := core.DB.GetAllJobs( &db.JobFilter{ diff --git a/db/archives.go b/db/archives.go index 3ef31c1d1..6ee487ff7 100644 --- a/db/archives.go +++ b/db/archives.go @@ -26,6 +26,7 @@ type Archive struct { StoreEndpoint string `json:"store_endpoint"` Job string `json:"job"` EncryptionType string `json:"encryption_type"` + Compression string `json:"compression"` TenantUUID uuid.UUID `json:"tenant_uuid"` Size int64 `json:"size"` } @@ -98,12 +99,12 @@ func (f *ArchiveFilter) Query() (string, []interface{}) { a.taken_at, a.expires_at, a.notes, t.uuid, t.name, t.plugin, t.endpoint, s.uuid, s.name, s.plugin, s.endpoint, - a.status, a.purge_reason, a.job, a.encryption_type, - a.tenant_uuid, a.size + a.status, a.purge_reason, a.job, a.encryption_type, + a.compression, a.tenant_uuid, a.size FROM archives a - INNER JOIN targets t ON t.uuid = a.target_uuid - INNER JOIN stores s ON s.uuid = a.store_uuid + INNER JOIN targets t ON t.uuid = a.target_uuid + INNER JOIN stores s ON s.uuid = a.store_uuid WHERE ` + strings.Join(wheres, " AND ") + ` ORDER BY a.taken_at DESC, a.uuid ASC @@ -153,7 +154,7 @@ func (db *DB) GetAllArchives(filter *ArchiveFilter) ([]*Archive, error) { &target, &targetName, &ann.TargetPlugin, &ann.TargetEndpoint, &store, &storeName, &ann.StorePlugin, &ann.StoreEndpoint, &ann.Status, &ann.PurgeReason, &ann.Job, &ann.EncryptionType, - &tenant, &size); err != nil { + &ann.Compression, &tenant, &size); err != nil { return l, err } @@ -189,8 +190,8 @@ func (db *DB) GetArchive(id uuid.UUID) (*Archive, error) { a.taken_at, a.expires_at, a.notes, t.uuid, t.name, t.plugin, t.endpoint, s.uuid, s.name, s.plugin, s.endpoint, a.status, - a.purge_reason, a.job, a.encryption_type, - a.tenant_uuid, a.size + a.purge_reason, a.job, a.encryption_type, + a.compression, a.tenant_uuid, a.size FROM archives a INNER JOIN targets t ON t.uuid = a.target_uuid @@ -215,7 +216,7 @@ func (db *DB) GetArchive(id uuid.UUID) (*Archive, error) { &target, &targetName, &ann.TargetPlugin, &ann.TargetEndpoint, &store, &storeName, &ann.StorePlugin, &ann.StoreEndpoint, &ann.Status, &ann.PurgeReason, &ann.Job, &ann.EncryptionType, - &tenant, &size); err != nil { + &ann.Compression, &tenant, &size); err != nil { return nil, err } diff --git a/db/archives_test.go b/db/archives_test.go index a1799f71f..e9e142e51 100644 --- a/db/archives_test.go +++ b/db/archives_test.go @@ -148,6 +148,7 @@ var _ = Describe("Archive Management", func() { Notes: "my_notes", Status: "valid", PurgeReason: "", + Compression: "none", TargetUUID: TARGET_UUID, TargetName: "target_name", TargetPlugin: "target_plugin", diff --git a/db/jobs.go b/db/jobs.go index 20ee71abc..201b0a271 100644 --- a/db/jobs.go +++ b/db/jobs.go @@ -26,10 +26,11 @@ type Job struct { FixedKey bool `json:"fixed_key"` Target struct { - UUID uuid.UUID `json:"uuid"` - Name string `json:"name"` - Agent string `json:"agent"` - Plugin string `json:"plugin"` + UUID uuid.UUID `json:"uuid"` + Name string `json:"name"` + Agent string `json:"agent"` + Plugin string `json:"plugin"` + Compression string `json:"compression"` Endpoint string `json:"endpoint,omitempty"` Config map[string]interface{} `json:"config,omitempty"` @@ -131,10 +132,11 @@ func (f *JobFilter) Query(driver string) (string, []interface{}, error) { GROUP BY job_uuid ) - SELECT j.uuid, j.name, j.summary, j.paused, j.schedule, j.tenant_uuid, j.fixed_key, + SELECT j.uuid, j.name, j.summary, j.paused, j.schedule, + j.tenant_uuid, j.fixed_key, r.name, r.summary, r.uuid, r.expiry, s.uuid, s.name, s.plugin, s.endpoint, s.summary, s.healthy, - t.uuid, t.name, t.plugin, t.endpoint, t.agent, + t.uuid, t.name, t.plugin, t.endpoint, t.agent, t.compression, k.started_at, k.status FROM jobs j @@ -171,11 +173,12 @@ func (db *DB) GetAllJobs(filter *JobFilter) ([]*Job, error) { status sql.NullString ) if err = r.Scan( - &this, &j.Name, &j.Summary, &j.Paused, &j.Schedule, &tenant, &j.FixedKey, + &this, &j.Name, &j.Summary, &j.Paused, &j.Schedule, + &tenant, &j.FixedKey, &j.Policy.Name, &j.Policy.Summary, &policy, &j.Expiry, &store, &j.Store.Name, &j.Store.Plugin, &j.Store.Endpoint, &j.Store.Summary, &j.Store.Healthy, &target, &j.Target.Name, &j.Target.Plugin, &j.Target.Endpoint, - &j.Agent, &last, &status); err != nil { + &j.Agent, &j.Target.Compression, &last, &status); err != nil { return l, err } j.UUID = this.UUID @@ -198,10 +201,11 @@ func (db *DB) GetAllJobs(filter *JobFilter) ([]*Job, error) { func (db *DB) GetJob(id uuid.UUID) (*Job, error) { r, err := db.Query(` - SELECT j.uuid, j.name, j.summary, j.paused, j.schedule, j.tenant_uuid, j.fixed_key, + SELECT j.uuid, j.name, j.summary, j.paused, j.schedule, + j.tenant_uuid, j.fixed_key, r.name, r.summary, r.uuid, r.expiry, s.uuid, s.name, s.plugin, s.endpoint, s.summary, s.healthy, - t.uuid, t.name, t.plugin, t.endpoint, t.agent + t.uuid, t.name, t.plugin, t.endpoint, t.agent, t.compression FROM jobs j INNER JOIN retention r ON r.uuid = j.retention_uuid @@ -221,11 +225,12 @@ func (db *DB) GetJob(id uuid.UUID) (*Job, error) { j := &Job{} var this, policy, store, target, tenant NullUUID if err = r.Scan( - &this, &j.Name, &j.Summary, &j.Paused, &j.Schedule, &tenant, &j.FixedKey, + &this, &j.Name, &j.Summary, &j.Paused, &j.Schedule, + &tenant, &j.FixedKey, &j.Policy.Name, &j.Policy.Summary, &policy, &j.Expiry, &store, &j.Store.Name, &j.Store.Plugin, &j.Store.Endpoint, &j.Store.Summary, &j.Store.Healthy, &target, &j.Target.Name, &j.Target.Plugin, &j.Target.Endpoint, - &j.Agent); err != nil { + &j.Agent, &j.Target.Compression); err != nil { return nil, err } j.UUID = this.UUID @@ -286,8 +291,8 @@ func (db *DB) UpdateJob(job *Job) error { schedule = ?, target_uuid = ?, store_uuid = ?, - retention_uuid = ?, - fixed_key = ? + retention_uuid = ?, + fixed_key = ? WHERE uuid = ?`, job.Name, job.Summary, job.Schedule, job.TargetUUID.String(), job.StoreUUID.String(), job.PolicyUUID.String(), diff --git a/db/schema.go b/db/schema.go index aceca81b1..c39bde627 100644 --- a/db/schema.go +++ b/db/schema.go @@ -13,6 +13,7 @@ var Schemas = map[int]Schema{ 2: v2Schema{}, 3: v3Schema{}, 4: v4Schema{}, + 5: v5Schema{}, } type Schema interface { diff --git a/db/schema_test.go b/db/schema_test.go index 8417cc32d..2b1466d3c 100644 --- a/db/schema_test.go +++ b/db/schema_test.go @@ -77,7 +77,7 @@ var _ = Describe("Database Schema", func() { var v int Ω(r.Scan(&v)).Should(Succeed()) - Ω(v).Should(Equal(4)) + Ω(v).Should(Equal(5)) }) It("creates the correct tables", func() { diff --git a/db/schema_v5.go b/db/schema_v5.go new file mode 100644 index 000000000..8b93fc38a --- /dev/null +++ b/db/schema_v5.go @@ -0,0 +1,44 @@ +package db + +type v5Schema struct{} + +func (s v5Schema) Deploy(db *DB) error { + var err error + + err = db.Exec(`ALTER TABLE targets ADD compression TEXT NOT NULL DEFAULT 'none'`) + if err != nil { + return err + } + + err = db.Exec(`UPDATE targets SET compression = 'bzip2'`) + if err != nil { + return err + } + + err = db.Exec(`ALTER TABLE archives ADD compression TEXT NOT NULL DEFAULT 'none'`) + if err != nil { + return err + } + + err = db.Exec(`UPDATE archives SET compression = 'bzip2'`) + if err != nil { + return err + } + + err = db.Exec(`ALTER TABLE tasks ADD compression TEXT NOT NULL DEFAULT ''`) + if err != nil { + return err + } + + err = db.Exec(`UPDATE tasks SET compression = 'bzip2' WHERE op = 'backup'`) + if err != nil { + return err + } + + err = db.Exec(`UPDATE schema_info set version = 5`) + if err != nil { + return err + } + + return nil +} diff --git a/db/targets.go b/db/targets.go index 65e3bbd58..62ad346a3 100644 --- a/db/targets.go +++ b/db/targets.go @@ -12,11 +12,12 @@ import ( type Target struct { TenantUUID uuid.UUID `json:"-"` - UUID uuid.UUID `json:"uuid"` - Name string `json:"name"` - Summary string `json:"summary"` - Plugin string `json:"plugin"` - Agent string `json:"agent"` + UUID uuid.UUID `json:"uuid"` + Name string `json:"name"` + Summary string `json:"summary"` + Plugin string `json:"plugin"` + Agent string `json:"agent"` + Compression string `json:"compression"` Config map[string]interface{} `json:"config,omitempty"` } @@ -63,7 +64,7 @@ func (f *TargetFilter) Query() (string, []interface{}) { if !f.SkipUsed && !f.SkipUnused { return ` SELECT t.uuid, t.tenant_uuid, t.name, t.summary, t.plugin, - t.endpoint, t.agent, -1 AS n + t.endpoint, t.agent, t.compression, -1 AS n FROM targets t WHERE ` + strings.Join(wheres, " AND ") + ` ORDER BY t.name, t.uuid ASC`, args @@ -76,7 +77,7 @@ func (f *TargetFilter) Query() (string, []interface{}) { return ` SELECT DISTINCT t.uuid, t.tenant_uuid, t.name, t.summary, t.plugin, - t.endpoint, t.agent, COUNT(j.uuid) AS n + t.endpoint, t.agent, t.compression, COUNT(j.uuid) AS n FROM targets t LEFT JOIN jobs j ON j.target_uuid = t.uuid @@ -126,7 +127,7 @@ func (db *DB) GetAllTargets(filter *TargetFilter) ([]*Target, error) { this, tenant NullUUID rawconfig []byte ) - if err = r.Scan(&this, &tenant, &t.Name, &t.Summary, &t.Plugin, &rawconfig, &t.Agent, &n); err != nil { + if err = r.Scan(&this, &tenant, &t.Name, &t.Summary, &t.Plugin, &rawconfig, &t.Agent, &t.Compression, &n); err != nil { return l, err } t.UUID = this.UUID @@ -146,7 +147,7 @@ func (db *DB) GetAllTargets(filter *TargetFilter) ([]*Target, error) { func (db *DB) GetTarget(id uuid.UUID) (*Target, error) { r, err := db.Query(` - SELECT uuid, tenant_uuid, name, summary, plugin, endpoint, agent + SELECT uuid, tenant_uuid, name, summary, plugin, endpoint, agent, compression FROM targets WHERE uuid = ?`, id.String()) if err != nil { @@ -163,7 +164,7 @@ func (db *DB) GetTarget(id uuid.UUID) (*Target, error) { this, tenant NullUUID rawconfig []byte ) - if err = r.Scan(&this, &tenant, &t.Name, &t.Summary, &t.Plugin, &rawconfig, &t.Agent); err != nil { + if err = r.Scan(&this, &tenant, &t.Name, &t.Summary, &t.Plugin, &rawconfig, &t.Agent, &t.Compression); err != nil { return nil, err } t.UUID = this.UUID @@ -186,9 +187,9 @@ func (db *DB) CreateTarget(in *Target) (*Target, error) { in.UUID = uuid.NewRandom() return in, db.Exec(` - INSERT INTO targets (uuid, tenant_uuid, name, summary, plugin, endpoint, agent) - VALUES (?, ?, ?, ?, ?, ?, ?)`, - in.UUID.String(), in.TenantUUID.String(), in.Name, in.Summary, in.Plugin, string(rawconfig), in.Agent) + INSERT INTO targets (uuid, tenant_uuid, name, summary, plugin, endpoint, agent, compression) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + in.UUID.String(), in.TenantUUID.String(), in.Name, in.Summary, in.Plugin, string(rawconfig), in.Agent, in.Compression) } func (db *DB) UpdateTarget(t *Target) error { @@ -199,13 +200,14 @@ func (db *DB) UpdateTarget(t *Target) error { return db.Exec(` UPDATE targets - SET name = ?, - summary = ?, - plugin = ?, - endpoint = ?, - agent = ? + SET name = ?, + summary = ?, + plugin = ?, + endpoint = ?, + agent = ?, + compression = ? WHERE uuid = ?`, - t.Name, t.Summary, t.Plugin, string(rawconfig), t.Agent, + t.Name, t.Summary, t.Plugin, string(rawconfig), t.Agent, t.Compression, t.UUID.String()) } diff --git a/db/tasks.go b/db/tasks.go index 5d3a254d0..439aa1f75 100644 --- a/db/tasks.go +++ b/db/tasks.go @@ -39,6 +39,7 @@ type Task struct { TargetUUID uuid.UUID `json:"-"` TargetPlugin string `json:"-"` TargetEndpoint string `json:"-"` + Compression string `json:"-"` Status string `json:"status"` RequestedAt int64 `json:"requested_at"` StartedAt int64 `json:"started_at"` @@ -172,7 +173,7 @@ func (f *TaskFilter) Query() (string, []interface{}) { t.target_uuid, t.target_plugin, t.target_endpoint, t.status, t.requested_at, t.started_at, t.stopped_at, t.timeout_at, t.restore_key, t.attempts, t.agent, t.log, - t.ok, t.notes, t.clear, t.fixed_key + t.ok, t.notes, t.clear, t.fixed_key, t.compression FROM tasks t @@ -207,7 +208,7 @@ func (db *DB) GetAllTasks(filter *TaskFilter) ([]*Task, error) { &target, &t.TargetPlugin, &t.TargetEndpoint, &t.Status, &t.RequestedAt, &started, &stopped, &deadline, &t.RestoreKey, &t.Attempts, &t.Agent, &log, - &t.OK, &t.Notes, &t.Clear, &t.FixedKey); err != nil { + &t.OK, &t.Notes, &t.Clear, &t.FixedKey, &t.Compression); err != nil { return l, err } t.UUID = this.UUID @@ -267,17 +268,17 @@ func (db *DB) CreateBackupTask(owner string, job *Job) (*Task, error) { `INSERT INTO tasks (uuid, owner, op, job_uuid, status, log, requested_at, store_uuid, store_plugin, store_endpoint, - target_uuid, target_plugin, target_endpoint, restore_key, - agent, attempts, tenant_uuid, fixed_key) + target_uuid, target_plugin, target_endpoint, restore_key, + agent, attempts, tenant_uuid, fixed_key, compression) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, - ?, ?, ?, ?, - ?, ?, ?, ?)`, + ?, ?, ?, ?, + ?, ?, ?, ?, ?)`, id.String(), owner, BackupOperation, job.UUID.String(), PendingStatus, "", time.Now().Unix(), job.Store.UUID.String(), job.Store.Plugin, job.Store.Endpoint, job.Target.UUID.String(), job.Target.Plugin, job.Target.Endpoint, "", - job.Agent, 0, job.TenantUUID.String(), job.FixedKey, + job.Agent, 0, job.TenantUUID.String(), job.FixedKey, job.Target.Compression, ) if err != nil { @@ -440,7 +441,7 @@ func (db *DB) UpdateTaskLog(id uuid.UUID, more string) error { ) } -func (db *DB) CreateTaskArchive(id uuid.UUID, archive_id uuid.UUID, key string, effective time.Time, encryptionType string, archive_size int64, tenant_uuid uuid.UUID) (uuid.UUID, error) { +func (db *DB) CreateTaskArchive(id uuid.UUID, archive_id uuid.UUID, key string, effective time.Time, encryptionType string, compression string, archive_size int64, tenant_uuid uuid.UUID) (uuid.UUID, error) { // fail on empty store_key, as '' seems to satisfy the NOT NULL constraint in postgres if key == "" { return nil, fmt.Errorf("cannot create an archive without a store_key") @@ -471,16 +472,23 @@ func (db *DB) CreateTaskArchive(id uuid.UUID, archive_id uuid.UUID, key string, // insert an archive with all proper references, expiration, etc. validtime := ValidateEffectiveUnix(effective) - err = db.Exec( - `INSERT INTO archives - (uuid, target_uuid, store_uuid, store_key, taken_at, expires_at, notes, status, purge_reason, job, encryption_type, size, tenant_uuid) - SELECT ?, t.uuid, s.uuid, ?, ?, ?, '', ?, '', j.Name, ?, ?, ? - FROM tasks - INNER JOIN jobs j ON j.uuid = tasks.job_uuid - INNER JOIN targets t ON t.uuid = j.target_uuid - INNER JOIN stores s ON s.uuid = j.store_uuid - WHERE tasks.uuid = ?`, - archive_id.String(), key, validtime, effective.Add(time.Duration(expiry)*time.Second).Unix(), "valid", encryptionType, archive_size, tenant_uuid.String(), id.String(), + err = db.Exec(` + INSERT INTO archives + (uuid, target_uuid, store_uuid, store_key, taken_at, + expires_at, notes, status, purge_reason, job, + compression, encryption_type, size, tenant_uuid) + + SELECT ?, t.uuid, s.uuid, ?, ?, + ?, '', 'valid', '', j.Name, + ?, ?, ?, ? + FROM tasks + INNER JOIN jobs j ON j.uuid = tasks.job_uuid + INNER JOIN targets t ON t.uuid = j.target_uuid + INNER JOIN stores s ON s.uuid = j.store_uuid + WHERE tasks.uuid = ?`, + archive_id.String(), key, + validtime, effective.Add(time.Duration(expiry)*time.Second).Unix(), + compression, encryptionType, archive_size, tenant_uuid.String(), id.String(), ) if err != nil { log.Errorf("failed to insert archive with UUID %s into database: %s", archive_id, err) diff --git a/db/tasks_test.go b/db/tasks_test.go index 0fb473537..01549bee6 100644 --- a/db/tasks_test.go +++ b/db/tasks_test.go @@ -250,7 +250,7 @@ var _ = Describe("Task Management", func() { Ω(db.StartTask(task.UUID, time.Now())).Should(Succeed()) Ω(db.CompleteTask(task.UUID, time.Now())).Should(Succeed()) - archive_id, err := db.CreateTaskArchive(task.UUID, uuid.NewRandom(), "SOME-KEY", time.Now(), "aes-256-ctr", 0, task.TenantUUID) + archive_id, err := db.CreateTaskArchive(task.UUID, uuid.NewRandom(), "SOME-KEY", time.Now(), "aes-256-ctr", "gz", 0, task.TenantUUID) Expect(err).ShouldNot(HaveOccurred()) Expect(archive_id).ShouldNot(BeNil()) @@ -272,7 +272,7 @@ var _ = Describe("Task Management", func() { Expect(db.StartTask(task.UUID, time.Now())).Should(Succeed()) Expect(db.CompleteTask(task.UUID, time.Now())).Should(Succeed()) - archive_id, err := db.CreateTaskArchive(task.UUID, uuid.NewRandom(), "", time.Now(), "aes-256-ctr", 0, task.TenantUUID) + archive_id, err := db.CreateTaskArchive(task.UUID, uuid.NewRandom(), "", time.Now(), "aes-256-ctr", "gz", 0, task.TenantUUID) Expect(err).Should(HaveOccurred()) Expect(archive_id).Should(BeNil()) diff --git a/docs/API.md b/docs/API.md index 2b0390147..bae509e19 100644 --- a/docs/API.md +++ b/docs/API.md @@ -3795,15 +3795,16 @@ value. Subject to the `exact=(t|f)` query string parameter. **Response** { - "uuid" : "30f34d8f-762e-402a-b7ce-769a4a68de90", - "name" : "Job Name", - "summary" : "A longer description", - "expiry" : 604800, - "schedule" : "daily 4am", - "paused" : false, - "agent" : "10.0.0.5:5444", - "last_run" : "2017-10-19 03:00:00", - "status" : "done", + "uuid" : "30f34d8f-762e-402a-b7ce-769a4a68de90", + "name" : "Job Name", + "summary" : "A longer description", + "compression" : "bzip2", + "expiry" : 604800, + "schedule" : "daily 4am", + "paused" : false, + "agent" : "10.0.0.5:5444", + "last_run" : "2017-10-19 03:00:00", + "status" : "done", "policy" : { "uuid" : "9a112894-10eb-439f-afd5-01597d8faf64", @@ -3867,14 +3868,15 @@ Configure a new backup job on a tenant. -X POST https://shield.host/v2/tenants/:tenant/jobs \ --data-binary ' { - "name" : "New Job Name", - "summary" : "A longer description...", - "schedule" : "daily 4am", - "paused" : false, + "name" : "New Job Name", + "summary" : "A longer description...", + "schedule" : "daily 4am", + "compression" : "bzip2", + "paused" : false, - "store" : "af1ad037-c8c1-4036-984a-3cf726b4081d", - "target" : "2c64d9ff-fc9f-4114-8e89-9f7c84fcaac7", - "policy" : "cb6b0503-4741-4cfd-9a1d-11b5a5aaadde" + "store" : "af1ad037-c8c1-4036-984a-3cf726b4081d", + "target" : "2c64d9ff-fc9f-4114-8e89-9f7c84fcaac7", + "policy" : "cb6b0503-4741-4cfd-9a1d-11b5a5aaadde" }' @@ -3886,15 +3888,17 @@ FIXME : allow non-UUIDs for all three. **Response** { - "uuid" : "30f34d8f-762e-402a-b7ce-769a4a68de90", - "name" : "Job Name", - "summary" : "A longer description", - "expiry" : 604800, - "schedule" : "daily 4am", - "paused" : false, - "agent" : "10.0.0.5:5444", - "last_run" : "2017-10-19 03:00:00", - "last_task_status": "", + "uuid" : "30f34d8f-762e-402a-b7ce-769a4a68de90", + "name" : "Job Name", + "summary" : "A longer description", + "compression" : "bzip2", + "expiry" : 604800, + "schedule" : "daily 4am", + "paused" : false, + "agent" : "10.0.0.5:5444", + + "last_run" : "2017-10-19 03:00:00", + "last_task_status" : "", "policy" : { "uuid" : "9a112894-10eb-439f-afd5-01597d8faf64", @@ -3962,15 +3966,17 @@ This endpoint takes no query string parameters. **Response** { - "uuid" : "30f34d8f-762e-402a-b7ce-769a4a68de90", - "name" : "Job Name", - "summary" : "A longer description", - "expiry" : 604800, - "schedule" : "daily 4am", - "paused" : false, - "agent" : "10.0.0.5:5444", - "last_run" : "2017-10-19 03:00:00", - "last_task_status": "", + "uuid" : "30f34d8f-762e-402a-b7ce-769a4a68de90", + "name" : "Job Name", + "summary" : "A longer description", + "compression" : "bzip2", + "expiry" : 604800, + "schedule" : "daily 4am", + "paused" : false, + "agent" : "10.0.0.5:5444", + + "last_run" : "2017-10-19 03:00:00", + "last_task_status" : "", "policy" : { "uuid" : "9a112894-10eb-439f-afd5-01597d8faf64", @@ -4038,12 +4044,14 @@ Update a single job on a tenant. -X PUT https://shield.host/v2/tenants/:tenant/jobs/:uuid \ --data-binary ' { - "name" : "New Name", - "summary" : "An updated summary", - "schedule" : "daily 4am", - "store" : "a6ef5aea-51f6-4e91-a490-3063395f879b", - "target" : "af1425ed-53fd-4ab6-a425-fb230c383901", - "policy" : "c16a4783-19b8-400d-8b51-f47dcdc11da3" + "name" : "New Name", + "summary" : "An updated summary", + "compression" : "bzip2", + "schedule" : "daily 4am", + + "store" : "a6ef5aea-51f6-4e91-a490-3063395f879b", + "target" : "af1425ed-53fd-4ab6-a425-fb230c383901", + "policy" : "c16a4783-19b8-400d-8b51-f47dcdc11da3" }' @@ -4551,25 +4559,29 @@ default) denotes an unlimited search. [ { - "uuid": "5c8cef06-190c-4b07-a0b7-8452f6faff26", - "key": "2017/10/27/2017-10-27-120512-948428f4-3a83-4b5d-8a41-7d27ca81ce8d", - "taken_at": "2017-10-27 16:05:25", - "expires_at": "2017-10-28 16:05:25", - "notes": "", - "status": "valid", - "purge_reason": "", - "target_uuid": "51d9cced-b11d-4b76-b9f3-fe0be4cd6087", - "target_name": "SHIELD", - "target_plugin": "fs", - "target_endpoint": "{\"base_dir\":\"/e/no/ent\",\"bsdtar\":\"bsdtar\",\"exclude\":\"var/*.db\"}", - "store_uuid": "828fccae-a11e-41ee-bc13-d33c4dff1241", - "store_name": "CloudStor", - "store_plugin": "fs", - "store_endpoint": "{\"base_dir\":\"/tmp/shield.testdev.storeNy0fewQ\",\"bsdtar\":\"bsdtar\"}", - "job": "Hourly", - "encryption_type": "aes256-ctr", - "tenant_uuid": "5524167e-cf56-4a8f-9580-cfca40949316", - "size": 43306681 + "uuid" : "5c8cef06-190c-4b07-a0b7-8452f6faff26", + "key" : "2017/10/27/2017-10-27-120512-948428f4-3a83-4b5d-8a41-7d27ca81ce8d", + "taken_at" : "2017-10-27 16:05:25", + "expires_at" : "2017-10-28 16:05:25", + "notes" : "", + "compression" : "bzip2", + "encryption_type" : "aes256-ctr", + "size" : 43306681, + "status" : "valid", + "purge_reason" : "", + "job" : "Hourly", + + "tenant_uuid" : "5524167e-cf56-4a8f-9580-cfca40949316", + + "target_uuid" : "51d9cced-b11d-4b76-b9f3-fe0be4cd6087", + "target_name" : "SHIELD", + "target_plugin" : "fs", + "target_endpoint" : "{\"base_dir\":\"/e/no/ent\",\"bsdtar\":\"bsdtar\",\"exclude\":\"var/*.db\"}", + + "store_uuid" : "828fccae-a11e-41ee-bc13-d33c4dff1241", + "store_name" : "CloudStor", + "store_plugin" : "fs", + "store_endpoint" : "{\"base_dir\":\"/tmp/shield.testdev.storeNy0fewQ\",\"bsdtar\":\"bsdtar\"}" } ] **Access Control** @@ -4618,25 +4630,29 @@ This endpoint takes no query string parameters. **Response** { - "uuid": "5c8cef06-190c-4b07-a0b7-8452f6faff26", - "key": "2017/10/27/2017-10-27-120512-948428f4-3a83-4b5d-8a41-7d27ca81ce8d", - "taken_at": "2017-10-27 16:05:25", - "expires_at": "2017-10-28 16:05:25", - "notes": "", - "status": "valid", - "purge_reason": "", - "target_uuid": "51d9cced-b11d-4b76-b9f3-fe0be4cd6087", - "target_name": "SHIELD", - "target_plugin": "fs", - "target_endpoint": "{\"base_dir\":\"/e/no/ent\",\"bsdtar\":\"bsdtar\",\"exclude\":\"var/*.db\"}", - "store_uuid": "828fccae-a11e-41ee-bc13-d33c4dff1241", - "store_name": "CloudStor", - "store_plugin": "fs", - "store_endpoint": "{\"base_dir\":\"/tmp/shield.testdev.storeNy0fewQ\",\"bsdtar\":\"bsdtar\"}", - "job": "Hourly", - "encryption_type": "aes256-ctr", - "tenant_uuid": "5524167e-cf56-4a8f-9580-cfca40949316", - "size": 43306681 + "uuid" : "5c8cef06-190c-4b07-a0b7-8452f6faff26", + "key" : "2017/10/27/2017-10-27-120512-948428f4-3a83-4b5d-8a41-7d27ca81ce8d", + "taken_at" : "2017-10-27 16:05:25", + "expires_at" : "2017-10-28 16:05:25", + "notes" : "", + "compression" : "bzip2", + "encryption_type" : "aes256-ctr", + "size" : 43306681, + "status" : "valid", + "purge_reason" : "", + "job" : "Hourly", + + "tenant_uuid" : "5524167e-cf56-4a8f-9580-cfca40949316", + + "target_uuid" : "51d9cced-b11d-4b76-b9f3-fe0be4cd6087", + "target_name" : "SHIELD", + "target_plugin" : "fs", + "target_endpoint" : "{\"base_dir\":\"/e/no/ent\",\"bsdtar\":\"bsdtar\",\"exclude\":\"var/*.db\"}", + + "store_uuid" : "828fccae-a11e-41ee-bc13-d33c4dff1241", + "store_name" : "CloudStor", + "store_plugin" : "fs", + "store_endpoint" : "{\"base_dir\":\"/tmp/shield.testdev.storeNy0fewQ\",\"bsdtar\":\"bsdtar\"}" } **Access Control** @@ -4688,25 +4704,29 @@ This endpoint takes no query string parameters. **Response** { - "uuid": "5c8cef06-190c-4b07-a0b7-8452f6faff26", - "key": "2017/10/27/2017-10-27-120512-948428f4-3a83-4b5d-8a41-7d27ca81ce8d", - "taken_at": "2017-10-27 16:05:25", - "expires_at": "2017-10-28 16:05:25", - "notes": "Notes for this specific archive", - "status": "valid", - "purge_reason": "", - "target_uuid": "51d9cced-b11d-4b76-b9f3-fe0be4cd6087", - "target_name": "SHIELD", - "target_plugin": "fs", - "target_endpoint": "{\"base_dir\":\"/e/no/ent\",\"bsdtar\":\"bsdtar\",\"exclude\":\"var/*.db\"}", - "store_uuid": "828fccae-a11e-41ee-bc13-d33c4dff1241", - "store_name": "CloudStor", - "store_plugin": "fs", - "store_endpoint": "{\"base_dir\":\"/tmp/shield.testdev.storeNy0fewQ\",\"bsdtar\":\"bsdtar\"}", - "job": "Hourly", - "encryption_type": "aes256-ctr", - "tenant_uuid": "5524167e-cf56-4a8f-9580-cfca40949316", - "size": 43306681 + "uuid" : "5c8cef06-190c-4b07-a0b7-8452f6faff26", + "key" : "2017/10/27/2017-10-27-120512-948428f4-3a83-4b5d-8a41-7d27ca81ce8d", + "taken_at" : "2017-10-27 16:05:25", + "expires_at" : "2017-10-28 16:05:25", + "notes" : "", + "compression" : "bzip2", + "encryption_type" : "aes256-ctr", + "size" : 43306681, + "status" : "valid", + "purge_reason" : "", + "job" : "Hourly", + + "tenant_uuid" : "5524167e-cf56-4a8f-9580-cfca40949316", + + "target_uuid" : "51d9cced-b11d-4b76-b9f3-fe0be4cd6087", + "target_name" : "SHIELD", + "target_plugin" : "fs", + "target_endpoint" : "{\"base_dir\":\"/e/no/ent\",\"bsdtar\":\"bsdtar\",\"exclude\":\"var/*.db\"}", + + "store_uuid" : "828fccae-a11e-41ee-bc13-d33c4dff1241", + "store_name" : "CloudStor", + "store_plugin" : "fs", + "store_endpoint" : "{\"base_dir\":\"/tmp/shield.testdev.storeNy0fewQ\",\"bsdtar\":\"bsdtar\"}" } **Access Control** diff --git a/docs/API.yml b/docs/API.yml index 711bbd7d7..f4691c8c6 100644 --- a/docs/API.yml +++ b/docs/API.yml @@ -2834,15 +2834,16 @@ sections: response: json: | { - "uuid" : "30f34d8f-762e-402a-b7ce-769a4a68de90", - "name" : "Job Name", - "summary" : "A longer description", - "expiry" : 604800, - "schedule" : "daily 4am", - "paused" : false, - "agent" : "10.0.0.5:5444", - "last_run" : "2017-10-19 03:00:00", - "status" : "done", + "uuid" : "30f34d8f-762e-402a-b7ce-769a4a68de90", + "name" : "Job Name", + "summary" : "A longer description", + "compression" : "bzip2", + "expiry" : 604800, + "schedule" : "daily 4am", + "paused" : false, + "agent" : "10.0.0.5:5444", + "last_run" : "2017-10-19 03:00:00", + "status" : "done", "policy" : { "uuid" : "9a112894-10eb-439f-afd5-01597d8faf64", @@ -2883,14 +2884,15 @@ sections: request: json: | { - "name" : "New Job Name", - "summary" : "A longer description...", - "schedule" : "daily 4am", - "paused" : false, + "name" : "New Job Name", + "summary" : "A longer description...", + "schedule" : "daily 4am", + "compression" : "bzip2", + "paused" : false, - "store" : "af1ad037-c8c1-4036-984a-3cf726b4081d", - "target" : "2c64d9ff-fc9f-4114-8e89-9f7c84fcaac7", - "policy" : "cb6b0503-4741-4cfd-9a1d-11b5a5aaadde" + "store" : "af1ad037-c8c1-4036-984a-3cf726b4081d", + "target" : "2c64d9ff-fc9f-4114-8e89-9f7c84fcaac7", + "policy" : "cb6b0503-4741-4cfd-9a1d-11b5a5aaadde" } summary: | @@ -2904,15 +2906,17 @@ sections: response: json: | { - "uuid" : "30f34d8f-762e-402a-b7ce-769a4a68de90", - "name" : "Job Name", - "summary" : "A longer description", - "expiry" : 604800, - "schedule" : "daily 4am", - "paused" : false, - "agent" : "10.0.0.5:5444", - "last_run" : "2017-10-19 03:00:00", - "last_task_status": "", + "uuid" : "30f34d8f-762e-402a-b7ce-769a4a68de90", + "name" : "Job Name", + "summary" : "A longer description", + "compression" : "bzip2", + "expiry" : 604800, + "schedule" : "daily 4am", + "paused" : false, + "agent" : "10.0.0.5:5444", + + "last_run" : "2017-10-19 03:00:00", + "last_task_status" : "", "policy" : { "uuid" : "9a112894-10eb-439f-afd5-01597d8faf64", @@ -2952,15 +2956,17 @@ sections: response: json: | { - "uuid" : "30f34d8f-762e-402a-b7ce-769a4a68de90", - "name" : "Job Name", - "summary" : "A longer description", - "expiry" : 604800, - "schedule" : "daily 4am", - "paused" : false, - "agent" : "10.0.0.5:5444", - "last_run" : "2017-10-19 03:00:00", - "last_task_status": "", + "uuid" : "30f34d8f-762e-402a-b7ce-769a4a68de90", + "name" : "Job Name", + "summary" : "A longer description", + "compression" : "bzip2", + "expiry" : 604800, + "schedule" : "daily 4am", + "paused" : false, + "agent" : "10.0.0.5:5444", + + "last_run" : "2017-10-19 03:00:00", + "last_task_status" : "", "policy" : { "uuid" : "9a112894-10eb-439f-afd5-01597d8faf64", @@ -3006,12 +3012,14 @@ sections: request: json: | { - "name" : "New Name", - "summary" : "An updated summary", - "schedule" : "daily 4am", - "store" : "a6ef5aea-51f6-4e91-a490-3063395f879b", - "target" : "af1425ed-53fd-4ab6-a425-fb230c383901", - "policy" : "c16a4783-19b8-400d-8b51-f47dcdc11da3" + "name" : "New Name", + "summary" : "An updated summary", + "compression" : "bzip2", + "schedule" : "daily 4am", + + "store" : "a6ef5aea-51f6-4e91-a490-3063395f879b", + "target" : "af1425ed-53fd-4ab6-a425-fb230c383901", + "policy" : "c16a4783-19b8-400d-8b51-f47dcdc11da3" } summary: | {{CURL}} @@ -3332,25 +3340,29 @@ sections: json: | [ { - "uuid": "5c8cef06-190c-4b07-a0b7-8452f6faff26", - "key": "2017/10/27/2017-10-27-120512-948428f4-3a83-4b5d-8a41-7d27ca81ce8d", - "taken_at": "2017-10-27 16:05:25", - "expires_at": "2017-10-28 16:05:25", - "notes": "", - "status": "valid", - "purge_reason": "", - "target_uuid": "51d9cced-b11d-4b76-b9f3-fe0be4cd6087", - "target_name": "SHIELD", - "target_plugin": "fs", - "target_endpoint": "{\"base_dir\":\"/e/no/ent\",\"bsdtar\":\"bsdtar\",\"exclude\":\"var/*.db\"}", - "store_uuid": "828fccae-a11e-41ee-bc13-d33c4dff1241", - "store_name": "CloudStor", - "store_plugin": "fs", - "store_endpoint": "{\"base_dir\":\"/tmp/shield.testdev.storeNy0fewQ\",\"bsdtar\":\"bsdtar\"}", - "job": "Hourly", - "encryption_type": "aes256-ctr", - "tenant_uuid": "5524167e-cf56-4a8f-9580-cfca40949316", - "size": 43306681 + "uuid" : "5c8cef06-190c-4b07-a0b7-8452f6faff26", + "key" : "2017/10/27/2017-10-27-120512-948428f4-3a83-4b5d-8a41-7d27ca81ce8d", + "taken_at" : "2017-10-27 16:05:25", + "expires_at" : "2017-10-28 16:05:25", + "notes" : "", + "compression" : "bzip2", + "encryption_type" : "aes256-ctr", + "size" : 43306681, + "status" : "valid", + "purge_reason" : "", + "job" : "Hourly", + + "tenant_uuid" : "5524167e-cf56-4a8f-9580-cfca40949316", + + "target_uuid" : "51d9cced-b11d-4b76-b9f3-fe0be4cd6087", + "target_name" : "SHIELD", + "target_plugin" : "fs", + "target_endpoint" : "{\"base_dir\":\"/e/no/ent\",\"bsdtar\":\"bsdtar\",\"exclude\":\"var/*.db\"}", + + "store_uuid" : "828fccae-a11e-41ee-bc13-d33c4dff1241", + "store_name" : "CloudStor", + "store_plugin" : "fs", + "store_endpoint" : "{\"base_dir\":\"/tmp/shield.testdev.storeNy0fewQ\",\"bsdtar\":\"bsdtar\"}" } ] @@ -3373,26 +3385,31 @@ sections: response: json: | { - "uuid": "5c8cef06-190c-4b07-a0b7-8452f6faff26", - "key": "2017/10/27/2017-10-27-120512-948428f4-3a83-4b5d-8a41-7d27ca81ce8d", - "taken_at": "2017-10-27 16:05:25", - "expires_at": "2017-10-28 16:05:25", - "notes": "", - "status": "valid", - "purge_reason": "", - "target_uuid": "51d9cced-b11d-4b76-b9f3-fe0be4cd6087", - "target_name": "SHIELD", - "target_plugin": "fs", - "target_endpoint": "{\"base_dir\":\"/e/no/ent\",\"bsdtar\":\"bsdtar\",\"exclude\":\"var/*.db\"}", - "store_uuid": "828fccae-a11e-41ee-bc13-d33c4dff1241", - "store_name": "CloudStor", - "store_plugin": "fs", - "store_endpoint": "{\"base_dir\":\"/tmp/shield.testdev.storeNy0fewQ\",\"bsdtar\":\"bsdtar\"}", - "job": "Hourly", - "encryption_type": "aes256-ctr", - "tenant_uuid": "5524167e-cf56-4a8f-9580-cfca40949316", - "size": 43306681 + "uuid" : "5c8cef06-190c-4b07-a0b7-8452f6faff26", + "key" : "2017/10/27/2017-10-27-120512-948428f4-3a83-4b5d-8a41-7d27ca81ce8d", + "taken_at" : "2017-10-27 16:05:25", + "expires_at" : "2017-10-28 16:05:25", + "notes" : "", + "compression" : "bzip2", + "encryption_type" : "aes256-ctr", + "size" : 43306681, + "status" : "valid", + "purge_reason" : "", + "job" : "Hourly", + + "tenant_uuid" : "5524167e-cf56-4a8f-9580-cfca40949316", + + "target_uuid" : "51d9cced-b11d-4b76-b9f3-fe0be4cd6087", + "target_name" : "SHIELD", + "target_plugin" : "fs", + "target_endpoint" : "{\"base_dir\":\"/e/no/ent\",\"bsdtar\":\"bsdtar\",\"exclude\":\"var/*.db\"}", + + "store_uuid" : "828fccae-a11e-41ee-bc13-d33c4dff1241", + "store_name" : "CloudStor", + "store_plugin" : "fs", + "store_endpoint" : "{\"base_dir\":\"/tmp/shield.testdev.storeNy0fewQ\",\"bsdtar\":\"bsdtar\"}" } + errors: - message: Unable to retrieve backup archive information summary: *internal @@ -3417,25 +3434,29 @@ sections: response: json: | { - "uuid": "5c8cef06-190c-4b07-a0b7-8452f6faff26", - "key": "2017/10/27/2017-10-27-120512-948428f4-3a83-4b5d-8a41-7d27ca81ce8d", - "taken_at": "2017-10-27 16:05:25", - "expires_at": "2017-10-28 16:05:25", - "notes": "Notes for this specific archive", - "status": "valid", - "purge_reason": "", - "target_uuid": "51d9cced-b11d-4b76-b9f3-fe0be4cd6087", - "target_name": "SHIELD", - "target_plugin": "fs", - "target_endpoint": "{\"base_dir\":\"/e/no/ent\",\"bsdtar\":\"bsdtar\",\"exclude\":\"var/*.db\"}", - "store_uuid": "828fccae-a11e-41ee-bc13-d33c4dff1241", - "store_name": "CloudStor", - "store_plugin": "fs", - "store_endpoint": "{\"base_dir\":\"/tmp/shield.testdev.storeNy0fewQ\",\"bsdtar\":\"bsdtar\"}", - "job": "Hourly", - "encryption_type": "aes256-ctr", - "tenant_uuid": "5524167e-cf56-4a8f-9580-cfca40949316", - "size": 43306681 + "uuid" : "5c8cef06-190c-4b07-a0b7-8452f6faff26", + "key" : "2017/10/27/2017-10-27-120512-948428f4-3a83-4b5d-8a41-7d27ca81ce8d", + "taken_at" : "2017-10-27 16:05:25", + "expires_at" : "2017-10-28 16:05:25", + "notes" : "", + "compression" : "bzip2", + "encryption_type" : "aes256-ctr", + "size" : 43306681, + "status" : "valid", + "purge_reason" : "", + "job" : "Hourly", + + "tenant_uuid" : "5524167e-cf56-4a8f-9580-cfca40949316", + + "target_uuid" : "51d9cced-b11d-4b76-b9f3-fe0be4cd6087", + "target_name" : "SHIELD", + "target_plugin" : "fs", + "target_endpoint" : "{\"base_dir\":\"/e/no/ent\",\"bsdtar\":\"bsdtar\",\"exclude\":\"var/*.db\"}", + + "store_uuid" : "828fccae-a11e-41ee-bc13-d33c4dff1241", + "store_name" : "CloudStor", + "store_plugin" : "fs", + "store_endpoint" : "{\"base_dir\":\"/tmp/shield.testdev.storeNy0fewQ\",\"bsdtar\":\"bsdtar\"}" } errors: diff --git a/t/travis b/t/travis index 32d146499..0591f135c 100755 --- a/t/travis +++ b/t/travis @@ -1,6 +1,7 @@ #!/bin/bash set -e +rm -f vault vault.zip curl -Lo vault.zip https://releases.hashicorp.com/vault/0.8.3/vault_0.8.3_linux_amd64.zip unzip vault.zip diff --git a/web2/htdocs/index.html b/web2/htdocs/index.html index d7cc84c6f..f89802163 100755 --- a/web2/htdocs/index.html +++ b/web2/htdocs/index.html @@ -219,7 +219,7 @@

Sign in with your SHIELD account