Skip to content

Commit

Permalink
Make Compression Optional (#437)
Browse files Browse the repository at this point in the history
Compression is now optional, and can be set on a per-target basis.
  • Loading branch information
jhunt committed Aug 1, 2018
1 parent 21f1ac8 commit c0f60c8
Show file tree
Hide file tree
Showing 29 changed files with 656 additions and 345 deletions.
5 changes: 5 additions & 0 deletions .gitignore
Expand Up @@ -6,6 +6,7 @@
/shield-schema
/buckler
/shield-crypt
/shield-report

# stuff we use in t/*
/bin/vault
Expand Down Expand Up @@ -48,6 +49,10 @@
# ignore tmp stuff from testdev
/tmp

# ignore travis test bins
/t/travis-bin
/vault.zip

# ignore testing files
cmd/shield-umc/shield-umc
cmd/shield-umc/main
Expand Down
13 changes: 8 additions & 5 deletions Makefile
Expand Up @@ -4,7 +4,7 @@
BUILD_TYPE?=build

# Everything; this is the default behavior
all: format shieldd shield shield-agent shield-schema shield-migrate shield-crypt plugins test
all: format shieldd shield shield-agent shield-schema shield-migrate shield-crypt shield-report plugins test

# go fmt ftw
format:
Expand All @@ -18,15 +18,15 @@ plugin-tests: plugins
@rm -f mock
go-tests:
export PATH=$$PATH:test/bin; go list ./... | grep -v vendor | xargs go test
api-tests:
api-tests: shieldd shield-schema shield-crypt shield-agent shield-report
./t/api

# Running Tests for race conditions
race:
ginkgo -race *

# Building Shield
shield: shieldd shield-agent shield-schema shield-crypt
shield: shieldd shield-agent shield-schema shield-crypt shield-report

shield-crypt:
go $(BUILD_TYPE) ./cmd/shield-crypt
Expand All @@ -38,6 +38,8 @@ shield-schema:
go $(BUILD_TYPE) ./cmd/shield-schema
shield-migrate:
go $(BUILD_TYPE) ./cmd/shield-migrate
shield-report:
go $(BUILD_TYPE) ./cmd/shield-report

shield: cmd/shield/help.go
go $(BUILD_TYPE) ./cmd/shield
Expand All @@ -63,7 +65,7 @@ docs/API.md: docs/API.yml
mv $@~ $@

clean:
rm -f shield shieldd shield-agent shield-schema shield-crypt shield-migrate
rm -f shield shieldd shield-agent shield-schema shield-crypt shield-migrate shield-report
rm -f $$(cat plugins) dummy


Expand Down Expand Up @@ -110,6 +112,7 @@ release:
done
go build -ldflags="$(LDFLAGS)" -o "$(ARTIFACTS)/crypter/shield-crypt" ./cmd/shield-crypt
go build -ldflags="$(LDFLAGS)" -o "$(ARTIFACTS)/agent/shield-agent" ./cmd/shield-agent
go build -ldflags="$(LDFLAGS)" -o "$(ARTIFACTS)/agent/shield-report" ./cmd/shield-report
go build -ldflags="$(LDFLAGS)" -o "$(ARTIFACTS)/cli/shield" ./cmd/shield
CGO_ENABLED=1 go build -ldflags="$(LDFLAGS)" -o "$(ARTIFACTS)/daemon/shield-schema" ./cmd/shield-schema
CGO_ENABLED=1 go build -ldflags="$(LDFLAGS)" -o "$(ARTIFACTS)/daemon/shield-migrate" ./cmd/shield-migrate
Expand All @@ -128,4 +131,4 @@ web2/htdocs/shield.js: $(JAVASCRIPTS)

web2: web2/htdocs/shield.js

.PHONY: plugins dev web2 shield shieldd shield-schema shield-agent shield-crypt demo
.PHONY: plugins dev web2 shield shieldd shield-schema shield-agent shield-crypt shield-report demo
3 changes: 3 additions & 0 deletions agent/command.go
Expand Up @@ -25,6 +25,7 @@ type Command struct {
EncryptType string `json:"encrypt_type,omitempty"`
EncryptKey string `json:"encrypt_key,omitempty"`
EncryptIV string `json:"encrypt_iv,omitempty"`
Compression string `json:"compression,omitempty"`
}

func ParseCommand(b []byte) (*Command, error) {
Expand Down Expand Up @@ -80,6 +81,7 @@ func ParseCommand(b []byte) (*Command, error) {
if cmd.RestoreKey == "" {
return nil, fmt.Errorf("missing required 'restore_key' value in payload (for purge operation)")
}

case "test-store":
if cmd.StorePlugin == "" {
return nil, fmt.Errorf("missing required 'store_plugin' value in payload")
Expand Down Expand Up @@ -153,6 +155,7 @@ func (agent *Agent) Execute(c *Command, out chan string) error {
fmt.Sprintf("SHIELD_ENCRYPT_TYPE=%s", c.EncryptType),
fmt.Sprintf("SHIELD_ENCRYPT_KEY=%s", c.EncryptKey),
fmt.Sprintf("SHIELD_ENCRYPT_IV=%s", c.EncryptIV),
fmt.Sprintf("SHIELD_COMPRESSION=%s", c.Compression),
}

if log.LogLevel() == syslog.LOG_DEBUG {
Expand Down
2 changes: 2 additions & 0 deletions agent/test/bin/shield-report
@@ -0,0 +1,2 @@
#!/bin/sh
exec cat
82 changes: 72 additions & 10 deletions bin/shield-pipe
Expand Up @@ -15,6 +15,7 @@
# SHIELD_STORE_PLUGIN Path to the store plugin to use
# SHIELD_STORE_ENDPOINT The store endpoint config (probably JSON)
# SHIELD_RESTORE_KEY Archive key for 'restore' operations
# SHIELD_COMPRESSION What type of compression to perform
#
# Temporary Environment Variables (Unset before call to shield plugin)
# ---------------------
Expand Down Expand Up @@ -106,6 +107,8 @@ if [ -n "$SHIELD_ENCRYPT_IV" ]; then
unset SHIELD_ENCRYPT_IV
fi

SHIELD_COMPRESSION=${SHIELD_COMPRESSION:-bzip2}

case ${SHIELD_OP} in
(status)
needenv SHIELD_OP \
Expand Down Expand Up @@ -210,16 +213,47 @@ exit 0
validate TARGET ${SHIELD_TARGET_PLUGIN} "${SHIELD_TARGET_ENDPOINT}"
validate STORE ${SHIELD_STORE_PLUGIN} "${SHIELD_STORE_ENDPOINT}"

header "Running backup task (using bzip2 compression)"
case $SHIELD_COMPRESSION in
bzip2) header "Running backup task (using bzip2 compression)" ;;
none) header "Running backup task (without compression)" ;;
*)
fail "Unrecognized compression scheme '$SHIELD_COMPRESSION'"
exit 145
;;
esac

PULSE=$(mktemp -t shield-pipe.XXXXX)
trap "rm -f ${PULSE}" QUIT TERM INT

set -o pipefail
#The use of 3<<< shown below is to write the encryption config as json to FD 3 thus allowing us to drop it from the environment
# and prevent further propogation
${SHIELD_TARGET_PLUGIN} backup -e "${SHIELD_TARGET_ENDPOINT}" | \
tee >(tail -c1 >$PULSE) | bzip2 | shield-crypt --encrypt 3<<<"{\"enc_key\":\"$enc_key\",\"enc_iv\":\"$enc_iv\",\"enc_type\":\"$enc_type\"}" | ${SHIELD_STORE_PLUGIN} store -e "${SHIELD_STORE_ENDPOINT}"

# The use of 3<<< shown below is to write the encryption
# config as JSON to fd3, allowing us to drop it from the
# environment and prevent further propogation

case $SHIELD_COMPRESSION in
bzip2)
${SHIELD_TARGET_PLUGIN} backup -e "${SHIELD_TARGET_ENDPOINT}" | \
tee >(tail -c1 >$PULSE) | \
bzip2 | \
shield-crypt --encrypt 3<<<"{\"enc_key\":\"$enc_key\",\"enc_iv\":\"$enc_iv\",\"enc_type\":\"$enc_type\"}" | \
${SHIELD_STORE_PLUGIN} store -e "${SHIELD_STORE_ENDPOINT}" | \
shield-report --compression bzip2
;;
none)
${SHIELD_TARGET_PLUGIN} backup -e "${SHIELD_TARGET_ENDPOINT}" | \
tee >(tail -c1 >$PULSE) | \
shield-crypt --encrypt 3<<<"{\"enc_key\":\"$enc_key\",\"enc_iv\":\"$enc_iv\",\"enc_type\":\"$enc_type\"}" | \
${SHIELD_STORE_PLUGIN} store -e "${SHIELD_STORE_ENDPOINT}" | \
shield-report --compression none
;;
*)
fail "Unrecognized compression scheme '$SHIELD_COMPRESSION'"
exit 145
;;
esac
if [[ ! -s ${PULSE} ]]; then
rm -f ${PULSE}
Expand All @@ -243,12 +277,40 @@ exit 0
validate TARGET ${SHIELD_TARGET_PLUGIN} "${SHIELD_TARGET_ENDPOINT}"
validate STORE ${SHIELD_STORE_PLUGIN} "${SHIELD_STORE_ENDPOINT}"
header "Running restore task (using bzip2 compression)"
case $SHIELD_COMPRESSION in
bzip2) header "Running restore task (using bzip2 compression)" ;;
none) header "Running restore task (without compression)" ;;
*)
fail "Unrecognized compression scheme '$SHIELD_COMPRESSION'"
exit 145
;;
esac
set -o pipefail
#The use of 3<<< shown below is to write the encryption config as json to FD 3 thus allowing us to drop it from the environment
# and prevent further propogation
${SHIELD_STORE_PLUGIN} retrieve -k "${SHIELD_RESTORE_KEY}" -e "${SHIELD_STORE_ENDPOINT}" | shield-crypt --decrypt 3<<<"{\"enc_key\":\"$enc_key\",\"enc_iv\":\"$enc_iv\",\"enc_type\":\"$enc_type\"}" | bunzip2 | \
${SHIELD_TARGET_PLUGIN} restore -e "${SHIELD_TARGET_ENDPOINT}"
# The use of 3<<< shown below is to write the encryption
# config as JSON to fd3, allowing us to drop it from the
# environment and prevent further propogation
case $SHIELD_COMPRESSION in
bzip2)
${SHIELD_STORE_PLUGIN} retrieve -k "${SHIELD_RESTORE_KEY}" -e "${SHIELD_STORE_ENDPOINT}" | \
shield-crypt --decrypt 3<<<"{\"enc_key\":\"$enc_key\",\"enc_iv\":\"$enc_iv\",\"enc_type\":\"$enc_type\"}" | \
bunzip2 | \
${SHIELD_TARGET_PLUGIN} restore -e "${SHIELD_TARGET_ENDPOINT}"
;;
none)
${SHIELD_STORE_PLUGIN} retrieve -k "${SHIELD_RESTORE_KEY}" -e "${SHIELD_STORE_ENDPOINT}" | \
shield-crypt --decrypt 3<<<"{\"enc_key\":\"$enc_key\",\"enc_iv\":\"$enc_iv\",\"enc_type\":\"$enc_type\"}" | \
${SHIELD_TARGET_PLUGIN} restore -e "${SHIELD_TARGET_ENDPOINT}"
;;
*)
fail "Unrecognized compression scheme '$SHIELD_COMPRESSION'"
exit 145
;;
esac
exit 0
;;
Expand Down
1 change: 1 addition & 0 deletions client/v2/shield/archives.go
Expand Up @@ -16,6 +16,7 @@ type Archive struct {
Store *Store `json:"store,omitempty"`
Policy *Policy `json:"policy,omitempty"`

Compression string `json:"compression"`
EncryptionType string `json:"encryption_type"`
Size int64 `json:"size"`
}
Expand Down
11 changes: 6 additions & 5 deletions client/v2/shield/targets.go
Expand Up @@ -8,11 +8,12 @@ import (
)

type Target struct {
UUID string `json:"uuid,omitempty"`
Name string `json:"name"`
Summary string `json:"summary"`
Plugin string `json:"plugin"`
Agent string `json:"agent"`
UUID string `json:"uuid,omitempty"`
Name string `json:"name"`
Summary string `json:"summary"`
Plugin string `json:"plugin"`
Agent string `json:"agent"`
Compression string `json:"compression"`

Config map[string]interface{} `json:"config"`
}
Expand Down
56 changes: 56 additions & 0 deletions cmd/shield-report/main.go
@@ -0,0 +1,56 @@
package main

import (
"encoding/json"
"fmt"
"io/ioutil"
"os"

"github.com/jhunt/go-cli"
)

var opt struct {
Help bool `cli:"-h, --help"`

Compression string `cli:"-c, --compression"`
}

func main() {
if _, _, err := cli.Parse(&opt); err != nil {
fmt.Fprintf(os.Stderr, "!!! shield-report utility failed to parse command-line flags: %s\n", err)
os.Exit(2)
}

if opt.Help {
fmt.Printf("echo '{\"some\":\"json\"}' | shield-report [OPTIONS]\n\n")
fmt.Printf("OPTIONS\n\n")
fmt.Printf(" -h, --help Show this help screen.\n")
fmt.Printf(" -c, --compression ... Set the \"compression\" key in the output JSON.\n")
fmt.Printf("\n")
os.Exit(0)
}

b, err := ioutil.ReadAll(os.Stdin)
if err != nil {
fmt.Fprintf(os.Stderr, "!!! shield-report utility failed to read standard input: %s\n", err)
os.Exit(3)
}

var data map[string]interface{}
if err := json.Unmarshal(b, &data); err != nil {
fmt.Fprintf(os.Stderr, "!!! shield-report utility failed to parse JSON from standard input: %s\n", err)
os.Exit(3)
}

if opt.Compression != "" {
data["compression"] = opt.Compression
}

b, err = json.Marshal(data)
if err != nil {
fmt.Fprintf(os.Stderr, "!!! shield-report utility failed to encode output JSON: %s\n", err)
os.Exit(4)
}
os.Stdout.Write(b)
os.Exit(0)
}

0 comments on commit c0f60c8

Please sign in to comment.