diff --git a/.container-diff-tests.sh b/.container-diff-tests.sh deleted file mode 100755 index 21705203..00000000 --- a/.container-diff-tests.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -while IFS=$' \n\r' read -r flag differ image1 image2 file; do - go run main.go diff $image1 $image2 $flag -j > $file - if [[ $? -ne 0 ]]; then - echo "ERROR container-diff diff $differ differ failed" - exit 1 - fi -done < tests/test_differ_runs.txt - -while IFS=$' \n\r' read -r flag analyzer image file; do - go run main.go analyze $image $flag -j > $file - if [[ $? -ne 0 ]]; then - echo "ERROR: container-diff analyze $analyzer analyzer failed" - exit 1 - fi -done < tests/test_analyzer_runs.txt - -success=0 -while IFS=$' \n\r' read -r type analyzer actual expected; do - diff=$(jq --argfile a "$actual" --argfile b "$expected" -n 'def walk(f): . as $in | if type == "object" then reduce keys[] as $key ( {}; . + { ($key): ($in[$key] | walk(f)) } ) | f elif type == "array" then map( walk(f) ) | f else f end; ($a | walk(if type == "array" then sort else . end)) as $a | ($b | walk(if type == "array" then sort else . end)) as $b | $a == $b') - if ! "$diff" ; then - echo "ERROR: container-diff analyze $analyzer $type: output is not as expected" - success=1 - fi -done < tests/test_run_comparisons.txt -if [[ "$success" -ne 0 ]]; then - exit 1 -fi - -go test `go list ./... | grep -v vendor` diff --git a/.gitignore b/.gitignore index 2c98f039..d42ab353 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1 @@ -tests/*_actual.json out/* diff --git a/.travis.yml b/.travis.yml index 0842f365..9e50e477 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,11 +1,8 @@ -sudo: required -dist: trusty - language: go +os: linux go: - - 1.8.1 + - 1.8.3 go_import_path: github.com/GoogleCloudPlatform/container-diff script: - - ./.gofmt.sh - - travis_wait ./.container-diff-tests.sh + - make test integration diff --git a/Makefile b/Makefile index 9d303303..c63ecb55 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ ORG := github.com/GoogleCloudPlatform PROJECT := container-diff REPOPATH ?= $(ORG)/$(PROJECT) -SUPPORTED_PLATFORMS := linux-amd64 darwin-amd64 +SUPPORTED_PLATFORMS := linux-amd64 darwin-amd64 windows-amd64 BUILD_PACKAGE = $(REPOPATH) # These build tags are from the containers/image library. @@ -33,7 +33,11 @@ cross: $(foreach platform, $(SUPPORTED_PLATFORMS), out/$(PROJECT)-$(platform)) .PHONY: test test: $(BUILD_DIR)/$(PROJECT) - ./.container-diff-tests.sh + @ ./test.sh + +.PHONY: integration +integration: $(BUILD_DIR)/$(PROJECT) + go test -v -tags integration $(REPOPATH)/tests .PHONY: clean clean: diff --git a/.gofmt.sh b/test.sh similarity index 88% rename from .gofmt.sh rename to test.sh index 5eed0939..ed9ad968 100755 --- a/.gofmt.sh +++ b/test.sh @@ -16,6 +16,10 @@ set -e +echo "Running go tests..." +go test `go list ./... | grep -v vendor` + +echo "Checking gofmt..." files=$(find . -name "*.go" | grep -v vendor/ | xargs gofmt -l -s) if [[ $files ]]; then echo "Gofmt errors in files: $files" diff --git a/tests/integration_test.go b/tests/integration_test.go new file mode 100644 index 00000000..da33a8f3 --- /dev/null +++ b/tests/integration_test.go @@ -0,0 +1,170 @@ +// +build integration + +package tests + +import ( + "bytes" + "fmt" + "io/ioutil" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +const ( + diffBase = "gcr.io/gcp-runtimes/diff-base" + diffModified = "gcr.io/gcp-runtimes/diff-modified" + + aptBase = "gcr.io/gcp-runtimes/apt-base" + aptModified = "gcr.io/gcp-runtimes/apt-modified" + + // Why is this node-modified:2.0? + nodeBase = "gcr.io/gcp-runtimes/node-modified:2.0" + nodeModified = "gcr.io/gcp-runtimes/node-modified" + + pipModified = "gcr.io/gcp-runtimes/pip-modified" + + multiBase = "gcr.io/gcp-runtimes/multi-base" + multiModified = "gcr.io/gcp-runtimes/multi-modified" +) + +type ContainerDiffRunner struct { + t *testing.T + binaryPath string +} + +func (c *ContainerDiffRunner) Run(command ...string) (string, error) { + path, err := filepath.Abs(c.binaryPath) + if err != nil { + c.t.Fatalf("Error finding container-diff binary: %s", err) + } + c.t.Logf("Running command: %s %s", path, command) + cmd := exec.Command(path, command...) + + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + return "", fmt.Errorf("Error running command %s: %s Stderr: %s", command, err, stderr.String()) + } + return stdout.String(), nil +} + +func TestDiffAndAnalysis(t *testing.T) { + runner := ContainerDiffRunner{ + t: t, + binaryPath: "../out/container-diff", + } + + var tests = []struct { + description string + imageA string + imageB string + differFlag string + subcommand string + + //TODO: Don't consume a json file + expectedFile string + }{ + { + description: "file differ", + subcommand: "diff", + imageA: diffBase, + imageB: diffModified, + differFlag: "-f", + expectedFile: "file_diff_expected.json", + }, + { + description: "apt differ", + subcommand: "diff", + imageA: aptBase, + imageB: aptModified, + differFlag: "-a", + expectedFile: "apt_diff_expected.json", + }, + { + description: "node differ", + subcommand: "diff", + imageA: nodeBase, + imageB: nodeModified, + differFlag: "-n", + expectedFile: "node_diff_order_expected.json", + }, + { + description: "multi differ", + subcommand: "diff", + imageA: multiBase, + imageB: multiModified, + differFlag: "-npa", + expectedFile: "multi_diff_expected.json", + }, + { + description: "history differ", + subcommand: "diff", + imageA: diffBase, + imageB: diffModified, + differFlag: "-d", + expectedFile: "hist_diff_expected.json", + }, + { + description: "apt sorted differ", + subcommand: "diff", + imageA: aptBase, + imageB: aptModified, + differFlag: "-ao", + expectedFile: "apt_sorted_diff_expected.json", + }, + { + description: "apt analysis", + subcommand: "analyze", + imageA: aptModified, + differFlag: "-a", + expectedFile: "apt_analysis_expected.json", + }, + { + description: "file sorted analysis", + subcommand: "analyze", + imageA: diffModified, + differFlag: "-fo", + expectedFile: "file_sorted_analysis_expected.json", + }, + { + description: "pip analysis", + subcommand: "analyze", + imageA: pipModified, + differFlag: "-p", + expectedFile: "pip_analysis_expected.json", + }, + { + description: "node analysis", + subcommand: "analyze", + imageA: nodeModified, + differFlag: "-n", + expectedFile: "node_analysis_expected.json", + }, + } + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + args := []string{test.subcommand, test.imageA} + if test.imageB != "" { + args = append(args, test.imageB) + } + args = append(args, test.differFlag) + args = append(args, "-j") + actual, err := runner.Run(args...) + if err != nil { + t.Fatalf("Error running command: %s", err) + } + e, err := ioutil.ReadFile(test.expectedFile) + if err != nil { + t.Fatalf("Error reading expected file output file: %s", err) + } + actual = strings.TrimSpace(actual) + expected := strings.TrimSpace(string(e)) + if actual != expected { + t.Errorf("Error actual output does not match expected. \n\nExpected: %s\n\n Actual: %s", expected, actual) + } + }) + } +} diff --git a/tests/test_run_comparisons.txt b/tests/test_run_comparisons.txt deleted file mode 100644 index 8764707c..00000000 --- a/tests/test_run_comparisons.txt +++ /dev/null @@ -1,10 +0,0 @@ -file diff tests/file_diff_actual.json tests/file_diff_expected.json -apt diff tests/apt_diff_actual.json tests/apt_diff_expected.json -nodeOrder diff tests/node_diff_order_actual.json tests/node_diff_order_expected.json -multi diff tests/multi_diff_actual.json tests/multi_diff_expected.json -history diff tests/hist_diff_actual.json tests/hist_diff_expected.json -pip analysis tests/pip_analysis_actual.json tests/pip_analysis_expected.json -apt analysis tests/apt_analysis_actual.json tests/apt_analysis_expected.json -node analysis tests/node_analysis_actual.json tests/node_analysis_expected.json -aptSort diff tests/apt_sorted_diff_actual.json tests/apt_sorted_diff_expected.json -fileSort analyis tests/file_sorted_analysis_actual.json tests/file_sorted_analysis_expected.json