Permalink
Browse files

*: add integration coverage stats [75%]

This now includes all of the coverage stats for umoci during integration
testing. This stilll needs to be combined with unit tests to get a
proper sense of the overall statistics.

Signed-off-by: Aleksa Sarai <asarai@suse.com>
  • Loading branch information...
1 parent 169b521 commit 2bcf87a19a396a70cb62bf0f4bbeafc743ec47ad @cyphar committed Dec 28, 2016
Showing with 176 additions and 9 deletions.
  1. +1 −0 .gitignore
  2. +1 −0 Dockerfile
  3. +17 −6 Makefile
  4. +56 −0 cmd/umoci/main_test.go
  5. +47 −0 hack/collate.awk
  6. +20 −3 test/helpers.bash
  7. +34 −0 test/run.sh
View
@@ -3,3 +3,4 @@
/vendor/src
/vendor/pkg
/vendor/github.com/cyphar/umoci
+/*.cov*
View
@@ -32,6 +32,7 @@ RUN zypper -n in \
go-mtree \
jq \
make \
+ moreutils \
oci-image-tools \
oci-runtime-tools \
python3-setuptools \
View
@@ -13,6 +13,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+# Use bash, so that we can do process substitution.
+SHELL = /bin/bash
+
+# Go tools.
GO ?= go
GO_MD2MAN ?= go-md2man
@@ -36,6 +40,9 @@ umoci: $(GO_SRC)
umoci.static: $(GO_SRC)
CGO_ENABLED=0 $(GO) build -ldflags "-s -w -extldflags '-static' -X main.gitCommit=${COMMIT} -X main.version=${VERSION}" -tags "$(BUILDTAGS)" -o $@ $(PROJECT)/cmd/umoci
+umoci.cover: $(GO_SRC)
+ $(GO) test -c -cover -coverpkg=$(PROJECT)/... -covermode=count -ldflags "-s -w -X main.gitCommit=${COMMIT} -X main.version=${VERSION}" -tags "$(BUILDTAGS)" -o $@ $(PROJECT)/cmd/umoci
+
.PHONY: update-deps
update-deps:
hack/vendor.sh
@@ -85,20 +92,24 @@ test-unit: umociimage
docker run --rm -it -v $(PWD):/go/src/$(PROJECT) -u 1000:1000 --cap-drop=all $(UMOCI_IMAGE) make local-test-unit
.PHONY: local-test-unit
-local-test-unit: umoci
+local-test-unit:
go test -cover -v $(PROJECT)/...
.PHONY: test-integration
+ifndef COVERAGE
+COVERAGE := $(shell mktemp umoci-integration.cov.XXXXXX)
+endif
test-integration: umociimage
- docker run --rm -it -v $(PWD):/go/src/$(PROJECT) $(UMOCI_IMAGE) make local-test-integration
- docker run --rm -it -v $(PWD):/go/src/$(PROJECT) -u 1000:1000 --cap-drop=all $(UMOCI_IMAGE) make local-test-integration
+ docker run --rm -it -v $(PWD):/go/src/$(PROJECT) -e COVERAGE=$(COVERAGE) $(UMOCI_IMAGE) make local-test-integration
+ docker run --rm -it -v $(PWD):/go/src/$(PROJECT) -e COVERAGE=$(COVERAGE) -u 1000:1000 --cap-drop=all $(UMOCI_IMAGE) make local-test-integration
+ $(GO) tool cover -func <(egrep -v 'vendor|third_party' $(COVERAGE))
.PHONY: local-test-integration
-local-test-integration: umoci
- bats -t test/*.bats
+local-test-integration: umoci.cover
+ test/run.sh
shell: umociimage
docker run --rm -it -v $(PWD):/go/src/$(PROJECT) $(UMOCI_IMAGE) bash
.PHONY: ci
-ci: umoci validate doc test-unit test-integration
+ci: umoci umoci.cover validate doc test-unit test-integration
@@ -0,0 +1,56 @@
+/*
+ * umoci: Umoci Modifies Open Containers' Images
+ * Copyright (C) 2016 SUSE LLC.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+ "os"
+ "strings"
+ "testing"
+)
+
+// Build:
+// $ go test -c -covermode=count -o umoci \
+// -cover -coverpkg=github.com/cyphar/umoci/... \
+// github.com/cyphar/umoci/cmd/umoci
+// Run:
+// $ ./umoci --i-heard-you-like-tests -test.coverprofile [file] [args]...
+
+// TestUmoci is a hack that allows us to figure out what the coverage is during
+// integration tests. I would not recommend that you use a binary built using
+// this hack outside of a test suite.
+func TestUmoci(t *testing.T) {
+ var (
+ args []string
+ run bool
+ )
+
+ for _, arg := range os.Args {
+ switch {
+ case arg == "~~i-heard-you-like-tests":
+ run = true
+ case strings.HasPrefix(arg, "-test"):
+ default:
+ args = append(args, arg)
+ }
+ }
+ os.Args = args
+
+ if run {
+ main()
+ }
+}
View
@@ -0,0 +1,47 @@
+#!/usr/bin/awk -f
+# Copyright (C) 2016 SUSE LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# collate.awk allows you to collate a bunch of Go coverprofiles for a given
+# binary (generated with -test.coverprofile), so that the statistics actually
+# make sense. The input to this function is just the concatenated versions of
+# the coverage reports, and the output is the combined coverage report.
+#
+# NOTE: This will _only_ work on coverage binaries compiles with
+# -covermode=count. The other modes aren't supported.
+
+{
+ # Every coverage file in the set will start with a "mode:" header. Just make
+ # sure they're all set to "count".
+ if ($1 == "mode:") {
+ if ($0 != "mode: count") {
+ print "Invalid coverage mode", $2 > "/dev/stderr"
+ exit 1
+ }
+ next
+ }
+
+ # The format of all other lines is as follows.
+ # <file>:<startline>.<startcol>,<endline>.<endcol> <numstmt> <count>
+ # We only care about the first field and the count.
+ statements[$1] = $2
+ counts[$1] += $3
+}
+
+END {
+ print "mode: count"
+ for (block in statements) {
+ print block, statements[block], counts[block]
+ }
+}
View
@@ -16,7 +16,7 @@
# Root directory of integration tests.
INTEGRATION_ROOT=$(dirname "$(readlink -f "$BASH_SOURCE")")
-UMOCI="${INTEGRATION_ROOT}/../umoci"
+UMOCI="${UMOCI:-${INTEGRATION_ROOT}/../umoci}"
GOMTREE="/usr/bin/gomtree" # For some reason $(whence ...) and $(where ...) are broken.
# The source OCI image path, which we will make a copy of for each test.
@@ -27,6 +27,9 @@ SOURCE_TAG="${SOURCE_TAG:-latest}"
IMAGE="${BATS_TMPDIR}/image"
TAG="${SOURCE_TAG}"
+# We need to store the coverage outputs somewhere.
+COVERAGE_DIR="${COVERAGE_DIR:-}"
+
# Are we rootless?
ROOTLESS="$(id -u)"
@@ -48,7 +51,7 @@ function requires() {
}
function image-verify() {
- oci-image-validate --type "$imageLayout" "$@"
+ oci-image-validate --type "imageLayout" "$@"
return $?
}
@@ -64,7 +67,13 @@ function bundle-verify() {
}
function umoci() {
- local args=("$1")
+ local args=()
+ if [ "$COVERAGE_DIR" ]; then
+ args+=("-test.coverprofile=$(mktemp -p "$COVERAGE_DIR" umoci.cov.XXXXXX)")
+ fi
+
+ # Set the first argument (the subcommand).
+ args+=("~~i-heard-you-like-tests" "$1")
# We're rootless if we're asked to unpack something.
if [[ "$ROOTLESS" != 0 && "$1" == "unpack" ]]; then
@@ -74,6 +83,14 @@ function umoci() {
shift
args+=("$@")
sane_run "$UMOCI" "${args[@]}"
+
+ # Because this is running as a -test.cover test, we need to remove the last
+ # two lines.
+ if [ "$status" -eq 0 ]; then
+ export output="$(echo "$output" | head -n-2)"
+ unset 'lines[${#lines[@]}-1]'
+ unset 'lines[${#lines[@]}-1]'
+ fi
}
function gomtree() {
View
@@ -0,0 +1,34 @@
+#!/bin/bash
+# umoci: Umoci Modifies Open Containers' Images
+# Copyright (C) 2016 SUSE LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+# Set up the root and coverage directories.
+export ROOT="$(readlink -f "$(dirname "$(readlink -f "$BASH_SOURCE")")/..")"
+export COVERAGE_DIR=$(mktemp --tmpdir -d umoci-coverage.XXXXXX)
+
+# Create a temporary symlink for umoci, since the --help tests require the
+# binary have the name "umoci". This is all just to make the Makefile nicer.
+UMOCI_DIR="$(mktemp --tmpdir -d umoci.XXXXXX)"
+export UMOCI="$UMOCI_DIR/umoci"
+ln -s "$ROOT/umoci.cover" "$UMOCI"
+
+# Run the tests and collate the results.
+bats -t $ROOT/test/*.bats
+$ROOT/hack/collate.awk $COVERAGE_DIR/* $COVERAGE | sponge $COVERAGE
+
+# Clean up the coverage directory.
+rm -rf "$COVERAGE_DIR"

0 comments on commit 2bcf87a

Please sign in to comment.