Skip to content
Permalink
Browse files

implement manual build selectors, wired to build tags in go buil… (#566)

* implement manual build selectors, wired to build tags in go builders.

This implements manual shim selection as specified in
the docs/EVOLVING_APIs.md specification. Users can now
set build tags to be applied when building group artifacts.

* implement build selectors tests.

* disable go proxy in tests.
  • Loading branch information
raulk committed Feb 28, 2020
1 parent 44d1cf1 commit cf00c1bb11a19a369834798b8bdab60ea96a2247
@@ -23,6 +23,7 @@ id = "bootstrappers"
instances = { count = 1 }

[groups.build]
selectors = ["foo"]
dependencies = [
{ module = "github.com/libp2p/go-libp2p-kad-dht", version = "995fee9e5345fdd7c151a5fe871252262db4e788"},
{ module = "github.com/libp2p/go-libp2p", version = "76944c4fc848530530f6be36fb22b70431ca506c"},
@@ -37,3 +38,6 @@ instances = { count = 49 }

[groups.run]
test_params = { random_walk = "true", n_bootstrap = "1" }

[groups.build]
selectors = ["bar", "bee"]
1 go.mod
@@ -38,6 +38,7 @@ require (
github.com/opencontainers/runc v0.1.1 // indirect
github.com/otiai10/copy v1.0.2
github.com/pborman/uuid v1.2.0
github.com/stretchr/testify v1.4.0
github.com/urfave/cli v1.22.1
github.com/vishvananda/netlink v1.0.0
github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df
@@ -33,6 +33,9 @@ type BuildInput struct {
Directories Directories
// TestPlan is the metadata of the test plan being built.
TestPlan *TestPlanDefinition
// Selectors specifies any source selection strings to be sent to the
// builder. In the case of go builders, this field maps to build tags.
Selectors []string
// Dependencies are the versions of upstream dependencies we want to build
// against. For a go build, this could be e.g.:
// github.com/ipfs/go-ipfs=v0.4.22
@@ -98,6 +98,10 @@ type Instances struct {
type Dependencies []Dependency

type Build struct {
// Selectors specifies any source selection strings to be sent to the
// builder. In the case of go builders, this field maps to build tags.
Selectors []string

// Dependencies specifies any upstream dependency overrides to apply to this
// build.
Dependencies Dependencies `toml:"dependencies" json:"dependencies"`
@@ -16,6 +16,8 @@ ARG GO_IPFS_VERSION
ARG TESTPLAN_EXEC_PKG
# GO_PROXY is the go proxy that will be used, or direct by default.
ARG GO_PROXY=direct
# BUILD_TAGS is either nothing, or when expanded, it expands to "-tags <comma-separated build tags>"
ARG BUILD_TAGS

ENV TESTPLAN_EXEC_PKG ${TESTPLAN_EXEC_PKG}
# PLAN_DIR is the location containing the plan source inside the container.
@@ -42,7 +44,7 @@ RUN cd ${PLAN_DIR} \
COPY . /
RUN cd ${PLAN_DIR} \
&& go env -w GOPROXY="${GO_PROXY}" \
&& CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o testplan ${TESTPLAN_EXEC_PKG}
&& CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o testplan ${BUILD_TAGS} ${TESTPLAN_EXEC_PKG}

# Store module dependencies
RUN cd ${PLAN_DIR} \
@@ -199,15 +199,24 @@ func (b *DockerGoBuilder) Build(ctx context.Context, in *api.BuildInput, output
return nil, fmt.Errorf("unable to add replace directives to go.mod; %w", err)
}

// initial go build args.
var args = map[string]*string{
"GO_VERSION": &cfg.GoVersion,
"GO_IPFS_VERSION": &cfg.GoIPFSVersion,
"TESTPLAN_EXEC_PKG": &cfg.ExecPkg,
"GO_PROXY": &proxyURL,
}

// set BUILD_TAGS arg if the user has provided selectors.
if len(in.Selectors) > 0 {
s := "-tags " + strings.Join(in.Selectors, ",")
args["BUILD_TAGS"] = &s
}

opts := types.ImageBuildOptions{
Tags: []string{id, in.BuildID},
NetworkMode: "testground-build",
BuildArgs: map[string]*string{
"GO_VERSION": &cfg.GoVersion,
"GO_IPFS_VERSION": &cfg.GoIPFSVersion,
"TESTPLAN_EXEC_PKG": &cfg.ExecPkg,
"GO_PROXY": &proxyURL,
},
BuildArgs: args,
}

tar, err := archive.TarWithOptions(tmp, &archive.TarOptions{})
@@ -118,8 +118,19 @@ func (b *ExecGoBuilder) Build(ctx context.Context, input *api.BuildInput, output
return nil, fmt.Errorf("unable to add replace directives to go.mod; %w", err)
}

// Calculate the arguments to go build.
// go build -o <output_path> [-tags <comma-separated tags>] <exec_pkg>
var args = []string{"build", "-o", path}
if len(input.Selectors) > 0 {
args = append(args, "-tags")
args = append(args, strings.Join(input.Selectors, ","))
}
args = append(args, cfg.ExecPkg)

fmt.Printf("%v\n", args)

// Execute the build.
cmd = exec.CommandContext(ctx, "go", "build", "-o", path, cfg.ExecPkg)
cmd = exec.CommandContext(ctx, "go", args...)
cmd.Dir = plandst
out, err := cmd.CombinedOutput()
if err != nil {
@@ -0,0 +1,84 @@
package golang_test

import (
"context"
"io/ioutil"
"os"
"testing"
"time"

"github.com/docker/docker/client"

"github.com/ipfs/testground/pkg/api"
"github.com/ipfs/testground/pkg/build/golang"
"github.com/ipfs/testground/pkg/config"
"github.com/ipfs/testground/pkg/engine"

"github.com/stretchr/testify/require"
)

func TestBuildSelector(t *testing.T) {
require := require.New(t)

tmp, err := ioutil.TempDir("", "")
require.NoError(err)
defer os.RemoveAll(tmp)

env, err := config.GetEnvConfig()
require.NoError(err)

cfg := &engine.EngineConfig{
Builders: []api.Builder{new(golang.DockerGoBuilder), new(golang.ExecGoBuilder)},
Runners: []api.Runner{},
EnvConfig: env,
}

engine, err := engine.NewEngine(cfg)
require.NoError(err)

buildFn := func(builder string, selectors []string, assertion func(err error, msgsAndArgs ...interface{})) func(t *testing.T) {
return func(t *testing.T) {
comp := &api.Composition{
Global: api.Global{
Builder: builder,
Plan: "placebo",
Case: "ok",
TotalInstances: 1,
BuildConfig: map[string]interface{}{
"go_proxy_mode": "direct",
},
},
Groups: []api.Group{
api.Group{
ID: "test",
Build: api.Build{Selectors: selectors},
Instances: api.Instances{Count: 1},
},
},
}

// this build is using the "foo" and "bar" selectors; it will fail.
_, err = engine.DoBuild(context.TODO(), comp, ioutil.Discard)
assertion(err)
}

}

t.Run("exec:go/selectors", buildFn("exec:go", []string{"foo", "bar"}, require.Error))
t.Run("exec:go/no_selectors", buildFn("exec:go", []string{}, require.NoError))

// if we have a docker daemon running, test the docker runner too.
cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
if err != nil {
return
}

ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
if _, err := cli.Ping(ctx); err != nil {
return
}

t.Run("docker:go/selectors", buildFn("docker:go", []string{"foo", "bar"}, require.Error))
t.Run("docker:go/no_selectors", buildFn("docker:go", []string{}, require.NoError))
}
@@ -79,6 +79,10 @@ func NewEngine(cfg *EngineConfig) (*Engine, error) {
e.runners[r.ID()] = r
}

if _, err := e.discoverTestPlans(); err != nil {
return nil, err
}

return e, nil
}

@@ -99,8 +103,6 @@ func NewDefaultEngine() (*Engine, error) {
return nil, err
}

_, _ = e.discoverTestPlans()

return e, nil
}

@@ -233,6 +235,7 @@ func (e *Engine) DoBuild(ctx context.Context, comp *api.Composition, output io.W
EnvConfig: *e.envcfg,
Directories: e.envcfg,
TestPlan: plan,
Selectors: grp.Build.Selectors,
Dependencies: grp.Build.Dependencies.AsMap(),
}

@@ -0,0 +1,4 @@
// +build foo,bar

this is garbage.
when the build tags are activated, this file will detonate the build.

0 comments on commit cf00c1b

Please sign in to comment.
You can’t perform that action at this time.