Merge branch 'multiple_build' into 'master'

Multiple build

See merge request !125

cluster/calcium/build_image.go

@@ -1,8 +1,10 @@
 package calcium
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
+	"html/template"
 	"io"
 	"io/ioutil"
 	"os"
@@ -18,31 +20,30 @@ import (
 	"gopkg.in/yaml.v2"
 )
 
-const dockerFile = `ARG Base
-FROM ${Base}
-ARG Appdir
-ARG Appname
-ARG BuildRun
-ARG Reponame
-ARG UID
-ENV UID ${UID}
-ENV Appname ${Appname}
-ENV Appdir ${Appdir}
+const (
+	FROM   = "FROM %s"
+	FROMAS = "FROM %s as %s"
+	// TODO 在alpine中，useradd不会成功
+	COMMON = `ENV UID {{.UID}}
+ENV Appname {{.Appname}}
+ENV Appdir {{.Appdir}}
 ENV ERU 1
-ADD ${Reponame} ${Appdir}/${Appname}
-WORKDIR ${Appdir}/${Appname}
-RUN useradd -u ${UID} -d /nonexistent -s /sbin/nologin -U ${Appname}
-RUN chown -R ${UID} ${Appdir}/${Appname}
-RUN sh -c "${BuildRun}"
-USER ${Appname}
-` // USER之后的layer都会以user的身份去RUN command，所以这里一定要把USER放到最下面
+{{ if .Source }}ADD {{.Reponame}} {{.Appdir}}/{{.Appname}}{{ else }}RUN mkdir -p {{.Appdir}}/{{.Appname}}{{ end }}
+WORKDIR {{.Appdir}}/{{.Appname}}
+RUN useradd -u {{.UID}} -d /nonexistent -s /sbin/nologin -U {{.Appname}}
+RUN chown -R {{.UID}} {{.Appdir}}/{{.Appname}}`
+	RUN  = "RUN sh -c \"%s\""
+	COPY = "COPY --from=%s %s %s"
+	USER = "USER %s"
+)
 
 // richSpecs is used to format templates
 type richSpecs struct {
 	types.Specs
-	Command string
-	Appdir  string
-	UID     string
+	UID      string
+	Appdir   string
+	Reponame string
+	Source   bool
 }
 
 // Get a random node from pod `podname`
@@ -86,9 +87,9 @@ func (c *calcium) BuildImage(repository, version, uid, artifact string) (chan *t
 
 	buildPodname := c.config.Docker.BuildPod
 	if buildPodname == "" {
-		// use pod `dev` to build image as default
-		buildPodname = "dev"
+		return ch, fmt.Errorf("No build pod set in config")
 	}
+
 	node, err := getRandomNode(c, buildPodname)
 	if err != nil {
 		return ch, err
@@ -116,8 +117,7 @@ func (c *calcium) BuildImage(repository, version, uid, artifact string) (chan *t
 
 	// ensure .git directory is removed
 	// we don't want any history files to be retrieved
-	if err := os.RemoveAll(filepath.Join(cloneDir, ".git")); err != nil {
-		log.Errorf("Error when removing .git dir")
+	if err := c.source.Security(cloneDir); err != nil {
 		return ch, err
 	}
 
@@ -138,14 +138,20 @@ func (c *calcium) BuildImage(repository, version, uid, artifact string) (chan *t
 		os.RemoveAll(cloneDir)
 		os.MkdirAll(cloneDir, os.ModeDir)
 		if err := c.source.Artifact(artifact, cloneDir); err != nil {
-			log.Errorf("Error when downloading artifact: %s", err.Error())
+			return ch, err
 		}
 	}
 
 	// create dockerfile
-	rs := richSpecs{specs, "", strings.TrimRight(c.config.AppDir, "/"), uid}
-	if err := createDockerfile(buildDir); err != nil {
-		return ch, err
+	rs := richSpecs{specs, uid, strings.TrimRight(c.config.AppDir, "/"), reponame, true}
+	if len(specs.Build) > 0 {
+		if err := makeSimpleDockerFile(rs, buildDir); err != nil {
+			return ch, err
+		}
+	} else {
+		if err := makeComplexDockerFile(rs, buildDir); err != nil {
+			return ch, err
+		}
 	}
 
 	// tag of image, later this will be used to push image to hub
@@ -157,9 +163,6 @@ func (c *calcium) BuildImage(repository, version, uid, artifact string) (chan *t
 		return ch, err
 	}
 
-	// generate build args
-	buildArgs := generateBuildArgs(reponame, specs, rs)
-
 	// must be put here because of that `defer os.RemoveAll(buildDir)`
 	buildOptions := enginetypes.ImageBuildOptions{
 		Tags:           []string{tag},
@@ -168,8 +171,8 @@ func (c *calcium) BuildImage(repository, version, uid, artifact string) (chan *t
 		Remove:         true,
 		ForceRemove:    true,
 		PullParent:     true,
-		BuildArgs:      buildArgs,
 	}
+
 	log.Infof("Building image %v with artifact %v at %v:%v", tag, artifact, buildPodname, node.Name)
 	resp, err := node.Engine.ImageBuild(context.Background(), buildContext, buildOptions)
 	if err != nil {
@@ -247,31 +250,85 @@ func createTarStream(path string) (io.ReadCloser, error) {
 	return archive.TarWithOptions(path, tarOpts)
 }
 
-// Dockerfile
-func createDockerfile(buildDir string) error {
-	f, err := os.Create(filepath.Join(buildDir, "Dockerfile"))
+func makeCommonPart(rs richSpecs) (string, error) {
+	tmpl := template.Must(template.New("dockerfile").Parse(COMMON))
+	out := bytes.Buffer{}
+	if err := tmpl.Execute(&out, rs); err != nil {
+		return "", err
+	}
+	return out.String(), nil
+}
+
+func makeMainPart(from, commands string, copys []string, rs richSpecs) (string, error) {
+	var buildTmpl []string
+	common, err := makeCommonPart(rs)
 	if err != nil {
-		return err
+		return "", err
 	}
-	defer f.Close()
-	if _, err := f.WriteString(dockerFile); err != nil {
+	buildTmpl = append(buildTmpl, from, common)
+	if len(copys) > 0 {
+		buildTmpl = append(buildTmpl, copys...)
+	}
+	buildTmpl = append(buildTmpl, commands, "")
+	return strings.Join(buildTmpl, "\n"), nil
+}
+
+func makeSimpleDockerFile(rs richSpecs, buildDir string) error {
+	from := fmt.Sprintf(FROM, rs.Base)
+	user := fmt.Sprintf(USER, rs.Appname)
+	commands := fmt.Sprintf(RUN, strings.Join(rs.Build, " && "))
+	// make sure add source code
+	rs.Source = true
+	mainPart, err := makeMainPart(from, commands, []string{}, rs)
+	if err != nil {
 		return err
 	}
-	return nil
+	dockerfile := fmt.Sprintf("%s\n%s", mainPart, user)
+	return createDockerfile(dockerfile, buildDir)
+}
+
+func makeComplexDockerFile(rs richSpecs, buildDir string) error {
+	var preArtifacts map[string]string
+	var preStage string
+	var buildTmpl []string
+
+	for _, stage := range rs.ComplexBuild.Stages {
+		build, ok := rs.ComplexBuild.Builds[stage]
+		if !ok {
+			log.Warnf("Complex build stage %s not defined", stage)
+			continue
+		}
+
+		from := fmt.Sprintf(FROMAS, build.Base, stage)
+		copys := []string{}
+		for src, dst := range preArtifacts {
+			copys = append(copys, fmt.Sprintf(COPY, preStage, src, dst))
+		}
+		commands := fmt.Sprintf(RUN, strings.Join(build.Commands, " && "))
+		// decide add source or not
+		rs.Source = build.Source
+		mainPart, err := makeMainPart(from, commands, copys, rs)
+		if err != nil {
+			return err
+		}
+		buildTmpl = append(buildTmpl, mainPart)
+		preStage = stage
+		preArtifacts = build.Artifacts
+	}
+	buildTmpl = append(buildTmpl, fmt.Sprintf(USER, rs.Appname))
+	dockerfile := strings.Join(buildTmpl, "\n")
+	return createDockerfile(dockerfile, buildDir)
 }
 
-// generate build args
-func generateBuildArgs(reponame string, specs types.Specs, rs richSpecs) map[string]*string {
-	buildArgs := map[string]*string{}
-	buildArgs["Base"] = &(specs.Base)
-	buildArgs["Appdir"] = &(rs.Appdir)
-	buildArgs["Appname"] = &(rs.Appname)
-	runCommands := strings.Join(specs.Build, " && ")
-	buildArgs["BuildRun"] = &runCommands
-	buildArgs["Reponame"] = &reponame
-	buildArgs["UID"] = &rs.UID
-
-	return buildArgs
+// Dockerfile
+func createDockerfile(dockerfile, buildDir string) error {
+	f, err := os.Create(filepath.Join(buildDir, "Dockerfile"))
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	_, err = f.WriteString(dockerfile)
+	return err
 }
 
 // Image tag

cluster/calcium/build_image_test.go

@@ -1,6 +1,9 @@
 package calcium
 
 import (
+	"fmt"
+	"io/ioutil"
+	"os"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -45,5 +48,60 @@ func TestGetRandomNodeFail(t *testing.T) {
 	assert.Nil(t, node)
 }
 
-// 后面的我实在不想写了
-// 让我们相信接口都是正确的吧
+func TestMultipleBuildDockerFile(t *testing.T) {
+	builds := types.ComplexBuild{
+		Stages: []string{"test", "step1", "setp2"},
+		Builds: map[string]types.Build{
+			"step1": types.Build{
+				Base:     "alpine:latest",
+				Commands: []string{"cp /bin/ls /root/artifact", "date > /root/something"},
+				Artifacts: map[string]string{
+					"/root/artifact":  "/root/artifact",
+					"/root/something": "/root/something",
+				},
+			},
+			"setp2": types.Build{
+				Base:     "centos:latest",
+				Commands: []string{"echo yooo", "sleep 1"},
+			},
+			"test": types.Build{
+				Base:     "ubuntu:latest",
+				Commands: []string{"date", "echo done"},
+			},
+		},
+	}
+	appname := "hello-app"
+	specs := types.Specs{
+		Appname:      appname,
+		ComplexBuild: builds,
+	}
+	rs := richSpecs{specs, "1001", "/app", appname, true}
+
+	tempDIR := os.TempDir()
+	err := makeComplexDockerFile(rs, tempDIR)
+	assert.NoError(t, err)
+	f, _ := os.Open(fmt.Sprintf("%s/Dockerfile", tempDIR))
+	bs, _ := ioutil.ReadAll(f)
+	fmt.Printf("%s", bs)
+	f.Close()
+	os.Remove(f.Name())
+}
+
+func TestSingleBuildDockerFile(t *testing.T) {
+	appname := "hello-app"
+	specs := types.Specs{
+		Appname: appname,
+		Build:   []string{"echo yes", "echo no"},
+		Base:    "alpine:latest",
+	}
+	rs := richSpecs{specs, "1001", "/app", appname, true}
+
+	tempDIR := os.TempDir()
+	err := makeSimpleDockerFile(rs, tempDIR)
+	assert.NoError(t, err)
+	f, _ := os.Open(fmt.Sprintf("%s/Dockerfile", tempDIR))
+	bs, _ := ioutil.ReadAll(f)
+	fmt.Printf("%s", bs)
+	f.Close()
+	os.Remove(f.Name())
+}

rpc/rpc_test.go

@@ -212,6 +212,7 @@ func initConfig(mStore *mockstore.MockStore) (types.Config, *vibranium) {
 		Docker: types.DockerConfig{
 			APIVersion: "v1.23",
 			LogDriver:  "none",
+			BuildPod:   "dev",
 		},
 	}
 

source/gitlab/cesium.go

@@ -139,6 +139,10 @@ func (c *cesium) Artifact(artifact, path string) error {
 	return nil
 }
 
+func (c *cesium) Security(path string) error {
+	return os.RemoveAll(filepath.Join(path, ".git"))
+}
+
 func New(config types.Config) *cesium {
 	return &cesium{config: config}
 }

source/source.go

@@ -5,4 +5,6 @@ type Source interface {
 	SourceCode(repository, path, revision string) error
 	// Get related artifact by artifact into path
 	Artifact(artifact, path string) error
+	// Keep code security
+	Security(path string) error
 }

types/specs.go

@@ -9,13 +9,26 @@ import (
 
 // correspond to app.yaml in repository
 type Specs struct {
-	Appname     string                `yaml:"appname,omitempty"`
-	Entrypoints map[string]Entrypoint `yaml:"entrypoints,omitempty,flow"`
-	Build       []string              `yaml:"build,omitempty,flow"`
-	Volumes     []string              `yaml:"volumes,omitempty,flow"`
-	Meta        map[string]string     `yaml:"meta,omitempty,flow"`
-	Base        string                `yaml:"base"`
-	DNS         []string              `yaml:"dns,omitempty,flow"`
+	Appname      string                `yaml:"appname,omitempty"`
+	Entrypoints  map[string]Entrypoint `yaml:"entrypoints,omitempty,flow"`
+	Build        []string              `yaml:"build,omitempty,flow"`
+	ComplexBuild ComplexBuild          `yaml:"complex_build,omitempty,flow"`
+	Volumes      []string              `yaml:"volumes,omitempty,flow"`
+	Meta         map[string]string     `yaml:"meta,omitempty,flow"`
+	Base         string                `yaml:"base"`
+	DNS          []string              `yaml:"dns,omitempty,flow"`
+}
+
+type ComplexBuild struct {
+	Stages []string         `yaml:"stages,omitempty,flow"`
+	Builds map[string]Build `yaml:"builds,omitempty,flow"`
+}
+
+type Build struct {
+	Base      string            `yaml:"base,omitempty"`
+	Source    bool              `yaml:"source,omitempty"`
+	Commands  []string          `yaml:"commands,omitempty,flow"`
+	Artifacts map[string]string `yaml:"artifacts,omitempty,flow"`
 }
 
 // single entrypoint