Skip to content

Commit

Permalink
Add ability to specify SLS info to docker image (#149)
Browse files Browse the repository at this point in the history 
* Add ability to specify SLS info to docker image

* Base64 Encode the sls info string

* simple commit to force update
  • Loading branch information
@yvdinesh @nmiyake
yvdinesh authored and nmiyake committed Jun 16, 2017
1 parent 2fe6b4c commit b43c62e
Show file tree
Hide file tree
Showing 12 changed files with 442 additions and 106 deletions.
45 changes: 1 addition & 44 deletions apps/distgo/cmd/dist/slsdist.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,10 @@ import (
"io"
"io/ioutil"
"path"
"strings"
"text/template"

"github.com/palantir/pkg/specdir"
"github.com/pkg/errors"
"gopkg.in/yaml.v2"

"github.com/palantir/godel/apps/distgo/params"
"github.com/palantir/godel/apps/distgo/pkg/binspec"
Expand Down Expand Up @@ -261,7 +259,7 @@ func (s *slsDister) writeSLSManifest(buildSpec params.ProductBuildSpec, distCfg
manifestTemplateString = manifestBuf.String()
} else {
var err error
manifestTemplateString, err = manifest(distCfg.Publish.GroupID, buildSpec.ProductName, buildSpec.ProductVersion, s.ProductType, s.ManifestExtensions)
manifestTemplateString, err = params.GetManifest(distCfg.Publish.GroupID, buildSpec.ProductName, buildSpec.ProductVersion, s.ProductType, s.ManifestExtensions)
if err != nil {
return errors.Wrapf(err, "failed to create manifest for SLS distribution")
}
Expand All @@ -272,47 +270,6 @@ func (s *slsDister) writeSLSManifest(buildSpec params.ProductBuildSpec, distCfg
return nil
}

type slsManifest struct {
ManifestVersion string `yaml:"manifest-version"`
ProductGroup string `yaml:"product-group"`
ProductName string `yaml:"product-name"`
ProductVersion string `yaml:"product-version"`
ProductType string `yaml:"product-type,omitempty"`
Extensions map[string]interface{} `yaml:"extensions,omitempty"`
}

func manifest(groupID, name, version, productType string, extensions map[string]interface{}) (string, error) {
var missingRequired []string
if groupID == "" {
missingRequired = append(missingRequired, "group-id")
}
if name == "" {
missingRequired = append(missingRequired, "product-name")
}
if version == "" {
missingRequired = append(missingRequired, "product-version")
}
if len(missingRequired) > 0 {
return "", errors.Errorf("required properties were missing: " + strings.Join(missingRequired, ", "))
}

m := slsManifest{
ManifestVersion: "1.0",
ProductGroup: groupID,
ProductName: name,
ProductVersion: version,
Extensions: extensions,
}
if productType != "" {
m.ProductType = productType
}
manifestBytes, err := yaml.Marshal(m)
if err != nil {
return "", errors.Wrapf(err, "failed to marshal %v as YAML", m)
}
return string(manifestBytes), nil
}

func (s *slsDister) writeSLSInitSh(buildSpec params.ProductBuildSpec, distCfg params.Dist, specDir specdir.SpecDir) error {
var initShTemplateBytes []byte
if s.InitShTemplateFile != "" {
Expand Down
40 changes: 17 additions & 23 deletions apps/distgo/cmd/docker/build.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ import (
"fmt"
"io"
"os"
"os/exec"
"path"

"github.com/pkg/errors"
Expand All @@ -39,7 +38,7 @@ func Build(cfg params.Project, wd string, baseRepo string, stdout io.Writer) err
productsToBuildImage[productName] = struct{}{}
}
for _, image := range productSpec.DockerImages {
for _, dep := range image.Deps {
for _, dep := range image.Dependencies() {
if isDist(dep.Type) {
productsToDist[dep.Product] = struct{}{}
}
Expand All @@ -65,8 +64,8 @@ func Build(cfg params.Project, wd string, baseRepo string, stdout io.Writer) err
// if base repo is specified, join it to each image's repo
for i := range orderedSpecs {
for j := range orderedSpecs[i].Spec.DockerImages {
orderedSpecs[i].Spec.DockerImages[j].Repository = path.Join(baseRepo,
orderedSpecs[i].Spec.DockerImages[j].Repository)
repo, _ := orderedSpecs[i].Spec.DockerImages[j].Coordinates()
orderedSpecs[i].Spec.DockerImages[j].SetRepository(path.Join(baseRepo, repo))
}
}
}
Expand All @@ -86,20 +85,13 @@ func RunBuild(buildSpecsWithDeps []params.ProductBuildSpecWithDeps, stdout io.Wr
}

func buildImage(image params.DockerImage, buildSpecsWithDeps params.ProductBuildSpecWithDeps, specsMap map[string]params.ProductBuildSpecWithDeps, stdout io.Writer) error {
if image.Repository == "" {
image.Repository = buildSpecsWithDeps.Spec.ProductName
}
if image.Tag == "" {
image.Tag = buildSpecsWithDeps.Spec.ProductVersion
}

fmt.Fprintf(stdout, "Building docker image for %s and tagging it as %s:%s\n", buildSpecsWithDeps.Spec.ProductName, image.Repository, image.Tag)
repo, tag := image.Coordinates()
fmt.Fprintf(stdout, "Building docker image for %s and tagging it as %s:%s\n", buildSpecsWithDeps.Spec.ProductName, repo, tag)

completeTag := fmt.Sprintf("%s:%s", image.Repository, image.Tag)
contextDir := path.Join(buildSpecsWithDeps.Spec.ProjectDir, image.ContextDir)
contextDir := path.Join(buildSpecsWithDeps.Spec.ProjectDir, image.ContextDirectory())

// link dependent dist artifacts into the context directory
for depProduct, depTypes := range image.Deps.ToMap() {
for depProduct, depTypes := range dockerDepsToMap(image.Dependencies()) {
for depType, targetFile := range depTypes {
if !isDist(depType) {
continue
Expand Down Expand Up @@ -134,16 +126,18 @@ func buildImage(image params.DockerImage, buildSpecsWithDeps params.ProductBuild
}
}

var args []string
args = append(args, "build")
args = append(args, "--tag", completeTag)
args = append(args, contextDir)
return image.Build(buildSpecsWithDeps)
}

buildCmd := exec.Command("docker", args...)
if output, err := buildCmd.CombinedOutput(); err != nil {
return errors.Wrap(err, fmt.Sprintf("docker build failed with error:\n%s\n", string(output)))
func dockerDepsToMap(deps []params.DockerDep) map[string]map[params.DockerDepType]string {
m := make(map[string]map[params.DockerDepType]string)
for _, dep := range deps {
if m[dep.Product] == nil {
m[dep.Product] = make(map[params.DockerDepType]string)
}
m[dep.Product][dep.Type] = dep.TargetFile
}
return nil
return m
}

func isDist(dep params.DockerDepType) bool {
Expand Down
124 changes: 121 additions & 3 deletions apps/distgo/cmd/docker/build_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ package docker_test

import (
"context"
"encoding/base64"
"fmt"
"io/ioutil"
"math/rand"
Expand Down Expand Up @@ -51,6 +52,9 @@ func main() {
}
`
dockerfile = `FROM alpine:3.5
`
configFile = `test_key: 'test_value'
test_key_2: test_value_2
`
slsDepDockerFile = `FROM alpine:3.5
Expand Down Expand Up @@ -134,7 +138,7 @@ func TestDockerDist(t *testing.T) {
},
},
DockerImages: []params.DockerImage{
{
&params.DefaultDockerImage{
Repository: fullRepoName("bar", pad),
Tag: "0.1.0",
ContextDir: "bar/docker",
Expand Down Expand Up @@ -164,7 +168,7 @@ func TestDockerDist(t *testing.T) {
},
},
DockerImages: []params.DockerImage{
{
&params.DefaultDockerImage{
Repository: fullRepoName("foo", pad),
Tag: "0.1.0",
ContextDir: "foo/docker",
Expand Down Expand Up @@ -212,6 +216,120 @@ func TestDockerDist(t *testing.T) {
require.True(t, len(images) > 0, "Case %d: %s", caseNum, name)
},
},
{
name: "sls docker dist",
setupProject: func(projectDir, pad string) error {
gittest.InitGitDir(t, projectDir)
// initialize foo
fooDir := path.Join(projectDir, "foo")
if err := os.Mkdir(fooDir, 0777); err != nil {
return err
}
if err := ioutil.WriteFile(path.Join(fooDir, "main.go"), []byte(testMain), 0644); err != nil {
return err
}
fooDockerDir := path.Join(fooDir, "docker")
if err = os.Mkdir(fooDockerDir, 0777); err != nil {
return err
}
if err = ioutil.WriteFile(path.Join(fooDockerDir, "Dockerfile"), []byte(dockerfile), 0777); err != nil {
return err
}
if err = ioutil.WriteFile(path.Join(fooDockerDir, params.ConfigurationFileName), []byte(configFile), 0777); err != nil {
return err
}

// commit
gittest.CommitAllFiles(t, projectDir, "Commit")
return nil
},
spec: func(projectDir string, pad string) []params.ProductBuildSpecWithDeps {
allSpec := make(map[string]params.ProductBuildSpec)
fooSpec := params.NewProductBuildSpec(
projectDir,
"foo",
git.ProjectInfo{
Version: "0.1.0",
},
params.Product{
Build: params.Build{
MainPkg: "./foo",
OSArchs: []osarch.OSArch{
{
OS: "linux",
Arch: "amd64",
},
},
},
DockerImages: []params.DockerImage{
&params.SLSDockerImage{
DefaultDockerImage: params.DefaultDockerImage{
Repository: fullRepoName("foo", pad),
Tag: "0.1.0",
ContextDir: "foo/docker",
Deps: []params.DockerDep{
{
Product: "bar",
Type: params.DockerDepDocker,
TargetFile: "",
},
},
},
ProuductType: "test_type",
GroupID: "com.palantir.godel",
Extensions: map[string]interface{}{
"test_key": "test_value",
},
},
},
},
params.Project{
GroupID: "com.test.group",
},
)
allSpec["foo"] = fooSpec
fooSpecWithDeps, err := params.NewProductBuildSpecWithDeps(fooSpec, allSpec)
require.NoError(t, err)

return []params.ProductBuildSpecWithDeps{fooSpecWithDeps}
},
cleanup: func(cli *dockercli.Client, projectDir, pad string) {
images := []string{fmt.Sprintf("%v:0.1.0", fullRepoName("foo", pad))}
err := removeImages(cli, images)
if err != nil {
t.Logf("Failed to remove images: %v", err)
}
},
preDistAction: func(projectDir string, buildSpec []params.ProductBuildSpecWithDeps) {
gittest.CreateGitTag(t, projectDir, "0.1.0")
},
validate: func(caseNum int, name string, pad string, cli *dockercli.Client) {
filter := filters.NewArgs()
filter.Add("reference", fmt.Sprintf("%v:0.1.0", fullRepoName("foo", pad)))
images, err := cli.ImageList(context.Background(), types.ImageListOptions{Filters: filter})
require.NoError(t, err, "Case %d: %s", caseNum, name)
require.True(t, len(images) > 0, "Case %d: %s", caseNum, name)
image := images[0]
inspect, _, err := cli.ImageInspectWithRaw(context.Background(), image.ID)
require.NoError(t, err, "Case %d: %s", caseNum, name)
actualManifestEncoded, ok := inspect.Config.Labels[params.ManifestLabel]
require.True(t, ok, "Case %d: %s", caseNum, name)
actualManifest, err := base64.StdEncoding.DecodeString(actualManifestEncoded)
require.NoError(t, err, "Case %d: %s", caseNum, name)
expectedManifest := "manifest-version: \"1.0\"\n" +
"product-group: com.palantir.godel\n" +
"product-name: foo\n" +
"product-version: 0.1.0\n" +
"product-type: test_type\n" +
"extensions:\n test_key: test_value\n"
require.Equal(t, expectedManifest, string(actualManifest), "Case %d: %s", caseNum, name)
actualConfigEncoded, ok := inspect.Config.Labels[params.ConfigurationLabel]
require.True(t, ok, "Case %d: %s", caseNum, name)
actualConfig, err := base64.StdEncoding.DecodeString(actualConfigEncoded)
require.NoError(t, err, "Case %d: %s", caseNum, name)
require.Equal(t, configFile, string(actualConfig), "Case %d: %s", caseNum, name)
},
},
{
name: "docker dist with dependent sls dist",
setupProject: func(projectDir, pad string) error {
Expand Down Expand Up @@ -296,7 +414,7 @@ func TestDockerDist(t *testing.T) {
},
},
DockerImages: []params.DockerImage{
{
&params.DefaultDockerImage{
Repository: fullRepoName("bar", pad),
Tag: "0.1.0",
ContextDir: "bar/docker",
Expand Down
2 changes: 1 addition & 1 deletion apps/distgo/cmd/docker/order_specs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ func OrderBuildSpecs(specsWithDeps []params.ProductBuildSpecWithDeps) ([]params.
graph[product] = make(map[string]struct{})
}
for _, curImage := range curSpec.Spec.DockerImages {
for depProduct, depTypes := range curImage.Deps.ToMap() {
for depProduct, depTypes := range dockerDepsToMap(curImage.Dependencies()) {
if !hasDockerDep(depTypes) {
// only add edge if its a docker image dependency
continue
Expand Down
22 changes: 11 additions & 11 deletions apps/distgo/cmd/docker/order_specs_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,12 @@ import (
"github.com/palantir/godel/apps/distgo/params"
)

func generateSpec(product string, deps params.DockerDeps) params.ProductBuildSpec {
func generateSpec(product string, deps []params.DockerDep) params.ProductBuildSpec {
return params.ProductBuildSpec{
ProductName: product,
Product: params.Product{
DockerImages: []params.DockerImage{
{
&params.DefaultDockerImage{
Deps: deps,
},
},
Expand All @@ -38,31 +38,31 @@ func generateSpec(product string, deps params.DockerDeps) params.ProductBuildSpe
}

func TestOrderBuildSpecs(t *testing.T) {
A := generateSpec("A", params.DockerDeps{
A := generateSpec("A", []params.DockerDep{
{Product: "B", Type: params.DockerDepDocker, TargetFile: ""},
{Product: "C", Type: params.DockerDepDocker, TargetFile: ""},
})
B := generateSpec("B", params.DockerDeps{
B := generateSpec("B", []params.DockerDep{
{Product: "D", Type: params.DockerDepDocker, TargetFile: ""},
})
C := generateSpec("C", params.DockerDeps{
C := generateSpec("C", []params.DockerDep{
{Product: "D", Type: params.DockerDepDocker, TargetFile: ""},
})
D := generateSpec("D", params.DockerDeps{})
E := generateSpec("E", params.DockerDeps{
D := generateSpec("D", []params.DockerDep{})
E := generateSpec("E", []params.DockerDep{
{Product: "DepE", Type: params.DockerDepDocker, TargetFile: ""},
})
DepE := generateSpec("DepE", params.DockerDeps{
DepE := generateSpec("DepE", []params.DockerDep{
{Product: "E", Type: params.DockerDepDocker, TargetFile: ""},
})

X := generateSpec("X", params.DockerDeps{
X := generateSpec("X", []params.DockerDep{
{Product: "Y", Type: params.DockerDepDocker, TargetFile: ""},
})
Y := generateSpec("Y", params.DockerDeps{
Y := generateSpec("Y", []params.DockerDep{
{Product: "Z", Type: params.DockerDepDocker, TargetFile: ""},
})
Z := generateSpec("Z", params.DockerDeps{})
Z := generateSpec("Z", []params.DockerDep{})

for _, testcase := range []struct {
input []params.ProductBuildSpecWithDeps
Expand Down
4 changes: 2 additions & 2 deletions apps/distgo/cmd/docker/publish.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,11 +41,11 @@ func Publish(cfg params.Project, wd string, baseRepo string, stdout io.Writer) e
for _, specWithDeps := range buildSpecsWithDeps {
versionTag := specWithDeps.Spec.ProductVersion
for _, image := range specWithDeps.Spec.DockerImages {
repo := image.Repository
repo, tag := image.Coordinates()
if baseRepo != "" {
repo = path.Join(baseRepo, repo)
}
buildTag := fmt.Sprintf("%s:%s", repo, image.Tag)
buildTag := fmt.Sprintf("%s:%s", repo, tag)
publishTag := fmt.Sprintf("%s:%s", repo, versionTag)
if err := tagImage(buildTag, publishTag); err != nil {
return err
Expand Down
Loading

0 comments on commit b43c62e

Please sign in to comment.