diff --git a/cmd/cmd.go b/cmd/cmd.go index 7e17ba37d..35b3a701b 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -34,6 +34,10 @@ func FlagLaunchDirSrc(dir *string) { flag.StringVar(dir, "launch-src", DefaultLaunchDir, "path to source launch directory for export step") } +func FlagDryRunDir(dir *string) { + flag.StringVar(dir, "dryrun", "", "path to store first stage output in. (Don't perform export)") +} + func FlagAppDir(dir *string) { flag.StringVar(dir, "app", DefaultAppDir, "path to app directory") } diff --git a/cmd/exporter/main.go b/cmd/exporter/main.go index a0283a1ff..1e9f449df 100644 --- a/cmd/exporter/main.go +++ b/cmd/exporter/main.go @@ -7,7 +7,6 @@ import ( "os" "github.com/BurntSushi/toml" - "github.com/buildpack/lifecycle" "github.com/buildpack/lifecycle/cmd" "github.com/buildpack/lifecycle/img" @@ -18,6 +17,7 @@ var ( runImage string launchDir string launchDirSrc string + dryrun string appDir string appDirSrc string groupPath string @@ -33,6 +33,7 @@ func init() { cmd.FlagLaunchDirSrc(&launchDirSrc) cmd.FlagAppDir(&appDir) cmd.FlagAppDirSrc(&appDirSrc) + cmd.FlagDryRunDir(&dryrun) cmd.FlagGroupPath(&groupPath) cmd.FlagUseDaemon(&useDaemon) cmd.FlagUseCredHelpers(&useHelpers) @@ -43,7 +44,7 @@ func init() { func main() { flag.Parse() if flag.NArg() > 1 || flag.Arg(0) == "" || runImage == "" { - args := map[string]interface{}{"narg": flag.NArg, "runImage": runImage, "launchDir": launchDir} + args := map[string]interface{}{"narg": flag.NArg(), "runImage": runImage, "launchDir": launchDir} cmd.Exit(cmd.FailCode(cmd.CodeInvalidArgs, "parse arguments", fmt.Sprintf("%+v", args))) } repoName = flag.Arg(0) @@ -51,6 +52,47 @@ func main() { } func export() error { + var group lifecycle.BuildpackGroup + var err error + if _, err := toml.DecodeFile(groupPath, &group); err != nil { + return cmd.FailErr(err, "read group") + } + + exporter := &lifecycle.Exporter{ + Buildpacks: group.Buildpacks, + Out: os.Stdout, + Err: os.Stderr, + UID: uid, + GID: gid, + } + + if dryrun != "" { + exporter.TmpDir = dryrun + if err := os.MkdirAll(exporter.TmpDir, 0777); err != nil { + return cmd.FailErr(err, "create temp directory") + } + } else { + exporter.TmpDir, err = ioutil.TempDir("", "lifecycle.exporter.layer") + if err != nil { + return cmd.FailErr(err, "create temp directory") + } + defer os.RemoveAll(exporter.TmpDir) + } + + _, err = exporter.PrepareExport( + launchDirSrc, + launchDir, + appDirSrc, + appDir, + ) + if err != nil { + return cmd.FailErrCode(err, cmd.CodeFailedBuild) + } + + if dryrun != "" { + return nil + } + if useHelpers { if err := img.SetupCredHelpers(repoName, runImage); err != nil { return cmd.FailErr(err, "setup credential helpers") @@ -88,30 +130,9 @@ func export() error { origImage = nil } - var group lifecycle.BuildpackGroup - if _, err := toml.DecodeFile(groupPath, &group); err != nil { - return cmd.FailErr(err, "read group") - } - - tmpDir, err := ioutil.TempDir("", "lifecycle.exporter.layer") - if err != nil { - return cmd.FailErr(err, "create temp directory") - } - defer os.RemoveAll(tmpDir) - - exporter := &lifecycle.Exporter{ - Buildpacks: group.Buildpacks, - TmpDir: tmpDir, - Out: os.Stdout, - Err: os.Stderr, - UID: uid, - GID: gid, - } - newImage, err := exporter.Export( + newImage, err := exporter.ExportImage( launchDirSrc, launchDir, - appDirSrc, - appDir, stackImage, origImage, ) diff --git a/exporter.go b/exporter.go index 7c8a5f792..b3266ebd7 100644 --- a/exporter.go +++ b/exporter.go @@ -1,21 +1,23 @@ package lifecycle import ( - "archive/tar" - "compress/gzip" + "crypto/sha256" + "encoding/hex" "encoding/json" "fmt" "io" + "io/ioutil" "os" "path/filepath" "strings" "github.com/BurntSushi/toml" + "github.com/buildpack/lifecycle/img" + "github.com/docker/docker/pkg/archive" + "github.com/docker/docker/pkg/idtools" "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/mutate" "github.com/pkg/errors" - - "github.com/buildpack/lifecycle/img" ) type Exporter struct { @@ -27,72 +29,139 @@ type Exporter struct { } func (e *Exporter) Export(launchDirSrc, launchDirDst, appDirSrc, appDirDst string, runImage, origImage v1.Image) (v1.Image, error) { - metadata := AppImageMetadata{} + dir, err := e.PrepareExport(launchDirSrc, launchDirDst, appDirSrc, appDirDst) + if err != nil { + return nil, errors.Wrapf(err, "prepare export") + } + return e.ExportImage(dir, launchDirDst, runImage, origImage) +} + +func (e *Exporter) PrepareExport(launchDirSrc, launchDirDst, appDirSrc, appDirDst string) (string, error) { + var err error + var metadata AppImageMetadata + + metadata.App.SHA, err = e.exportTar(appDirSrc, appDirDst, "") + if err != nil { + return "", errors.Wrap(err, "exporting app layer tar") + } + metadata.Config.SHA, err = e.exportTar(launchDirSrc, launchDirDst, "config") + if err != nil { + return "", errors.Wrap(err, "exporting config layer tar") + } + + for _, buildpack := range e.Buildpacks { + bpMetadata := BuildpackMetadata{ID: buildpack.ID, Version: buildpack.Version, Layers: make(map[string]LayerMetadata)} + tomls, err := filepath.Glob(filepath.Join(launchDirSrc, buildpack.ID, "*.toml")) + if err != nil { + return "", errors.Wrapf(err, "finding layer tomls") + } + for _, tomlFile := range tomls { + var bpLayer LayerMetadata + if filepath.Base(tomlFile) == "launch.toml" { + continue + } + dir := strings.TrimSuffix(tomlFile, ".toml") + layerName := filepath.Base(dir) + _, err := os.Stat(dir) + if !os.IsNotExist(err) { + bpLayer.SHA, err = e.exportTar(launchDirSrc, launchDirDst, filepath.Join(buildpack.ID, layerName)) + if err != nil { + return "", errors.Wrapf(err, "exporting tar for layer '%s/%s'", buildpack.ID, layerName) + } + } + var metadata map[string]interface{} + if _, err := toml.DecodeFile(tomlFile, &metadata); err != nil { + return "", errors.Wrapf(err, "read metadata for layer %s/%s", buildpack.ID, layerName) + } + bpLayer.Data = metadata + bpMetadata.Layers[layerName] = bpLayer + } + metadata.Buildpacks = append(metadata.Buildpacks, bpMetadata) + } + + data, err := json.Marshal(metadata) + if err != nil { + return "", errors.Wrap(err, "marshal metadata") + } + err = ioutil.WriteFile(filepath.Join(e.TmpDir, "metadata.json"), data, 0600) + if err != nil { + return "", errors.Wrap(err, "write metadata") + } + + return e.TmpDir, nil +} + +func rawSHA(prefixedSHA string) string { + return strings.TrimPrefix(prefixedSHA, "sha256:") +} + +func (e *Exporter) ExportImage(exportDir, launchDirDst string, runImage, origImage v1.Image) (v1.Image, error) { + data, err := ioutil.ReadFile(filepath.Join(exportDir, "metadata.json")) + if err != nil { + return nil, err + } + var metadata AppImageMetadata + if err := json.Unmarshal(data, &metadata); err != nil { + return nil, err + } if err := addRunImageMetadata(runImage, &metadata); err != nil { return nil, err } - repoImage, appLayerDigest, err := e.addDirAsLayer( - runImage, - filepath.Join(e.TmpDir, "app.tgz"), - appDirSrc, - appDirDst, - ) + repoImage, _, err := img.Append(runImage, filepath.Join(exportDir, fmt.Sprintf("%s.tar", rawSHA(metadata.App.SHA)))) if err != nil { - return nil, errors.Wrap(err, "append app layer to run image") + return nil, errors.Wrap(err, "append app layer") } - metadata.App.SHA = appLayerDigest - - repoImage, configLayerDigest, err := e.addDirAsLayer( - repoImage, - filepath.Join(e.TmpDir, "config.tgz"), - filepath.Join(launchDirSrc, "config"), - filepath.Join(launchDirDst, "config"), - ) + repoImage, _, err = img.Append(repoImage, filepath.Join(exportDir, fmt.Sprintf("%s.tar", rawSHA(metadata.Config.SHA)))) if err != nil { - return nil, errors.Wrap(err, "append config layer to run image") + return nil, errors.Wrap(err, "append config layer") } - metadata.Config.SHA = configLayerDigest - var bpMetadata []BuildpackMetadata + var origMetadata *AppImageMetadata if origImage != nil { - data, err := e.GetMetadata(origImage) + origMetadata, err = e.GetMetadata(origImage) if err != nil { return nil, errors.Wrap(err, "find metadata") } - bpMetadata = data.Buildpacks } - for _, buildpack := range e.Buildpacks { - var origLayers map[string]LayerMetadata - for _, md := range bpMetadata { - if md.ID == buildpack.ID { - origLayers = md.Layers + + for _, bpMetadata := range metadata.Buildpacks { + for layerName, data := range bpMetadata.Layers { + tar := filepath.Join(exportDir, fmt.Sprintf("%s.tar", rawSHA(data.SHA))) + _, err := os.Stat(tar) + if os.IsNotExist(err) { + data.SHA, err = origLayerDiffID(origMetadata, bpMetadata.ID, layerName) + if err != nil { + return nil, err + } + hash, err := v1.NewHash(data.SHA) + topLayer, err := origImage.LayerByDiffID(hash) + if err != nil { + return nil, errors.Wrapf(err, "find previous layer %s/%s", bpMetadata.ID, layerName) + } + repoImage, err = mutate.AppendLayers(repoImage, topLayer) + if err != nil { + return nil, errors.Wrapf(err, "append layer %s/%s from previous image", bpMetadata.ID, layerName) + } + bpMetadata.Layers[layerName] = data + } else { + repoImage, _, err = img.Append(repoImage, tar) + if err != nil { + return nil, errors.Wrapf(err, "append new layer %s/%s", bpMetadata.ID, layerName) + } } } - bpMetadata := BuildpackMetadata{ID: buildpack.ID, Version: buildpack.Version} - repoImage, bpMetadata.Layers, err = e.addBuildpackLayer( - buildpack.ID, - launchDirSrc, - launchDirDst, - repoImage, - origImage, - origLayers, - ) - if err != nil { - return nil, errors.Wrap(err, "append layers") - } - metadata.Buildpacks = append(metadata.Buildpacks, bpMetadata) } - - buildJSON, err := json.Marshal(metadata) + metadataJSON, err := json.Marshal(metadata) if err != nil { return nil, errors.Wrap(err, "get encoded metadata") } - repoImage, err = img.Label(repoImage, MetadataLabel, string(buildJSON)) + repoImage, err = img.Label(repoImage, MetadataLabel, string(metadataJSON)) if err != nil { return nil, errors.Wrap(err, "set metadata label") } + repoImage, err = img.Env(repoImage, EnvLaunchDir, launchDirDst) if err != nil { return nil, errors.Wrap(err, "set launch dir env var") @@ -100,6 +169,23 @@ func (e *Exporter) Export(launchDirSrc, launchDirDst, appDirSrc, appDirDst strin return repoImage, nil } + +func origLayerDiffID(metadata *AppImageMetadata, buildpackID, layerName string) (string, error) { + if metadata == nil { + return "", fmt.Errorf("cannot reuse layer, missing previous image metadata") + } + for _, buildpack := range metadata.Buildpacks { + if buildpack.ID == buildpackID { + data, ok := buildpack.Layers[layerName] + if !ok { + return "", fmt.Errorf("previous image has no layer '%s/%s'", buildpackID, layerName) + } + return data.SHA, nil + } + } + return "", fmt.Errorf("cannot reuse layer '%s/%s', previous image has no layers for buildpack '%s'", buildpackID, layerName, buildpackID) +} + func addRunImageMetadata(runImage v1.Image, metadata *AppImageMetadata) error { runLayerDiffID, err := img.TopLayerDiffID(runImage) if err != nil { @@ -116,64 +202,8 @@ func addRunImageMetadata(runImage v1.Image, metadata *AppImageMetadata) error { return nil } -func (e *Exporter) addBuildpackLayer( - id, launchDir, launchDirDst string, - repoImage, origImage v1.Image, - origLayers map[string]LayerMetadata, -) (v1.Image, map[string]LayerMetadata, error) { - - metadata := map[string]LayerMetadata{} - layers, err := filepath.Glob(filepath.Join(launchDir, id, "*.toml")) - if err != nil { - return nil, nil, err - } - for _, layer := range layers { - if filepath.Base(layer) == "launch.toml" { - continue - } - var layerDiffID string - dir := strings.TrimSuffix(layer, ".toml") - layerName := filepath.Base(dir) - dirInfo, err := os.Stat(dir) - if os.IsNotExist(err) { - if origImage == nil || origLayers == nil || origLayers[layerName].SHA == "" { - return nil, nil, errors.Errorf("layer TOML found, but no available contents for %s %s", id, layerName) - } - layerDiffID = origLayers[layerName].SHA - hash, err := v1.NewHash(layerDiffID) - if err != nil { - return nil, nil, errors.Wrapf(err, "parse hash: %s", origLayers[layerName].SHA) - } - topLayer, err := origImage.LayerByDiffID(hash) - if err != nil { - return nil, nil, errors.Wrapf(err, "find previous layer %s/%s", id, layerName) - } - repoImage, err = mutate.AppendLayers(repoImage, topLayer) - if err != nil { - return nil, nil, errors.Wrapf(err, "append layer %s/%s from previous image", id, layerName) - } - } else if err != nil { - return nil, nil, err - } else if !dirInfo.IsDir() { - return nil, nil, errors.Errorf("expected %s to be a directory", dir) - } else { - tarFile := filepath.Join(e.TmpDir, fmt.Sprintf("layer.%s.%s.tgz", id, layerName)) - repoImage, layerDiffID, err = e.addDirAsLayer(repoImage, tarFile, dir, filepath.Join(launchDirDst, id, layerName)) - if err != nil { - return nil, nil, errors.Wrap(err, "append dir as layer") - } - } - var tomlData map[string]interface{} - if _, err := toml.DecodeFile(layer, &tomlData); err != nil { - return nil, nil, errors.Wrap(err, "read layer TOML data") - } - metadata[layerName] = LayerMetadata{SHA: layerDiffID, Data: tomlData} - } - return repoImage, metadata, nil -} - -func (e *Exporter) GetMetadata(image v1.Image) (AppImageMetadata, error) { - var metadata AppImageMetadata +func (e *Exporter) GetMetadata(image v1.Image) (*AppImageMetadata, error) { + var metadata *AppImageMetadata cfg, err := image.ConfigFile() if err != nil { return metadata, err @@ -185,77 +215,50 @@ func (e *Exporter) GetMetadata(image v1.Image) (AppImageMetadata, error) { return metadata, nil } -func (e *Exporter) addDirAsLayer(image v1.Image, tarFile, fsDir, tarDir string) (v1.Image, string, error) { - if err := e.createTarFile(tarFile, fsDir, tarDir); err != nil { - return nil, "", errors.Wrapf(err, "tar %s to %s", fsDir, tarFile) - } - newImage, topLayer, err := img.Append(image, tarFile) - if err != nil { - return nil, "", errors.Wrap(err, "append layers to run image") - } - diffID, err := topLayer.DiffID() +func (e *Exporter) writeWithSHA(r io.Reader) (string, error) { + hasher := sha256.New() + + f, err := ioutil.TempFile(e.TmpDir, "tarfile") if err != nil { - return nil, "", errors.Wrap(err, "calculate layer diff ID") + return "", err } - return newImage, diffID.String(), nil -} + defer f.Close() -func (e *Exporter) createTarFile(tarFile, fsDir, tarDir string) error { - fh, err := os.Create(tarFile) - if err != nil { - return errors.Wrap(err, "create file for tar") + w := io.MultiWriter(hasher, f) + + if _, err := io.Copy(w, r); err != nil { + return "", err } - defer fh.Close() - gzw := gzip.NewWriter(fh) - defer gzw.Close() - tw := tar.NewWriter(gzw) - defer tw.Close() - return filepath.Walk(fsDir, func(file string, fi os.FileInfo, err error) error { - if err != nil { - return err - } - relPath, err := filepath.Rel(fsDir, file) - if err != nil { - return err - } + sha := hex.EncodeToString(hasher.Sum(make([]byte, 0, hasher.Size()))) - var header *tar.Header - if fi.Mode()&os.ModeSymlink != 0 { - target, err := os.Readlink(file) - if err != nil { - return err - } - header, err = tar.FileInfoHeader(fi, target) - if err != nil { - return err - } - } else { - header, err = tar.FileInfoHeader(fi, fi.Name()) - if err != nil { - return err - } - } - header.Name = filepath.Join(tarDir, relPath) + if err := f.Close(); err != nil { + return "", err + } + if err := os.Rename(f.Name(), filepath.Join(e.TmpDir, sha+".tar")); err != nil { + return "", err + } - if e.UID > 0 && e.GID > 0 { - header.Uid = e.UID - header.Gid = e.GID - } + return "sha256:" + sha, nil +} - if err := tw.WriteHeader(header); err != nil { - return err - } - if fi.Mode().IsRegular() { - f, err := os.Open(file) - if err != nil { - return err - } - defer f.Close() - if _, err := io.Copy(tw, f); err != nil { - return err - } +func (e *Exporter) exportTar(launchDirSrc, launchDirDst, name string) (string, error) { + tarOptions := &archive.TarOptions{ + IncludeFiles: []string{name}, + RebaseNames: map[string]string{ + name: launchDirDst + "/" + name, + }, + } + if e.UID > 0 && e.GID > 0 { + tarOptions.ChownOpts = &idtools.Identity{ + UID: e.UID, + GID: e.GID, } - return nil - }) + } + rc, err := archive.TarWithOptions(filepath.Join(launchDirSrc), tarOptions) + if err != nil { + return "", err + } + defer rc.Close() + return e.writeWithSHA(rc) } diff --git a/exporter_test.go b/exporter_test.go index 2e2f73bf4..e2b94925a 100644 --- a/exporter_test.go +++ b/exporter_test.go @@ -10,19 +10,19 @@ import ( "math/rand" "os" "os/user" + "path/filepath" "strconv" "strings" "testing" "time" + "github.com/buildpack/lifecycle" + "github.com/buildpack/lifecycle/img" "github.com/google/go-cmp/cmp" "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/mutate" "github.com/sclevine/spec" "github.com/sclevine/spec/report" - - "github.com/buildpack/lifecycle" - "github.com/buildpack/lifecycle/img" ) func TestExporter(t *testing.T) { @@ -60,6 +60,45 @@ func testExporter(t *testing.T, when spec.G, it spec.S) { } }) + when("#PrepareExport", func() { + it("creates fs representation of image", func() { + dir, err := exporter.PrepareExport("testdata/exporter/first/launch", "/launch/dest", "testdata/exporter/first/launch/app", "/app/dest") + assertNil(t, err) + var metadata lifecycle.AppImageMetadata + b, err := ioutil.ReadFile(filepath.Join(dir, "metadata.json")) + assertNil(t, err) + assertNil(t, json.Unmarshal(b, &metadata)) + + t.Log("generates app tar") + assertTarFileContents(t, + filepath.Join(dir, strings.Replace(metadata.App.SHA, "sha256:", "", -1)+".tar"), + "/app/dest/.hidden.txt", "some-hidden-text\n") + + t.Log("generate config tar") + assertTarFileContents(t, + filepath.Join(dir, strings.Replace(metadata.Config.SHA, "sha256:", "", -1)+".tar"), + "/launch/dest/config/metadata.toml", "[[processes]]\n type = \"web\"\n command = \"npm start\"\n") + + t.Log("generates buildpacks layers") + assertEq(t, len(metadata.Buildpacks), 1) + assertTarFileContents(t, + filepath.Join(dir, strings.Replace(metadata.Buildpacks[0].Layers["layer1"].SHA, "sha256:", "", -1)+".tar"), + "/launch/dest/buildpack.id/layer1/file-from-layer-1", "echo text from layer 1\n") + assertEq(t, + metadata.Buildpacks[0].Layers["layer1"].Data, + map[string]interface{}{ + "mykey": "myval", + }) + + assertTarFileContents(t, + filepath.Join(dir, strings.Replace(metadata.Buildpacks[0].Layers["layer2"].SHA, "sha256:", "", -1)+".tar"), + "/launch/dest/buildpack.id/layer2/file-from-layer-2", "echo text from layer 2\n") + assertEq(t, + metadata.Buildpacks[0].Layers["layer2"].Data, + map[string]interface{}{}) + }) + }) + when("#Export", func() { var runImage v1.Image @@ -73,7 +112,7 @@ func testExporter(t *testing.T, when spec.G, it spec.S) { it("should process a simple launch directory", func() { image, err := exporter.Export("testdata/exporter/first/launch", "/launch/dest", - "testdata/exporter/first/launch/app", "/launch/dest/app", runImage, nil) + "testdata/exporter/first/launch/app", "/app/dest", runImage, nil) if err != nil { t.Fatalf("Error: %s\n", err) } @@ -111,10 +150,10 @@ func testExporter(t *testing.T, when spec.G, it spec.S) { } t.Log("adds app layer to image") - if txt, err := getImageFile(image, data.App.SHA, "/launch/dest/app/subdir/myfile.txt"); err != nil { + if txt, err := getImageFile(image, data.App.SHA, "/app/dest/subdir/myfile.txt"); err != nil { t.Fatalf("Error: %s\n", err) } else if diff := cmp.Diff(strings.TrimSpace(txt), "mycontents"); diff != "" { - t.Fatalf(`/launch/dest/app/subdir/myfile.txt: (-got +want)\n%s`, diff) + t.Fatalf(`/app/dest/subdir/myfile.txt: (-got +want)\n%s`, diff) } t.Log("adds config layer to image") @@ -150,12 +189,12 @@ func testExporter(t *testing.T, when spec.G, it spec.S) { if err != nil { t.Fatalf("Error: %s\n", err) } - if uid, gid, err := getImageFileOwner(image, data.App.SHA, "/launch/dest/app/subdir/myfile.txt"); err != nil { + if uid, gid, err := getImageFileOwner(image, data.App.SHA, "/app/dest/subdir/myfile.txt"); err != nil { t.Fatalf("Error: %s\n", err) } else if diff := cmp.Diff(strconv.Itoa(uid), currentUser.Uid); diff != "" { - t.Fatalf(`/launch/dest/app/subdir/myfile.txt: (-got +want)\n%s`, diff) + t.Fatalf(`/app/dest/subdir/myfile.txt: (-got +want)\n%s`, diff) } else if diff := cmp.Diff(strconv.Itoa(gid), currentUser.Gid); diff != "" { - t.Fatalf(`/launch/dest/app/subdir/myfile.txt: (-got +want)\n%s`, diff) + t.Fatalf(`/app/dest/subdir/myfile.txt: (-got +want)\n%s`, diff) } }) @@ -166,7 +205,7 @@ func testExporter(t *testing.T, when spec.G, it spec.S) { }) it("sets uid/gid on the layer files", func() { image, err := exporter.Export("testdata/exporter/first/launch", "/launch/dest", - "testdata/exporter/first/launch/app", "/launch/dest/app", runImage, nil) + "testdata/exporter/first/launch/app", "/app/dest", runImage, nil) if err != nil { t.Fatalf("Error: %s\n", err) } @@ -176,21 +215,21 @@ func testExporter(t *testing.T, when spec.G, it spec.S) { } // File - if uid, gid, err := getImageFileOwner(image, data.App.SHA, "/launch/dest/app/subdir/myfile.txt"); err != nil { + if uid, gid, err := getImageFileOwner(image, data.App.SHA, "/app/dest/subdir/myfile.txt"); err != nil { t.Fatalf("Error: %s\n", err) } else if diff := cmp.Diff(uid, 1234); diff != "" { - t.Fatalf(`/launch/dest/app/subdir/myfile.txt: (-got +want)\n%s`, diff) + t.Fatalf(`/app/dest/subdir/myfile.txt: (-got +want)\n%s`, diff) } else if diff := cmp.Diff(gid, 5678); diff != "" { - t.Fatalf(`/launch/dest/app/subdir/myfile.txt: (-got +want)\n%s`, diff) + t.Fatalf(`/app/dest/subdir/myfile.txt: (-got +want)\n%s`, diff) } // Directory - if uid, gid, err := getImageFileOwner(image, data.App.SHA, "/launch/dest/app/subdir"); err != nil { + if uid, gid, err := getImageFileOwner(image, data.App.SHA, "/app/dest/subdir/"); err != nil { t.Fatalf("Error: %s\n", err) } else if diff := cmp.Diff(uid, 1234); diff != "" { - t.Fatalf(`/launch/dest/app/subdir: (-got +want)\n%s`, diff) + t.Fatalf(`/app/dest/subdir/: (-got +want)\n%s`, diff) } else if diff := cmp.Diff(gid, 5678); diff != "" { - t.Fatalf(`/launch/dest/app/subdir: (-got +want)\n%s`, diff) + t.Fatalf(`/app/dest/subdir/: (-got +want)\n%s`, diff) } }) }) @@ -200,7 +239,7 @@ func testExporter(t *testing.T, when spec.G, it spec.S) { it.Before(func() { var err error firstImage, err = exporter.Export("testdata/exporter/first/launch", "/launch/dest", - "testdata/exporter/first/launch/app", "/launch/dest/app", runImage, nil) + "testdata/exporter/first/launch/app", "/app/dest", runImage, nil) if err != nil { t.Fatalf("Error: %s\n", err) } @@ -208,7 +247,7 @@ func testExporter(t *testing.T, when spec.G, it spec.S) { it("should reuse layers if there is a layer TOML file", func() { image, err := exporter.Export("testdata/exporter/second/launch", "/launch/dest", - "testdata/exporter/first/launch/app", "/launch/dest/app", runImage, firstImage) + "testdata/exporter/first/launch/app", "/app/dest", runImage, firstImage) if err != nil { t.Fatalf("Error: %s\n", err) } @@ -380,3 +419,26 @@ func envVar(image v1.Image, key string) (string, error) { } return "", fmt.Errorf("image ENV did not contain variable '%s'", key) } + +func assertTarFileContents(t *testing.T, tarfile, path, expected string) { + r, err := os.Open(tarfile) + assertNil(t, err) + defer r.Close() + + tr := tar.NewReader(r) + for { + header, err := tr.Next() + if err == io.EOF { + break + } + assertNil(t, err) + + if header.Name == path { + buf, err := ioutil.ReadAll(tr) + assertNil(t, err) + assertEq(t, string(buf), expected) + return + } + } + t.Fatalf("%s does not exist in %s", path, tarfile) +} diff --git a/go.mod b/go.mod index 67f9f45b3..739ab15ba 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,9 @@ module github.com/buildpack/lifecycle require ( github.com/BurntSushi/toml v0.3.0 github.com/Microsoft/go-winio v0.4.11 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/containerd/continuity v0.0.0-20181027224239-bea7585dbfac // indirect github.com/docker/distribution v0.0.0-20180327202408-83389a148052 // indirect - github.com/docker/docker v0.7.3-0.20180531152204-71cd53e4a197 // indirect + github.com/docker/docker v0.7.3-0.20181027010111-b8e87cfdad8d github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.3.3 // indirect github.com/gogo/protobuf v1.1.1 // indirect @@ -15,10 +15,10 @@ require ( github.com/gotestyourself/gotestyourself v2.1.0+incompatible // indirect github.com/opencontainers/go-digest v1.0.0-rc1 // indirect github.com/opencontainers/image-spec v1.0.1 // indirect + github.com/opencontainers/runc v0.1.1 // indirect github.com/pkg/errors v0.8.0 - github.com/pmezard/go-difflib v1.0.0 // indirect github.com/sclevine/spec v1.0.0 - github.com/stretchr/testify v1.2.2 // indirect + github.com/sirupsen/logrus v1.1.1 // indirect golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3 // indirect golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e // indirect gotest.tools v2.1.0+incompatible // indirect diff --git a/go.sum b/go.sum index 20172a567..41146100b 100644 --- a/go.sum +++ b/go.sum @@ -3,12 +3,17 @@ github.com/BurntSushi/toml v0.3.0/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/Microsoft/go-winio v0.4.9 h1:3RbgqgGVqmcpbOiwrjbVtDHLlJBGF6aE+yHmNtBNsFQ= github.com/Microsoft/go-winio v0.4.9/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= +github.com/containerd/continuity v0.0.0-20181027224239-bea7585dbfac h1:PThQaO4yCvJzJBUW1XoFQxLotWRhvX2fgljJX8yrhFI= +github.com/containerd/continuity v0.0.0-20181027224239-bea7585dbfac/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/docker/distribution v0.0.0-20180327202408-83389a148052 h1:bYklS+YB8BZreSEY+/WqaH+S8upfuYf0Hq/EmNOQMIA= github.com/docker/distribution v0.0.0-20180327202408-83389a148052/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v0.7.3-0.20180531152204-71cd53e4a197 h1:MuoHurKAPwdDBHScO79akveR0nbIUlWWebjWIqK+DqM= github.com/docker/docker v0.7.3-0.20180531152204-71cd53e4a197/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v0.7.3-0.20181027010111-b8e87cfdad8d h1:/4OivNB4IJIue2ZUKccE2zgp/2C8xPBShMGETIlauCA= +github.com/docker/docker v0.7.3-0.20181027010111-b8e87cfdad8d/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v1.13.1 h1:5VBhsO6ckUxB0A8CE5LlUJdXzik9cbEbBTQ/ggeml7M= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.3.3 h1:Xk8S3Xj5sLGlG5g67hJmYMmUgXv5N4PhkjJHHqrwnTk= @@ -25,10 +30,13 @@ github.com/google/go-containerregistry v0.0.0-20180912122137-74aef1a35cfa h1:y5n github.com/google/go-containerregistry v0.0.0-20180912122137-74aef1a35cfa/go.mod h1:yZAFP63pRshzrEYLXLGPmUt0Ay+2zdjmMN1loCnRLUk= github.com/gotestyourself/gotestyourself v2.1.0+incompatible h1:JdX/5sh/7yF7jRW5Xpvh1wlkAlgZS+X3HVCMlYqlxmw= github.com/gotestyourself/gotestyourself v2.1.0+incompatible/go.mod h1:zZKM6oeNM8k+FRljX1mnzVYeS8wiGgQyvST1/GafPbY= +github.com/konsorten/go-windows-terminal-sequences v0.0.0-20180402223658-b729f2633dfe/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/runc v0.1.1 h1:GlxAyO6x8rfZYN9Tt0Kti5a/cP41iuiO2yYT0IJGY8Y= +github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -37,14 +45,20 @@ github.com/sclevine/spec v0.0.0-20180404042546-a925ac4bfbc9 h1:xh7f/httJstXLcqjP github.com/sclevine/spec v0.0.0-20180404042546-a925ac4bfbc9/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= github.com/sclevine/spec v1.0.0 h1:ILQ08A/CHCz8GGqivOvI54Hy1U40wwcpkf7WtB1MQfY= github.com/sclevine/spec v1.0.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= +github.com/sirupsen/logrus v1.1.1 h1:VzGj7lhU7KEB9e9gMpAV/v5XT2NVSvLJhJLCWbnkgXg= +github.com/sirupsen/logrus v1.1.1/go.mod h1:zrgwTnHtNr00buQ1vSptGe8m1f/BbgsPukg8qsT7A+A= github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 h1:u+LnwYTOOW7Ukr/fppxEb1Nwz0AtPflrblfvUudpo+I= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/net v0.0.0-20180611182652-db08ff08e862 h1:JZi6BqOZ+iSgmLWe6llhGrNnEnK+YB/MRkStwnEfbqM= golang.org/x/net v0.0.0-20180611182652-db08ff08e862/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3 h1:czFLhve3vsQetD6JOJ8NZZvGQIXlnN3/yXxbT6/awxI= golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/sys v0.0.0-20180724212812-e072cadbbdc8 h1:7T3bTJEttnfJdEY+NY/VYT7IXRaul8potWiyw/n7LB8= golang.org/x/sys v0.0.0-20180724212812-e072cadbbdc8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e h1:o3PsSEY8E4eXWkXrIP9YJALUkVZqzHJT5DOasTyn8Vs= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= gotest.tools v2.1.0+incompatible h1:5USw7CrJBYKqjg9R7QlA6jzqZKEAtvW82aNmsxxGPxw= gotest.tools v2.1.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= diff --git a/testdata/exporter/first/launch/app/.hidden.txt b/testdata/exporter/first/launch/app/.hidden.txt new file mode 100644 index 000000000..fed58d0c4 --- /dev/null +++ b/testdata/exporter/first/launch/app/.hidden.txt @@ -0,0 +1 @@ +some-hidden-text