diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 3f7a8aedd9..b280e38b12 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -1,6 +1,6 @@ { "ImportPath": "github.com/coreos/rkt", - "GoVersion": "go1.4.2", + "GoVersion": "go1.4.3", "Packages": [ "./..." ], @@ -72,11 +72,11 @@ }, { "ImportPath": "github.com/appc/docker2aci/lib", - "Rev": "9c1b472ce804899a5db94a98fb2ad791f4bce2b2" + "Rev": "fce83d1375e33a840e455f51a102491111984924" }, { "ImportPath": "github.com/appc/docker2aci/tarball", - "Rev": "9c1b472ce804899a5db94a98fb2ad791f4bce2b2" + "Rev": "fce83d1375e33a840e455f51a102491111984924" }, { "ImportPath": "github.com/appc/goaci/proj2aci", diff --git a/Godeps/_workspace/src/github.com/appc/docker2aci/lib/common/common.go b/Godeps/_workspace/src/github.com/appc/docker2aci/lib/common/common.go index 6b658cd15a..0fb14f620b 100644 --- a/Godeps/_workspace/src/github.com/appc/docker2aci/lib/common/common.go +++ b/Godeps/_workspace/src/github.com/appc/docker2aci/lib/common/common.go @@ -329,6 +329,7 @@ func writeACI(layer io.ReadSeeker, manifest schema.ImageManifest, curPwl []strin return nil, fmt.Errorf("error writing rootfs entry: %v", err) } + fileMap := make(map[string]struct{}) var whiteouts []string convWalker := func(t *tarball.TarFile) error { name := t.Name() @@ -337,6 +338,12 @@ func writeACI(layer io.ReadSeeker, manifest schema.ImageManifest, curPwl []strin } t.Header.Name = path.Join("rootfs", name) absolutePath := strings.TrimPrefix(t.Header.Name, "rootfs") + + if filepath.Clean(absolutePath) == "/dev" && t.Header.Typeflag != tar.TypeDir { + return fmt.Errorf(`invalid layer: "/dev" is not a directory`) + } + + fileMap[absolutePath] = struct{}{} if strings.Contains(t.Header.Name, "/.wh.") { whiteouts = append(whiteouts, strings.Replace(absolutePath, ".wh.", "", 1)) return nil @@ -373,7 +380,11 @@ func writeACI(layer io.ReadSeeker, manifest schema.ImageManifest, curPwl []strin } newPwl := subtractWhiteouts(curPwl, whiteouts) - manifest.PathWhitelist = newPwl + manifest.PathWhitelist, err = writeStdioSymlinks(trw, fileMap, newPwl) + if err != nil { + return nil, err + } + if err := WriteManifest(trw, manifest); err != nil { return nil, fmt.Errorf("error writing manifest: %v", err) } @@ -471,6 +482,41 @@ func WriteRootfsDir(tarWriter *tar.Writer) error { return tarWriter.WriteHeader(hdr) } +// writeStdioSymlinks adds the /dev/stdin, /dev/stdout, /dev/stderr, and +// /dev/fd symlinks expected by Docker to the converted ACIs so apps can find +// them as expected +func writeStdioSymlinks(tarWriter *tar.Writer, fileMap map[string]struct{}, pwl []string) ([]string, error) { + stdioSymlinks := map[string]string{ + "/dev/stdin": "/proc/self/fd/0", + // Docker makes /dev/{stdout,stderr} point to /proc/self/fd/{1,2} but + // we point to /dev/console instead in order to support the case when + // stdout/stderr is a Unix socket (e.g. for the journal). + "/dev/stdout": "/dev/console", + "/dev/stderr": "/dev/console", + "/dev/fd": "/proc/self/fd", + } + + for name, target := range stdioSymlinks { + if _, exists := fileMap[name]; exists { + continue + } + hdr := &tar.Header{ + Name: filepath.Join("rootfs", name), + Mode: 0777, + Typeflag: tar.TypeSymlink, + Linkname: target, + } + if err := tarWriter.WriteHeader(hdr); err != nil { + return nil, err + } + if !util.In(pwl, name) { + pwl = append(pwl, name) + } + } + + return pwl, nil +} + func getGenericTarHeader() *tar.Header { // FIXME(iaguis) Use docker image time instead of the Unix Epoch? hdr := &tar.Header{ diff --git a/Godeps/_workspace/src/github.com/appc/docker2aci/lib/conversion_store.go b/Godeps/_workspace/src/github.com/appc/docker2aci/lib/conversion_store.go index f71334428b..ea8774df3a 100644 --- a/Godeps/_workspace/src/github.com/appc/docker2aci/lib/conversion_store.go +++ b/Godeps/_workspace/src/github.com/appc/docker2aci/lib/conversion_store.go @@ -45,6 +45,7 @@ func (ms *ConversionStore) WriteACI(path string) (string, error) { if err != nil { return "", err } + defer cr.Close() h := sha512.New() r := io.TeeReader(cr, h) @@ -98,7 +99,7 @@ func (ms *ConversionStore) ReadStream(key string) (io.ReadCloser, error) { return nil, err } - return ioutil.NopCloser(tr), nil + return tr, nil } func (ms *ConversionStore) ResolveKey(key string) (string, error) { diff --git a/Godeps/_workspace/src/github.com/appc/docker2aci/lib/docker2aci.go b/Godeps/_workspace/src/github.com/appc/docker2aci/lib/docker2aci.go index 123474c37a..b61a55fabe 100644 --- a/Godeps/_workspace/src/github.com/appc/docker2aci/lib/docker2aci.go +++ b/Godeps/_workspace/src/github.com/appc/docker2aci/lib/docker2aci.go @@ -230,6 +230,27 @@ func writeSquashedImage(outputFile *os.File, renderedACI acirenderer.RenderedACI outputWriter := tar.NewWriter(gw) defer outputWriter.Close() + finalManifest := mergeManifests(manifests) + + if err := common.WriteManifest(outputWriter, finalManifest); err != nil { + return err + } + + if err := common.WriteRootfsDir(outputWriter); err != nil { + return err + } + + type hardLinkEntry struct { + firstLinkCleanName string + firstLinkHeader tar.Header + keepOriginal bool + walked bool + } + // map aciFileKey -> cleanTarget -> hardLinkEntry + hardLinks := make(map[string]map[string]hardLinkEntry) + + // first pass: read all the entries and build the hardLinks map in memory + // but don't write on disk for _, aciFile := range renderedACI { rs, err := aciProvider.ReadStream(aciFile.Key) if err != nil { @@ -237,19 +258,20 @@ func writeSquashedImage(outputFile *os.File, renderedACI acirenderer.RenderedACI } defer rs.Close() + hardLinks[aciFile.Key] = map[string]hardLinkEntry{} + squashWalker := func(t *tarball.TarFile) error { cleanName := filepath.Clean(t.Name()) - - if _, ok := aciFile.FileMap[cleanName]; ok { - // we generate and add the squashed manifest later - if cleanName == "manifest" { - return nil - } - if err := outputWriter.WriteHeader(t.Header); err != nil { - return fmt.Errorf("error writing header: %v", err) - } - if _, err := io.Copy(outputWriter, t.TarStream); err != nil { - return fmt.Errorf("error copying file into the tar out: %v", err) + // the rootfs and the squashed manifest are added separately + if cleanName == "manifest" || cleanName == "rootfs" { + return nil + } + _, keep := aciFile.FileMap[cleanName] + if keep && t.Header.Typeflag == tar.TypeLink { + cleanTarget := filepath.Clean(t.Linkname()) + if _, ok := hardLinks[aciFile.Key][cleanTarget]; !ok { + _, keepOriginal := aciFile.FileMap[cleanTarget] + hardLinks[aciFile.Key][cleanTarget] = hardLinkEntry{cleanName, *t.Header, keepOriginal, false} } } return nil @@ -261,14 +283,82 @@ func writeSquashedImage(outputFile *os.File, renderedACI acirenderer.RenderedACI } } - if err := common.WriteRootfsDir(outputWriter); err != nil { - return err - } + // second pass: write on disk + for _, aciFile := range renderedACI { + rs, err := aciProvider.ReadStream(aciFile.Key) + if err != nil { + return err + } + defer rs.Close() - finalManifest := mergeManifests(manifests) + squashWalker := func(t *tarball.TarFile) error { + cleanName := filepath.Clean(t.Name()) + // the rootfs and the squashed manifest are added separately + if cleanName == "manifest" || cleanName == "rootfs" { + return nil + } + _, keep := aciFile.FileMap[cleanName] - if err := common.WriteManifest(outputWriter, finalManifest); err != nil { - return err + if link, ok := hardLinks[aciFile.Key][cleanName]; ok { + if keep != link.keepOriginal { + return fmt.Errorf("logic error: should we keep file %q?", cleanName) + } + if keep { + if err := outputWriter.WriteHeader(t.Header); err != nil { + return fmt.Errorf("error writing header: %v", err) + } + if _, err := io.Copy(outputWriter, t.TarStream); err != nil { + return fmt.Errorf("error copying file into the tar out: %v", err) + } + } else { + // The current file does not remain but there is a hard link pointing to + // it. Write the current file but with the filename of the first hard link + // pointing to it. That first hard link will not be written later, see + // variable "alreadyWritten". + link.firstLinkHeader.Size = t.Header.Size + link.firstLinkHeader.Typeflag = t.Header.Typeflag + link.firstLinkHeader.Linkname = "" + + if err := outputWriter.WriteHeader(&link.firstLinkHeader); err != nil { + return fmt.Errorf("error writing header: %v", err) + } + if _, err := io.Copy(outputWriter, t.TarStream); err != nil { + return fmt.Errorf("error copying file into the tar out: %v", err) + } + } + } else if keep { + alreadyWritten := false + if t.Header.Typeflag == tar.TypeLink { + cleanTarget := filepath.Clean(t.Linkname()) + if link, ok := hardLinks[aciFile.Key][cleanTarget]; ok { + if !link.keepOriginal { + if link.walked { + t.Header.Linkname = link.firstLinkCleanName + } else { + alreadyWritten = true + } + } + link.walked = true + hardLinks[aciFile.Key][cleanTarget] = link + } + } + + if !alreadyWritten { + if err := outputWriter.WriteHeader(t.Header); err != nil { + return fmt.Errorf("error writing header: %v", err) + } + if _, err := io.Copy(outputWriter, t.TarStream); err != nil { + return fmt.Errorf("error copying file into the tar out: %v", err) + } + } + } + return nil + } + + tr := tar.NewReader(rs) + if err := tarball.Walk(*tr, squashWalker); err != nil { + return err + } } return nil diff --git a/Godeps/_workspace/src/github.com/golang/protobuf/proto/testdata/test.pb.go b/Godeps/_workspace/src/github.com/golang/protobuf/proto/testdata/test.pb.go index c22fe31239..7278537b44 100644 --- a/Godeps/_workspace/src/github.com/golang/protobuf/proto/testdata/test.pb.go +++ b/Godeps/_workspace/src/github.com/golang/protobuf/proto/testdata/test.pb.go @@ -39,7 +39,7 @@ It has these top-level messages: */ package testdata -import proto "github.com/golang/protobuf/proto" +import proto "github.com/coreos/rkt/Godeps/_workspace/src/github.com/golang/protobuf/proto" import fmt "fmt" import math "math"