handle hard links and whiteouts correctly (v3)

lib/docker2aci.go

@@ -230,26 +230,48 @@ func writeSquashedImage(outputFile *os.File, renderedACI acirenderer.RenderedACI
 	outputWriter := tar.NewWriter(gw)
 	defer outputWriter.Close()
 
+	finalManifest := mergeManifests(manifests)
+
+	if err := common.WriteManifest(outputWriter, finalManifest); err != nil {
+		return err
+	}
+
+	if err := common.WriteRootfsDir(outputWriter); err != nil {
+		return err
+	}
+
+	type hardLinkEntry struct {
+		firstLinkCleanName string
+		firstLinkHeader    tar.Header
+		keepOriginal       bool
+		walked             bool
+	}
+	// map aciFileKey -> cleanTarget -> hardLinkEntry
+	hardLinks := make(map[string]map[string]hardLinkEntry)
+
+	// first pass: read all the entries and build the hardLinks map in memory
+	// but don't write on disk
 	for _, aciFile := range renderedACI {
 		rs, err := aciProvider.ReadStream(aciFile.Key)
 		if err != nil {
 			return err
 		}
 		defer rs.Close()
 
+		hardLinks[aciFile.Key] = map[string]hardLinkEntry{}
+
 		squashWalker := func(t *tarball.TarFile) error {
 			cleanName := filepath.Clean(t.Name())
-
-			if _, ok := aciFile.FileMap[cleanName]; ok {
-				// we generate and add rootfs and the squashed manifest later
-				if cleanName == "manifest" || cleanName == "rootfs" {
-					return nil
-				}
-				if err := outputWriter.WriteHeader(t.Header); err != nil {
-					return fmt.Errorf("error writing header: %v", err)
-				}
-				if _, err := io.Copy(outputWriter, t.TarStream); err != nil {
-					return fmt.Errorf("error copying file into the tar out: %v", err)
+			// the rootfs and the squashed manifest are added separately
+			if cleanName == "manifest" || cleanName == "rootfs" {
+				return nil
+			}
+			_, keep := aciFile.FileMap[cleanName]
+			if keep && t.Header.Typeflag == tar.TypeLink {
+				cleanTarget := filepath.Clean(t.Linkname())
+				if _, ok := hardLinks[aciFile.Key][cleanTarget]; !ok {
+					_, keepOriginal := aciFile.FileMap[cleanTarget]
+					hardLinks[aciFile.Key][cleanTarget] = hardLinkEntry{cleanName, *t.Header, keepOriginal, false}
 				}
 			}
 			return nil
@@ -261,14 +283,82 @@ func writeSquashedImage(outputFile *os.File, renderedACI acirenderer.RenderedACI
 		}
 	}
 
-	if err := common.WriteRootfsDir(outputWriter); err != nil {
-		return err
-	}
+	// second pass: write on disk
+	for _, aciFile := range renderedACI {
+		rs, err := aciProvider.ReadStream(aciFile.Key)
+		if err != nil {
+			return err
+		}
+		defer rs.Close()
 
-	finalManifest := mergeManifests(manifests)
+		squashWalker := func(t *tarball.TarFile) error {
+			cleanName := filepath.Clean(t.Name())
+			// the rootfs and the squashed manifest are added separately
+			if cleanName == "manifest" || cleanName == "rootfs" {
+				return nil
+			}
+			_, keep := aciFile.FileMap[cleanName]
 
-	if err := common.WriteManifest(outputWriter, finalManifest); err != nil {
-		return err
+			if link, ok := hardLinks[aciFile.Key][cleanName]; ok {
+				if keep != link.keepOriginal {
+					return fmt.Errorf("logic error: should we keep file %q?", cleanName)
+				}
+				if keep {
+					if err := outputWriter.WriteHeader(t.Header); err != nil {
+						return fmt.Errorf("error writing header: %v", err)
+					}
+					if _, err := io.Copy(outputWriter, t.TarStream); err != nil {
+						return fmt.Errorf("error copying file into the tar out: %v", err)
+					}
+				} else {
+					// The current file does not remain but there is a hard link pointing to
+					// it. Write the current file but with the filename of the first hard link
+					// pointing to it. That first hard link will not be written later, see
+					// variable "alreadyWritten".
+					link.firstLinkHeader.Size = t.Header.Size
+					link.firstLinkHeader.Typeflag = t.Header.Typeflag
+					link.firstLinkHeader.Linkname = ""
+
+					if err := outputWriter.WriteHeader(&link.firstLinkHeader); err != nil {
+						return fmt.Errorf("error writing header: %v", err)
+					}
+					if _, err := io.Copy(outputWriter, t.TarStream); err != nil {
+						return fmt.Errorf("error copying file into the tar out: %v", err)
+					}
+				}
+			} else if keep {
+				alreadyWritten := false
+				if t.Header.Typeflag == tar.TypeLink {
+					cleanTarget := filepath.Clean(t.Linkname())
+					if link, ok := hardLinks[aciFile.Key][cleanTarget]; ok {
+						if !link.keepOriginal {
+							if link.walked {
+								t.Header.Linkname = link.firstLinkCleanName
+							} else {
+								alreadyWritten = true
+							}
+						}
+						link.walked = true
+						hardLinks[aciFile.Key][cleanTarget] = link
+					}
+				}
+
+				if !alreadyWritten {
+					if err := outputWriter.WriteHeader(t.Header); err != nil {
+						return fmt.Errorf("error writing header: %v", err)
+					}
+					if _, err := io.Copy(outputWriter, t.TarStream); err != nil {
+						return fmt.Errorf("error copying file into the tar out: %v", err)
+					}
+				}
+			}
+			return nil
+		}
+
+		tr := tar.NewReader(rs)
+		if err := tarball.Walk(*tr, squashWalker); err != nil {
+			return err
+		}
 	}
 
 	return nil

tests/test-basic/Dockerfile

@@ -3,4 +3,4 @@ COPY check.sh /
 RUN echo file1 > file1 ; ln file1 file2
 RUN echo file3 > file3
 RUN echo file4 > file4
-CMD /check.sh
+CMD /check.sh 2>&1

tests/test-basic/check.sh

@@ -1,13 +1,15 @@
 #!/bin/sh
 set -e
+set -x
+
 grep -q file1 file1
 grep -q file1 file2
 grep -q file3 file3
 grep -q file4 file4
-if [ "$rendered" != "true" ] ; then
-	test -e file1
-	test -e file2
-	test $(stat -c %i file1) -eq $(stat -c %i file2)
-	test $(stat -c %i file3) -ne $(stat -c %i file4)
+if [ "$CHECK" != "rkt-rendered" ] ; then
+	# Skip this test because of:
+	# https://github.com/coreos/rkt/issues/1774
+	test $(ls -i file1 |awk '{print $1}') -eq $(ls -i file2 |awk '{print $1}')
+	test $(ls -i file3 |awk '{print $1}') -ne $(ls -i file4 |awk '{print $1}')
 fi
 echo "SUCCESS"

tests/test-whiteouts/Dockerfile

@@ -0,0 +1,21 @@
+FROM busybox
+COPY check.sh /
+
+RUN echo yes > layer0-file1 ; ln layer0-file1 layer0-file2 ; ln layer0-file1 layer0-file3
+RUN echo yes > layer1-file1 ; ln layer1-file1 layer1-file2 ; ln layer1-file1 layer1-file3
+RUN echo yes > layer2-file1 ; ln layer2-file1 layer2-file2 ; ln layer2-file1 layer2-file3
+RUN echo yes > layer3-file1 ; ln layer3-file1 layer3-file2 ; ln layer3-file1 layer3-file3
+RUN rm -f layer1-file1 layer2-file2 layer3-file3
+
+RUN echo yes > layer4-file1 ; ln layer4-file1 layer4-file2 ; ln layer4-file1 layer4-file3
+RUN echo yes > layer5-file1 ; ln layer5-file1 layer5-file2 ; ln layer5-file1 layer5-file3
+RUN echo yes > layer6-file1 ; ln layer6-file1 layer6-file2 ; ln layer6-file1 layer6-file3
+RUN rm -f layer4-file2 layer5-file1 layer6-file1 layer4-file3 layer5-file3 layer6-file2
+
+RUN echo OLD > layer10-file1 ; ln layer10-file1 layer10-file2 ; ln layer10-file1 layer10-file3
+RUN echo NEW > layer10-file1 ; ln -f layer10-file1 layer10-file2 ; ln -f layer10-file1 layer10-file3 ; echo foo > foo
+
+RUN echo line1 >  layer11-file1 ; ln layer11-file1 layer11-file2 ; ln layer11-file1 layer11-file3
+RUN echo line2 >> layer11-file1
+
+CMD /check.sh 2>&1

tests/test-whiteouts/check.sh

@@ -0,0 +1,66 @@
+#!/bin/sh
+set -e
+set -x
+
+grep -q yes layer0-file1
+grep -q yes layer0-file2
+grep -q yes layer0-file3
+
+test ! -e layer1-file1
+test   -e layer1-file2
+test   -e layer1-file3
+
+test   -e layer2-file1
+test ! -e layer2-file2
+test   -e layer2-file3
+
+test   -e layer3-file1
+test   -e layer3-file2
+test ! -e layer3-file3
+
+grep -q yes layer1-file2
+grep -q yes layer1-file3
+
+grep -q yes layer2-file1
+grep -q yes layer2-file3
+
+grep -q yes layer3-file1
+grep -q yes layer3-file2
+
+
+test   -e layer4-file1
+test ! -e layer4-file2
+test ! -e layer4-file3
+
+test ! -e layer5-file1
+test   -e layer5-file2
+test ! -e layer5-file3
+
+test ! -e layer6-file1
+test ! -e layer6-file2
+test   -e layer6-file3
+
+grep -q yes layer4-file1
+grep -q yes layer5-file2
+grep -q yes layer6-file3
+
+
+grep -q NEW layer10-file1
+grep -q NEW layer10-file2
+grep -q NEW layer10-file3
+
+grep -q line1 layer11-file1
+grep -q line1 layer11-file2
+grep -q line1 layer11-file3
+
+# # Docker with AUFS or overlay storage backend does not handle this test
+# # correctly and Semaphore uses AUFS
+if [ "$DOCKER_STORAGE_BACKEND" == devicemapper ] ; then
+	grep -q line2 layer11-file1
+	grep -q line2 layer11-file2
+	grep -q line2 layer11-file3
+	cmp layer11-file1 layer11-file2
+	cmp layer11-file1 layer11-file3
+fi
+
+echo "SUCCESS"

tests/test.sh

@@ -25,19 +25,40 @@ if ! which rkt > /dev/null ; then
 fi
 RKT=$(which rkt)
 
+DOCKER_STORAGE_BACKEND=$(sudo docker info|grep '^Storage Driver:'|sed 's/Storage Driver: //')
+
 for i in $(find . -maxdepth 1 -type d -name 'test-*') ; do
 	TESTNAME=$(basename $i)
-	echo "### Test case: ${TESTNAME}"
-	sudo docker build --tag=$PREFIX/${TESTNAME} --quiet ${TESTNAME}
-	sudo docker run $PREFIX/${TESTNAME}
+	echo "### Test case ${TESTNAME}: build..."
+	sudo docker build --tag=$PREFIX/${TESTNAME} --no-cache=true ${TESTNAME}
+
+	echo "### Test case ${TESTNAME}: test in Docker..."
+	sudo docker run --rm \
+	                --env=CHECK=docker-run \
+	                --env=DOCKER_STORAGE_BACKEND=$DOCKER_STORAGE_BACKEND \
+	                $PREFIX/${TESTNAME}
+
+	echo "### Test case ${TESTNAME}: converting to ACI..."
 	sudo docker save -o ${TESTNAME}.docker $PREFIX/${TESTNAME}
 	$DOCKER2ACI ${TESTNAME}.docker
-	sudo $RKT run --insecure-skip-verify ./${PREFIX}-${TESTNAME}-latest.aci
+
+	echo "### Test case ${TESTNAME}: test in rkt..."
+	sudo $RKT prepare --insecure-skip-verify \
+	                  --set-env=CHECK=rkt-run \
+	                  --set-env=DOCKER_STORAGE_BACKEND=$DOCKER_STORAGE_BACKEND \
+	                  ./${PREFIX}-${TESTNAME}-latest.aci \
+	                  > rkt-uuid-${TESTNAME}
+	sudo $RKT run-prepared $(cat rkt-uuid-${TESTNAME})
+	sudo $RKT status $(cat rkt-uuid-${TESTNAME}) | grep app-${TESTNAME}=0
+	sudo $RKT rm $(cat rkt-uuid-${TESTNAME})
+
+	echo "### Test case ${TESTNAME}: test with 'rkt image render'..."
 	sudo $RKT image render --overwrite ${PREFIX}/${TESTNAME} ./rendered-${TESTNAME}
-	if [ -x $TESTDIR/${TESTNAME}/check.sh ] ; then
-		pushd rendered-${TESTNAME}/rootfs
-		rendered=true $TESTDIR/${TESTNAME}/check.sh
-		popd
-	fi
+	pushd rendered-${TESTNAME}/rootfs
+	CHECK=rkt-rendered DOCKER_STORAGE_BACKEND=$DOCKER_STORAGE_BACKEND $TESTDIR/${TESTNAME}/check.sh
+	popd
+	echo "### Test case ${TESTNAME}: SUCCESS"
+
+	sudo docker rmi $PREFIX/${TESTNAME}
 done