Merge pull request containerd#591 from imeoer/converter-bootstrap-append

converter: support appending files to bootstrap layer

pkg/converter/convert_unix.go

@@ -25,6 +25,7 @@ import (
 	"github.com/containerd/containerd/archive"
 	"github.com/containerd/containerd/archive/compression"
 	"github.com/containerd/containerd/content"
+	"github.com/containerd/containerd/content/local"
 	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/images"
 	"github.com/containerd/containerd/images/converter"
@@ -624,13 +625,23 @@ func Merge(ctx context.Context, layers []Layer, dest io.Writer, opt MergeOption)
 		return nil, errors.Wrap(err, "merge bootstrap")
 	}
 
+	bootstrapRa, err := local.OpenReader(targetBootstrapPath)
+	if err != nil {
+		return nil, errors.Wrap(err, "open bootstrap reader")
+	}
+	defer bootstrapRa.Close()
+
+	files := append([]File{
+		{
+			Name:   EntryBootstrap,
+			Reader: content.NewReader(bootstrapRa),
+			Size:   bootstrapRa.Size(),
+		},
+	}, opt.AppendFiles...)
 	var rc io.ReadCloser
 
 	if opt.WithTar {
-		rc, err = packToTar(targetBootstrapPath, fmt.Sprintf("image/%s", EntryBootstrap), false)
-		if err != nil {
-			return nil, errors.Wrap(err, "pack bootstrap to tar")
-		}
+		rc = packToTar(files, false)
 	} else {
 		rc, err = os.Open(targetBootstrapPath)
 		if err != nil {

pkg/converter/types.go

@@ -125,6 +125,8 @@ type MergeOption struct {
 	Timeout *time.Duration
 	// Encrypt encrypts the bootstrap layer if it's specified.
 	Encrypt Encrypter
+	// AppendFiles specifies the files that need to be appended to the bootstrap layer.
+	AppendFiles []File
 }
 
 type UnpackOption struct {

pkg/converter/utils.go

@@ -14,14 +14,19 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"os"
 	"path/filepath"
 
 	"github.com/containerd/containerd/content"
 	"github.com/opencontainers/go-digest"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
+type File struct {
+	Name   string
+	Reader io.Reader
+	Size   int64
+}
+
 type writeCloser struct {
 	closed bool
 	io.WriteCloser
@@ -83,39 +88,27 @@ func newSeekReader(ra io.ReaderAt) *seekReader {
 	}
 }
 
-// packToTar makes .tar(.gz) stream of file named `name` and return reader.
-func packToTar(src string, name string, compress bool) (io.ReadCloser, error) {
-	fi, err := os.Stat(src)
-	if err != nil {
-		return nil, err
-	}
-
+// packToTar packs files to .tar(.gz) stream then return reader.
+func packToTar(files []File, compress bool) io.ReadCloser {
 	dirHdr := &tar.Header{
-		Name:     filepath.Dir(name),
+		Name:     "image",
 		Mode:     0755,
 		Typeflag: tar.TypeDir,
 	}
 
-	hdr := &tar.Header{
-		Name: name,
-		Mode: 0444,
-		Size: fi.Size(),
-	}
-
-	reader, writer := io.Pipe()
+	pr, pw := io.Pipe()
 
 	go func() {
 		// Prepare targz writer
 		var tw *tar.Writer
 		var gw *gzip.Writer
 		var err error
-		var file *os.File
 
 		if compress {
-			gw = gzip.NewWriter(writer)
+			gw = gzip.NewWriter(pw)
 			tw = tar.NewWriter(gw)
 		} else {
-			tw = tar.NewWriter(writer)
+			tw = tar.NewWriter(pw)
 		}
 
 		defer func() {
@@ -137,30 +130,30 @@ func packToTar(src string, name string, compress bool) (io.ReadCloser, error) {
 				finalErr = err2
 			}
 
-			writer.CloseWithError(finalErr)
+			pw.CloseWithError(finalErr)
 		}()
 
-		file, err = os.Open(src)
-		if err != nil {
-			return
-		}
-		defer file.Close()
-
 		// Write targz stream
 		if err = tw.WriteHeader(dirHdr); err != nil {
 			return
 		}
 
-		if err = tw.WriteHeader(hdr); err != nil {
-			return
-		}
-
-		if _, err = io.Copy(tw, file); err != nil {
-			return
+		for _, file := range files {
+			hdr := tar.Header{
+				Name: filepath.Join("image", file.Name),
+				Mode: 0444,
+				Size: file.Size,
+			}
+			if err = tw.WriteHeader(&hdr); err != nil {
+				return
+			}
+			if _, err = io.Copy(tw, file.Reader); err != nil {
+				return
+			}
 		}
 	}()
 
-	return reader, nil
+	return pr
 }
 
 // Copied from containerd/containerd project, copyright The containerd Authors.

tools/optimizer-server/Cargo.toml

@@ -11,6 +11,6 @@ clap = "4.1.8"
 lazy_static = "1.4.0"
 libc = "0.2.140"
 nix = "0.26.2"
-serde = { version="1.0.152", features = ["derive"] }
-serde_json = "1.0.93"
+serde = { version="1.0.198", features = ["derive"] }
+serde_json = "1.0.116"
 signal-hook = "0.3.15"

tools/optimizer-server/src/main.rs

@@ -74,12 +74,14 @@ const FAN_OPEN: u64 = 0x0000_0020;
 const FAN_OPEN_EXEC: u64 = 0x00001000;
 const AT_FDCWD: i32 = -100;
 
+#[allow(dead_code)]
 #[derive(Debug)]
 enum SetnsError {
     IO(io::Error),
     Nix(nix::Error),
 }
 
+#[allow(dead_code)]
 #[derive(Debug)]
 enum SendError {
     IO(io::Error),