converter: support FromDir and AttributesPath options

Signed-off-by: Yan Song <yansong.ys@antgroup.com>

pkg/converter/convert_unix.go

@@ -314,7 +314,7 @@ func seekFile(ra content.ReaderAt, targetName string, handle func(io.Reader, *ta
 // provided by the caller.
 //
 // Important: the caller must check `io.WriteCloser.Close() == nil` to ensure
-// the conversion workflow is finished.
+// the conversion workflow is finished if `io.WriteCloser == nil`.
 func Pack(ctx context.Context, dest io.Writer, opt PackOption) (io.WriteCloser, error) {
 	if opt.FsVersion == "" {
 		opt.FsVersion = "6"
@@ -347,14 +347,93 @@ func Pack(ctx context.Context, dest io.Writer, opt PackOption) (io.WriteCloser,
 		return nil, fmt.Errorf("'--batch-size' can only be supported by fs version 6")
 	}
 
+	if opt.FromDir != "" {
+		return nil, packFromDirectory(ctx, dest, opt, builderPath, opt.FromDir)
+	}
+
 	if opt.features.Contains(tool.FeatureTar2Rafs) {
 		return packFromTar(ctx, dest, opt)
 	}
 
-	return packFromDirectory(ctx, dest, opt, builderPath)
+	return packFromUnpackedTar(ctx, dest, opt, builderPath)
+}
+
+func packFromDirectory(ctx context.Context, dest io.Writer, opt PackOption, builderPath, sourceDir string) error {
+	if opt.ExternalBlobWriter == nil {
+		return fmt.Errorf("the 'ExternalBlobWriter' option requires the 'AttributesPath' option be specified")
+	}
+
+	workDir, err := ensureWorkDir(opt.WorkDir)
+	if err != nil {
+		return errors.Wrap(err, "ensure work directory")
+	}
+	defer func() {
+		if err != nil {
+			os.RemoveAll(workDir)
+		}
+	}()
+
+	blobPath := filepath.Join(workDir, "blob")
+	blobFifo, err := fifo.OpenFifo(ctx, blobPath, syscall.O_CREAT|syscall.O_RDONLY|syscall.O_NONBLOCK, 0640)
+	if err != nil {
+		return errors.Wrapf(err, "create fifo file for blob")
+	}
+	defer blobFifo.Close()
+
+	externalBlobPath := filepath.Join(workDir, "external-blob")
+	externalBlobFifo, err := fifo.OpenFifo(ctx, externalBlobPath, syscall.O_CREAT|syscall.O_RDONLY|syscall.O_NONBLOCK, 0640)
+	if err != nil {
+		return errors.Wrapf(err, "create fifo file for external blob")
+	}
+	defer externalBlobFifo.Close()
+
+	go func() {
+		err := tool.Pack(tool.PackOption{
+			BuilderPath: builderPath,
+
+			BlobPath:         blobPath,
+			ExternalBlobPath: externalBlobPath,
+			FsVersion:        opt.FsVersion,
+			SourcePath:       sourceDir,
+			ChunkDictPath:    opt.ChunkDictPath,
+			AttributesPath:   opt.AttributesPath,
+			PrefetchPatterns: opt.PrefetchPatterns,
+			AlignedChunk:     opt.AlignedChunk,
+			ChunkSize:        opt.ChunkSize,
+			BatchSize:        opt.BatchSize,
+			Compressor:       opt.Compressor,
+			Timeout:          opt.Timeout,
+			Encrypt:          opt.Encrypt,
+
+			Features: opt.features,
+		})
+		if err != nil {
+			blobFifo.Close()
+		}
+	}()
+
+	eg := errgroup.Group{}
+	eg.Go(func() error {
+		buffer := bufPool.Get().(*[]byte)
+		defer bufPool.Put(buffer)
+		if _, err := io.CopyBuffer(dest, blobFifo, *buffer); err != nil {
+			return errors.Wrap(err, "pack to nydus blob")
+		}
+		return nil
+	})
+	eg.Go(func() error {
+		buffer := bufPool.Get().(*[]byte)
+		defer bufPool.Put(buffer)
+		if _, err := io.CopyBuffer(opt.ExternalBlobWriter, externalBlobFifo, *buffer); err != nil {
+			return errors.Wrap(err, "pack to nydus external blob")
+		}
+		return nil
+	})
+
+	return eg.Wait()
 }
 
-func packFromDirectory(ctx context.Context, dest io.Writer, opt PackOption, builderPath string) (io.WriteCloser, error) {
+func packFromUnpackedTar(ctx context.Context, dest io.Writer, opt PackOption, builderPath string) (io.WriteCloser, error) {
 	workDir, err := ensureWorkDir(opt.WorkDir)
 	if err != nil {
 		return nil, errors.Wrap(err, "ensure work directory")

pkg/converter/tool/builder.go

@@ -29,11 +29,12 @@ func isSignalKilled(err error) bool {
 type PackOption struct {
 	BuilderPath string
 
-	BootstrapPath    string
 	BlobPath         string
+	ExternalBlobPath string
 	FsVersion        string
 	SourcePath       string
 	ChunkDictPath    string
+	AttributesPath   string
 	PrefetchPatterns string
 	Compressor       string
 	OCIRef           bool
@@ -97,10 +98,21 @@ func buildPackArgs(option PackOption) []string {
 	if option.Features.Contains(FeatureTar2Rafs) {
 		args = append(
 			args,
-			"--type",
-			"tar-rafs",
 			"--blob-inline-meta",
 		)
+		if option.AttributesPath != "" {
+			args = append(
+				args,
+				"--type",
+				"dir-rafs",
+			)
+		} else {
+			args = append(
+				args,
+				"--type",
+				"tar-rafs",
+			)
+		}
 		if option.FsVersion == "6" {
 			args = append(
 				args,
@@ -140,6 +152,12 @@ func buildPackArgs(option PackOption) []string {
 	if option.Encrypt {
 		args = append(args, "--encrypt")
 	}
+	if option.AttributesPath != "" {
+		args = append(args, "--attributes", option.AttributesPath)
+	}
+	if option.ExternalBlobPath != "" {
+		args = append(args, "--external-blob", option.ExternalBlobPath)
+	}
 	args = append(args, option.SourcePath)
 
 	return args

pkg/converter/types.go

@@ -10,6 +10,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io"
 	"strings"
 	"time"
 
@@ -83,6 +84,12 @@ type PackOption struct {
 	Timeout *time.Duration
 	// Whether the generated Nydus blobs should be encrypted.
 	Encrypt bool
+	// Pack to nydus blob from a directory.
+	FromDir string
+	// Path to specified nydus attributes configuration file.
+	AttributesPath string
+	// Path to outputted nydus external blob file.
+	ExternalBlobWriter io.Writer
 
 	// Features keeps a feature list supported by newer version of builder,
 	// It is detected automatically, so don't export it.