feat: add nydusify commit command

.github/workflows/smoke.yml

@@ -165,7 +165,8 @@ jobs:
         sudo mkdir -p /usr/bin/nydus-latest /home/runner/work/workdir
         sudo install -D -m 755 contrib/nydusify/cmd/nydusify /usr/bin/nydus-latest
         sudo install -D -m 755 target/release/nydusd target/release/nydus-image /usr/bin/nydus-latest
-
+        sudo bash misc/prepare.sh
+
         export NYDUS_STABLE_VERSION=$(curl https://api.github.com/repos/Dragonflyoss/nydus/releases/latest | jq -r '.tag_name')
         export NYDUS_STABLE_VERSION_EXPORT="${NYDUS_STABLE_VERSION//./_}"
 
@@ -302,7 +303,7 @@ jobs:
         path: contrib/nydusify/cmd
     - name: Prepare Nydus Container Environment
       run: |
-        sudo bash misc/performance/prepare.sh
+        sudo bash misc/prepare.sh
     - name: Performance Test
       run: |
         export PERFORMANCE_TEST_MODE=${{ matrix.mode }}
@@ -326,7 +327,7 @@ jobs:
         path: contrib/nydusify/cmd
     - name: Prepare Nydus Container Environment
       run: |
-        sudo bash misc/takeover/prepare.sh
+        sudo bash misc/prepare.sh takeover_test
     - name: Takeover Test
       run: |
         export NEW_NYDUSD_BINARY_PATH=target/release/nydusd

contrib/nydusify/cmd/nydusify.go

@@ -26,6 +26,7 @@ import (
 	"github.com/dragonflyoss/nydus/contrib/nydusify/pkg/checker"
 	"github.com/dragonflyoss/nydus/contrib/nydusify/pkg/checker/rule"
 	"github.com/dragonflyoss/nydus/contrib/nydusify/pkg/chunkdict/generator"
+	"github.com/dragonflyoss/nydus/contrib/nydusify/pkg/committer"
 	"github.com/dragonflyoss/nydus/contrib/nydusify/pkg/converter"
 	"github.com/dragonflyoss/nydus/contrib/nydusify/pkg/copier"
 	"github.com/dragonflyoss/nydus/contrib/nydusify/pkg/packer"
@@ -1103,6 +1104,108 @@ func main() {
 				return copier.Copy(context.Background(), opt)
 			},
 		},
+		{
+			Name:  "commit",
+			Usage: "Create and push a new nydus image from a container's changes that use a nydus image",
+			Flags: []cli.Flag{
+				&cli.StringFlag{
+					Name:    "work-dir",
+					Value:   "./tmp",
+					Usage:   "Working directory for commit workflow",
+					EnvVars: []string{"WORK_DIR"},
+				},
+				&cli.StringFlag{
+					Name:    "nydus-image",
+					Value:   "nydus-image",
+					Usage:   "Path to the nydus-image binary, default to search in PATH",
+					EnvVars: []string{"NYDUS_IMAGE"},
+				},
+				&cli.StringFlag{
+					Name:    "containerd-address",
+					Value:   "/run/containerd/containerd.sock",
+					Usage:   "Containerd address, optionally with \"unix://\" prefix [$CONTAINERD_ADDRESS] (default \"/run/containerd/containerd.sock\")",
+					EnvVars: []string{"CONTAINERD_ADDR"},
+				},
+				&cli.StringFlag{
+					Name:     "container",
+					Required: true,
+					Usage:    "Target container id",
+					EnvVars:  []string{"CONTAINER"},
+				},
+				&cli.StringFlag{
+					Name:     "target",
+					Required: true,
+					Usage:    "Target nydus image reference",
+					EnvVars:  []string{"TARGET"},
+				},
+				&cli.BoolFlag{
+					Name:     "source-insecure",
+					Required: false,
+					Usage:    "Skip verifying server certs for HTTPS source registry",
+					EnvVars:  []string{"SOURCE_INSECURE"},
+				},
+				&cli.BoolFlag{
+					Name:     "target-insecure",
+					Required: false,
+					Usage:    "Skip verifying server certs for HTTPS target registry",
+					EnvVars:  []string{"TARGET_INSECURE"},
+				},
+				&cli.IntFlag{
+					Name:        "maximum-times",
+					Required:    false,
+					DefaultText: "400",
+					Value:       400,
+					Usage:       "The maximum times allowed to be committed",
+					EnvVars:     []string{"MAXIMUM_TIMES"},
+				},
+				&cli.StringSliceFlag{
+					Name:     "with-path",
+					Aliases:  []string{"with-mount-path"},
+					Required: false,
+					Usage:    "The external directory (for example mountpoint) in container that need to be committed",
+					EnvVars:  []string{"WITH_PATH"},
+				},
+			},
+			Action: func(c *cli.Context) error {
+				setupLogLevel(c)
+				parsePaths := func(paths []string) ([]string, []string) {
+					withPaths := []string{}
+					withoutPaths := []string{}
+
+					for _, path := range paths {
+						path = strings.TrimSpace(path)
+						if strings.HasPrefix(path, "!") {
+							path = strings.TrimLeft(path, "!")
+							path = strings.TrimRight(path, "/")
+							withoutPaths = append(withoutPaths, path)
+						} else {
+							withPaths = append(withPaths, path)
+						}
+					}
+
+					return withPaths, withoutPaths
+				}
+
+				withPaths, withoutPaths := parsePaths(c.StringSlice("with-path"))
+				opt := committer.Opt{
+					WorkDir:           c.String("work-dir"),
+					NydusImagePath:    c.String("nydus-image"),
+					ContainerdAddress: c.String("containerd-address"),
+					ContainerID:       c.String("container"),
+					TargetRef:         c.String("target"),
+					SourceInsecure:    c.Bool("source-insecure"),
+					TargetInsecure:    c.Bool("target-insecure"),
+					MaximumTimes:      c.Int("maximum-times"),
+					WithPaths:         withPaths,
+					WithoutPaths:      withoutPaths,
+				}
+				cm, err := committer.NewCommitter(opt)
+				if err != nil {
+					return errors.Wrap(err, "create commiter")
+				}
+				return cm.Commit(c.Context, opt)
+			},
+		},
 	}
 
 	if !utils.IsSupportedArch(runtime.GOARCH) {

contrib/nydusify/go.mod

@@ -4,20 +4,22 @@ go 1.21
 
 require (
 	github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible
-	github.com/aws/aws-sdk-go-v2 v1.24.0
-	github.com/aws/aws-sdk-go-v2/config v1.26.2
-	github.com/aws/aws-sdk-go-v2/credentials v1.16.13
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.9
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.47.7
-	github.com/containerd/containerd v1.7.11
-	github.com/containerd/nydus-snapshotter v0.13.4
+	github.com/aws/aws-sdk-go-v2 v1.24.1
+	github.com/aws/aws-sdk-go-v2/config v1.26.6
+	github.com/aws/aws-sdk-go-v2/credentials v1.16.16
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.15
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.48.1
+	github.com/containerd/containerd v1.7.13
+	github.com/containerd/continuity v0.4.3
+	github.com/containerd/nydus-snapshotter v0.13.7
 	github.com/distribution/reference v0.5.0
-	github.com/docker/cli v24.0.7+incompatible
+	github.com/docker/cli v25.0.3+incompatible
 	github.com/dustin/go-humanize v1.0.1
 	github.com/goharbor/acceleration-service v0.2.12
 	github.com/google/uuid v1.5.0
 	github.com/hashicorp/go-hclog v1.6.2
 	github.com/hashicorp/go-plugin v1.6.0
+	github.com/moby/buildkit v0.13.0
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.1.0-rc5
 	github.com/pkg/errors v0.9.1
@@ -37,23 +39,22 @@ require (
 	github.com/Microsoft/go-winio v0.6.1 // indirect
 	github.com/Microsoft/hcsshim v0.11.4 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.9 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.7.3 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.10 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.9 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.9 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.18.5 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.26.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.10 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.10 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.18.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 // indirect
 	github.com/aws/smithy-go v1.19.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/containerd/cgroups v1.1.0 // indirect
-	github.com/containerd/continuity v0.4.3 // indirect
 	github.com/containerd/fifo v1.1.0 // indirect
 	github.com/containerd/log v0.1.0 // indirect
 	github.com/containerd/stargz-snapshotter v0.15.1 // indirect
@@ -63,7 +64,7 @@ require (
 	github.com/containers/ocicrypt v1.1.9 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/docker/docker v24.0.7+incompatible // indirect
+	github.com/docker/docker v25.0.3+incompatible // indirect
 	github.com/docker/docker-credential-helpers v0.8.0 // indirect
 	github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
 	github.com/fatih/color v1.16.0 // indirect
@@ -102,7 +103,7 @@ require (
 	github.com/stretchr/objx v0.5.1 // indirect
 	github.com/vbatts/tar-split v0.11.5 // indirect
 	github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e // indirect
-	go.etcd.io/bbolt v1.3.8 // indirect
+	go.etcd.io/bbolt v1.3.9 // indirect
 	go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/otel v1.21.0 // indirect