From 0273c008d9ace85b347eed80bb359818304c2ca0 Mon Sep 17 00:00:00 2001 From: Manu Gupta Date: Mon, 5 Sep 2022 15:37:13 -0700 Subject: [PATCH] Implements logURI to support 3rd party ctrd shim logger Testing details Custom built the journld shim logger ``` nerdctl run -d --log-driver binary:/home/manu/go/src/github.com/manugupt1/ctr-journald-shim/ctr-journald-shim docker.io/library/hello-world:latest ``` Journalctl logs ``` ep 05 15:36:24 fedora default:10585b461231fb19106451008cc013f14bfd0fc0fc6bfaa11c3dd81a8f962674[189472]: Sep 05 15:36:24 fedora default:10585b461231fb19106451008cc013f14bfd0fc0fc6bfaa11c3dd81a8f962674[189472]: Hello from Docker! Sep 05 15:36:24 fedora default:10585b461231fb19106451008cc013f14bfd0fc0fc6bfaa11c3dd81a8f962674[189472]: This message shows that your installation appears to be working correctly. Sep 05 15:36:24 fedora default:10585b461231fb19106451008cc013f14bfd0fc0fc6bfaa11c3dd81a8f962674[189472]: Sep 05 15:36:24 fedora default:10585b461231fb19106451008cc013f14bfd0fc0fc6bfaa11c3dd81a8f962674[189472]: To generate this message, Docker took the following steps: Sep 05 15:36:24 fedora default:10585b461231fb19106451008cc013f14bfd0fc0fc6bfaa11c3dd81a8f962674[189472]: 1. The Docker client contacted the Docker daemon. Sep 05 15:36:24 fedora default:10585b461231fb19106451008cc013f14bfd0fc0fc6bfaa11c3dd81a8f962674[189472]: 2. The Docker daemon pulled the "hello-world" image from the Docker Hub. Sep 05 15:36:24 fedora default:10585b461231fb19106451008cc013f14bfd0fc0fc6bfaa11c3dd81a8f962674[189472]: (amd64) Sep 05 15:36:24 fedora default:10585b461231fb19106451008cc013f14bfd0fc0fc6bfaa11c3dd81a8f962674[189472]: 3. The Docker daemon created a new container from that image which runs the Sep 05 15:36:24 fedora default:10585b461231fb19106451008cc013f14bfd0fc0fc6bfaa11c3dd81a8f962674[189472]: executable that produces the output you are currently reading. Sep 05 15:36:24 fedora default:10585b461231fb19106451008cc013f14bfd0fc0fc6bfaa11c3dd81a8f962674[189472]: 4. The Docker daemon streamed that output to the Docker client, which sent it Sep 05 15:36:24 fedora default:10585b461231fb19106451008cc013f14bfd0fc0fc6bfaa11c3dd81a8f962674[189472]: to your terminal. Sep 05 15:36:24 fedora default:10585b461231fb19106451008cc013f14bfd0fc0fc6bfaa11c3dd81a8f962674[189472]: Sep 05 15:36:24 fedora default:10585b461231fb19106451008cc013f14bfd0fc0fc6bfaa11c3dd81a8f962674[189472]: To try something more ambitious, you can run an Ubuntu container with: Sep 05 15:36:24 fedora default:10585b461231fb19106451008cc013f14bfd0fc0fc6bfaa11c3dd81a8f962674[189472]: $ docker run -it ubuntu bash Sep 05 15:36:24 fedora default:10585b461231fb19106451008cc013f14bfd0fc0fc6bfaa11c3dd81a8f962674[189472]: Sep 05 15:36:24 fedora default:10585b461231fb19106451008cc013f14bfd0fc0fc6bfaa11c3dd81a8f962674[189472]: Share images, automate workflows, and more with a free Docker ID: Sep 05 15:36:24 fedora default:10585b461231fb19106451008cc013f14bfd0fc0fc6bfaa11c3dd81a8f962674[189472]: https://hub.docker.com/ Sep 05 15:36:24 fedora default:10585b461231fb19106451008cc013f14bfd0fc0fc6bfaa11c3dd81a8f962674[189472]: Sep 05 15:36:24 fedora default:10585b461231fb19106451008cc013f14bfd0fc0fc6bfaa11c3dd81a8f962674[189472]: For more examples and ideas, visit: Sep 05 15:36:24 fedora default:10585b461231fb19106451008cc013f14bfd0fc0fc6bfaa11c3dd81a8f962674[189472]: https://docs.docker.com/get-started/ Sep 05 15:36:24 fedora default:10585b461231fb19106451008cc013f14bfd0fc0fc6bfaa11c3dd81a8f962674[189472]: Sep 05 15:36:24 fedora containerd-rootless.sh[3083]: time="2022-09-05T15:36:24.411080375-07:00" level=warning msg="error from *cgroupsv2.Manager.EventChan" error="failed to add inotify watch for \"/sys/fs/cgroup/user ``` Co-authored-by: Akihiro Suda Signed-off-by: Manu Gupta --- README.md | 2 + cmd/nerdctl/run.go | 64 +++++++++++++++++------------- cmd/nerdctl/run_test.go | 88 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 126 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 06231bf203..8932d9ae80 100644 --- a/README.md +++ b/README.md @@ -508,6 +508,8 @@ Logging flags: - :whale: `--log-opt=fluentd-sub-second-precision=`: Enable sub-second precision for fluentd. The default value is false. - :nerd_face: `--log-opt=fluentd-async-reconnect-interval=<1s|1ms>`: The time to wait before retrying to reconnect to fluentd. The default value is 0s. - :nerd_face: `--log-opt=fluentd-request-ack=`: Enable request ack for fluentd. The default value is false. + - :nerd_face: Accepts a LogURI which is a containerd shim logger. A scheme must be specified for the URI. Example: `nerdctl run -d --log-driver binary:///usr/bin/ctr-journald-shim docker.io/library/hello-world:latest`. An implementation of shim logger can be found at (https://github.com/containerd/containerd/tree/dbef1d56d7ebc05bc4553d72c419ed5ce025b05d/runtime/v2#logging) + Shared memory flags: - :whale: `--ipc`: IPC namespace to use diff --git a/cmd/nerdctl/run.go b/cmd/nerdctl/run.go index 5d2dffac27..616c867e5a 100644 --- a/cmd/nerdctl/run.go +++ b/cmd/nerdctl/run.go @@ -250,7 +250,7 @@ func setCreateFlags(cmd *cobra.Command) { // #region logging flags // log-opt needs to be StringArray, not StringSlice, to prevent "env=os,customer" from being split to {"env=os", "customer"} - cmd.Flags().String("log-driver", "json-file", "Logging driver for the container") + cmd.Flags().String("log-driver", "json-file", "Logging driver for the container. Default is json-file. It also supports logURI (eg: --log-driver binary://)") cmd.RegisterFlagCompletionFunc("log-driver", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return logging.Drivers(), cobra.ShellCompDirectiveNoFileComp }) @@ -497,33 +497,41 @@ func createContainer(cmd *cobra.Command, ctx context.Context, client *containerd if err != nil { return nil, nil, err } - logOptMap, err := parseKVStringsMapFromLogOpt(cmd, logDriver) - if err != nil { - return nil, nil, err - } - logDriverInst, err := logging.GetDriver(logDriver, logOptMap) - if err != nil { - return nil, nil, err - } - if err := logDriverInst.Init(dataStore, ns, id); err != nil { - return nil, nil, err - } - logConfig := &logging.LogConfig{ - Driver: logDriver, - Opts: logOptMap, - } - logConfigB, err := json.Marshal(logConfig) - if err != nil { - return nil, nil, err - } - logConfigFilePath := logging.LogConfigFilePath(dataStore, ns, id) - if err = os.WriteFile(logConfigFilePath, logConfigB, 0600); err != nil { - return nil, nil, err - } - if lu, err := generateLogURI(dataStore); err != nil { - return nil, nil, err - } else if lu != nil { - logURI = lu.String() + + // check if log driver is a valid uri. If it is a valid uri and scheme is not + if u, err := url.Parse(logDriver); err == nil && u.Scheme != "" { + logURI = logDriver + } else { + logOptMap, err := parseKVStringsMapFromLogOpt(cmd, logDriver) + if err != nil { + return nil, nil, err + } + logDriverInst, err := logging.GetDriver(logDriver, logOptMap) + if err != nil { + return nil, nil, err + } + if err := logDriverInst.Init(dataStore, ns, id); err != nil { + return nil, nil, err + } + logConfig := &logging.LogConfig{ + Driver: logDriver, + Opts: logOptMap, + } + logConfigB, err := json.Marshal(logConfig) + if err != nil { + return nil, nil, err + } + logConfigFilePath := logging.LogConfigFilePath(dataStore, ns, id) + if err = os.WriteFile(logConfigFilePath, logConfigB, 0600); err != nil { + return nil, nil, err + } + if lu, err := generateLogURI(dataStore); err != nil { + return nil, nil, err + } else if lu != nil { + logrus.Debugf("generated log driver: %s", lu.String()) + + logURI = lu.String() + } } } diff --git a/cmd/nerdctl/run_test.go b/cmd/nerdctl/run_test.go index 5336f08875..996068b20b 100644 --- a/cmd/nerdctl/run_test.go +++ b/cmd/nerdctl/run_test.go @@ -305,3 +305,91 @@ func TestRunWithJournaldLogDriverAndLogOpt(t *testing.T) { poll.WaitOn(t, check, poll.WithDelay(100*time.Microsecond), poll.WithTimeout(20*time.Second)) assert.Equal(t, 1, found) } + +func TestRunWithLogBinary(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("buildkit is not enabled on windows, this feature may work on windows.") + } + testutil.DockerIncompatible(t) + t.Parallel() + base := testutil.NewBase(t) + imageName := testutil.Identifier(t) + "-image" + containerName := testutil.Identifier(t) + + const dockerfile = ` +FROM golang:latest as builder +WORKDIR /go/src/ +RUN mkdir -p logger +WORKDIR /go/src/logger +RUN echo '\ + package main \n\ + \n\ + import ( \n\ + "bufio" \n\ + "context" \n\ + "fmt" \n\ + "io" \n\ + "os" \n\ + "path/filepath" \n\ + "sync" \n\ + \n\ + "github.com/containerd/containerd/runtime/v2/logging"\n\ + )\n\ + + func main() {\n\ + logging.Run(log)\n\ + }\n\ + + func log(ctx context.Context, config *logging.Config, ready func() error) error {\n\ + var wg sync.WaitGroup \n\ + wg.Add(2) \n\ + // forward both stdout and stderr to temp files \n\ + go copy(&wg, config.Stdout, config.ID, "stdout") \n\ + go copy(&wg, config.Stderr, config.ID, "stderr") \n\ + + // signal that we are ready and setup for the container to be started \n\ + if err := ready(); err != nil { \n\ + return err \n\ + } \n\ + wg.Wait() \n\ + return nil \n\ + }\n\ + \n\ + func copy(wg *sync.WaitGroup, r io.Reader, id string, kind string) { \n\ + f, _ := os.Create(filepath.Join(os.TempDir(), fmt.Sprintf("%s_%s.log", id, kind))) \n\ + defer f.Close() \n\ + defer wg.Done() \n\ + s := bufio.NewScanner(r) \n\ + for s.Scan() { \n\ + f.WriteString(s.Text()) \n\ + } \n\ + }\n' >> main.go + + +RUN go mod init +RUN go mod tidy +RUN go build . + +FROM scratch +COPY --from=builder /go/src/logger/logger / + ` + + buildCtx, err := createBuildContext(dockerfile) + assert.NilError(t, err) + defer os.RemoveAll(buildCtx) + tmpDir := t.TempDir() + base.Cmd("build", buildCtx, "--output", fmt.Sprintf("type=local,src=/go/src/logger/logger,dest=%s", tmpDir)).AssertOK() + defer base.Cmd("image", "rm", "-f", imageName).AssertOK() + + base.Cmd("container", "rm", "-f", containerName).AssertOK() + base.Cmd("run", "-d", "--log-driver", fmt.Sprintf("binary://%s/logger", tmpDir), "--name", containerName, testutil.CommonImage, + "sh", "-euxc", "echo foo; echo bar").AssertOK() + defer base.Cmd("container", "rm", "-f", containerName) + + inspectedContainer := base.InspectContainer(containerName) + bytes, err := os.ReadFile(filepath.Join(os.TempDir(), fmt.Sprintf("%s_%s.log", inspectedContainer.ID, "stdout"))) + assert.NilError(t, err) + log := string(bytes) + assert.Check(t, strings.Contains(log, "foo")) + assert.Check(t, strings.Contains(log, "bar")) +}