fix(host-cleanup): don't stop removing werf containers on paused or running ones

Signed-off-by: Alexandr Zaytsev <alexandr.zaytsev@flant.com>

Author: nervgh

pkg/container_backend/docker_server_backend.go

@@ -2,6 +2,7 @@ package container_backend
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -272,7 +273,15 @@ func (backend *DockerServerBackend) Rmi(ctx context.Context, ref string, opts Rm
 }
 
 func (backend *DockerServerBackend) Rm(ctx context.Context, ref string, opts RmOpts) error {
-	return docker.ContainerRemove(ctx, ref, types.ContainerRemoveOptions{Force: opts.Force})
+	err := docker.ContainerRemove(ctx, ref, types.ContainerRemoveOptions{Force: opts.Force})
+	switch {
+	case docker.IsErrContainerPaused(err):
+		return errors.Join(ErrCannotRemovePausedContainer, err)
+	case docker.IsErrContainerRunning(err):
+		return errors.Join(ErrCannotRemoveRunningContainer, err)
+	default:
+		return err // err or nil
+	}
 }
 
 func (backend *DockerServerBackend) PushImage(ctx context.Context, img LegacyImageInterface) error {

pkg/container_backend/errors.go

@@ -2,4 +2,8 @@ package container_backend
 
 import "errors"
 
-var ErrUnsupportedFeature = errors.New("unsupported feature")
+var (
+	ErrUnsupportedFeature           = errors.New("unsupported feature")
+	ErrCannotRemovePausedContainer  = errors.New("cannot remove paused container")
+	ErrCannotRemoveRunningContainer = errors.New("cannot remove running container")
+)

pkg/docker/errors.go

@@ -0,0 +1,33 @@
+package docker
+
+import (
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+// IsErrContainerPaused returns true if err is "container is paused" error
+// https://github.com/moby/moby/blob/25.0/daemon/delete.go#L94
+func IsErrContainerPaused(err error) bool {
+	if err == nil {
+		return false
+	}
+	cause := errors.Cause(err)
+	if !strings.HasPrefix(cause.Error(), "cannot remove container") {
+		return false
+	}
+	return strings.HasSuffix(cause.Error(), "container is paused and must be unpaused first")
+}
+
+// IsErrContainerRunning returns true if err is "container is running" error
+// https://github.com/moby/moby/blob/25.0/daemon/delete.go#L96
+func IsErrContainerRunning(err error) bool {
+	if err == nil {
+		return false
+	}
+	cause := errors.Cause(err)
+	if !strings.HasPrefix(cause.Error(), "cannot remove container") {
+		return false
+	}
+	return strings.HasSuffix(cause.Error(), "container is running: stop the container before removing or force remove")
+}

pkg/docker/errors_test.go

@@ -0,0 +1,34 @@
+package docker
+
+import (
+	"errors"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	errorsPkg "github.com/pkg/errors"
+)
+
+var _ = Describe("docker errors", func() {
+	Describe("IsErrContainerPaused", func() {
+		It("should return true if err is \"container is paused\" error", func() {
+			causeErr := errors.New("cannot remove container XXXX: container is paused and must be unpaused first")
+			err0 := errorsPkg.WithMessage(causeErr, "some text")
+			Expect(IsErrContainerPaused(err0)).To(BeTrue())
+		})
+		It("should return false otherwise", func() {
+			err0 := errors.New("some err")
+			Expect(IsErrContainerPaused(err0)).To(BeFalse())
+		})
+	})
+	Describe("IsErrContainerRunning", func() {
+		It("should return true if err is \"container is paused\" error", func() {
+			causeErr := errors.New("cannot remove container XXXX: container is running: stop the container before removing or force remove")
+			err0 := errorsPkg.WithMessage(causeErr, "some text")
+			Expect(IsErrContainerRunning(err0)).To(BeTrue())
+		})
+		It("should return false otherwise", func() {
+			err0 := errors.New("some err")
+			Expect(IsErrContainerRunning(err0)).To(BeFalse())
+		})
+	})
+})

pkg/docker/suite_test.go

@@ -0,0 +1,13 @@
+package docker
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestDocker(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Docker Suite")
+}

pkg/host_cleaning/local_backend_cleaner.go

@@ -416,7 +416,7 @@ func (cleaner *LocalBackendCleaner) RunGC(ctx context.Context, options RunGCOpti
 	}
 
 	if vu.Percentage() <= options.AllowedStorageVolumeUsagePercentage {
-		logboek.Context(ctx).LogBlock("Check storage", cleaner.BackendName()).Do(func() {
+		logboek.Context(ctx).LogBlock("Check storage").Do(func() {
 			logboek.Context(ctx).LogF("Total freed space: %s\n", utils.RedF("%s", humanize.Bytes(vuBefore.UsedBytes-vu.UsedBytes)))
 			logboek.Context(ctx).LogF("Volume usage: %s / %s\n", humanize.Bytes(vu.UsedBytes), humanize.Bytes(vu.TotalBytes))
 			logboek.Context(ctx).LogF("Allowed percentage level exceeded: %s > %s — %s\n", utils.RedF("%0.2f%%", vu.Percentage()), utils.YellowF("%0.2f%%", options.AllowedStorageVolumeUsagePercentage), utils.RedF("HIGH VOLUME USAGE"))
@@ -574,7 +574,14 @@ func (cleaner *LocalBackendCleaner) doSafeCleanupWerfContainers(ctx context.Cont
 			err := cleaner.backend.Rm(ctx, container.ID, container_backend.RmOpts{
 				Force: options.Force,
 			})
-			if err != nil {
+			switch {
+			case errors.Is(err, container_backend.ErrCannotRemovePausedContainer):
+				logboek.Context(ctx).Warn().LogF("Ignore paused container %s\n", logContainerName(container))
+				return nil
+			case errors.Is(err, container_backend.ErrCannotRemoveRunningContainer):
+				logboek.Context(ctx).Warn().LogF("Ignore running container %s\n", logContainerName(container))
+				return nil
+			case err != nil:
 				return fmt.Errorf("failed to remove container %s: %w", logContainerName(container), err)
 			}
 			return nil

pkg/host_cleaning/local_backend_cleaner_test.go

@@ -187,6 +187,33 @@ var _ = Describe("LocalBackendCleaner", func() {
 		})
 	})
 
+	Describe("doSafeCleanupWerfContainers", func() {
+		It("should not return err if backend.Rm() returns 'container is paused' error", func() {
+			container := image.Container{
+				ID:    "id-stage",
+				Names: []string{fmt.Sprintf("/%s", image.StageContainerNamePrefix)},
+			}
+
+			backend.EXPECT().Rm(ctx, container.ID, container_backend.RmOpts{Force: false}).Return(container_backend.ErrCannotRemovePausedContainer)
+			stubs.StubFunc(&cleaner.volumeutilsGetVolumeUsageByPath, volumeutils.VolumeUsage{}, nil)
+
+			_, err := cleaner.doSafeCleanupWerfContainers(ctx, RunGCOptions{}, volumeutils.VolumeUsage{}, image.ContainerList{container})
+			Expect(err).To(Succeed())
+		})
+		It("should not return err if backend.Rm() returns 'container is running' error", func() {
+			container := image.Container{
+				ID:    "id-import-server",
+				Names: []string{fmt.Sprintf("/%s", image.ImportServerContainerNamePrefix)},
+			}
+
+			backend.EXPECT().Rm(ctx, container.ID, container_backend.RmOpts{Force: false}).Return(container_backend.ErrCannotRemoveRunningContainer)
+			stubs.StubFunc(&cleaner.volumeutilsGetVolumeUsageByPath, volumeutils.VolumeUsage{}, nil)
+
+			_, err := cleaner.doSafeCleanupWerfContainers(ctx, RunGCOptions{}, volumeutils.VolumeUsage{}, image.ContainerList{container})
+			Expect(err).To(Succeed())
+		})
+	})
+
 	Describe("werfImages", func() {
 		It("should return images as merged and sorted result of several backend calls", func() {
 			expectedImages := []image.Summary{