fix(host-cleanup): fix locking doesn't work as expected #6725

Signed-off-by: Alexandr Zaytsev <alexandr.zaytsev@flant.com>

Author: nervgh

Diff for: pkg/host_cleaning/host_lock.go

@@ -5,9 +5,26 @@ import (
 
 	chart "github.com/werf/common-go/pkg/lock"
 	"github.com/werf/lockgate"
+	"github.com/werf/logboek"
 )
 
-func withHostLock(ctx context.Context, lockName string, fn func() error) error {
+// withHostLockOrNothing executes callback function if "soft" (NonBlocking=true) lock is acquired. Otherwise, does nothing.
+func withHostLockOrNothing(ctx context.Context, lockName string, callback func() error) (err error) {
 	lockOptions := lockgate.AcquireOptions{NonBlocking: true}
-	return chart.WithHostLock(ctx, lockName, lockOptions, fn)
+
+	acquired, lock, err := chart.AcquireHostLock(ctx, lockName, lockOptions)
+	if err != nil {
+		return err
+	}
+
+	if !acquired {
+		logboek.Context(ctx).Warn().LogF("Ignore locked %s\n", lockName)
+		return nil
+	}
+
+	defer func() {
+		err = chart.ReleaseHostLock(lock)
+	}()
+
+	return callback()
 }

Diff for: pkg/host_cleaning/host_lock_test.go

@@ -0,0 +1,40 @@
+package host_cleaning
+
+import (
+	"context"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("withHostLockOrNothing", func() {
+	var ctx context.Context
+	var spy *spyHostLockCallback
+	BeforeEach(func() {
+		ctx = context.Background()
+		spy = &spyHostLockCallback{}
+	})
+	It("should call callback function if lock is acquired", func() {
+		lockName := "test"
+		err := withHostLockOrNothing(ctx, lockName, spy.Method)
+		Expect(err).To(Succeed())
+		Expect(spy.callsCount).To(Equal(1))
+	})
+	It("should not call callback function if lock isn't acquired", func() {
+		lockName := "test"
+		err := withHostLockOrNothing(ctx, lockName, func() error {
+			return withHostLockOrNothing(ctx, lockName, spy.Method) // lock is already acquired in parent function
+		})
+		Expect(err).To(Succeed())
+		Expect(spy.callsCount).To(Equal(0))
+	})
+})
+
+type spyHostLockCallback struct {
+	callsCount int
+}
+
+func (s *spyHostLockCallback) Method() error {
+	s.callsCount++
+	return nil
+}

Diff for: pkg/host_cleaning/local_backend_cleaner.go

@@ -570,7 +570,7 @@ func (cleaner *LocalBackendCleaner) doSafeCleanupWerfContainers(ctx context.Cont
 			continue
 		}
 
-		err := withHostLock(ctx, container_backend.ContainerLockName(containerName), func() error {
+		err := withHostLockOrNothing(ctx, container_backend.ContainerLockName(containerName), func() error {
 			err := cleaner.backend.Rm(ctx, container.ID, container_backend.RmOpts{
 				Force: options.Force,
 			})
@@ -693,7 +693,7 @@ func (cleaner *LocalBackendCleaner) removeImageByRepoTags(ctx context.Context, o
 				unRemovedCount++
 			}
 		} else {
-			err := withHostLock(ctx, container_backend.ImageLockName(ref), func() error {
+			err := withHostLockOrNothing(ctx, container_backend.ImageLockName(ref), func() error {
 				err := cleaner.backend.Rmi(ctx, ref, container_backend.RmiOpts{
 					Force: options.Force,
 				})

Diff for: pkg/host_cleaning/suite_test.go

@@ -7,7 +7,7 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-func TestStage(t *testing.T) {
+func TestHostCleaning(t *testing.T) {
 	RegisterFailHandler(Fail)
 	RunSpecs(t, "Host Cleaning Suite")
 }