Skip to content
Permalink
Browse files
fix(build): cleanup orphan build containers on ctrl-c or gitlab cancel
Signed-off-by: Timofey Kirillov <timofey.kirillov@flant.com>
  • Loading branch information
distorhead committed Apr 20, 2022
1 parent 5a9bb4a commit 8702efaf5cae11da9c60e174397a280fa9537ba2
@@ -4,6 +4,8 @@ import (
"os"
"os/signal"
"syscall"

"github.com/werf/werf/pkg/container_backend"
)

var (
@@ -21,6 +23,8 @@ func EnableTerminationSignalsTrap() {
go func() {
select {
case <-terminationSignalsChan:
container_backend.TerminateRunningDockerContainers()

TerminateWithError("interrupted", 17)
case <-disableTerminationSignalsTrapChan:
return
@@ -0,0 +1,58 @@
package container_backend

import (
"context"
"sync"

"github.com/docker/docker/api/types"

"github.com/werf/logboek"
"github.com/werf/werf/pkg/docker"
)

var (
runningDockerContainers []*runningDockerContainer
mu sync.Mutex
)

type runningDockerContainer struct {
Name string
Ctx context.Context
}

func RegisterRunningContainer(name string, ctx context.Context) {
mu.Lock()
defer mu.Unlock()

runningDockerContainers = append(runningDockerContainers, &runningDockerContainer{
Name: name,
Ctx: ctx,
})
}

func UnregisterRunningContainer(name string) {
mu.Lock()
defer mu.Unlock()

var res []*runningDockerContainer
for _, cont := range runningDockerContainers {
if cont.Name != name {
res = append(res, cont)
}
}
runningDockerContainers = res
}

func TerminateRunningDockerContainers() error {
mu.Lock()
defer mu.Unlock()

for _, container := range runningDockerContainers {
err := docker.ContainerRemove(container.Ctx, container.Name, types.ContainerRemoveOptions{RemoveVolumes: true, Force: true})
if err != nil {
logboek.Context(container.Ctx).Error().LogF("WARNING: Unable to remove container %q: %w\n", container.Name, err.Error())
}
}

return nil
}
@@ -91,7 +91,7 @@ func (i *LegacyStageImage) Build(ctx context.Context, options BuildOptions) erro
}

if err := i.container.rm(ctx); err != nil {
return fmt.Errorf("introspect error failed: %w", err)
return fmt.Errorf("unable to remove container (original error %s): %w", containerRunErr, err)
}
}

@@ -267,10 +267,12 @@ func (c *LegacyStageImageContainer) run(ctx context.Context) error {
return err
}

if err := docker.CliRun_LiveOutput(ctx, runArgs...); err != nil {
RegisterRunningContainer(c.name, ctx)
err = docker.CliRun_LiveOutput(ctx, runArgs...)
UnregisterRunningContainer(c.name)
if err != nil {
return fmt.Errorf("container run failed: %w", err)
}

return nil
}

@@ -339,5 +341,12 @@ func (c *LegacyStageImageContainer) commit(ctx context.Context) (string, error)
func (c *LegacyStageImageContainer) rm(ctx context.Context) error {
_ = c.image.ContainerBackend.(*DockerServerBackend)

return docker.ContainerRemove(ctx, c.name, types.ContainerRemoveOptions{})
err := docker.ContainerRemove(ctx, c.name, types.ContainerRemoveOptions{RemoveVolumes: true, Force: true})
if err != nil {
if strings.Contains(err.Error(), fmt.Sprintf("removal of container %s is already in progress", c.name)) {
return nil
}
return fmt.Errorf("unable to remove container %s: %w", c.name, err)
}
return nil
}
@@ -13,6 +13,7 @@ import (
"syscall"
"time"

"github.com/werf/werf/pkg/container_backend"
"github.com/werf/werf/pkg/werf"
)

@@ -45,6 +46,7 @@ func run(parentsPids []int) {
fmt.Fprintf(os.Stderr, "Process exterminator error: %s\n", err)
}

container_backend.TerminateRunningDockerContainers()
syscall.Kill(ownPid, syscall.SIGINT)

return

0 comments on commit 8702efa

Please sign in to comment.