diff --git a/frontend/gateway/container.go b/frontend/gateway/container.go
index 824a503ffb55..45cf2d90eb50 100644
--- a/frontend/gateway/container.go
+++ b/frontend/gateway/container.go
@@ -361,6 +361,8 @@ func (gwCtr *gatewayContainer) Start(ctx context.Context, req client.StartReques
 }
 
 func (gwCtr *gatewayContainer) Release(ctx context.Context) error {
+	gwCtr.mu.Lock()
+	defer gwCtr.mu.Unlock()
 	gwCtr.cancel()
 	err1 := gwCtr.errGroup.Wait()
 
@@ -371,6 +373,7 @@ func (gwCtr *gatewayContainer) Release(ctx context.Context) error {
 			err2 = err
 		}
 	}
+	gwCtr.cleanup = nil
 
 	if err1 != nil {
 		return stack.Enable(err1)
diff --git a/frontend/gateway/forwarder/forward.go b/frontend/gateway/forwarder/forward.go
index c4b32e961b0e..b9b88098feb5 100644
--- a/frontend/gateway/forwarder/forward.go
+++ b/frontend/gateway/forwarder/forward.go
@@ -50,6 +50,7 @@ type bridgeClient struct {
 	workers       worker.Infos
 	workerRefByID map[string]*worker.WorkerRef
 	buildOpts     client.BuildOpts
+	ctrs          []client.Container
 }
 
 func (c *bridgeClient) Solve(ctx context.Context, req client.SolveRequest) (*client.Result, error) {
@@ -208,6 +209,10 @@ func (c *bridgeClient) toFrontendResult(r *client.Result) (*frontend.Result, err
 }
 
 func (c *bridgeClient) discard(err error) {
+	for _, ctr := range c.ctrs {
+		ctr.Release(context.TODO())
+	}
+
 	for id, workerRef := range c.workerRefByID {
 		workerRef.ImmutableRef.Release(context.TODO())
 		delete(c.workerRefByID, id)
@@ -299,6 +304,7 @@ func (c *bridgeClient) NewContainer(ctx context.Context, req client.NewContainer
 	if err != nil {
 		return nil, err
 	}
+	c.ctrs = append(c.ctrs, ctr)
 	return ctr, nil
 }
 
diff --git a/frontend/gateway/gateway.go b/frontend/gateway/gateway.go
index b983d37a60be..83b4b1fca4f4 100644
--- a/frontend/gateway/gateway.go
+++ b/frontend/gateway/gateway.go
@@ -345,6 +345,13 @@ func (b *bindMount) IdentityMapping() *idtools.IdentityMapping {
 func (lbf *llbBridgeForwarder) Discard() {
 	lbf.mu.Lock()
 	defer lbf.mu.Unlock()
+
+	for ctr := range lbf.ctrs {
+		lbf.ReleaseContainer(context.TODO(), &pb.ReleaseContainerRequest{
+			ContainerID: ctr,
+		})
+	}
+
 	for id, workerRef := range lbf.workerRefByID {
 		workerRef.ImmutableRef.Release(context.TODO())
 		delete(lbf.workerRefByID, id)