Merge pull request containers#8724 from bblenard/support-volume-filte…

…rs-in-system-prune

Add volume filters to system prune

cmd/podman/system/prune.go

@@ -4,16 +4,15 @@ import (
 	"bufio"
 	"context"
 	"fmt"
-	"net/url"
 	"os"
 	"strings"
 
 	"github.com/containers/common/pkg/completion"
 	"github.com/containers/podman/v2/cmd/podman/registry"
 	"github.com/containers/podman/v2/cmd/podman/utils"
 	"github.com/containers/podman/v2/cmd/podman/validate"
+	lpfilters "github.com/containers/podman/v2/libpod/filters"
 	"github.com/containers/podman/v2/pkg/domain/entities"
-	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 )
 
@@ -55,6 +54,8 @@ func init() {
 }
 
 func prune(cmd *cobra.Command, args []string) error {
+	var err error
+
 	// Prompt for confirmation if --force is not set
 	if !force {
 		reader := bufio.NewReader(os.Stdin)
@@ -79,16 +80,11 @@ Are you sure you want to continue? [y/N] `, volumeString)
 		}
 	}
 
-	pruneOptions.ContainerPruneOptions = entities.ContainerPruneOptions{}
-	for _, f := range filters {
-		t := strings.SplitN(f, "=", 2)
-		pruneOptions.ContainerPruneOptions.Filters = make(url.Values)
-		if len(t) < 2 {
-			return errors.Errorf("filter input must be in the form of filter=value: %s is invalid", f)
-		}
-		pruneOptions.ContainerPruneOptions.Filters.Add(t[0], t[1])
+	pruneOptions.Filters, err = lpfilters.ParseFilterArgumentsIntoFilters(filters)
+	if err != nil {
+		return err
 	}
-	// TODO: support for filters in system prune
+
 	response, err := registry.ContainerEngine().SystemPrune(context.Background(), pruneOptions)
 	if err != nil {
 		return err

libpod/filters/helpers.go

@@ -0,0 +1,20 @@
+package lpfilters
+
+import (
+	"net/url"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+func ParseFilterArgumentsIntoFilters(filters []string) (url.Values, error) {
+	parsedFilters := make(url.Values)
+	for _, f := range filters {
+		t := strings.SplitN(f, "=", 2)
+		if len(t) < 2 {
+			return parsedFilters, errors.Errorf("filter input must be in the form of filter=value: %s is invalid", f)
+		}
+		parsedFilters.Add(t[0], t[1])
+	}
+	return parsedFilters, nil
+}

pkg/bindings/test/system_test.go

@@ -158,4 +158,61 @@ var _ = Describe("Podman system", func() {
 		// Volume should be pruned now as flag set true
 		Expect(len(systemPruneResponse.VolumePruneReport)).To(Equal(1))
 	})
+
+	It("podman system prune running alpine container volume prune --filter", func() {
+		// Start a pod and leave it running
+		_, err := pods.Start(bt.conn, newpod, nil)
+		Expect(err).To(BeNil())
+
+		// Start and stop a container to enter in exited state.
+		var name = "top"
+		_, err = bt.RunTopContainer(&name, bindings.PFalse, nil)
+		Expect(err).To(BeNil())
+		err = containers.Stop(bt.conn, name, nil)
+		Expect(err).To(BeNil())
+
+		// Start second container and leave in running
+		var name2 = "top2"
+		_, err = bt.RunTopContainer(&name2, bindings.PFalse, nil)
+		Expect(err).To(BeNil())
+
+		// Adding an unused volume should work
+		_, err = volumes.Create(bt.conn, entities.VolumeCreateOptions{}, nil)
+		Expect(err).To(BeNil())
+
+		// Adding an unused volume with label should work
+		_, err = volumes.Create(bt.conn, entities.VolumeCreateOptions{Label: map[string]string{
+			"label1": "value1",
+		}}, nil)
+		Expect(err).To(BeNil())
+
+		f := make(map[string][]string)
+		f["label"] = []string{"label1=idontmatch"}
+
+		options := new(system.PruneOptions).WithAll(true).WithVolumes(true).WithFilters(f)
+		systemPruneResponse, err := system.Prune(bt.conn, options)
+		Expect(err).To(BeNil())
+		Expect(len(systemPruneResponse.PodPruneReport)).To(Equal(0))
+		// TODO fix system filter handling so all components can handle filters
+		//      This check **should** be "Equal(0)" since we are passing label
+		//      filters however the Prune function doesn't seem to pass filters
+		//      to each component.
+		Expect(len(systemPruneResponse.ContainerPruneReport.ID)).To(Equal(1))
+		Expect(len(systemPruneResponse.ImagePruneReport.Report.Id)).
+			To(BeNumerically(">", 0))
+		// Alpine image should not be pruned as used by running container
+		Expect(systemPruneResponse.ImagePruneReport.Report.Id).
+			ToNot(ContainElement("docker.io/library/alpine:latest"))
+		// Volume shouldn't be pruned because the PruneOptions filters doesn't match
+		Expect(len(systemPruneResponse.VolumePruneReport)).To(Equal(0))
+
+		// Fix filter and re prune
+		f["label"] = []string{"label1=value1"}
+		options = new(system.PruneOptions).WithAll(true).WithVolumes(true).WithFilters(f)
+		systemPruneResponse, err = system.Prune(bt.conn, options)
+		Expect(err).To(BeNil())
+
+		// Volume should be pruned because the PruneOptions filters now match
+		Expect(len(systemPruneResponse.VolumePruneReport)).To(Equal(1))
+	})
 })

pkg/domain/entities/system.go

@@ -17,9 +17,9 @@ type ServiceOptions struct {
 
 // SystemPruneOptions provides options to prune system.
 type SystemPruneOptions struct {
-	All    bool
-	Volume bool
-	ContainerPruneOptions
+	All     bool
+	Volume  bool
+	Filters map[string][]string `json:"filters" schema:"filters"`
 }
 
 // SystemPruneReport provides report after system prune is executed.

pkg/domain/infra/abi/system.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"io/ioutil"
+	"net/url"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -180,7 +181,12 @@ func (ic *ContainerEngine) SystemPrune(ctx context.Context, options entities.Sys
 			found = true
 		}
 		systemPruneReport.PodPruneReport = append(systemPruneReport.PodPruneReport, podPruneReport...)
-		containerPruneReport, err := ic.ContainerPrune(ctx, options.ContainerPruneOptions)
+
+		// TODO: Figure out cleaner way to handle all of the different PruneOptions
+		containerPruneOptions := entities.ContainerPruneOptions{}
+		containerPruneOptions.Filters = (url.Values)(options.Filters)
+
+		containerPruneReport, err := ic.ContainerPrune(ctx, containerPruneOptions)
 		if err != nil {
 			return nil, err
 		}
@@ -217,7 +223,9 @@ func (ic *ContainerEngine) SystemPrune(ctx context.Context, options entities.Sys
 			systemPruneReport.ImagePruneReport.Report.Id = append(systemPruneReport.ImagePruneReport.Report.Id, results...)
 		}
 		if options.Volume {
-			volumePruneReport, err := ic.pruneVolumesHelper(ctx, nil)
+			volumePruneOptions := entities.VolumePruneOptions{}
+			volumePruneOptions.Filters = (url.Values)(options.Filters)
+			volumePruneReport, err := ic.VolumePrune(ctx, volumePruneOptions)
 			if err != nil {
 				return nil, err
 			}

pkg/domain/infra/tunnel/system.go

@@ -20,7 +20,7 @@ func (ic *ContainerEngine) SetupRootless(_ context.Context, cmd *cobra.Command)
 
 // SystemPrune prunes unused data from the system.
 func (ic *ContainerEngine) SystemPrune(ctx context.Context, opts entities.SystemPruneOptions) (*entities.SystemPruneReport, error) {
-	options := new(system.PruneOptions).WithAll(opts.All).WithVolumes(opts.Volume).WithFilters(opts.ContainerPruneOptions.Filters)
+	options := new(system.PruneOptions).WithAll(opts.All).WithVolumes(opts.Volume).WithFilters(opts.Filters)
 	return system.Prune(ic.ClientCxt, options)
 }
 

test/apiv2/45-system.at

@@ -0,0 +1,67 @@
+# -*- sh -*-
+#
+# system related tests
+#
+
+## ensure system is clean
+t POST 'libpod/system/prune?volumes=true&all=true' params='' 200
+
+## podman system df
+t GET system/df 200 '{"LayersSize":0,"Images":[],"Containers":[],"Volumes":[],"BuildCache":[],"BuilderSize":0}'
+t GET libpod/system/df 200 '{"Images":[],"Containers":[],"Volumes":[]}'
+
+# Create volume. We expect df to report this volume next invocation of system/df
+t GET libpod/info 200
+volumepath=$(jq -r ".store.volumePath" <<<"$output")
+t POST libpod/volumes/create name=foo1  201 \
+    .Name=foo1 \
+    .Driver=local \
+    .Mountpoint=$volumepath/foo1/_data \
+    .CreatedAt~[0-9]\\{4\\}-[0-9]\\{2\\}-[0-9]\\{2\\}.* \
+    .Labels={} \
+    .Options=null
+
+t GET system/df 200 '.Volumes[0].Name=foo1'
+
+t GET libpod/system/df 200 '.Volumes[0].VolumeName=foo1'
+
+# Create two more volumes to test pruneing
+t POST libpod/volumes/create \
+    '"Name":"foo2","Label":{"testlabel1":""},"Options":{"type":"tmpfs","o":"nodev,noexec"}}' 201 \
+    .Name=foo2 \
+    .Driver=local \
+    .Mountpoint=$volumepath/foo2/_data \
+    .CreatedAt~[0-9]\\{4\\}-[0-9]\\{2\\}-[0-9]\\{2\\}.* \
+    .Labels.testlabel1="" \
+    .Options.o=nodev,noexec
+
+t POST libpod/volumes/create \
+    '"Name":"foo3","Label":{"testlabel1":"testonly"},"Options":{"type":"tmpfs","o":"nodev,noexec"}}' 201 \
+    .Name=foo3 \
+    .Driver=local \
+    .Mountpoint=$volumepath/foo3/_data \
+    .CreatedAt~[0-9]\\{4\\}-[0-9]\\{2\\}-[0-9]\\{2\\}.* \
+    .Labels.testlabel1=testonly \
+    .Options.o=nodev,noexec
+
+t GET system/df 200 '.Volumes | length=3'
+t GET libpod/system/df 200 '.Volumes | length=3'
+
+# Prune volumes
+
+# -G --data-urlencode 'volumes=true&filters={"label":["testlabel1=idontmatch"]}'
+t POST 'libpod/system/prune?volumes=true&filters=%7B%22label%22:%5B%22testlabel1=idontmatch%22%5D%7D' params='' 200
+
+# nothing should have been pruned
+t GET system/df 200 '.Volumes | length=3'
+t GET libpod/system/df 200 '.Volumes | length=3'
+
+# -G --data-urlencode 'volumes=true&filters={"label":["testlabel1=testonly"]}'
+# only foo3 should be pruned because of filter
+t POST 'libpod/system/prune?volumes=true&filters=%7B%22label%22:%5B%22testlabel1=testonly%22%5D%7D' params='' 200 .VolumePruneReport[0].Id=foo3
+# only foo2 should be pruned because of filter
+t POST 'libpod/system/prune?volumes=true&filters=%7B%22label%22:%5B%22testlabel1%22%5D%7D' params='' 200 .VolumePruneReport[0].Id=foo2
+# foo1, the last remaining volume should be pruned without any filters applied
+t POST 'libpod/system/prune?volumes=true' params='' 200 .VolumePruneReport[0].Id=foo1
+
+# TODO add other system prune tests for pods / images

test/e2e/prune_test.go

@@ -349,4 +349,64 @@ var _ = Describe("Podman prune", func() {
 		// all images are unused, so they all should be deleted!
 		Expect(len(images.OutputToStringArray())).To(Equal(len(CACHE_IMAGES)))
 	})
+
+	It("podman system prune --volumes --filter", func() {
+		session := podmanTest.Podman([]string{"volume", "create", "--label", "label1=value1", "myvol1"})
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+
+		session = podmanTest.Podman([]string{"volume", "create", "--label", "sharedlabel1=slv1", "myvol2"})
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+
+		session = podmanTest.Podman([]string{"volume", "create", "--label", "sharedlabel1=slv2", "myvol3"})
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+
+		session = podmanTest.Podman([]string{"volume", "create", "--label", "sharedlabel1", "myvol4"})
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+
+		session = podmanTest.Podman([]string{"create", "-v", "myvol5:/myvol5", ALPINE, "ls"})
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+
+		session = podmanTest.Podman([]string{"create", "-v", "myvol6:/myvol6", ALPINE, "ls"})
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+
+		session = podmanTest.Podman([]string{"volume", "ls"})
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+		Expect(len(session.OutputToStringArray())).To(Equal(7))
+
+		session = podmanTest.Podman([]string{"system", "prune", "--force", "--volumes", "--filter", "label=label1=value1"})
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+
+		session = podmanTest.Podman([]string{"volume", "ls"})
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+		Expect(len(session.OutputToStringArray())).To(Equal(6))
+
+		session = podmanTest.Podman([]string{"system", "prune", "--force", "--volumes", "--filter", "label=sharedlabel1=slv1"})
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+
+		session = podmanTest.Podman([]string{"volume", "ls"})
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+		Expect(len(session.OutputToStringArray())).To(Equal(5))
+
+		session = podmanTest.Podman([]string{"system", "prune", "--force", "--volumes", "--filter", "label=sharedlabel1"})
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+
+		session = podmanTest.Podman([]string{"volume", "ls"})
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+		Expect(len(session.OutputToStringArray())).To(Equal(3))
+
+		podmanTest.Cleanup()
+	})
 })