Merge pull request containers#17688 from mheon/add_sqlite_ci

Add SQLite job to CI

.cirrus.yml

@@ -55,6 +55,7 @@ env:
     DISTRO_NV:               # any {PRIOR_,}{FEDORA,DEBIAN}_NAME value
     VM_IMAGE_NAME:           # One of the "Google-cloud VM Images" (above)
     CTR_FQIN:                # One of the "Container FQIN's" (above)
+    CI_DESIRED_DATABASE: boltdb # One of "", "sqlite", "boltdb"
 
     # Curl-command prefix for downloading task artifacts, simply add the
     # the url-encoded task name, artifact name, and path as a suffix.
@@ -103,6 +104,16 @@ build_task:
               # ID for re-use of build output
               CI_DESIRED_RUNTIME: crun
               CI_DESIRED_NETWORK: netavark
+              CI_DESIRED_DATABASE: boltdb
+        - env: &sqliteenvvars
+              DISTRO_NV: ${FEDORA_NAME}
+              # Not used here, is used in other tasks
+              VM_IMAGE_NAME: ${FEDORA_CACHE_IMAGE_NAME}
+              CTR_FQIN: ${FEDORA_CONTAINER_FQIN}
+              # ID for re-use of build output
+              CI_DESIRED_RUNTIME: crun
+              CI_DESIRED_NETWORK: netavark
+              CI_DESIRED_DATABASE: sqlite
         - env: &priorfedora_envvars
               DISTRO_NV: ${PRIOR_FEDORA_NAME}
               VM_IMAGE_NAME: ${PRIOR_FEDORA_CACHE_IMAGE_NAME}
@@ -553,7 +564,7 @@ windows_smoke_test_task:
 local_integration_test_task: &local_integration_test_task
     # Integration-test task name convention:
     # <int.|sys.> <podman|remote> <Distro NV> <root|rootless>
-    name: &std_name_fmt "$TEST_FLAVOR $PODBIN_NAME $DISTRO_NV $PRIV_NAME $TEST_ENVIRON"
+    name: &std_name_fmt "$TEST_FLAVOR $PODBIN_NAME $DISTRO_NV $PRIV_NAME $TEST_ENVIRON ${CI_DESIRED_DATABASE}"
     alias: local_integration_test
     # Docs: ./contrib/cirrus/CIModes.md
     only_if: *not_tag_branch_build_docs

contrib/cirrus/lib.sh

@@ -93,7 +93,7 @@ EPOCH_TEST_COMMIT="$CIRRUS_BASE_SHA"
 # contexts, such as host->container or root->rootless user
 #
 # List of envariables which must be EXACT matches
-PASSTHROUGH_ENV_EXACT='CGROUP_MANAGER|DEST_BRANCH|DISTRO_NV|GOCACHE|GOPATH|GOSRC|NETWORK_BACKEND|OCI_RUNTIME|ROOTLESS_USER|SCRIPT_BASE|SKIP_USERNS|EC2_INST_TYPE'
+PASSTHROUGH_ENV_EXACT='CGROUP_MANAGER|DEST_BRANCH|DISTRO_NV|GOCACHE|GOPATH|GOSRC|NETWORK_BACKEND|OCI_RUNTIME|ROOTLESS_USER|SCRIPT_BASE|SKIP_USERNS|EC2_INST_TYPE|PODMAN_DB'
 
 # List of envariable patterns which must match AT THE BEGINNING of the name.
 PASSTHROUGH_ENV_ATSTART='CI|TEST'

contrib/cirrus/setup_environment.sh

@@ -142,6 +142,26 @@ case "$CI_DESIRED_NETWORK" in
     *)          die_unknown CI_DESIRED_NETWORK ;;
 esac
 
+# Database: force SQLite or BoltDB as requested in .cirrus.yml.
+# If unset, will default to BoltDB.
+# shellcheck disable=SC2154
+case "$CI_DESIRED_DATABASE" in
+    sqlite)
+        warn "Forcing PODMAN_DB=sqlite"
+        echo "PODMAN_DB=sqlite" >> /etc/ci_environment
+	;;
+    boltdb)
+        warn "Forcing PODMAN_DB=boltdb"
+        echo "PODMAN_DB=boltdb" >> /etc/ci_environment
+	;;
+    "")
+        warn "Using default Podman database"
+        ;;
+    *)
+        die_unknown CI_DESIRED_DATABASE
+        ;;
+esac
+
 # Required to be defined by caller: The environment where primary testing happens
 # shellcheck disable=SC2154
 case "$TEST_ENVIRON" in

libpod/container_internal_common.go

@@ -2745,7 +2745,7 @@ func (c *Container) fixVolumePermissions(v *ContainerNamedVolume) error {
 
 	// Volumes owned by a volume driver are not chowned - we don't want to
 	// mess with a mount not managed by us.
-	if vol.state.NeedsChown && !vol.UsesVolumeDriver() {
+	if vol.state.NeedsChown && (!vol.UsesVolumeDriver() && vol.config.Driver != "image") {
 		vol.state.NeedsChown = false
 
 		uid := int(c.config.Spec.Process.User.UID)

libpod/runtime_volume_common.go

@@ -120,7 +120,9 @@ func (r *Runtime) newVolume(ctx context.Context, noCreatePluginVolume bool, opti
 		volume.config.StorageImageID = image.ID()
 
 		// Create a backing container in c/storage.
-		storageConfig := storage.ContainerOptions{}
+		storageConfig := storage.ContainerOptions{
+			LabelOpts: []string{"filetype:container_file_t", "level:s0"},
+		}
 		if len(volume.config.MountLabel) > 0 {
 			context, err := selinux.NewContext(volume.config.MountLabel)
 			if err != nil {

libpod/sqlite_state.go

@@ -4,6 +4,7 @@ import (
 	"database/sql"
 	"errors"
 	"fmt"
+	"os"
 	"path/filepath"
 	goruntime "runtime"
 	"strings"
@@ -32,7 +33,19 @@ type SQLiteState struct {
 func NewSqliteState(runtime *Runtime) (_ State, defErr error) {
 	state := new(SQLiteState)
 
-	conn, err := sql.Open("sqlite3", filepath.Join(runtime.storageConfig.GraphRoot, "db.sql?_loc=auto"))
+	basePath := runtime.storageConfig.GraphRoot
+	if runtime.storageConfig.TransientStore {
+		basePath = runtime.storageConfig.RunRoot
+	}
+
+	// c/storage is set up *after* the DB - so even though we use the c/s
+	// root (or, for transient, runroot) dir, we need to make the dir
+	// ourselves.
+	if err := os.MkdirAll(basePath, 0700); err != nil {
+		return nil, fmt.Errorf("creating root directory: %w", err)
+	}
+
+	conn, err := sql.Open("sqlite3", filepath.Join(basePath, "db.sql?_loc=auto"))
 	if err != nil {
 		return nil, fmt.Errorf("initializing sqlite database: %w", err)
 	}
@@ -1482,7 +1495,7 @@ func (s *SQLiteState) AddPod(pod *Pod) (defErr error) {
 			return fmt.Errorf("checking if pod name %s exists in database: %w", pod.Name(), err)
 		}
 	} else if check != 0 {
-		return fmt.Errorf("name \"%s\" is in use: %w", pod.Name(), define.ErrPodExists)
+		return fmt.Errorf("name %q is in use: %w", pod.Name(), define.ErrPodExists)
 	}
 
 	if _, err := tx.Exec("INSERT INTO IDNamespace VALUES (?);", pod.ID()); err != nil {
@@ -1816,6 +1829,17 @@ func (s *SQLiteState) AddVolume(volume *Volume) (defErr error) {
 		}
 	}()
 
+	// TODO: There has to be a better way of doing this
+	var check int
+	row := tx.QueryRow("SELECT 1 FROM VolumeConfig WHERE Name=?;", volume.Name())
+	if err := row.Scan(&check); err != nil {
+		if !errors.Is(err, sql.ErrNoRows) {
+			return fmt.Errorf("checking if volume name %s exists in database: %w", volume.Name(), err)
+		}
+	} else if check != 0 {
+		return fmt.Errorf("name %q is in use: %w", volume.Name(), define.ErrVolumeExists)
+	}
+
 	if _, err := tx.Exec("INSERT INTO VolumeConfig VALUES (?, ?, ?);", volume.Name(), storageID, cfgJSON); err != nil {
 		return fmt.Errorf("adding volume %s config to database: %w", volume.Name(), err)
 	}

libpod/sqlite_state_internal.go

@@ -295,6 +295,21 @@ func finalizeVolumeSqlite(vol *Volume) error {
 	}
 	vol.lock = lock
 
+	// Retrieve volume driver
+	if vol.UsesVolumeDriver() {
+		plugin, err := vol.runtime.getVolumePlugin(vol.config)
+		if err != nil {
+			// We want to fail gracefully here, to ensure that we
+			// can still remove volumes even if their plugin is
+			// missing. Otherwise, we end up with volumes that
+			// cannot even be retrieved from the database and will
+			// cause things like `volume ls` to fail.
+			logrus.Errorf("Volume %s uses volume plugin %s, but it cannot be accessed - some functionality may not be available: %v", vol.Name(), vol.config.Driver, err)
+		} else {
+			vol.plugin = plugin
+		}
+	}
+
 	vol.valid = true
 
 	return nil
@@ -374,6 +389,17 @@ func (s *SQLiteState) addContainer(ctr *Container) (defErr error) {
 		}
 	}()
 
+	// TODO: There has to be a better way of doing this
+	var check int
+	row := tx.QueryRow("SELECT 1 FROM ContainerConfig WHERE Name=?;", ctr.Name())
+	if err := row.Scan(&check); err != nil {
+		if !errors.Is(err, sql.ErrNoRows) {
+			return fmt.Errorf("checking if container name %s exists in database: %w", ctr.Name(), err)
+		}
+	} else if check != 0 {
+		return fmt.Errorf("name %q is in use: %w", ctr.Name(), define.ErrCtrExists)
+	}
+
 	if _, err := tx.Exec("INSERT INTO IDNamespace VALUES (?);", ctr.ID()); err != nil {
 		return fmt.Errorf("adding container id to database: %w", err)
 	}
@@ -502,7 +528,7 @@ func (s *SQLiteState) networkModify(ctr *Container, network string, opts types.P
 
 	_, ok := newCfg.Networks[network]
 	if new && ok {
-		return fmt.Errorf("container %s is already connected to network %s: %w", ctr.ID(), network, define.ErrNoSuchNetwork)
+		return fmt.Errorf("container %s is already connected to network %s: %w", ctr.ID(), network, define.ErrNetworkConnected)
 	}
 	if !ok && (!new || disconnect) {
 		return fmt.Errorf("container %s is not connected to network %s: %w", ctr.ID(), network, define.ErrNoSuchNetwork)

libpod/volume.go

@@ -136,7 +136,7 @@ func (v *Volume) Labels() map[string]string {
 // MountPoint returns the volume's mountpoint on the host
 func (v *Volume) MountPoint() (string, error) {
 	// For the sake of performance, avoid locking unless we have to.
-	if v.UsesVolumeDriver() {
+	if v.UsesVolumeDriver() || v.config.Driver == define.VolumeDriverImage {
 		v.lock.Lock()
 		defer v.lock.Unlock()
 

pkg/specgenutil/util.go

@@ -282,6 +282,7 @@ func CreateExitCommandArgs(storageConfig storageTypes.StoreOptions, config *conf
 		"--network-config-dir", config.Network.NetworkConfigDir,
 		"--network-backend", config.Network.NetworkBackend,
 		"--volumepath", config.Engine.VolumePath,
+		"--db-backend", config.Engine.DBBackend,
 		fmt.Sprintf("--transient-store=%t", storageConfig.TransientStore),
 	}
 	if config.Engine.OCIRuntime != "" {

test/e2e/common_test.go

@@ -249,6 +249,11 @@ func PodmanTestCreateUtil(tempDir string, remote bool) *PodmanTestIntegration {
 	}
 	os.Setenv("DISABLE_HC_SYSTEMD", "true")
 
+	dbBackend := "boltdb"
+	if os.Getenv("PODMAN_DB") == "sqlite" {
+		dbBackend = "sqlite"
+	}
+
 	networkBackend := CNI
 	networkConfigDir := "/etc/cni/net.d"
 	if isRootless() {
@@ -291,6 +296,7 @@ func PodmanTestCreateUtil(tempDir string, remote bool) *PodmanTestIntegration {
 			ImageCacheFS:       storageFs,
 			ImageCacheDir:      ImageCacheDir,
 			NetworkBackend:     networkBackend,
+			DatabaseBackend:    dbBackend,
 		},
 		ConmonBinary:        conmonBinary,
 		QuadletBinary:       quadletBinary,
@@ -918,8 +924,8 @@ func (p *PodmanTestIntegration) makeOptions(args []string, noEvents, noCache boo
 		eventsType = "none"
 	}
 
-	podmanOptions := strings.Split(fmt.Sprintf("%s--root %s --runroot %s --runtime %s --conmon %s --network-config-dir %s --network-backend %s --cgroup-manager %s --tmpdir %s --events-backend %s",
-		debug, p.Root, p.RunRoot, p.OCIRuntime, p.ConmonBinary, p.NetworkConfigDir, p.NetworkBackend.ToString(), p.CgroupManager, p.TmpDir, eventsType), " ")
+	podmanOptions := strings.Split(fmt.Sprintf("%s--root %s --runroot %s --runtime %s --conmon %s --network-config-dir %s --network-backend %s --cgroup-manager %s --tmpdir %s --events-backend %s --db-backend %s",
+		debug, p.Root, p.RunRoot, p.OCIRuntime, p.ConmonBinary, p.NetworkConfigDir, p.NetworkBackend.ToString(), p.CgroupManager, p.TmpDir, eventsType, p.DatabaseBackend), " ")
 
 	podmanOptions = append(podmanOptions, strings.Split(p.StorageOptions, " ")...)
 	if !noCache {

test/e2e/libpod_suite_remote_test.go

@@ -134,8 +134,8 @@ func (p *PodmanTestIntegration) StopRemoteService() {
 // MakeOptions assembles all the podman main options
 func getRemoteOptions(p *PodmanTestIntegration, args []string) []string {
 	networkDir := p.NetworkConfigDir
-	podmanOptions := strings.Split(fmt.Sprintf("--root %s --runroot %s --runtime %s --conmon %s --network-config-dir %s --network-backend %s --cgroup-manager %s",
-		p.Root, p.RunRoot, p.OCIRuntime, p.ConmonBinary, networkDir, p.NetworkBackend.ToString(), p.CgroupManager), " ")
+	podmanOptions := strings.Split(fmt.Sprintf("--root %s --runroot %s --runtime %s --conmon %s --network-config-dir %s --network-backend %s --cgroup-manager %s --db-backend %s",
+		p.Root, p.RunRoot, p.OCIRuntime, p.ConmonBinary, networkDir, p.NetworkBackend.ToString(), p.CgroupManager, p.DatabaseBackend), " ")
 	podmanOptions = append(podmanOptions, strings.Split(p.StorageOptions, " ")...)
 	podmanOptions = append(podmanOptions, args...)
 	return podmanOptions

test/e2e/pod_create_test.go

@@ -76,20 +76,6 @@ var _ = Describe("Podman pod create", func() {
 		Expect(check.OutputToStringArray()).To(HaveLen(1))
 	})
 
-	It("podman create pod with same name as ctr", func() {
-		name := "test"
-		session := podmanTest.Podman([]string{"create", "--name", name, ALPINE, "ls"})
-		session.WaitWithDefaultTimeout()
-		Expect(session).Should(Exit(0))
-
-		_, ec, _ := podmanTest.CreatePod(map[string][]string{"--name": {name}})
-		Expect(ec).To(Not(Equal(0)))
-
-		check := podmanTest.Podman([]string{"pod", "ps", "-q"})
-		check.WaitWithDefaultTimeout()
-		Expect(check.OutputToStringArray()).To(BeEmpty())
-	})
-
 	It("podman create pod without network portbindings", func() {
 		name := "test"
 		session := podmanTest.Podman([]string{"pod", "create", "--name", name})

test/e2e/prune_test.go

@@ -563,7 +563,6 @@ var _ = Describe("Podman prune", func() {
 		SkipIfRemote("Can't drop database while daemon running")
 
 		containerStorageDir := filepath.Join(podmanTest.Root, podmanTest.ImageCacheFS+"-containers")
-		dbDir := filepath.Join(podmanTest.Root, "libpod")
 
 		// Create container 1
 		create := podmanTest.Podman([]string{"create", "--name", "test", BB})
@@ -580,8 +579,15 @@ var _ = Describe("Podman prune", func() {
 		// Drop podman database and storage, losing track of container 1 (but directory remains)
 		err = os.Remove(filepath.Join(containerStorageDir, "containers.json"))
 		Expect(err).ToNot(HaveOccurred())
-		err = os.RemoveAll(dbDir)
-		Expect(err).ToNot(HaveOccurred())
+
+		if podmanTest.DatabaseBackend == "sqlite" {
+			err = os.Remove(filepath.Join(podmanTest.Root, "db.sql"))
+			Expect(err).ToNot(HaveOccurred())
+		} else {
+			dbDir := filepath.Join(podmanTest.Root, "libpod")
+			err = os.RemoveAll(dbDir)
+			Expect(err).ToNot(HaveOccurred())
+		}
 
 		Expect(podmanTest.NumberOfContainers()).To(Equal(0))
 

test/e2e/run_transient_test.go

@@ -54,8 +54,14 @@ var _ = Describe("Podman run with volumes", func() {
 		Expect(filepath.Join(containerStorageDir, "volatile-containers.json")).Should(Not(BeAnExistingFile()))
 		Expect(filepath.Join(runContainerStorageDir, "containers.json")).Should(Not(BeAnExistingFile()))
 		Expect(filepath.Join(runContainerStorageDir, "volatile-containers.json")).Should(Not(BeAnExistingFile()))
-		Expect(filepath.Join(dbDir, "bolt_state.db")).Should(BeARegularFile())
-		Expect(filepath.Join(runDBDir, "bolt_state.db")).Should(Not(BeAnExistingFile()))
+
+		if podmanTest.DatabaseBackend == "sqlite" {
+			Expect(filepath.Join(podmanTest.Root, "db.sql")).Should(BeARegularFile())
+			Expect(filepath.Join(podmanTest.RunRoot, "db.sql")).Should(Not(BeAnExistingFile()))
+		} else {
+			Expect(filepath.Join(dbDir, "bolt_state.db")).Should(BeARegularFile())
+			Expect(filepath.Join(runDBDir, "bolt_state.db")).Should(Not(BeAnExistingFile()))
+		}
 	})
 
 	It("podman run --rm with no transient-store", func() {
@@ -68,8 +74,14 @@ var _ = Describe("Podman run with volumes", func() {
 		Expect(filepath.Join(containerStorageDir, "volatile-containers.json")).Should(BeARegularFile())
 		Expect(filepath.Join(runContainerStorageDir, "containers.json")).Should(Not(BeAnExistingFile()))
 		Expect(filepath.Join(runContainerStorageDir, "volatile-containers.json")).Should(Not(BeAnExistingFile()))
-		Expect(filepath.Join(dbDir, "bolt_state.db")).Should(BeARegularFile())
-		Expect(filepath.Join(runDBDir, "bolt_state.db")).Should(Not(BeAnExistingFile()))
+
+		if podmanTest.DatabaseBackend == "sqlite" {
+			Expect(filepath.Join(podmanTest.Root, "db.sql")).Should(BeARegularFile())
+			Expect(filepath.Join(podmanTest.RunRoot, "db.sql")).Should(Not(BeAnExistingFile()))
+		} else {
+			Expect(filepath.Join(dbDir, "bolt_state.db")).Should(BeARegularFile())
+			Expect(filepath.Join(runDBDir, "bolt_state.db")).Should(Not(BeAnExistingFile()))
+		}
 	})
 
 	It("podman run --transient-store", func() {
@@ -83,8 +95,14 @@ var _ = Describe("Podman run with volumes", func() {
 		Expect(filepath.Join(containerStorageDir, "volatile-containers.json")).Should(Not(BeAnExistingFile()))
 		Expect(filepath.Join(runContainerStorageDir, "containers.json")).Should(Not(BeAnExistingFile()))
 		Expect(filepath.Join(runContainerStorageDir, "volatile-containers.json")).Should(BeARegularFile())
-		Expect(filepath.Join(dbDir, "bolt_state.db")).Should(Not(BeAnExistingFile()))
-		Expect(filepath.Join(runDBDir, "bolt_state.db")).Should(BeARegularFile())
+
+		if podmanTest.DatabaseBackend == "sqlite" {
+			Expect(filepath.Join(podmanTest.Root, "db.sql")).Should(Not(BeAnExistingFile()))
+			Expect(filepath.Join(podmanTest.RunRoot, "db.sql")).Should(BeARegularFile())
+		} else {
+			Expect(filepath.Join(dbDir, "bolt_state.db")).Should(Not(BeAnExistingFile()))
+			Expect(filepath.Join(runDBDir, "bolt_state.db")).Should(BeARegularFile())
+		}
 	})
 
 })

test/utils/utils.go

@@ -65,6 +65,7 @@ type PodmanTest struct {
 	ImageCacheDir      string
 	ImageCacheFS       string
 	NetworkBackend     NetworkBackend
+	DatabaseBackend    string
 	PodmanBinary       string
 	PodmanMakeOptions  func(args []string, noEvents, noCache bool) []string
 	RemoteCommand      *exec.Cmd