Fix handling of quota on volumes

This patch fixes the handling of inodes and sizes, currently if
user sets indoes and sizes together, quota is only set on sizes.

Second problem with quota is that we have to have unigue projectids
for each directory. Originally container/storage only did quota on
rootfs, now we want to support it on volumes as well. We need to be
able to get unigue projectids for these two different parent
directories. The added function, attempts to maintain at least 10,000
unigue id's based on the inode of the parent directory. I know that this
is not perfect and we have a potential for overlay.  If you have a
better algorythm, I would love to use it.

Signed-off-by: Daniel J Walsh <dwalsh@redhat.com>

docs/containers-storage.conf.5.md

@@ -263,13 +263,45 @@ The semanage command above tells SELinux to setup the default labeling of `NEWST
 
 Now all new content created in these directories will automatically be created with the correct label.
 
-## SEE ALSO
-`semanage(8)`, `restorecon(8)`, `mount(8)`, `fuse-overlayfs(1)`
+## QUOTAS
+
+Container storage implements `XFS project quota controls` for overlay storage
+containers and volumes. The directory used to store the containers must be an
+`XFS` file system and be mounted with the `pquota` option.
+
+Example /etc/fstab entry:
+```
+/dev/podman/podman-var /var xfs defaults,x-systemd.device-timeout=0,pquota 1 2
+```
+
+Container storage generates project ids for each container and builtin volume, but these project ids need to be unique for the XFS file system.
+
+The xfs_quota tool can be used to assign a project id to the storage driver directory, e.g.:
+
+```
+echo 100000:/var/lib/containers/storage/overlay >> /etc/projects
+echo 200000:/var/lib/containers/storage/volumes >> /etc/projects
+echo storage:100000 >> /etc/projid
+echo volumes:200000 >> /etc/projid
+xfs_quota -x -c 'project -s storage volumes' /<xfs mount point>
+```
+
+In the example above, the storage directory project id will be used as a "start offset"
+and all containers will be assigned larger project ids (e.g. >= 100000).
+Then the volumes directory project id will be used as a "start offset"
+and all volumes will be assigned larger project ids (e.g. >= 200000).
+This is a way to prevent xfs_quota management from conflicting with containers/storage.
 
 ## FILES
 
 Distributions often provide a `/usr/share/containers/storage.conf` file to define default storage configuration. Administrators can override this file by creating `/etc/containers/storage.conf` to specify their own configuration. The storage.conf file for rootless users is stored in the `$XDG_CONFIG_HOME/containers/storage.conf` file.  If `$XDG_CONFIG_HOME` is not set then the file `$HOME/.config/containers/storage.conf` is used.
 
+/etc/projects - XFS persistent project root definition
+/etc/projid -  XFS project name mapping file
+
+## SEE ALSO
+`semanage(8)`, `restorecon(8)`, `mount(8)`, `fuse-overlayfs(1)`, `xfs_quota(8)`, `projects(5)`, `projid(5)`
+
 ## HISTORY
 May 2017, Originally compiled by Dan Walsh <dwalsh@redhat.com>
 Format copied from crio.conf man page created by Aleksa Sarai <asarai@suse.de>

drivers/quota/projectquota.go

@@ -52,15 +52,20 @@ import "C"
 import (
 	"fmt"
 	"io/ioutil"
+	"math"
+	"os"
 	"path"
 	"path/filepath"
+	"syscall"
 	"unsafe"
 
 	"github.com/containers/storage/pkg/directory"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
 )
 
+const projectIDsAllocatedPerQuotaHome = 10000
+
 // Quota limit params - currently we only control blocks hard limit and inodes
 type Quota struct {
 	Size   uint64
@@ -75,23 +80,48 @@ type Control struct {
 	quotas            map[string]uint32
 }
 
+// Attempt to generate a unigue projectid.  Multiple directories
+// per file system can have quota and they need a group of unique
+// ids. This function attempts to allocate at least projectIDsAllocatedPerQuotaHome(10000)
+// unique projectids, based on the inode of the basepath.
+func generateUniqueProjectID(path string) (uint32, error) {
+	fileinfo, err := os.Stat(path)
+	if err != nil {
+		return 0, err
+	}
+	stat, ok := fileinfo.Sys().(*syscall.Stat_t)
+	if !ok {
+		return 0, fmt.Errorf("Not a syscall.Stat_t %s", path)
+
+	}
+	projectID := projectIDsAllocatedPerQuotaHome + (stat.Ino*projectIDsAllocatedPerQuotaHome)%(math.MaxUint32-projectIDsAllocatedPerQuotaHome)
+	return uint32(projectID), nil
+}
+
 // NewControl - initialize project quota support.
 // Test to make sure that quota can be set on a test dir and find
 // the first project id to be used for the next container create.
 //
 // Returns nil (and error) if project quota is not supported.
 //
-// First get the project id of the home directory.
+// First get the project id of the basePath directory.
 // This test will fail if the backing fs is not xfs.
 //
 // xfs_quota tool can be used to assign a project id to the driver home directory, e.g.:
-//    echo 999:/var/lib/containers/storage/overlay >> /etc/projects
-//    echo storage:999 >> /etc/projid
-//    xfs_quota -x -c 'project -s storage' /<xfs mount point>
+//    echo 100000:/var/lib/containers/storage/overlay >> /etc/projects
+//    echo 200000:/var/lib/containers/storage/volumes >> /etc/projects
+//    echo storage:100000 >> /etc/projid
+//    echo volumes:200000 >> /etc/projid
+//    xfs_quota -x -c 'project -s storage volumes' /<xfs mount point>
 //
-// In that case, the home directory project id will be used as a "start offset"
-// and all containers will be assigned larger project ids (e.g. >= 1000).
-// This is a way to prevent xfs_quota management from conflicting with containers/storage.
+// In the example above, the storage directory project id will be used as a
+// "start offset" and all containers will be assigned larger project ids
+// (e.g. >= 100000). Then the volumes directory project id will be used as a
+// "start offset" and all volumes will be assigned larger project ids
+// (e.g. >= 200000).
+// This is a way to prevent xfs_quota management from conflicting with
+// containers/storage.
+
 //
 // Then try to create a test directory with the next project id and set a quota
 // on it. If that works, continue to scan existing containers to map allocated
@@ -105,8 +135,15 @@ func NewControl(basePath string) (*Control, error) {
 	if err != nil {
 		return nil, err
 	}
-	minProjectID++
+	if minProjectID == 0 {
+		// Indicates the storage was never initialized
+		// Generate a unique range of Projectids for this basepath
+		minProjectID, err = generateUniqueProjectID(basePath)
+		if err != nil {
+			return nil, err
+		}
 
+	}
 	//
 	// create backing filesystem device node
 	//
@@ -180,12 +217,12 @@ func setProjectQuota(backingFsBlockDev string, projectID uint32, quota Quota) er
 	d.d_flags = C.FS_PROJ_QUOTA
 
 	if quota.Size > 0 {
-		d.d_fieldmask = C.FS_DQ_BHARD | C.FS_DQ_BSOFT
+		d.d_fieldmask = d.d_fieldmask | C.FS_DQ_BHARD | C.FS_DQ_BSOFT
 		d.d_blk_hardlimit = C.__u64(quota.Size / 512)
 		d.d_blk_softlimit = d.d_blk_hardlimit
 	}
 	if quota.Inodes > 0 {
-		d.d_fieldmask = C.FS_DQ_IHARD | C.FS_DQ_ISOFT
+		d.d_fieldmask = d.d_fieldmask | C.FS_DQ_IHARD | C.FS_DQ_ISOFT
 		d.d_ino_hardlimit = C.__u64(quota.Inodes)
 		d.d_ino_softlimit = d.d_ino_hardlimit
 	}