Add support for all-projects to incus image list and API

client/incus_images.go

@@ -34,6 +34,26 @@ func (r *ProtocolIncus) GetImages() ([]api.Image, error) {
 	return images, nil
 }
 
+// GetImagesAllProjects returns a list of images across all projects as Image structs.
+func (r *ProtocolIncus) GetImagesAllProjects() ([]api.Image, error) {
+	images := []api.Image{}
+
+	v := url.Values{}
+	v.Set("recursion", "1")
+	v.Set("all-projects", "true")
+
+	if !r.HasExtension("images_all_projects") {
+		return nil, fmt.Errorf("The server is missing the required \"images_all_projects\" API extension")
+	}
+
+	_, err := r.queryStruct("GET", fmt.Sprintf("/images?%s", v.Encode()), nil, "", &images)
+	if err != nil {
+		return nil, err
+	}
+
+	return images, nil
+}
+
 // GetImagesWithFilter returns a filtered list of available images as Image structs.
 func (r *ProtocolIncus) GetImagesWithFilter(filters []string) ([]api.Image, error) {
 	if !r.HasExtension("api_filtering") {

client/interfaces.go

@@ -48,6 +48,7 @@ type ImageServer interface {
 
 	// Image handling functions
 	GetImages() (images []api.Image, err error)
+	GetImagesAllProjects() (images []api.Image, err error)
 	GetImageFingerprints() (fingerprints []string, err error)
 	GetImagesWithFilter(filters []string) (images []api.Image, err error)
 

client/simplestreams_images.go

@@ -24,6 +24,11 @@ func (r *ProtocolSimpleStreams) GetImages() ([]api.Image, error) {
 	return r.ssClient.ListImages()
 }
 
+// GetImagesAllProjects returns a list of available images as Image structs.
+func (r *ProtocolSimpleStreams) GetImagesAllProjects() ([]api.Image, error) {
+	return r.GetImages()
+}
+
 // GetImageFingerprints returns a list of available image fingerprints.
 func (r *ProtocolSimpleStreams) GetImageFingerprints() ([]string, error) {
 	// Get all the images from simplestreams

cmd/incus/image.go

@@ -1061,8 +1061,9 @@ type cmdImageList struct {
 	global *cmdGlobal
 	image  *cmdImage
 
-	flagFormat  string
-	flagColumns string
+	flagFormat      string
+	flagColumns     string
+	flagAllProjects bool
 }
 
 func (c *cmdImageList) Command() *cobra.Command {
@@ -1090,13 +1091,15 @@ Column shorthand chars:
     F - Fingerprint (long)
     p - Whether image is public
     d - Description
+    e - Project
     a - Architecture
     s - Size
     u - Upload date
     t - Type`))
 
-	cmd.Flags().StringVarP(&c.flagColumns, "columns", "c", "lfpdatsu", i18n.G("Columns")+"``")
+	cmd.Flags().StringVarP(&c.flagColumns, "columns", "c", defaultImagesColumns, i18n.G("Columns")+"``")
 	cmd.Flags().StringVarP(&c.flagFormat, "format", "f", "table", i18n.G("Format (csv|json|table|yaml|compact)")+"``")
+	cmd.Flags().BoolVar(&c.flagAllProjects, "all-projects", false, i18n.G("Display images from all projects"))
 	cmd.RunE = c.Run
 
 	cmd.ValidArgsFunction = func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
@@ -1110,23 +1113,33 @@ Column shorthand chars:
 	return cmd
 }
 
+const defaultImagesColumns = "lfpdatsu"
+const defaultImagesColumnsAllProjects = "elfpdatsu"
+
 func (c *cmdImageList) parseColumns() ([]imageColumn, error) {
 	columnsShorthandMap := map[rune]imageColumn{
-		'l': {i18n.G("ALIAS"), c.aliasColumnData},
-		'L': {i18n.G("ALIASES"), c.aliasesColumnData},
+		'a': {i18n.G("ARCHITECTURE"), c.architectureColumnData},
+		'd': {i18n.G("DESCRIPTION"), c.descriptionColumnData},
+		'e': {i18n.G("PROJECT"), c.projectColumnData},
 		'f': {i18n.G("FINGERPRINT"), c.fingerprintColumnData},
 		'F': {i18n.G("FINGERPRINT"), c.fingerprintFullColumnData},
+		'l': {i18n.G("ALIAS"), c.aliasColumnData},
+		'L': {i18n.G("ALIASES"), c.aliasesColumnData},
 		'p': {i18n.G("PUBLIC"), c.publicColumnData},
-		'd': {i18n.G("DESCRIPTION"), c.descriptionColumnData},
-		'a': {i18n.G("ARCHITECTURE"), c.architectureColumnData},
 		's': {i18n.G("SIZE"), c.sizeColumnData},
-		'u': {i18n.G("UPLOAD DATE"), c.uploadDateColumnData},
 		't': {i18n.G("TYPE"), c.typeColumnData},
+		'u': {i18n.G("UPLOAD DATE"), c.uploadDateColumnData},
 	}
 
 	columnList := strings.Split(c.flagColumns, ",")
 
 	columns := []imageColumn{}
+	// Add project column if --all-projects flag specified and
+	// no --c was passed
+	if c.flagAllProjects && c.flagColumns == defaultImagesColumns {
+		c.flagColumns = defaultImagesColumnsAllProjects
+	}
+
 	for _, columnEntry := range columnList {
 		if columnEntry == "" {
 			return nil, fmt.Errorf(i18n.G("Empty column entry (redundant, leading or trailing command) in '%s'"), c.flagColumns)
@@ -1184,6 +1197,10 @@ func (c *cmdImageList) descriptionColumnData(image api.Image) string {
 	return c.findDescription(image.Properties)
 }
 
+func (c *cmdImageList) projectColumnData(image api.Image) string {
+	return image.Project
+}
+
 func (c *cmdImageList) architectureColumnData(image api.Image) string {
 	return image.Architecture
 }
@@ -1338,15 +1355,22 @@ func (c *cmdImageList) Run(cmd *cobra.Command, args []string) error {
 
 	serverFilters, clientFilters := getServerSupportedFilters(filters, api.Image{})
 
-	var images []api.Image
-	allImages, err := remoteServer.GetImagesWithFilter(serverFilters)
-	if err != nil {
-		allImages, err = remoteServer.GetImages()
+	var allImages, images []api.Image
+	if c.flagAllProjects {
+		allImages, err = remoteServer.GetImagesAllProjects()
 		if err != nil {
 			return err
 		}
+	} else {
+		allImages, err = remoteServer.GetImagesWithFilter(serverFilters)
+		if err != nil {
+			allImages, err = remoteServer.GetImages()
+			if err != nil {
+				return err
+			}
 
-		clientFilters = filters
+			clientFilters = filters
+		}
 	}
 
 	for _, image := range allImages {

cmd/incusd/images.go

@@ -1339,30 +1339,46 @@ func getImageMetadata(fname string) (*api.ImageMetadata, string, error) {
 	return &result, imageType, nil
 }
 
-func doImagesGet(ctx context.Context, tx *db.ClusterTx, recursion bool, projectName string, public bool, clauses *filter.ClauseSet, hasPermission auth.PermissionChecker) (any, error) {
+func doImagesGet(ctx context.Context, tx *db.ClusterTx, recursion bool, projectName string, public bool, clauses *filter.ClauseSet, hasPermission auth.PermissionChecker, allProjects bool) (any, error) {
 	mustLoadObjects := recursion || (clauses != nil && len(clauses.Clauses) > 0)
 
-	fingerprints, err := tx.GetImagesFingerprints(ctx, projectName, public)
-	if err != nil {
-		return err, err
+	imagesProjectsMap := map[string][]string{}
+	if allProjects {
+		var err error
+
+		imagesProjectsMap, err = tx.GetImages(ctx)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		fingerprints, err := tx.GetImagesFingerprints(ctx, projectName, public)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, fp := range fingerprints {
+			imagesProjectsMap[fp] = []string{projectName}
+		}
 	}
 
 	var resultString []string
 	var resultMap []*api.Image
 
 	if recursion {
-		resultMap = make([]*api.Image, 0, len(fingerprints))
+		resultMap = make([]*api.Image, 0, len(imagesProjectsMap))
 	} else {
-		resultString = make([]string, 0, len(fingerprints))
+		resultString = make([]string, 0, len(imagesProjectsMap))
 	}
 
-	for _, fingerprint := range fingerprints {
-		image, err := doImageGet(ctx, tx, projectName, fingerprint, public)
+	for fingerprint, projects := range imagesProjectsMap {
+		curProjectName := projects[0]
+
+		image, err := doImageGet(ctx, tx, curProjectName, fingerprint, public)
 		if err != nil {
 			continue
 		}
 
-		if !image.Public && !hasPermission(auth.ObjectImage(projectName, fingerprint)) {
+		if !image.Public && !hasPermission(auth.ObjectImage(curProjectName, fingerprint)) {
 			continue
 		}
 
@@ -1415,6 +1431,10 @@ func doImagesGet(ctx context.Context, tx *db.ClusterTx, recursion bool, projectN
 //      description: Collection filter
 //      type: string
 //      example: default
+//    - in: query
+//      name: all-projects
+//      description: Retrieve images from all projects
+//      type: boolean
 //  responses:
 //    "200":
 //      description: API endpoints
@@ -1469,6 +1489,10 @@ func doImagesGet(ctx context.Context, tx *db.ClusterTx, recursion bool, projectN
 //      description: Collection filter
 //      type: string
 //      example: default
+//    - in: query
+//      name: all-projects
+//      description: Retrieve images from all projects
+//      type: boolean
 //  responses:
 //    "200":
 //      description: API endpoints
@@ -1518,6 +1542,10 @@ func doImagesGet(ctx context.Context, tx *db.ClusterTx, recursion bool, projectN
 //      description: Collection filter
 //      type: string
 //      example: default
+//    - in: query
+//      name: all-projects
+//      description: Retrieve images from all projects
+//      type: boolean
 //  responses:
 //    "200":
 //      description: API endpoints
@@ -1572,6 +1600,11 @@ func doImagesGet(ctx context.Context, tx *db.ClusterTx, recursion bool, projectN
 //	    description: Collection filter
 //	    type: string
 //	    example: default
+//	  - in: query
+//	    name: all-projects
+//	    description: Retrieve images from all projects
+//	    type: boolean
+//	    example: default
 //	responses:
 //	  "200":
 //	    description: API endpoints
@@ -1602,8 +1635,14 @@ func doImagesGet(ctx context.Context, tx *db.ClusterTx, recursion bool, projectN
 //	    $ref: "#/responses/InternalServerError"
 func imagesGet(d *Daemon, r *http.Request) response.Response {
 	projectName := request.ProjectParam(r)
+	allProjects := util.IsTrue(r.FormValue("all-projects"))
 	filterStr := r.FormValue("filter")
 
+	// ProjectParam returns default if not set
+	if allProjects && projectName != api.ProjectDefaultName {
+		return response.BadRequest(fmt.Errorf("Cannot specify a project when requesting all projects"))
+	}
+
 	s := d.State()
 
 	hasPermission, authorizationErr := s.Authorizer.GetPermissionChecker(r.Context(), r, auth.EntitlementCanView, auth.ObjectTypeImage)
@@ -1620,7 +1659,7 @@ func imagesGet(d *Daemon, r *http.Request) response.Response {
 
 	var result any
 	err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error {
-		result, err = doImagesGet(ctx, tx, localUtil.IsRecursionRequest(r), projectName, public, clauses, hasPermission)
+		result, err = doImagesGet(ctx, tx, localUtil.IsRecursionRequest(r), projectName, public, clauses, hasPermission, allProjects)
 		if err != nil {
 			return err
 		}

doc/api-extensions.md

@@ -1901,6 +1901,10 @@ client authentication.
 
 Adds ability to copy image to a project different from the source.
 
+## `images_all_projects`
+
+This adds support for listing images across all projects through the `all-projects` parameter on the `GET /1.0/images`API.
+
 ## `cluster_migration_inconsistent_copy`
 
 Adds `allow_inconsistent` field to `POST /1.0/instances/<name>`. Set to `true` to allow inconsistent copying between cluster

doc/rest-api.yaml

@@ -689,6 +689,11 @@ definitions:
                     type: string
                 type: array
                 x-go-name: Profiles
+            project:
+                description: Project name
+                example: project1
+                type: string
+                x-go-name: Project
             properties:
                 additionalProperties:
                     type: string
@@ -7304,6 +7309,10 @@ paths:
                   in: query
                   name: filter
                   type: string
+                - description: Retrieve images from all projects
+                  in: query
+                  name: all-projects
+                  type: boolean
             produces:
                 - application/json
             responses:
@@ -8055,6 +8064,10 @@ paths:
                   in: query
                   name: filter
                   type: string
+                - description: Retrieve images from all projects
+                  in: query
+                  name: all-projects
+                  type: boolean
             produces:
                 - application/json
             responses:
@@ -8143,6 +8156,10 @@ paths:
                   in: query
                   name: filter
                   type: string
+                - description: Retrieve images from all projects
+                  in: query
+                  name: all-projects
+                  type: boolean
             produces:
                 - application/json
             responses:
@@ -8191,6 +8208,11 @@ paths:
                   in: query
                   name: filter
                   type: string
+                - description: Retrieve images from all projects
+                  example: default
+                  in: query
+                  name: all-projects
+                  type: boolean
             produces:
                 - application/json
             responses:

internal/server/db/images.go

@@ -400,6 +400,7 @@ func (c *ClusterTx) GetImageByFingerprintPrefix(ctx context.Context, fingerprint
 	image.Cached = object.Cached
 	image.Public = object.Public
 	image.AutoUpdate = object.AutoUpdate
+	image.Project = object.Project
 
 	err = c.imageFill(
 		ctx, object.ID, &image,

internal/version/api.go

@@ -317,6 +317,7 @@ var APIExtensions = []string{
 	"projects_restricted_intercept",
 	"metrics_authentication",
 	"images_target_project",
+	"images_all_projects",
 	"cluster_migration_inconsistent_copy",
 	"cluster_ovn_chassis",
 	"container_syscall_intercept_sched_setscheduler",