Merge pull request #1827 from pwlandoll/feat/815-keyset-pagination

Add keyset pagination options

examples/pagination.go

@@ -56,3 +56,46 @@ func pagination() {
 		opt.Page = resp.NextPage
 	}
 }
+
+func keysetPagination() {
+	git, err := gitlab.NewClient("yourtokengoeshere")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	opt := &gitlab.ListProjectsOptions{
+		ListOptions: gitlab.ListOptions{
+			OrderBy:    "id",
+			Pagination: "keyset",
+			PerPage:    5,
+			Sort:       "asc",
+		},
+		Owned: gitlab.Bool(true),
+	}
+
+	options := []gitlab.RequestOptionFunc{}
+
+	for {
+		// Get the first page with projects.
+		ps, resp, err := git.Projects.ListProjects(opt, options...)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		// List all the projects we've found so far.
+		for _, p := range ps {
+			log.Printf("Found project: %s", p.Name)
+		}
+
+		// Exit the loop when we've seen all pages.
+		if resp.NextLink == "" {
+			break
+		}
+
+		// Set all query parameters in the next request to values in the
+		// returned parameters. This could go along with any existing options.
+		options = []gitlab.RequestOptionFunc{
+			gitlab.WithKeysetPaginationParameters(resp.NextLink),
+		}
+	}
+}

gitlab.go

@@ -227,11 +227,17 @@ type Client struct {
 // ListOptions specifies the optional parameters to various List methods that
 // support pagination.
 type ListOptions struct {
-	// For paginated result sets, page of results to retrieve.
+	// For offset-based paginated result sets, page of results to retrieve.
 	Page int `url:"page,omitempty" json:"page,omitempty"`
-
-	// For paginated result sets, the number of results to include per page.
+	// For offset-based and keyset-based paginated result sets, the number of results to include per page.
 	PerPage int `url:"per_page,omitempty" json:"per_page,omitempty"`
+
+	// For keyset-based paginated result sets, name of the column by which to order
+	OrderBy string `url:"order_by,omitempty" json:"order_by,omitempty"`
+	// For keyset-based paginated result sets, the value must be `"keyset"`
+	Pagination string `url:"pagination,omitempty" json:"pagination,omitempty"`
+	// For keyset-based paginated result sets, sort order (`"asc"`` or `"desc"`)
+	Sort string `url:"sort,omitempty" json:"sort,omitempty"`
 }
 
 // RateLimiter describes the interface that all (custom) rate limiters must implement.
@@ -710,32 +716,43 @@ func (c *Client) UploadRequest(method, path string, content io.Reader, filename
 type Response struct {
 	*http.Response
 
-	// These fields provide the page values for paginating through a set of
-	// results. Any or all of these may be set to the zero value for
-	// responses that are not part of a paginated set, or for which there
-	// are no additional pages.
+	// Fields used for offset-based pagination.
 	TotalItems   int
 	TotalPages   int
 	ItemsPerPage int
 	CurrentPage  int
 	NextPage     int
 	PreviousPage int
+
+	// Fields used for keyset-based pagination.
+	PreviousLink string
+	NextLink     string
+	FirstLink    string
+	LastLink     string
 }
 
 // newResponse creates a new Response for the provided http.Response.
 func newResponse(r *http.Response) *Response {
 	response := &Response{Response: r}
 	response.populatePageValues()
+	response.populateLinkValues()
 	return response
 }
 
 const (
+	// Headers used for offset-based pagination.
 	xTotal      = "X-Total"
 	xTotalPages = "X-Total-Pages"
 	xPerPage    = "X-Per-Page"
 	xPage       = "X-Page"
 	xNextPage   = "X-Next-Page"
 	xPrevPage   = "X-Prev-Page"
+
+	// Headers used for keyset-based pagination.
+	linkPrev  = "prev"
+	linkNext  = "next"
+	linkFirst = "first"
+	linkLast  = "last"
 )
 
 // populatePageValues parses the HTTP Link response headers and populates the
@@ -761,6 +778,31 @@ func (r *Response) populatePageValues() {
 	}
 }
 
+func (r *Response) populateLinkValues() {
+	if link := r.Header.Get("Link"); link != "" {
+		for _, link := range strings.Split(link, ",") {
+			parts := strings.Split(link, ";")
+			if len(parts) < 2 {
+				continue
+			}
+
+			linkType := strings.Trim(strings.Split(parts[1], "=")[1], "\"")
+			linkValue := strings.Trim(parts[0], "< >")
+
+			switch linkType {
+			case linkPrev:
+				r.PreviousLink = linkValue
+			case linkNext:
+				r.NextLink = linkValue
+			case linkFirst:
+				r.FirstLink = linkValue
+			case linkLast:
+				r.LastLink = linkValue
+			}
+		}
+	}
+}
+
 // Do sends an API request and returns the API response. The API response is
 // JSON decoded and stored in the value pointed to by v, or returned as an
 // error if an API error has occurred. If v implements the io.Writer

gitlab_test.go

@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"context"
 	"errors"
+	"fmt"
 	"io"
 	"log"
 	"net/http"
@@ -236,3 +237,155 @@ func TestPathEscape(t *testing.T) {
 		t.Errorf("Expected: %s, got %s", want, got)
 	}
 }
+
+func TestPaginationPopulatePageValuesEmpty(t *testing.T) {
+	wantPageHeaders := map[string]int{
+		xTotal:      0,
+		xTotalPages: 0,
+		xPerPage:    0,
+		xPage:       0,
+		xNextPage:   0,
+		xPrevPage:   0,
+	}
+	wantLinkHeaders := map[string]string{
+		linkPrev:  "",
+		linkNext:  "",
+		linkFirst: "",
+		linkLast:  "",
+	}
+
+	r := newResponse(&http.Response{
+		Header: http.Header{},
+	})
+
+	gotPageHeaders := map[string]int{
+		xTotal:      r.TotalItems,
+		xTotalPages: r.TotalPages,
+		xPerPage:    r.ItemsPerPage,
+		xPage:       r.CurrentPage,
+		xNextPage:   r.NextPage,
+		xPrevPage:   r.PreviousPage,
+	}
+	for k, v := range wantPageHeaders {
+		if v != gotPageHeaders[k] {
+			t.Errorf("For %s, expected %d, got %d", k, v, gotPageHeaders[k])
+		}
+	}
+
+	gotLinkHeaders := map[string]string{
+		linkPrev:  r.PreviousLink,
+		linkNext:  r.NextLink,
+		linkFirst: r.FirstLink,
+		linkLast:  r.LastLink,
+	}
+	for k, v := range wantLinkHeaders {
+		if v != gotLinkHeaders[k] {
+			t.Errorf("For %s, expected %s, got %s", k, v, gotLinkHeaders[k])
+		}
+	}
+}
+
+func TestPaginationPopulatePageValuesOffset(t *testing.T) {
+	wantPageHeaders := map[string]int{
+		xTotal:      100,
+		xTotalPages: 5,
+		xPerPage:    20,
+		xPage:       2,
+		xNextPage:   3,
+		xPrevPage:   1,
+	}
+	wantLinkHeaders := map[string]string{
+		linkPrev:  "https://gitlab.example.com/api/v4/projects/8/issues/8/notes?page=1&per_page=3",
+		linkNext:  "https://gitlab.example.com/api/v4/projects/8/issues/8/notes?page=3&per_page=3",
+		linkFirst: "https://gitlab.example.com/api/v4/projects/8/issues/8/notes?page=1&per_page=3",
+		linkLast:  "https://gitlab.example.com/api/v4/projects/8/issues/8/notes?page=3&per_page=3",
+	}
+
+	h := http.Header{}
+	for k, v := range wantPageHeaders {
+		h.Add(k, fmt.Sprint(v))
+	}
+	var linkHeaderComponents []string
+	for k, v := range wantLinkHeaders {
+		if v != "" {
+			linkHeaderComponents = append(linkHeaderComponents, fmt.Sprintf("<%s>; rel=\"%s\"", v, k))
+		}
+	}
+	h.Add("Link", strings.Join(linkHeaderComponents, ", "))
+
+	r := newResponse(&http.Response{
+		Header: h,
+	})
+
+	gotPageHeaders := map[string]int{
+		xTotal:      r.TotalItems,
+		xTotalPages: r.TotalPages,
+		xPerPage:    r.ItemsPerPage,
+		xPage:       r.CurrentPage,
+		xNextPage:   r.NextPage,
+		xPrevPage:   r.PreviousPage,
+	}
+	for k, v := range wantPageHeaders {
+		if v != gotPageHeaders[k] {
+			t.Errorf("For %s, expected %d, got %d", k, v, gotPageHeaders[k])
+		}
+	}
+
+	gotLinkHeaders := map[string]string{
+		linkPrev:  r.PreviousLink,
+		linkNext:  r.NextLink,
+		linkFirst: r.FirstLink,
+		linkLast:  r.LastLink,
+	}
+	for k, v := range wantLinkHeaders {
+		if v != gotLinkHeaders[k] {
+			t.Errorf("For %s, expected %s, got %s", k, v, gotLinkHeaders[k])
+		}
+	}
+}
+
+func TestPaginationPopulatePageValuesKeyset(t *testing.T) {
+	wantPageHeaders := map[string]int{
+		xTotal:      0,
+		xTotalPages: 0,
+		xPerPage:    0,
+		xPage:       0,
+		xNextPage:   0,
+		xPrevPage:   0,
+	}
+	wantLinkHeaders := map[string]string{
+		linkPrev:  "",
+		linkFirst: "",
+		linkLast:  "",
+	}
+
+	h := http.Header{}
+	for k, v := range wantPageHeaders {
+		h.Add(k, fmt.Sprint(v))
+	}
+	var linkHeaderComponents []string
+	for k, v := range wantLinkHeaders {
+		if v != "" {
+			linkHeaderComponents = append(linkHeaderComponents, fmt.Sprintf("<%s>; rel=\"%s\"", v, k))
+		}
+	}
+	h.Add("Link", strings.Join(linkHeaderComponents, ", "))
+
+	r := newResponse(&http.Response{
+		Header: h,
+	})
+
+	gotPageHeaders := map[string]int{
+		xTotal:      r.TotalItems,
+		xTotalPages: r.TotalPages,
+		xPerPage:    r.ItemsPerPage,
+		xPage:       r.CurrentPage,
+		xNextPage:   r.NextPage,
+		xPrevPage:   r.PreviousPage,
+	}
+	for k, v := range wantPageHeaders {
+		if v != gotPageHeaders[k] {
+			t.Errorf("For %s, expected %d, got %d", k, v, gotPageHeaders[k])
+		}
+	}
+}

invites_test.go

@@ -16,7 +16,7 @@ func TestListGroupPendingInvites(t *testing.T) {
 	})
 
 	opt := &ListPendingInvitationsOptions{
-		ListOptions: ListOptions{2, 3},
+		ListOptions: ListOptions{Page: 2, PerPage: 3},
 	}
 
 	projects, _, err := client.Invites.ListPendingGroupInvitations("test", opt)
@@ -85,7 +85,7 @@ func TestListProjectPendingInvites(t *testing.T) {
 	})
 
 	opt := &ListPendingInvitationsOptions{
-		ListOptions: ListOptions{2, 3},
+		ListOptions: ListOptions{Page: 2, PerPage: 3},
 	}
 
 	projects, _, err := client.Invites.ListPendingProjectInvitations("test", opt)

project_vulnerabilities_test.go

@@ -32,7 +32,7 @@ func TestListProjectVulnerabilities(t *testing.T) {
 	})
 
 	opt := &ListProjectVulnerabilitiesOptions{
-		ListOptions: ListOptions{2, 3},
+		ListOptions: ListOptions{Page: 2, PerPage: 3},
 	}
 
 	projectVulnerabilities, _, err := client.ProjectVulnerabilities.ListProjectVulnerabilities(1, opt)