diff --git a/examples/pagination.go b/examples/pagination.go index 3d4d62f94..9da6f815c 100644 --- a/examples/pagination.go +++ b/examples/pagination.go @@ -56,3 +56,46 @@ func pagination() { opt.Page = resp.NextPage } } + +func keysetPagination() { + git, err := gitlab.NewClient("yourtokengoeshere") + if err != nil { + log.Fatal(err) + } + + opt := &gitlab.ListProjectsOptions{ + ListOptions: gitlab.ListOptions{ + OrderBy: "id", + Pagination: "keyset", + PerPage: 5, + Sort: "asc", + }, + Owned: gitlab.Bool(true), + } + + options := []gitlab.RequestOptionFunc{} + + for { + // Get the first page with projects. + ps, resp, err := git.Projects.ListProjects(opt, options...) + if err != nil { + log.Fatal(err) + } + + // List all the projects we've found so far. + for _, p := range ps { + log.Printf("Found project: %s", p.Name) + } + + // Exit the loop when we've seen all pages. + if resp.NextLink == "" { + break + } + + // Set all query parameters in the next request to values in the + // returned parameters. This could go along with any existing options. + options = []gitlab.RequestOptionFunc{ + gitlab.WithKeysetPaginationParameters(resp.NextLink), + } + } +} diff --git a/gitlab.go b/gitlab.go index 88d55057c..0fb72e68a 100644 --- a/gitlab.go +++ b/gitlab.go @@ -227,11 +227,17 @@ type Client struct { // ListOptions specifies the optional parameters to various List methods that // support pagination. type ListOptions struct { - // For paginated result sets, page of results to retrieve. + // For offset-based paginated result sets, page of results to retrieve. Page int `url:"page,omitempty" json:"page,omitempty"` - - // For paginated result sets, the number of results to include per page. + // For offset-based and keyset-based paginated result sets, the number of results to include per page. PerPage int `url:"per_page,omitempty" json:"per_page,omitempty"` + + // For keyset-based paginated result sets, name of the column by which to order + OrderBy string `url:"order_by,omitempty" json:"order_by,omitempty"` + // For keyset-based paginated result sets, the value must be `"keyset"` + Pagination string `url:"pagination,omitempty" json:"pagination,omitempty"` + // For keyset-based paginated result sets, sort order (`"asc"`` or `"desc"`) + Sort string `url:"sort,omitempty" json:"sort,omitempty"` } // RateLimiter describes the interface that all (custom) rate limiters must implement. @@ -710,32 +716,43 @@ func (c *Client) UploadRequest(method, path string, content io.Reader, filename type Response struct { *http.Response - // These fields provide the page values for paginating through a set of - // results. Any or all of these may be set to the zero value for - // responses that are not part of a paginated set, or for which there - // are no additional pages. + // Fields used for offset-based pagination. TotalItems int TotalPages int ItemsPerPage int CurrentPage int NextPage int PreviousPage int + + // Fields used for keyset-based pagination. + PreviousLink string + NextLink string + FirstLink string + LastLink string } // newResponse creates a new Response for the provided http.Response. func newResponse(r *http.Response) *Response { response := &Response{Response: r} response.populatePageValues() + response.populateLinkValues() return response } const ( + // Headers used for offset-based pagination. xTotal = "X-Total" xTotalPages = "X-Total-Pages" xPerPage = "X-Per-Page" xPage = "X-Page" xNextPage = "X-Next-Page" xPrevPage = "X-Prev-Page" + + // Headers used for keyset-based pagination. + linkPrev = "prev" + linkNext = "next" + linkFirst = "first" + linkLast = "last" ) // populatePageValues parses the HTTP Link response headers and populates the @@ -761,6 +778,31 @@ func (r *Response) populatePageValues() { } } +func (r *Response) populateLinkValues() { + if link := r.Header.Get("Link"); link != "" { + for _, link := range strings.Split(link, ",") { + parts := strings.Split(link, ";") + if len(parts) < 2 { + continue + } + + linkType := strings.Trim(strings.Split(parts[1], "=")[1], "\"") + linkValue := strings.Trim(parts[0], "< >") + + switch linkType { + case linkPrev: + r.PreviousLink = linkValue + case linkNext: + r.NextLink = linkValue + case linkFirst: + r.FirstLink = linkValue + case linkLast: + r.LastLink = linkValue + } + } + } +} + // Do sends an API request and returns the API response. The API response is // JSON decoded and stored in the value pointed to by v, or returned as an // error if an API error has occurred. If v implements the io.Writer diff --git a/gitlab_test.go b/gitlab_test.go index d12defe7d..ae08a4936 100644 --- a/gitlab_test.go +++ b/gitlab_test.go @@ -20,6 +20,7 @@ import ( "bytes" "context" "errors" + "fmt" "io" "log" "net/http" @@ -236,3 +237,155 @@ func TestPathEscape(t *testing.T) { t.Errorf("Expected: %s, got %s", want, got) } } + +func TestPaginationPopulatePageValuesEmpty(t *testing.T) { + wantPageHeaders := map[string]int{ + xTotal: 0, + xTotalPages: 0, + xPerPage: 0, + xPage: 0, + xNextPage: 0, + xPrevPage: 0, + } + wantLinkHeaders := map[string]string{ + linkPrev: "", + linkNext: "", + linkFirst: "", + linkLast: "", + } + + r := newResponse(&http.Response{ + Header: http.Header{}, + }) + + gotPageHeaders := map[string]int{ + xTotal: r.TotalItems, + xTotalPages: r.TotalPages, + xPerPage: r.ItemsPerPage, + xPage: r.CurrentPage, + xNextPage: r.NextPage, + xPrevPage: r.PreviousPage, + } + for k, v := range wantPageHeaders { + if v != gotPageHeaders[k] { + t.Errorf("For %s, expected %d, got %d", k, v, gotPageHeaders[k]) + } + } + + gotLinkHeaders := map[string]string{ + linkPrev: r.PreviousLink, + linkNext: r.NextLink, + linkFirst: r.FirstLink, + linkLast: r.LastLink, + } + for k, v := range wantLinkHeaders { + if v != gotLinkHeaders[k] { + t.Errorf("For %s, expected %s, got %s", k, v, gotLinkHeaders[k]) + } + } +} + +func TestPaginationPopulatePageValuesOffset(t *testing.T) { + wantPageHeaders := map[string]int{ + xTotal: 100, + xTotalPages: 5, + xPerPage: 20, + xPage: 2, + xNextPage: 3, + xPrevPage: 1, + } + wantLinkHeaders := map[string]string{ + linkPrev: "https://gitlab.example.com/api/v4/projects/8/issues/8/notes?page=1&per_page=3", + linkNext: "https://gitlab.example.com/api/v4/projects/8/issues/8/notes?page=3&per_page=3", + linkFirst: "https://gitlab.example.com/api/v4/projects/8/issues/8/notes?page=1&per_page=3", + linkLast: "https://gitlab.example.com/api/v4/projects/8/issues/8/notes?page=3&per_page=3", + } + + h := http.Header{} + for k, v := range wantPageHeaders { + h.Add(k, fmt.Sprint(v)) + } + var linkHeaderComponents []string + for k, v := range wantLinkHeaders { + if v != "" { + linkHeaderComponents = append(linkHeaderComponents, fmt.Sprintf("<%s>; rel=\"%s\"", v, k)) + } + } + h.Add("Link", strings.Join(linkHeaderComponents, ", ")) + + r := newResponse(&http.Response{ + Header: h, + }) + + gotPageHeaders := map[string]int{ + xTotal: r.TotalItems, + xTotalPages: r.TotalPages, + xPerPage: r.ItemsPerPage, + xPage: r.CurrentPage, + xNextPage: r.NextPage, + xPrevPage: r.PreviousPage, + } + for k, v := range wantPageHeaders { + if v != gotPageHeaders[k] { + t.Errorf("For %s, expected %d, got %d", k, v, gotPageHeaders[k]) + } + } + + gotLinkHeaders := map[string]string{ + linkPrev: r.PreviousLink, + linkNext: r.NextLink, + linkFirst: r.FirstLink, + linkLast: r.LastLink, + } + for k, v := range wantLinkHeaders { + if v != gotLinkHeaders[k] { + t.Errorf("For %s, expected %s, got %s", k, v, gotLinkHeaders[k]) + } + } +} + +func TestPaginationPopulatePageValuesKeyset(t *testing.T) { + wantPageHeaders := map[string]int{ + xTotal: 0, + xTotalPages: 0, + xPerPage: 0, + xPage: 0, + xNextPage: 0, + xPrevPage: 0, + } + wantLinkHeaders := map[string]string{ + linkPrev: "", + linkFirst: "", + linkLast: "", + } + + h := http.Header{} + for k, v := range wantPageHeaders { + h.Add(k, fmt.Sprint(v)) + } + var linkHeaderComponents []string + for k, v := range wantLinkHeaders { + if v != "" { + linkHeaderComponents = append(linkHeaderComponents, fmt.Sprintf("<%s>; rel=\"%s\"", v, k)) + } + } + h.Add("Link", strings.Join(linkHeaderComponents, ", ")) + + r := newResponse(&http.Response{ + Header: h, + }) + + gotPageHeaders := map[string]int{ + xTotal: r.TotalItems, + xTotalPages: r.TotalPages, + xPerPage: r.ItemsPerPage, + xPage: r.CurrentPage, + xNextPage: r.NextPage, + xPrevPage: r.PreviousPage, + } + for k, v := range wantPageHeaders { + if v != gotPageHeaders[k] { + t.Errorf("For %s, expected %d, got %d", k, v, gotPageHeaders[k]) + } + } +} diff --git a/invites_test.go b/invites_test.go index 56b27c6b6..d3178a274 100644 --- a/invites_test.go +++ b/invites_test.go @@ -16,7 +16,7 @@ func TestListGroupPendingInvites(t *testing.T) { }) opt := &ListPendingInvitationsOptions{ - ListOptions: ListOptions{2, 3}, + ListOptions: ListOptions{Page: 2, PerPage: 3}, } projects, _, err := client.Invites.ListPendingGroupInvitations("test", opt) @@ -85,7 +85,7 @@ func TestListProjectPendingInvites(t *testing.T) { }) opt := &ListPendingInvitationsOptions{ - ListOptions: ListOptions{2, 3}, + ListOptions: ListOptions{Page: 2, PerPage: 3}, } projects, _, err := client.Invites.ListPendingProjectInvitations("test", opt) diff --git a/notifications_test.go b/notifications_test.go index 1b971fcaf..a05651b1b 100644 --- a/notifications_test.go +++ b/notifications_test.go @@ -86,7 +86,7 @@ func TestGetProjectSettings(t *testing.T) { } want := &NotificationSettings{ - Level: 5, //custom + Level: 5, // custom Events: &NotificationEvents{ NewEpic: true, NewNote: true, diff --git a/project_vulnerabilities_test.go b/project_vulnerabilities_test.go index 7f21bc4b0..5bf0b8dd5 100644 --- a/project_vulnerabilities_test.go +++ b/project_vulnerabilities_test.go @@ -32,7 +32,7 @@ func TestListProjectVulnerabilities(t *testing.T) { }) opt := &ListProjectVulnerabilitiesOptions{ - ListOptions: ListOptions{2, 3}, + ListOptions: ListOptions{Page: 2, PerPage: 3}, } projectVulnerabilities, _, err := client.ProjectVulnerabilities.ListProjectVulnerabilities(1, opt) diff --git a/projects_test.go b/projects_test.go index cf4965a75..c3e9b9a19 100644 --- a/projects_test.go +++ b/projects_test.go @@ -39,7 +39,7 @@ func TestListProjects(t *testing.T) { }) opt := &ListProjectsOptions{ - ListOptions: ListOptions{2, 3}, + ListOptions: ListOptions{Page: 2, PerPage: 3}, Archived: Bool(true), OrderBy: String("name"), Sort: String("asc"), @@ -68,7 +68,7 @@ func TestListUserProjects(t *testing.T) { }) opt := &ListProjectsOptions{ - ListOptions: ListOptions{2, 3}, + ListOptions: ListOptions{Page: 2, PerPage: 3}, Archived: Bool(true), OrderBy: String("name"), Sort: String("asc"), @@ -97,7 +97,7 @@ func TestListUserContributedProjects(t *testing.T) { }) opt := &ListProjectsOptions{ - ListOptions: ListOptions{2, 3}, + ListOptions: ListOptions{Page: 2, PerPage: 3}, Archived: Bool(true), OrderBy: String("name"), Sort: String("asc"), @@ -126,7 +126,7 @@ func TestListUserStarredProjects(t *testing.T) { }) opt := &ListProjectsOptions{ - ListOptions: ListOptions{2, 3}, + ListOptions: ListOptions{Page: 2, PerPage: 3}, Archived: Bool(true), OrderBy: String("name"), Sort: String("asc"), @@ -156,7 +156,7 @@ func TestListProjectsUsersByID(t *testing.T) { }) opt := &ListProjectUserOptions{ - ListOptions: ListOptions{2, 3}, + ListOptions: ListOptions{Page: 2, PerPage: 3}, Search: String("query"), } @@ -181,7 +181,7 @@ func TestListProjectsUsersByName(t *testing.T) { }) opt := &ListProjectUserOptions{ - ListOptions: ListOptions{2, 3}, + ListOptions: ListOptions{Page: 2, PerPage: 3}, Search: String("query"), } @@ -206,7 +206,7 @@ func TestListProjectsGroupsByID(t *testing.T) { }) opt := &ListProjectGroupOptions{ - ListOptions: ListOptions{2, 3}, + ListOptions: ListOptions{Page: 2, PerPage: 3}, Search: String("query"), } @@ -231,7 +231,7 @@ func TestListProjectsGroupsByName(t *testing.T) { }) opt := &ListProjectGroupOptions{ - ListOptions: ListOptions{2, 3}, + ListOptions: ListOptions{Page: 2, PerPage: 3}, Search: String("query"), } @@ -255,7 +255,7 @@ func TestListOwnedProjects(t *testing.T) { }) opt := &ListProjectsOptions{ - ListOptions: ListOptions{2, 3}, + ListOptions: ListOptions{Page: 2, PerPage: 3}, Archived: Bool(true), OrderBy: String("name"), Sort: String("asc"), @@ -285,7 +285,7 @@ func TestListStarredProjects(t *testing.T) { }) opt := &ListProjectsOptions{ - ListOptions: ListOptions{2, 3}, + ListOptions: ListOptions{Page: 2, PerPage: 3}, Archived: Bool(true), OrderBy: String("name"), Sort: String("asc"), @@ -572,7 +572,7 @@ func TestListProjectForks(t *testing.T) { }) opt := &ListProjectsOptions{} - opt.ListOptions = ListOptions{2, 3} + opt.ListOptions = ListOptions{Page: 2, PerPage: 3} opt.Archived = Bool(true) opt.OrderBy = String("name") opt.Sort = String("asc") diff --git a/request_options.go b/request_options.go index bc01b9180..d158047f6 100644 --- a/request_options.go +++ b/request_options.go @@ -18,6 +18,7 @@ package gitlab import ( "context" + "net/url" retryablehttp "github.com/hashicorp/go-retryablehttp" ) @@ -52,6 +53,27 @@ func WithHeaders(headers map[string]string) RequestOptionFunc { } } +// WithKeysetPaginationParameters takes a "next" link from the Link header of a +// response to a keyset-based paginated request and modifies the values of each +// query parameter in the request with its corresponding response parameter. +func WithKeysetPaginationParameters(nextLink string) RequestOptionFunc { + return func(req *retryablehttp.Request) error { + nextUrl, err := url.Parse(nextLink) + if err != nil { + return err + } + q := req.URL.Query() + for k, values := range nextUrl.Query() { + q.Del(k) + for _, v := range values { + q.Add(k, v) + } + } + req.URL.RawQuery = q.Encode() + return nil + } +} + // WithSudo takes either a username or user ID and sets the SUDO request header. func WithSudo(uid interface{}) RequestOptionFunc { return func(req *retryablehttp.Request) error { diff --git a/request_options_test.go b/request_options_test.go index 9e6a3fdfa..88e0aef29 100644 --- a/request_options_test.go +++ b/request_options_test.go @@ -19,6 +19,7 @@ import ( "testing" "time" + "github.com/hashicorp/go-retryablehttp" "github.com/stretchr/testify/assert" ) @@ -161,3 +162,23 @@ func TestWithHeaders(t *testing.T) { _, err = client.Do(req, nil) assert.NoError(t, err) } + +func TestWithKeysetPaginationParameters(t *testing.T) { + req, err := retryablehttp.NewRequest("GET", "https://gitlab.example.com/api/v4/groups?pagination=keyset&per_page=50&order_by=name&sort=asc", nil) + assert.NoError(t, err) + + linkNext := "https://gitlab.example.com/api/v4/groups?pagination=keyset&per_page=50&order_by=name&sort=asc&cursor=eyJuYW1lIjoiRmxpZ2h0anMiLCJpZCI6IjI2IiwiX2tkIjoibiJ9" + + err = WithKeysetPaginationParameters(linkNext)(req) + assert.NoError(t, err) + + values := req.URL.Query() + // Ensure all original parameters remain + assert.Equal(t, "keyset", values.Get("pagination")) + assert.Equal(t, "50", values.Get("per_page")) + assert.Equal(t, "name", values.Get("order_by")) + assert.Equal(t, "asc", values.Get("sort")) + + // Ensure cursor gets properly pulled from "next link" header + assert.Equal(t, "eyJuYW1lIjoiRmxpZ2h0anMiLCJpZCI6IjI2IiwiX2tkIjoibiJ9", values.Get("cursor")) +}