Add pagination for known tests API

civisibility/utils/net/known_tests_api.go

@@ -6,6 +6,7 @@
 package net
 
 import (
+	"encoding/json"
 	"fmt"
 )
 
@@ -26,10 +27,15 @@ type (
 	}
 
 	KnownTestsRequestData struct {
-		Service        string             `json:"service"`
-		Env            string             `json:"env"`
-		RepositoryURL  string             `json:"repository_url"`
-		Configurations testConfigurations `json:"configurations"`
+		Service        string                     `json:"service"`
+		Env            string                     `json:"env"`
+		RepositoryURL  string                     `json:"repository_url"`
+		Configurations testConfigurations         `json:"configurations"`
+		PageInfo       *knownTestsRequestPageInfo `json:"page_info,omitempty"`
+	}
+
+	knownTestsRequestPageInfo struct {
+		PageState string `json:"page_state,omitempty"`
 	}
 
 	knownTestsResponse struct {
@@ -41,7 +47,14 @@ type (
 	}
 
 	KnownTestsResponseData struct {
-		Tests KnownTestsResponseDataModules `json:"tests"`
+		Tests    KnownTestsResponseDataModules `json:"tests"`
+		PageInfo *knownTestsResponsePageInfo   `json:"page_info,omitempty"`
+	}
+
+	knownTestsResponsePageInfo struct {
+		Cursor  string `json:"cursor,omitempty"`
+		Size    int    `json:"size,omitempty"`
+		HasNext bool   `json:"has_next"`
 	}
 
 	KnownTestsResponseDataModules map[string]KnownTestsResponseDataSuites
@@ -54,33 +67,86 @@ func (c *client) GetKnownTests() (*KnownTestsResponseData, error) {
 	}
 	c.knownTestsRawResponse = nil
 
-	body := knownTestsRequest{
-		Data: knownTestsRequestHeader{
-			ID:   c.id,
-			Type: knownTestsRequestType,
-			Attributes: KnownTestsRequestData{
-				Service:        c.serviceName,
-				Env:            c.environment,
-				RepositoryURL:  c.repositoryURL,
-				Configurations: c.testConfigurations,
+	allKnownTests := KnownTestsResponseData{
+		Tests: KnownTestsResponseDataModules{},
+	}
+	var allResponse knownTestsResponse
+	var firstRawResponse json.RawMessage
+
+	cursor := ""
+	pageCount := 0
+	for {
+		body := knownTestsRequest{
+			Data: knownTestsRequestHeader{
+				ID:   c.id,
+				Type: knownTestsRequestType,
+				Attributes: KnownTestsRequestData{
+					Service:        c.serviceName,
+					Env:            c.environment,
+					RepositoryURL:  c.repositoryURL,
+					Configurations: c.testConfigurations,
+				},
 			},
-		},
+		}
+		if cursor != "" {
+			body.Data.Attributes.PageInfo = &knownTestsRequestPageInfo{PageState: cursor}
+		}
+
+		request := c.getPostRequestConfig(knownTestsURLPath, body)
+		response, err := c.handler.SendRequest(*request)
+
+		if err != nil {
+			return nil, fmt.Errorf("sending known tests request: %s", err)
+		}
+		pageCount++
+		if pageCount == 1 {
+			firstRawResponse = cloneRawMessage(response.Body)
+		}
+
+		var responseObject knownTestsResponse
+		err = response.Unmarshal(&responseObject)
+		if err != nil {
+			return nil, fmt.Errorf("unmarshalling known tests response: %s", err)
+		}
+
+		if pageCount == 1 {
+			allResponse.Data.ID = responseObject.Data.ID
+			allResponse.Data.Type = responseObject.Data.Type
+		}
+
+		mergeKnownTests(allKnownTests.Tests, responseObject.Data.Attributes.Tests)
+		allKnownTests.PageInfo = responseObject.Data.Attributes.PageInfo
+
+		if allKnownTests.PageInfo == nil || !allKnownTests.PageInfo.HasNext {
+			break
+		}
+		if allKnownTests.PageInfo.Cursor == "" {
+			return nil, fmt.Errorf("known tests response page_info is missing cursor")
+		}
+		cursor = allKnownTests.PageInfo.Cursor
 	}
 
-	request := c.getPostRequestConfig(knownTestsURLPath, body)
-
-	response, err := c.handler.SendRequest(*request)
-
-	if err != nil {
-		return nil, fmt.Errorf("sending known tests request: %s", err)
+	if pageCount == 1 {
+		c.knownTestsRawResponse = firstRawResponse
+	} else {
+		allResponse.Data.Attributes = allKnownTests
+		rawResponse, err := json.Marshal(allResponse)
+		if err != nil {
+			return nil, fmt.Errorf("marshalling known tests response: %s", err)
+		}
+		c.knownTestsRawResponse = cloneRawMessage(rawResponse)
 	}
-	c.knownTestsRawResponse = cloneRawMessage(response.Body)
 
-	var responseObject knownTestsResponse
-	err = response.Unmarshal(&responseObject)
-	if err != nil {
-		return nil, fmt.Errorf("unmarshalling known tests response: %s", err)
-	}
+	return &allKnownTests, nil
+}
 
-	return &responseObject.Data.Attributes, nil
+func mergeKnownTests(dst, src KnownTestsResponseDataModules) {
+	for module, suites := range src {
+		if _, ok := dst[module]; !ok {
+			dst[module] = KnownTestsResponseDataSuites{}
+		}
+		for suite, tests := range suites {
+			dst[module][suite] = append(dst[module][suite], tests...)
+		}
+	}
 }

civisibility/utils/net/known_tests_api_test.go

@@ -0,0 +1,146 @@
+package net
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+)
+
+func TestClientGetKnownTestsPaginatesWithAttributesPageInfo(t *testing.T) {
+	requests := []map[string]any{}
+	responses := []string{
+		`{"data":{"id":"known-tests-id","type":"ci_app_libraries_tests","attributes":{"tests":{"module-a":{"suite-a":["test-a"]}},"page_info":{"cursor":"page-2","size":1,"has_next":true}}}}`,
+		`{"data":{"id":"known-tests-id","type":"ci_app_libraries_tests","attributes":{"tests":{"module-a":{"suite-a":["test-b"]},"module-b":{"suite-b":["test-c"]}},"page_info":{"size":1,"has_next":false}}}}`,
+	}
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != http.MethodPost {
+			t.Fatalf("expected POST request, got %s", r.Method)
+		}
+		if strings.TrimPrefix(r.URL.Path, "/") != knownTestsURLPath {
+			t.Fatalf("unexpected request path %s", r.URL.Path)
+		}
+		if len(requests) >= len(responses) {
+			t.Fatalf("unexpected extra request")
+		}
+
+		var request map[string]any
+		if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
+			t.Fatalf("failed to decode request: %v", err)
+		}
+		requests = append(requests, request)
+
+		w.Header().Set(HeaderContentType, ContentTypeJSON)
+		_, _ = w.Write([]byte(responses[len(requests)-1]))
+	}))
+	defer server.Close()
+
+	client := newRawResponseTestClient(server)
+	knownTests, err := client.GetKnownTests()
+	if err != nil {
+		t.Fatalf("GetKnownTests() returned error: %v", err)
+	}
+
+	if len(requests) != 2 {
+		t.Fatalf("expected 2 known-tests requests, got %d", len(requests))
+	}
+
+	assertKnownTestsRequestPageInfo(t, requests[0], "", false)
+	assertKnownTestsRequestPageInfo(t, requests[1], "page-2", true)
+
+	if tests := knownTests.Tests["module-a"]["suite-a"]; len(tests) != 2 || tests[0] != "test-a" || tests[1] != "test-b" {
+		t.Fatalf("expected module-a/suite-a tests to be merged, got %#v", tests)
+	}
+	if tests := knownTests.Tests["module-b"]["suite-b"]; len(tests) != 1 || tests[0] != "test-c" {
+		t.Fatalf("expected module-b/suite-b tests from page 2, got %#v", tests)
+	}
+
+	var rawResponse knownTestsResponse
+	if err := json.Unmarshal(client.GetKnownTestsRawResponse(), &rawResponse); err != nil {
+		t.Fatalf("failed to decode merged raw response: %v", err)
+	}
+	if tests := rawResponse.Data.Attributes.Tests["module-a"]["suite-a"]; len(tests) != 2 {
+		t.Fatalf("expected raw response to include merged known tests, got %#v", tests)
+	}
+}
+
+func TestClientGetKnownTestsDoesNotCachePartialRawResponseWhenFollowUpFails(t *testing.T) {
+	responses := []string{
+		`{"data":{"id":"known-tests-id","type":"ci_app_libraries_tests","attributes":{"tests":{"module-a":{"suite-a":["test-a"]}},"page_info":{"cursor":"page-2","size":1,"has_next":true}}}}`,
+		`{"data":`,
+	}
+	requests := 0
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		requests++
+		if requests > len(responses) {
+			t.Fatalf("unexpected extra request")
+		}
+
+		w.Header().Set(HeaderContentType, ContentTypeJSON)
+		_, _ = w.Write([]byte(responses[requests-1]))
+	}))
+	defer server.Close()
+
+	client := newRawResponseTestClient(server)
+	if _, err := client.GetKnownTests(); err == nil {
+		t.Fatalf("GetKnownTests() should fail when a later page is invalid")
+	}
+	if rawResponse := client.GetKnownTestsRawResponse(); rawResponse != nil {
+		t.Fatalf("known tests raw response should stay unset after partial pagination failure, got %s", string(rawResponse))
+	}
+}
+
+func TestClientGetKnownTestsDoesNotCachePartialRawResponseWithoutFollowUpCursor(t *testing.T) {
+	response := `{"data":{"id":"known-tests-id","type":"ci_app_libraries_tests","attributes":{"tests":{"module-a":{"suite-a":["test-a"]}},"page_info":{"size":1,"has_next":true}}}}`
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set(HeaderContentType, ContentTypeJSON)
+		_, _ = w.Write([]byte(response))
+	}))
+	defer server.Close()
+
+	client := newRawResponseTestClient(server)
+	if _, err := client.GetKnownTests(); err == nil {
+		t.Fatalf("GetKnownTests() should fail when a paginated response omits the follow-up cursor")
+	}
+	if rawResponse := client.GetKnownTestsRawResponse(); rawResponse != nil {
+		t.Fatalf("known tests raw response should stay unset after missing cursor failure, got %s", string(rawResponse))
+	}
+}
+
+func assertKnownTestsRequestPageInfo(t *testing.T, request map[string]any, expectedPageState string, expectPageInfo bool) {
+	t.Helper()
+
+	if _, ok := request["page_info"]; ok {
+		t.Fatalf("request should not send top-level page_info: %#v", request)
+	}
+
+	data, ok := request["data"].(map[string]any)
+	if !ok {
+		t.Fatalf("request data has unexpected shape: %#v", request["data"])
+	}
+	attributes, ok := data["attributes"].(map[string]any)
+	if !ok {
+		t.Fatalf("request attributes have unexpected shape: %#v", data["attributes"])
+	}
+
+	pageInfo, ok := attributes["page_info"].(map[string]any)
+	if !expectPageInfo {
+		if ok {
+			t.Fatalf("first request should let the backend choose pagination without page_info, got %#v", pageInfo)
+		}
+		return
+	}
+	if !ok {
+		t.Fatalf("follow-up request should include data.attributes.page_info")
+	}
+	if pageInfo["page_state"] != expectedPageState {
+		t.Fatalf("expected page_state %q, got %#v", expectedPageState, pageInfo["page_state"])
+	}
+	if _, ok := pageInfo["page_size"]; ok {
+		t.Fatalf("request should not send page_size: %#v", pageInfo)
+	}
+}