cherry-pick #6601 fix: github grapahql increment collector

backend/helpers/pluginhelper/api/api_collector_with_state.go

@@ -157,6 +157,7 @@ func (m *ApiCollectorStateManager) InitCollector(args ApiCollectorArgs) errors.E
 // InitGraphQLCollector init the embedded collector
 func (m *ApiCollectorStateManager) InitGraphQLCollector(args GraphqlCollectorArgs) errors.Error {
 	args.RawDataSubTaskArgs = m.RawDataSubTaskArgs
+	args.Incremental = args.Incremental || m.IsIncremental
 	graphqlCollector, err := NewGraphqlCollector(args)
 	if err != nil {
 		return err

backend/plugins/github_graphql/tasks/deployment_collector.go

@@ -125,7 +125,7 @@ func CollectDeployments(taskCtx plugin.SubTaskContext) errors.Error {
 			deployments := query.Repository.Deployments.Deployments
 			for _, rawL := range deployments {
 				if collectorWithState.Since != nil && !collectorWithState.Since.Before(rawL.UpdatedAt) {
-					break
+					return nil, helper.ErrFinishCollect
 				}
 			}
 			return nil, nil

backend/plugins/github_graphql/tasks/issue_collector.go

@@ -39,7 +39,7 @@ type GraphqlQueryIssueWrapper struct {
 			TotalCount graphql.Int
 			Issues     []GraphqlQueryIssue `graphql:"nodes"`
 			PageInfo   *helper.GraphqlQueryPageInfo
-		} `graphql:"issues(first: $pageSize, after: $skipCursor, orderBy: {field: UPDATED_AT, direction: DESC}, filterBy: {since: $since})"`
+		} `graphql:"issues(first: $pageSize, after: $skipCursor, orderBy: {field: UPDATED_AT, direction: DESC})"`
 	} `graphql:"repository(owner: $owner, name: $name)"`
 }
 
@@ -96,19 +96,14 @@ func CollectIssues(taskCtx plugin.SubTaskContext) errors.Error {
 
 	err = collectorWithState.InitGraphQLCollector(helper.GraphqlCollectorArgs{
 		GraphqlClient: data.GraphqlClient,
-		PageSize:      100,
+		PageSize:      10,
 		BuildQuery: func(reqData *helper.GraphqlRequestData) (interface{}, map[string]interface{}, error) {
 			query := &GraphqlQueryIssueWrapper{}
 			if reqData == nil {
 				return query, map[string]interface{}{}, nil
 			}
-			since := helper.DateTime{}
-			if collectorWithState.Since != nil {
-				since = helper.DateTime{Time: *collectorWithState.Since}
-			}
 			ownerName := strings.Split(data.Options.Name, "/")
 			variables := map[string]interface{}{
-				"since":      since,
 				"pageSize":   graphql.Int(reqData.Pager.Size),
 				"skipCursor": (*graphql.String)(reqData.Pager.SkipCursor),
 				"owner":      graphql.String(ownerName[0]),
@@ -121,6 +116,13 @@ func CollectIssues(taskCtx plugin.SubTaskContext) errors.Error {
 			return query.Repository.IssueList.PageInfo, nil
 		},
 		ResponseParser: func(iQuery interface{}, variables map[string]interface{}) ([]interface{}, error) {
+			query := iQuery.(*GraphqlQueryIssueWrapper)
+			issues := query.Repository.IssueList.Issues
+			for _, rawL := range issues {
+				if collectorWithState.Since != nil && !collectorWithState.Since.Before(rawL.UpdatedAt) {
+					return nil, helper.ErrFinishCollect
+				}
+			}
 			return nil, nil
 		},
 	})

backend/plugins/github_graphql/tasks/pr_collector.go

@@ -43,7 +43,7 @@ type GraphqlQueryPrWrapper struct {
 			PageInfo   *api.GraphqlQueryPageInfo
 			Prs        []GraphqlQueryPr `graphql:"nodes"`
 			TotalCount graphql.Int
-		} `graphql:"pullRequests(first: $pageSize, after: $skipCursor, orderBy: {field: UPDATED_AT, direction: DESC})"`
+		} `graphql:"pullRequests(first: $pageSize, after: $skipCursor, orderBy: {field: CREATED_AT, direction: DESC})"`
 	} `graphql:"repository(owner: $owner, name: $name)"`
 }
 
@@ -169,20 +169,12 @@ func CollectPrs(taskCtx plugin.SubTaskContext) errors.Error {
 		ResponseParser: func(iQuery interface{}, variables map[string]interface{}) ([]interface{}, error) {
 			query := iQuery.(*GraphqlQueryPrWrapper)
 			prs := query.Repository.PullRequests.Prs
-
-			isFinish := false
 			for _, rawL := range prs {
-				// collect data even though in increment mode because of updating existing data
-				if collectorWithState.Since != nil && !collectorWithState.Since.Before(rawL.UpdatedAt) {
-					isFinish = true
-					break
+				if collectorWithState.Since != nil && !collectorWithState.Since.Before(rawL.CreatedAt) {
+					return nil, api.ErrFinishCollect
 				}
 			}
-			if isFinish {
-				return nil, api.ErrFinishCollect
-			} else {
-				return nil, nil
-			}
+			return nil, nil
 		},
 	})
 	if err != nil {