feat: added gitlab user crawling

backend/plugins/gitlab/e2e/account_test.go

@@ -19,9 +19,11 @@ package e2e
 
 import (
 	"testing"
+	"time"
 
 	"github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain"
 	"github.com/apache/incubator-devlake/helpers/e2ehelper"
+	"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
 	"github.com/apache/incubator-devlake/plugins/gitlab/impl"
 	"github.com/apache/incubator-devlake/plugins/gitlab/models"
 	"github.com/apache/incubator-devlake/plugins/gitlab/tasks"
@@ -31,13 +33,23 @@ func TestGitlabAccountDataFlow(t *testing.T) {
 
 	var gitlab impl.Gitlab
 	dataflowTester := e2ehelper.NewDataFlowTester(t, "gitlab", gitlab)
+	apiClient := &api.ApiClient{}
+	apiClient.Setup(
+		"https://gitlab.com",
+		make(map[string]string),
+		time.Hour,
+	)
 
 	taskData := &tasks.GitlabTaskData{
 		Options: &tasks.GitlabOptions{
 			ConnectionId: 1,
 			ProjectId:    12345678,
 			ScopeConfig:  new(models.GitlabScopeConfig),
 		},
+
+		ApiClient: &api.ApiAsyncClient{
+			ApiClient: apiClient,
+		},
 	}
 
 	// import raw data table
@@ -81,3 +93,68 @@ func TestGitlabAccountDataFlow(t *testing.T) {
 		),
 	)
 }
+
+func TestGitlabAccountDataFlowUsersApi(t *testing.T) {
+
+	var gitlab impl.Gitlab
+	dataflowTester := e2ehelper.NewDataFlowTester(t, "gitlab", gitlab)
+	apiClient := &api.ApiClient{}
+	apiClient.Setup(
+		"https://custom.gitlab.com",
+		make(map[string]string),
+		time.Hour,
+	)
+
+	taskData := &tasks.GitlabTaskData{
+		Options: &tasks.GitlabOptions{
+			ConnectionId: 1,
+			ProjectId:    12345678,
+			ScopeConfig:  new(models.GitlabScopeConfig),
+		},
+
+		ApiClient: &api.ApiAsyncClient{
+			ApiClient: apiClient,
+		},
+	}
+
+	// import raw data table
+	dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_gitlab_api_users_direct_api.csv",
+		"_raw_gitlab_api_users")
+
+	// verify extraction
+	dataflowTester.FlushTabler(&models.GitlabAccount{})
+	dataflowTester.Subtask(tasks.ExtractAccountsMeta, taskData)
+	dataflowTester.VerifyTable(
+		models.GitlabAccount{},
+		"./snapshot_tables/_tool_gitlab_accounts_direct_api.csv",
+		e2ehelper.ColumnWithRawData(
+			"connection_id",
+			"gitlab_id",
+			"username",
+			"email",
+			"name",
+			"state",
+			"avatar_url",
+			"web_url",
+			"created_user_at",
+		),
+	)
+
+	// verify conversion
+	dataflowTester.FlushTabler(&crossdomain.Account{})
+	dataflowTester.Subtask(tasks.ConvertAccountsMeta, taskData)
+	dataflowTester.VerifyTable(
+		crossdomain.Account{},
+		"./snapshot_tables/accounts_direct_api.csv",
+		e2ehelper.ColumnWithRawData(
+			"id",
+			"email",
+			"full_name",
+			"user_name",
+			"avatar_url",
+			"organization",
+			"created_date",
+			"status",
+		),
+	)
+}

backend/plugins/gitlab/e2e/raw_tables/_raw_gitlab_api_users_direct_api.csv

@@ -0,0 +1,7 @@
+id,params,data,url,input,created_at
+1,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":2994198,""username"":""abc1"",""name"":""abc1"",""state"":""active"",""locked"":false,""avatar_url"":"""",""web_url"":""https://custom.gitlab.com/abc1"",""created_at"":""2018-03-25T09:43:17.048+01:00"",""bio"":"""",""location"":"""",""public_email"":"""",""skype"":"""",""linkedin"":"""",""twitter"":"""",""discord"":"""",""website_url"":"""",""organization"":"""",""job_title"":"""",""pronouns"":null,""bot"":false,""work_information"":null,""followers"":0,""following"":1,""is_followed"":false,""local_time"":null,""last_sign_in_at"":""2024-01-04T09:43:11.817+01:00"",""confirmed_at"":""2021-03-25T09:43:16.650+01:00"",""last_activity_on"":""2024-01-09"",""email"":""abc1@dev.com"",""theme_id"":11,""color_scheme_id"":2,""projects_limit"":10,""current_sign_in_at"":""2024-01-08T16:16:41.386+01:00"",""identities"":[],""can_create_group"":false,""can_create_project"":true,""two_factor_enabled"":false,""external"":false,""private_profile"":false,""commit_email"":""abc1@dev.com"",""shared_runners_minutes_limit"":null,""extra_shared_runners_minutes_limit"":null,""scim_identities"":[],""is_admin"":false,""note"":null,""namespace_id"":351,""created_by"":{""id"":139,""username"":""abc2"",""name"":""abc2"",""state"":""active"",""locked"":false,""avatar_url"":"""",""web_url"":""https://custom.gitlab.com/abc2""},""using_license_seat"":true,""is_auditor"":false,""provisioned_by_group_id"":null,""enterprise_group_id"":null,""enterprise_group_associated_at"":null}",https://custom.gitlab.com/api/v4/users,null,2022-07-13 13:15:05.261
+2,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":2994199,""username"":""abc2"",""name"":""abc2"",""state"":""active"",""locked"":false,""avatar_url"":"""",""web_url"":""https://custom.gitlab.com/abc2"",""created_at"":""2019-03-25T09:43:17.048+01:00"",""bio"":"""",""location"":"""",""public_email"":"""",""skype"":"""",""linkedin"":"""",""twitter"":"""",""discord"":"""",""website_url"":"""",""organization"":"""",""job_title"":"""",""pronouns"":null,""bot"":false,""work_information"":null,""followers"":0,""following"":1,""is_followed"":false,""local_time"":null,""last_sign_in_at"":""2024-01-04T09:43:11.817+01:00"",""confirmed_at"":""2021-03-25T09:43:16.650+01:00"",""last_activity_on"":""2024-01-09"",""email"":""abc2@dev.com"",""theme_id"":11,""color_scheme_id"":2,""projects_limit"":10,""current_sign_in_at"":""2024-01-08T16:16:41.386+01:00"",""identities"":[],""can_create_group"":false,""can_create_project"":true,""two_factor_enabled"":false,""external"":false,""private_profile"":false,""commit_email"":""abc2@dev.com"",""shared_runners_minutes_limit"":null,""extra_shared_runners_minutes_limit"":null,""scim_identities"":[],""is_admin"":false,""note"":null,""namespace_id"":351,""created_by"":{""id"":139,""username"":""abc2"",""name"":""abc2"",""state"":""active"",""locked"":false,""avatar_url"":"""",""web_url"":""https://custom.gitlab.com/abc2""},""using_license_seat"":true,""is_auditor"":false,""provisioned_by_group_id"":null,""enterprise_group_id"":null,""enterprise_group_associated_at"":null}",https://custom.gitlab.com/api/v4/users,null,2022-07-13 13:15:05.261
+3,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":2994200,""username"":""abc3"",""name"":""abc3"",""state"":""active"",""locked"":false,""avatar_url"":"""",""web_url"":""https://custom.gitlab.com/abc3"",""created_at"":""2020-03-25T09:43:17.048+01:00"",""bio"":"""",""location"":"""",""public_email"":"""",""skype"":"""",""linkedin"":"""",""twitter"":"""",""discord"":"""",""website_url"":"""",""organization"":"""",""job_title"":"""",""pronouns"":null,""bot"":false,""work_information"":null,""followers"":0,""following"":1,""is_followed"":false,""local_time"":null,""last_sign_in_at"":""2024-01-04T09:43:11.817+01:00"",""confirmed_at"":""2021-03-25T09:43:16.650+01:00"",""last_activity_on"":""2024-01-09"",""email"":""abc3@dev.com"",""theme_id"":11,""color_scheme_id"":2,""projects_limit"":10,""current_sign_in_at"":""2024-01-08T16:16:41.386+01:00"",""identities"":[],""can_create_group"":false,""can_create_project"":true,""two_factor_enabled"":false,""external"":false,""private_profile"":false,""commit_email"":""abc2@dev.com"",""shared_runners_minutes_limit"":null,""extra_shared_runners_minutes_limit"":null,""scim_identities"":[],""is_admin"":false,""note"":null,""namespace_id"":351,""created_by"":{""id"":139,""username"":""abc2"",""name"":""abc2"",""state"":""active"",""locked"":false,""avatar_url"":"""",""web_url"":""https://custom.gitlab.com/abc2""},""using_license_seat"":true,""is_auditor"":false,""provisioned_by_group_id"":null,""enterprise_group_id"":null,""enterprise_group_associated_at"":null}",https://custom.gitlab.com/api/v4/users,null,2022-07-13 13:15:05.261
+4,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":2944200,""username"":""abc4"",""name"":""abc4"",""state"":""active"",""locked"":false,""avatar_url"":"""",""web_url"":""https://custom.gitlab.com/abc4"",""created_at"":""2021-03-25T09:43:17.048+01:00"",""bio"":"""",""location"":"""",""public_email"":"""",""skype"":"""",""linkedin"":"""",""twitter"":"""",""discord"":"""",""website_url"":"""",""organization"":"""",""job_title"":"""",""pronouns"":null,""bot"":false,""work_information"":null,""followers"":0,""following"":1,""is_followed"":false,""local_time"":null,""last_sign_in_at"":""2024-01-04T09:43:11.817+01:00"",""confirmed_at"":""2021-03-25T09:43:16.650+01:00"",""last_activity_on"":""2024-01-09"",""email"":""abc4@dev.com"",""theme_id"":11,""color_scheme_id"":2,""projects_limit"":10,""current_sign_in_at"":""2024-01-08T16:16:41.386+01:00"",""identities"":[],""can_create_group"":false,""can_create_project"":true,""two_factor_enabled"":false,""external"":false,""private_profile"":false,""commit_email"":""abc2@dev.com"",""shared_runners_minutes_limit"":null,""extra_shared_runners_minutes_limit"":null,""scim_identities"":[],""is_admin"":false,""note"":null,""namespace_id"":351,""created_by"":{""id"":139,""username"":""abc2"",""name"":""abc2"",""state"":""active"",""locked"":false,""avatar_url"":"""",""web_url"":""https://custom.gitlab.com/abc2""},""using_license_seat"":true,""is_auditor"":false,""provisioned_by_group_id"":null,""enterprise_group_id"":null,""enterprise_group_associated_at"":null}",https://custom.gitlab.com/api/v4/users,null,2022-07-13 13:15:05.261
+5,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":2944220,""username"":""abc5"",""name"":""abc5"",""state"":""active"",""locked"":false,""avatar_url"":"""",""web_url"":""https://custom.gitlab.com/abc5"",""created_at"":""2022-03-25T09:43:17.048+01:00"",""bio"":"""",""location"":"""",""public_email"":"""",""skype"":"""",""linkedin"":"""",""twitter"":"""",""discord"":"""",""website_url"":"""",""organization"":"""",""job_title"":"""",""pronouns"":null,""bot"":false,""work_information"":null,""followers"":0,""following"":1,""is_followed"":false,""local_time"":null,""last_sign_in_at"":""2024-01-04T09:43:11.817+01:00"",""confirmed_at"":""2021-03-25T09:43:16.650+01:00"",""last_activity_on"":""2024-01-09"",""email"":""abc5@dev.com"",""theme_id"":11,""color_scheme_id"":2,""projects_limit"":10,""current_sign_in_at"":""2024-01-08T16:16:41.386+01:00"",""identities"":[],""can_create_group"":false,""can_create_project"":true,""two_factor_enabled"":false,""external"":false,""private_profile"":false,""commit_email"":""abc2@dev.com"",""shared_runners_minutes_limit"":null,""extra_shared_runners_minutes_limit"":null,""scim_identities"":[],""is_admin"":false,""note"":null,""namespace_id"":351,""created_by"":{""id"":139,""username"":""abc2"",""name"":""abc2"",""state"":""active"",""locked"":false,""avatar_url"":"""",""web_url"":""https://custom.gitlab.com/abc2""},""using_license_seat"":true,""is_auditor"":false,""provisioned_by_group_id"":null,""enterprise_group_id"":null,""enterprise_group_associated_at"":null}",https://custom.gitlab.com/api/v4/users,null,2022-07-13 13:15:05.261
+6,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":2944221,""username"":""abc6"",""name"":""abc6"",""state"":""active"",""locked"":false,""avatar_url"":"""",""web_url"":""https://custom.gitlab.com/abc6"",""created_at"":""2023-03-25T09:43:17.048+01:00"",""bio"":"""",""location"":"""",""public_email"":"""",""skype"":"""",""linkedin"":"""",""twitter"":"""",""discord"":"""",""website_url"":"""",""organization"":"""",""job_title"":"""",""pronouns"":null,""bot"":false,""work_information"":null,""followers"":0,""following"":1,""is_followed"":false,""local_time"":null,""last_sign_in_at"":""2024-01-04T09:43:11.817+01:00"",""confirmed_at"":""2021-03-25T09:43:16.650+01:00"",""last_activity_on"":""2024-01-09"",""email"":""abc6@dev.com"",""theme_id"":11,""color_scheme_id"":2,""projects_limit"":10,""current_sign_in_at"":""2024-01-08T16:16:41.386+01:00"",""identities"":[],""can_create_group"":false,""can_create_project"":true,""two_factor_enabled"":false,""external"":false,""private_profile"":false,""commit_email"":""abc2@dev.com"",""shared_runners_minutes_limit"":null,""extra_shared_runners_minutes_limit"":null,""scim_identities"":[],""is_admin"":false,""note"":null,""namespace_id"":351,""created_by"":{""id"":139,""username"":""abc2"",""name"":""abc2"",""state"":""active"",""locked"":false,""avatar_url"":"""",""web_url"":""https://custom.gitlab.com/abc2""},""using_license_seat"":true,""is_auditor"":false,""provisioned_by_group_id"":null,""enterprise_group_id"":null,""enterprise_group_associated_at"":null}",https://custom.gitlab.com/api/v4/users,null,2022-07-13 13:15:05.261

backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_accounts_direct_api.csv

@@ -0,0 +1,7 @@
+connection_id,gitlab_id,username,email,name,state,membership_state,avatar_url,web_url,created_user_at,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+1,2994198,abc1,abc1@dev.com,abc1,active,active,,https://custom.gitlab.com/abc1,2018-03-25T08:43:17.048+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_users,1,
+1,2994199,abc2,abc2@dev.com,abc2,active,active,,https://custom.gitlab.com/abc2,2019-03-25T08:43:17.048+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_users,2,
+1,2994200,abc3,abc3@dev.com,abc3,active,active,,https://custom.gitlab.com/abc3,2020-03-25T08:43:17.048+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_users,3,
+1,2944200,abc4,abc4@dev.com,abc4,active,active,,https://custom.gitlab.com/abc4,2021-03-25T08:43:17.048+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_users,4,
+1,2944220,abc5,abc5@dev.com,abc5,active,active,,https://custom.gitlab.com/abc5,2022-03-25T08:43:17.048+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_users,5,
+1,2944221,abc6,abc6@dev.com,abc6,active,active,,https://custom.gitlab.com/abc6,2023-03-25T08:43:17.048+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_users,6,

backend/plugins/gitlab/e2e/snapshot_tables/accounts_direct_api.csv

@@ -0,0 +1,7 @@
+id,email,full_name,user_name,avatar_url,organization,created_date,status,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+gitlab:GitlabAccount:1:2994198,abc1@dev.com,abc1,abc1,,,2018-03-25T08:43:17.048+00:00,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_users,1,
+gitlab:GitlabAccount:1:2994199,abc2@dev.com,abc2,abc2,,,2019-03-25T08:43:17.048+00:00,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_users,2,
+gitlab:GitlabAccount:1:2994200,abc3@dev.com,abc3,abc3,,,2020-03-25T08:43:17.048+00:00,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_users,3,
+gitlab:GitlabAccount:1:2944200,abc4@dev.com,abc4,abc4,,,2021-03-25T08:43:17.048+00:00,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_users,4,
+gitlab:GitlabAccount:1:2944220,abc5@dev.com,abc5,abc5,,,2022-03-25T08:43:17.048+00:00,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_users,5,
+gitlab:GitlabAccount:1:2944221,abc6@dev.com,abc6,abc6,,,2023-03-25T08:43:17.048+00:00,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_users,6,

backend/plugins/gitlab/models/account.go

@@ -18,19 +18,22 @@ limitations under the License.
 package models
 
 import (
+	"time"
+
 	"github.com/apache/incubator-devlake/core/models/common"
 )
 
 type GitlabAccount struct {
-	ConnectionId    uint64 `gorm:"primaryKey"`
-	GitlabId        int    `gorm:"primaryKey" json:"id"`
-	Username        string `gorm:"type:varchar(255)"`
-	Email           string `gorm:"type:varchar(255)"`
-	Name            string `gorm:"type:varchar(255)"`
-	State           string `gorm:"type:varchar(255)"`
-	MembershipState string `json:"membership_state" gorm:"type:varchar(255)"`
-	AvatarUrl       string `json:"avatar_url" gorm:"type:varchar(255)"`
-	WebUrl          string `json:"web_url" gorm:"type:varchar(255)"`
+	ConnectionId    uint64     `gorm:"primaryKey"`
+	GitlabId        int        `gorm:"primaryKey" json:"id"`
+	Username        string     `gorm:"type:varchar(255)"`
+	Email           string     `gorm:"type:varchar(255)"`
+	Name            string     `gorm:"type:varchar(255)"`
+	State           string     `gorm:"type:varchar(255)"`
+	MembershipState string     `json:"membership_state" gorm:"type:varchar(255)"`
+	AvatarUrl       string     `json:"avatar_url" gorm:"type:varchar(255)"`
+	WebUrl          string     `json:"web_url" gorm:"type:varchar(255)"`
+	CreatedUserAt   *time.Time `json:"created_at"`
 
 	common.NoPKModel
 }

backend/plugins/gitlab/models/migrationscripts/20231220_add_created_user_at_tool_gitlab_account.go

@@ -0,0 +1,48 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+	"time"
+
+	"github.com/apache/incubator-devlake/core/context"
+	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/helpers/migrationhelper"
+)
+
+type gitlabAccount20231120 struct {
+	CreatedUserAt *time.Time
+}
+
+func (gitlabAccount20231120) TableName() string {
+	return "_tool_gitlab_accounts"
+}
+
+type addUserCreationAt20231120 struct{}
+
+func (*addUserCreationAt20231120) Up(basicRes context.BasicRes) errors.Error {
+	return migrationhelper.AutoMigrateTables(basicRes, &gitlabAccount20231120{})
+}
+
+func (*addUserCreationAt20231120) Version() uint64 {
+	return 20231226140000
+}
+
+func (*addUserCreationAt20231120) Name() string {
+	return "add created_user_at column to _tool_gitlab_accounts"
+}

backend/plugins/gitlab/models/migrationscripts/register.go

@@ -41,6 +41,7 @@ func All() []plugin.MigrationScript {
 		new(addRawParamTableForScope),
 		new(addProjectArchived),
 		new(addDeployment),
+		new(addUserCreationAt20231120),
 		new(addEnvNamePattern),
 		new(addQueuedDuration20231129),
 		new(modifyDeploymentMessageType),

backend/plugins/gitlab/tasks/account_collector.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"net/http"
 	"net/url"
+	"strings"
 
 	"github.com/apache/incubator-devlake/core/errors"
 	"github.com/apache/incubator-devlake/core/plugin"
@@ -56,6 +57,11 @@ func CollectAccounts(taskCtx plugin.SubTaskContext) errors.Error {
 		urlTemplate = "/projects/{{ .Params.ProjectId }}/members/"
 	}
 
+	// Collect all users if endpoint is private gitlab instance
+	if !strings.HasPrefix(data.ApiClient.GetEndpoint(), "https://gitlab.com") {
+		urlTemplate = "/users"
+	}
+
 	collector, err := api.NewApiCollector(api.ApiCollectorArgs{
 		RawDataSubTaskArgs: *rawDataSubTaskArgs,
 		ApiClient:          data.ApiClient,

backend/plugins/gitlab/tasks/account_convertor.go

@@ -73,6 +73,7 @@ func ConvertAccounts(taskCtx plugin.SubTaskContext) errors.Error {
 				FullName:     GitlabAccount.Name,
 				Email:        GitlabAccount.Email,
 				AvatarUrl:    GitlabAccount.AvatarUrl,
+				CreatedDate:  GitlabAccount.CreatedUserAt,
 			}
 
 			return []interface{}{