feat(customize): add account handling for imported issues

backend/plugins/customize/e2e/import_issues_test.go

@@ -21,6 +21,7 @@ import (
 	"os"
 	"testing"
 
+	"github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain"
 	"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
 	"github.com/apache/incubator-devlake/helpers/e2ehelper"
 	"github.com/apache/incubator-devlake/plugins/customize/impl"
@@ -37,6 +38,7 @@ func TestImportIssueDataFlow(t *testing.T) {
 	dataflowTester.FlushTabler(&models.CustomizedField{})
 	dataflowTester.FlushTabler(&ticket.IssueLabel{})
 	dataflowTester.FlushTabler(&ticket.BoardIssue{})
+	dataflowTester.FlushTabler(&crossdomain.Account{})
 	svc := service.NewService(dataflowTester.Dal)
 	err := svc.CreateField(&models.CustomizedField{
 		TbName:      "issues",
@@ -172,4 +174,13 @@ func TestImportIssueDataFlow(t *testing.T) {
 			"board_id",
 			"issue_id",
 		})
+	dataflowTester.VerifyTableWithRawData(
+		&crossdomain.Account{},
+		"snapshot_tables/accounts.csv",
+		[]string{
+			"id",
+			"full_name",
+			"user_name",
+		},
+	)
 }

backend/plugins/customize/e2e/raw_tables/issues_input.csv

@@ -1,4 +1,4 @@
 id,url,issue_key,title,original_type,original_status,created_date,resolution_date,story_point,priority,severity,original_estimate_minutes,time_spent_minutes,component,epic_key,creator_name,assignee_name,x_int,x_time,x_varchar,x_float,labels
-csv:1,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/1,1,issue test,BUG,new,2022-07-17 07:15:55.959+00:00,NULL,0,major,,0,0,,,tgp,tgp,10,2022-09-15 15:27:56,world,8,NULL
-csv:10,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/10,10,issue test007,BUG,new,2022-08-12 13:43:00.783+00:00,NULL,0,trivial,,0,0,,,tgp,tgp,30,2022-09-15 15:27:56,abc,24590,hello worlds
-csv:11,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/11,11,issue test011,REQUIREMENT,new,2022-08-10 13:44:46.508+00:00,NULL,0,major,,0,0,,,tgp,,1,2022-09-15 15:27:56,NULL,0.00014,NULL
+csv:1,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/1,1,issue test,BUG,new,2022-07-17 07:15:55.959+00:00,NULL,0,major,,0,0,,,tgp,klesh,10,2022-09-15 15:27:56,world,8,NULL
+csv:10,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/10,10,issue test007,BUG,new,2022-08-12 13:43:00.783+00:00,NULL,0,trivial,,0,0,,,tgp,warren,30,2022-09-15 15:27:56,abc,24590,hello worlds
+csv:11,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/11,11,issue test011,REQUIREMENT,new,2022-08-10 13:44:46.508+00:00,NULL,0,major,,0,0,,,tgp,abeizn,1,2022-09-15 15:27:56,NULL,0.00014,NULL

backend/plugins/customize/e2e/snapshot_tables/accounts.csv

@@ -0,0 +1,5 @@
+id,full_name,user_name,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+csv:CsvAccount:0:abeizn,abeizn,abeizn,csv-board,,0,
+csv:CsvAccount:0:klesh,klesh,klesh,csv-board,,0,
+csv:CsvAccount:0:tgp,tgp,tgp,csv-board2,,0,
+csv:CsvAccount:0:warren,warren,warren,csv-board,,0,

backend/plugins/customize/e2e/snapshot_tables/issues_output.csv

@@ -1,7 +1,7 @@
 id,url,icon_url,issue_key,title,description,epic_key,type,original_type,status,original_status,story_point,resolution_date,created_date,updated_date,lead_time_minutes,parent_issue_id,priority,original_estimate_minutes,time_spent_minutes,time_remaining_minutes,creator_id,creator_name,assignee_id,assignee_name,severity,component,original_project,x_varchar,x_text,x_time,x_float,x_int,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-csv:1,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/1,,1,issue test,,,,BUG,,new,0,,2022-07-17T07:15:55.959+00:00,,,,major,0,0,,,tgp,,tgp,,,,world,,2022-09-15T15:27:56.000+00:00,8,10,csv-board2,,,
-csv:10,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/10,,10,issue title edited,,,,BUG,,new,0,,2022-08-12T13:43:00.783+00:00,,,,trivial,0,0,,,tgp,,tgp,,,,abc,,2022-09-15T15:27:56.000+00:00,24590,30,csv-board2,,,
-csv:11,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/11,,11,issue test011,,,,REQUIREMENT,,new,0,,2022-08-10T13:44:46.508+00:00,,,,major,0,0,,,tgp,,,,,,,,2022-09-15T15:27:56.000+00:00,0.00014,1,csv-board,,,
-csv:12,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/12,,12,issue test012,,,,REQUIREMENT,,new,0,,2022-08-11T13:44:46.508+00:00,,,,major,0,0,,,tgp,,,,,,,,2022-09-15T15:27:56.000+00:00,0.00014,1,csv-board,,,
-csv:13,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/13,,13,issue test013,,,,REQUIREMENT,,new,0,,2022-08-12T13:44:46.508+00:00,,,,critical,0,0,,,tgp,,,,,,,,2022-09-15T15:27:56.000+00:00,0.00014,1,csv-board2,,,
-csv:14,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/14,,14,issue test014,,,,INCIDENT,,new,0,,2022-08-12T13:45:12.810+00:00,,,,blocker,0,0,,,tgp,,tgp,,,,,,2022-09-15T15:27:56.000+00:00,,41534568464351,csv-board2,,,
+csv:1,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/1,,1,issue test,,,,BUG,,new,0,,2022-07-17T07:15:55.959+00:00,,,,major,0,0,,csv:CsvAccount:0:tgp,tgp,csv:CsvAccount:0:tgp,tgp,,,,world,,2022-09-15T15:27:56.000+00:00,8,10,csv-board2,,,
+csv:10,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/10,,10,issue title edited,,,,BUG,,new,0,,2022-08-12T13:43:00.783+00:00,,,,trivial,0,0,,csv:CsvAccount:0:tgp,tgp,csv:CsvAccount:0:tgp,tgp,,,,abc,,2022-09-15T15:27:56.000+00:00,24590,30,csv-board2,,,
+csv:11,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/11,,11,issue test011,,,,REQUIREMENT,,new,0,,2022-08-10T13:44:46.508+00:00,,,,major,0,0,,csv:CsvAccount:0:tgp,tgp,csv:CsvAccount:0:abeizn,abeizn,,,,,,2022-09-15T15:27:56.000+00:00,0.00014,1,csv-board,,,
+csv:12,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/12,,12,issue test012,,,,REQUIREMENT,,new,0,,2022-08-11T13:44:46.508+00:00,,,,major,0,0,,csv:CsvAccount:0:tgp,tgp,,,,,,,,2022-09-15T15:27:56.000+00:00,0.00014,1,csv-board,,,
+csv:13,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/13,,13,issue test013,,,,REQUIREMENT,,new,0,,2022-08-12T13:44:46.508+00:00,,,,critical,0,0,,csv:CsvAccount:0:tgp,tgp,,,,,,,,2022-09-15T15:27:56.000+00:00,0.00014,1,csv-board2,,,
+csv:14,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/14,,14,issue test014,,,,INCIDENT,,new,0,,2022-08-12T13:45:12.810+00:00,,,,blocker,0,0,,csv:CsvAccount:0:tgp,tgp,csv:CsvAccount:0:tgp,tgp,,,,,,2022-09-15T15:27:56.000+00:00,,41534568464351,csv-board2,,,

backend/plugins/customize/service/service.go

@@ -156,6 +156,7 @@ func (s *Service) getCustomizedFields(table string) ([]models.CustomizedField, e
 // issue could exist in multiple boards, so we should only delete an old records when it doesn't belong to another board
 func (s *Service) ImportIssue(boardId string, file io.ReadCloser, incremental bool) errors.Error {
 	if !incremental {
+		// not delete accounts data since account may be referenced by others
 		err := s.dal.Delete(
 			&ticket.Issue{},
 			dal.Where("id IN (SELECT issue_id FROM board_issues WHERE board_id=? AND issue_id NOT IN (SELECT issue_id FROM board_issues WHERE board_id!=?))", boardId, boardId),
@@ -262,23 +263,74 @@ func (s *Service) importCSV(file io.ReadCloser, rawDataParams string, recordHand
 	}
 }
 
+// createOrUpdateAccount creates or updates an account based on the provided name.
+// It returns the account ID and an error if any occurred.
+func (s *Service) createOrUpdateAccount(accountName string, rawDataParams string) (string, errors.Error) {
+	if accountName == "" {
+		return "", nil // Return empty ID if name is empty, no error needed here.
+	}
+	now := time.Now()
+	accountId := fmt.Sprintf("csv:CsvAccount:0:%s", accountName)
+	account := &crossdomain.Account{
+		DomainEntity: domainlayer.DomainEntity{
+			Id: accountId,
+			NoPKModel: common.NoPKModel{
+				RawDataOrigin: common.RawDataOrigin{
+					RawDataParams: rawDataParams,
+				},
+			},
+		},
+		FullName:    accountName,
+		UserName:    accountName,
+		CreatedDate: &now,
+	}
+	err := s.dal.CreateOrUpdate(account)
+	if err != nil {
+		return "", errors.Default.Wrap(err, fmt.Sprintf("failed to create or update account for %s", accountName))
+	}
+	return accountId, nil
+}
+
+// getStringField extracts a string field from a record map.
+// If required is true, it returns an error if the field is missing, nil, empty, or not a string.
+// If required is false, it returns an empty string without error if the field is missing or nil,
+// but returns an error if the field exists and is not a string.
+func getStringField(record map[string]interface{}, fieldName string, required bool) (string, errors.Error) {
+	value, ok := record[fieldName]
+	if !ok || value == nil {
+		if required {
+			return "", errors.Default.New(fmt.Sprintf("record without required field %s", fieldName))
+		}
+		return "", nil // Field missing or nil, but not required
+	}
+
+	strValue, ok := value.(string)
+	if !ok {
+		return "", errors.Default.New(fmt.Sprintf("%s is not a string", fieldName))
+	}
+
+	if required && strValue == "" {
+		return "", errors.Default.New(fmt.Sprintf("invalid or empty required field %s", fieldName))
+	}
+
+	return strValue, nil
+}
+
 // issueHandlerFactory returns a handler that save record into `issues`, `board_issues` and `issue_labels` table
 func (s *Service) issueHandlerFactory(boardId string, incremental bool) func(record map[string]interface{}) errors.Error {
 	return func(record map[string]interface{}) errors.Error {
 		var err errors.Error
-		var id string
-		if record["id"] == nil {
-			return errors.Default.New("record without id")
+		id, err := getStringField(record, "id", true)
+		if err != nil {
+			return err
 		}
-		id, _ = record["id"].(string)
-		if id == "" {
-			return errors.Default.New("empty id")
+
+		// Handle labels
+		labels, err := getStringField(record, "labels", false)
+		if err != nil {
+			return err
 		}
-		if record["labels"] != nil {
-			labels, ok := record["labels"].(string)
-			if !ok {
-				return errors.Default.New("labels is not string")
-			}
+		if labels != "" {
 			var issueLabels []*ticket.IssueLabel
 			appearedLabels := make(map[string]struct{}) // record the labels that have appeared
 			for _, label := range strings.Split(labels, ",") {
@@ -307,15 +359,56 @@ func (s *Service) issueHandlerFactory(boardId string, incremental bool) func(rec
 				}
 			}
 		}
-		delete(record, "labels")
+		delete(record, "labels") // Remove labels from record map as it's handled
+
+		// Handle creator and assignee accounts
+		rawDataParams, err := getStringField(record, "_raw_data_params", true)
+		if err != nil {
+			// This should ideally not happen as it's set in importCSV, but good to check
+			return err
+		}
+
+		// Handle creator
+		creatorName, err := getStringField(record, "creator_name", false)
+		if err != nil {
+			return err
+		}
+		creatorId, err := s.createOrUpdateAccount(creatorName, rawDataParams)
+		if err != nil {
+			return err
+		}
+		if creatorId != "" {
+			record["creator_id"] = creatorId
+		}
+
+		// Handle assignee
+		assigneeName, err := getStringField(record, "assignee_name", false)
+		if err != nil {
+			return err
+		}
+		assigneeId, err := s.createOrUpdateAccount(assigneeName, rawDataParams)
+		if err != nil {
+			return err
+		}
+		if assigneeId != "" {
+			record["assignee_id"] = assigneeId
+		}
+
+		// Handle issues
 		err = s.dal.CreateWithMap(&ticket.Issue{}, record)
 		if err != nil {
 			return err
 		}
-		return s.dal.CreateOrUpdate(&ticket.BoardIssue{
+
+		// Handle board_issues
+		err = s.dal.CreateOrUpdate(&ticket.BoardIssue{
 			BoardId: boardId,
 			IssueId: id,
 		})
+		if err != nil {
+			return err
+		}
+		return nil
 	}
 }
 

backend/plugins/customize/service/service_test.go

@@ -21,6 +21,8 @@ import (
 	"regexp"
 	"strings"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
 )
 
 func TestService_checkFieldName(t *testing.T) {
@@ -75,3 +77,116 @@ func TestService_checkFieldName(t *testing.T) {
 		})
 	}
 }
+
+func TestGetStringField(t *testing.T) {
+	testCases := []struct {
+		name        string
+		record      map[string]interface{}
+		fieldName   string
+		required    bool
+		expectValue string
+		expectError bool
+		errorMsg    string
+	}{
+		// Required field tests
+		{
+			name:        "Required field exists and valid",
+			record:      map[string]interface{}{"id": "123"},
+			fieldName:   "id",
+			required:    true,
+			expectValue: "123",
+			expectError: false,
+		},
+		{
+			name:        "Required field exists but empty",
+			record:      map[string]interface{}{"id": ""},
+			fieldName:   "id",
+			required:    true,
+			expectValue: "",
+			expectError: true,
+			errorMsg:    "invalid or empty required field id",
+		},
+		{
+			name:        "Required field exists but wrong type",
+			record:      map[string]interface{}{"id": 123},
+			fieldName:   "id",
+			required:    true,
+			expectValue: "",
+			expectError: true,
+			errorMsg:    "id is not a string",
+		},
+		{
+			name:        "Required field missing",
+			record:      map[string]interface{}{"name": "test"},
+			fieldName:   "id",
+			required:    true,
+			expectValue: "",
+			expectError: true,
+			errorMsg:    "record without required field id",
+		},
+		{
+			name:        "Required field nil",
+			record:      map[string]interface{}{"id": nil},
+			fieldName:   "id",
+			required:    true,
+			expectValue: "",
+			expectError: true,
+			errorMsg:    "record without required field id",
+		},
+		// Optional field tests
+		{
+			name:        "Optional field exists and valid",
+			record:      map[string]interface{}{"label": "bug"},
+			fieldName:   "label",
+			required:    false,
+			expectValue: "bug",
+			expectError: false,
+		},
+		{
+			name:        "Optional field exists but empty",
+			record:      map[string]interface{}{"label": ""},
+			fieldName:   "label",
+			required:    false,
+			expectValue: "",
+			expectError: false,
+		},
+		{
+			name:        "Optional field exists but wrong type",
+			record:      map[string]interface{}{"label": 123},
+			fieldName:   "label",
+			required:    false,
+			expectValue: "",
+			expectError: true,
+			errorMsg:    "label is not a string",
+		},
+		{
+			name:        "Optional field missing",
+			record:      map[string]interface{}{"name": "test"},
+			fieldName:   "label",
+			required:    false,
+			expectValue: "",
+			expectError: false,
+		},
+		{
+			name:        "Optional field nil",
+			record:      map[string]interface{}{"label": nil},
+			fieldName:   "label",
+			required:    false,
+			expectValue: "",
+			expectError: false,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			value, err := getStringField(tc.record, tc.fieldName, tc.required)
+			assert.Equal(t, tc.expectValue, value)
+			if tc.expectError {
+				assert.Error(t, err)
+				assert.Contains(t, err.Error(), tc.errorMsg)
+			} else {
+				assert.NoError(t, err)
+			}
+		})
+	}
+}