(cherry-pick #5957) Fix azure devops migration from 0.17 to 0.18

backend/python/plugins/azuredevops/azuredevops/models.py

@@ -158,7 +158,11 @@ def add_raw_data_params_table_to_scope(b: MigrationScriptBuilder):
 
 @migration(20230802000001, name="rename startTime/finishTime to start_time/finish_time")
 def rename_starttime_and_finishtime_for_job(b: MigrationScriptBuilder):
-    b.execute(f'ALTER TABLE _tool_azuredevops_jobs RENAME COLUMN startTime TO start_time', Dialect.MYSQL, ignore_error=True)
-    b.execute(f'ALTER TABLE _tool_azuredevops_jobs RENAME COLUMN finishTime TO finish_time', Dialect.MYSQL, ignore_error=True)
-    b.execute(f'ALTER TABLE _tool_azuredevops_jobs RENAME COLUMN `startTime` TO start_time', Dialect.POSTGRESQL, ignore_error=True)
-    b.execute(f'ALTER TABLE _tool_azuredevops_jobs RENAME COLUMN `finishTime` TO finish_time', Dialect.POSTGRESQL, ignore_error=True)
+    b.rename_column('_tool_azuredevops_jobs', 'startTime', 'start_time')
+    b.rename_column('_tool_azuredevops_jobs', 'finishTime', 'finish_time')
+
+
+@migration(20230825150421, name="add missing migrations from 0.17 to 0.18")
+def add_missing_migrations_0_17_to_0_18(b: MigrationScriptBuilder):
+    b.rename_column('_tool_azuredevops_gitrepositories', 'transformation_rule_id', 'scope_config_id')
+    b.add_column('_tool_azuredevops_gitrepositories', 'provider', 'varchar(255)')

backend/python/pydevlake/pydevlake/model.py

@@ -87,6 +87,12 @@ class ScopeConfig(ToolTable, Model):
     name: str = Field(default="default")
     domain_types: list[DomainType] = Field(default=list(DomainType), alias="entities")
 
+    @validator('domain_types', pre=True, always=True)
+    def set_default_domain_types(cls, v):
+        if v is None:
+            return list(DomainType)
+        return v
+
 
 class RawModel(SQLModel):
     id: int = Field(primary_key=True)

backend/server/services/remote/models/migration.go

@@ -97,6 +97,27 @@ func (o DropTableOperation) Execute(dal dal.Dal) errors.Error {
 
 var _ Operation = (*DropTableOperation)(nil)
 
+type RenameColumnOperation struct {
+	Table   string `json:"table"`
+	OldName string `json:"old_name"`
+	NewName string `json:"new_name"`
+}
+
+func (o RenameColumnOperation) Execute(dal dal.Dal) errors.Error {
+	if !dal.HasColumn(o.Table, o.OldName) {
+		return nil
+	}
+	if dal.HasColumn(o.Table, o.NewName) {
+		err := dal.DropColumns(o.Table, o.NewName)
+		if err != nil {
+			return err
+		}
+	}
+	return dal.RenameColumn(o.Table, o.OldName, o.NewName)
+}
+
+var _ Operation = (*RenameTableOperation)(nil)
+
 type RenameTableOperation struct {
 	OldName string `json:"old_name"`
 	NewName string `json:"new_name"`
@@ -155,10 +176,12 @@ func (s *RemoteMigrationScript) UnmarshalJSON(data []byte) error {
 			operation = &DropColumnOperation{}
 		case "drop_table":
 			operation = &DropTableOperation{}
+		case "rename_column":
+			operation = &RenameColumnOperation{}
 		case "rename_table":
 			operation = &RenameTableOperation{}
 		default:
-			return errors.BadInput.New("unsupported operation type")
+			return errors.BadInput.New("unsupported operation type: " + operationType)
 		}
 		err = json.Unmarshal(operationRaw, operation)
 		if err != nil {

backend/server/services/remote/models/migrationscripts/azuredevops/azuredevops_20230714_decrypt_azdo_fields.go

@@ -0,0 +1,96 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package azuredevops
+
+import (
+	"github.com/apache/incubator-devlake/core/context"
+	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
+	"github.com/apache/incubator-devlake/core/plugin"
+	"github.com/apache/incubator-devlake/helpers/migrationhelper"
+)
+
+var _ plugin.MigrationScript = (*DecryptConnectionFields)(nil)
+
+type azureDevopsConnection20230825 struct {
+	archived.Model
+	Name         string
+	Token        string
+	Proxy        *string
+	Organization *string
+}
+
+type DecryptConnectionFields struct{}
+
+func (script *DecryptConnectionFields) Up(basicRes context.BasicRes) errors.Error {
+	encryptionSecret := basicRes.GetConfig(plugin.EncodeKeyEnvStr)
+	if encryptionSecret == "" {
+		return errors.BadInput.New("invalid encryptionSecret")
+	}
+
+	err := migrationhelper.TransformColumns(
+		basicRes,
+		script,
+		"_tool_azuredevops_azuredevopsconnections",
+		[]string{"name", "proxy", "organization"},
+		func(src *azureDevopsConnection20230825) (*azureDevopsConnection20230825, errors.Error) {
+			encName := src.Name
+			name, err := plugin.Decrypt(encryptionSecret, encName)
+			if err != nil {
+				return src, nil
+			}
+			src.Name = name
+
+			if src.Proxy != nil {
+				encProxy := *src.Proxy
+				decProxy, err := plugin.Decrypt(encryptionSecret, encProxy)
+				if err != nil {
+					return src, nil
+				}
+				if decProxy == "" {
+					src.Proxy = nil
+				} else {
+					src.Proxy = &decProxy
+				}
+			}
+
+			if src.Organization != nil {
+				encOrg := *src.Organization
+				decOrg, err := plugin.Decrypt(encryptionSecret, encOrg)
+				if err != nil {
+					return src, nil
+				}
+				if decOrg == "" {
+					src.Organization = nil
+				} else {
+					src.Organization = &decOrg
+				}
+			}
+			return src, nil
+		},
+	)
+	return err
+}
+
+func (*DecryptConnectionFields) Version() uint64 {
+	return 20230825090504
+}
+
+func (script *DecryptConnectionFields) Name() string {
+	return "Decrypt Azure DevOps connection fields"
+}

backend/server/services/remote/models/migrationscripts/register.go

@@ -25,6 +25,7 @@ import (
 var allMigrations = map[string][]plugin.MigrationScript{
 	"azuredevops": {
 		&azuredevops.AddRawDataForScope{},
+		&azuredevops.DecryptConnectionFields{},
 	},
 }