diff --git a/backend/python/plugins/azuredevops/azuredevops/models.py b/backend/python/plugins/azuredevops/azuredevops/models.py index d77d9e6916f..63765e5e223 100644 --- a/backend/python/plugins/azuredevops/azuredevops/models.py +++ b/backend/python/plugins/azuredevops/azuredevops/models.py @@ -158,7 +158,11 @@ def add_raw_data_params_table_to_scope(b: MigrationScriptBuilder): @migration(20230802000001, name="rename startTime/finishTime to start_time/finish_time") def rename_starttime_and_finishtime_for_job(b: MigrationScriptBuilder): - b.execute(f'ALTER TABLE _tool_azuredevops_jobs RENAME COLUMN startTime TO start_time', Dialect.MYSQL, ignore_error=True) - b.execute(f'ALTER TABLE _tool_azuredevops_jobs RENAME COLUMN finishTime TO finish_time', Dialect.MYSQL, ignore_error=True) - b.execute(f'ALTER TABLE _tool_azuredevops_jobs RENAME COLUMN `startTime` TO start_time', Dialect.POSTGRESQL, ignore_error=True) - b.execute(f'ALTER TABLE _tool_azuredevops_jobs RENAME COLUMN `finishTime` TO finish_time', Dialect.POSTGRESQL, ignore_error=True) \ No newline at end of file + b.rename_column('_tool_azuredevops_jobs', 'startTime', 'start_time') + b.rename_column('_tool_azuredevops_jobs', 'finishTime', 'finish_time') + + +@migration(20230825150421, name="add missing migrations from 0.17 to 0.18") +def add_missing_migrations_0_17_to_0_18(b: MigrationScriptBuilder): + b.rename_column('_tool_azuredevops_gitrepositories', 'transformation_rule_id', 'scope_config_id') + b.add_column('_tool_azuredevops_gitrepositories', 'provider', 'varchar(255)') \ No newline at end of file diff --git a/backend/python/pydevlake/pydevlake/model.py b/backend/python/pydevlake/pydevlake/model.py index 4b7f9bfcc79..dc801e35eb5 100644 --- a/backend/python/pydevlake/pydevlake/model.py +++ b/backend/python/pydevlake/pydevlake/model.py @@ -87,6 +87,12 @@ class ScopeConfig(ToolTable, Model): name: str = Field(default="default") domain_types: list[DomainType] = Field(default=list(DomainType), alias="entities") + @validator('domain_types', pre=True, always=True) + def set_default_domain_types(cls, v): + if v is None: + return list(DomainType) + return v + class RawModel(SQLModel): id: int = Field(primary_key=True) diff --git a/backend/server/services/remote/models/migration.go b/backend/server/services/remote/models/migration.go index 129b66ce4f5..0d92d606b01 100644 --- a/backend/server/services/remote/models/migration.go +++ b/backend/server/services/remote/models/migration.go @@ -97,6 +97,27 @@ func (o DropTableOperation) Execute(dal dal.Dal) errors.Error { var _ Operation = (*DropTableOperation)(nil) +type RenameColumnOperation struct { + Table string `json:"table"` + OldName string `json:"old_name"` + NewName string `json:"new_name"` +} + +func (o RenameColumnOperation) Execute(dal dal.Dal) errors.Error { + if !dal.HasColumn(o.Table, o.OldName) { + return nil + } + if dal.HasColumn(o.Table, o.NewName) { + err := dal.DropColumns(o.Table, o.NewName) + if err != nil { + return err + } + } + return dal.RenameColumn(o.Table, o.OldName, o.NewName) +} + +var _ Operation = (*RenameTableOperation)(nil) + type RenameTableOperation struct { OldName string `json:"old_name"` NewName string `json:"new_name"` @@ -155,10 +176,12 @@ func (s *RemoteMigrationScript) UnmarshalJSON(data []byte) error { operation = &DropColumnOperation{} case "drop_table": operation = &DropTableOperation{} + case "rename_column": + operation = &RenameColumnOperation{} case "rename_table": operation = &RenameTableOperation{} default: - return errors.BadInput.New("unsupported operation type") + return errors.BadInput.New("unsupported operation type: " + operationType) } err = json.Unmarshal(operationRaw, operation) if err != nil { diff --git a/backend/server/services/remote/models/migrationscripts/azuredevops/azuredevops_20230714_decrypt_azdo_fields.go b/backend/server/services/remote/models/migrationscripts/azuredevops/azuredevops_20230714_decrypt_azdo_fields.go new file mode 100644 index 00000000000..d02cb10b00f --- /dev/null +++ b/backend/server/services/remote/models/migrationscripts/azuredevops/azuredevops_20230714_decrypt_azdo_fields.go @@ -0,0 +1,96 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azuredevops + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +var _ plugin.MigrationScript = (*DecryptConnectionFields)(nil) + +type azureDevopsConnection20230825 struct { + archived.Model + Name string + Token string + Proxy *string + Organization *string +} + +type DecryptConnectionFields struct{} + +func (script *DecryptConnectionFields) Up(basicRes context.BasicRes) errors.Error { + encryptionSecret := basicRes.GetConfig(plugin.EncodeKeyEnvStr) + if encryptionSecret == "" { + return errors.BadInput.New("invalid encryptionSecret") + } + + err := migrationhelper.TransformColumns( + basicRes, + script, + "_tool_azuredevops_azuredevopsconnections", + []string{"name", "proxy", "organization"}, + func(src *azureDevopsConnection20230825) (*azureDevopsConnection20230825, errors.Error) { + encName := src.Name + name, err := plugin.Decrypt(encryptionSecret, encName) + if err != nil { + return src, nil + } + src.Name = name + + if src.Proxy != nil { + encProxy := *src.Proxy + decProxy, err := plugin.Decrypt(encryptionSecret, encProxy) + if err != nil { + return src, nil + } + if decProxy == "" { + src.Proxy = nil + } else { + src.Proxy = &decProxy + } + } + + if src.Organization != nil { + encOrg := *src.Organization + decOrg, err := plugin.Decrypt(encryptionSecret, encOrg) + if err != nil { + return src, nil + } + if decOrg == "" { + src.Organization = nil + } else { + src.Organization = &decOrg + } + } + return src, nil + }, + ) + return err +} + +func (*DecryptConnectionFields) Version() uint64 { + return 20230825090504 +} + +func (script *DecryptConnectionFields) Name() string { + return "Decrypt Azure DevOps connection fields" +} diff --git a/backend/server/services/remote/models/migrationscripts/register.go b/backend/server/services/remote/models/migrationscripts/register.go index af1e9f94950..15565c30a90 100644 --- a/backend/server/services/remote/models/migrationscripts/register.go +++ b/backend/server/services/remote/models/migrationscripts/register.go @@ -25,6 +25,7 @@ import ( var allMigrations = map[string][]plugin.MigrationScript{ "azuredevops": { &azuredevops.AddRawDataForScope{}, + &azuredevops.DecryptConnectionFields{}, }, }