fix: increase max URL length (#7493)

Supports longer azure devops organisaion names

backend/core/models/migrationscripts/20240521_modify_cicd_deployment_commits_repo_url_length.go

@@ -0,0 +1,48 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+	"github.com/apache/incubator-devlake/core/context"
+	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/core/plugin"
+)
+
+var _ plugin.MigrationScript = (*modifyCicdDeploymentCommitsRepoUrlLength)(nil)
+
+type modifyCicdDeploymentCommitsRepoUrlLength struct{}
+
+type cicdDeploymentCommitsRepoUrl20240521 struct {
+	RepoUrl string `gorm:"type:varchar(500)"`
+}
+
+func (cicdDeploymentCommitsRepoUrl20240521) TableName() string {
+	return "cicd_deployment_commits"
+}
+
+func (*modifyCicdDeploymentCommitsRepoUrlLength) Up(basicRes context.BasicRes) errors.Error {
+	return basicRes.GetDal().AutoMigrate(&cicdDeploymentCommitsRepoUrl20240521{})
+}
+
+func (*modifyCicdDeploymentCommitsRepoUrlLength) Version() uint64 {
+	return 20240521200500
+}
+
+func (*modifyCicdDeploymentCommitsRepoUrlLength) Name() string {
+	return "modify cicd_deployment_commits repo_url from 191 to 500"
+}

backend/core/models/migrationscripts/20240521_modify_cicd_pipeline_commits_repo_url_length.go

@@ -0,0 +1,48 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+	"github.com/apache/incubator-devlake/core/context"
+	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/core/plugin"
+)
+
+var _ plugin.MigrationScript = (*modifyCicdPipelineCommitsRepoUrlLength)(nil)
+
+type modifyCicdPipelineCommitsRepoUrlLength struct{}
+
+type cicdPipelineCommitsRepoUrl20240521 struct {
+	RepoUrl string `gorm:"type:varchar(500)"`
+}
+
+func (cicdPipelineCommitsRepoUrl20240521) TableName() string {
+	return "cicd_pipeline_commits"
+}
+
+func (*modifyCicdPipelineCommitsRepoUrlLength) Up(basicRes context.BasicRes) errors.Error {
+	return basicRes.GetDal().AutoMigrate(&cicdPipelineCommitsRepoUrl20240521{})
+}
+
+func (*modifyCicdPipelineCommitsRepoUrlLength) Version() uint64 {
+	return 20240521201000
+}
+
+func (*modifyCicdPipelineCommitsRepoUrlLength) Name() string {
+	return "modify cicd_pipeline_commits repo_url from 255 to 500"
+}

backend/core/models/migrationscripts/register.go

@@ -117,5 +117,7 @@ func All() []plugin.MigrationScript {
 		new(addCommitShaToCicdRelease),
 		new(updateIssueKeyType),
 		new(updatePluginOptionInProjectMetricSetting),
+		new(modifyCicdDeploymentCommitsRepoUrlLength),
+		new(modifyCicdPipelineCommitsRepoUrlLength),
 	}
 }

backend/python/plugins/azuredevops/azuredevops/migrations.py

@@ -204,4 +204,46 @@ def add_updated_date_field_in_tool_azuredevops_gitrepositories(b: MigrationScrip
 def add_queue_time_field_in_tool_azuredevops_builds(b: MigrationScriptBuilder):
     table = '_tool_azuredevops_builds'
     b.add_column(table, 'display_title', 'TEXT')
-    b.add_column(table, 'url', 'TEXT')
+    b.add_column(table, 'url', 'TEXT')
+
+@migration(20240521133000, name="Update url column in _tool_azuredevops_builds")
+def add_updated_url_column_length_in_tool_azuredevops_builds(b: MigrationScriptBuilder):
+    table = '_tool_azuredevops_builds'
+    b.execute(f'ALTER TABLE {table} MODIFY COLUMN url TEXT', Dialect.MYSQL)
+    b.execute(f'ALTER TABLE {table} ALTER COLUMN url TYPE TEXT', Dialect.POSTGRESQL)
+
+@migration(20240527113400, name="Update url column in _raw_azuredevops_builds ")
+def add_updated_url_column_length_in_raw_azuredevops_builds(b: MigrationScriptBuilder):
+    table = '_raw_azuredevops_builds'
+
+    b.execute(f"""
+    CREATE PROCEDURE alter_url_column_if_exists()
+    BEGIN
+        DECLARE table_exists INT DEFAULT 0;
+        SELECT COUNT(*) INTO table_exists
+        FROM information_schema.tables 
+        WHERE table_schema = DATABASE() 
+          AND table_name = '{table}';
+        IF table_exists > 0 THEN
+            ALTER TABLE {table} MODIFY COLUMN url TEXT;
+        ELSE
+            SELECT 'Table {table} does not exist' AS message;
+        END IF;
+    END;
+    """, Dialect.MYSQL)
+
+    b.execute(f"CALL alter_url_column_if_exists();", Dialect.MYSQL)
+    b.execute(f"DROP PROCEDURE alter_url_column_if_exists;", Dialect.MYSQL)
+
+    b.execute(f"""
+    DO $$
+    BEGIN
+        IF EXISTS (SELECT FROM information_schema.tables 
+                   WHERE table_schema = 'public' AND table_name = '{table}') THEN
+            ALTER TABLE {table} ALTER COLUMN url TYPE TEXT;
+        ELSE
+            RAISE NOTICE 'Table {table} does not exist';
+        END IF;
+    END $$;
+    """, Dialect.POSTGRESQL)
+

backend/python/pydevlake/pydevlake/model.py

@@ -103,7 +103,7 @@ class RawModel(SQLModel):
     id: int = Field(primary_key=True)
     params: str = b''
     data: bytes
-    url: str = b''
+    url: str = Field(default=b'', sa_column=Column(Text))
     input: bytes = b''
     created_at: datetime = Field(default_factory=datetime.now)