diff --git a/backend/plugins/github/api/scope_duplicates_api.go b/backend/plugins/github/api/scope_duplicates_api.go new file mode 100644 index 00000000000..e3cc2312b09 --- /dev/null +++ b/backend/plugins/github/api/scope_duplicates_api.go @@ -0,0 +1,205 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "net/http" + "strconv" + "strings" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +// ScopeDuplicateConnection is a connection that shares a repository scope. +type ScopeDuplicateConnection struct { + ConnectionId uint64 `json:"connectionId"` + ConnectionName string `json:"connectionName"` +} + +// ScopeDuplicateGroup is one repository that appears under multiple connections +// (diagnostics) or already exists under another connection (pre-add check). +type ScopeDuplicateGroup struct { + GithubId int `json:"githubId"` + HTMLUrl string `json:"htmlUrl"` + FullName string `json:"fullName"` + Connections []ScopeDuplicateConnection `json:"connections"` +} + +// ScopeDuplicatesOutput is the response body for GetScopeDuplicates. +type ScopeDuplicatesOutput struct { + Duplicates []ScopeDuplicateGroup `json:"duplicates"` +} + +// scopeDuplicateRow is one joined row from the scoped SQL query. +type scopeDuplicateRow struct { + GithubId int `gorm:"column:github_id"` + HTMLUrl string `gorm:"column:html_url"` + FullName string `gorm:"column:full_name"` + ConnectionId uint64 `gorm:"column:connection_id"` + ConnectionName string `gorm:"column:connection_name"` +} + +// GetScopeDuplicates returns GitHub repositories registered under more than one +// connection, or (with connectionId + githubIds) candidates already present on +// other connections. +// @Summary Find GitHub scopes duplicated across connections +// @Description Diagnostics: groups where the same githubId appears on more than one connection. +// @Description Pre-add check: pass connectionId and githubIds to find candidates already registered elsewhere. +// @Tags plugins/github +// @Param connectionId query int false "Current connection id (pre-add check)" +// @Param githubIds query string false "Comma-separated GitHub repo ids to check (pre-add check)" +// @Success 200 {object} ScopeDuplicatesOutput +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/github/scope-duplicates [GET] +func GetScopeDuplicates(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connectionId, githubIds, err := parseScopeDuplicateQuery(input) + if err != nil { + return nil, err + } + + // Pre-add check with an empty selection: nothing to warn about. + if connectionId != nil && len(githubIds) == 0 { + return &plugin.ApiResourceOutput{ + Body: ScopeDuplicatesOutput{Duplicates: []ScopeDuplicateGroup{}}, + Status: http.StatusOK, + }, nil + } + + rows, err := queryScopeDuplicateRows(basicRes.GetDal(), connectionId, githubIds) + if err != nil { + return nil, err + } + + return &plugin.ApiResourceOutput{ + Body: ScopeDuplicatesOutput{Duplicates: groupScopeDuplicateRows(rows)}, + Status: http.StatusOK, + }, nil +} + +func parseScopeDuplicateQuery(input *plugin.ApiResourceInput) (*uint64, []int, errors.Error) { + var connectionId *uint64 + if v := input.Query.Get("connectionId"); v != "" { + id, err := strconv.ParseUint(v, 10, 64) + if err != nil { + return nil, nil, errors.BadInput.Wrap(err, "invalid connectionId") + } + connectionId = &id + } + + var githubIds []int + if v := input.Query.Get("githubIds"); v != "" { + for _, part := range strings.Split(v, ",") { + part = strings.TrimSpace(part) + if part == "" { + continue + } + id, err := strconv.Atoi(part) + if err != nil { + return nil, nil, errors.BadInput.Wrap(err, "invalid githubIds") + } + githubIds = append(githubIds, id) + } + } + + if len(githubIds) > 0 && connectionId == nil { + return nil, nil, errors.BadInput.New("connectionId is required when githubIds is provided") + } + + return connectionId, githubIds, nil +} + +// queryScopeDuplicateRows loads only the rows needed for the requested mode. +// Check mode: selected githubIds on any connection other than connectionId. +// Diagnostics: githubIds that already appear on more than one connection. +func queryScopeDuplicateRows(db dal.Dal, connectionId *uint64, githubIds []int) ([]scopeDuplicateRow, errors.Error) { + clauses := []dal.Clause{ + dal.Select("r.github_id, r.html_url, r.full_name, r.connection_id, c.name AS connection_name"), + dal.From("_tool_github_repos r"), + dal.Join("INNER JOIN _tool_github_connections c ON c.id = r.connection_id"), + dal.Orderby("r.github_id ASC, r.connection_id ASC"), + } + + if connectionId != nil { + clauses = append(clauses, dal.Where( + "r.github_id IN ? AND r.connection_id != ?", + githubIds, + *connectionId, + )) + } else { + clauses = append(clauses, dal.Where(`r.github_id IN ( + SELECT github_id FROM _tool_github_repos + GROUP BY github_id + HAVING COUNT(DISTINCT connection_id) > 1 + )`)) + } + + var rows []scopeDuplicateRow + if err := db.All(&rows, clauses...); err != nil { + return nil, err + } + return rows, nil +} + +// groupScopeDuplicateRows collapses already-filtered SQL rows into API groups. +func groupScopeDuplicateRows(rows []scopeDuplicateRow) []ScopeDuplicateGroup { + if len(rows) == 0 { + return []ScopeDuplicateGroup{} + } + + result := make([]ScopeDuplicateGroup, 0) + var current *ScopeDuplicateGroup + seenConns := make(map[uint64]struct{}) + + flush := func() { + if current != nil { + result = append(result, *current) + } + } + + for _, row := range rows { + if current == nil || current.GithubId != row.GithubId { + flush() + current = &ScopeDuplicateGroup{ + GithubId: row.GithubId, + HTMLUrl: row.HTMLUrl, + FullName: row.FullName, + Connections: make([]ScopeDuplicateConnection, 0, 2), + } + seenConns = make(map[uint64]struct{}) + } + if current.HTMLUrl == "" && row.HTMLUrl != "" { + current.HTMLUrl = row.HTMLUrl + } + if current.FullName == "" && row.FullName != "" { + current.FullName = row.FullName + } + if _, ok := seenConns[row.ConnectionId]; ok { + continue + } + seenConns[row.ConnectionId] = struct{}{} + current.Connections = append(current.Connections, ScopeDuplicateConnection{ + ConnectionId: row.ConnectionId, + ConnectionName: row.ConnectionName, + }) + } + flush() + return result +} diff --git a/backend/plugins/github/api/scope_duplicates_api_test.go b/backend/plugins/github/api/scope_duplicates_api_test.go new file mode 100644 index 00000000000..87cc33456e3 --- /dev/null +++ b/backend/plugins/github/api/scope_duplicates_api_test.go @@ -0,0 +1,114 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "net/url" + "testing" + + "github.com/apache/incubator-devlake/core/plugin" + "github.com/stretchr/testify/assert" +) + +func TestGroupScopeDuplicateRows_Empty(t *testing.T) { + assert.Empty(t, groupScopeDuplicateRows(nil)) + assert.Empty(t, groupScopeDuplicateRows([]scopeDuplicateRow{})) +} + +func TestGroupScopeDuplicateRows_GroupsConnections(t *testing.T) { + rows := []scopeDuplicateRow{ + {GithubId: 100, HTMLUrl: "https://github.com/o/a", FullName: "o/a", ConnectionId: 1, ConnectionName: "GitHub Production"}, + {GithubId: 100, HTMLUrl: "https://github.com/o/a", FullName: "o/a", ConnectionId: 2, ConnectionName: "GitHub Staging"}, + {GithubId: 200, HTMLUrl: "https://github.com/o/b", FullName: "o/b", ConnectionId: 3, ConnectionName: "Other"}, + } + + got := groupScopeDuplicateRows(rows) + assert.Equal(t, []ScopeDuplicateGroup{ + { + GithubId: 100, + HTMLUrl: "https://github.com/o/a", + FullName: "o/a", + Connections: []ScopeDuplicateConnection{ + {ConnectionId: 1, ConnectionName: "GitHub Production"}, + {ConnectionId: 2, ConnectionName: "GitHub Staging"}, + }, + }, + { + GithubId: 200, + HTMLUrl: "https://github.com/o/b", + FullName: "o/b", + Connections: []ScopeDuplicateConnection{ + {ConnectionId: 3, ConnectionName: "Other"}, + }, + }, + }, got) +} + +func TestGroupScopeDuplicateRows_DedupesSameConnection(t *testing.T) { + rows := []scopeDuplicateRow{ + {GithubId: 100, FullName: "o/a", ConnectionId: 1, ConnectionName: "Prod"}, + {GithubId: 100, FullName: "o/a", ConnectionId: 1, ConnectionName: "Prod"}, + } + + got := groupScopeDuplicateRows(rows) + assert.Len(t, got, 1) + assert.Equal(t, []ScopeDuplicateConnection{ + {ConnectionId: 1, ConnectionName: "Prod"}, + }, got[0].Connections) +} + +func TestGroupScopeDuplicateRows_FillsMissingLabels(t *testing.T) { + rows := []scopeDuplicateRow{ + {GithubId: 100, ConnectionId: 1, ConnectionName: "A"}, + {GithubId: 100, HTMLUrl: "https://github.com/o/a", FullName: "o/a", ConnectionId: 2, ConnectionName: "B"}, + } + + got := groupScopeDuplicateRows(rows) + assert.Len(t, got, 1) + assert.Equal(t, "https://github.com/o/a", got[0].HTMLUrl) + assert.Equal(t, "o/a", got[0].FullName) +} + +func TestParseScopeDuplicateQuery(t *testing.T) { + input := &plugin.ApiResourceInput{Query: url.Values{}} + connId, ids, err := parseScopeDuplicateQuery(input) + assert.Nil(t, err) + assert.Nil(t, connId) + assert.Empty(t, ids) + + input = &plugin.ApiResourceInput{Query: url.Values{ + "connectionId": []string{"3"}, + "githubIds": []string{"10, 20,30"}, + }} + connId, ids, err = parseScopeDuplicateQuery(input) + assert.Nil(t, err) + assert.Equal(t, uint64(3), *connId) + assert.Equal(t, []int{10, 20, 30}, ids) + + input = &plugin.ApiResourceInput{Query: url.Values{ + "githubIds": []string{"10"}, + }} + _, _, err = parseScopeDuplicateQuery(input) + assert.Error(t, err) + + input = &plugin.ApiResourceInput{Query: url.Values{ + "connectionId": []string{"abc"}, + }} + _, _, err = parseScopeDuplicateQuery(input) + assert.Error(t, err) +} diff --git a/backend/plugins/github/impl/impl.go b/backend/plugins/github/impl/impl.go index e09779a3dcd..ada24da0da8 100644 --- a/backend/plugins/github/impl/impl.go +++ b/backend/plugins/github/impl/impl.go @@ -225,6 +225,9 @@ func (p Github) ApiResources() map[string]map[string]plugin.ApiResourceHandler { "scope-config/:scopeConfigId/projects": { "GET": api.GetProjectsByScopeConfig, }, + "scope-duplicates": { + "GET": api.GetScopeDuplicates, + }, } } diff --git a/config-ui/src/api/scope/index.ts b/config-ui/src/api/scope/index.ts index 7007314f694..8996f466265 100644 --- a/config-ui/src/api/scope/index.ts +++ b/config-ui/src/api/scope/index.ts @@ -96,3 +96,27 @@ export const searchRemote = ( method: 'get', data, }); + +export type ScopeDuplicateConnection = { + connectionId: ID; + connectionName: string; +}; + +export type ScopeDuplicateGroup = { + githubId: number; + htmlUrl: string; + fullName: string; + connections: ScopeDuplicateConnection[]; +}; + +export const scopeDuplicates = ( + plugin: string, + data?: { + connectionId?: ID; + githubIds?: string; + }, +): Promise<{ duplicates: ScopeDuplicateGroup[] }> => + request(`/plugins/${plugin}/scope-duplicates`, { + method: 'get', + data, + }); diff --git a/config-ui/src/plugins/components/data-scope-remote/data-scope-remote.tsx b/config-ui/src/plugins/components/data-scope-remote/data-scope-remote.tsx index 6bf1509161b..7453cbedc39 100644 --- a/config-ui/src/plugins/components/data-scope-remote/data-scope-remote.tsx +++ b/config-ui/src/plugins/components/data-scope-remote/data-scope-remote.tsx @@ -17,9 +17,10 @@ */ import { useState, useEffect, useMemo } from 'react'; -import { Flex, Button } from 'antd'; +import { Flex, Button, Alert } from 'antd'; import API from '@/api'; +import type { ScopeDuplicateGroup } from '@/api/scope'; import { getPluginConfig } from '@/plugins'; import { operator } from '@/utils'; @@ -38,6 +39,39 @@ interface Props { onSubmit?: (origin: any) => void; } +const getGithubId = (scope: any): number | undefined => { + const fromData = scope?.data?.githubId; + if (typeof fromData === 'number' && fromData > 0) { + return fromData; + } + const fromId = Number(scope?.id); + if (!Number.isNaN(fromId) && fromId > 0) { + return fromId; + } + return undefined; +}; + +const buildDuplicateWarning = (duplicates: ScopeDuplicateGroup[]): string => { + const connectionNames = Array.from( + new Set( + duplicates.flatMap((d) => d.connections.map((c) => c.connectionName).filter(Boolean)), + ), + ); + const repoLabel = + duplicates.length === 1 + ? duplicates[0].fullName || duplicates[0].htmlUrl || 'This repository' + : 'One or more selected repositories'; + + const via = + connectionNames.length === 1 + ? `Connection "${connectionNames[0]}"` + : connectionNames.length > 1 + ? `Connections ${connectionNames.map((n) => `"${n}"`).join(', ')}` + : 'another connection'; + + return `${repoLabel} is already connected via ${via}. Collecting it here will create duplicate pull requests and issue records, which will inflate all metrics for this repository.`; +}; + export const DataScopeRemote = ({ mode = 'multiple', plugin, @@ -51,6 +85,8 @@ export const DataScopeRemote = ({ }: Props) => { const [selectedScope, setSelectedScope] = useState([]); const [operating, setOperating] = useState(false); + const [duplicates, setDuplicates] = useState([]); + const [warningDismissed, setWarningDismissed] = useState(false); useEffect(() => { setSelectedScope(props.selectedScope ?? []); @@ -58,6 +94,48 @@ export const DataScopeRemote = ({ const config = useMemo(() => getPluginConfig(plugin).dataScope, [plugin]); + const githubIdsKey = useMemo(() => { + if (plugin !== 'github') { + return ''; + } + return selectedScope + .map(getGithubId) + .filter((id): id is number => id !== undefined) + .sort((a, b) => a - b) + .join(','); + }, [plugin, selectedScope]); + + useEffect(() => { + if (plugin !== 'github' || !githubIdsKey) { + setDuplicates([]); + setWarningDismissed(false); + return; + } + + let cancelled = false; + setWarningDismissed(false); + + API.scope + .scopeDuplicates(plugin, { + connectionId, + githubIds: githubIdsKey, + }) + .then((res) => { + if (!cancelled) { + setDuplicates(res.duplicates ?? []); + } + }) + .catch(() => { + if (!cancelled) { + setDuplicates([]); + } + }); + + return () => { + cancelled = true; + }; + }, [plugin, connectionId, githubIdsKey]); + const handleSubmit = async () => { const [success, res] = await operator( () => API.scope.batch(plugin, connectionId, { data: selectedScope.map((it) => it.data) }), @@ -72,8 +150,20 @@ export const DataScopeRemote = ({ } }; + const showWarning = plugin === 'github' && duplicates.length > 0 && !warningDismissed; + return ( + {showWarning && ( + setWarningDismissed(true)} + style={{ marginBottom: 16 }} + message={buildDuplicateWarning(duplicates)} + /> + )} {config.render ? ( config.render({ connectionId,