Skip to content

chore: migrate-repo-reads-to-public-repositories [CM-900] #3765

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mbani01 merged 5 commits into from
Jan 21, 2026
Merged

chore: migrate-repo-reads-to-public-repositories [CM-900] #3765

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
55eb365
chore: migrate-repo-reads-to-public-repositories
mbani01 Jan 19, 2026
262d100
fix: lint
mbani01 Jan 20, 2026
5af423d
fix: formatting
mbani01 Jan 20, 2026
334b48e
feat: migrate integrationService readers to repositories table
mbani01 Jan 20, 2026
d71977e
fix: code review suggestions
mbani01 Jan 20, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 0 additions & 53 deletions backend/src/database/repositories/githubReposRepository.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,59 +105,6 @@ export default class GithubReposRepository {
await this.getCache(options).deleteAll()
}

static async getMapping(integrationId, options: IRepositoryOptions) {
const transaction = SequelizeRepository.getTransaction(options)

const results = await options.database.sequelize.query(
`
SELECT
r.url,
JSONB_BUILD_OBJECT(
'id', s.id,
'name', s.name
) as "segment"
FROM "githubRepos" r
JOIN segments s ON s.id = r."segmentId"
WHERE r."integrationId" = :integrationId
AND r."deletedAt" is null
`,
{
replacements: {
integrationId,
},
type: QueryTypes.SELECT,
transaction,
},
)

return results
}

static async hasMappedRepos(segmentId: string, options: IRepositoryOptions) {
const transaction = SequelizeRepository.getTransaction(options)

const result = await options.database.sequelize.query(
`
SELECT EXISTS (
SELECT 1
FROM "githubRepos" r
WHERE r."segmentId" = :segmentId
AND r."deletedAt" is null
LIMIT 1
) as has_repos
`,
{
replacements: {
segmentId,
},
type: QueryTypes.SELECT,
transaction,
},
)

return result[0].has_repos
}

static async delete(integrationId, options: IRepositoryOptions) {
const seq = SequelizeRepository.getSequelize(options)
const transaction = SequelizeRepository.getTransaction(options)
Expand Down
53 changes: 0 additions & 53 deletions backend/src/database/repositories/gitlabReposRepository.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,59 +96,6 @@ export default class GitlabReposRepository {
await this.getCache(options).deleteAll()
}

static async getMapping(integrationId, options: IRepositoryOptions) {
const transaction = SequelizeRepository.getTransaction(options)

const results = await options.database.sequelize.query(
`
SELECT
r.url,
JSONB_BUILD_OBJECT(
'id', s.id,
'name', s.name
) as "segment"
FROM "gitlabRepos" r
JOIN segments s ON s.id = r."segmentId"
WHERE r."integrationId" = :integrationId
AND r."deletedAt" is null
`,
{
replacements: {
integrationId,
},
type: QueryTypes.SELECT,
transaction,
},
)

return results
}

static async hasMappedRepos(segmentId: string, options: IRepositoryOptions) {
const transaction = SequelizeRepository.getTransaction(options)

const result = await options.database.sequelize.query(
`
SELECT EXISTS (
SELECT 1
FROM "gitlabRepos" r
WHERE r."segmentId" = :segmentId
AND r."deletedAt" is null
LIMIT 1
) as has_repos
`,
{
replacements: {
segmentId,
},
type: QueryTypes.SELECT,
transaction,
},
)

return result[0].has_repos
}

static async delete(integrationId, options: IRepositoryOptions) {
const seq = SequelizeRepository.getSequelize(options)
const transaction = SequelizeRepository.getTransaction(options)
Expand Down
69 changes: 7 additions & 62 deletions backend/src/database/repositories/segmentRepository.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,17 @@ import { Error404 } from '@crowd/common'
import {
buildSegmentActivityTypes,
findSegmentById,
getMappedWithSegmentName,
getSegmentActivityTypes,
hasMappedRepos,
isSegmentProject,
isSegmentProjectGroup,
populateSegmentRelations,
} from '@crowd/data-access-layer/src/segments'
import {
ActivityTypeSettings,
PageData,
PlatformType,
QueryData,
SegmentCreateData,
SegmentData,
Expand Down Expand Up @@ -633,11 +636,13 @@ class SegmentRepository extends RepositoryBase<

const subprojects = projects.map((p) => p.subprojects).flat()
const integrationsBySegments = await this.queryIntegrationsForSubprojects(subprojects)
const qx = SequelizeRepository.getQueryExecutor(this.options)
const githubPlatforms = [PlatformType.GITHUB, PlatformType.GITHUB_NANGO]
const mappedGithubReposBySegments = (
await Promise.all(
subprojects.map(async (s) => ({
segmentId: s.id,
hasMappedRepo: await this.hasMappedRepos(s.id),
hasMappedRepo: await hasMappedRepos(qx, s.id, githubPlatforms),
})),
)
).reduce((acc, { segmentId, hasMappedRepo }) => {
Expand Down Expand Up @@ -666,7 +671,7 @@ class SegmentRepository extends RepositoryBase<
platform: 'github',
segmentId: subproject.id,
type: 'mapped',
mappedWith: await this.mappedWith(subproject.id),
mappedWith: await getMappedWithSegmentName(qx, subproject.id, githubPlatforms),
})
}

Expand Down Expand Up @@ -890,66 +895,6 @@ class SegmentRepository extends RepositoryBase<

return segments.map((i: any) => i.id)
}

async hasMappedRepos(segmentId: string) {
const transaction = SequelizeRepository.getTransaction(this.options)
const tenantId = this.options.currentTenant.id

const result = await this.options.database.sequelize.query(
`
SELECT EXISTS (
SELECT 1
FROM "githubRepos" r
LEFT JOIN "integrations" i ON r."integrationId" = i.id
WHERE r."segmentId" = :segmentId
AND r."tenantId" = :tenantId
AND r."deletedAt" IS NULL
AND (i.id IS NULL OR i."segmentId" != :segmentId)
LIMIT 1
) as has_repos
`,
{
replacements: {
segmentId,
tenantId,
},
type: QueryTypes.SELECT,
transaction,
},
)

return !!result[0].has_repos
}

async mappedWith(segmentId: string) {
const transaction = SequelizeRepository.getTransaction(this.options)
const tenantId = this.options.currentTenant.id

const result = await this.options.database.sequelize.query(
`
select
s.name as segment_name
from
"githubRepos" r
left join "integrations" i on r."integrationId" = i.id
left join "segments" s on i."segmentId" = s.id
where r."segmentId" = :segmentId
and r."tenantId" = :tenantId
and (i.id is null or i."segmentId" != :segmentId)
limit 1
`,
{
replacements: {
segmentId,
tenantId,
},
type: QueryTypes.SELECT,
transaction,
},
)

return result[0].segment_name as string
}
}

export default SegmentRepository
11 changes: 6 additions & 5 deletions backend/src/services/collectionService.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,13 +35,12 @@ import {
listRepositoryGroups,
updateRepositoryGroup,
} from '@crowd/data-access-layer/src/repositoryGroups'
import { findSegmentById } from '@crowd/data-access-layer/src/segments'
import { findSegmentById, hasMappedRepos } from '@crowd/data-access-layer/src/segments'
import { QueryResult } from '@crowd/data-access-layer/src/utils'
import { GithubIntegrationSettings } from '@crowd/integrations'
import { LoggerBase } from '@crowd/logging'
import { DEFAULT_WIDGET_VALUES, PlatformType, Widgets } from '@crowd/types'

import SegmentRepository from '@/database/repositories/segmentRepository'
import SequelizeRepository from '@/database/repositories/sequelizeRepository'
import { IGithubInsights } from '@/types/githubTypes'

Expand Down Expand Up @@ -582,9 +581,11 @@ export class CollectionService extends LoggerBase {
]

// Check for mapped repositories and add GitHub if there are any
const segmentRepository = new SegmentRepository(this.options)
const hasMappedRepos = await segmentRepository.hasMappedRepos(segmentId)
if (hasMappedRepos && !platforms.includes(PlatformType.GITHUB)) {
const hasGithubMappedRepos = await hasMappedRepos(qx, segmentId, [
PlatformType.GITHUB,
PlatformType.GITHUB_NANGO,
])
if (hasGithubMappedRepos && !platforms.includes(PlatformType.GITHUB)) {
platforms.push(PlatformType.GITHUB)
}

Expand Down
70 changes: 20 additions & 50 deletions backend/src/services/integrationService.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,11 @@ import {
restoreRepositories,
softDeleteRepositories,
} from '@crowd/data-access-layer/src/repositories'
import { getGithubMappedRepos, getGitlabMappedRepos } from '@crowd/data-access-layer/src/segments'
import {
getMappedRepos,
getMappedWithSegmentName,
hasMappedRepos,
} from '@crowd/data-access-layer/src/segments'
import {
NangoIntegration,
connectNangoIntegration,
Expand Down Expand Up @@ -1146,25 +1150,6 @@ export default class IntegrationService {
}
}

async getGithubRepos(integrationId): Promise<any[]> {
const transaction = await SequelizeRepository.createTransaction(this.options)

const txOptions = {
...this.options,
transaction,
}

try {
const mapping = await GithubReposRepository.getMapping(integrationId, txOptions)

await SequelizeRepository.commitTransaction(transaction)
return mapping
} catch (err) {
await SequelizeRepository.rollbackTransaction(transaction)
throw err
}
}

/**
* Get repository mappings for an integration
* Uses the unified public.repositories table instead of legacy githubRepos table
Expand Down Expand Up @@ -2638,8 +2623,9 @@ export default class IntegrationService {
updatedAt: string
}[]

const githubRepos = await this.getGithubRepos(integrationId)
const mappedSegments = githubRepos.map((repo) => repo.segment.id)
const qx = SequelizeRepository.getQueryExecutor(this.options)
const githubRepos = await getRepositoriesBySourceIntegrationId(qx, integrationId)
const mappedSegments = githubRepos.map((repo) => repo.segmentId)

const cacheRemote = new RedisCache(
'github-progress-remote',
Expand Down Expand Up @@ -2858,22 +2844,25 @@ export default class IntegrationService {
}

async getIntegrationMappedRepos(segmentId: string) {
const segmentRepository = new SegmentRepository(this.options)
const hasMappedRepos = await segmentRepository.hasMappedRepos(segmentId)
const qx = SequelizeRepository.getQueryExecutor(this.options)
const githubPlatforms = [PlatformType.GITHUB, PlatformType.GITHUB_NANGO]

const hasRepos = await hasMappedRepos(qx, segmentId, githubPlatforms)

if (!hasMappedRepos) {
if (!hasRepos) {
return null
}

const qx = SequelizeRepository.getQueryExecutor(this.options)

const gitlabMappedRepos = await getGitlabMappedRepos(qx, segmentId)
const githubMappedRepos = await getGithubMappedRepos(qx, segmentId)
const project = await segmentRepository.mappedWith(segmentId)
const [githubMappedRepos, githubNangoMappedRepos, gitlabMappedRepos] = await Promise.all([
getMappedRepos(qx, segmentId, PlatformType.GITHUB),
getMappedRepos(qx, segmentId, PlatformType.GITHUB_NANGO),
getMappedRepos(qx, segmentId, PlatformType.GITLAB),
])
const project = await getMappedWithSegmentName(qx, segmentId, githubPlatforms)

return {
project,
repositories: [...githubMappedRepos, ...gitlabMappedRepos],
repositories: [...githubMappedRepos, ...githubNangoMappedRepos, ...gitlabMappedRepos],
}
}

Expand Down Expand Up @@ -3077,25 +3066,6 @@ export default class IntegrationService {
}
}

async getGitlabRepos(integrationId): Promise<any[]> {
const transaction = await SequelizeRepository.createTransaction(this.options)

const txOptions = {
...this.options,
transaction,
}

try {
const mapping = await GitlabReposRepository.getMapping(integrationId, txOptions)

await SequelizeRepository.commitTransaction(transaction)
return mapping
} catch (err) {
await SequelizeRepository.rollbackTransaction(transaction)
throw err
}
}

async updateGithubIntegrationSettings(installId: string) {
this.options.log.info(`Updating GitHub integration settings for installId: ${installId}`)

Expand Down
Loading