diff --git a/CHANGELOG.md b/CHANGELOG.md index 52207d31..e4c1f9f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fixed incorrect shutdown of PostHog SDK in the worker. [#609](https://github.com/sourcebot-dev/sourcebot/pull/609) - Fixed race condition in job schedulers. [#607](https://github.com/sourcebot-dev/sourcebot/pull/607) +### Added +- Added force resync buttons for connections and repositories. [#610](https://github.com/sourcebot-dev/sourcebot/pull/610) + ## [4.9.1] - 2025-11-07 ### Added diff --git a/packages/backend/package.json b/packages/backend/package.json index 5369bde1..201bd886 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -40,6 +40,7 @@ "cross-fetch": "^4.0.0", "dotenv": "^16.4.5", "express": "^4.21.2", + "express-async-errors": "^3.1.1", "git-url-parse": "^16.1.0", "gitea-js": "^1.22.0", "glob": "^11.0.0", diff --git a/packages/backend/src/api.ts b/packages/backend/src/api.ts new file mode 100644 index 00000000..5c7e2547 --- /dev/null +++ b/packages/backend/src/api.ts @@ -0,0 +1,103 @@ +import { PrismaClient, RepoIndexingJobType } from '@sourcebot/db'; +import { createLogger } from '@sourcebot/shared'; +import express, { Request, Response } from 'express'; +import 'express-async-errors'; +import * as http from "http"; +import z from 'zod'; +import { ConnectionManager } from './connectionManager.js'; +import { PromClient } from './promClient.js'; +import { RepoIndexManager } from './repoIndexManager.js'; + +const logger = createLogger('api'); +const PORT = 3060; + +export class Api { + private server: http.Server; + + constructor( + promClient: PromClient, + private prisma: PrismaClient, + private connectionManager: ConnectionManager, + private repoIndexManager: RepoIndexManager, + ) { + const app = express(); + app.use(express.json()); + app.use(express.urlencoded({ extended: true })); + + // Prometheus metrics endpoint + app.use('/metrics', async (_req: Request, res: Response) => { + res.set('Content-Type', promClient.registry.contentType); + const metrics = await promClient.registry.metrics(); + res.end(metrics); + }); + + app.post('/api/sync-connection', this.syncConnection.bind(this)); + app.post('/api/index-repo', this.indexRepo.bind(this)); + + this.server = app.listen(PORT, () => { + logger.info(`API server is running on port ${PORT}`); + }); + } + + private async syncConnection(req: Request, res: Response) { + const schema = z.object({ + connectionId: z.number(), + }).strict(); + + const parsed = schema.safeParse(req.body); + if (!parsed.success) { + res.status(400).json({ error: parsed.error.message }); + return; + } + + const { connectionId } = parsed.data; + const connection = await this.prisma.connection.findUnique({ + where: { + id: connectionId, + } + }); + + if (!connection) { + res.status(404).json({ error: 'Connection not found' }); + return; + } + + const [jobId] = await this.connectionManager.createJobs([connection]); + + res.status(200).json({ jobId }); + } + + private async indexRepo(req: Request, res: Response) { + const schema = z.object({ + repoId: z.number(), + }).strict(); + + const parsed = schema.safeParse(req.body); + if (!parsed.success) { + res.status(400).json({ error: parsed.error.message }); + return; + } + + const { repoId } = parsed.data; + const repo = await this.prisma.repo.findUnique({ + where: { id: repoId }, + }); + + if (!repo) { + res.status(404).json({ error: 'Repo not found' }); + return; + } + + const [jobId] = await this.repoIndexManager.createJobs([repo], RepoIndexingJobType.INDEX); + res.status(200).json({ jobId }); + } + + public async dispose() { + return new Promise((resolve, reject) => { + this.server.close((err) => { + if (err) reject(err); + else resolve(undefined); + }); + }); + } +} \ No newline at end of file diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index be0ddb01..5e6d6ba0 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -1,21 +1,21 @@ import "./instrument.js"; import { PrismaClient } from "@sourcebot/db"; -import { createLogger } from "@sourcebot/shared"; -import { env, getConfigSettings, hasEntitlement, getDBConnectionString } from '@sourcebot/shared'; +import { createLogger, env, getConfigSettings, getDBConnectionString, hasEntitlement } from "@sourcebot/shared"; +import 'express-async-errors'; import { existsSync } from 'fs'; import { mkdir } from 'fs/promises'; import { Redis } from 'ioredis'; +import { Api } from "./api.js"; import { ConfigManager } from "./configManager.js"; import { ConnectionManager } from './connectionManager.js'; import { INDEX_CACHE_DIR, REPOS_CACHE_DIR } from './constants.js'; +import { AccountPermissionSyncer } from "./ee/accountPermissionSyncer.js"; import { GithubAppManager } from "./ee/githubAppManager.js"; import { RepoPermissionSyncer } from './ee/repoPermissionSyncer.js'; -import { AccountPermissionSyncer } from "./ee/accountPermissionSyncer.js"; +import { shutdownPosthog } from "./posthog.js"; import { PromClient } from './promClient.js'; import { RepoIndexManager } from "./repoIndexManager.js"; -import { shutdownPosthog } from "./posthog.js"; - const logger = createLogger('backend-entrypoint'); @@ -74,6 +74,13 @@ else if (env.EXPERIMENT_EE_PERMISSION_SYNC_ENABLED === 'true' && hasEntitlement( accountPermissionSyncer.startScheduler(); } +const api = new Api( + promClient, + prisma, + connectionManager, + repoIndexManager, +); + logger.info('Worker started.'); const cleanup = async (signal: string) => { @@ -88,7 +95,6 @@ const cleanup = async (signal: string) => { connectionManager.dispose(), repoPermissionSyncer.dispose(), accountPermissionSyncer.dispose(), - promClient.dispose(), configManager.dispose(), ]), new Promise((_, reject) => @@ -102,6 +108,7 @@ const cleanup = async (signal: string) => { await prisma.$disconnect(); await redis.quit(); + await api.dispose(); await shutdownPosthog(); } diff --git a/packages/backend/src/promClient.ts b/packages/backend/src/promClient.ts index 2fa7718f..7beaac84 100644 --- a/packages/backend/src/promClient.ts +++ b/packages/backend/src/promClient.ts @@ -1,14 +1,6 @@ -import express, { Request, Response } from 'express'; -import { Server } from 'http'; import client, { Registry, Counter, Gauge } from 'prom-client'; -import { createLogger } from "@sourcebot/shared"; - -const logger = createLogger('prometheus-client'); - export class PromClient { - private registry: Registry; - private app: express.Application; - private server: Server; + public registry: Registry; public activeRepoIndexJobs: Gauge; public pendingRepoIndexJobs: Gauge; @@ -22,8 +14,6 @@ export class PromClient { public connectionSyncJobFailTotal: Counter; public connectionSyncJobSuccessTotal: Counter; - public readonly PORT = 3060; - constructor() { this.registry = new Registry(); @@ -100,26 +90,5 @@ export class PromClient { client.collectDefaultMetrics({ register: this.registry, }); - - this.app = express(); - this.app.get('/metrics', async (req: Request, res: Response) => { - res.set('Content-Type', this.registry.contentType); - - const metrics = await this.registry.metrics(); - res.end(metrics); - }); - - this.server = this.app.listen(this.PORT, () => { - logger.info(`Prometheus metrics server is running on port ${this.PORT}`); - }); - } - - async dispose() { - return new Promise((resolve, reject) => { - this.server.close((err) => { - if (err) reject(err); - else resolve(); - }); - }); } } \ No newline at end of file diff --git a/packages/backend/src/repoIndexManager.ts b/packages/backend/src/repoIndexManager.ts index 6be40d70..17ed2d8a 100644 --- a/packages/backend/src/repoIndexManager.ts +++ b/packages/backend/src/repoIndexManager.ts @@ -192,7 +192,7 @@ export class RepoIndexManager { } } - private async createJobs(repos: Repo[], type: RepoIndexingJobType) { + public async createJobs(repos: Repo[], type: RepoIndexingJobType) { // @note: we don't perform this in a transaction because // we want to avoid the situation where a job is created and run // prior to the transaction being committed. @@ -221,6 +221,8 @@ export class RepoIndexManager { const jobTypeLabel = getJobTypePrometheusLabel(type); this.promClient.pendingRepoIndexJobs.inc({ repo: job.repo.name, type: jobTypeLabel }); } + + return jobs.map(job => job.id); } private async runJob(job: ReservedJob) { diff --git a/packages/web/src/app/[domain]/repos/[id]/page.tsx b/packages/web/src/app/[domain]/repos/[id]/page.tsx index 8986f7f6..0c2ddfa1 100644 --- a/packages/web/src/app/[domain]/repos/[id]/page.tsx +++ b/packages/web/src/app/[domain]/repos/[id]/page.tsx @@ -1,4 +1,4 @@ -import { sew } from "@/actions" +import { getCurrentUserRole, sew } from "@/actions" import { Badge } from "@/components/ui/badge" import { Button } from "@/components/ui/button" import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card" @@ -19,6 +19,7 @@ import { BackButton } from "../../components/backButton" import { DisplayDate } from "../../components/DisplayDate" import { RepoBranchesTable } from "../components/repoBranchesTable" import { RepoJobsTable } from "../components/repoJobsTable" +import { OrgRole } from "@sourcebot/db" export default async function RepoDetailPage({ params }: { params: Promise<{ id: string }> }) { const { id } = await params @@ -51,6 +52,11 @@ export default async function RepoDetailPage({ params }: { params: Promise<{ id: const repoMetadata = repoMetadataSchema.parse(repo.metadata); + const userRole = await getCurrentUserRole(SINGLE_TENANT_ORG_DOMAIN); + if (isServiceError(userRole)) { + throw new ServiceErrorException(userRole); + } + return ( <>
@@ -172,7 +178,11 @@ export default async function RepoDetailPage({ params }: { params: Promise<{ id: }> - + diff --git a/packages/web/src/app/[domain]/repos/components/repoJobsTable.tsx b/packages/web/src/app/[domain]/repos/components/repoJobsTable.tsx index 1f8e290e..a108d1b1 100644 --- a/packages/web/src/app/[domain]/repos/components/repoJobsTable.tsx +++ b/packages/web/src/app/[domain]/repos/components/repoJobsTable.tsx @@ -18,7 +18,7 @@ import { useReactTable, } from "@tanstack/react-table" import { cva } from "class-variance-authority" -import { AlertCircle, ArrowUpDown, RefreshCwIcon } from "lucide-react" +import { AlertCircle, ArrowUpDown, PlusCircleIcon, RefreshCwIcon } from "lucide-react" import * as React from "react" import { CopyIconButton } from "../../components/copyIconButton" import { useMemo } from "react" @@ -26,6 +26,9 @@ import { LightweightCodeHighlighter } from "../../components/lightweightCodeHigh import { useRouter } from "next/navigation" import { useToast } from "@/components/hooks/use-toast" import { DisplayDate } from "../../components/DisplayDate" +import { LoadingButton } from "@/components/ui/loading-button" +import { indexRepo } from "@/features/workerApi/actions" +import { isServiceError } from "@/lib/utils" // @see: https://v0.app/chat/repo-indexing-status-uhjdDim8OUS @@ -129,7 +132,7 @@ export const columns: ColumnDef[] = [ ) }, - cell: ({ row }) => , + cell: ({ row }) => , }, { accessorKey: "completedAt", @@ -147,7 +150,7 @@ export const columns: ColumnDef[] = [ return "-"; } - return + return }, }, { @@ -176,13 +179,41 @@ export const columns: ColumnDef[] = [ }, ] -export const RepoJobsTable = ({ data }: { data: RepoIndexingJob[] }) => { +export const RepoJobsTable = ({ + data, + repoId, + isIndexButtonVisible, +}: { + data: RepoIndexingJob[], + repoId: number, + isIndexButtonVisible: boolean, +}) => { const [sorting, setSorting] = React.useState([{ id: "createdAt", desc: true }]) const [columnFilters, setColumnFilters] = React.useState([]) const [columnVisibility, setColumnVisibility] = React.useState({}) const router = useRouter(); const { toast } = useToast(); + const [isIndexSubmitting, setIsIndexSubmitting] = React.useState(false); + const onIndexButtonClick = React.useCallback(async () => { + setIsIndexSubmitting(true); + const response = await indexRepo(repoId); + + if (!isServiceError(response)) { + const { jobId } = response; + toast({ + description: `✅ Repository sync triggered successfully. Job ID: ${jobId}`, + }) + router.refresh(); + } else { + toast({ + description: `❌ Failed to index repository. ${response.message}`, + }); + } + + setIsIndexSubmitting(false); + }, [repoId, router, toast]); + const table = useReactTable({ data, columns, @@ -247,19 +278,31 @@ export const RepoJobsTable = ({ data }: { data: RepoIndexingJob[] }) => { - +
+ + + {isIndexButtonVisible && ( + + + Trigger sync + + )} +
diff --git a/packages/web/src/app/[domain]/settings/connections/[id]/page.tsx b/packages/web/src/app/[domain]/settings/connections/[id]/page.tsx index 4cebd7db..b3c7fe79 100644 --- a/packages/web/src/app/[domain]/settings/connections/[id]/page.tsx +++ b/packages/web/src/app/[domain]/settings/connections/[id]/page.tsx @@ -4,13 +4,13 @@ import { DisplayDate } from "@/app/[domain]/components/DisplayDate"; import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; import { Skeleton } from "@/components/ui/skeleton"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; -import { env } from "@sourcebot/shared"; import { SINGLE_TENANT_ORG_DOMAIN } from "@/lib/constants"; -import { notFound, ServiceErrorException } from "@/lib/serviceError"; +import { notFound as notFoundServiceError, ServiceErrorException } from "@/lib/serviceError"; +import { notFound } from "next/navigation"; import { isServiceError } from "@/lib/utils"; import { withAuthV2 } from "@/withAuthV2"; import { AzureDevOpsConnectionConfig, BitbucketConnectionConfig, GenericGitHostConnectionConfig, GerritConnectionConfig, GiteaConnectionConfig, GithubConnectionConfig, GitlabConnectionConfig } from "@sourcebot/schemas/v3/index.type"; -import { getConfigSettings } from "@sourcebot/shared"; +import { env, getConfigSettings } from "@sourcebot/shared"; import { Info } from "lucide-react"; import Link from "next/link"; import { Suspense } from "react"; @@ -22,12 +22,16 @@ interface ConnectionDetailPageProps { }> } - export default async function ConnectionDetailPage(props: ConnectionDetailPageProps) { const params = await props.params; const { id } = params; - const connection = await getConnectionWithJobs(Number.parseInt(id)); + const connectionId = Number.parseInt(id); + if (isNaN(connectionId)) { + return notFound(); + } + + const connection = await getConnectionWithJobs(connectionId); if (isServiceError(connection)) { throw new ServiceErrorException(connection); } @@ -172,7 +176,10 @@ export default async function ConnectionDetailPage(props: ConnectionDetailPagePr }> - + @@ -197,7 +204,7 @@ const getConnectionWithJobs = async (id: number) => sew(() => }); if (!connection) { - return notFound(); + return notFoundServiceError(); } return connection; diff --git a/packages/web/src/app/[domain]/settings/connections/components/connectionJobsTable.tsx b/packages/web/src/app/[domain]/settings/connections/components/connectionJobsTable.tsx index fec991cb..fd5df81e 100644 --- a/packages/web/src/app/[domain]/settings/connections/components/connectionJobsTable.tsx +++ b/packages/web/src/app/[domain]/settings/connections/components/connectionJobsTable.tsx @@ -18,7 +18,7 @@ import { useReactTable, } from "@tanstack/react-table" import { cva } from "class-variance-authority" -import { AlertCircle, AlertTriangle, ArrowUpDown, RefreshCwIcon } from "lucide-react" +import { AlertCircle, AlertTriangle, ArrowUpDown, PlusCircleIcon, RefreshCwIcon } from "lucide-react" import * as React from "react" import { CopyIconButton } from "@/app/[domain]/components/copyIconButton" import { useMemo } from "react" @@ -26,6 +26,9 @@ import { LightweightCodeHighlighter } from "@/app/[domain]/components/lightweigh import { useRouter } from "next/navigation" import { useToast } from "@/components/hooks/use-toast" import { DisplayDate } from "@/app/[domain]/components/DisplayDate" +import { LoadingButton } from "@/components/ui/loading-button" +import { syncConnection } from "@/features/workerApi/actions" +import { isServiceError } from "@/lib/utils" export type ConnectionSyncJob = { @@ -181,13 +184,33 @@ export const columns: ColumnDef[] = [ }, ] -export const ConnectionJobsTable = ({ data }: { data: ConnectionSyncJob[] }) => { +export const ConnectionJobsTable = ({ data, connectionId }: { data: ConnectionSyncJob[], connectionId: number }) => { const [sorting, setSorting] = React.useState([{ id: "createdAt", desc: true }]) const [columnFilters, setColumnFilters] = React.useState([]) const [columnVisibility, setColumnVisibility] = React.useState({}) const router = useRouter(); const { toast } = useToast(); + const [isSyncSubmitting, setIsSyncSubmitting] = React.useState(false); + const onSyncButtonClick = React.useCallback(async () => { + setIsSyncSubmitting(true); + const response = await syncConnection(connectionId); + + if (!isServiceError(response)) { + const { jobId } = response; + toast({ + description: `✅ Connection synced successfully. Job ID: ${jobId}`, + }) + router.refresh(); + } else { + toast({ + description: `❌ Failed to sync connection. ${response.message}`, + }); + } + + setIsSyncSubmitting(false); + }, [connectionId, router, toast]); + const table = useReactTable({ data, columns, @@ -238,19 +261,29 @@ export const ConnectionJobsTable = ({ data }: { data: ConnectionSyncJob[] }) => - +
+ + + + + Trigger sync + +
diff --git a/packages/web/src/features/workerApi/README.md b/packages/web/src/features/workerApi/README.md new file mode 100644 index 00000000..3134c95c --- /dev/null +++ b/packages/web/src/features/workerApi/README.md @@ -0,0 +1 @@ +This folder contains utilities to interact with the internal worker REST api. See packages/backend/api.ts \ No newline at end of file diff --git a/packages/web/src/features/workerApi/actions.ts b/packages/web/src/features/workerApi/actions.ts new file mode 100644 index 00000000..a9f1fc46 --- /dev/null +++ b/packages/web/src/features/workerApi/actions.ts @@ -0,0 +1,59 @@ +'use server'; + +import { sew } from "@/actions"; +import { unexpectedError } from "@/lib/serviceError"; +import { withAuthV2, withMinimumOrgRole } from "@/withAuthV2"; +import { OrgRole } from "@sourcebot/db"; +import z from "zod"; + +const WORKER_API_URL = 'http://localhost:3060'; + +export const syncConnection = async (connectionId: number) => sew(() => + withAuthV2(({ role }) => + withMinimumOrgRole(role, OrgRole.OWNER, async () => { + const response = await fetch(`${WORKER_API_URL}/api/sync-connection`, { + method: 'POST', + body: JSON.stringify({ + connectionId + }), + headers: { + 'Content-Type': 'application/json', + }, + }); + + if (!response.ok) { + return unexpectedError('Failed to sync connection'); + } + + const data = await response.json(); + const schema = z.object({ + jobId: z.string(), + }); + return schema.parse(data); + }) + ) +); + +export const indexRepo = async (repoId: number) => sew(() => + withAuthV2(({ role }) => + withMinimumOrgRole(role, OrgRole.OWNER, async () => { + const response = await fetch(`${WORKER_API_URL}/api/index-repo`, { + method: 'POST', + body: JSON.stringify({ repoId }), + headers: { + 'Content-Type': 'application/json', + }, + }); + + if (!response.ok) { + return unexpectedError('Failed to index repo'); + } + + const data = await response.json(); + const schema = z.object({ + jobId: z.string(), + }); + return schema.parse(data); + }) + ) +); diff --git a/packages/web/src/withAuthV2.ts b/packages/web/src/withAuthV2.ts index 65ebb054..1b055533 100644 --- a/packages/web/src/withAuthV2.ts +++ b/packages/web/src/withAuthV2.ts @@ -181,7 +181,7 @@ export const withMinimumOrgRole = async ( userRole: OrgRole, minRequiredRole: OrgRole = OrgRole.MEMBER, fn: () => Promise, -) => { +): Promise => { const getAuthorizationPrecedence = (role: OrgRole): number => { switch (role) { diff --git a/yarn.lock b/yarn.lock index 2f19bfa9..e47ad883 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7908,6 +7908,7 @@ __metadata: cross-fetch: "npm:^4.0.0" dotenv: "npm:^16.4.5" express: "npm:^4.21.2" + express-async-errors: "npm:^3.1.1" git-url-parse: "npm:^16.1.0" gitea-js: "npm:^1.22.0" glob: "npm:^11.0.0" @@ -12538,6 +12539,15 @@ __metadata: languageName: node linkType: hard +"express-async-errors@npm:^3.1.1": + version: 3.1.1 + resolution: "express-async-errors@npm:3.1.1" + peerDependencies: + express: ^4.16.2 + checksum: 10c0/56c4e90c44e98c7edc5bd38e18dd23b0d9a7139cb94ff3e25943ba257415b433e0e52ea8f9bc1fb5b70a5e6c5246eaace4fb69ab171edfb8896580928bb97ec6 + languageName: node + linkType: hard + "express-rate-limit@npm:^7.5.0": version: 7.5.0 resolution: "express-rate-limit@npm:7.5.0"