feat(worker,web): Improved connection management

CHANGELOG.md

@@ -28,6 +28,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Changed navbar indexing indicator to only report progress for first time indexing jobs. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563)
 - Improved repo indexing job stability and robustness. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563)
 - Improved repositories table. [#572](https://github.com/sourcebot-dev/sourcebot/pull/572)
+- Improved connections table. [#579](https://github.com/sourcebot-dev/sourcebot/pull/579)
 
 ### Removed
 - Removed spam "login page loaded" log. [#552](https://github.com/sourcebot-dev/sourcebot/pull/552)

docs/snippets/schemas/v3/app.schema.mdx

@@ -3,11 +3,9 @@
 {
   "$schema": "http://json-schema.org/draft-07/schema#",
   "title": "AppConfig",
-  "oneOf": [
-    {
-      "$schema": "http://json-schema.org/draft-07/schema#",
+  "definitions": {
+    "GitHubAppConfig": {
       "type": "object",
-      "title": "GithubAppConfig",
       "properties": {
         "type": {
           "const": "githubApp",
@@ -61,19 +59,70 @@
       },
       "required": [
         "type",
-        "id"
+        "id",
+        "privateKey"
       ],
-      "oneOf": [
-        {
-          "required": [
-            "privateKey"
+      "additionalProperties": false
+    }
+  },
+  "oneOf": [
+    {
+      "type": "object",
+      "properties": {
+        "type": {
+          "const": "githubApp",
+          "description": "GitHub App Configuration"
+        },
+        "deploymentHostname": {
+          "type": "string",
+          "format": "hostname",
+          "default": "github.com",
+          "description": "The hostname of the GitHub App deployment.",
+          "examples": [
+            "github.com",
+            "github.example.com"
           ]
         },
-        {
-          "required": [
-            "privateKeyPath"
+        "id": {
+          "type": "string",
+          "description": "The ID of the GitHub App."
+        },
+        "privateKey": {
+          "description": "The private key of the GitHub App.",
+          "anyOf": [
+            {
+              "type": "object",
+              "properties": {
+                "secret": {
+                  "type": "string",
+                  "description": "The name of the secret that contains the token."
+                }
+              },
+              "required": [
+                "secret"
+              ],
+              "additionalProperties": false
+            },
+            {
+              "type": "object",
+              "properties": {
+                "env": {
+                  "type": "string",
+                  "description": "The name of the environment variable that contains the token. Only supported in declarative connection configs."
+                }
+              },
+              "required": [
+                "env"
+              ],
+              "additionalProperties": false
+            }
           ]
         }
+      },
+      "required": [
+        "type",
+        "id",
+        "privateKey"
       ],
       "additionalProperties": false
     }

docs/snippets/schemas/v3/index.schema.mdx

@@ -4280,11 +4280,9 @@
       "items": {
         "$schema": "http://json-schema.org/draft-07/schema#",
         "title": "AppConfig",
-        "oneOf": [
-          {
-            "$schema": "http://json-schema.org/draft-07/schema#",
+        "definitions": {
+          "GitHubAppConfig": {
             "type": "object",
-            "title": "GithubAppConfig",
             "properties": {
               "type": {
                 "const": "githubApp",
@@ -4338,19 +4336,70 @@
             },
             "required": [
               "type",
-              "id"
+              "id",
+              "privateKey"
             ],
-            "oneOf": [
-              {
-                "required": [
-                  "privateKey"
-                ]
+            "additionalProperties": false
+          }
+        },
+        "oneOf": [
+          {
+            "type": "object",
+            "properties": {
+              "type": {
+                "const": "githubApp",
+                "description": "GitHub App Configuration"
               },
-              {
-                "required": [
-                  "privateKeyPath"
+              "deploymentHostname": {
+                "type": "string",
+                "format": "hostname",
+                "default": "github.com",
+                "description": "The hostname of the GitHub App deployment.",
+                "examples": [
+                  "github.com",
+                  "github.example.com"
                 ]
+              },
+              "id": {
+                "type": "string",
+                "description": "The ID of the GitHub App."
+              },
+              "privateKey": {
+                "anyOf": [
+                  {
+                    "type": "object",
+                    "properties": {
+                      "secret": {
+                        "type": "string",
+                        "description": "The name of the secret that contains the token."
+                      }
+                    },
+                    "required": [
+                      "secret"
+                    ],
+                    "additionalProperties": false
+                  },
+                  {
+                    "type": "object",
+                    "properties": {
+                      "env": {
+                        "type": "string",
+                        "description": "The name of the environment variable that contains the token. Only supported in declarative connection configs."
+                      }
+                    },
+                    "required": [
+                      "env"
+                    ],
+                    "additionalProperties": false
+                  }
+                ],
+                "description": "The private key of the GitHub App."
               }
+            },
+            "required": [
+              "type",
+              "id",
+              "privateKey"
             ],
             "additionalProperties": false
           }

packages/backend/package.json

@@ -40,6 +40,7 @@
         "argparse": "^2.0.1",
         "azure-devops-node-api": "^15.1.1",
         "bullmq": "^5.34.10",
+        "chokidar": "^4.0.3",
         "cross-fetch": "^4.0.0",
         "dotenv": "^16.4.5",
         "express": "^4.21.2",

packages/backend/src/azuredevops.ts

@@ -1,13 +1,14 @@
 import { AzureDevOpsConnectionConfig } from "@sourcebot/schemas/v3/azuredevops.type";
 import { createLogger } from "@sourcebot/logger";
-import { getTokenFromConfig, measure, fetchWithRetry } from "./utils.js";
+import { measure, fetchWithRetry } from "./utils.js";
 import micromatch from "micromatch";
 import { PrismaClient } from "@sourcebot/db";
 import { BackendException, BackendError } from "@sourcebot/error";
 import { processPromiseResults, throwIfAnyFailed } from "./connectionUtils.js";
 import * as Sentry from "@sentry/node";
 import * as azdev from "azure-devops-node-api";
 import { GitRepository } from "azure-devops-node-api/interfaces/GitInterfaces.js";
+import { getTokenFromConfig } from "@sourcebot/crypto";
 
 const logger = createLogger('azuredevops');
 const AZUREDEVOPS_CLOUD_HOSTNAME = "dev.azure.com";
@@ -34,7 +35,7 @@ export const getAzureDevOpsReposFromConfig = async (
     const baseUrl = config.url || `https://${AZUREDEVOPS_CLOUD_HOSTNAME}`;
 
     const token = config.token ?
-        await getTokenFromConfig(config.token, orgId, db, logger) :
+        await getTokenFromConfig(config.token, orgId, db) :
         undefined;
 
     if (!token) {
@@ -47,47 +48,39 @@ export const getAzureDevOpsReposFromConfig = async (
 
     const useTfsPath = config.useTfsPath || false;
     let allRepos: GitRepository[] = [];
-    let notFound: {
-        users: string[],
-        orgs: string[],
-        repos: string[],
-    } = {
-        users: [],
-        orgs: [],
-        repos: [],
-    };
+    let allWarnings: string[] = [];
 
     if (config.orgs) {
-        const { validRepos, notFoundOrgs } = await getReposForOrganizations(
+        const { repos, warnings } = await getReposForOrganizations(
             config.orgs, 
             baseUrl,
             token,
             useTfsPath
         );
-        allRepos = allRepos.concat(validRepos);
-        notFound.orgs = notFoundOrgs;
+        allRepos = allRepos.concat(repos);
+        allWarnings = allWarnings.concat(warnings);
     }
 
     if (config.projects) {
-        const { validRepos, notFoundProjects } = await getReposForProjects(
+        const { repos, warnings } = await getReposForProjects(
             config.projects,
             baseUrl,
             token,
             useTfsPath
         );
-        allRepos = allRepos.concat(validRepos);
-        notFound.repos = notFound.repos.concat(notFoundProjects);
+        allRepos = allRepos.concat(repos);
+        allWarnings = allWarnings.concat(warnings);
     }
 
     if (config.repos) {
-        const { validRepos, notFoundRepos } = await getRepos(
+        const { repos, warnings } = await getRepos(
             config.repos,
             baseUrl,
             token,
             useTfsPath
         );
-        allRepos = allRepos.concat(validRepos);
-        notFound.repos = notFound.repos.concat(notFoundRepos);
+        allRepos = allRepos.concat(repos);
+        allWarnings = allWarnings.concat(warnings);
     }
 
     let repos = allRepos
@@ -103,8 +96,8 @@ export const getAzureDevOpsReposFromConfig = async (
     logger.debug(`Found ${repos.length} total repositories.`);
 
     return {
-        validRepos: repos,
-        notFound,
+        repos,
+        warnings: allWarnings,
     };
 };
 
@@ -221,22 +214,23 @@ async function getReposForOrganizations(
 
             // Check if it's a 404-like error (organization not found)
             if (error && typeof error === 'object' && 'statusCode' in error && error.statusCode === 404) {
-                logger.error(`Organization ${org} not found or no access`);
+                const warning = `Organization ${org} not found or no access`;
+                logger.warn(warning);
                 return {
-                    type: 'notFound' as const,
-                    value: org
+                    type: 'warning' as const,
+                    warning
                 };
             }
             throw error;
         }
     }));
 
     throwIfAnyFailed(results);
-    const { validItems: validRepos, notFoundItems: notFoundOrgs } = processPromiseResults<GitRepository>(results);
+    const { validItems: repos, warnings } = processPromiseResults<GitRepository>(results);
 
     return {
-        validRepos,
-        notFoundOrgs,
+        repos,
+        warnings,
     };
 }
 
@@ -274,22 +268,23 @@ async function getReposForProjects(
             logger.error(`Failed to fetch repositories for project ${project}.`, error);
 
             if (error && typeof error === 'object' && 'statusCode' in error && error.statusCode === 404) {
-                logger.error(`Project ${project} not found or no access`);
+                const warning = `Project ${project} not found or no access`;
+                logger.warn(warning);
                 return {
-                    type: 'notFound' as const,
-                    value: project
+                    type: 'warning' as const,
+                    warning
                 };
             }
             throw error;
         }
     }));
 
     throwIfAnyFailed(results);
-    const { validItems: validRepos, notFoundItems: notFoundProjects } = processPromiseResults<GitRepository>(results);
+    const { validItems: repos, warnings } = processPromiseResults<GitRepository>(results);
 
     return {
-        validRepos,
-        notFoundProjects,
+        repos,
+        warnings,
     };
 }
 
@@ -328,21 +323,22 @@ async function getRepos(
             logger.error(`Failed to fetch repository ${repo}.`, error);
 
             if (error && typeof error === 'object' && 'statusCode' in error && error.statusCode === 404) {
-                logger.error(`Repository ${repo} not found or no access`);
+                const warning = `Repository ${repo} not found or no access`;
+                logger.warn(warning);
                 return {
-                    type: 'notFound' as const,
-                    value: repo
+                    type: 'warning' as const,
+                    warning
                 };
             }
             throw error;
         }
     }));
 
     throwIfAnyFailed(results);
-    const { validItems: validRepos, notFoundItems: notFoundRepos } = processPromiseResults<GitRepository>(results);
+    const { validItems: repos, warnings } = processPromiseResults<GitRepository>(results);
 
     return {
-        validRepos,
-        notFoundRepos,
+        repos,
+        warnings,
     };
 }