Merge pull request #130 from fensak-io/main

Release

config/custom-environment-variables.json5

@@ -45,6 +45,13 @@
       host: "FENSAK_LOGGING_LOKI_HOST",
       basicAuth: "FENSAK_LOGGING_LOKI_BASIC_AUTH",
     },
+    sentry: {
+      enabled: {
+        __name: "FENSAK_LOGGING_SENTRY_ENABLED",
+        __format: "boolean",
+      },
+      dsn: "FENSAK_LOGGING_SENTRY_DSN",
+    },
   },
   github: {
     allowedOrgs: {

config/default.json5

@@ -60,7 +60,16 @@
 
       // Basic authentication credentials to access the Loki instance.
       basicAuth: "",
-    }
+    },
+
+    // Settings/Secrets for configuring Sentry for error reporting.
+    sentry: {
+      // Whether Sentry for error reporting should be enabled.
+      enabled: false,
+
+      // The DSN for the project to report errors to.
+      dsn: "",
+    },
   },
 
   /**

deployments/release/integration_test.ts

@@ -537,8 +537,9 @@ Deno.test("passed required rule and failed automerge requires review", async (t)
 });
 
 /**
- * Wait for the Fensak Staging check to show up on the head commit of the given branch. This will timeout after 1
- * minute.
+ * Wait for the Fensak Staging check to show up on the head commit of the given branch. This will timeout after
+ * 3 minutes. 3 minutes feels very long, but unfortunately sometimes GitHub has significant delays in the webhooks so it
+ * makes sense to wait that long.
  */
 async function waitForFensakStagingCheck(
   octokit: Octokit,
@@ -548,7 +549,7 @@ async function waitForFensakStagingCheck(
   previousCheckRuns?: number[],
 ): Promise<GitHubCheckRun> {
   const maxRetries = 60;
-  const sleepBetweenRetries = 1000;
+  const sleepBetweenRetries = 3000;
 
   const headSHA = await getHeadSHA(octokit, owner, repoName, branchName);
   for (let i = 0; i < maxRetries; i++) {

deps.ts

@@ -58,6 +58,10 @@ export {
   babelPresetTypescript,
   config,
 };
+
+import * as Sentry from "npm:@sentry/node@^7.74.1";
+export { Sentry };
+
 // Must use esm.sh version for auth-app. See https://github.com/octokit/auth-app.js/issues/465
 export { createAppAuth as octokitCreateAppAuth } from "https://esm.sh/@octokit/auth-app@6.0.1";
 // Must use esm.sh version for auth-oauth-app. See https://github.com/octokit/auth-app.js/issues/465

logging/logger.ts

@@ -5,9 +5,11 @@ import { config, winston, WinstonLoki, WinstonTransport } from "../deps.ts";
 
 const serviceEnv = config.get("env");
 const loggingLevel = config.get("logging.level");
+const sentryEnabled = config.get("logging.sentry.enabled");
 const lokiEnabled = config.get("logging.loki.enabled");
 const lokiHost = config.get("logging.loki.host");
 const lokiAuth = config.get("logging.loki.basicAuth");
+
 export let lokiTransport: WinstonLoki;
 if (lokiEnabled) {
   lokiTransport = new WinstonLoki({
@@ -31,6 +33,9 @@ export function logConfig(): void {
   if (lokiEnabled) {
     logger.info(`Shipping logs to Loki host: ${lokiHost}`);
   }
+  if (sentryEnabled) {
+    logger.info(`Reporting errors to Sentry`);
+  }
 
   const repoLimits = config.get("planRepoLimits");
   logger.info("planRepoLimits:");

middlewares/mod.ts

@@ -7,3 +7,4 @@
  */
 export * from "./ghevent.ts";
 export * from "./mgmt.ts";
+export * from "./sentry.ts";

middlewares/sentry.ts

@@ -0,0 +1,28 @@
+// Copyright (c) Fensak, LLC.
+// SPDX-License-Identifier: AGPL-3.0-or-later OR BUSL-1.1
+
+import { config, Context, Sentry } from "../deps.ts";
+import type { Middleware, Next } from "../deps.ts";
+
+const appEnv = config.get("env");
+const sentryEnabled = config.get("logging.sentry.enabled");
+const sentryDSN = config.get("logging.sentry.dsn");
+
+if (sentryEnabled) {
+  Sentry.init({
+    dsn: sentryDSN,
+    environment: appEnv,
+  });
+}
+
+export const sentryReportError: Middleware = async (
+  _ctx: Context,
+  next: Next,
+): Promise<void> => {
+  try {
+    await next();
+  } catch (e) {
+    Sentry.captureException(e);
+    throw e;
+  }
+};

web/app.ts

@@ -4,6 +4,9 @@
 import { Application, basemiddlewares, config, Router } from "../deps.ts";
 
 import { logger } from "../logging/mod.ts";
+import {
+  sentryReportError as sentryReportErrorMiddleware,
+} from "../middlewares/mod.ts";
 
 import { attachRoutes } from "./routes.ts";
 import { attachMgmtAPIRoutes } from "./mgmt_routes.ts";
@@ -15,6 +18,7 @@ export async function startWebServer(): Promise<void> {
 
   app.use(basemiddlewares.newLoggerMiddleware(logger));
   app.use(basemiddlewares.newErrorMiddleware(logger));
+  app.use(sentryReportErrorMiddleware);
   app.use(basemiddlewares.timing);
   app.use(basemiddlewares.requestId);
   app.use(basemiddlewares.unsupportedRoute);

web/mgmt_routes.ts

@@ -10,13 +10,17 @@ import {
   handleSubscriptionEvent,
 } from "../mgmt/mod.ts";
 import {
+  enqueueMsg,
   FensakConfigSource,
   getComputedFensakConfig,
   getSubscription,
+  MessageType,
   mustGetGitHubOrgWithSubscription,
+  waitForHealthCheckResult,
 } from "../svcdata/mod.ts";
 import type { GitHubOrgWithSubscription } from "../svcdata/mod.ts";
 import { isOrgManager } from "../ghstd/mod.ts";
+import { getRandomString } from "../xtd/mod.ts";
 
 interface APIOrganization {
   slug: string;
@@ -38,6 +42,8 @@ export function attachMgmtAPIRoutes(router: Router): void {
   const corsMW = oakCors({ origin: corsOrigins });
 
   router
+    .get("/healthz", healthCheck)
+    .get("/sentry-test", testSentry)
     .post(
       "/hooks/mgmt",
       middlewares.assertMgmtEvent,
@@ -59,6 +65,35 @@ export function attachMgmtAPIRoutes(router: Router): void {
     );
 }
 
+async function healthCheck(ctx: Context): Promise<void> {
+  const requestID = getRandomString(6);
+  await enqueueMsg({
+    type: MessageType.HealthCheck,
+    payload: {
+      requestID: requestID,
+    },
+  });
+  const result = await waitForHealthCheckResult(requestID);
+  if (!result) {
+    ctx.response.status = Status.InternalServerError;
+    ctx.response.body = {
+      status: Status.InternalServerError,
+      msg: "timed out waiting for worker health result",
+    };
+    return;
+  }
+
+  ctx.response.status = Status.OK;
+  ctx.response.body = {
+    status: Status.OK,
+    msg: "system ok",
+  };
+}
+
+function testSentry(_ctx: Context): void {
+  throw new Error("Test error to ensure sentry is working");
+}
+
 async function handleGetOrganizations(ctx: Context): Promise<void> {
   const token = ctx.state.apiToken;
   const octokit = new Octokit({ auth: token });

web/routes.ts

@@ -5,45 +5,14 @@ import { Context, GitHubWebhookEventName, Router, Status } from "../deps.ts";
 
 import { logger } from "../logging/mod.ts";
 import * as middlewares from "../middlewares/mod.ts";
-import {
-  enqueueMsg,
-  MessageType,
-  waitForHealthCheckResult,
-} from "../svcdata/mod.ts";
+import { enqueueMsg, MessageType } from "../svcdata/mod.ts";
 import { fastRejectEvent } from "../ghevent/mod.ts";
-import { getRandomString } from "../xtd/mod.ts";
 
 export function attachRoutes(router: Router): void {
   router
-    .get("/healthz", healthCheck)
     .post("/hooks/gh", middlewares.assertGitHubWebhook, handleGitHubWebhooks);
 }
 
-async function healthCheck(ctx: Context): Promise<void> {
-  const requestID = getRandomString(6);
-  await enqueueMsg({
-    type: MessageType.HealthCheck,
-    payload: {
-      requestID: requestID,
-    },
-  });
-  const result = await waitForHealthCheckResult(requestID);
-  if (!result) {
-    ctx.response.status = Status.InternalServerError;
-    ctx.response.body = {
-      status: Status.InternalServerError,
-      msg: "timed out waiting for worker health result",
-    };
-    return;
-  }
-
-  ctx.response.status = Status.OK;
-  ctx.response.body = {
-    status: Status.OK,
-    msg: "system ok",
-  };
-}
-
 async function handleGitHubWebhooks(ctx: Context): Promise<void> {
   const ghEventName = ctx.request.headers.get("X-GitHub-Event");
   if (ghEventName == null) {

worker/app.ts

@@ -1,7 +1,7 @@
 // Copyright (c) Fensak, LLC.
 // SPDX-License-Identifier: AGPL-3.0-or-later OR BUSL-1.1
 
-import { config } from "../deps.ts";
+import { config, Sentry } from "../deps.ts";
 
 import { logger, lokiTransport } from "../logging/mod.ts";
 import { handleGitHubEvent } from "../ghevent/mod.ts";
@@ -24,6 +24,9 @@ export function startWorker(): void {
 async function handler(msg: Message): Promise<void> {
   try {
     await runHandler(msg);
+  } catch (e) {
+    Sentry.captureException(e);
+    throw e;
   } finally {
     if (lokiEnabled) {
       await lokiTransport.flush();