[server] Delete prebuild records when purging workspaces

components/gitpod-db/src/index.ts

@@ -39,3 +39,4 @@ export * from "./project-db";
 export * from "./team-db";
 export * from "./installation-admin-db";
 export * from "./webhook-event-db";
+export * from "./typeorm/metrics";

components/gitpod-db/src/typeorm/entity/db-prebuild-info-entry.ts

@@ -31,4 +31,8 @@ export class DBPrebuildInfo {
         })(),
     })
     info: PrebuildInfo;
+
+    // This column triggers the db-sync deletion mechanism. It's not intended for public consumption.
+    @Column()
+    deleted?: boolean;
 }

components/gitpod-db/src/typeorm/entity/db-prebuilt-workspace-updatable.ts

@@ -59,4 +59,8 @@ export class DBPrebuiltWorkspaceUpdatable implements PrebuiltWorkspaceUpdatable
         transformer: Transformer.MAP_EMPTY_STR_TO_UNDEFINED,
     })
     label?: string;
+
+    // This column triggers the db-sync deletion mechanism. It's not intended for public consumption.
+    @Column()
+    deleted?: boolean;
 }

components/gitpod-db/src/typeorm/entity/db-prebuilt-workspace.ts

@@ -77,4 +77,8 @@ export class DBPrebuiltWorkspace implements PrebuiltWorkspace {
         transformer: Transformer.MAP_BIGINT_TO_NUMBER,
     })
     statusVersion: number;
+
+    // This column triggers the db-sync deletion mechanism. It's not intended for public consumption.
+    @Column()
+    deleted?: boolean;
 }

components/gitpod-db/src/typeorm/metrics.ts

@@ -0,0 +1,59 @@
+/**
+ * Copyright (c) 2021 Gitpod GmbH. All rights reserved.
+ * Licensed under the GNU Affero General Public License (AGPL).
+ * See License-AGPL.txt in the project root for license information.
+ */
+
+import * as prometheusClient from "prom-client";
+
+export function registerDBMetrics(registry: prometheusClient.Registry) {
+    registry.registerMetric(workspacesPurgedTotal);
+    registry.registerMetric(prebuildWorkspacesPurgedTotal);
+    registry.registerMetric(prebuildInfoPurgedTotal);
+    registry.registerMetric(workspaceInstancePurgedTotal);
+}
+
+const workspacesPurgedTotal = new prometheusClient.Counter({
+    name: "gitpod_server_workspaces_purged_total",
+    help: "Counter of workspaces hard deleted by periodic gc.",
+});
+
+export function reportWorkspacePurged(count: number) {
+    workspacesPurgedTotal.inc(count);
+}
+
+const prebuildWorkspacesPurgedTotal = new prometheusClient.Counter({
+    name: "gitpod_server_prebuild_workspaces_purged_total",
+    help: "Counter of prebuild workspaces hard deleted by periodic gc.",
+});
+
+export function reportPrebuiltWorkspacePurged(count: number) {
+    prebuildWorkspacesPurgedTotal.inc(count);
+}
+
+const prebuildInfoPurgedTotal = new prometheusClient.Counter({
+    name: "gitpod_server_prebuild_info_purged_total",
+    help: "Counter of prebuild info records hard deleted by periodic gc.",
+});
+
+export function reportPrebuildInfoPurged(count: number) {
+    prebuildInfoPurgedTotal.inc(count);
+}
+
+const workspaceInstancePurgedTotal = new prometheusClient.Counter({
+    name: "gitpod_server_workspace_instances_purged_total",
+    help: "Counter of workspace instances records hard deleted by periodic gc.",
+});
+
+export function reportWorkspaceInstancePurged(count: number) {
+    workspaceInstancePurgedTotal.inc(count);
+}
+
+const prebuiltWorkspaceUpdatablePurgedTotal = new prometheusClient.Counter({
+    name: "gitpod_server_prebuilt_workspace_updatable_purged_total",
+    help: "Counter of prebuilt workspace updatable records hard deleted by periodic gc.",
+});
+
+export function reportPrebuiltWorkspaceUpdatablePurged(count: number) {
+    prebuiltWorkspaceUpdatablePurgedTotal.inc(count);
+}

components/gitpod-db/src/typeorm/workspace-db-impl.ts

@@ -53,6 +53,13 @@ import { DBPrebuiltWorkspaceUpdatable } from "./entity/db-prebuilt-workspace-upd
 import { BUILTIN_WORKSPACE_PROBE_USER_ID } from "../user-db";
 import { DBPrebuildInfo } from "./entity/db-prebuild-info-entry";
 import { daysBefore } from "@gitpod/gitpod-protocol/lib/util/timeutil";
+import {
+    reportPrebuildInfoPurged,
+    reportPrebuiltWorkspacePurged,
+    reportPrebuiltWorkspaceUpdatablePurged,
+    reportWorkspaceInstancePurged,
+    reportWorkspacePurged,
+} from "./metrics";
 
 type RawTo<T> = (instance: WorkspaceInstance, ws: Workspace) => T;
 interface OrderBy {
@@ -981,8 +988,35 @@ export abstract class AbstractTypeORMWorkspaceDBImpl implements WorkspaceDB {
      *       around to deleting them.
      */
     public async hardDeleteWorkspace(workspaceId: string): Promise<void> {
-        await (await this.getWorkspaceInstanceRepo()).update({ workspaceId }, { deleted: true });
-        await (await this.getWorkspaceRepo()).update(workspaceId, { deleted: true });
+        const logCtx = { workspaceId };
+        const prebuild = await this.findPrebuildByWorkspaceID(workspaceId);
+        if (prebuild !== undefined) {
+            // There are prebuilds linked to this workspace. We need to delete these first.
+            const prebuildsDeleted = await (
+                await this.getPrebuiltWorkspaceRepo()
+            ).update({ id: prebuild.id }, { deleted: true });
+            log.info(logCtx, `Hard deleted ${prebuildsDeleted.affected} prebuilds.`);
+            reportPrebuiltWorkspacePurged(prebuildsDeleted.affected || 0);
+
+            const updatableDeletes = await (
+                await this.getPrebuiltWorkspaceUpdatableRepo()
+            ).update({ id: prebuild.id }, { deleted: true });
+            log.info(logCtx, `Hard deleted ${updatableDeletes.affected} prebuild updatables.`);
+            reportPrebuiltWorkspaceUpdatablePurged(updatableDeletes.affected || 0);
+
+            const prebuildInfos = await (
+                await this.getPrebuildInfoRepo()
+            ).update({ prebuildId: prebuild.id }, { deleted: true });
+            log.info(logCtx, `Hard deleted ${prebuildInfos.affected} prebuild infos.`);
+            reportPrebuildInfoPurged(prebuildInfos.affected || 0);
+        }
+        const instances = await (await this.getWorkspaceInstanceRepo()).update({ workspaceId }, { deleted: true });
+        log.info(logCtx, `Hard deleted ${instances.affected} workspace instances.`);
+        reportWorkspaceInstancePurged(instances.affected || 0);
+
+        const workspaces = await (await this.getWorkspaceRepo()).update(workspaceId, { deleted: true });
+        log.info(logCtx, `Hard deleted ${workspaces.affected} workspaces.`);
+        reportWorkspacePurged(workspaces.affected || 0);
     }
 
     public async findAllWorkspaces(

components/server/ee/src/monitoring-endpoint-ee.ts

@@ -9,18 +9,26 @@ import { WorkspaceHealthMonitoring } from "./workspace/workspace-health-monitori
 import { TraceContext } from "@gitpod/gitpod-protocol/lib/util/tracing";
 import { log } from "@gitpod/gitpod-protocol/lib/util/logging";
 import { injectable, inject } from "inversify";
-import { register } from "../../src/prometheus-metrics";
+import { registerServerMetrics } from "../../src/prometheus-metrics";
+import * as prometheusClient from "prom-client";
+import { registerDBMetrics } from "@gitpod/gitpod-db/lib";
 
 @injectable()
 export class MonitoringEndpointsAppEE extends WorkspaceHealthMonitoring {
     @inject(WorkspaceHealthMonitoring) protected readonly workspaceHealthMonitoring: WorkspaceHealthMonitoring;
 
     public create(): express.Application {
+        const registry = prometheusClient.register;
+
+        prometheusClient.collectDefaultMetrics({ register: registry });
+        registerDBMetrics(registry);
+        registerServerMetrics(registry);
+
         const monApp = express();
         monApp.get("/metrics", async (req, res) => {
             try {
-                res.set("Content-Type", register.contentType);
-                res.end(await register.metrics());
+                res.set("Content-Type", registry.contentType);
+                res.end(await registry.metrics());
             } catch (ex) {
                 res.status(500).end(ex);
             }

components/server/src/prometheus-metrics.ts

@@ -10,11 +10,26 @@ import * as prometheusClient from "prom-client";
 prometheusClient.collectDefaultMetrics();
 export const register = prometheusClient.register;
 
+export function registerServerMetrics(registry: prometheusClient.Registry) {
+    registry.registerMetric(loginCounter);
+    registry.registerMetric(apiConnectionCounter);
+    registry.registerMetric(apiConnectionClosedCounter);
+    registry.registerMetric(apiCallCounter);
+    registry.registerMetric(apiCallDurationHistogram);
+    registry.registerMetric(apiCallUserCounter);
+    registry.registerMetric(httpRequestTotal);
+    registry.registerMetric(httpRequestDuration);
+    registry.registerMetric(messagebusTopicReads);
+    registry.registerMetric(gitpodVersionInfo);
+    registry.registerMetric(instanceStartsSuccessTotal);
+    registry.registerMetric(instanceStartsFailedTotal);
+    registry.registerMetric(prebuildsStartedTotal);
+}
+
 const loginCounter = new prometheusClient.Counter({
     name: "gitpod_server_login_requests_total",
     help: "Total amount of login requests",
     labelNames: ["status", "auth_host"],
-    registers: [prometheusClient.register],
 });
 
 export function increaseLoginCounter(status: string, auth_host: string) {
@@ -27,7 +42,6 @@ export function increaseLoginCounter(status: string, auth_host: string) {
 const apiConnectionCounter = new prometheusClient.Counter({
     name: "gitpod_server_api_connections_total",
     help: "Total amount of established API connections",
-    registers: [prometheusClient.register],
 });
 
 export function increaseApiConnectionCounter() {
@@ -37,7 +51,6 @@ export function increaseApiConnectionCounter() {
 const apiConnectionClosedCounter = new prometheusClient.Counter({
     name: "gitpod_server_api_connections_closed_total",
     help: "Total amount of closed API connections",
-    registers: [prometheusClient.register],
 });
 
 export function increaseApiConnectionClosedCounter() {
@@ -48,7 +61,6 @@ const apiCallCounter = new prometheusClient.Counter({
     name: "gitpod_server_api_calls_total",
     help: "Total amount of API calls per method",
     labelNames: ["method", "statusCode"],
-    registers: [prometheusClient.register],
 });
 
 export function increaseApiCallCounter(method: string, statusCode: number) {
@@ -60,7 +72,6 @@ export const apiCallDurationHistogram = new prometheusClient.Histogram({
     help: "Duration of API calls in seconds",
     labelNames: ["method"],
     buckets: [0.1, 0.5, 1, 5, 10, 15, 30],
-    registers: [prometheusClient.register],
 });
 
 export function observeAPICallsDuration(method: string, duration: number) {
@@ -71,7 +82,6 @@ const apiCallUserCounter = new prometheusClient.Counter({
     name: "gitpod_server_api_calls_user_total",
     help: "Total amount of API calls per user",
     labelNames: ["method", "user"],
-    registers: [prometheusClient.register],
 });
 
 export function increaseApiCallUserCounter(method: string, user: string) {
@@ -82,7 +92,6 @@ const httpRequestTotal = new prometheusClient.Counter({
     name: "gitpod_server_http_requests_total",
     help: "Total amount of HTTP requests per express route",
     labelNames: ["method", "route", "statusCode"],
-    registers: [prometheusClient.register],
 });
 
 export function increaseHttpRequestCounter(method: string, route: string, statusCode: number) {
@@ -94,7 +103,6 @@ const httpRequestDuration = new prometheusClient.Histogram({
     help: "Duration of HTTP requests in seconds",
     labelNames: ["method", "route", "statusCode"],
     buckets: [0.01, 0.05, 0.1, 0.5, 1, 5, 10],
-    registers: [prometheusClient.register],
 });
 
 export function observeHttpRequestDuration(
@@ -110,7 +118,6 @@ const messagebusTopicReads = new prometheusClient.Counter({
     name: "gitpod_server_topic_reads_total",
     help: "The amount of reads from messagebus topics.",
     labelNames: ["topic"],
-    registers: [prometheusClient.register],
 });
 
 export function increaseMessagebusTopicReads(topic: string) {
@@ -123,7 +130,6 @@ const gitpodVersionInfo = new prometheusClient.Gauge({
     name: "gitpod_version_info",
     help: "Gitpod's version",
     labelNames: ["gitpod_version"],
-    registers: [prometheusClient.register],
 });
 
 export function setGitpodVersion(gitpod_version: string) {
@@ -134,7 +140,6 @@ const instanceStartsSuccessTotal = new prometheusClient.Counter({
     name: "gitpod_server_instance_starts_success_total",
     help: "Total amount of successfully performed instance starts",
     labelNames: ["retries"],
-    registers: [prometheusClient.register],
 });
 
 export function increaseSuccessfulInstanceStartCounter(retries: number = 0) {
@@ -145,7 +150,6 @@ const instanceStartsFailedTotal = new prometheusClient.Counter({
     name: "gitpod_server_instance_starts_failed_total",
     help: "Total amount of failed performed instance starts",
     labelNames: ["reason"],
-    registers: [prometheusClient.register],
 });
 
 export type FailedInstanceStartReason =
@@ -160,19 +164,8 @@ export function increaseFailedInstanceStartCounter(reason: FailedInstanceStartRe
 const prebuildsStartedTotal = new prometheusClient.Counter({
     name: "gitpod_prebuilds_started_total",
     help: "Counter of total prebuilds started.",
-    registers: [prometheusClient.register],
 });
 
 export function increasePrebuildsStartedCounter() {
     prebuildsStartedTotal.inc();
 }
-
-const workspacesPurgedTotal = new prometheusClient.Counter({
-    name: "gitpod_server_workspaces_purged_total",
-    help: "Counter of workspaces hard deleted by periodic job running on server.",
-    registers: [prometheusClient.register],
-});
-
-export function reportWorkspacePurged() {
-    workspacesPurgedTotal.inc();
-}

components/server/src/workspace/workspace-deletion-service.ts

@@ -19,7 +19,6 @@ import { TraceContext } from "@gitpod/gitpod-protocol/lib/util/tracing";
 import { WorkspaceManagerClientProvider } from "@gitpod/ws-manager/lib/client-provider";
 import { DeleteVolumeSnapshotRequest } from "@gitpod/ws-manager/lib";
 import { log } from "@gitpod/gitpod-protocol/lib/util/logging";
-import { reportWorkspacePurged } from "../prometheus-metrics";
 
 @injectable()
 export class WorkspaceDeletionService {
@@ -57,7 +56,6 @@ export class WorkspaceDeletionService {
     public async hardDeleteWorkspace(ctx: TraceContext, workspaceId: string): Promise<void> {
         await this.db.trace(ctx).hardDeleteWorkspace(workspaceId);
         log.info(`Purged Workspace ${workspaceId} and all WorkspaceInstances for this workspace`, { workspaceId });
-        reportWorkspacePurged();
     }
 
     /**