diff --git a/components/gitpod-db/src/typeorm/entity/db-prebuilt-workspace-updatable.ts b/components/gitpod-db/src/typeorm/entity/db-prebuilt-workspace-updatable.ts new file mode 100644 index 00000000000000..9460f92883a6dd --- /dev/null +++ b/components/gitpod-db/src/typeorm/entity/db-prebuilt-workspace-updatable.ts @@ -0,0 +1,53 @@ +/** + * Copyright (c) 2020 Gitpod GmbH. All rights reserved. + * Licensed under the GNU Affero General Public License (AGPL). + * See License-AGPL.txt in the project root for license information. + */ + +import { PrimaryColumn, Column, Entity } from "typeorm"; + +import { PrebuiltWorkspaceUpdatable } from "@gitpod/gitpod-protocol"; +import { TypeORM } from "../typeorm"; +import { Transformer } from "../transformer"; + +@Entity() +export class DBPrebuiltWorkspaceUpdatable implements PrebuiltWorkspaceUpdatable { + + @PrimaryColumn(TypeORM.UUID_COLUMN_TYPE) + id: string; + + @Column(TypeORM.UUID_COLUMN_TYPE) + prebuiltWorkspaceId: string; + + @Column() + owner: string; + + @Column() + repo: string; + + @Column() + isResolved: boolean; + + @Column() + installationId: string; + + + @Column({ + default: '', + transformer: Transformer.MAP_EMPTY_STR_TO_UNDEFINED + }) + contextUrl?: string; + + @Column({ + default: '', + transformer: Transformer.MAP_EMPTY_STR_TO_UNDEFINED + }) + issue?: string; + + @Column({ + default: '', + transformer: Transformer.MAP_EMPTY_STR_TO_UNDEFINED + }) + label?: string; + +} \ No newline at end of file diff --git a/components/gitpod-db/src/typeorm/workspace-db-impl.ts b/components/gitpod-db/src/typeorm/workspace-db-impl.ts index eeed0cd4254d95..0386444bab6ece 100644 --- a/components/gitpod-db/src/typeorm/workspace-db-impl.ts +++ b/components/gitpod-db/src/typeorm/workspace-db-impl.ts @@ -6,8 +6,8 @@ import { injectable, inject } from "inversify"; import { Repository, EntityManager, DeepPartial, UpdateQueryBuilder, Brackets } from "typeorm"; -import { MaybeWorkspace, MaybeWorkspaceInstance, WorkspaceDB, FindWorkspacesOptions, WorkspaceInstanceSessionWithWorkspace, PrebuildWithWorkspace, WorkspaceAndOwner, WorkspacePortsAuthData, WorkspaceOwnerAndSoftDeleted } from "../workspace-db"; -import { Workspace, WorkspaceInstance, WorkspaceInfo, WorkspaceInstanceUser, WhitelistedRepository, Snapshot, LayoutData, PrebuiltWorkspace, RunningWorkspaceInfo, WorkspaceAndInstance, WorkspaceType, PrebuildInfo, AdminGetWorkspacesQuery, SnapshotState } from "@gitpod/gitpod-protocol"; +import { MaybeWorkspace, MaybeWorkspaceInstance, WorkspaceDB, FindWorkspacesOptions, PrebuiltUpdatableAndWorkspace, WorkspaceInstanceSessionWithWorkspace, PrebuildWithWorkspace, WorkspaceAndOwner, WorkspacePortsAuthData, WorkspaceOwnerAndSoftDeleted } from "../workspace-db"; +import { Workspace, WorkspaceInstance, WorkspaceInfo, WorkspaceInstanceUser, WhitelistedRepository, Snapshot, LayoutData, PrebuiltWorkspace, RunningWorkspaceInfo, PrebuiltWorkspaceUpdatable, WorkspaceAndInstance, WorkspaceType, PrebuildInfo, AdminGetWorkspacesQuery, SnapshotState } from "@gitpod/gitpod-protocol"; import { TypeORM } from "./typeorm"; import { DBWorkspace } from "./entity/db-workspace"; import { DBWorkspaceInstance } from "./entity/db-workspace-instance"; @@ -17,6 +17,7 @@ import { DBWorkspaceInstanceUser } from "./entity/db-workspace-instance-user"; import { DBRepositoryWhiteList } from "./entity/db-repository-whitelist"; import { log } from '@gitpod/gitpod-protocol/lib/util/logging'; import { DBPrebuiltWorkspace } from "./entity/db-prebuilt-workspace"; +import { DBPrebuiltWorkspaceUpdatable } from "./entity/db-prebuilt-workspace-updatable"; import { BUILTIN_WORKSPACE_PROBE_USER_ID } from "../user-db"; import { DBPrebuildInfo } from "./entity/db-prebuild-info-entry"; @@ -59,6 +60,10 @@ export abstract class AbstractTypeORMWorkspaceDBImpl implements WorkspaceDB { return await (await this.getManager()).getRepository(DBPrebuildInfo); } + protected async getPrebuiltWorkspaceUpdatableRepo(): Promise> { + return await (await this.getManager()).getRepository(DBPrebuiltWorkspaceUpdatable); + } + protected async getLayoutDataRepo(): Promise> { return await (await this.getManager()).getRepository(DBLayoutData); } @@ -652,6 +657,31 @@ export abstract class AbstractTypeORMWorkspaceDBImpl implements WorkspaceDB { } }); } + public async attachUpdatableToPrebuild(pwsid: string, update: PrebuiltWorkspaceUpdatable): Promise { + const repo = await this.getPrebuiltWorkspaceUpdatableRepo(); + await repo.save(update); + } + public async findUpdatablesForPrebuild(pwsid: string): Promise { + const repo = await this.getPrebuiltWorkspaceUpdatableRepo(); + return await repo.createQueryBuilder('pwsu') + .where('pwsu.prebuiltWorkspaceId = :pwsid', { pwsid }) + .getMany(); + } + public async markUpdatableResolved(updatableId: string): Promise { + const repo = await this.getPrebuiltWorkspaceUpdatableRepo(); + await repo.update(updatableId, { isResolved: true }); + } + public async getUnresolvedUpdatables(): Promise { + const pwsuRepo = await this.getPrebuiltWorkspaceUpdatableRepo(); + + // select * from d_b_prebuilt_workspace_updatable as pwsu left join d_b_prebuilt_workspace pws ON pws.id = pwsu.prebuiltWorkspaceId left join d_b_workspace ws on pws.buildWorkspaceId = ws.id left join d_b_workspace_instance wsi on ws.id = wsi.workspaceId where pwsu.isResolved = 0 + return await pwsuRepo.createQueryBuilder("pwsu") + .innerJoinAndMapOne('pwsu.prebuild', DBPrebuiltWorkspace, 'pws', 'pwsu.prebuiltWorkspaceId = pws.id') + .innerJoinAndMapOne('pwsu.workspace', DBWorkspace, 'ws', 'pws.buildWorkspaceId = ws.id') + .innerJoinAndMapOne('pwsu.instance', DBWorkspaceInstance, 'wsi', 'ws.id = wsi.workspaceId') + .where('pwsu.isResolved = 0') + .getMany() as any; + } public async findLayoutDataByWorkspaceId(workspaceId: string): Promise { const layoutDataRepo = await this.getLayoutDataRepo(); diff --git a/components/gitpod-db/src/workspace-db.ts b/components/gitpod-db/src/workspace-db.ts index 8d7800ff52c3cf..dd93e9d1345fcf 100644 --- a/components/gitpod-db/src/workspace-db.ts +++ b/components/gitpod-db/src/workspace-db.ts @@ -6,7 +6,7 @@ import { DeepPartial } from 'typeorm'; -import { Workspace, WorkspaceInfo, WorkspaceInstance, WorkspaceInstanceUser, WhitelistedRepository, Snapshot, LayoutData, PrebuiltWorkspace, RunningWorkspaceInfo, WorkspaceAndInstance, WorkspaceType, PrebuildInfo, AdminGetWorkspacesQuery, SnapshotState } from '@gitpod/gitpod-protocol'; +import { Workspace, WorkspaceInfo, WorkspaceInstance, WorkspaceInstanceUser, WhitelistedRepository, Snapshot, LayoutData, PrebuiltWorkspace, PrebuiltWorkspaceUpdatable, RunningWorkspaceInfo, WorkspaceAndInstance, WorkspaceType, PrebuildInfo, AdminGetWorkspacesQuery, SnapshotState } from '@gitpod/gitpod-protocol'; export type MaybeWorkspace = Workspace | undefined; export type MaybeWorkspaceInstance = WorkspaceInstance | undefined; @@ -21,6 +21,12 @@ export interface FindWorkspacesOptions { pinnedOnly?: boolean } +export interface PrebuiltUpdatableAndWorkspace extends PrebuiltWorkspaceUpdatable { + prebuild: PrebuiltWorkspace + workspace: Workspace + instance: WorkspaceInstance +} + export type WorkspaceAuthData = Pick; export type WorkspaceInstancePortsAuthData = Pick; export interface WorkspacePortsAuthData { @@ -100,6 +106,10 @@ export interface WorkspaceDB { findPrebuildByID(pwsid: string): Promise; countRunningPrebuilds(cloneURL: string): Promise; findQueuedPrebuilds(cloneURL?: string): Promise; + attachUpdatableToPrebuild(pwsid: string, update: PrebuiltWorkspaceUpdatable): Promise; + findUpdatablesForPrebuild(pwsid: string): Promise; + markUpdatableResolved(updatableId: string): Promise; + getUnresolvedUpdatables(): Promise; findLayoutDataByWorkspaceId(workspaceId: string): Promise; storeLayoutData(layoutData: LayoutData): Promise; diff --git a/components/gitpod-protocol/go/gitpod-service.go b/components/gitpod-protocol/go/gitpod-service.go index a5a7a4dc8bc0fa..8c0660a3d34efd 100644 --- a/components/gitpod-protocol/go/gitpod-service.go +++ b/components/gitpod-protocol/go/gitpod-service.go @@ -1826,6 +1826,7 @@ type GithubAppConfig struct { // GithubAppPrebuildConfig is the GithubAppPrebuildConfig message type type GithubAppPrebuildConfig struct { AddBadge bool `json:"addBadge,omitempty"` + AddCheck bool `json:"addCheck,omitempty"` AddComment bool `json:"addComment,omitempty"` AddLabel interface{} `json:"addLabel,omitempty"` Branches bool `json:"branches,omitempty"` diff --git a/components/gitpod-protocol/src/headless-workspace-log.ts b/components/gitpod-protocol/src/headless-workspace-log.ts index db887fdfd11ecc..7cac2a0bc7b91c 100644 --- a/components/gitpod-protocol/src/headless-workspace-log.ts +++ b/components/gitpod-protocol/src/headless-workspace-log.ts @@ -4,6 +4,30 @@ * See License-AGPL.txt in the project root for license information. */ + +export enum HeadlessWorkspaceEventType { + LogOutput = "log-output", + FinishedSuccessfully = "finish-success", + FinishedButFailed = "finish-fail", + AbortedTimedOut = "aborted-timeout", + Aborted = "aborted", + Started = "started" +} +export namespace HeadlessWorkspaceEventType { + export function isRunning(t: HeadlessWorkspaceEventType) { + return t === HeadlessWorkspaceEventType.LogOutput; + } + export function didFinish(t: HeadlessWorkspaceEventType) { + return t === HeadlessWorkspaceEventType.FinishedButFailed || t === HeadlessWorkspaceEventType.FinishedSuccessfully; + } +} + +export interface HeadlessWorkspaceEvent { + workspaceID: string; + text: string; + type: HeadlessWorkspaceEventType; +} + export interface HeadlessLogUrls { // A map of id to URL streams: { [streamID: string]: string }; diff --git a/components/gitpod-protocol/src/protocol.ts b/components/gitpod-protocol/src/protocol.ts index 965635dbaecaee..3ebc6714af89f6 100644 --- a/components/gitpod-protocol/src/protocol.ts +++ b/components/gitpod-protocol/src/protocol.ts @@ -591,6 +591,7 @@ export interface GithubAppPrebuildConfig { branches?: boolean pullRequests?: boolean pullRequestsFromForks?: boolean + addCheck?: boolean addBadge?: boolean addLabel?: boolean | string addComment?: boolean @@ -662,6 +663,17 @@ export namespace PrebuiltWorkspace { } } +export interface PrebuiltWorkspaceUpdatable { + id: string; + prebuiltWorkspaceId: string; + owner: string; + repo: string; + isResolved: boolean; + installationId: string; + issue?: string; + contextUrl?: string; +} + export interface WhitelistedRepository { url: string name: string diff --git a/components/server/ee/src/container-module.ts b/components/server/ee/src/container-module.ts index a35199d9c5d2de..8c9c96d35a7da7 100644 --- a/components/server/ee/src/container-module.ts +++ b/components/server/ee/src/container-module.ts @@ -22,6 +22,7 @@ import { HostContainerMappingEE } from "./auth/host-container-mapping"; import { PrebuildManager } from "./prebuilds/prebuild-manager"; import { GithubApp } from "./prebuilds/github-app"; import { GithubAppRules } from "./prebuilds/github-app-rules"; +import { PrebuildStatusMaintainer } from "./prebuilds/prebuilt-status-maintainer"; import { GitLabApp } from "./prebuilds/gitlab-app"; import { BitbucketApp } from "./prebuilds/bitbucket-app"; import { IPrefixContextParser } from "../../src/workspace/context-parser"; @@ -64,6 +65,7 @@ export const productionEEContainerModule = new ContainerModule((bind, unbind, is bind(GithubApp).toSelf().inSingletonScope(); bind(GitHubAppSupport).toSelf().inSingletonScope(); bind(GithubAppRules).toSelf().inSingletonScope(); + bind(PrebuildStatusMaintainer).toSelf().inSingletonScope(); bind(GitLabApp).toSelf().inSingletonScope(); bind(GitLabAppSupport).toSelf().inSingletonScope(); bind(BitbucketApp).toSelf().inSingletonScope(); diff --git a/components/server/ee/src/prebuilds/github-app-rules.ts b/components/server/ee/src/prebuilds/github-app-rules.ts index dd13d551bc5396..d8cf7e7da14b79 100644 --- a/components/server/ee/src/prebuilds/github-app-rules.ts +++ b/components/server/ee/src/prebuilds/github-app-rules.ts @@ -11,6 +11,7 @@ import { log } from '@gitpod/gitpod-protocol/lib/util/logging'; const defaultConfig: GithubAppConfig = { prebuilds: { + addCheck: true, addBadge: false, addComment: false, addLabel: false, @@ -59,7 +60,7 @@ export class GithubAppRules { } } - public shouldDo(cfg: WorkspaceConfig | undefined, action: 'addBadge' | 'addLabel' | 'addComment'): boolean { + public shouldDo(cfg: WorkspaceConfig | undefined, action: 'addCheck' | 'addBadge' | 'addLabel' | 'addComment'): boolean { const config = this.mergeWithDefaultConfig(cfg); const prebuildCfg = config.prebuilds!; @@ -67,7 +68,9 @@ export class GithubAppRules { return !!prebuildCfg; } - if (action === 'addBadge') { + if (action === 'addCheck') { + return !!prebuildCfg.addCheck; + } else if (action === 'addBadge') { return !!prebuildCfg.addBadge; } else if (action === 'addLabel') { return !!prebuildCfg.addLabel; diff --git a/components/server/ee/src/prebuilds/github-app.ts b/components/server/ee/src/prebuilds/github-app.ts index a76e2da6c73146..fc3cbfa53978a0 100644 --- a/components/server/ee/src/prebuilds/github-app.ts +++ b/components/server/ee/src/prebuilds/github-app.ts @@ -16,6 +16,7 @@ import { WorkspaceConfig, User, Project, StartPrebuildResult } from '@gitpod/git import { GithubAppRules } from './github-app-rules'; import { TraceContext } from '@gitpod/gitpod-protocol/lib/util/tracing'; import { PrebuildManager } from './prebuild-manager'; +import { PrebuildStatusMaintainer } from './prebuilt-status-maintainer'; import { Options, ApplicationFunctionOptions } from 'probot/lib/types'; import { asyncHandler } from '../../../src/express-util'; @@ -43,6 +44,7 @@ export class GithubApp { constructor( @inject(Config) protected readonly config: Config, + @inject(PrebuildStatusMaintainer) protected readonly statusMaintainer: PrebuildStatusMaintainer, ) { if (config.githubApp?.enabled) { this.server = new Server({ @@ -65,6 +67,15 @@ export class GithubApp { } protected async buildApp(app: Probot, options: ApplicationFunctionOptions) { + this.statusMaintainer.start(async (id) => { + try { + const githubApi = await app.auth(id); + return githubApi; + } catch (error) { + log.error("Failes to authorize GH API for Probot", { error }) + } + }); + // Backward-compatibility: Redirect old badge URLs (e.g. "/api/apps/github/pbs/github.com/gitpod-io/gitpod/5431d5735c32ab7d5d840a4d1a7d7c688d1f0ce9.svg") options.getRouter && options.getRouter('/pbs').get('/*', (req: express.Request, res: express.Response, next: express.NextFunction) => { res.redirect(301, this.getBadgeImageURL()); @@ -223,7 +234,8 @@ export class GithubApp { const contextURL = pr.html_url; const config = await this.prebuildManager.fetchConfig({ span }, owner.user, contextURL); - this.onPrStartPrebuild({ span }, config, owner, ctx); + const prebuildStartPromise = this.onPrStartPrebuild({ span }, config, owner, ctx); + this.onPrAddCheck({ span }, config, ctx, prebuildStartPromise); this.onPrAddBadge(config, ctx); this.onPrAddComment(config, ctx); } catch (e) { @@ -234,6 +246,41 @@ export class GithubApp { } } + protected async onPrAddCheck(tracecContext: TraceContext, config: WorkspaceConfig | undefined, ctx: Context<'pull_request.opened' | 'pull_request.synchronize' | 'pull_request.reopened'>, start: Promise | undefined) { + if (!start) { + return; + } + + if (!this.appRules.shouldDo(config, 'addCheck')) { + return; + } + + const span = TraceContext.startSpan("onPrAddCheck", tracecContext); + try { + const spr = await start; + const pws = await this.workspaceDB.trace({ span }).findPrebuildByWorkspaceID(spr.wsid); + if (!pws) { + return; + } + + const installationId = ctx.payload.installation?.id; + if (!installationId) { + log.info("Did not find user for installation. Probably an incomplete app installation.", { repo: ctx.payload.repository, installationId }); + return; + } + await this.statusMaintainer.registerCheckRun({ span }, installationId, pws, { + ...ctx.repo(), + head_sha: ctx.payload.pull_request.head.sha, + details_url: this.config.hostUrl.withContext(ctx.payload.pull_request.html_url).toString() + }); + } catch (err) { + TraceContext.setError({ span }, err); + throw err; + } finally { + span.finish(); + } + } + protected onPrStartPrebuild(tracecContext: TraceContext, config: WorkspaceConfig | undefined, owner: {user: User, project?: Project}, ctx: Context<'pull_request.opened' | 'pull_request.synchronize' | 'pull_request.reopened'>): Promise | undefined { const { user, project } = owner; const pr = ctx.payload.pull_request; diff --git a/components/server/ee/src/prebuilds/prebuilt-status-maintainer.ts b/components/server/ee/src/prebuilds/prebuilt-status-maintainer.ts new file mode 100644 index 00000000000000..d1b886237defad --- /dev/null +++ b/components/server/ee/src/prebuilds/prebuilt-status-maintainer.ts @@ -0,0 +1,229 @@ +/** + * Copyright (c) 2020 Gitpod GmbH. All rights reserved. + * Licensed under the Gitpod Enterprise Source Code License, + * See License.enterprise.txt in the project root folder. + */ + +import { ProbotOctokit } from 'probot'; +import { injectable, inject } from 'inversify'; +import { WorkspaceDB, TracedWorkspaceDB, DBWithTracing } from '@gitpod/gitpod-db/lib'; +import { v4 as uuidv4 } from 'uuid'; +import { HeadlessWorkspaceEvent } from '@gitpod/gitpod-protocol/lib/headless-workspace-log'; +import { log } from '@gitpod/gitpod-protocol/lib/util/logging'; +import { PrebuiltWorkspaceUpdatable, PrebuiltWorkspace, Disposable, DisposableCollection } from '@gitpod/gitpod-protocol'; +import { TraceContext } from '@gitpod/gitpod-protocol/lib/util/tracing'; +import { LocalMessageBroker } from '../../../src/messaging/local-message-broker'; +import { repeat } from "@gitpod/gitpod-protocol/lib/util/repeat"; + +export interface CheckRunInfo { + owner: string; + repo: string; + head_sha: string; + details_url: string; +} + +// 6 hours +const MAX_UPDATABLE_AGE = 6 * 60 * 60 * 1000; +const DEFAULT_STATUS_DESCRIPTION = "Open a prebuilt online workspace in Gitpod"; +const NON_PREBUILT_STATUS_DESCRIPTION = "Open an online workspace in Gitpod"; + +export type AuthenticatedGithubProvider = (installationId: number) => Promise | undefined>; + +@injectable() +export class PrebuildStatusMaintainer implements Disposable { + @inject(TracedWorkspaceDB) protected readonly workspaceDB: DBWithTracing; + @inject(LocalMessageBroker) protected readonly localMessageBroker: LocalMessageBroker; + protected githubApiProvider: AuthenticatedGithubProvider; + protected readonly disposables = new DisposableCollection(); + + start(githubApiProvider: AuthenticatedGithubProvider): void { + // set github before registering the msgbus listener - otherwise an incoming message and the github set might race + this.githubApiProvider = githubApiProvider; + + this.disposables.push( + this.localMessageBroker.listenForPrebuildUpdatableEvents((ctx, msg) => this.handlePrebuildFinished(ctx, msg)) + ); + this.disposables.push( + repeat(this.periodicUpdatableCheck.bind(this), 60 * 1000) + ); + log.debug("prebuild updatatable status maintainer started"); + } + + public async registerCheckRun(ctx: TraceContext, installationId: number, pws: PrebuiltWorkspace, cri: CheckRunInfo) { + const span = TraceContext.startSpan("registerCheckRun", ctx); + span.setTag("pws-state", pws.state); + + try { + const githubApi = await this.getGitHubApi(installationId); + if (!githubApi) { + throw new Error("unable to authenticate GitHub app"); + } + + if (pws.state == 'queued' || pws.state == "building") { + await this.workspaceDB.trace({span}).attachUpdatableToPrebuild(pws.id, { + id: uuidv4(), + owner: cri.owner, + repo: cri.repo, + isResolved: false, + installationId: installationId.toString(), + contextUrl: cri.details_url, + prebuiltWorkspaceId: pws.id, + }); + await githubApi.repos.createCommitStatus({ + repo: cri.repo, + owner: cri.owner, + sha: cri.head_sha, + target_url: cri.details_url, + context: "Gitpod", + description: "prebuilding an online workspace for this PR", + state: "pending", + }); + } else { + // prebuild isn't running - mark with check + const conclusion = this.getConclusionFromPrebuildState(pws); + await githubApi.repos.createCommitStatus({ + repo: cri.repo, + owner: cri.owner, + sha: cri.head_sha, + target_url: cri.details_url, + context: "Gitpod", + description: conclusion == 'success' ? DEFAULT_STATUS_DESCRIPTION : NON_PREBUILT_STATUS_DESCRIPTION, + + // at the moment we run in 'evergreen' mode where we always report success for status checks + state: "success", + }); + } + } catch (err) { + TraceContext.setError({span}, err); + throw err; + } finally { + span.finish(); + } + } + + protected getConclusionFromPrebuildState(pws: PrebuiltWorkspace): "error" | "failure" | "pending" | "success" { + if (pws.state === "aborted") { + return "error"; + } else if (pws.state === "queued") { + return "pending"; + } else if (pws.state === "timeout") { + return "error"; + } else if (pws.state === "available" && !pws.error) { + return "success"; + } else if (pws.state === "available" && !!pws.error) { + // Not sure if this is the right choice - do we really want the check to fail if the prebuild fails? + return "failure"; + } else if (pws.state === "building") { + return "pending"; + } else { + log.warn("Should have updated prebuilt workspace updatable, but don't know how. Resorting to error conclusion.", { pws }); + return "error"; + } + } + + protected async handlePrebuildFinished(ctx: TraceContext, msg: HeadlessWorkspaceEvent) { + const span = TraceContext.startSpan("PrebuildStatusMaintainer.handlePrebuildFinished", ctx) + + try { + // this code assumes that the prebuild is updated in the database before the msgbus msg is received + const prebuild = await this.workspaceDB.trace({span}).findPrebuildByWorkspaceID(msg.workspaceID); + if (!prebuild) { + log.warn("received headless log message without associated prebuild", msg); + return; + } + + const updatatables = await this.workspaceDB.trace({span}).findUpdatablesForPrebuild(prebuild.id); + await Promise.all(updatatables.filter(u => !u.isResolved).map(u => this.doUpdate({span}, u, prebuild))); + } catch (err) { + TraceContext.setError({span}, err); + throw err; + } finally { + span.finish(); + } + } + + protected async doUpdate(ctx: TraceContext, updatatable: PrebuiltWorkspaceUpdatable, pws: PrebuiltWorkspace): Promise { + const span = TraceContext.startSpan("doUpdate", ctx); + + try { + const githubApi = await this.getGitHubApi(Number.parseInt(updatatable.installationId)); + if (!githubApi) { + log.error("unable to authenticate GitHub app - this leaves user-facing checks dangling."); + return; + } + + if (!!updatatable.contextUrl) { + const conclusion = this.getConclusionFromPrebuildState(pws); + + let found = true; + try { + await githubApi.repos.createCommitStatus({ + owner: updatatable.owner, + repo: updatatable.repo, + context: "Gitpod", + sha: pws.commit, + target_url: updatatable.contextUrl, + // at the moment we run in 'evergreen' mode where we always report success for status checks + description: conclusion == 'success' ? DEFAULT_STATUS_DESCRIPTION : NON_PREBUILT_STATUS_DESCRIPTION, + state: "success" + }); + } catch (err) { + if (err.message == "Not Found") { + log.info("Did not find repository while updating updatable. Probably we lost the GitHub permission for the repo.", {owner: updatatable.owner, repo: updatatable.repo}); + found = true; + } else { + throw err; + } + } + TraceContext.addNestedTags({ span }, { + doUpdate: { + update: 'done', + found, + }, + }); + + await this.workspaceDB.trace({span}).markUpdatableResolved(updatatable.id); + log.info(`Resolved updatable. Marked check on ${updatatable.contextUrl} as ${conclusion}`); + } else if (!!updatatable.issue) { + // this updatatable updates a label + log.debug("Update label on a PR - we're not using this yet"); + } + } catch (err) { + TraceContext.setError({span}, err); + throw err; + } finally { + span.finish(); + } + } + + protected async getGitHubApi(installationId: number): Promise | undefined> { + const api = await this.githubApiProvider(installationId); + if (!api) { + return undefined + } + return (api as InstanceType); + } + + protected async periodicUpdatableCheck() { + const ctx = TraceContext.childContext("periodicUpdatableCheck", {}); + + try { + const unresolvedUpdatables = await this.workspaceDB.trace(ctx).getUnresolvedUpdatables(); + for (const updatable of unresolvedUpdatables) { + if ((Date.now() - Date.parse(updatable.workspace.creationTime)) > MAX_UPDATABLE_AGE) { + log.info("found unresolved updatable that's older than MAX_UPDATABLE_AGE and is inconclusive. Resolving.", updatable); + await this.doUpdate(ctx, updatable, updatable.prebuild); + } + } + } catch (err) { + TraceContext.setError(ctx, err); + throw err; + } finally { + ctx.span?.finish(); + } + } + + dispose(): void { + this.disposables.dispose(); + } +} diff --git a/components/server/src/messaging/local-message-broker.ts b/components/server/src/messaging/local-message-broker.ts index bb3577aaeca639..c0e11c637489bc 100644 --- a/components/server/src/messaging/local-message-broker.ts +++ b/components/server/src/messaging/local-message-broker.ts @@ -4,7 +4,7 @@ * See License-AGPL.txt in the project root for license information. */ -import { Disposable, DisposableCollection, PrebuildWithStatus, WorkspaceInstance } from "@gitpod/gitpod-protocol"; +import { Disposable, DisposableCollection, HeadlessWorkspaceEvent, PrebuildWithStatus, WorkspaceInstance } from "@gitpod/gitpod-protocol"; import { CreditAlert } from "@gitpod/gitpod-protocol/lib/accounting-protocol"; import { log } from "@gitpod/gitpod-protocol/lib/util/logging"; import { TraceContext } from "@gitpod/gitpod-protocol/lib/util/tracing"; @@ -18,6 +18,9 @@ export interface PrebuildUpdateListener { export interface CreditAlertListener { (ctx: TraceContext, alert: CreditAlert): void; } +export interface HeadlessWorkspaceEventListener { + (ctx: TraceContext, evt: HeadlessWorkspaceEvent): void; +} export interface WorkspaceInstanceUpdateListener { (ctx: TraceContext, instance: WorkspaceInstance): void; } @@ -32,6 +35,8 @@ export interface LocalMessageBroker { listenToCreditAlerts(userId: string, listener: CreditAlertListener): Disposable; + listenForPrebuildUpdatableEvents(listener: HeadlessWorkspaceEventListener): Disposable; + listenForWorkspaceInstanceUpdates(userId: string, listener: WorkspaceInstanceUpdateListener): Disposable; } @@ -58,6 +63,7 @@ export class LocalRabbitMQBackedMessageBroker implements LocalMessageBroker { protected prebuildUpdateListeners: Map = new Map(); protected creditAlertsListeners: Map = new Map(); + protected headlessWorkspaceEventListeners: Map = new Map(); protected workspaceInstanceUpdateListeners: Map = new Map(); protected readonly disposables = new DisposableCollection(); @@ -95,6 +101,21 @@ export class LocalRabbitMQBackedMessageBroker implements LocalMessageBroker { } } )); + this.disposables.push(this.messageBusIntegration.listenForPrebuildUpdatableQueue( + (ctx: TraceContext, evt: HeadlessWorkspaceEvent) => { + TraceContext.setOWI(ctx, { workspaceId: evt.workspaceID }); + + const listeners = this.headlessWorkspaceEventListeners.get(LocalRabbitMQBackedMessageBroker.UNDEFINED_KEY) || []; + for (const l of listeners) { + try { + l(ctx, evt); + } catch (err) { + TraceContext.setError(ctx, err); + log.error({ workspaceId: evt.workspaceID }, "listenForPrebuildUpdatableQueue", err); + } + } + } + )); this.disposables.push(this.messageBusIntegration.listenForWorkspaceInstanceUpdates( undefined, (ctx: TraceContext, instance: WorkspaceInstance, userId: string | undefined) => { @@ -129,6 +150,11 @@ export class LocalRabbitMQBackedMessageBroker implements LocalMessageBroker { return this.doRegister(userId, listener, this.creditAlertsListeners); } + listenForPrebuildUpdatableEvents(listener: HeadlessWorkspaceEventListener): Disposable { + // we're being cheap here in re-using a map where it just needs to be a plain array. + return this.doRegister(LocalRabbitMQBackedMessageBroker.UNDEFINED_KEY, listener, this.headlessWorkspaceEventListeners); + } + listenForWorkspaceInstanceUpdates(userId: string, listener: WorkspaceInstanceUpdateListener): Disposable { return this.doRegister(userId, listener, this.workspaceInstanceUpdateListeners); } diff --git a/components/server/src/workspace/messagebus-integration.ts b/components/server/src/workspace/messagebus-integration.ts index 5dfc25f581fa4e..410a0611bbf6e5 100644 --- a/components/server/src/workspace/messagebus-integration.ts +++ b/components/server/src/workspace/messagebus-integration.ts @@ -5,9 +5,13 @@ */ import { injectable } from "inversify"; -import { AbstractMessageBusIntegration, MessageBusHelper, AbstractTopicListener, TopicListener, MessageBusHelperImpl } from "@gitpod/gitpod-messagebus/lib"; +import { AbstractMessageBusIntegration, MessageBusHelper, AbstractTopicListener, TopicListener, MessageBusHelperImpl, MessagebusListener } from "@gitpod/gitpod-messagebus/lib"; import { Disposable, PrebuildWithStatus, WorkspaceInstance } from "@gitpod/gitpod-protocol"; +import { log } from '@gitpod/gitpod-protocol/lib/util/logging'; +import { HeadlessWorkspaceEvent, HeadlessWorkspaceEventType } from "@gitpod/gitpod-protocol/lib/headless-workspace-log"; +import { Channel, Message } from "amqplib"; import { TraceContext } from "@gitpod/gitpod-protocol/lib/util/tracing"; +import * as opentracing from "opentracing"; import { CancellationTokenSource } from "vscode-ws-jsonrpc"; import { increaseMessagebusTopicReads } from '../prometheus-metrics'; import { CreditAlert } from "@gitpod/gitpod-protocol/lib/accounting-protocol"; @@ -52,6 +56,69 @@ export class CreditAlertListener extends AbstractTopicListener { } } +export class PrebuildUpdatableQueueListener implements MessagebusListener { + protected channel: Channel | undefined; + protected consumerTag: string | undefined; + constructor(protected readonly callback: (ctx: TraceContext, evt: HeadlessWorkspaceEvent) => void) { } + + async establish(channel: Channel): Promise { + this.channel = channel; + + await MessageBusHelperImpl.assertPrebuildWorkspaceUpdatableQueue(this.channel); + const consumer = await channel.consume(MessageBusHelperImpl.PREBUILD_UPDATABLE_QUEUE, message => { + this.handleMessage(message); + }, { noAck: false }); + this.consumerTag = consumer.consumerTag; + } + + protected handleMessage(message: Message | null) { + if (message === null) return; + if (this.channel !== undefined) { + this.channel.ack(message); + } + + const spanCtx = opentracing.globalTracer().extract(opentracing.FORMAT_HTTP_HEADERS, message.properties.headers); + const span = !!spanCtx ? opentracing.globalTracer().startSpan(`/messagebus/${MessageBusHelperImpl.PREBUILD_UPDATABLE_QUEUE}`, {references: [opentracing.childOf(spanCtx!)]}) : undefined; + + let msg: any | undefined; + try { + const content = message.content; + const jsonContent = JSON.parse(content.toString()); + msg = jsonContent as HeadlessWorkspaceEvent; + } catch (e) { + log.warn('Caught message without or with invalid JSON content', e, { message }); + } + + if (msg) { + try { + this.callback({ span }, msg); + } catch (e) { + log.error('Error while executing message handler', e, { message }); + } finally { + if (span) { + span.finish(); + } + } + } + } + + async dispose(): Promise { + if (!this.channel || !this.consumerTag) return; + + try { + // cancel our subscription on the queue + await this.channel.cancel(this.consumerTag); + this.channel = this.consumerTag = undefined; + } catch (e) { + if (e instanceof Error && e.toString().includes('Channel closed')) { + // This is expected behavior when the message bus server goes down. + } else { + throw e; + } + } + } +} + @injectable() export class MessageBusIntegration extends AbstractMessageBusIntegration { @@ -65,6 +132,13 @@ export class MessageBusIntegration extends AbstractMessageBusIntegration { } } + listenForPrebuildUpdatableQueue(callback: (ctx: TraceContext, evt: HeadlessWorkspaceEvent) => void): Disposable { + const listener = new PrebuildUpdatableQueueListener(callback); + const cancellationTokenSource = new CancellationTokenSource() + this.listen(listener, cancellationTokenSource.token); + return Disposable.create(() => cancellationTokenSource.cancel()) + } + listenForWorkspaceInstanceUpdates(userId: string | undefined, callback: WorkspaceInstanceUpdateCallback): Disposable { const listener = new WorkspaceInstanceUpdateListener(this.messageBusHelper, callback, userId); const cancellationTokenSource = new CancellationTokenSource() @@ -115,4 +189,28 @@ export class MessageBusIntegration extends AbstractMessageBusIntegration { await super.publish(MessageBusHelperImpl.WORKSPACE_EXCHANGE_LOCAL, topic, Buffer.from(JSON.stringify(instance))); } + // copied from ws-manager-bridge/messagebus-integration + async notifyHeadlessUpdate(ctx: TraceContext, userId: string, workspaceId: string, evt: HeadlessWorkspaceEvent) { + if (!this.channel) { + throw new Error("Not connected to message bus"); + } + + const topic = this.messageBusHelper.getWsTopicForPublishing(userId, workspaceId, 'headless-log'); + const msg = Buffer.from(JSON.stringify(evt)); + await this.messageBusHelper.assertWorkspaceExchange(this.channel); + await super.publish(MessageBusHelperImpl.WORKSPACE_EXCHANGE_LOCAL, topic, msg, { + trace: ctx, + }); + + // Prebuild updatables use a single queue to implement round-robin handling of updatables. + // We need to write to that queue in addition to the regular log exchange. + if (!HeadlessWorkspaceEventType.isRunning(evt.type)) { + await MessageBusHelperImpl.assertPrebuildWorkspaceUpdatableQueue(this.channel!); + await super.publishToQueue(MessageBusHelperImpl.PREBUILD_UPDATABLE_QUEUE, msg, { + persistent: true, + trace: ctx, + }); + } + } + } diff --git a/components/server/src/workspace/workspace-starter.ts b/components/server/src/workspace/workspace-starter.ts index 7195613824fc34..267fd06d6fb450 100644 --- a/components/server/src/workspace/workspace-starter.ts +++ b/components/server/src/workspace/workspace-starter.ts @@ -7,7 +7,7 @@ import { CloneTargetMode, FileDownloadInitializer, GitAuthMethod, GitConfig, GitInitializer, PrebuildInitializer, SnapshotInitializer, WorkspaceInitializer } from "@gitpod/content-service/lib"; import { CompositeInitializer, FromBackupInitializer } from "@gitpod/content-service/lib/initializer_pb"; import { DBUser, DBWithTracing, TracedUserDB, TracedWorkspaceDB, UserDB, WorkspaceDB } from '@gitpod/gitpod-db/lib'; -import { CommitContext, Disposable, GitpodToken, GitpodTokenType, IssueContext, NamedWorkspaceFeatureFlag, PullRequestContext, RefType, SnapshotContext, StartWorkspaceResult, User, UserEnvVar, UserEnvVarValue, WithEnvvarsContext, WithPrebuild, Workspace, WorkspaceContext, WorkspaceImageSource, WorkspaceImageSourceDocker, WorkspaceImageSourceReference, WorkspaceInstance, WorkspaceInstanceConfiguration, WorkspaceInstanceStatus, WorkspaceProbeContext, Permission, DisposableCollection, AdditionalContentContext, ImageConfigFile } from "@gitpod/gitpod-protocol"; +import { CommitContext, Disposable, GitpodToken, GitpodTokenType, IssueContext, NamedWorkspaceFeatureFlag, PullRequestContext, RefType, SnapshotContext, StartWorkspaceResult, User, UserEnvVar, UserEnvVarValue, WithEnvvarsContext, WithPrebuild, Workspace, WorkspaceContext, WorkspaceImageSource, WorkspaceImageSourceDocker, WorkspaceImageSourceReference, WorkspaceInstance, WorkspaceInstanceConfiguration, WorkspaceInstanceStatus, WorkspaceProbeContext, Permission, HeadlessWorkspaceEvent, HeadlessWorkspaceEventType, DisposableCollection, AdditionalContentContext, ImageConfigFile } from "@gitpod/gitpod-protocol"; import { IAnalyticsWriter } from '@gitpod/gitpod-protocol/lib/analytics'; import { log } from '@gitpod/gitpod-protocol/lib/util/logging'; import { TraceContext } from "@gitpod/gitpod-protocol/lib/util/tracing"; @@ -285,6 +285,10 @@ export class WorkspaceStarter { prebuild.error = err.toString(); await this.workspaceDb.trace({ span }).storePrebuiltWorkspace(prebuild) + await this.messageBus.notifyHeadlessUpdate({span}, workspace.ownerId, workspace.id, { + type: HeadlessWorkspaceEventType.Aborted, + // TODO: `workspaceID: workspace.id` not needed here? (found in ee/src/prebuilds/prebuild-queue-maintainer.ts and ee/src/bridge.ts) + }); } } } catch (err) { diff --git a/components/ws-manager-bridge/ee/src/bridge.ts b/components/ws-manager-bridge/ee/src/bridge.ts index d65287b0b9385c..598ec79cec837d 100644 --- a/components/ws-manager-bridge/ee/src/bridge.ts +++ b/components/ws-manager-bridge/ee/src/bridge.ts @@ -8,6 +8,7 @@ import { WorkspaceManagerBridge } from "../../src/bridge"; import { injectable } from "inversify"; import { TraceContext } from "@gitpod/gitpod-protocol/lib/util/tracing"; import { WorkspaceStatus, WorkspaceType, WorkspacePhase } from "@gitpod/ws-manager/lib"; +import { HeadlessWorkspaceEvent, HeadlessWorkspaceEventType } from "@gitpod/gitpod-protocol/lib/headless-workspace-log"; import { WorkspaceInstance } from "@gitpod/gitpod-protocol"; import { log } from "@gitpod/gitpod-protocol/lib/util/logging"; @@ -61,27 +62,42 @@ export class WorkspaceManagerBridgeEE extends WorkspaceManagerBridge { prebuild.state = "building"; await this.workspaceDB.trace({span}).storePrebuiltWorkspace(prebuild); + await this.messagebus.notifyHeadlessUpdate({span}, userId, workspaceId, { + type: HeadlessWorkspaceEventType.Started, + workspaceID: workspaceId, + }); } if (status.phase === WorkspacePhase.STOPPING) { + let headlessUpdateType: HeadlessWorkspaceEventType = HeadlessWorkspaceEventType.Aborted; if (!!status.conditions!.timeout) { prebuild.state = "timeout"; prebuild.error = status.conditions!.timeout; + headlessUpdateType = HeadlessWorkspaceEventType.AbortedTimedOut; } else if (!!status.conditions!.failed) { prebuild.state = "aborted"; prebuild.error = status.conditions!.failed; + headlessUpdateType = HeadlessWorkspaceEventType.Aborted; } else if (!!status.conditions!.stoppedByRequest) { prebuild.state = "aborted"; prebuild.error = "Cancelled"; + headlessUpdateType = HeadlessWorkspaceEventType.Aborted; } else if (!!status.conditions!.headlessTaskFailed) { prebuild.state = "available"; prebuild.error = status.conditions!.headlessTaskFailed; prebuild.snapshot = status.conditions!.snapshot; + headlessUpdateType = HeadlessWorkspaceEventType.FinishedButFailed; } else { prebuild.state = "available"; prebuild.snapshot = status.conditions!.snapshot; + headlessUpdateType = HeadlessWorkspaceEventType.FinishedSuccessfully; } await this.workspaceDB.trace({span}).storePrebuiltWorkspace(prebuild); + + await this.messagebus.notifyHeadlessUpdate({span}, userId, workspaceId, { + type: headlessUpdateType, + workspaceID: workspaceId, + }); } { // notify about prebuild updated diff --git a/components/ws-manager-bridge/src/messagebus-integration.ts b/components/ws-manager-bridge/src/messagebus-integration.ts index 135c7ce593a881..08ab721797a351 100644 --- a/components/ws-manager-bridge/src/messagebus-integration.ts +++ b/components/ws-manager-bridge/src/messagebus-integration.ts @@ -8,7 +8,7 @@ import { injectable, inject } from 'inversify'; import { MessageBusHelper, AbstractMessageBusIntegration, TopicListener, AbstractTopicListener, MessageBusHelperImpl } from "@gitpod/gitpod-messagebus/lib"; import { Disposable, CancellationTokenSource } from 'vscode-jsonrpc'; import { WorkspaceStatus } from '@gitpod/ws-manager/lib'; -import { WorkspaceInstance, PrebuildWithStatus } from '@gitpod/gitpod-protocol'; +import { HeadlessWorkspaceEventType, WorkspaceInstance, HeadlessWorkspaceEvent, PrebuildWithStatus } from '@gitpod/gitpod-protocol'; import { TraceContext } from '@gitpod/gitpod-protocol/lib/util/tracing'; @injectable() @@ -64,6 +64,29 @@ export class MessageBusIntegration extends AbstractMessageBusIntegration { } } + async notifyHeadlessUpdate(ctx: TraceContext, userId: string, workspaceId: string, evt: HeadlessWorkspaceEvent) { + if (!this.channel) { + throw new Error("Not connected to message bus"); + } + + const topic = this.messageBusHelper.getWsTopicForPublishing(userId, workspaceId, 'headless-log'); + const msg = new Buffer(JSON.stringify(evt)); + await this.messageBusHelper.assertWorkspaceExchange(this.channel); + await super.publish(MessageBusHelperImpl.WORKSPACE_EXCHANGE_LOCAL, topic, msg, { + trace: ctx, + }); + + // Prebuild updatables use a single queue to implement round-robin handling of updatables. + // We need to write to that queue in addition to the regular log exchange. + if (!HeadlessWorkspaceEventType.isRunning(evt.type)) { + await MessageBusHelperImpl.assertPrebuildWorkspaceUpdatableQueue(this.channel!); + await super.publishToQueue(MessageBusHelperImpl.PREBUILD_UPDATABLE_QUEUE, msg, { + persistent: true, + trace: ctx, + }); + } + } + async disconnect(): Promise { if (this.channel) { this.channel.close();