Skip to content

Commit c8257e3

Browse files
author
Forest Hoffman
committed
Scaffold bucket uploader for e2e CI-screenshots
1 parent a77b72a commit c8257e3

File tree

6 files changed

+415
-25
lines changed

6 files changed

+415
-25
lines changed

‎test/package.json‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
"devDependencies": {
1414
"@types/jest": "^24.0.11",
1515
"@types/jest-environment-puppeteer": "^4.0.0",
16+
"@types/node-fetch": "^2.3.2",
1617
"@types/puppeteer": "^1.12.3",
1718
"@types/xml2js": "^0.4.3"
1819
},
@@ -45,6 +46,8 @@
4546
},
4647
"dependencies": {
4748
"@coder/logger": "^1.1.3",
48-
"jest": "^24.7.1"
49+
"@google-cloud/storage": "^2.5.0",
50+
"jest": "^24.7.1",
51+
"node-fetch": "^2.3.0"
4952
}
5053
}

‎test/src/index.ts‎

Lines changed: 29 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,11 @@ import * as os from "os";
44
import * as path from "path";
55
import * as puppeteer from "puppeteer";
66
import { ChildProcess, spawn } from "child_process";
7-
import { logger, field, Level } from "@coder/logger";
7+
import { logger, field } from "@coder/logger";
8+
9+
import { GoogleCloudBucket } from "../storage/gcloud";
10+
11+
const bucket = new GoogleCloudBucket();
812

913
interface IServerOptions {
1014
host: string;
@@ -142,19 +146,31 @@ export class TestPage {
142146
* `<screenshotDir>/[<page-tag>_]<screenshot-number>_<screenshot-name>.jpg`.
143147
*/
144148
public async screenshot(name: string, options?: puppeteer.ScreenshotOptions): Promise<string | Buffer> {
145-
let debugCI = logger.level === Level.Debug && process.env.TRAVIS_BUILD_NUMBER;
146-
let tag = this.tag ? `${this.tag}_` : "";
147-
if (debugCI) {
148-
tag = `TRAVIS-${process.env.TRAVIS_BUILD_NUMBER}_${tag}`;
149-
}
150-
options = Object.assign({ path: path.resolve(TestServer.puppeteerDir, `./${tag}${this.screenshotCount}_${name}.jpg`), fullPage: true }, options);
149+
const fileName = `${this.tag ? `${this.tag}_` : ""}${this.screenshotCount}_${name}.jpg`;
150+
options = Object.assign({
151+
path: path.resolve(TestServer.puppeteerDir, `./${fileName}`),
152+
fullPage: true,
153+
type: "jpeg",
154+
}, options);
151155
const img = await this.rootPage.screenshot(options);
152156
this.screenshotCount++;
153157

154-
if (debugCI) {
155-
// TODO: upload to imgur.
156-
const url = "";
157-
logger.debug("captured screenshot", field("path", options.path), field("url", url));
158+
if (process.env.TRAVIS_OS_NAME && process.env.TRAVIS_BUILD_NUMBER) {
159+
const bucketPath = `Travis-${process.env.TRAVIS_BUILD_NUMBER}/${fileName}`;
160+
let buf: Buffer = typeof img === "string" ? Buffer.from(img as string) : img;
161+
try {
162+
const url = await bucket.write(bucketPath, buf);
163+
logger.info("captured screenshot",
164+
field("localPath", options.path),
165+
field("bucketPath", bucketPath),
166+
field("url", url),
167+
);
168+
} catch (ex) {
169+
logger.warn("failed to capture screenshot",
170+
field("exception", ex),
171+
field("targetPath", bucketPath),
172+
);
173+
}
158174
}
159175

160176
return img;
@@ -224,8 +240,10 @@ export class TestServer {
224240

225241
// @ts-ignore
226242
private child: ChildProcess;
243+
227244
// The directory to load the IDE with.
228245
public static readonly workingDir = path.resolve(__dirname, "../tmp/workspace/");
246+
// The directory to store puppeteer related files.
229247
public static readonly puppeteerDir = path.resolve(TestServer.workingDir, "../puppeteer/", Date.now().toString());
230248

231249
public constructor(opts?: {

‎test/storage/bucket.ts‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
/**
2+
* Storage Bucket Object.
3+
*/
4+
export class File {
5+
public constructor(
6+
public readonly name: string,
7+
public readonly createdAt: Date,
8+
public readonly updatedAt: Date,
9+
public readonly size: number,
10+
public readonly metadata: object,
11+
) { }
12+
13+
public get isFolder(): boolean {
14+
return this.name.endsWith("/");
15+
}
16+
}
17+
18+
export interface IMetadata {
19+
readonly contentType: string;
20+
readonly contentEncoding: string;
21+
readonly cacheControl: string;
22+
}
23+
24+
/**
25+
* Storage Bucket I/O.
26+
*/
27+
export abstract class Bucket {
28+
public abstract read(path: string): Promise<Buffer>;
29+
public abstract list(prefix?: string): Promise<File[]>;
30+
public abstract write(path: string, value: Buffer, makePublic?: true, metadata?: IMetadata): Promise<string>;
31+
public abstract write(path: string, value: Buffer, makePublic?: false, metadata?: IMetadata): Promise<void>;
32+
}

‎test/storage/gcloud.test.ts‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
import fetch from "node-fetch";
2+
import { GoogleCloudBucket } from "./gcloud";
3+
4+
describe("gcloud bucket", () => {
5+
const bucket = new GoogleCloudBucket();
6+
7+
const expectObjectContent = async (objUrl: string, content: string): Promise<void> => {
8+
expect(await fetch(objUrl).then((resp) => resp.text())).toEqual(content);
9+
};
10+
11+
const expectWrite = async (path: string, content: string): Promise<string> => {
12+
const publicUrl = await bucket.write(path, Buffer.from(content), true);
13+
await expectObjectContent(publicUrl, content);
14+
15+
return publicUrl;
16+
};
17+
18+
it("should write file", async () => {
19+
await expectWrite("/test", "hi");
20+
});
21+
});

‎test/storage/gcloud.ts‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
import * as Storage from "@google-cloud/storage";
2+
import { File, Bucket, IMetadata } from "./bucket";
3+
4+
const ScreenshotBucketName = "coder-dev-1-ci-screenshots";
5+
6+
/**
7+
* GCP Storage Bucket Wrapper.
8+
*/
9+
export class GoogleCloudBucket extends Bucket {
10+
private readonly bucket: Storage.Bucket;
11+
12+
public constructor() {
13+
super();
14+
const storage = new Storage.Storage({
15+
projectId: "coder-dev-1",
16+
});
17+
this.bucket = storage.bucket(ScreenshotBucketName);
18+
}
19+
20+
/**
21+
* Read object in bucket to a Buffer.
22+
*/
23+
public read(path: string): Promise<Buffer> {
24+
return new Promise<Buffer>((resolve, reject): void => {
25+
const stream = this.bucket.file(path).createReadStream();
26+
const chunks: Uint8Array[] = [];
27+
28+
stream.once("error", (err) => reject(err));
29+
30+
stream.on("data", (data: Uint8Array) => {
31+
chunks.push(data);
32+
});
33+
34+
stream.on("end", () => {
35+
resolve(Buffer.concat(chunks));
36+
});
37+
});
38+
}
39+
40+
/**
41+
* Move object in bucket.
42+
*/
43+
public move(oldPath: string, newPath: string): Promise<void> {
44+
return new Promise((res, rej): void => this.bucket.file(oldPath).move(newPath, {}, (err) => err ? rej(err) : res()));
45+
}
46+
47+
/**
48+
* Make object publicly accessible via URL.
49+
*/
50+
public makePublic(path: string): Promise<void> {
51+
return new Promise((res, rej): void => this.bucket.file(path).makePublic((err) => err ? rej(err) : res()));
52+
}
53+
54+
/**
55+
* Update bucket object metadata.
56+
*/
57+
public update(path: string, metadata: IMetadata): Promise<void> {
58+
return new Promise(async (res, rej): Promise<void> => {
59+
await this.bucket.file(path).setMetadata(metadata, (err: Error) => err ? rej(err) : res());
60+
});
61+
}
62+
63+
public async write(path: string, data: Buffer, makePublic?: false, metadata?: IMetadata): Promise<void>;
64+
public async write(path: string, data: Buffer, makePublic?: true, metadata?: IMetadata): Promise<string>;
65+
/**
66+
* Write to bucket.
67+
*/
68+
public async write(path: string, data: Buffer, makePublic: true | false = false, metadata?: IMetadata): Promise<void | string> {
69+
return new Promise<void | string>((resolve, reject): void => {
70+
const file = this.bucket.file(path);
71+
const stream = file.createWriteStream();
72+
73+
stream.on("error", (err) => {
74+
reject(err);
75+
});
76+
77+
stream.on("finish", async () => {
78+
if (makePublic) {
79+
try {
80+
await this.makePublic(path);
81+
} catch (ex) {
82+
reject(ex);
83+
84+
return;
85+
}
86+
}
87+
if (metadata) {
88+
try {
89+
await this.update(path, metadata);
90+
} catch (ex) {
91+
reject(ex);
92+
93+
return;
94+
}
95+
}
96+
resolve(makePublic ? `https://storage.googleapis.com/${ScreenshotBucketName}${path}` : undefined);
97+
});
98+
99+
stream.end(data);
100+
});
101+
}
102+
103+
/**
104+
* List files in bucket.
105+
*/
106+
public list(prefix?: string): Promise<File[]> {
107+
return new Promise<File[]>((resolve, reject): void => {
108+
this.bucket.getFiles({
109+
prefix,
110+
}).then((results) => {
111+
resolve(results[0].map((r) => new File(
112+
r.name,
113+
new Date(r.metadata.timeCreated),
114+
new Date(r.metadata.updated),
115+
parseInt(r.metadata.size, 10),
116+
{},
117+
)));
118+
}).catch((e) => {
119+
reject(e);
120+
});
121+
});
122+
}
123+
124+
}

0 commit comments

Comments
 (0)