From 1ad532cdfb0ec2757c88bfa48ad29c6d97fc3dbd Mon Sep 17 00:00:00 2001 From: Luca Casonato Date: Wed, 12 Aug 2020 15:04:36 +0200 Subject: [PATCH 1/3] Dependency analysis --- api/async/publish.ts | 122 ++++++++++++++++++++--- api/async/publish_test.ts | 199 +++++++++++++++++++++++++++++++++++++- run-tests.sh | 2 + template.yaml | 1 + utils/database.ts | 12 ++- utils/deno.ts | 56 +++++++++++ utils/storage.ts | 12 +++ 7 files changed, 383 insertions(+), 21 deletions(-) create mode 100644 utils/deno.ts diff --git a/api/async/publish.ts b/api/async/publish.ts index 76ce8d6..ec37a2d 100644 --- a/api/async/publish.ts +++ b/api/async/publish.ts @@ -10,19 +10,23 @@ */ import { join, walk, SQSEvent, Context } from "../../deps.ts"; -import { Build, Database } from "../../utils/database.ts"; +import { Build, Database, BuildStats } from "../../utils/database.ts"; import { clone } from "../../utils/git.ts"; import { uploadVersionMetaJson, uploadVersionRaw, uploadMetaJson, getMeta, + getVersionMetaJson, } from "../../utils/storage.ts"; import type { DirectoryListingFile } from "../../utils/types.ts"; import { asyncPool } from "../../utils/util.ts"; - +import { runDenoInfo } from "../../utils/deno.ts"; +import type { Dep } from "../../utils/deno.ts"; const database = new Database(Deno.env.get("MONGO_URI")!); +const remoteURL = Deno.env.get("REMOTE_URL")!; + const MAX_FILE_SIZE = 2_500_000; const decoder = new TextDecoder(); @@ -37,10 +41,13 @@ export async function handler( if (build === null) { throw new Error("Build does not exist!"); } + + let stats: BuildStats | undefined = undefined; + switch (build.options.type) { case "github": try { - await publishGithub(build); + stats = await publishGithub(build); } catch (err) { console.log("error", err, err?.response); await database.saveBuild({ @@ -48,17 +55,37 @@ export async function handler( status: "error", message: err.message, }); + return; } break; default: throw new Error(`Unknown build type: ${build.options.type}`); } + + const graph = await analyzeDependencies(build); + await uploadVersionMetaJson( + build.options.moduleName, + build.options.version, + "deps.json", + { graph }, + ); + + await database.saveBuild({ + ...build, + status: "success", + message: `Finished processing module`, + stats, + }); } } async function publishGithub( build: Build, -) { +): Promise<{ + total_files: number; + skipped_due_to_size: string[]; + total_size: number; +}> { console.log( `Publishing ${build.options.moduleName} at ${build.options.ref} from GitHub`, ); @@ -182,18 +209,85 @@ async function publishGithub( }, ); - await database.saveBuild({ - ...build, - status: "success", - message: `Finished uploading`, - stats: { - total_files: directory.filter((f) => f.type === "file").length, - skipped_due_to_size: skippedFiles, - total_size: totalSize, - }, - }); + return { + total_files: directory.filter((f) => f.type === "file").length, + skipped_due_to_size: skippedFiles, + total_size: totalSize, + }; } finally { // Remove checkout await Deno.remove(clonePath, { recursive: true }); } } + +interface DependencyGraph { + nodes: { + [url: string]: { + imports: string[]; + }; + }; +} + +async function analyzeDependencies(build: Build): Promise { + console.log( + `Analyzing dependencies for ${build.options.moduleName}@${build.options.version}`, + ); + await database.saveBuild({ + ...build, + status: "analyzing_dependencies", + }); + + const { options: { moduleName, version } } = build; + const denoDir = await Deno.makeTempDir(); + const prefix = remoteURL.replace("%m", moduleName).replace("%v", version); + + const rawMeta = await getVersionMetaJson(moduleName, version, "meta.json"); + if (!rawMeta) { + throw new Error("Invalid module"); + } + const meta: { directory_listing: DirectoryListingFile[] } = JSON.parse( + decoder.decode(rawMeta), + ); + + const depsTrees = []; + + for await (const file of meta.directory_listing) { + if (file.type !== "file") { + continue; + } + if ( + !(file.path.endsWith(".js") || file.path.endsWith(".jsx") || + file.path.endsWith(".ts") || file.path.endsWith(".tsx")) + ) { + continue; + } + const entrypoint = join( + prefix, + file.path, + ); + depsTrees.push(await runDenoInfo({ entrypoint, denoDir })); + } + + const graph: DependencyGraph = { nodes: {} }; + + for (const dep of depsTrees) { + treeToGraph(graph, dep); + } + + await Deno.remove(denoDir, { recursive: true }); + + return graph; +} + +function treeToGraph(graph: DependencyGraph, dep: Dep) { + const url = dep[0]; + if (!graph.nodes[url]) { + graph.nodes[url] = { imports: [] }; + } + dep[1].forEach((dep) => { + if (!graph.nodes[url].imports.includes(dep[0])) { + graph.nodes[url].imports.push(dep[0]); + } + }); + dep[1].forEach((dep) => treeToGraph(graph, dep)); +} diff --git a/api/async/publish_test.ts b/api/async/publish_test.ts index df2c467..fb53368 100644 --- a/api/async/publish_test.ts +++ b/api/async/publish_test.ts @@ -42,7 +42,7 @@ Deno.test({ version: "0.0.7", }, status: "success", - message: "Finished uploading", + message: "Finished processing module", stats: { skipped_due_to_size: [], total_files: 7, @@ -210,7 +210,7 @@ Deno.test({ subdir: "subproject/", }, status: "success", - message: "Finished uploading", + message: "Finished processing module", stats: { skipped_due_to_size: [], total_files: 2, @@ -288,3 +288,198 @@ Deno.test({ await s3.deleteObject("ltest/versions/0.0.7/raw/README.md"); }, }); + +Deno.test({ + name: "publish success 0.0.9", + async fn() { + const id = await database.createBuild({ + options: { + moduleName: "ltest", + ref: "0.0.9", + repository: "luca-rand/testing", + type: "github", + version: "0.0.9", + }, + status: "queued", + }); + + await handler( + createSQSEvent({ buildID: id }), + createContext(), + ); + + assertEquals({ ...await database.getBuild(id), created_at: undefined }, { + created_at: undefined, + id, + options: { + moduleName: "ltest", + ref: "0.0.9", + repository: "luca-rand/testing", + type: "github", + version: "0.0.9", + }, + status: "success", + message: "Finished processing module", + stats: { + skipped_due_to_size: [], + total_files: 11, + total_size: 2735, + }, + }); + + // Check that versions.json file exists + let versions = await s3.getObject("ltest/meta/versions.json"); + assertEquals(versions?.cacheControl, "max-age=10, must-revalidate"); + assertEquals(versions?.contentType, "application/json"); + assertEquals( + JSON.parse(decoder.decode(versions?.body)), + { latest: "0.0.9", versions: ["0.0.9"] }, + ); + + let meta = await s3.getObject("ltest/versions/0.0.9/meta/meta.json"); + assertEquals(meta?.cacheControl, "public, max-age=31536000, immutable"); + assertEquals(meta?.contentType, "application/json"); + // Check that meta file exists + assertEquals( + { + ...JSON.parse( + decoder.decode( + meta?.body, + ), + ), + uploaded_at: undefined, + }, + { + directory_listing: [ + { + path: "", + size: 2735, + type: "dir", + }, + { + path: "/.github", + size: 716, + type: "dir", + }, + { + path: "/.github/README.md", + size: 304, + type: "file", + }, + { + path: "/.github/workflows", + size: 412, + type: "dir", + }, + { + path: "/.github/workflows/ci.yml", + size: 412, + type: "file", + }, + { + path: "/.vscode", + size: 26, + type: "dir", + }, + { + path: "/.vscode/settings.json", + size: 26, + type: "file", + }, + { + path: "/LICENSE", + size: 1066, + type: "file", + }, + { + path: "/deps.ts", + size: 63, + type: "file", + }, + { + path: "/example.ts", + size: 50, + type: "file", + }, + { + path: "/fixtures", + size: 23, + type: "dir", + }, + { + path: "/fixtures/%", + size: 23, + type: "file", + }, + { + path: "/mod.ts", + size: 139, + type: "file", + }, + { + path: "/mod_test.ts", + size: 227, + type: "file", + }, + { + path: "/subproject", + size: 425, + type: "dir", + }, + { + path: "/subproject/README.md", + size: 354, + type: "file", + }, + { + path: "/subproject/mod.ts", + size: 71, + type: "file", + }, + ], + upload_options: { + ref: "0.0.9", + repository: "luca-rand/testing", + type: "github", + }, + uploaded_at: undefined, + }, + ); + + // Check the yml file was uploaded + let yml = await s3.getObject( + "ltest/versions/0.0.9/raw/.github/workflows/ci.yml", + ); + assertEquals(yml?.cacheControl, "public, max-age=31536000, immutable"); + assertEquals(yml?.contentType, "text/yaml"); + assertEquals(yml?.body.length, 412); + + // Check the ts file was uploaded + let ts = await s3.getObject("ltest/versions/0.0.9/raw/mod.ts"); + assertEquals(ts?.cacheControl, "public, max-age=31536000, immutable"); + assertEquals(ts?.contentType, "application/typescript; charset=utf-8"); + assertEquals(ts?.body.length, 139); + + // Check the ts file was uploaded + let readme = await s3.getObject( + "ltest/versions/0.0.9/raw/.github/README.md", + ); + assertEquals(readme?.cacheControl, "public, max-age=31536000, immutable"); + assertEquals(readme?.contentType, "text/markdown"); + assertEquals(readme?.body.length, 304); + + // Cleanup + await database._builds.deleteMany({}); + await s3.deleteObject("ltest/meta/versions.json"); + await s3.deleteObject("ltest/versions/0.0.9/meta/meta.json"); + await s3.deleteObject("ltest/versions/0.0.9/raw/.github/workflows/ci.yml"); + await s3.deleteObject("ltest/versions/0.0.9/raw/.vscode/settings.json"); + await s3.deleteObject("ltest/versions/0.0.9/raw/LICENCE"); + await s3.deleteObject("ltest/versions/0.0.9/raw/deps.ts"); + await s3.deleteObject("ltest/versions/0.0.9/raw/fixtures/%"); + await s3.deleteObject("ltest/versions/0.0.9/raw/mod.ts"); + await s3.deleteObject("ltest/versions/0.0.9/raw/mod_test.md"); + await s3.deleteObject("ltest/versions/0.0.9/raw/subproject/README.md"); + await s3.deleteObject("ltest/versions/0.0.9/raw/subproject/mod.ts"); + }, +}); diff --git a/run-tests.sh b/run-tests.sh index c2d04fa..b46654d 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -7,6 +7,7 @@ sleep 10 aws --endpoint-url=http://s3:9000 s3 rm --recursive s3://deno-registry2 || true aws --endpoint-url=http://s3:9000 s3 rb s3://deno-registry2 || true aws --endpoint-url=http://s3:9000 s3 mb s3://deno-registry2 +aws --endpoint-url=http://s3:9000 s3api put-bucket-policy --bucket deno-registry2 --policy '{ "Version":"2012-10-17", "Statement":[ { "Sid":"PublicRead", "Effect":"Allow", "Principal": "*", "Action":["s3:GetObject","s3:GetObjectVersion"], "Resource":["arn:aws:s3:::deno-registry2/*"] } ] }' # Set up SQS aws --endpoint-url=http://sqs:9324 sqs delete-queue --queue-url http://sqs:9324/000000000000/builds --region us-east-1|| true @@ -17,5 +18,6 @@ export MONGO_URI=mongodb://root:rootpassword@mongo export BUILD_QUEUE=http://sqs:9324/000000000000/builds export STORAGE_BUCKET=deno-registry2 export S3_ENDPOINT_URL=http://s3:9000 +export REMOTE_URL=http://s3:9000/deno-registry2/%m/versions/%v/raw deno test --unstable -A \ No newline at end of file diff --git a/template.yaml b/template.yaml index 3832e0b..ae4a3f3 100644 --- a/template.yaml +++ b/template.yaml @@ -71,6 +71,7 @@ Resources: HANDLER_EXT: js MONGO_URI: "{{resolve:secretsmanager:mongodb/atlas/deno_registry2:SecretString:MongoURI}}" STORAGE_BUCKET: !Ref StorageBucket + REMOTE_URL: https://deno.land/x/%m@%v Timeout: 300 Events: BuildQueueTrigger: diff --git a/utils/database.ts b/utils/database.ts index a7737db..e85b2ca 100644 --- a/utils/database.ts +++ b/utils/database.ts @@ -32,11 +32,13 @@ export interface Build { }; status: string; message?: string; - stats?: { - total_files: number; - total_size: number; - skipped_due_to_size: string[]; - }; + stats?: BuildStats; +} + +export interface BuildStats { + total_files: number; + total_size: number; + skipped_due_to_size: string[]; } export class Database { diff --git a/utils/deno.ts b/utils/deno.ts new file mode 100644 index 0000000..099bb68 --- /dev/null +++ b/utils/deno.ts @@ -0,0 +1,56 @@ +const decoder = new TextDecoder(); + +export type Dep = [string, Dep[]]; + +export async function cacheDependencies( + options: { entrypoint: string; denoDir: string; importmap: string }, +): Promise { + const p = Deno.run({ + cmd: [ + "deno", + "cache", + "--importmap", + options.importmap, + options.entrypoint, + ], + env: { + "DENO_DIR": options.denoDir, + }, + stdout: "inherit", + stderr: "inherit", + }); + const status = await p.status(); + p.close(); + if (!status.success) { + throw new Error(`Failed to gather dependencies for ${options.entrypoint}`); + } + return; +} + +export async function runDenoInfo( + options: { entrypoint: string; denoDir: string }, +): Promise { + const p = Deno.run({ + cmd: [ + "deno", + "info", + "--json", + "--unstable", + options.entrypoint, + ], + env: { + "DENO_DIR": options.denoDir, + }, + stdout: "piped", + stderr: "inherit", + }); + const status = await p.status(); + const file = await p.output(); + p.close(); + if (!status.success) { + throw new Error(`Failed to run deno info for ${options.entrypoint}`); + } + const text = decoder.decode(file); + const { deps } = JSON.parse(text); + return deps; +} diff --git a/utils/storage.ts b/utils/storage.ts index 60579ed..b715c2c 100644 --- a/utils/storage.ts +++ b/utils/storage.ts @@ -24,6 +24,18 @@ export async function getMeta( return resp?.body; } +export async function getVersionMetaJson( + module: string, + version: string, + file: string, +): Promise { + const resp = await s3.getObject( + join(module, "versions", version, "meta", file), + {}, + ); + return resp?.body; +} + const encoder = new TextEncoder(); export async function uploadMetaJson( From 89214b6e60b79bbeb44dfa03e9c34b0879f099a9 Mon Sep 17 00:00:00 2001 From: Luca Casonato Date: Wed, 12 Aug 2020 15:23:18 +0200 Subject: [PATCH 2/3] Fixed tests --- api/async/publish.ts | 24 +- api/async/publish_test.ts | 390 ++++++++++++++------------------ api/webhook/github_push_test.ts | 2 - utils/deno.ts | 26 +-- 4 files changed, 182 insertions(+), 260 deletions(-) diff --git a/api/async/publish.ts b/api/async/publish.ts index ec37a2d..1736b5f 100644 --- a/api/async/publish.ts +++ b/api/async/publish.ts @@ -62,18 +62,17 @@ export async function handler( throw new Error(`Unknown build type: ${build.options.type}`); } - const graph = await analyzeDependencies(build); - await uploadVersionMetaJson( - build.options.moduleName, - build.options.version, - "deps.json", - { graph }, - ); + let message = "Published module."; + + await analyzeDependencies(build).catch((err) => { + console.error("failed dependency analysis", build, err, err?.response); + message += " Failed to run dependency analysis."; + }); await database.saveBuild({ ...build, status: "success", - message: `Finished processing module`, + message: message, stats, }); } @@ -228,7 +227,7 @@ interface DependencyGraph { }; } -async function analyzeDependencies(build: Build): Promise { +async function analyzeDependencies(build: Build): Promise { console.log( `Analyzing dependencies for ${build.options.moduleName}@${build.options.version}`, ); @@ -276,7 +275,12 @@ async function analyzeDependencies(build: Build): Promise { await Deno.remove(denoDir, { recursive: true }); - return graph; + await uploadVersionMetaJson( + build.options.moduleName, + build.options.version, + "deps.json", + { graph }, + ); } function treeToGraph(graph: DependencyGraph, dep: Dep) { diff --git a/api/async/publish_test.ts b/api/async/publish_test.ts index fb53368..80e785d 100644 --- a/api/async/publish_test.ts +++ b/api/async/publish_test.ts @@ -3,10 +3,9 @@ import { createContext, createSQSEvent, } from "../../utils/test_utils.ts"; -import { join } from "../../deps.ts"; import { assertEquals } from "../../test_deps.ts"; import { Database } from "../../utils/database.ts"; -import { s3, getMeta } from "../../utils/storage.ts"; +import { s3 } from "../../utils/storage.ts"; const database = new Database(Deno.env.get("MONGO_URI")!); @@ -18,10 +17,10 @@ Deno.test({ const id = await database.createBuild({ options: { moduleName: "ltest", - ref: "0.0.7", + ref: "0.0.9", repository: "luca-rand/testing", type: "github", - version: "0.0.7", + version: "0.0.9", }, status: "queued", }); @@ -36,17 +35,17 @@ Deno.test({ id, options: { moduleName: "ltest", - ref: "0.0.7", + ref: "0.0.9", repository: "luca-rand/testing", type: "github", - version: "0.0.7", + version: "0.0.9", }, status: "success", message: "Finished processing module", stats: { skipped_due_to_size: [], - total_files: 7, - total_size: 2317, + total_files: 11, + total_size: 2735, }, }); @@ -56,10 +55,10 @@ Deno.test({ assertEquals(versions?.contentType, "application/json"); assertEquals( JSON.parse(decoder.decode(versions?.body)), - { latest: "0.0.7", versions: ["0.0.7"] }, + { latest: "0.0.9", versions: ["0.0.9"] }, ); - let meta = await s3.getObject("ltest/versions/0.0.7/meta/meta.json"); + let meta = await s3.getObject("ltest/versions/0.0.9/meta/meta.json"); assertEquals(meta?.cacheControl, "public, max-age=31536000, immutable"); assertEquals(meta?.contentType, "application/json"); // Check that meta file exists @@ -76,14 +75,19 @@ Deno.test({ directory_listing: [ { path: "", - size: 2317, + size: 2735, type: "dir", }, { path: "/.github", - size: 412, + size: 716, type: "dir", }, + { + path: "/.github/README.md", + size: 304, + type: "file", + }, { path: "/.github/workflows", size: 412, @@ -94,14 +98,29 @@ Deno.test({ size: 412, type: "file", }, + { + path: "/.vscode", + size: 26, + type: "dir", + }, + { + path: "/.vscode/settings.json", + size: 26, + type: "file", + }, { path: "/LICENSE", size: 1066, type: "file", }, { - path: "/README.md", - size: 304, + path: "/deps.ts", + size: 63, + type: "file", + }, + { + path: "/example.ts", + size: 50, type: "file", }, { @@ -116,7 +135,12 @@ Deno.test({ }, { path: "/mod.ts", - size: 87, + size: 139, + type: "file", + }, + { + path: "/mod_test.ts", + size: 227, type: "file", }, { @@ -136,7 +160,7 @@ Deno.test({ }, ], upload_options: { - ref: "0.0.7", + ref: "0.0.9", repository: "luca-rand/testing", type: "github", }, @@ -144,22 +168,134 @@ Deno.test({ }, ); + let deps = await s3.getObject("ltest/versions/0.0.9/meta/deps.json"); + assertEquals(deps?.cacheControl, "public, max-age=31536000, immutable"); + assertEquals(deps?.contentType, "application/json"); + // Check that meta file exists + assertEquals( + JSON.parse( + decoder.decode( + deps?.body, + ), + ), + { + graph: { + nodes: { + "http://s3:9000/deno-registry2/ltest/versions/0.0.9/raw/deps.ts": { + "imports": [ + "https://deno.land/std@0.64.0/uuid/mod.ts", + ], + }, + "https://deno.land/std@0.64.0/uuid/mod.ts": { + "imports": [ + "https://deno.land/std@0.64.0/uuid/v1.ts", + "https://deno.land/std@0.64.0/uuid/v4.ts", + "https://deno.land/std@0.64.0/uuid/v5.ts", + ], + }, + "https://deno.land/std@0.64.0/uuid/v1.ts": { + "imports": [ + "https://deno.land/std@0.64.0/uuid/_common.ts", + ], + }, + "https://deno.land/std@0.64.0/uuid/_common.ts": { + "imports": [], + }, + "https://deno.land/std@0.64.0/uuid/v4.ts": { + "imports": [ + "https://deno.land/std@0.64.0/uuid/_common.ts", + ], + }, + "https://deno.land/std@0.64.0/uuid/v5.ts": { + "imports": [ + "https://deno.land/std@0.64.0/uuid/_common.ts", + "https://deno.land/std@0.64.0/hash/sha1.ts", + "https://deno.land/std@0.64.0/node/util.ts", + "https://deno.land/std@0.64.0/_util/assert.ts", + ], + }, + "https://deno.land/std@0.64.0/hash/sha1.ts": { + "imports": [], + }, + "https://deno.land/std@0.64.0/node/util.ts": { + "imports": [ + "https://deno.land/std@0.64.0/node/_util/_util_promisify.ts", + "https://deno.land/std@0.64.0/node/_util/_util_callbackify.ts", + "https://deno.land/std@0.64.0/node/_util/_util_types.ts", + "https://deno.land/std@0.64.0/node/_utils.ts", + ], + }, + "https://deno.land/std@0.64.0/node/_util/_util_promisify.ts": { + "imports": [], + }, + "https://deno.land/std@0.64.0/node/_util/_util_callbackify.ts": { + "imports": [], + }, + "https://deno.land/std@0.64.0/node/_util/_util_types.ts": { + "imports": [], + }, + "https://deno.land/std@0.64.0/node/_utils.ts": { + "imports": [], + }, + "https://deno.land/std@0.64.0/_util/assert.ts": { + "imports": [], + }, + "http://s3:9000/deno-registry2/ltest/versions/0.0.9/raw/example.ts": + { + "imports": [ + "http://s3:9000/deno-registry2/ltest/versions/0.0.9/raw/mod.ts", + ], + }, + "http://s3:9000/deno-registry2/ltest/versions/0.0.9/raw/mod.ts": { + "imports": [ + "http://s3:9000/deno-registry2/ltest/versions/0.0.9/raw/deps.ts", + ], + }, + "http://s3:9000/deno-registry2/ltest/versions/0.0.9/raw/mod_test.ts": + { + "imports": [ + "https://deno.land/std@0.64.0/testing/asserts.ts", + ], + }, + "https://deno.land/std@0.64.0/testing/asserts.ts": { + "imports": [ + "https://deno.land/std@0.64.0/fmt/colors.ts", + "https://deno.land/std@0.64.0/testing/diff.ts", + ], + }, + "https://deno.land/std@0.64.0/fmt/colors.ts": { + "imports": [], + }, + "https://deno.land/std@0.64.0/testing/diff.ts": { + "imports": [], + }, + "http://s3:9000/deno-registry2/ltest/versions/0.0.9/raw/subproject/mod.ts": + { + "imports": [], + }, + }, + }, + }, + ); + // Check the yml file was uploaded let yml = await s3.getObject( - "ltest/versions/0.0.7/raw/.github/workflows/ci.yml", + "ltest/versions/0.0.9/raw/.github/workflows/ci.yml", ); assertEquals(yml?.cacheControl, "public, max-age=31536000, immutable"); assertEquals(yml?.contentType, "text/yaml"); assertEquals(yml?.body.length, 412); // Check the ts file was uploaded - let ts = await s3.getObject("ltest/versions/0.0.7/raw/mod.ts"); + let ts = await s3.getObject("ltest/versions/0.0.9/raw/mod.ts"); assertEquals(ts?.cacheControl, "public, max-age=31536000, immutable"); assertEquals(ts?.contentType, "application/typescript; charset=utf-8"); - assertEquals(ts?.body.length, 87); + assertEquals(ts?.body.length, 139); // Check the ts file was uploaded - let readme = await s3.getObject("ltest/versions/0.0.7/raw/README.md"); + let readme = await s3.getObject( + "ltest/versions/0.0.9/raw/.github/README.md", + ); assertEquals(readme?.cacheControl, "public, max-age=31536000, immutable"); assertEquals(readme?.contentType, "text/markdown"); assertEquals(readme?.body.length, 304); @@ -167,14 +303,17 @@ Deno.test({ // Cleanup await database._builds.deleteMany({}); await s3.deleteObject("ltest/meta/versions.json"); - await s3.deleteObject("ltest/versions/0.0.7/meta/meta.json"); - await s3.deleteObject("ltest/versions/0.0.7/raw/.github/workflows/ci.yml"); - await s3.deleteObject("ltest/versions/0.0.7/raw/LICENCE"); - await s3.deleteObject("ltest/versions/0.0.7/raw/README.md"); - await s3.deleteObject("ltest/versions/0.0.7/raw/fixtures/%"); - await s3.deleteObject("ltest/versions/0.0.7/raw/mod.ts"); - await s3.deleteObject("ltest/versions/0.0.7/raw/subproject/README.md"); - await s3.deleteObject("ltest/versions/0.0.7/raw/subproject/mod.ts"); + await s3.deleteObject("ltest/versions/0.0.9/meta/meta.json"); + await s3.deleteObject("ltest/versions/0.0.9/meta/deps.json"); + await s3.deleteObject("ltest/versions/0.0.9/raw/.github/workflows/ci.yml"); + await s3.deleteObject("ltest/versions/0.0.9/raw/.vscode/settings.json"); + await s3.deleteObject("ltest/versions/0.0.9/raw/LICENCE"); + await s3.deleteObject("ltest/versions/0.0.9/raw/deps.ts"); + await s3.deleteObject("ltest/versions/0.0.9/raw/fixtures/%"); + await s3.deleteObject("ltest/versions/0.0.9/raw/mod.ts"); + await s3.deleteObject("ltest/versions/0.0.9/raw/mod_test.md"); + await s3.deleteObject("ltest/versions/0.0.9/raw/subproject/README.md"); + await s3.deleteObject("ltest/versions/0.0.9/raw/subproject/mod.ts"); }, }); @@ -288,198 +427,3 @@ Deno.test({ await s3.deleteObject("ltest/versions/0.0.7/raw/README.md"); }, }); - -Deno.test({ - name: "publish success 0.0.9", - async fn() { - const id = await database.createBuild({ - options: { - moduleName: "ltest", - ref: "0.0.9", - repository: "luca-rand/testing", - type: "github", - version: "0.0.9", - }, - status: "queued", - }); - - await handler( - createSQSEvent({ buildID: id }), - createContext(), - ); - - assertEquals({ ...await database.getBuild(id), created_at: undefined }, { - created_at: undefined, - id, - options: { - moduleName: "ltest", - ref: "0.0.9", - repository: "luca-rand/testing", - type: "github", - version: "0.0.9", - }, - status: "success", - message: "Finished processing module", - stats: { - skipped_due_to_size: [], - total_files: 11, - total_size: 2735, - }, - }); - - // Check that versions.json file exists - let versions = await s3.getObject("ltest/meta/versions.json"); - assertEquals(versions?.cacheControl, "max-age=10, must-revalidate"); - assertEquals(versions?.contentType, "application/json"); - assertEquals( - JSON.parse(decoder.decode(versions?.body)), - { latest: "0.0.9", versions: ["0.0.9"] }, - ); - - let meta = await s3.getObject("ltest/versions/0.0.9/meta/meta.json"); - assertEquals(meta?.cacheControl, "public, max-age=31536000, immutable"); - assertEquals(meta?.contentType, "application/json"); - // Check that meta file exists - assertEquals( - { - ...JSON.parse( - decoder.decode( - meta?.body, - ), - ), - uploaded_at: undefined, - }, - { - directory_listing: [ - { - path: "", - size: 2735, - type: "dir", - }, - { - path: "/.github", - size: 716, - type: "dir", - }, - { - path: "/.github/README.md", - size: 304, - type: "file", - }, - { - path: "/.github/workflows", - size: 412, - type: "dir", - }, - { - path: "/.github/workflows/ci.yml", - size: 412, - type: "file", - }, - { - path: "/.vscode", - size: 26, - type: "dir", - }, - { - path: "/.vscode/settings.json", - size: 26, - type: "file", - }, - { - path: "/LICENSE", - size: 1066, - type: "file", - }, - { - path: "/deps.ts", - size: 63, - type: "file", - }, - { - path: "/example.ts", - size: 50, - type: "file", - }, - { - path: "/fixtures", - size: 23, - type: "dir", - }, - { - path: "/fixtures/%", - size: 23, - type: "file", - }, - { - path: "/mod.ts", - size: 139, - type: "file", - }, - { - path: "/mod_test.ts", - size: 227, - type: "file", - }, - { - path: "/subproject", - size: 425, - type: "dir", - }, - { - path: "/subproject/README.md", - size: 354, - type: "file", - }, - { - path: "/subproject/mod.ts", - size: 71, - type: "file", - }, - ], - upload_options: { - ref: "0.0.9", - repository: "luca-rand/testing", - type: "github", - }, - uploaded_at: undefined, - }, - ); - - // Check the yml file was uploaded - let yml = await s3.getObject( - "ltest/versions/0.0.9/raw/.github/workflows/ci.yml", - ); - assertEquals(yml?.cacheControl, "public, max-age=31536000, immutable"); - assertEquals(yml?.contentType, "text/yaml"); - assertEquals(yml?.body.length, 412); - - // Check the ts file was uploaded - let ts = await s3.getObject("ltest/versions/0.0.9/raw/mod.ts"); - assertEquals(ts?.cacheControl, "public, max-age=31536000, immutable"); - assertEquals(ts?.contentType, "application/typescript; charset=utf-8"); - assertEquals(ts?.body.length, 139); - - // Check the ts file was uploaded - let readme = await s3.getObject( - "ltest/versions/0.0.9/raw/.github/README.md", - ); - assertEquals(readme?.cacheControl, "public, max-age=31536000, immutable"); - assertEquals(readme?.contentType, "text/markdown"); - assertEquals(readme?.body.length, 304); - - // Cleanup - await database._builds.deleteMany({}); - await s3.deleteObject("ltest/meta/versions.json"); - await s3.deleteObject("ltest/versions/0.0.9/meta/meta.json"); - await s3.deleteObject("ltest/versions/0.0.9/raw/.github/workflows/ci.yml"); - await s3.deleteObject("ltest/versions/0.0.9/raw/.vscode/settings.json"); - await s3.deleteObject("ltest/versions/0.0.9/raw/LICENCE"); - await s3.deleteObject("ltest/versions/0.0.9/raw/deps.ts"); - await s3.deleteObject("ltest/versions/0.0.9/raw/fixtures/%"); - await s3.deleteObject("ltest/versions/0.0.9/raw/mod.ts"); - await s3.deleteObject("ltest/versions/0.0.9/raw/mod_test.md"); - await s3.deleteObject("ltest/versions/0.0.9/raw/subproject/README.md"); - await s3.deleteObject("ltest/versions/0.0.9/raw/subproject/mod.ts"); - }, -}); diff --git a/api/webhook/github_push_test.ts b/api/webhook/github_push_test.ts index 9866412..08f21f0 100644 --- a/api/webhook/github_push_test.ts +++ b/api/webhook/github_push_test.ts @@ -153,8 +153,6 @@ Deno.test({ createContext(), ); - console.log(resp); - const builds = await database._builds.find({}); // Check that a new build was queued diff --git a/utils/deno.ts b/utils/deno.ts index 099bb68..227f177 100644 --- a/utils/deno.ts +++ b/utils/deno.ts @@ -2,31 +2,6 @@ const decoder = new TextDecoder(); export type Dep = [string, Dep[]]; -export async function cacheDependencies( - options: { entrypoint: string; denoDir: string; importmap: string }, -): Promise { - const p = Deno.run({ - cmd: [ - "deno", - "cache", - "--importmap", - options.importmap, - options.entrypoint, - ], - env: { - "DENO_DIR": options.denoDir, - }, - stdout: "inherit", - stderr: "inherit", - }); - const status = await p.status(); - p.close(); - if (!status.success) { - throw new Error(`Failed to gather dependencies for ${options.entrypoint}`); - } - return; -} - export async function runDenoInfo( options: { entrypoint: string; denoDir: string }, ): Promise { @@ -36,6 +11,7 @@ export async function runDenoInfo( "info", "--json", "--unstable", + "--no-check", options.entrypoint, ], env: { From f5d924b1a418a8ae2e7957dd1e097870c21da6e7 Mon Sep 17 00:00:00 2001 From: Luca Casonato Date: Wed, 12 Aug 2020 17:55:28 +0200 Subject: [PATCH 3/3] fixed tests --- api/async/publish_test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/async/publish_test.ts b/api/async/publish_test.ts index 80e785d..47ab55c 100644 --- a/api/async/publish_test.ts +++ b/api/async/publish_test.ts @@ -41,7 +41,7 @@ Deno.test({ version: "0.0.9", }, status: "success", - message: "Finished processing module", + message: "Published module.", stats: { skipped_due_to_size: [], total_files: 11, @@ -349,7 +349,7 @@ Deno.test({ subdir: "subproject/", }, status: "success", - message: "Finished processing module", + message: "Published module.", stats: { skipped_due_to_size: [], total_files: 2,