diff --git a/api/async/publish.ts b/api/async/publish.ts index 76ce8d6..1736b5f 100644 --- a/api/async/publish.ts +++ b/api/async/publish.ts @@ -10,19 +10,23 @@ */ import { join, walk, SQSEvent, Context } from "../../deps.ts"; -import { Build, Database } from "../../utils/database.ts"; +import { Build, Database, BuildStats } from "../../utils/database.ts"; import { clone } from "../../utils/git.ts"; import { uploadVersionMetaJson, uploadVersionRaw, uploadMetaJson, getMeta, + getVersionMetaJson, } from "../../utils/storage.ts"; import type { DirectoryListingFile } from "../../utils/types.ts"; import { asyncPool } from "../../utils/util.ts"; - +import { runDenoInfo } from "../../utils/deno.ts"; +import type { Dep } from "../../utils/deno.ts"; const database = new Database(Deno.env.get("MONGO_URI")!); +const remoteURL = Deno.env.get("REMOTE_URL")!; + const MAX_FILE_SIZE = 2_500_000; const decoder = new TextDecoder(); @@ -37,10 +41,13 @@ export async function handler( if (build === null) { throw new Error("Build does not exist!"); } + + let stats: BuildStats | undefined = undefined; + switch (build.options.type) { case "github": try { - await publishGithub(build); + stats = await publishGithub(build); } catch (err) { console.log("error", err, err?.response); await database.saveBuild({ @@ -48,17 +55,36 @@ export async function handler( status: "error", message: err.message, }); + return; } break; default: throw new Error(`Unknown build type: ${build.options.type}`); } + + let message = "Published module."; + + await analyzeDependencies(build).catch((err) => { + console.error("failed dependency analysis", build, err, err?.response); + message += " Failed to run dependency analysis."; + }); + + await database.saveBuild({ + ...build, + status: "success", + message: message, + stats, + }); } } async function publishGithub( build: Build, -) { +): Promise<{ + total_files: number; + skipped_due_to_size: string[]; + total_size: number; +}> { console.log( `Publishing ${build.options.moduleName} at ${build.options.ref} from GitHub`, ); @@ -182,18 +208,90 @@ async function publishGithub( }, ); - await database.saveBuild({ - ...build, - status: "success", - message: `Finished uploading`, - stats: { - total_files: directory.filter((f) => f.type === "file").length, - skipped_due_to_size: skippedFiles, - total_size: totalSize, - }, - }); + return { + total_files: directory.filter((f) => f.type === "file").length, + skipped_due_to_size: skippedFiles, + total_size: totalSize, + }; } finally { // Remove checkout await Deno.remove(clonePath, { recursive: true }); } } + +interface DependencyGraph { + nodes: { + [url: string]: { + imports: string[]; + }; + }; +} + +async function analyzeDependencies(build: Build): Promise { + console.log( + `Analyzing dependencies for ${build.options.moduleName}@${build.options.version}`, + ); + await database.saveBuild({ + ...build, + status: "analyzing_dependencies", + }); + + const { options: { moduleName, version } } = build; + const denoDir = await Deno.makeTempDir(); + const prefix = remoteURL.replace("%m", moduleName).replace("%v", version); + + const rawMeta = await getVersionMetaJson(moduleName, version, "meta.json"); + if (!rawMeta) { + throw new Error("Invalid module"); + } + const meta: { directory_listing: DirectoryListingFile[] } = JSON.parse( + decoder.decode(rawMeta), + ); + + const depsTrees = []; + + for await (const file of meta.directory_listing) { + if (file.type !== "file") { + continue; + } + if ( + !(file.path.endsWith(".js") || file.path.endsWith(".jsx") || + file.path.endsWith(".ts") || file.path.endsWith(".tsx")) + ) { + continue; + } + const entrypoint = join( + prefix, + file.path, + ); + depsTrees.push(await runDenoInfo({ entrypoint, denoDir })); + } + + const graph: DependencyGraph = { nodes: {} }; + + for (const dep of depsTrees) { + treeToGraph(graph, dep); + } + + await Deno.remove(denoDir, { recursive: true }); + + await uploadVersionMetaJson( + build.options.moduleName, + build.options.version, + "deps.json", + { graph }, + ); +} + +function treeToGraph(graph: DependencyGraph, dep: Dep) { + const url = dep[0]; + if (!graph.nodes[url]) { + graph.nodes[url] = { imports: [] }; + } + dep[1].forEach((dep) => { + if (!graph.nodes[url].imports.includes(dep[0])) { + graph.nodes[url].imports.push(dep[0]); + } + }); + dep[1].forEach((dep) => treeToGraph(graph, dep)); +} diff --git a/api/async/publish_test.ts b/api/async/publish_test.ts index df2c467..47ab55c 100644 --- a/api/async/publish_test.ts +++ b/api/async/publish_test.ts @@ -3,10 +3,9 @@ import { createContext, createSQSEvent, } from "../../utils/test_utils.ts"; -import { join } from "../../deps.ts"; import { assertEquals } from "../../test_deps.ts"; import { Database } from "../../utils/database.ts"; -import { s3, getMeta } from "../../utils/storage.ts"; +import { s3 } from "../../utils/storage.ts"; const database = new Database(Deno.env.get("MONGO_URI")!); @@ -18,10 +17,10 @@ Deno.test({ const id = await database.createBuild({ options: { moduleName: "ltest", - ref: "0.0.7", + ref: "0.0.9", repository: "luca-rand/testing", type: "github", - version: "0.0.7", + version: "0.0.9", }, status: "queued", }); @@ -36,17 +35,17 @@ Deno.test({ id, options: { moduleName: "ltest", - ref: "0.0.7", + ref: "0.0.9", repository: "luca-rand/testing", type: "github", - version: "0.0.7", + version: "0.0.9", }, status: "success", - message: "Finished uploading", + message: "Published module.", stats: { skipped_due_to_size: [], - total_files: 7, - total_size: 2317, + total_files: 11, + total_size: 2735, }, }); @@ -56,10 +55,10 @@ Deno.test({ assertEquals(versions?.contentType, "application/json"); assertEquals( JSON.parse(decoder.decode(versions?.body)), - { latest: "0.0.7", versions: ["0.0.7"] }, + { latest: "0.0.9", versions: ["0.0.9"] }, ); - let meta = await s3.getObject("ltest/versions/0.0.7/meta/meta.json"); + let meta = await s3.getObject("ltest/versions/0.0.9/meta/meta.json"); assertEquals(meta?.cacheControl, "public, max-age=31536000, immutable"); assertEquals(meta?.contentType, "application/json"); // Check that meta file exists @@ -76,14 +75,19 @@ Deno.test({ directory_listing: [ { path: "", - size: 2317, + size: 2735, type: "dir", }, { path: "/.github", - size: 412, + size: 716, type: "dir", }, + { + path: "/.github/README.md", + size: 304, + type: "file", + }, { path: "/.github/workflows", size: 412, @@ -94,14 +98,29 @@ Deno.test({ size: 412, type: "file", }, + { + path: "/.vscode", + size: 26, + type: "dir", + }, + { + path: "/.vscode/settings.json", + size: 26, + type: "file", + }, { path: "/LICENSE", size: 1066, type: "file", }, { - path: "/README.md", - size: 304, + path: "/deps.ts", + size: 63, + type: "file", + }, + { + path: "/example.ts", + size: 50, type: "file", }, { @@ -116,7 +135,12 @@ Deno.test({ }, { path: "/mod.ts", - size: 87, + size: 139, + type: "file", + }, + { + path: "/mod_test.ts", + size: 227, type: "file", }, { @@ -136,7 +160,7 @@ Deno.test({ }, ], upload_options: { - ref: "0.0.7", + ref: "0.0.9", repository: "luca-rand/testing", type: "github", }, @@ -144,22 +168,134 @@ Deno.test({ }, ); + let deps = await s3.getObject("ltest/versions/0.0.9/meta/deps.json"); + assertEquals(deps?.cacheControl, "public, max-age=31536000, immutable"); + assertEquals(deps?.contentType, "application/json"); + // Check that meta file exists + assertEquals( + JSON.parse( + decoder.decode( + deps?.body, + ), + ), + { + graph: { + nodes: { + "http://s3:9000/deno-registry2/ltest/versions/0.0.9/raw/deps.ts": { + "imports": [ + "https://deno.land/std@0.64.0/uuid/mod.ts", + ], + }, + "https://deno.land/std@0.64.0/uuid/mod.ts": { + "imports": [ + "https://deno.land/std@0.64.0/uuid/v1.ts", + "https://deno.land/std@0.64.0/uuid/v4.ts", + "https://deno.land/std@0.64.0/uuid/v5.ts", + ], + }, + "https://deno.land/std@0.64.0/uuid/v1.ts": { + "imports": [ + "https://deno.land/std@0.64.0/uuid/_common.ts", + ], + }, + "https://deno.land/std@0.64.0/uuid/_common.ts": { + "imports": [], + }, + "https://deno.land/std@0.64.0/uuid/v4.ts": { + "imports": [ + "https://deno.land/std@0.64.0/uuid/_common.ts", + ], + }, + "https://deno.land/std@0.64.0/uuid/v5.ts": { + "imports": [ + "https://deno.land/std@0.64.0/uuid/_common.ts", + "https://deno.land/std@0.64.0/hash/sha1.ts", + "https://deno.land/std@0.64.0/node/util.ts", + "https://deno.land/std@0.64.0/_util/assert.ts", + ], + }, + "https://deno.land/std@0.64.0/hash/sha1.ts": { + "imports": [], + }, + "https://deno.land/std@0.64.0/node/util.ts": { + "imports": [ + "https://deno.land/std@0.64.0/node/_util/_util_promisify.ts", + "https://deno.land/std@0.64.0/node/_util/_util_callbackify.ts", + "https://deno.land/std@0.64.0/node/_util/_util_types.ts", + "https://deno.land/std@0.64.0/node/_utils.ts", + ], + }, + "https://deno.land/std@0.64.0/node/_util/_util_promisify.ts": { + "imports": [], + }, + "https://deno.land/std@0.64.0/node/_util/_util_callbackify.ts": { + "imports": [], + }, + "https://deno.land/std@0.64.0/node/_util/_util_types.ts": { + "imports": [], + }, + "https://deno.land/std@0.64.0/node/_utils.ts": { + "imports": [], + }, + "https://deno.land/std@0.64.0/_util/assert.ts": { + "imports": [], + }, + "http://s3:9000/deno-registry2/ltest/versions/0.0.9/raw/example.ts": + { + "imports": [ + "http://s3:9000/deno-registry2/ltest/versions/0.0.9/raw/mod.ts", + ], + }, + "http://s3:9000/deno-registry2/ltest/versions/0.0.9/raw/mod.ts": { + "imports": [ + "http://s3:9000/deno-registry2/ltest/versions/0.0.9/raw/deps.ts", + ], + }, + "http://s3:9000/deno-registry2/ltest/versions/0.0.9/raw/mod_test.ts": + { + "imports": [ + "https://deno.land/std@0.64.0/testing/asserts.ts", + ], + }, + "https://deno.land/std@0.64.0/testing/asserts.ts": { + "imports": [ + "https://deno.land/std@0.64.0/fmt/colors.ts", + "https://deno.land/std@0.64.0/testing/diff.ts", + ], + }, + "https://deno.land/std@0.64.0/fmt/colors.ts": { + "imports": [], + }, + "https://deno.land/std@0.64.0/testing/diff.ts": { + "imports": [], + }, + "http://s3:9000/deno-registry2/ltest/versions/0.0.9/raw/subproject/mod.ts": + { + "imports": [], + }, + }, + }, + }, + ); + // Check the yml file was uploaded let yml = await s3.getObject( - "ltest/versions/0.0.7/raw/.github/workflows/ci.yml", + "ltest/versions/0.0.9/raw/.github/workflows/ci.yml", ); assertEquals(yml?.cacheControl, "public, max-age=31536000, immutable"); assertEquals(yml?.contentType, "text/yaml"); assertEquals(yml?.body.length, 412); // Check the ts file was uploaded - let ts = await s3.getObject("ltest/versions/0.0.7/raw/mod.ts"); + let ts = await s3.getObject("ltest/versions/0.0.9/raw/mod.ts"); assertEquals(ts?.cacheControl, "public, max-age=31536000, immutable"); assertEquals(ts?.contentType, "application/typescript; charset=utf-8"); - assertEquals(ts?.body.length, 87); + assertEquals(ts?.body.length, 139); // Check the ts file was uploaded - let readme = await s3.getObject("ltest/versions/0.0.7/raw/README.md"); + let readme = await s3.getObject( + "ltest/versions/0.0.9/raw/.github/README.md", + ); assertEquals(readme?.cacheControl, "public, max-age=31536000, immutable"); assertEquals(readme?.contentType, "text/markdown"); assertEquals(readme?.body.length, 304); @@ -167,14 +303,17 @@ Deno.test({ // Cleanup await database._builds.deleteMany({}); await s3.deleteObject("ltest/meta/versions.json"); - await s3.deleteObject("ltest/versions/0.0.7/meta/meta.json"); - await s3.deleteObject("ltest/versions/0.0.7/raw/.github/workflows/ci.yml"); - await s3.deleteObject("ltest/versions/0.0.7/raw/LICENCE"); - await s3.deleteObject("ltest/versions/0.0.7/raw/README.md"); - await s3.deleteObject("ltest/versions/0.0.7/raw/fixtures/%"); - await s3.deleteObject("ltest/versions/0.0.7/raw/mod.ts"); - await s3.deleteObject("ltest/versions/0.0.7/raw/subproject/README.md"); - await s3.deleteObject("ltest/versions/0.0.7/raw/subproject/mod.ts"); + await s3.deleteObject("ltest/versions/0.0.9/meta/meta.json"); + await s3.deleteObject("ltest/versions/0.0.9/meta/deps.json"); + await s3.deleteObject("ltest/versions/0.0.9/raw/.github/workflows/ci.yml"); + await s3.deleteObject("ltest/versions/0.0.9/raw/.vscode/settings.json"); + await s3.deleteObject("ltest/versions/0.0.9/raw/LICENCE"); + await s3.deleteObject("ltest/versions/0.0.9/raw/deps.ts"); + await s3.deleteObject("ltest/versions/0.0.9/raw/fixtures/%"); + await s3.deleteObject("ltest/versions/0.0.9/raw/mod.ts"); + await s3.deleteObject("ltest/versions/0.0.9/raw/mod_test.md"); + await s3.deleteObject("ltest/versions/0.0.9/raw/subproject/README.md"); + await s3.deleteObject("ltest/versions/0.0.9/raw/subproject/mod.ts"); }, }); @@ -210,7 +349,7 @@ Deno.test({ subdir: "subproject/", }, status: "success", - message: "Finished uploading", + message: "Published module.", stats: { skipped_due_to_size: [], total_files: 2, diff --git a/api/webhook/github_push_test.ts b/api/webhook/github_push_test.ts index 9866412..08f21f0 100644 --- a/api/webhook/github_push_test.ts +++ b/api/webhook/github_push_test.ts @@ -153,8 +153,6 @@ Deno.test({ createContext(), ); - console.log(resp); - const builds = await database._builds.find({}); // Check that a new build was queued diff --git a/run-tests.sh b/run-tests.sh index c2d04fa..b46654d 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -7,6 +7,7 @@ sleep 10 aws --endpoint-url=http://s3:9000 s3 rm --recursive s3://deno-registry2 || true aws --endpoint-url=http://s3:9000 s3 rb s3://deno-registry2 || true aws --endpoint-url=http://s3:9000 s3 mb s3://deno-registry2 +aws --endpoint-url=http://s3:9000 s3api put-bucket-policy --bucket deno-registry2 --policy '{ "Version":"2012-10-17", "Statement":[ { "Sid":"PublicRead", "Effect":"Allow", "Principal": "*", "Action":["s3:GetObject","s3:GetObjectVersion"], "Resource":["arn:aws:s3:::deno-registry2/*"] } ] }' # Set up SQS aws --endpoint-url=http://sqs:9324 sqs delete-queue --queue-url http://sqs:9324/000000000000/builds --region us-east-1|| true @@ -17,5 +18,6 @@ export MONGO_URI=mongodb://root:rootpassword@mongo export BUILD_QUEUE=http://sqs:9324/000000000000/builds export STORAGE_BUCKET=deno-registry2 export S3_ENDPOINT_URL=http://s3:9000 +export REMOTE_URL=http://s3:9000/deno-registry2/%m/versions/%v/raw deno test --unstable -A \ No newline at end of file diff --git a/template.yaml b/template.yaml index 3832e0b..ae4a3f3 100644 --- a/template.yaml +++ b/template.yaml @@ -71,6 +71,7 @@ Resources: HANDLER_EXT: js MONGO_URI: "{{resolve:secretsmanager:mongodb/atlas/deno_registry2:SecretString:MongoURI}}" STORAGE_BUCKET: !Ref StorageBucket + REMOTE_URL: https://deno.land/x/%m@%v Timeout: 300 Events: BuildQueueTrigger: diff --git a/utils/database.ts b/utils/database.ts index a7737db..e85b2ca 100644 --- a/utils/database.ts +++ b/utils/database.ts @@ -32,11 +32,13 @@ export interface Build { }; status: string; message?: string; - stats?: { - total_files: number; - total_size: number; - skipped_due_to_size: string[]; - }; + stats?: BuildStats; +} + +export interface BuildStats { + total_files: number; + total_size: number; + skipped_due_to_size: string[]; } export class Database { diff --git a/utils/deno.ts b/utils/deno.ts new file mode 100644 index 0000000..227f177 --- /dev/null +++ b/utils/deno.ts @@ -0,0 +1,32 @@ +const decoder = new TextDecoder(); + +export type Dep = [string, Dep[]]; + +export async function runDenoInfo( + options: { entrypoint: string; denoDir: string }, +): Promise { + const p = Deno.run({ + cmd: [ + "deno", + "info", + "--json", + "--unstable", + "--no-check", + options.entrypoint, + ], + env: { + "DENO_DIR": options.denoDir, + }, + stdout: "piped", + stderr: "inherit", + }); + const status = await p.status(); + const file = await p.output(); + p.close(); + if (!status.success) { + throw new Error(`Failed to run deno info for ${options.entrypoint}`); + } + const text = decoder.decode(file); + const { deps } = JSON.parse(text); + return deps; +} diff --git a/utils/storage.ts b/utils/storage.ts index 60579ed..b715c2c 100644 --- a/utils/storage.ts +++ b/utils/storage.ts @@ -24,6 +24,18 @@ export async function getMeta( return resp?.body; } +export async function getVersionMetaJson( + module: string, + version: string, + file: string, +): Promise { + const resp = await s3.getObject( + join(module, "versions", version, "meta", file), + {}, + ); + return resp?.body; +} + const encoder = new TextEncoder(); export async function uploadMetaJson(