npm (unpkg) backed registry (denoland#25)

* Add np,m backed registry

* Added tests

* Fixed worker tests

components/Registry.tsx

@@ -11,6 +11,7 @@ import {
   denoDocAvailableForURL,
   isReadme,
   getVersionList,
+  getDefaultVersion,
 } from "../util/registry_utils";
 import Header from "./Header";
 import Footer from "./Footer";
@@ -46,6 +47,7 @@ const Registry = () => {
     path,
     version,
   ]);
+  const defaultVersion = useMemo(() => getDefaultVersion(name), [name]);
 
   const documentationURL = useMemo(() => {
     const doc = `https://doc.deno.land/https/deno.land/${canonicalPath}`;
@@ -232,7 +234,7 @@ const Registry = () => {
                         </option>
                       )}
                       <option key="" value="">
-                        master
+                        {defaultVersion}
                       </option>
                       {versions.map((v) => (
                         <option key={v} value={v}>

database.json

@@ -1211,10 +1211,8 @@
     "desc": "Update Deno Dependencies to their latest published versions"
   },
   "unexpected": {
-    "type": "url",
-    "url": "https://unpkg.com/unexpected@${v}/unexpected.esm.js",
-    "repo": "https://github.com/unexpectedjs/unexpected/",
-    "default_version": "latest",
+    "type": "npm",
+    "package": "unexpected",
     "desc": "Extensible BDD assertion toolkit"
   },
   "username": {

util/registries.ts

@@ -20,4 +20,5 @@ export interface Registry<T extends Entry> {
     version?: string
   ): Promise<DirEntry[] | null>;
   getVersionList(entry: T): Promise<string[] | null>;
+  getDefaultVersion(entry: T): string;
 }

util/registries/deno_std.ts

@@ -57,6 +57,9 @@ export class DenoStdRegistry implements Registry<DenoStdEntry> {
   async getVersionList(_entry: DenoStdEntry): Promise<string[] | null> {
     return stdVersions;
   }
+  getDefaultVersion(_entry: DenoStdEntry): string {
+    return "master";
+  }
 }
 
 export const denoStd = new DenoStdRegistry();

util/registries/github.ts

@@ -73,6 +73,9 @@ export class GithubRegistry implements Registry<GithubEntry> {
     const tags: string[] | undefined = data?.map((tag: any) => tag.name);
     return tags ?? null;
   }
+  getDefaultVersion(entry: GithubEntry): string {
+    return entry.default_version ?? "master";
+  }
 }
 
 export const github = new GithubRegistry();

util/registries/npm.test.ts

@@ -0,0 +1,77 @@
+import { npm, NPMEntry } from "./npm";
+import "isomorphic-unfetch";
+
+/* eslint-env jest */
+
+const testEntry: NPMEntry = {
+  type: "npm",
+  desc: "A entry for testing",
+  package: "example",
+};
+
+test("source url", () => {
+  expect(npm.getSourceURL(testEntry, "/index.js", "1.0.0")).toEqual(
+    "https://unpkg.com/example@1.0.0/index.js"
+  );
+});
+
+test("source url with default version", () => {
+  expect(npm.getSourceURL(testEntry, "/index.js", undefined)).toEqual(
+    "https://unpkg.com/example@latest/index.js"
+  );
+});
+
+test("source url with empty path", () => {
+  expect(npm.getSourceURL(testEntry, "", "1.0.0")).toEqual(
+    "https://unpkg.com/example@1.0.0"
+  );
+});
+
+test("source url with subdirectory", () => {
+  expect(
+    npm.getSourceURL({ ...testEntry, path: "/test" }, "/index.js", "1.0.0")
+  ).toEqual("https://unpkg.com/example@1.0.0/test/index.js");
+});
+
+test("repo url", () => {
+  expect(npm.getRepositoryURL(testEntry, "/index.js", "1.0.0")).toEqual(
+    "https://unpkg.com/browse/example@1.0.0/index.js"
+  );
+});
+
+test("repo url with default version", () => {
+  expect(npm.getRepositoryURL(testEntry, "/index.js", undefined)).toEqual(
+    "https://unpkg.com/browse/example@latest/index.js"
+  );
+});
+
+test("repo url with empty path", () => {
+  expect(npm.getRepositoryURL(testEntry, "", "1.0.0")).toEqual(
+    "https://unpkg.com/browse/example@1.0.0/"
+  );
+});
+
+test("repo url with subdirectory", () => {
+  expect(
+    npm.getRepositoryURL({ ...testEntry, path: "/test" }, "/index.js", "1.0.0")
+  ).toEqual("https://unpkg.com/browse/example@1.0.0/test/index.js");
+});
+
+test("directory listing", async () => {
+  expect(await npm.getDirectoryListing(testEntry, "", "0.0.0")).toEqual([
+    {
+      name: "package.json",
+      size: 291,
+      type: "file",
+    },
+    {
+      name: "github.js",
+      size: 886,
+      type: "file",
+    },
+  ]);
+});
+
+test("version list", async () => {
+  expect(await npm.getVersionList(testEntry)).toEqual(["0.0.0"]);
+});

util/registries/npm.ts

@@ -0,0 +1,83 @@
+/* Copyright 2020 the Deno authors. All rights reserved. MIT license. */
+
+import { Entry, Registry, DirEntry } from "../registries";
+
+export interface NPMEntry extends Entry {
+  type: "npm";
+  package: string;
+  path?: string;
+}
+
+export class NPMRegistry implements Registry<NPMEntry> {
+  getSourceURL(entry: NPMEntry, path: string, version?: string): string {
+    return `https://unpkg.com/${entry.package}@${version ?? "latest"}${
+      entry.path ?? ""
+    }${path}`;
+  }
+  getRepositoryURL(entry: NPMEntry, path: string, version?: string): string {
+    return `https://unpkg.com/browse/${entry.package}@${version ?? "latest"}${
+      entry.path ?? ""
+    }${path || "/"}`;
+  }
+  async getDirectoryListing(
+    entry: NPMEntry,
+    path: string,
+    version?: string
+  ): Promise<DirEntry[] | null> {
+    const url = `https://unpkg.com/${entry.package}@${version ?? "latest"}${
+      entry.path ?? ""
+    }${path}/?meta`;
+    const res = await fetch(url, {
+      headers: {
+        accept: "application/json",
+      },
+    });
+    if (res.status !== 200) {
+      throw Error(
+        `Got an error (${
+          res.status
+        }) while querying unpkg:\n${await res.text()}`
+      );
+    }
+    const data = await res.json();
+    if (data.type !== "directory") {
+      return null;
+    }
+    const prefix = (data.path === "/" ? "" : data.path) + "/";
+    const files: DirEntry[] = data.files.map((file: any) => {
+      return {
+        name: file.path?.substring(prefix.length),
+        type: file.type === "directory" ? "dir" : file.type, // "file" | "dir"
+        size: file.size, // file only
+      };
+    });
+    return files;
+  }
+  async getVersionList(entry: NPMEntry): Promise<string[] | null> {
+    const url = `https://${
+      process ? "" : "cors-anywhere.herokuapp.com/"
+    }registry.npmjs.org/${entry.package}`;
+    const res = await fetch(url, {
+      headers: {
+        accept: "application/vnd.npm.install-v1+json",
+      },
+    });
+    if (res.status !== 200) {
+      throw Error(
+        `Got an error (${
+          res.status
+        }) while querying the NPM registry API:\n${await res.text()}`
+      );
+    }
+    const data = await res.json();
+    const tags: string[] | undefined = data
+      ? Object.keys(data.versions).reverse()
+      : [];
+    return tags ?? null;
+  }
+  getDefaultVersion(_entry: NPMEntry): string {
+    return "latest";
+  }
+}
+
+export const npm = new NPMRegistry();

util/registries/url.ts

@@ -30,6 +30,9 @@ export class URLRegistry implements Registry<URLEntry> {
   async getVersionList(_entry: URLEntry): Promise<string[] | null> {
     return null;
   }
+  getDefaultVersion(entry: URLEntry): string {
+    return entry.default_version;
+  }
 }
 
 export const url = new URLRegistry();

util/registry_utils.ts

@@ -4,77 +4,70 @@ import DATABASE from "../database.json";
 import { github, GithubEntry } from "./registries/github";
 import { denoStd, DenoStdEntry } from "./registries/deno_std";
 import { url, URLEntry } from "./registries/url";
-import { DirEntry, Entry } from "./registries";
+import { npm, NPMEntry } from "./registries/npm";
+import { DirEntry, Entry, Registry } from "./registries";
 
-export function find(name: string): GithubEntry | DenoStdEntry | URLEntry {
+export function find(
+  name: string
+): GithubEntry | DenoStdEntry | URLEntry | NPMEntry {
   // @ts-ignore
   return DATABASE[name];
 }
 
-export function getSourceURL(
-  name: string,
-  path: string,
-  version?: string
-): string | null {
-  const entry = find(name);
-  switch (entry?.type) {
+function registryByType(type?: string): Registry<Entry> | null {
+  switch (type) {
     case "github":
-      return github.getSourceURL(entry, path, version);
+      return github;
     case "deno_std":
-      return denoStd.getSourceURL(entry, path, version);
+      return denoStd;
     case "url":
-      return url.getSourceURL(entry, path, version);
+      return url;
+    case "npm":
+      return npm;
     default:
       return null;
   }
 }
+export function getSourceURL(
+  name: string,
+  path: string,
+  version?: string
+): string | null {
+  const entry = find(name);
+  const registry = registryByType(entry?.type);
+  return registry?.getSourceURL(entry, path, version) ?? null;
+}
+
 export function getRepositoryURL(
   name: string,
   path: string,
   version?: string
 ): string | null {
   const entry = find(name);
-  switch (entry?.type) {
-    case "github":
-      return github.getRepositoryURL(entry, path, version);
-    case "deno_std":
-      return denoStd.getRepositoryURL(entry, path, version);
-    case "url":
-      return url.getRepositoryURL(entry, path, version);
-    default:
-      return null;
-  }
+  const registry = registryByType(entry?.type);
+  return registry?.getRepositoryURL(entry, path, version) ?? null;
 }
+
 export async function getDirectoryListing(
   name: string,
   path: string,
   version?: string
 ): Promise<DirEntry[] | null> {
   const entry = find(name);
-  switch (entry?.type) {
-    case "github":
-      return github.getDirectoryListing(entry, path, version);
-    case "deno_std":
-      return denoStd.getDirectoryListing(entry, path, version);
-    case "url":
-      return url.getDirectoryListing(entry, path, version);
-    default:
-      return null;
-  }
+  const registry = registryByType(entry?.type);
+  return registry?.getDirectoryListing(entry, path, version) ?? null;
 }
 
 export async function getVersionList(name: string): Promise<string[] | null> {
   const entry = find(name);
-  switch (entry?.type) {
-    case "github":
-      return github.getVersionList(entry);
-    case "deno_std":
-      return denoStd.getVersionList(entry);
-    case "url":
-      return url.getVersionList(entry);
-    default:
-      return null;
-  }
+  const registry = registryByType(entry?.type);
+  return registry?.getVersionList(entry) ?? null;
+}
+
+export function getDefaultVersion(name: string): string | undefined {
+  const entry = find(name);
+  const registry = registryByType(entry?.type);
+  return registry?.getDefaultVersion(entry);
 }
 
 export function parseNameVersion(

worker/package.json

@@ -17,10 +17,11 @@
     "@types/node": "^13.13.0",
     "jest": "25",
     "prettier": "^2.0.4",
-    "ts-loader": "^7.0.3",
     "ts-jest": "^25.5.1",
+    "ts-loader": "^7.0.3",
     "typescript": "^3.8.3",
     "webpack": "^4.42.1",
-    "webpack-cli": "^3.3.11"
+    "webpack-cli": "^3.3.11",
+    "source-map-loader": "^0.2.4"
   }
 }