Full "webDependencies" management

src/config.ts

@@ -2,7 +2,7 @@ import path from 'path';
 import {cosmiconfigSync} from 'cosmiconfig';
 import {Plugin} from 'rollup';
 import {validate} from 'jsonschema';
-import merge from 'deepmerge';
+import {all as merge} from 'deepmerge';
 
 const CONFIG_NAME = 'snowpack';
 
@@ -17,7 +17,8 @@ type DeepPartial<T> = {
 // interface this library uses internally
 export interface SnowpackConfig {
   source: 'local' | 'pika';
-  webDependencies?: string[];
+  webDependencies?: {[packageName: string]: string};
+  entrypoints?: string[];
   dedupe?: string[];
   namedExports?: {[filepath: string]: string[]};
   installOptions: {
@@ -52,8 +53,7 @@ export interface CLIFlags extends Partial<SnowpackConfig['installOptions']> {
 }
 
 // default settings
-const DEFAULT_CONFIG: SnowpackConfig = {
-  source: 'local',
+const DEFAULT_CONFIG: Partial<SnowpackConfig> = {
   dedupe: [],
   installOptions: {
     clean: false,
@@ -74,7 +74,13 @@ const configSchema = {
   type: 'object',
   properties: {
     source: {type: 'string'},
-    webDependencies: {type: 'array', items: {type: 'string'}},
+    entrypoints: {type: 'array', items: {type: 'string'}},
+    // TODO: Array of strings data format is deprecated, remove for v2
+    webDependencies: {
+      type: ['array', 'object'],
+      additionalProperties: {type: 'string'},
+      items: {type: 'string'},
+    },
     dedupe: {
       type: 'array',
       items: {type: 'string'},
@@ -128,13 +134,21 @@ function expandCliFlags(flags: CLIFlags): DeepPartial<SnowpackConfig> {
   return result;
 }
 
-/** resolve --dest relative to cwd */
-function normalizeDest(config: SnowpackConfig) {
+/** resolve --dest relative to cwd, and set the default "source" */
+function normalizeConfig(config: SnowpackConfig): SnowpackConfig {
   config.installOptions.dest = path.resolve(process.cwd(), config.installOptions.dest);
+  if (Array.isArray(config.webDependencies)) {
+    config.entrypoints = config.webDependencies;
+    delete config.webDependencies;
+  }
+  if (!config.source) {
+    const isDetailedObject = config.webDependencies && typeof config.webDependencies === 'object';
+    config.source = isDetailedObject ? 'pika' : 'local';
+  }
   return config;
 }
 
-export default function loadConfig(flags: CLIFlags) {
+export default function loadConfig(flags: CLIFlags, pkgManifest: any) {
   const cliConfig = expandCliFlags(flags);
 
   const explorerSync = cosmiconfigSync(CONFIG_NAME, {
@@ -162,7 +176,13 @@ export default function loadConfig(flags: CLIFlags) {
   if (!result || !result.config || result.isEmpty) {
     // if CLI flags present, apply those as overrides
     return {
-      config: normalizeDest(merge<any>(DEFAULT_CONFIG, cliConfig)),
+      config: normalizeConfig(
+        merge<SnowpackConfig>([
+          DEFAULT_CONFIG,
+          {webDependencies: pkgManifest.webDependencies},
+          cliConfig as any,
+        ]),
+      ),
       errors,
     };
   }
@@ -176,11 +196,16 @@ export default function loadConfig(flags: CLIFlags) {
   });
 
   // if valid, apply config over defaults
-  const mergedConfig = merge(DEFAULT_CONFIG, config);
+  const mergedConfig = merge<SnowpackConfig>([
+    DEFAULT_CONFIG,
+    {webDependencies: pkgManifest.webDependencies},
+    config,
+    cliConfig as any,
+  ]);
 
   // if CLI flags present, apply those as overrides
   return {
-    config: normalizeDest(merge<any>(mergedConfig, cliConfig)),
+    config: normalizeConfig(mergedConfig),
     errors: validation.errors.map((msg) => `${path.basename(result.filepath)}: ${msg.toString()}`),
   };
 }

src/index.ts

@@ -404,7 +404,7 @@ export async function install(
   if (Object.keys(installEntrypoints).length > 0) {
     try {
       const packageBundle = await rollup(inputOptions);
-      logUpdate('');
+      logUpdate(formatInstallResults(skipFailures));
       await packageBundle.write(outputOptions);
     } catch (err) {
       const {loc} = err as RollupError;
@@ -499,8 +499,17 @@ export async function cli(args: string[]) {
     await clearCache();
   }
 
+  // Load the current package manifest
+  let pkgManifest: any;
+  try {
+    pkgManifest = require(path.join(cwd, 'package.json'));
+  } catch (err) {
+    console.log(chalk.red('[ERROR] package.json required but no file was found.'));
+    process.exit(0);
+  }
+
   // load config
-  const {config, errors} = loadConfig(cliFlags);
+  const {config, errors} = loadConfig(cliFlags, pkgManifest);
 
   // handle config errors (if any)
   if (Array.isArray(errors) && errors.length) {
@@ -512,7 +521,7 @@ export async function cli(args: string[]) {
     console.log(
       `${chalk.yellow(
         'ℹ',
-      )} "source" configuration is still experimental. Behavior may change before the next major version...`,
+      )} "source: pika" mode enabled. Behavior is still experimental and may change before the next major version...`,
     );
     await clearCache();
   }
@@ -523,18 +532,11 @@ export async function cli(args: string[]) {
 
   const {
     installOptions: {clean, dest, exclude, include},
-    webDependencies,
+    entrypoints: configEntrypoints,
     source,
+    webDependencies,
   } = config;
 
-  let pkgManifest: any;
-  try {
-    pkgManifest = require(path.join(cwd, 'package.json'));
-  } catch (err) {
-    console.log(chalk.red('[ERROR] package.json required but no file was found.'));
-    process.exit(0);
-  }
-
   const implicitDependencies = [
     ...Object.keys(pkgManifest.peerDependencies || {}),
     ...Object.keys(pkgManifest.dependencies || {}),
@@ -548,15 +550,19 @@ export async function cli(args: string[]) {
   let isExplicit = false;
   const installTargets: InstallTarget[] = [];
 
+  if (configEntrypoints) {
+    isExplicit = true;
+    installTargets.push(...scanDepList(configEntrypoints, cwd));
+  }
   if (webDependencies) {
     isExplicit = true;
-    installTargets.push(...scanDepList(webDependencies, cwd));
+    installTargets.push(...scanDepList(Object.keys(webDependencies), cwd));
   }
   if (include) {
     isExplicit = true;
     installTargets.push(...(await scanImports({include, exclude})));
   }
-  if (!webDependencies && !include) {
+  if (!isExplicit) {
     installTargets.push(...scanDepList(implicitDependencies, cwd));
   }
   if (installTargets.length === 0) {
@@ -567,7 +573,15 @@ export async function cli(args: string[]) {
   spinner.start();
   const startTime = Date.now();
   if (source === 'pika') {
-    newLockfile = await resolveTargetsFromRemoteCDN(installTargets, lockfile, pkgManifest, config);
+    newLockfile = await resolveTargetsFromRemoteCDN(
+      installTargets,
+      lockfile,
+      pkgManifest,
+      config,
+    ).catch((err) => {
+      logError(err.message || err);
+      process.exit(1);
+    });
   }
 
   if (clean) {

src/resolve-remote.ts

@@ -1,5 +1,6 @@
 import cacache from 'cacache';
 import PQueue from 'p-queue';
+import chalk from 'chalk';
 import validatePackageName from 'validate-npm-package-name';
 import {SnowpackConfig} from './config.js';
 import {InstallTarget} from './scan-imports.js';
@@ -18,6 +19,7 @@ async function resolveDependency(
   installSpecifier: string,
   packageSemver: string,
   lockfile: ImportMap | null,
+  canRetry = true,
 ): Promise<null | string> {
   // Right now, the CDN is only for top-level JS packages. The CDN doesn't support CSS,
   // non-JS assets, and has limited support for deep package imports. Snowpack
@@ -29,8 +31,10 @@ async function resolveDependency(
 
   // Grab the installUrl from our lockfile if it exists, otherwise resolve it yourself.
   let installUrl: string;
+  let installUrlType: 'pin' | 'lookup';
   if (lockfile && lockfile.imports[installSpecifier]) {
     installUrl = lockfile.imports[installSpecifier];
+    installUrlType = 'pin';
   } else {
     if (packageSemver === 'latest') {
       console.warn(
@@ -39,7 +43,7 @@ async function resolveDependency(
     }
     if (packageSemver.startsWith('npm:@reactesm') || packageSemver.startsWith('npm:@pika/react')) {
       throw new Error(
-        `React workarounds no longer needed in --source=pika mode. Revert to the official React & React-DOM packages.`,
+        `React workaround packages no longer needed! Revert to the official React & React-DOM packages.`,
       );
     }
     if (packageSemver.includes(' ') || packageSemver.includes(':')) {
@@ -48,12 +52,13 @@ async function resolveDependency(
       );
       return null;
     }
+    installUrlType = 'lookup';
     installUrl = `${PIKA_CDN}/${installSpecifier}@${packageSemver}`;
   }
 
   // Hashed CDN urls never change, so its safe to grab them directly from the local cache
   // without a network request.
-  if (HAS_CDN_HASH_REGEX.test(installUrl)) {
+  if (installUrlType === 'pin') {
     const cachedResult = await cacache.get.info(RESOURCE_CACHE, installUrl).catch(() => null);
     if (cachedResult) {
       if (cachedResult.metadata) {
@@ -70,15 +75,46 @@ async function resolveDependency(
     console.warn(`Falling back to local copy...`);
     return null;
   }
-  const _pinnedUrl = headers['x-pinned-url'] as string;
-  if (!_pinnedUrl) {
-    throw new Error('X-Pinned-URL Header expected, but none received.');
+  if (installUrlType === 'pin') {
+    const pinnedUrl = installUrl;
+    await cacache.put(RESOURCE_CACHE, pinnedUrl, body, {
+      metadata: {pinnedUrl},
+    });
+    return pinnedUrl;
+  }
+  let importUrlPath = headers['x-import-url'] as string;
+  let pinnedUrlPath = headers['x-pinned-url'] as string;
+  const buildStatus = headers['x-import-status'] as string;
+  if (pinnedUrlPath) {
+    const pinnedUrl = `${PIKA_CDN}${pinnedUrlPath}`;
+    await cacache.put(RESOURCE_CACHE, pinnedUrl, body, {
+      metadata: {pinnedUrl},
+    });
+    return pinnedUrl;
+  }
+  if (buildStatus === 'SUCCESS') {
+    console.warn(`Failed to lookup [${statusCode}]: ${installUrl}`);
+    console.warn(`Falling back to local copy...`);
+    return null;
   }
-  const pinnedUrl = `${PIKA_CDN}${_pinnedUrl}`;
-  await cacache.put(RESOURCE_CACHE, installUrl, body, {
-    metadata: {pinnedUrl},
-  });
-  return pinnedUrl;
+  if (!canRetry || buildStatus === 'FAIL') {
+    console.warn(`Failed to build: ${installSpecifier}@${packageSemver}`);
+    console.warn(`Falling back to local copy...`);
+    return null;
+  }
+  console.log(
+    chalk.cyan(
+      `Building ${installSpecifier}@${packageSemver}... (This takes a moment, but will be cached for future use)`,
+    ),
+  );
+  if (!importUrlPath) {
+    throw new Error('X-Import-URL header expected, but none received.');
+  }
+  const {statusCode: lookupStatusCode} = await fetchCDNResource(importUrlPath);
+  if (lookupStatusCode !== 200) {
+    throw new Error(`Unexpected response [${lookupStatusCode}]: ${PIKA_CDN}${importUrlPath}`);
+  }
+  return resolveDependency(installSpecifier, packageSemver, lockfile, false);
 }
 
 export async function resolveTargetsFromRemoteCDN(
@@ -89,23 +125,33 @@ export async function resolveTargetsFromRemoteCDN(
 ) {
   const downloadQueue = new PQueue({concurrency: 16});
   const newLockfile: ImportMap = {imports: {}};
+  let resolutionError: Error | undefined;
 
   const allInstallSpecifiers = new Set(installTargets.map((dep) => dep.specifier));
   for (const installSpecifier of allInstallSpecifiers) {
     const installSemver: string =
+      (config.webDependencies || {})[installSpecifier] ||
+      (pkgManifest.webDependencies || {})[installSpecifier] ||
       (pkgManifest.dependencies || {})[installSpecifier] ||
       (pkgManifest.devDependencies || {})[installSpecifier] ||
       (pkgManifest.peerDependencies || {})[installSpecifier] ||
       'latest';
     downloadQueue.add(async () => {
-      const resolvedUrl = await resolveDependency(installSpecifier, installSemver, lockfile);
-      if (resolvedUrl) {
-        newLockfile.imports[installSpecifier] = resolvedUrl;
+      try {
+        const resolvedUrl = await resolveDependency(installSpecifier, installSemver, lockfile);
+        if (resolvedUrl) {
+          newLockfile.imports[installSpecifier] = resolvedUrl;
+        }
+      } catch (err) {
+        resolutionError = resolutionError || err;
       }
     });
   }
 
   await downloadQueue.onIdle();
+  if (resolutionError) {
+    throw resolutionError;
+  }
 
   return newLockfile;
 }

src/rollup-plugin-remote-cdn.ts

@@ -3,7 +3,7 @@ import cacache from 'cacache';
 import {RESOURCE_CACHE, fetchCDNResource, PIKA_CDN, HAS_CDN_HASH_REGEX} from './util';
 
 const CACHED_FILE_ID_PREFIX = 'snowpack-pkg-cache:';
-
+const PIKA_CDN_TRIM_LENGTH = PIKA_CDN.length;
 /**
  * rollup-plugin-remote-cdn
  *
@@ -21,15 +21,17 @@ export function rollupPluginDependencyCache({log}: {log: (url: string) => void})
         cacheKey = source;
       } else if (source.startsWith('/-/')) {
         cacheKey = PIKA_CDN + source;
+      } else if (source.startsWith('/pin/')) {
+        cacheKey = PIKA_CDN + source;
       } else {
         return null;
       }
 
       // If the source path is a CDN path including a hash, it's assumed the
       // file will never change and it is safe to pull from our local cache
       // without a network request.
-      log(source);
-      if (HAS_CDN_HASH_REGEX.test(source)) {
+      log(cacheKey);
+      if (HAS_CDN_HASH_REGEX.test(cacheKey)) {
         const cachedResult = await cacache.get
           .info(RESOURCE_CACHE, cacheKey)
           .catch((/* ignore */) => null);
@@ -47,7 +49,11 @@ export function rollupPluginDependencyCache({log}: {log: (url: string) => void})
 
       // If lookup failed, skip this plugin and resolve the import locally instead.
       // TODO: Log that this has happened (if some sort of verbose mode is enabled).
-      const packageName = source.substring(3).split('@')[0];
+      const packageName = cacheKey
+        .substring(PIKA_CDN_TRIM_LENGTH)
+        .replace('/-/', '')
+        .replace('/pin/', '')
+        .split('@')[0];
       return this.resolve(packageName, importer!, {skipSelf: true}).then((resolved) => {
         let finalResult = resolved;
         if (!finalResult) {
@@ -61,6 +67,7 @@ export function rollupPluginDependencyCache({log}: {log: (url: string) => void})
         return null;
       }
       const cacheKey = id.substring(CACHED_FILE_ID_PREFIX.length);
+      log(cacheKey);
       const cachedResult = await cacache.get(RESOURCE_CACHE, cacheKey);
       return cachedResult.data.toString('utf8');
     },