Merge pull request #440 from TobyAndToby/ts/http-replacements

Http replacement source

e2e/config-file/test/cli/replacement/__snapshots__/replacement.e2e.spec.ts.snap

@@ -4,30 +4,33 @@ exports[`cli should match snapshot when a package is replaced 1`] = `
 "This file was generated with the generate-license-file npm package!
 https://www.npmjs.com/package/generate-license-file
 
-The following npm package may be included in this product:
+The following npm packages may be included in this product:
 
- - dep-four@1.0.0
+ - dep-two-duplicate@1.0.0
+ - dep-two@1.0.0
 
-This package contains the following license and notice below:
+These packages each contain the following license and notice below:
 
-# Dep Four
+# Dep Two
 
-This license file is spelt \`LICENCE\`.
+This license file is spelt \`LICENCE.md\`.
 This license should be found.
 
 -----------
 
-The following npm packages may be included in this product:
+The following npm package may be included in this product:
 
- - dep-two-duplicate@1.0.0
- - dep-two@1.0.0
+ - dep-four@1.0.0
 
-These packages each contain the following license and notice below:
+This package contains the following license and notice below:
 
-# Dep Two
+# Remote license
 
-This license file is spelt \`LICENCE.md\`.
-This license should be found.
+This file is NOT an actual license for this package.
+
+It is used as a resource for our e2e's, as a file we can fetch over the network, and have control over.
+
+This content should appear in the generated license file output!
 
 -----------
 

e2e/config-file/test/cli/replacement/config.js

@@ -8,5 +8,7 @@ module.exports = {
 
     "dep-three": "./some-path-that-we-dont-want-to-use.txt",
     "dep-three@1.0.0": "./name-and-version-replacement-content.txt",
+    "dep-four":
+      "https://raw.githubusercontent.com/TobyAndToby/generate-license-file/main/e2e/.remote-licenses/license.md",
   },
 };

src/packages/generate-license-file/src/lib/cli/config/index.ts

@@ -17,17 +17,20 @@ export const loadConfigFile = async (path?: string): Promise<ConfigSchema> => {
 const findAndParseConfig = async (directory: string): Promise<ConfigSchema> => {
   const configFile = await findConfig(directory);
 
-  return parseConfig(configFile, directory);
+  return await parseConfig(configFile, directory);
 };
 
 const loadAndParseConfig = async (filePath: string) => {
   const configFile = await loadConfig(filePath);
   const directory = dirname(filePath);
 
-  return parseConfig(configFile, directory);
+  return await parseConfig(configFile, directory);
 };
 
-const parseConfig = (configFile: ConfigFile | undefined, directory: string): ConfigSchema => {
+const parseConfig = async (
+  configFile: ConfigFile | undefined,
+  directory: string,
+): Promise<ConfigSchema> => {
   const config = parseSchema(configFile?.config);
 
   if (config === undefined) {
@@ -37,12 +40,16 @@ const parseConfig = (configFile: ConfigFile | undefined, directory: string): Con
   for (const replacement in config?.replace) {
     const replacementPath = config.replace[replacement];
 
-    if (isAbsolute(replacementPath)) {
-      continue;
-    }
-
+    // The replacement value could be multiple things (e.g. file path, or a
+    // URL). If it's a file path, then at this stage the CLI is aware of the
+    // execution directory and needs to make the path absolute before handing
+    // it off to the library implementation. Otherwise, pass the raw replacement
+    // value into the library so it can handle it however it wants.
     const absolutePath = join(directory, replacementPath);
-    config.replace[replacement] = absolutePath;
+
+    if (await doesFileExist(absolutePath)) {
+      config.replace[replacement] = absolutePath;
+    }
   }
 
   if (config.append) {

src/packages/generate-license-file/src/lib/internal/resolveDependencies/resolveNpmDependencies.ts

@@ -3,6 +3,7 @@ import { resolveLicenseContent } from "../resolveLicenseContent";
 import { dirname, isAbsolute, join } from "path";
 import { Dependency, LicenseContent } from "../resolveLicenses";
 import { readPackageJson } from "../../utils/packageJson.utils";
+import logger from "../../utils/console.utils";
 
 type ResolveLicensesOptions = {
   replace?: Record<string, string>;
@@ -36,9 +37,9 @@ export const resolveDependenciesForNpmProject = async (
       return;
     }
 
-    const licenseContent = await resolveLicenseContent(node.realpath, packageJson, replacements);
+    try {
+      const licenseContent = await resolveLicenseContent(node.realpath, packageJson, replacements);
 
-    if (licenseContent) {
       const dependencies = licensesMap.get(licenseContent) ?? [];
 
       const alreadyExists = dependencies.find(
@@ -50,6 +51,14 @@ export const resolveDependenciesForNpmProject = async (
       }
 
       licensesMap.set(licenseContent, dependencies);
+    } catch (error) {
+      const warningLines = [
+        `Unable to determine license content for ${packageJson.name}@${packageJson.version} with error:`,
+        error instanceof Error ? error.message : error?.toString(),
+        "", // Empty line for spacing
+      ];
+
+      logger.warn(warningLines.join("\n"));
     }
 
     for (const child of node.children.values()) {

src/packages/generate-license-file/src/lib/internal/resolveDependencies/resolvePnpmDependencies.ts

@@ -3,6 +3,7 @@ import { dirname, join } from "path";
 import { getPnpmProjectDependencies, getPnpmVersion } from "../../utils/pnpmCli.utils";
 import { Dependency, LicenseContent } from "../resolveLicenses";
 import { readPackageJson } from "../../utils/packageJson.utils";
+import logger from "../../utils/console.utils";
 
 type ResolveLicensesOptions = {
   replace?: Record<string, string>;
@@ -34,9 +35,13 @@ export const resolveDependenciesForPnpmProject = async (
         continue;
       }
 
-      const licenseContent = await resolveLicenseContent(dependencyPath, packageJson, replacements);
+      try {
+        const licenseContent = await resolveLicenseContent(
+          dependencyPath,
+          packageJson,
+          replacements,
+        );
 
-      if (licenseContent) {
         const dependencies = licensesMap.get(licenseContent) ?? [];
 
         const alreadyExists = dependencies.find(
@@ -48,6 +53,14 @@ export const resolveDependenciesForPnpmProject = async (
         }
 
         licensesMap.set(licenseContent, dependencies);
+      } catch (error) {
+        const warningLines = [
+          `Unable to determine license content for ${packageJson.name}@${packageJson.version} with error:`,
+          error instanceof Error ? error.message : error?.toString(),
+          "", // Empty line for spacing
+        ];
+
+        logger.warn(warningLines.join("\n"));
       }
     }
   }

src/packages/generate-license-file/src/lib/internal/resolveLicenseContent/index.ts

@@ -2,37 +2,59 @@ import { PackageJson } from "../../utils/packageJson.utils";
 import { packageJsonLicense } from "./packageJsonLicense";
 import { licenseFile } from "./licenseFile";
 import { spdxExpression } from "./spdxExpression";
-import { readFile } from "../../utils/file.utils";
+import { replacementFile } from "./replacementFile";
+import { replacementHttp } from "./replacementHttp";
 
 export interface ResolutionInputs {
   directory: string;
   packageJson: PackageJson;
 }
 
 export type Resolution = (inputs: ResolutionInputs) => Promise<string | null>;
-
 const resolutions: Resolution[] = [packageJsonLicense, licenseFile, spdxExpression];
 
+export type ReplacementResolution = (location: string) => Promise<string | null>;
+const replacementResolutions: ReplacementResolution[] = [replacementHttp, replacementFile];
+
 export const resolveLicenseContent = async (
   directory: string,
   packageJson: PackageJson,
   replacements: Record<string, string>,
-): Promise<string | null> => {
+): Promise<string> => {
   const replacementPath =
     replacements[`${packageJson.name}@${packageJson.version}`] ||
     replacements[`${packageJson.name}`];
 
   if (replacementPath) {
-    return await readFile(replacementPath, { encoding: "utf-8" });
+    return runReplacementResolutions(replacementPath, packageJson);
+  }
+
+  const resolutionInputs: ResolutionInputs = { directory, packageJson };
+  return runResolutions(resolutionInputs, packageJson);
+};
+
+const runReplacementResolutions = async (replacementPath: string, packageJson: PackageJson) => {
+  for (const resolution of replacementResolutions) {
+    const result = await resolution(replacementPath);
+
+    if (result) {
+      return result;
+    }
   }
 
+  throw new Error(
+    `Could not find replacement content at ${replacementPath} for ${packageJson.name}@${packageJson.version}`,
+  );
+};
+
+const runResolutions = async (inputs: ResolutionInputs, packageJson: PackageJson) => {
   for (const resolution of resolutions) {
-    const result = await resolution({ directory, packageJson });
+    const result = await resolution(inputs);
 
     if (result) {
       return result;
     }
   }
 
-  return null;
+  throw new Error(`Could not find license content for ${packageJson.name}@${packageJson.version}`);
 };

src/packages/generate-license-file/src/lib/internal/resolveLicenseContent/packageJsonLicense.ts

@@ -86,7 +86,7 @@ const parseArrayLicense = (license: PackageJsonLicense[], packageJson: PackageJs
   }
 
   const warningLines = [
-    `The license key for ${packageJson.name}@${packageJson.version} contains multiple licenses"`,
+    `The license key for ${packageJson.name}@${packageJson.version} contains multiple licenses`,
     "We suggest you determine which license applies to your project and replace the license content",
     `for ${packageJson.name}@${packageJson.version} using a generate-license-file config file.`,
     "See: https://generate-license-file.js.org/docs/cli/config-file for more information.",

src/packages/generate-license-file/src/lib/internal/resolveLicenseContent/replacementFile.ts

@@ -0,0 +1,10 @@
+import { ReplacementResolution } from ".";
+import { readFile, doesFileExist } from "../../utils/file.utils";
+
+export const replacementFile: ReplacementResolution = async location => {
+  if (!(await doesFileExist(location))) {
+    return null;
+  }
+
+  return readFile(location, { encoding: "utf-8" });
+};

src/packages/generate-license-file/src/lib/internal/resolveLicenseContent/replacementHttp.ts

@@ -0,0 +1,10 @@
+import { ReplacementResolution } from ".";
+import { fetchString } from "../../utils/http.utils";
+
+export const replacementHttp: ReplacementResolution = async location => {
+  if (!location.startsWith("http") && !location.startsWith("www")) {
+    return null;
+  }
+
+  return fetchString(location);
+};

src/packages/generate-license-file/src/lib/utils/http.utils.ts

@@ -0,0 +1,6 @@
+// istanbul ignore file
+
+export const fetchString = async (url: string): Promise<string> => {
+  const response = await fetch(url);
+  return response.text();
+};

src/packages/generate-license-file/test/internal/resolveDependencies/resolveNpmDependencies.spec.ts

@@ -6,19 +6,22 @@ import { Dependency, LicenseContent } from "../../../src/lib/internal/resolveLic
 import { PackageJson } from "../../../src/lib/utils/packageJson.utils";
 import { join } from "path";
 import { doesFileExist, readFile } from "../../../src/lib/utils/file.utils";
+import logger from "../../../src/lib/utils/console.utils";
 
 jest.mock("@npmcli/arborist", () => ({
   __esModule: true,
   default: jest.fn(),
 }));
 
 jest.mock("../../../src/lib/utils/file.utils");
+jest.mock("../../../src/lib/utils/console.utils");
 
 jest.mock("../../../src/lib/internal/resolveLicenseContent", () => ({
   resolveLicenseContent: jest.fn(),
 }));
 
 describe("resolveNpmDependencies", () => {
+  const mockedLogger = jest.mocked(logger);
   const mockedReadFile = jest.mocked(readFile);
   const mockedDoesFileExist = jest.mocked(doesFileExist);
 
@@ -258,6 +261,22 @@ describe("resolveNpmDependencies", () => {
     expect(replacements3).toBe(replacements);
   });
 
+  it.each([new Error("Something went wrong"), "Something went wrong"])(
+    "should warning log if resolveLicenseContent throws an error",
+    async error => {
+      when(mockedResolveLicenseContent)
+        .calledWith(child1Realpath, expect.anything(), expect.anything())
+        .mockRejectedValue(error);
+
+      await resolveDependenciesForNpmProject("/some/path/package.json", new Map());
+
+      expect(mockedLogger.warn).toHaveBeenCalledTimes(1);
+      expect(mockedLogger.warn).toHaveBeenCalledWith(
+        `Unable to determine license content for ${child1Name}@${child1Version} with error:\nSomething went wrong\n`,
+      );
+    },
+  );
+
   describe("when no options are provided", () => {
     it("should include non-dev dependencies in the result", async () => {
       const licensesMap = new Map<LicenseContent, Dependency[]>();

src/packages/generate-license-file/test/internal/resolveDependencies/resolvePnpmDependencies.spec.ts

@@ -8,6 +8,7 @@ import { resolveLicenseContent } from "../../../src/lib/internal/resolveLicenseC
 import { when } from "jest-when";
 import { doesFileExist, readFile } from "../../../src/lib/utils/file.utils";
 import { PackageJson } from "../../../src/lib/utils/packageJson.utils";
+import logger from "../../../src/lib/utils/console.utils";
 import { join } from "path";
 import { Dependency, LicenseContent } from "../../../src/lib/internal/resolveLicenses";
 
@@ -17,6 +18,7 @@ jest.mock("../../../src/lib/utils/pnpmCli.utils", () => ({
 }));
 
 jest.mock("../../../src/lib/utils/file.utils");
+jest.mock("../../../src/lib/utils/console.utils");
 
 jest.mock("../../../src/lib/internal/resolveLicenseContent", () => ({
   resolveLicenseContent: jest.fn(),
@@ -39,8 +41,8 @@ describe("resolveDependenciesForPnpmProject", () => {
     name: "dependency3",
     paths: ["/some/path/dependency3"],
   };
-  const dependency3LicenseContent = null as unknown as string;
 
+  const mockedLogger = jest.mocked(logger);
   const mockedReadFile = jest.mocked(readFile);
   const mockedDoesFileExist = jest.mocked(doesFileExist);
   const mockedGetPnpmVersion = jest.mocked(getPnpmVersion);
@@ -67,7 +69,9 @@ describe("resolveDependenciesForPnpmProject", () => {
 
     when(mockedResolveLicenseContent)
       .calledWith(dependency3.paths[0], expect.anything(), expect.anything())
-      .mockResolvedValue(dependency3LicenseContent);
+      .mockImplementation(() => {
+        throw new Error("Cannot find license content");
+      });
     setUpPackageJson(dependency3.paths[0], { name: dependency3.name, version: "1.0.0" });
   });
 
@@ -198,9 +202,28 @@ describe("resolveDependenciesForPnpmProject", () => {
           .get(dependency2LicenseContent)
           ?.find(d => d.name === "dependency2" && d.version === "2.0.0"),
       ).toBeDefined();
-      expect(licensesMap.get(dependency3LicenseContent)).toBeUndefined();
     });
 
+    it.each([new Error("Something went wrong"), "Something went wrong"])(
+      "should warning log if resolveLicenseContent throws an error",
+      async error => {
+        mockedGetPnpmVersion.mockResolvedValue(pnpmVersion);
+        mockedGetPnpmProjectDependencies.mockResolvedValue([dependency1, dependency2, dependency3]);
+
+        when(mockedResolveLicenseContent)
+          .calledWith(dependency1.paths[0], expect.anything(), expect.anything())
+          .mockRejectedValue(error);
+
+        const licensesMap = new Map<LicenseContent, Dependency[]>();
+
+        await resolveDependenciesForPnpmProject("/some/path/package.json", licensesMap);
+
+        expect(mockedLogger.warn).toHaveBeenCalledWith(
+          `Unable to determine license content for ${dependency1.name}@1.0.0 with error:\nSomething went wrong\n`,
+        );
+      },
+    );
+
     describe("when the dependency is in the exclude list", () => {
       it("should not call resolveLicenseContent", async () => {
         mockedGetPnpmVersion.mockResolvedValue(pnpmVersion);