feat(gh-18): support extracting python library dependencies (#237)

extractor/src/extractors/ExtractorRegistry.ts

@@ -12,6 +12,8 @@ import PackageJsonExtractor from "./PackageJsonExtractor";
 import PomXmlExtractor from "./PomXmlExtractor";
 import VendorConfExtractor from "./VendorConfExtractor";
 import JsonnetfileJsonExtractor from "./JsonnetfileJsonExtractor";
+import PipfileExtractor from "./PipfileExtractor";
+import RequirementsTxtExtractor from "./RequirementsTxtExtractor";
 
 const ExtractorRegistry = new Registry<Extractor>("Extractor");
 
@@ -28,6 +30,8 @@ ExtractorRegistry.registerAll({
     "bower.json": async () => new BowerJsonExtractor(),
     "vendor.conf": async () => new VendorConfExtractor(),
     "jsonnetfile.json": async() => new JsonnetfileJsonExtractor(),
+    "Pipfile": async() => new PipfileExtractor(),
+    "requirements.txt": async() => new RequirementsTxtExtractor(),
 });
 
 export default ExtractorRegistry;

extractor/src/extractors/GopkgTomlExtractor.ts

@@ -1,6 +1,6 @@
 import Extractor from "./Extractor";
 import ExtractorFile from "./ExtractorFile";
-import inferImportPath from "./goutils/inferImportPath";
+import inferGoImportPath from "./utils/inferGoImportPath";
 import Languages from "./Languages";
 import MatchConfig from "../matcher/MatchConfig";
 import {ManifestDependency, ManifestFile} from "@depscloud/api/v1beta";
@@ -57,7 +57,7 @@ export default class GopkgTomlExtractor implements Extractor {
     }
 
     public async extract(url: string, files: { [p: string]: ExtractorFile }): Promise<ManifestFile> {
-        const name = inferImportPath(url);
+        const name = inferGoImportPath(url);
 
         const toml = files["Gopkg.toml"].toml();
 

extractor/src/extractors/Languages.ts

@@ -25,7 +25,11 @@ const Languages = {
 
     get JSONNET() : string {
         return "jsonnet";
-    }
+    },
+
+    get PYTHON() : string {
+        return "python";
+    },
 };
 
 export default Languages;

extractor/src/extractors/PipfileExtractor.test.ts

@@ -0,0 +1,21 @@
+import {readFile} from "fs";
+import {promisify} from "util";
+import ExtractorFile from "./ExtractorFile";
+import PipfileExtractor from "./PipfileExtractor";
+
+const readFileAsync = promisify(readFile);
+
+describe("PipfileExtractor", () => {
+    test("fullParse", async () => {
+        const filePath = require.resolve("./testdata/Pipfile");
+        const buffer = await readFileAsync(filePath);
+        const content = buffer.toString();
+
+        const parser = new PipfileExtractor();
+
+        const actual = await parser.extract("", { "Pipfile": new ExtractorFile(content) });
+
+        expect(actual).toMatchSnapshot();
+        expect(JSON.stringify(actual, null, 2)).toMatchSnapshot();
+    });
+});

extractor/src/extractors/PipfileExtractor.ts

@@ -0,0 +1,71 @@
+import Extractor from "./Extractor";
+import ExtractorFile from "./ExtractorFile";
+import MatchConfig from "../matcher/MatchConfig";
+import {ManifestDependency, ManifestFile} from "@depscloud/api/v1beta";
+import Languages from "./Languages";
+import inferPythonName from "./utils/inferPythonName";
+
+const transform = (scope: string, name: string, val: any): ManifestDependency => {
+    let versionConstraint = "*";
+    if (typeof val == "string") {
+        versionConstraint = val;
+    } else if (val.version) {
+        versionConstraint = val.version;
+    } else if (val.ref) { // git
+        versionConstraint = val.ref;
+    }
+
+    return {
+        name,
+        versionConstraint,
+        scopes: [ scope ],
+    }
+}
+
+export default class PipfileExtractor implements Extractor {
+    public async extract(url: string, files: { [p: string]: ExtractorFile }): Promise<ManifestFile> {
+        const {
+            source,
+            packages,
+            "dev-packages": devPackages,
+        } = files["Pipfile"].toml();
+
+        let [ name, sourceUrl ] = [ "", "" ];
+        if (source) {
+            name = source[0].name;
+            sourceUrl = source[0].url;
+        } else {
+            name = inferPythonName(url);
+        }
+
+        const dependencies = [];
+
+        Object.keys(packages)
+            .map((key) => transform("", key, packages[key]))
+            .forEach((dep) => dependencies.push(dep));
+
+        Object.keys(devPackages)
+            .map((key) => transform("dev", key, devPackages[key]))
+            .forEach((dep) => dependencies.push(dep));
+
+        return {
+            language: Languages.PYTHON,
+            system: "pipfile",
+            name,
+            version: "*",
+            sourceUrl,
+            dependencies,
+        };
+    }
+
+    public matchConfig(): MatchConfig {
+        return {
+            includes: [ "**/Pipfile" ],
+            excludes: [],
+        };
+    }
+
+    public requires(): string[] {
+        return [ "Pipfile" ];
+    }
+}

extractor/src/extractors/RequirementsTxtExtractor.test.ts

@@ -0,0 +1,22 @@
+import {readFile} from "fs";
+import {promisify} from "util";
+import ExtractorFile from "./ExtractorFile";
+import RequirementsTxtExtractor from "./RequirementsTxtExtractor";
+
+const readFileAsync = promisify(readFile);
+
+describe("RequirementsTxtExtractor", () => {
+    test("fullParse", async () => {
+        const filePath = require.resolve("./testdata/requirements.txt");
+        const buffer = await readFileAsync(filePath);
+        const content = buffer.toString();
+
+        const parser = new RequirementsTxtExtractor();
+
+        const url = "git@example.com:common/python-web.git";
+        const actual = await parser.extract(url, { "requirements.txt": new ExtractorFile(content) });
+
+        expect(actual).toMatchSnapshot();
+        expect(JSON.stringify(actual, null, 2)).toMatchSnapshot();
+    });
+});

extractor/src/extractors/RequirementsTxtExtractor.ts

@@ -0,0 +1,70 @@
+import Extractor from "./Extractor";
+import ExtractorFile from "./ExtractorFile";
+import MatchConfig from "../matcher/MatchConfig";
+import {ManifestFile} from "@depscloud/api/v1beta";
+import Languages from "./Languages";
+import inferPythonName from "./utils/inferPythonName";
+import {ManifestDependency} from "@depscloud/api/depscloud/api/v1beta/manifest";
+
+const operators = [
+    "~", "=", "!", ">", "<"
+];
+
+export default class RequirementsTxtExtractor implements Extractor {
+    public async extract(url: string, files: { [p: string]: ExtractorFile }): Promise<ManifestFile> {
+        const dependencies: Array<ManifestDependency> = [];
+
+        const contents = files["requirements.txt"].raw();
+        const lines = contents.split("\n");
+
+        lines.forEach((line) => {
+            const commentIndex = line.indexOf("#");
+            if (commentIndex > -1) {
+                line = line.substring(0, commentIndex);
+            }
+
+            line = line.trim()
+
+            if (!line || line[0] === "-") {
+                return
+            }
+
+            let l = line.length;
+            for (const operator of operators) {
+                const i = line.indexOf(operator);
+                if (i > -1) {
+                    l = Math.min(l, i);
+                }
+            }
+
+            const name = line.substring(0, l).trim();
+            const versionConstraint = line.substring(l).trim();
+
+            dependencies.push({
+                name,
+                versionConstraint: (versionConstraint || "*"),
+                scopes: [""],
+            });
+        });
+
+        return {
+            language: Languages.PYTHON,
+            system: "pip",
+            sourceUrl: "",
+            name: inferPythonName(url),
+            version: "*",
+            dependencies,
+        };
+    }
+
+    public matchConfig(): MatchConfig {
+        return {
+            includes: [ "**/requirements.txt" ],
+            excludes: [],
+        };
+    }
+
+    public requires(): string[] {
+        return [ "requirements.txt" ];
+    }
+}

extractor/src/extractors/__snapshots__/PipfileExtractor.test.ts.snap

@@ -0,0 +1,137 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`PipfileExtractor fullParse 1`] = `
+Object {
+  "dependencies": Array [
+    Object {
+      "name": "requests",
+      "scopes": Array [
+        "",
+      ],
+      "versionConstraint": "*",
+    },
+    Object {
+      "name": "records",
+      "scopes": Array [
+        "",
+      ],
+      "versionConstraint": ">0.5.0",
+    },
+    Object {
+      "name": "django",
+      "scopes": Array [
+        "",
+      ],
+      "versionConstraint": "1.11.4",
+    },
+    Object {
+      "name": "e682b37",
+      "scopes": Array [
+        "",
+      ],
+      "versionConstraint": "*",
+    },
+    Object {
+      "name": "e1839a8",
+      "scopes": Array [
+        "",
+      ],
+      "versionConstraint": "*",
+    },
+    Object {
+      "name": "pywinusb",
+      "scopes": Array [
+        "",
+      ],
+      "versionConstraint": "*",
+    },
+    Object {
+      "name": "nose",
+      "scopes": Array [
+        "dev",
+      ],
+      "versionConstraint": "*",
+    },
+    Object {
+      "name": "unittest2",
+      "scopes": Array [
+        "dev",
+      ],
+      "versionConstraint": ">=1.0,<3.0",
+    },
+  ],
+  "language": "python",
+  "name": "pypi",
+  "sourceUrl": "https://pypi.python.org/simple",
+  "system": "pipfile",
+  "version": "*",
+}
+`;
+
+exports[`PipfileExtractor fullParse 2`] = `
+"{
+  \\"language\\": \\"python\\",
+  \\"system\\": \\"pipfile\\",
+  \\"name\\": \\"pypi\\",
+  \\"version\\": \\"*\\",
+  \\"sourceUrl\\": \\"https://pypi.python.org/simple\\",
+  \\"dependencies\\": [
+    {
+      \\"name\\": \\"requests\\",
+      \\"versionConstraint\\": \\"*\\",
+      \\"scopes\\": [
+        \\"\\"
+      ]
+    },
+    {
+      \\"name\\": \\"records\\",
+      \\"versionConstraint\\": \\">0.5.0\\",
+      \\"scopes\\": [
+        \\"\\"
+      ]
+    },
+    {
+      \\"name\\": \\"django\\",
+      \\"versionConstraint\\": \\"1.11.4\\",
+      \\"scopes\\": [
+        \\"\\"
+      ]
+    },
+    {
+      \\"name\\": \\"e682b37\\",
+      \\"versionConstraint\\": \\"*\\",
+      \\"scopes\\": [
+        \\"\\"
+      ]
+    },
+    {
+      \\"name\\": \\"e1839a8\\",
+      \\"versionConstraint\\": \\"*\\",
+      \\"scopes\\": [
+        \\"\\"
+      ]
+    },
+    {
+      \\"name\\": \\"pywinusb\\",
+      \\"versionConstraint\\": \\"*\\",
+      \\"scopes\\": [
+        \\"\\"
+      ]
+    },
+    {
+      \\"name\\": \\"nose\\",
+      \\"versionConstraint\\": \\"*\\",
+      \\"scopes\\": [
+        \\"dev\\"
+      ]
+    },
+    {
+      \\"name\\": \\"unittest2\\",
+      \\"versionConstraint\\": \\">=1.0,<3.0\\",
+      \\"scopes\\": [
+        \\"dev\\"
+      ]
+    }
+  ]
+}"
+`;