Add support for git-hosted urls as sibling package dependencies (#1033)

Coauthored with @synaptiko

README.md

@@ -673,7 +673,6 @@ Running `lerna` without arguments will show all commands/options.
 - `commands.bootstrap.scope`: an array of globs that restricts which packages will be bootstrapped when running the `lerna bootstrap` command.
 - `packages`: Array of globs to use as package locations.
 
-
 ### Common `devDependencies`
 
 Most `devDependencies` can be pulled up to the root of a Lerna repo.
@@ -896,9 +895,9 @@ May also be configured in `lerna.json`:
 
 #### --use-workspaces
 
-Enables integration with [Yarn Workspaces](https://github.com/yarnpkg/rfcs/blob/master/implemented/0000-workspaces-install-phase-1.md) (available since yarn@0.27+).  
-The values in the array are the commands in which Lerna will delegate operation to Yarn (currently only bootstrapping).    
-If `--use-workspaces` is true then `packages` will be overridden by the value from `package.json/workspaces.`  
+Enables integration with [Yarn Workspaces](https://github.com/yarnpkg/rfcs/blob/master/implemented/0000-workspaces-install-phase-1.md) (available since yarn@0.27+).
+The values in the array are the commands in which Lerna will delegate operation to Yarn (currently only bootstrapping).
+If `--use-workspaces` is true then `packages` will be overridden by the value from `package.json/workspaces.`
 May also be configured in `lerna.json`:
 
 ```js
@@ -924,6 +923,63 @@ The root-level package.json must also include a `workspaces` array:
 This list is broadly similar to lerna's `packages` config (a list of globs matching directories with a package.json),
 except it does not support recursive globs (`"**"`, a.k.a. "globstars").
 
+#### --use-git-version
+
+Allow target versions of dependent packages to be written as [git hosted urls](https://github.com/npm/hosted-git-info) instead of a plain version number.
+If enabled, Lerna will attempt to extract and save the interpackage dependency versions from `package.json` files using git url-aware parser.
+
+Eg. assuming monorepo with 2 packages where `my-package-1` depends on `my-package-2`, `package.json` of `my-package-1` could be:
+```
+// packages/my-package-1/package.json
+{
+  name: "my-package-1",
+  version: "1.0.0",
+  bin: "bin.js",
+  dependencies: {
+    "my-package-2": "github:example-user/my-package-2#v1.0.0"
+  },
+  devDependencies: {
+    "my-dev-dependency": "^1.0.0"
+  },
+  peerDependencies: {
+    "my-peer-dependency": "^1.0.0"
+  }
+}
+```
+For the case above Lerna will read the version of `my-package-2` dependency as `1.0.0`.
+
+This allows packages to be distributed via git repos if eg. packages are private and [private npm repo is not an option](https://www.dotconferences.com/2016/05/fabien-potencier-monolithic-repositories-vs-many-repositories).
+
+Please note that using `--use-git-version`
+- is limited to urls with [`committish`](https://docs.npmjs.com/files/package.json#git-urls-as-dependencies) part present (ie. `github:example-user/my-package-2` is invalid)
+- requires `publish` command to be used with `--exact`
+
+May also be configured in `lerna.json`:
+```js
+{
+  ...
+  "useGitVersion": true
+}
+```
+
+#### --git-version-prefix
+
+Defines version prefix string (defaults to 'v') ignored when extracting version number from a commitish part of git url.
+Everything after the prefix will be considered a version.
+
+
+Eg. given `github:example-user/my-package-2#v1.0.0` and `gitVersionPrefix: 'v'` version will be read as `1.0.0`.
+
+Only used if `--use-git-version` is set to `true`.
+
+May also be configured in `lerna.json`:
+```js
+{
+  ...
+  "gitVersionPrefix": "v"
+}
+```
+
 #### --stream
 
 Stream output from child processes immediately, prefixed with the originating

package.json

@@ -46,6 +46,7 @@
     "glob-parent": "^3.1.0",
     "globby": "^6.1.0",
     "graceful-fs": "^4.1.11",
+    "hosted-git-info": "^2.5.0",
     "inquirer": "^3.2.2",
     "is-ci": "^1.0.10",
     "load-json-file": "^3.0.0",

src/Command.js

@@ -5,11 +5,13 @@ import log from "npmlog";
 import ChildProcessUtilities from "./ChildProcessUtilities";
 import FileSystemUtilities from "./FileSystemUtilities";
 import GitUtilities from "./GitUtilities";
+import GitVersionParser from "./GitVersionParser";
 import PackageUtilities from "./PackageUtilities";
 import Repository from "./Repository";
 import filterFlags from "./utils/filterFlags";
 import writeLogFile from "./utils/writeLogFile";
 import UpdatedPackagesCollector from "./UpdatedPackagesCollector";
+import VersionSerializer from "./VersionSerializer";
 
 // handle log.success()
 log.addLevel("success", 3001, { fg: "green", bold: true });
@@ -294,7 +296,8 @@ export default class Command {
   }
 
   runPreparations() {
-    const { scope, ignore, registry, since } = this.options;
+    const { rootPath, packageConfigs } = this.repository;
+    const { scope, ignore, registry, since, useGitVersion, gitVersionPrefix } = this.options;
 
     if (scope) {
       log.info("scope", scope);
@@ -309,10 +312,22 @@ export default class Command {
     }
 
     try {
-      this.repository.buildPackageGraph();
-      this.packages = this.repository.packages;
-      this.packageGraph = this.repository.packageGraph;
-      this.filteredPackages = PackageUtilities.filterPackages(this.packages, { scope, ignore });
+      const versionParser = useGitVersion && new GitVersionParser(gitVersionPrefix);
+      const packages = PackageUtilities.getPackages({ rootPath, packageConfigs });
+      const packageGraph = PackageUtilities.getPackageGraph(packages, false, versionParser);
+
+      if (useGitVersion) {
+        packages.forEach((pkg) => {
+          pkg.versionSerializer = new VersionSerializer({
+            graphDependencies: packageGraph.get(pkg.name).dependencies,
+            versionParser
+          });
+        });
+      }
+
+      this.packages = packages;
+      this.packageGraph = packageGraph;
+      this.filteredPackages = PackageUtilities.filterPackages(packages, { scope, ignore });
 
       // The UpdatedPackagesCollector requires that filteredPackages be present prior to checking for
       // updates. That's okay because it further filters based on what's already been filtered.

src/GitVersionParser.js

@@ -0,0 +1,22 @@
+import { escapeRegExp } from "lodash";
+import hostedGitInfo from "hosted-git-info";
+
+export default class GitVersionParser {
+  constructor(versionPrefix = "v") {
+    this._gitUrlPattern = new RegExp(`(.+?#${escapeRegExp(versionPrefix)})(.+)$`);
+  }
+
+  parseVersion(version) {
+    const gitInfo = hostedGitInfo.fromUrl(version);
+    let targetMatches;
+
+    if (gitInfo && gitInfo.committish) {
+      targetMatches = this._gitUrlPattern.exec(version);
+    }
+
+    return {
+      prefix: targetMatches ? targetMatches[1] : null,
+      version: targetMatches ? targetMatches[2] : version
+    };
+  }
+}

src/Package.js

@@ -2,6 +2,7 @@ import dedent from "dedent";
 import log from "npmlog";
 import path from "path";
 import semver from "semver";
+import _ from "lodash";
 
 import dependencyIsSatisfied from "./utils/dependencyIsSatisfied";
 import NpmUtilities from "./NpmUtilities";
@@ -60,12 +61,21 @@ export default class Package {
     return this._package.scripts || {};
   }
 
+  set versionSerializer(versionSerializer) {
+    this._versionSerializer = versionSerializer;
+
+    if (versionSerializer) {
+      this._package = versionSerializer.deserialize(this._package);
+    }
+  }
+
   isPrivate() {
     return !!this._package.private;
   }
 
   toJSON() {
-    return this._package;
+    const pkg = _.cloneDeep(this._package);
+    return this._versionSerializer ? this._versionSerializer.serialize(pkg) : pkg;
   }
 
   /**

src/PackageGraph.js

@@ -20,7 +20,7 @@ export class PackageGraphNode {
  *    devDependencies that would normally be included.
  */
 export default class PackageGraph {
-  constructor(packages, depsOnly = false) {
+  constructor(packages, depsOnly = false, versionParser) {
     this.nodes = [];
     this.nodesByName = {};
 
@@ -38,11 +38,17 @@ export default class PackageGraph {
 
       for (let d = 0; d < depNames.length; d++) {
         const depName = depNames[d];
-        const depVersion = dependencies[depName];
         const packageNode = this.nodesByName[depName];
 
-        if (packageNode && semver.satisfies(packageNode.package.version, depVersion)) {
-          node.dependencies.push(depName);
+        if (packageNode) {
+          const depVersion = (versionParser
+            ? versionParser.parseVersion(dependencies[depName]).version
+            : dependencies[depName]
+          );
+
+          if (semver.satisfies(packageNode.package.version, depVersion)) {
+            node.dependencies.push(depName);
+          }
         }
       }
     }

src/PackageUtilities.js

@@ -87,8 +87,8 @@ export default class PackageUtilities {
     return packages;
   }
 
-  static getPackageGraph(packages, depsOnly) {
-    return new PackageGraph(packages, depsOnly);
+  static getPackageGraph(packages, depsOnly, versionParser) {
+    return new PackageGraph(packages, depsOnly, versionParser);
   }
 
   /**

src/Repository.js

@@ -8,7 +8,6 @@ import semver from "semver";
 
 import dependencyIsSatisfied from "./utils/dependencyIsSatisfied";
 import Package from "./Package";
-import PackageUtilities from "./PackageUtilities";
 
 const DEFAULT_PACKAGE_GLOB = "packages/*";
 
@@ -67,22 +66,6 @@ export default class Repository {
       .map(parentDir => path.resolve(this.rootPath, parentDir));
   }
 
-  get packages() {
-    if (!this._packages) {
-      this.buildPackageGraph();
-    }
-
-    return this._packages;
-  }
-
-  get packageGraph() {
-    if (!this._packageGraph) {
-      this.buildPackageGraph();
-    }
-
-    return this._packageGraph;
-  }
-
   get packageJson() {
     if (!this._packageJson) {
       try {
@@ -117,11 +100,6 @@ export default class Repository {
     return this.version === "independent";
   }
 
-  buildPackageGraph() {
-    this._packages = PackageUtilities.getPackages(this);
-    this._packageGraph = PackageUtilities.getPackageGraph(this._packages);
-  }
-
   hasDependencyInstalled(depName, version) {
     log.silly("hasDependencyInstalled", "ROOT", depName, version);
 

src/UpdatedPackagesCollector.js

@@ -37,7 +37,7 @@ export default class UpdatedPackagesCollector {
     this.logger = command.logger;
     this.repository = command.repository;
     this.packages = command.filteredPackages;
-    this.packageGraph = command.repository.packageGraph;
+    this.packageGraph = command.packageGraph;
     this.options = command.options;
   }
 

src/VersionSerializer.js

@@ -0,0 +1,46 @@
+export default class VersionSerializer {
+  constructor({ graphDependencies, versionParser }) {
+    this._graphDependencies = graphDependencies;
+    this._versionParser = versionParser;
+    this._dependenciesKeys = ["dependencies", "devDependencies", "peerDependencies"];
+    this._strippedPrefixes = new Map();
+  }
+
+  serialize(pkg) {
+    this._dependenciesKeys.forEach((key) => {
+      this._prependPrefix(pkg[key] || {});
+    });
+
+    return pkg;
+  }
+
+  deserialize(pkg) {
+    this._dependenciesKeys.forEach((key) => {
+      this._stripPrefix(pkg[key] || {});
+    });
+
+    return pkg;
+  }
+
+  _prependPrefix(dependencies) {
+    this._strippedPrefixes.forEach((prefix, name) => {
+      const version = dependencies[name];
+      if (version) {
+        dependencies[name] = `${prefix}${version}`;
+      }
+    });
+  }
+
+  _stripPrefix(dependencies) {
+    Object.keys(dependencies).forEach((name) => {
+      if (this._graphDependencies.includes(name)) {
+        const result = this._versionParser.parseVersion(dependencies[name]);
+
+        if (result.prefix) {
+          dependencies[name] = result.version;
+          this._strippedPrefixes.set(name, result.prefix);
+        }
+      }
+    });
+  }
+}

src/commands/PublishCommand.js

@@ -156,6 +156,13 @@ export default class PublishCommand extends Command {
     this.gitRemote = this.options.gitRemote || "origin";
     this.gitEnabled = !(this.options.canary || this.options.skipGit);
 
+    if (this.options.useGitVersion && !this.options.exact) {
+      throw new Error(dedent`
+        Using git version without 'exact' option is not recommended.
+        Please make sure you publish with --exact.
+      `);
+    }
+
     if (this.options.canary) {
       this.logger.info("canary", "enabled");
     }