Add "state save" and "state load" commands (#610)

* A first implementation of lockfile generation

* Add help for the lock command

* cleanup

* Feedback

- fix crash for libraries with no non-dev version
- output --lib instead of -L for compatibility and clarity
- move the command to the Misc category

* Allow people to work with dev versions without breaking haxelib lock output

* [mercurial] add support for cloning revisions, add impl for lock command

* [tests] restore mercurial tests

* [tests] update tests to work with mercurial too

* Rebuild run.n

* Server is still build with haxe 3.4.7, can't use final there...

* Wording

* Move lock command above deprecated commands

* Fix lock file generation

* Rename 'lock' command to 'pin generate'

Also write directly to file instead of stdout, and added 'pin apply' command as alias to 'install --always --skip-dependencies'

* Move pin commands above deprecated commands

* Pretend we're professionals by using corporate terminology

* Rebuild run.n

* Ok last rename, I swear

* Rebuild run.n

* Rebuild run.n

---------

Co-authored-by: k <k@klabz.org>

src/haxelib/VersionData.hx

@@ -42,6 +42,18 @@ class VcsData {
 	**/
 	@:optional
 	var subDir:Null<String>;
+
+	public function toString(): String {
+		var qualifier =
+			if (ref != null) ref
+			else if (tag != null) tag
+			else if (branch != null) branch
+			else null;
+		return if (qualifier != null)
+			'$url#$qualifier'
+		else
+			url;
+	}
 }
 
 /** Data required to reproduce a library version **/
@@ -84,4 +96,10 @@ class VersionDataHelper {
 			}
 		}
 	}
+
+	public static function toString(data: VersionData): String
+		return switch data {
+			case Haxelib(semver): semver;
+			case VcsInstall(vcsId, vcsData): '$vcsId:${vcsData.toString()}';
+		}
 }

src/haxelib/api/LibFlagData.hx

@@ -71,7 +71,7 @@ function fromHxml(path:String):List<{name:ProjectName, data:Option<VersionData>}
 	final libsData = new List<{name:ProjectName, data:Option<VersionData>}>();
 	final targetLibraries:Map<ProjectName, Null<Bool>> = [for (_ => lib in TARGETS) lib => null];
 
-	final lines = [path];
+	final lines = normalizeHxml(File.getContent(path)).split("\n");
 
 	while (lines.length > 0) {
 		final line = lines.shift().trim();

src/haxelib/api/RepoManager.hx

@@ -77,7 +77,7 @@ class RepoManager {
 
 		Returns the directory path if it is found, otherwise returns null.
 	**/
-	static function getLocalPath(dir:String):Null<String> {
+	public static function getLocalPath(dir:String):Null<String> {
 		if (dir == "")
 			return null;
 		final repo = Path.join([dir, LOCAL_REPO_DIR]);

src/haxelib/api/Repository.hx

@@ -1,5 +1,6 @@
 package haxelib.api;
 
+import haxelib.VersionData.VcsData;
 import sys.FileSystem;
 import sys.io.File;
 
@@ -470,4 +471,8 @@ class Repository {
 	function getDevFilePath(name:ProjectName):String {
 		return addToRepoPath(name, DEV_FILE);
 	}
+
+	public function getVcsData(name: ProjectName, version: VcsID): VcsData {
+		return Vcs.get(version).getReproducibleVersion(getProjectVersionPath(name, version));
+	}
 }

src/haxelib/api/Vcs.hx

@@ -21,6 +21,7 @@
  */
 package haxelib.api;
 
+import haxelib.VersionData.VcsData;
 import sys.FileSystem;
 import sys.thread.Thread;
 import sys.thread.Lock;
@@ -232,6 +233,8 @@ abstract class Vcs implements IVcs {
 	public abstract function clone(libPath:String, vcsPath:String, ?branch:String, ?version:String, ?debugLog:(msg:String)->Void):Void;
 
 	public abstract function update(?confirm:() -> Bool, ?debugLog:(msg:String) -> Void, ?summaryLog:(msg:String) -> Void):Bool;
+
+	public abstract function getReproducibleVersion(libPath: String): VcsData;
 }
 
 /** Class wrapping `git` operations. **/
@@ -347,6 +350,20 @@ class Git extends Vcs {
 		// return prev. cwd:
 		Sys.setCwd(oldCwd);
 	}
+
+	public function getReproducibleVersion(libPath: String): VcsData {
+		final oldCwd = Sys.getCwd();
+		Sys.setCwd(libPath);
+
+		final ret: VcsData = {
+			// could the remote's name not be "origin"?
+			url: run(["remote", "get-url", "origin"], true).out.trim(),
+			ref: run(["rev-parse", "HEAD"], true).out.trim(),
+		};
+
+		Sys.setCwd(oldCwd);
+		return ret;
+	}
 }
 
 /** Class wrapping `hg` operations. **/
@@ -407,17 +424,34 @@ class Mercurial extends Vcs {
 	public function clone(libPath:String, url:String, ?branch:String, ?version:String, ?debugLog:(msg:String)->Void):Void {
 		final vcsArgs = ["clone", url, libPath];
 
-		if (branch != null) {
+		if (branch != null && version != null) {
 			vcsArgs.push("--branch");
 			vcsArgs.push(branch);
-		}
 
-		if (version != null) {
 			vcsArgs.push("--rev");
 			vcsArgs.push(version);
+		} else if (branch != null) {
+			vcsArgs.push("--updaterev");
+			vcsArgs.push(branch);
+		} else if (version != null) {
+			vcsArgs.push("--updaterev");
+			vcsArgs.push(version);
 		}
 
 		if (run(vcsArgs, debugLog).code != 0)
 			throw VcsError.CantCloneRepo(this, url/*, ret.out*/);
 	}
+
+	public function getReproducibleVersion(libPath: String): VcsData {
+		final oldCwd = Sys.getCwd();
+		Sys.setCwd(libPath);
+
+		final ret: VcsData = {
+			url: run(["paths", "default"], true).out.trim(),
+			ref: run(["identify", "-i", "--debug"], true).out.trim(),
+		};
+
+		Sys.setCwd(oldCwd);
+		return ret;
+	}
 }

src/haxelib/client/Args.hx

@@ -52,6 +52,9 @@ enum abstract Command(String) to String {
 	final ConvertXml = "convertxml";
 	final Run = "run";
 	final Proxy = "proxy";
+	final StateSave = "state save";
+	final StateLoad = "state load";
+
 	// deprecated commands
 	final Local = "local";
 	final SelfUpdate = "selfupdate";
@@ -173,12 +176,18 @@ class Args {
 
 		validate(flags);
 
-		final commandStr = mainArgs.shift();
+		var commandStr = mainArgs.shift();
 		if (commandStr == null)
 			throw new InvalidCommand("No command specified");
 
-		final command = Command.ofString(commandStr);
-		if(command == null)
+		// Allow multi words commands
+		var command = Command.ofString(commandStr);
+		while (command == null && mainArgs.length > 0) {
+			commandStr += ' ' + mainArgs.shift();
+			command = Command.ofString(commandStr);
+		}
+
+		if (command == null)
 			throw new InvalidCommand('Unknown command $commandStr');
 
 		return {
@@ -276,6 +285,8 @@ class Args {
 		addCommand(Setup, "set the haxelib repository path", Miscellaneous);
 		addCommand(NewRepo, "create a new local repository", Miscellaneous);
 		addCommand(DeleteRepo, "delete the local repository", Miscellaneous);
+		addCommand(StateSave, "generate a manifest file to load with haxelib state load", Miscellaneous);
+		addCommand(StateLoad, "install libraries from manifest file", Miscellaneous);
 		addCommand(ConvertXml, "convert haxelib.xml file to haxelib.json", Miscellaneous);
 		addCommand(Run, "run the specified library with parameters", Miscellaneous);
 		addCommand(Proxy, "setup the Http proxy", Miscellaneous);

src/haxelib/client/Main.hx

@@ -21,6 +21,8 @@
  */
 package haxelib.client;
 
+import haxe.display.Server.ConfigurePrintParams;
+import haxelib.VersionData.VersionDataHelper;
 import haxe.crypto.Md5;
 import haxe.iterators.ArrayIterator;
 
@@ -229,6 +231,8 @@ class Main {
 			Proxy => create(proxy, 5, true),
 			#end
 			FixRepo => create(fixRepo, 0),
+			StateSave => create(stateSave, 1),
+			StateLoad => create(stateLoad, 1),
 			// deprecated commands
 			Local => create(local, 1, 'haxelib install <file>'),
 			SelfUpdate => create(updateSelf, 0, true, 'haxelib --global update $HAXELIB_LIBNAME'),
@@ -843,6 +847,66 @@ class Main {
 		Cli.print('Local repository deleted ($path)');
 	}
 
+	/** Outputs a manifest file with all the installed libraries and their current version **/
+	function stateSave() {
+		final manifest = getArgument("Manifest file to save");
+
+		final dir = haxe.io.Path.directory(manifest);
+		if (!sys.FileSystem.exists(dir)) sys.FileSystem.createDirectory(dir);
+		final out = sys.io.File.write(manifest);
+
+		if (RepoManager.getLocalPath(Sys.getCwd()) == null) {
+			Cli.printWarning("no local repository found, using global repository");
+		}
+
+		final scope = getScope();
+		final libraryInfo = scope.getArrayOfLibraryInfo();
+
+		// sort projects alphabetically
+		libraryInfo.sort(function(a, b) return Reflect.compare(a.name.toLowerCase(), b.name.toLowerCase()));
+
+		for (library in libraryInfo) {
+			if (library.devPath != null) {
+				Cli.printWarning('Version of "${library.name}" is "dev". The manifest file may not work properly in other environments!');
+			}
+
+			final version = try scope.getVersion(library.name) catch (e) {
+				Cli.printWarning(e.message);
+				continue;
+			};
+
+			var versionData: VersionData;
+			if (VcsID.isValid(version)) {
+				final vcsId = VcsID.ofString(version);
+				versionData = VcsInstall(
+					vcsId,
+					scope.repository.getVcsData(library.name, vcsId)
+				);
+			} else {
+				versionData = Haxelib(SemVer.ofString(version));
+			}
+			final versionStr = VersionDataHelper.toString(versionData);
+			out.writeString('-lib ${library.name}:$versionStr\n');
+		}
+
+		out.close();
+	}
+
+	/** Reads a manifest file and install all specified library versions **/
+	function stateLoad() {
+		final manifest = getArgument("Manifest file to load");
+		final scope = getScope();
+		final installer = setupAndGetInstaller(scope);
+		Cli.defaultAnswer = Always;
+		Installer.skipDependencies = true;
+
+		if (sys.FileSystem.exists(manifest) && !sys.FileSystem.isDirectory(manifest)) {
+			return installer.installFromHxml(manifest, confirmHxmlInstall);
+		}
+
+		throw 'Expected a manifest file.';
+	}
+
 	// ----------------------------------
 
 	static function main() {

test/HaxelibTests.hx

@@ -47,14 +47,13 @@ class HaxelibTests {
 
 		final isCI = Sys.getEnv("CI") != null;
 
-		// The test repo https://bitbucket.org/fzzr/hx.signal is gone.
-		// if (isCI || cmdSucceed("hg", ["version"])) {
-		// 	// Hg impl. suports tags & revs. Here "78edb4b" is a first revision "initial import" at that repo:
-		// 	TestHg.init();
-		// 	r.add(new TestHg());
-		// } else {
-		// 	Sys.println("hg not found.");
-		// }
+		if (isCI || cmdSucceed("hg", ["version"])) {
+			// Hg impl. suports tags & revs. Here "b022617bccfb" is a first revision at that repo:
+			TestHg.init();
+			r.add(new TestHg());
+		} else {
+			Sys.println("hg not found.");
+		}
 		if (isCI || cmdSucceed("git", ["version"])) {
 			// Git impl. suports only tags. Here "0.9.2" is a first revision too ("initial import"):
 			TestGit.init();

test/tests/TestGit.hx

@@ -12,6 +12,6 @@ class TestGit extends TestVcs {
 	}
 
 	public function new():Void {
-		super(VcsID.Git, "Git", FileSystem.fullPath(REPO_PATH), "0.9.2");
+		super(VcsID.Git, "Git", FileSystem.fullPath(REPO_PATH), "develop", "0.9.2");
 	}
 }

test/tests/TestHg.hx

@@ -1,17 +1,17 @@
 package tests;
 
 import sys.FileSystem;
-import haxelib.api.Vcs;
+import haxelib.VersionData.VcsID;
 
 class TestHg extends TestVcs {
 	static final REPO_PATH = 'test/repo/hg';
 
 	static public function init() {
 		HaxelibTests.deleteDirectory(REPO_PATH);
-		HaxelibTests.runCommand('hg', ['clone', 'https://bitbucket.org/fzzr/hx.signal', REPO_PATH]);
+		HaxelibTests.runCommand('hg', ['clone', 'http://hg.code.sf.net/p/hx-signal/mercurial', REPO_PATH]);
 	}
 
 	public function new():Void {
-		super(VcsID.Hg, "Mercurial", FileSystem.fullPath(REPO_PATH), "78edb4b");
+		super(VcsID.Hg, "Mercurial", FileSystem.fullPath(REPO_PATH), "default", "b022617bccfb");
 	}
 }

test/tests/TestVcs.hx

@@ -19,15 +19,17 @@ class TestVcs extends TestBase
 	final id:VcsID = null;
 	final vcsName:String = null;
 	final url:String = null;
+	final branch:String = null;
 	final rev:String = null;
 	var counter:Int = 0;
 
 	//--------------- constructor ---------------//
 
-	public function new(id:VcsID, vcsName:String, url:String, ?rev:String) {
+	public function new(id:VcsID, vcsName:String, url:String, ?branch:String, ?rev:String) {
 		super();
 		this.id = id;
 		this.url = url;
+		this.branch = branch;
 		this.rev = rev;
 		this.vcsName = vcsName;
 
@@ -92,7 +94,7 @@ class TestVcs extends TestBase
 	public function testCloneBranch():Void {
 		final vcs = getVcs();
 		final dir = vcs.directory + counter++;
-		vcs.clone(dir, url, "develop");
+		vcs.clone(dir, url, branch);
 
 		assertTrue(FileSystem.exists(dir));
 		assertTrue(FileSystem.isDirectory(dir));
@@ -104,7 +106,7 @@ class TestVcs extends TestBase
 	public function testCloneBranchTag_0_9_2():Void {
 		final vcs = getVcs();
 		final dir = vcs.directory + counter++;
-		vcs.clone(dir, url, "develop", "0.9.2");
+		vcs.clone(dir, url, branch, "0.9.2");
 		Sys.sleep(3);
 		assertTrue(FileSystem.exists(dir));
 		assertTrue(FileSystem.exists('$dir/.${vcs.directory}'));
@@ -116,7 +118,7 @@ class TestVcs extends TestBase
 	public function testCloneBranchTag_0_9_3():Void {
 		final vcs = getVcs();
 		final dir = vcs.directory + counter++;
-		vcs.clone(dir, url, "develop", "0.9.3");
+		vcs.clone(dir, url, branch, "0.9.3");
 
 		assertTrue(FileSystem.exists(dir));
 		assertTrue(FileSystem.exists('$dir/.${vcs.directory}'));
@@ -128,7 +130,7 @@ class TestVcs extends TestBase
 	public function testCloneBranchRev():Void {
 		final vcs = getVcs();
 		final dir = vcs.directory + counter++;
-		vcs.clone(dir, url, "develop", rev);
+		vcs.clone(dir, url, branch, rev);
 
 		assertTrue(FileSystem.exists(dir));
 		assertTrue(FileSystem.exists('$dir/.${vcs.directory}'));