Stream files during code transform

lib/api-helper/project/fileCopy.ts

@@ -23,7 +23,7 @@ import {
     SimpleProjectEditor,
 } from "@atomist/automation-client";
 import axios from "axios";
-import { CodeTransform } from "../../api/registration/CodeTransform";
+import {CodeTransform} from "../../api/registration/CodeTransform";
 
 /**
  * Add the downloaded content to the given project
@@ -43,6 +43,18 @@ export interface FileMapping {
     recipientPath: string;
 }
 
+export interface FileGlobMapping {
+    /**
+     * See https://github.com/gulpjs/glob-stream for implementation details
+     */
+    globPatterns: string[];
+
+    /**
+     * This recipientPath will only be prefixed verbatim to any path returned from the globs.
+     */
+    recipientPath?: string;
+}
+
 /**
  * Take the specified files from the donor project
  * @param {RemoteRepoRef} donorProjectId
@@ -74,3 +86,41 @@ export function copyFiles(donorProject: Project,
         return p;
     };
 }
+
+/**
+ * Take the specified files from the donor project
+ * @param {RemoteRepoRef} donorProjectId
+ * @param {FileGlobMapping} fileGlobMapping - treated as globs as defined in Project.streamFiles
+ * @return {SimpleProjectEditor}
+ */
+export function streamFilesFrom(donorProjectId: RemoteRepoRef,
+                                fileGlobMapping: FileGlobMapping): CodeTransform {
+    return async (p, i) => {
+        const donorProject = await GitCommandGitProject.cloned(i.credentials, donorProjectId);
+        return streamFiles(donorProject, fileGlobMapping)(p, i);
+    };
+}
+
+export function streamFiles(donorProject: Project,
+                            fileGlobMapping: FileGlobMapping): CodeTransform {
+    return async p => {
+        const fileStream = donorProject.streamFiles(...fileGlobMapping.globPatterns);
+
+        await new Promise((resolve, reject) => {
+            fileStream
+                .on("end", () => {
+                    logger.debug("end of file stream reached, using glob: ", fileGlobMapping);
+                    resolve();
+                })
+                .on("data", donorFile => {
+                    const newPath = (fileGlobMapping.recipientPath || "") + donorFile.path;
+                    p.addFileSync(newPath, donorFile.getContentSync());
+                    logger.log("silly", "file added: ", donorFile.path);
+                })
+                .on("error", e => {
+                    logger.warn("Error copying file: ", e);
+                    reject(e);
+                });
+        });
+    };
+}

test/api-helper/project/fileCopy.test.ts

@@ -22,10 +22,11 @@ import * as assert from "power-assert";
 import {
     copyFileFromUrl,
     copyFilesFrom,
+    FileGlobMapping,
+    streamFiles,
 } from "../../../lib/api-helper/project/fileCopy";
 
 describe("fileCopy", () => {
-
     it("should copy file from url", async () => {
         const recipient = InMemoryProject.of();
         await (copyFileFromUrl("https://raw.githubusercontent.com/spring-team/spring-rest-seed/master/pom.xml", "pom.xml"))(recipient, undefined);
@@ -48,4 +49,44 @@ describe("fileCopy", () => {
         assert(!(await recipient.getFile(filesToSteal[0].donorPath)));
         assert(!!(await recipient.getFile(filesToSteal[0].recipientPath)));
     }).timeout(5000);
+
+    it("should copy a subset of files from donor project according to glob", async () => {
+        const donorProject = InMemoryProject.of(
+            {path: "a/b/a", content: "a/b/a file"},
+            {path: "a/b/b", content: "a/b/b file"},
+            {path: "x/b/b", content: "x/b/b file"},
+        );
+
+        const filesToSteal: FileGlobMapping = { globPatterns: ["a/**"], recipientPath: "c/" };
+        const recipient = InMemoryProject.of();
+
+        await (streamFiles(donorProject, filesToSteal))(recipient, undefined);
+
+        assert(3 === (await donorProject.totalFileCount()));
+
+        assert(!!(await recipient.getFile("c/a/b/a")));
+        assert(!!(await recipient.getFile("c/a/b/b")));
+
+        assert(2 === (await recipient.totalFileCount()));
+    });
+
+    it("should copy a subset of files from donor project handling undefined recipient path", async () => {
+        const donorProject = InMemoryProject.of(
+            {path: "a/b/a", content: "a/b/a file"},
+            {path: "a/b/b", content: "a/b/b file"},
+            {path: "x/b/b", content: "x/b/b file"},
+        );
+
+        const filesToSteal: FileGlobMapping = { globPatterns: ["a/**"] };
+        const recipient = InMemoryProject.of();
+
+        await (streamFiles(donorProject, filesToSteal))(recipient, undefined);
+
+        assert(3 === (await donorProject.totalFileCount()));
+
+        assert(!!(await recipient.getFile("a/b/a")));
+        assert(!!(await recipient.getFile("a/b/b")));
+
+        assert(2 === (await recipient.totalFileCount()));
+    });
 });