Add container goal

Add container goal definition and working schedulers for Kubernetes
jobs and Docker CLI.  Use first container as goal container, which
means it is the only one waited on and its status sets the goal
status.  Will throw error if no containers defined.  It will do its
best to make the goal and fulfillment names unique.

Co-opt KubernetesGoalScheduler for k8s container goal implementation,
extracting some of its functionality into functions that can be used
to define the init container.  Collect logs from k8s into goal
progress log, adding a delay because log delivery from the k8s API can
occur after the container terminates.  Add tests for some scheduler
and environment variable functionality.

Add CacheEntry interface and do some minor clean up to caching
interface.  Correct goal caching restore interface to only require
options it uses.  Clean up cache tests.

Properly handle project directory and cache use by copying its
contents to a location usable by container, mounting that location in
the container, setting the working directory in the first container to
that location, and copying the contents back after the containers
exit.

Closes #162

index.ts

@@ -27,6 +27,26 @@ export {
     Version,
     ProjectVersionerRegistration,
 } from "./lib/goal/common/Version";
+export {
+    Container,
+    containerGoal,
+    ContainerPort,
+    ContainerProjectHome,
+    ContainerRegistration,
+    ContainerScheduler,
+    ContainerSpecCallback,
+    ContainerVolumeMount,
+    GoalContainer,
+    GoalContainerVolume,
+} from "./lib/goal/container/container";
+export {
+    DockerContainerRegistration,
+} from "./lib/goal/container/docker";
+export {
+    K8sContainerRegistration,
+    K8sContainerSpecCallback,
+    K8sGoalContainerSpec,
+} from "./lib/goal/container/k8s";
 export {
     DisplayDeployEnablement,
 } from "./lib/handlers/commands/DisplayDeployEnablement";

lib/goal/cache/CompressingGoalCache.ts

@@ -15,10 +15,8 @@
  */
 
 import {
-    GitProject,
     guid,
     LocalProject,
-    Project,
 } from "@atomist/automation-client";
 import {
     GoalInvocation,
@@ -65,17 +63,17 @@ export class CompressingGoalCache implements GoalCache {
         this.store = store;
     }
 
-    public async put(gi: GoalInvocation, project: GitProject, files: string[], classifier?: string): Promise<void> {
+    public async put(gi: GoalInvocation, project: LocalProject, files: string[], classifier?: string): Promise<void> {
         const archiveName = "atomist-cache";
         const teamArchiveFileName = path.join(project.baseDir, `${archiveName}.${guid().slice(0, 7)}`);
 
         await spawnLog("tar", ["-cf", teamArchiveFileName, ...files], {
             log: gi.progressLog,
-            cwd: (project as LocalProject).baseDir,
+            cwd: project.baseDir,
         });
         await spawnLog("gzip", ["-3", teamArchiveFileName], {
             log: gi.progressLog,
-            cwd: (project as LocalProject).baseDir,
+            cwd: project.baseDir,
         });
         await this.store.store(gi, classifier, teamArchiveFileName + ".gz");
     }
@@ -84,14 +82,14 @@ export class CompressingGoalCache implements GoalCache {
         await this.store.delete(gi, classifier);
     }
 
-    public async retrieve(gi: GoalInvocation, project: Project, classifier?: string): Promise<void> {
+    public async retrieve(gi: GoalInvocation, project: LocalProject, classifier?: string): Promise<void> {
         const archiveName = "atomist-cache";
-        const teamArchiveFileName = path.join((project as LocalProject).baseDir, `${archiveName}.${guid().slice(0, 7)}`);
+        const teamArchiveFileName = path.join(project.baseDir, `${archiveName}.${guid().slice(0, 7)}`);
         await this.store.retrieve(gi, classifier, teamArchiveFileName);
         if (fs.existsSync(teamArchiveFileName)) {
             await spawnLog("tar", ["-xzf", teamArchiveFileName], {
                 log: gi.progressLog,
-                cwd: (project as LocalProject).baseDir,
+                cwd: project.baseDir,
             });
         } else {
             throw Error("No cache entry");

lib/goal/cache/goalCaching.ts

@@ -79,25 +79,44 @@ export interface DirectoryPattern {
     directory: string;
 }
 
+export interface CacheEntry {
+    classifier: string;
+    pattern: GlobFilePattern | DirectoryPattern;
+}
+
 /**
- * Options for goal caching
+ * Core options for goal caching.
  */
-export interface GoalCacheOptions {
+export interface GoalCacheCoreOptions {
     /**
      * Optional push test on when to trigger caching
      */
     pushTest?: PushTest;
-    /**
-     * Collection of glob patterns with classifiers to determine which files need to be cached between
-     * goal invocations, possibly excluding paths using regular expressions.
-     */
-    entries: Array<{ classifier: string, pattern: GlobFilePattern | DirectoryPattern }>;
     /**
      * Optional listener functions that should be called when no cache entry is found.
      */
     onCacheMiss?: GoalProjectListenerRegistration | GoalProjectListenerRegistration[];
 }
 
+/**
+ * Options for putting goal cache entries.
+ */
+export interface GoalCacheOptions extends GoalCacheCoreOptions {
+    /**
+     * Collection of glob patterns with classifiers to determine which
+     * files need to be cached between goal invocations, possibly
+     * excluding paths using regular expressions.
+     */
+    entries: CacheEntry[];
+}
+
+/**
+ * Options for restoring goal cache entries.
+ */
+export interface GoalCacheRestoreOptions extends GoalCacheCoreOptions {
+    entries?: Array<{ classifier: string }>;
+}
+
 const DefaultGoalCache = new NoOpGoalCache();
 
 /**
@@ -158,7 +177,7 @@ async function pushTestSucceeds(pushTest: PushTest, gi: GoalInvocation, p: GitPr
     });
 }
 
-async function invokeCacheMissListeners(optsToUse: GoalCacheOptions,
+async function invokeCacheMissListeners(optsToUse: GoalCacheOptions | GoalCacheRestoreOptions,
                                         p: GitProject,
                                         gi: GoalInvocation,
                                         event: GoalProjectListenerEvent): Promise<void> {
@@ -173,7 +192,7 @@ async function invokeCacheMissListeners(optsToUse: GoalCacheOptions,
 
 export const NoOpGoalProjectListenerRegistration: GoalProjectListenerRegistration = {
     name: "NoOpListener",
-    listener: async () => {},
+    listener: async () => { },
     pushTest: AnyPush,
 };
 
@@ -184,15 +203,15 @@ export const NoOpGoalProjectListenerRegistration: GoalProjectListenerRegistratio
  * needs to be restored. If omitted, all classifiers defined in the options are restored.
  * @param classifiers Additional classifiers that need to be restored.
  */
-export function cacheRestore(options: GoalCacheOptions,
+export function cacheRestore(options: GoalCacheRestoreOptions,
                              classifier?: string,
                              ...classifiers: string[]): GoalProjectListenerRegistration {
     const allClassifiers = [];
     if (classifier) {
         allClassifiers.push(...[classifier, ...classifiers]);
     }
     const listenerName = `restoring ${classifier ? "caches: " + allClassifiers.join(",") : "caches"}`;
-    const optsToUse: GoalCacheOptions = {
+    const optsToUse: GoalCacheRestoreOptions = {
         onCacheMiss: NoOpGoalProjectListenerRegistration,
         ...options,
     };
@@ -207,7 +226,7 @@ export function cacheRestore(options: GoalCacheOptions,
                 if (allClassifiers.length > 0) {
                     classifiersToBeRestored.push(...allClassifiers);
                 } else {
-                    classifiersToBeRestored.push(...options.entries.map(entry => entry.classifier));
+                    classifiersToBeRestored.push(...optsToUse.entries.map(entry => entry.classifier));
                 }
                 for (const c of classifiersToBeRestored) {
                     try {
@@ -266,5 +285,5 @@ async function getFilePathsThroughPattern(project: Project, globPattern: string
 }
 
 function isCacheEnabled(gi: GoalInvocation): boolean {
-    return _.get(gi.configuration, "sdm.cache.enabled") || false;
+    return _.get(gi.configuration, "sdm.cache.enabled", false);
 }