Skip to content

Commit

Permalink
Add support for skill output
Browse files Browse the repository at this point in the history
  • Loading branch information
@cdupuis
cdupuis committed Feb 14, 2020
1 parent d51736d commit 42fd790
Show file tree
Hide file tree
Showing 39 changed files with 32,717 additions and 27,052 deletions.
44 changes: 10 additions & 34 deletions lib/goal/cache/CompressingGoalCache.ts
View file
Expand Up @@ -14,7 +14,6 @@
* limitations under the License.
*/

import { resolvePlaceholders } from "@atomist/automation-client/lib/configuration";
import { Deferred } from "@atomist/automation-client/lib/internal/util/Deferred";
import { guid } from "@atomist/automation-client/lib/internal/util/string";
import { GitProject } from "@atomist/automation-client/lib/project/git/GitProject";
Expand All @@ -25,7 +24,6 @@ import * as fs from "fs-extra";
import * as JSZip from "jszip";
import * as os from "os";
import * as path from "path";
import { resolvePlaceholder } from "../../machine/yaml/resolvePlaceholder";
import { FileSystemGoalCacheArchiveStore } from "./FileSystemGoalCacheArchiveStore";
import { GoalCache } from "./goalCaching";

Expand All @@ -36,7 +34,7 @@ export interface GoalCacheArchiveStore {
* @param classifier The classifier of the cache
* @param archivePath The path of the archive to be stored.
*/
store(gi: GoalInvocation, classifier: string, archivePath: string): Promise<void>;
store(gi: GoalInvocation, classifier: string, archivePath: string): Promise<string>;

/**
* Remove a compressed goal archive
Expand Down Expand Up @@ -69,7 +67,10 @@ export class CompressingGoalCache implements GoalCache {
private readonly method: CompressionMethod = CompressionMethod.TAR) {
}

public async put(gi: GoalInvocation, project: GitProject, files: string[], classifier?: string): Promise<void> {
public async put(gi: GoalInvocation,
project: GitProject,
files: string[],
classifier?: string): Promise<string> {
const archiveName = "atomist-cache";
const teamArchiveFileName = path.join(os.tmpdir(), `${archiveName}.${guid().slice(0, 7)}`);
const slug = `${gi.id.owner}/${gi.id.repo}`;
Expand All @@ -83,12 +84,12 @@ export class CompressingGoalCache implements GoalCache {
const tarResult = await spawnLog("tar", ["-cf", teamArchiveFileName, ...files], spawnLogOpts);
if (tarResult.code) {
gi.progressLog.write(`Failed to create tar archive '${teamArchiveFileName}' for ${slug}`);
return;
return undefined;
}
const gzipResult = await spawnLog("gzip", ["-3", teamArchiveFileName], spawnLogOpts);
if (gzipResult.code) {
gi.progressLog.write(`Failed to gzip tar archive '${teamArchiveFileName}' for ${slug}`);
return;
return undefined;
}
teamArchiveFileNameWithSuffix += ".gz";
} else if (this.method === CompressionMethod.ZIP) {
Expand Down Expand Up @@ -129,20 +130,17 @@ export class CompressingGoalCache implements GoalCache {
await defer.promise;
}
}
const resolvedClassifier = await resolveClassifierPath(classifier, gi);
await this.store.store(gi, resolvedClassifier, teamArchiveFileNameWithSuffix);
return this.store.store(gi, classifier, teamArchiveFileNameWithSuffix);
}

public async remove(gi: GoalInvocation, classifier?: string): Promise<void> {
const resolvedClassifier = await resolveClassifierPath(classifier, gi);
await this.store.delete(gi, resolvedClassifier);
await this.store.delete(gi, classifier);
}

public async retrieve(gi: GoalInvocation, project: GitProject, classifier?: string): Promise<void> {
const archiveName = "atomist-cache";
const teamArchiveFileName = path.join(os.tmpdir(), `${archiveName}.${guid().slice(0, 7)}`);
const resolvedClassifier = await resolveClassifierPath(classifier, gi);
await this.store.retrieve(gi, resolvedClassifier, teamArchiveFileName);
await this.store.retrieve(gi, classifier, teamArchiveFileName);
if (fs.existsSync(teamArchiveFileName)) {
if (this.method === CompressionMethod.TAR) {
await spawnLog("tar", ["-xzf", teamArchiveFileName], {
Expand All @@ -168,25 +166,3 @@ export class CompressingGoalCache implements GoalCache {
}

}

/**
* Interpolate information from goal invocation into the classifier.
*/
export async function resolveClassifierPath(classifier: string | undefined, gi: GoalInvocation): Promise<string> {
if (!classifier) {
return gi.context.workspaceId;
}
const wrapper = { classifier };
await resolvePlaceholders(wrapper, v => resolvePlaceholder(v, gi.goalEvent, gi, {}));
return gi.context.workspaceId + "/" + sanitizeClassifier(wrapper.classifier);
}

/**
* Sanitize classifier for use in path. Replace any characters
* which might cause problems on POSIX or MS Windows with "_",
* including path separators. Ensure resulting file is not "hidden".
*/
export function sanitizeClassifier(classifier: string): string {
return classifier.replace(/[^-.0-9A-Za-z_+]/g, "_")
.replace(/^\.+/, ""); // hidden
}
5 changes: 3 additions & 2 deletions lib/goal/cache/FileSystemGoalCacheArchiveStore.ts
View file
@@ -1,5 +1,5 @@
/*
* Copyright © 2019 Atomist, Inc.
* Copyright © 2020 Atomist, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -28,13 +28,14 @@ import { GoalCacheArchiveStore } from "./CompressingGoalCache";
export class FileSystemGoalCacheArchiveStore implements GoalCacheArchiveStore {
private static readonly archiveName: string = "cache.tar.gz";

public async store(gi: GoalInvocation, classifier: string, archivePath: string): Promise<void> {
public async store(gi: GoalInvocation, classifier: string, archivePath: string): Promise<string> {
const cacheDir = await FileSystemGoalCacheArchiveStore.getCacheDirectory(gi, classifier);
const archiveName = FileSystemGoalCacheArchiveStore.archiveName;
const archiveFileName = path.join(cacheDir, archiveName);
await spawnLog("mv", [archivePath, archiveFileName], {
log: gi.progressLog,
});
return archiveFileName;
}

public async delete(gi: GoalInvocation, classifier: string): Promise<void> {
Expand Down
5 changes: 3 additions & 2 deletions lib/goal/cache/NoOpGoalCache.ts
View file
@@ -1,5 +1,5 @@
/*
* Copyright © 2019 Atomist, Inc.
* Copyright © 2020 Atomist, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand All @@ -24,8 +24,9 @@ import { GoalCache } from "./goalCaching";
* Cache implementation that doesn't cache anything and will always trigger the fallback.
*/
export class NoOpGoalCache implements GoalCache {
public async put(gi: GoalInvocation, project: GitProject, files: string[], classifier?: string): Promise<void> {
public async put(gi: GoalInvocation, project: GitProject, files: string[], classifier?: string): Promise<string> {
logger.warn(`No-Op goal cache in use; no cache will be preserved!`);
return undefined;
}

public async remove(gi: GoalInvocation, classifier?: string): Promise<void> {
Expand Down
52 changes: 43 additions & 9 deletions lib/goal/cache/goalCaching.ts
View file
@@ -1,5 +1,5 @@
/*
* Copyright © 2019 Atomist, Inc.
* Copyright © 2020 Atomist, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand All @@ -14,6 +14,7 @@
* limitations under the License.
*/

import { resolvePlaceholders } from "@atomist/automation-client/lib/configuration";
import { DefaultExcludes } from "@atomist/automation-client/lib/project/fileGlobs";
import { GitProject } from "@atomist/automation-client/lib/project/git/GitProject";
import { Project } from "@atomist/automation-client/lib/project/Project";
Expand All @@ -27,6 +28,7 @@ import {
import { PushTest } from "@atomist/sdm/lib/api/mapping/PushTest";
import { AnyPush } from "@atomist/sdm/lib/api/mapping/support/commonPushTests";
import * as _ from "lodash";
import { resolvePlaceholder } from "../../machine/yaml/resolvePlaceholder";
import { toArray } from "../../util/misc/array";
import { CompressingGoalCache } from "./CompressingGoalCache";

Expand All @@ -46,8 +48,9 @@ export interface GoalCache {
* @param p The project where the files (or directories) reside.
* @param files The files (or directories) to be cached.
* @param classifier An optional classifier to identify the set of files (or directories to be cached).
* @param type An optional output type
*/
put(gi: GoalInvocation, p: GitProject, files: string | string[], classifier?: string): Promise<void>;
put(gi: GoalInvocation, p: GitProject, files: string | string[], classifier?: string): Promise<string>;

/**
* Retrieve files from the cache.
Expand Down Expand Up @@ -109,7 +112,7 @@ export interface GoalCacheOptions extends GoalCacheCoreOptions {
* files need to be cached between goal invocations, possibly
* excluding paths using regular expressions.
*/
entries: CacheEntry[];
entries: Array<CacheEntry & { type?: string }>;
}

/**
Expand Down Expand Up @@ -146,27 +149,33 @@ export function cachePut(options: GoalCacheOptions,
name: listenerName,
listener: async (p: GitProject,
gi: GoalInvocation): Promise<void | ExecuteGoalResult> => {
const { goalEvent } = gi;
if (!!isCacheEnabled(gi) && !process.env.ATOMIST_ISOLATED_GOAL_INIT) {
const cloneEntries = _.cloneDeep(entries);
const goalCache = cacheStore(gi);
for (const entry of entries) {
for (const entry of cloneEntries) {
const files = [];
if (isGlobFilePattern(entry.pattern)) {
files.push(...(await getFilePathsThroughPattern(p, entry.pattern.globPattern)));
} else if (isDirectoryPattern(entry.pattern)) {
files.push(entry.pattern.directory);
}
if (!_.isEmpty(files)) {
await goalCache.put(gi, p, files, entry.classifier);
const resolvedClassifier = await resolveClassifierPath(entry.classifier, gi);
const uri = await goalCache.put(gi, p, files, resolvedClassifier);
if (!!resolvedClassifier && !!uri) {
entry.classifier = resolvedClassifier;
(entry as any).uri = uri;
}
}
}

// Set outputs on the goal data
const { goalEvent } = gi;
const data = JSON.parse(goalEvent.data || "{}");
const newData = {
[CacheOutputGoalDataKey]: [
...(data[CacheOutputGoalDataKey] || []),
...entries,
...cloneEntries,
],
};
goalEvent.data = JSON.stringify({
Expand Down Expand Up @@ -198,6 +207,7 @@ async function pushTestSucceeds(pushTest: PushTest, gi: GoalInvocation, p: GitPr
context: gi.context,
preferences: gi.preferences,
credentials: gi.credentials,
skill: gi.skill,
});
}

Expand Down Expand Up @@ -259,7 +269,8 @@ export function cacheRestore(options: GoalCacheRestoreOptions,
const goalCache = cacheStore(gi);
for (const c of classifiersToBeRestored) {
try {
await goalCache.retrieve(gi, p, c);
const resolvedClassifier = await resolveClassifierPath(c, gi);
await goalCache.retrieve(gi, p, resolvedClassifier);
} catch (e) {
await invokeCacheMissListeners(optsToUse, p, gi, event);
}
Expand Down Expand Up @@ -320,7 +331,8 @@ export function cacheRemove(options: GoalCacheOptions,
const goalCache = cacheStore(gi);

for (const c of classifiersToBeRemoved) {
await goalCache.remove(gi, c);
const resolvedClassifier = await resolveClassifierPath(c, gi);
await goalCache.remove(gi, resolvedClassifier);
}
}
},
Expand All @@ -346,3 +358,25 @@ function isCacheEnabled(gi: GoalInvocation): boolean {
function cacheStore(gi: GoalInvocation): GoalCache {
return _.get(gi.configuration, "sdm.cache.store", DefaultGoalCache);
}

/**
* Interpolate information from goal invocation into the classifier.
*/
export async function resolveClassifierPath(classifier: string | undefined, gi: GoalInvocation): Promise<string> {
if (!classifier) {
return gi.context.workspaceId;
}
const wrapper = { classifier };
await resolvePlaceholders(wrapper, v => resolvePlaceholder(v, gi.goalEvent, gi, {}));
return gi.context.workspaceId + "/" + sanitizeClassifier(wrapper.classifier);
}

/**
* Sanitize classifier for use in path. Replace any characters
* which might cause problems on POSIX or MS Windows with "_",
* including path separators. Ensure resulting file is not "hidden".
*/
export function sanitizeClassifier(classifier: string): string {
return classifier.replace(/[^-.0-9A-Za-z_+]/g, "_")
.replace(/^\.+/, ""); // hidden
}
22 changes: 19 additions & 3 deletions lib/goal/container/k8s.ts
View file
Expand Up @@ -59,6 +59,7 @@ import {
import { SdmGoalState } from "../../typings/types";
import { toArray } from "../../util/misc/array";
import {
CacheEntry,
CacheOutputGoalDataKey,
cachePut,
cacheRestore,
Expand Down Expand Up @@ -374,7 +375,10 @@ export const scheduleK8sJob: ExecuteGoal = async gi => {
const schedulableGoalEvent = await k8sFulfillmentCallback(gi.goal as Container, containerReg)(goalEvent, gi);
const scheduleResult = await k8sScheduler.schedule({ ...gi, goalEvent: schedulableGoalEvent });
if (scheduleResult.code) {
return { ...scheduleResult, message: `Failed to schedule container goal ${uniqueName}: ${scheduleResult.message}` };
return {
...scheduleResult,
message: `Failed to schedule container goal ${uniqueName}: ${scheduleResult.message}`,
};
}
schedulableGoalEvent.state = SdmGoalState.in_process;
return schedulableGoalEvent;
Expand Down Expand Up @@ -526,15 +530,27 @@ export function executeK8sJob(): ExecuteGoal {
}
}

const cacheEntriesToPut = [
const cacheEntriesToPut: CacheEntry[] = [
...(registration.output || []),
...((gi.parameters || {})[CacheOutputGoalDataKey] || []),
];
if (cacheEntriesToPut.length > 0) {
try {
const project = GitCommandGitProject.fromBaseDir(id, projectDir, credentials, async () => {
});
const cp = cachePut({ entries: cacheEntriesToPut });
const cp = cachePut({
entries: cacheEntriesToPut.map(e => {
// Prevent the type on the entry to get passed along when goal actually failed
if (status.code !== 0) {
return {
classifier: e.classifier,
pattern: e.pattern,
};
} else {
return e;
}
}),
});
await cp.listener(project, gi, GoalProjectListenerEvent.after);
} catch (e) {
const message = `Failed to put cache output from container: ${e.message}`;
Expand Down

0 comments on commit 42fd790

Please sign in to comment.