Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Move k8s goal launching into extension pack
[changelog:changed]
  • Loading branch information
cdupuis committed Jan 21, 2019
1 parent 786feb1 commit ccb6fbc
Show file tree
Hide file tree
Showing 7 changed files with 147 additions and 5 deletions.
11 changes: 11 additions & 0 deletions index.ts
Expand Up @@ -136,3 +136,14 @@ export {
} from "./lib/internal/preferences/AbstractPreferenceStore";
export { FilePreferenceStoreFactory } from "./lib/internal/preferences/FilePreferenceStore";
export { GraphQLPreferenceStoreFactory } from "./lib/internal/preferences/GraphQLPreferenceStore";
export {
KubernetesGoalScheduler,
sanitizeName,
isConfiguredInEnv,
} from "./lib/pack/k8s/KubernetesGoalScheduler";
export {
KubernetesJobDeletingGoalCompletionListenerFactory,
} from "./lib/pack/k8s/KubernetesJobDeletingGoalCompletionListener";
export {
goalScheduling,
} from "./lib/pack/k8s/goalScheduling";
3 changes: 1 addition & 2 deletions lib/machine/defaultSoftwareDeliveryMachineConfiguration.ts
Expand Up @@ -23,7 +23,6 @@ import {
import * as _ from "lodash";
import { DefaultRepoRefResolver } from "../handlers/common/DefaultRepoRefResolver";
import { GitHubCredentialsResolver } from "../handlers/common/GitHubCredentialsResolver";
import { KubernetesGoalScheduler } from "../handlers/events/delivery/goals/k8s/KubernetesGoalScheduler";
import { EphemeralLocalArtifactStore } from "../internal/artifact/local/EphemeralLocalArtifactStore";
import { LocalSoftwareDeliveryMachineConfiguration } from "../internal/machine/LocalSoftwareDeliveryMachineOptions";
import { GraphQLPreferenceStoreFactory } from "../internal/preferences/GraphQLPreferenceStore";
Expand All @@ -44,7 +43,7 @@ export function defaultSoftwareDeliveryMachineConfiguration(configuration: Confi
repoRefResolver,
repoFinder: allReposInTeam(repoRefResolver),
projectPersister: RemoteGitProjectPersister,
goalScheduler: [new KubernetesGoalScheduler()],
goalScheduler: [],
preferenceStoreFactory: GraphQLPreferenceStoreFactory,
},
local: {
Expand Down
4 changes: 2 additions & 2 deletions lib/machine/machineFactory.ts
Expand Up @@ -62,8 +62,8 @@ import { exposeInfo } from "../pack/info/exposeInfo";
* ```
*/
export function createSoftwareDeliveryMachine(config: MachineConfiguration<SoftwareDeliveryMachineConfiguration>,
// tslint:disable-next-line:max-line-length
...goalSetters: Array<GoalSetter | GoalSetter[]>): SoftwareDeliveryMachine<SoftwareDeliveryMachineConfiguration> {
...goalSetters: Array<GoalSetter | GoalSetter[]>)
: SoftwareDeliveryMachine<SoftwareDeliveryMachineConfiguration> {
const machine = new HandlerBasedSoftwareDeliveryMachine(config.name, config.configuration,
goalSetters);
machine.addExtensionPacks(exposeInfo());
Expand Down
Expand Up @@ -152,7 +152,7 @@ export class KubernetesGoalScheduler implements GoalScheduler {
}

private init(): void {
if (cluster.isMaster && isConfiguredInEnv("kubernetes", "kubernetes-all")) {
if (cluster.isMaster) {
setInterval(() => {
return this.cleanUp()
.then(() => {
Expand Down
108 changes: 108 additions & 0 deletions lib/pack/k8s/KubernetesJobDeletingGoalCompletionListener.ts
@@ -0,0 +1,108 @@
/*
* Copyright © 2019 Atomist, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

import { logger } from "@atomist/automation-client";
import {
GoalCompletionListener,
SdmGoalState,
SoftwareDeliveryMachine,
} from "@atomist/sdm";
import * as k8s from "@kubernetes/client-node";
import * as _ from "lodash";
import { loadKubeConfig } from "./k8config";
import { sanitizeName } from "./KubernetesGoalScheduler";

/**
* GoalCompletionListener that puts completed goal jobs into a ttl cache for later deletion.
*/
export class KubernetesJobDeletingGoalCompletionListenerFactory {

private readonly cache: Map<string, { ttl: number, name: string, namespace: string }> = new Map();
private readonly batch: k8s.Batch_v1Api;

constructor(private readonly sdm: SoftwareDeliveryMachine) {
const kc = loadKubeConfig();
this.batch = kc.makeApiClient(k8s.Batch_v1Api);

this.init();
}

public create(): GoalCompletionListener {
return async gi => {
const goalEvent = gi.completedGoal;

if (goalEvent.state === SdmGoalState.in_process) {
return;
}

const selector = `goalSetId=${goalEvent.goalSetId},creator=${sanitizeName(this.sdm.configuration.name)}`;

const jobs = await this.batch.listJobForAllNamespaces(
undefined,
undefined,
undefined,
selector);

const goalJobs = jobs.body.items.filter(j => {
const annotations = j.metadata.annotations;
if (!!annotations && !!annotations["atomist.com/sdm"]) {
const sdmAnnotation = JSON.parse(annotations["atomist.com/sdm"]);
return sdmAnnotation.goal.uniqueName === goalEvent.uniqueName;
}
return false;
});

const ttl: number = _.get(this.sdm.configuration, "sdm.kubernetes.job.ttl", 1000 * 60 * 2);

for (const goalJob of goalJobs) {
this.cache.set(
goalJob.metadata.uid,
{
ttl: Date.now() + ttl,
name: goalJob.metadata.name,
namespace: goalJob.metadata.namespace,
});
}
};
}

private init(): void {
setInterval(async () => {
const now = Date.now();
for (const uid of this.cache.keys()) {
const job = this.cache.get(uid);
if (job.ttl <= now) {
logger.debug(`Deleting k8s job '${job.namespace}:${job.name}'`);
try {
await this.batch.readNamespacedJob(job.name, job.namespace);
try {
await this.batch.deleteNamespacedJob(
job.name,
job.namespace,
{ propagationPolicy: "Background" } as any);
} catch (e) {
logger.warn(`Failed to delete k8s jobs '${job.namespace}:${job.name}': ${e.message}`);
}
} catch (e) {
// This is ok to ignore because the job doesn't exist any more
}
this.cache.delete(uid);
}
}
},
_.get(this.sdm.configuration, "sdm.kubernetes.job.ttlCheckInterval", 15000));
}
}
24 changes: 24 additions & 0 deletions lib/pack/k8s/goalScheduling.ts
@@ -0,0 +1,24 @@
import {
ExtensionPack,
metadata,
} from "@atomist/sdm";
import {
isConfiguredInEnv,
KubernetesGoalScheduler,
} from "./KubernetesGoalScheduler";
import { KubernetesJobDeletingGoalCompletionListenerFactory } from "./KubernetesJobDeletingGoalCompletionListener";

/**
* Extension pack to schedule goals as k8s jobs when marked as isolated = true.
*/
export function goalScheduling(): ExtensionPack {
return {
...metadata("k8s-goal-scheduling"),
configure: sdm => {
if (!process.env.ATOMIST_ISOLATED_GOAL && isConfiguredInEnv("kubernetes", "kubernetes-all")) {
sdm.addGoalCompletionListener(new KubernetesJobDeletingGoalCompletionListenerFactory(sdm).create());
sdm.configuration.sdm.goalScheduler = [new KubernetesGoalScheduler()];
}
},
};
}
File renamed without changes.

0 comments on commit ccb6fbc

Please sign in to comment.