Skip to content

Commit

Permalink
Merge pull request #22 from KleeUT/mainandmaster
Browse files Browse the repository at this point in the history 
❇️ adds protection to main branch as well as master
  • Loading branch information
@angelling11
angelling11 committed Dec 30, 2020
2 parents fe75aef + f4191c8 commit bbb1091
Show file tree
Hide file tree
Showing 3 changed files with 198 additions and 151 deletions.
4 changes: 2 additions & 2 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ Images stored in ECR incur monthly data storage charges, this means paying to st

Per Lambda invocation:

- 100 images that are older than 30 days and that do not have tags that contain 'master'
- 100 images that are older than 30 days and that do not have tags that contain 'master' or 'main'

If you need to delete more than 100 images, rather than complicating this script so that it can paginate
through all pages of images, we suggest you simply run the lambda multiple times.
Expand Down Expand Up @@ -58,7 +58,7 @@ $ npm run tail-logs

## TODO

- Only keep the last 10 master images (justification: we should be using the last images only, last 10 gives us something to rollback to if needed.)
- Only keep the last 10 master/main images (justification: we should be using the last images only, last 10 gives us something to rollback to if needed.)
- Add some more documentation to this readme
- Delete all untagged images
- Make tagging convention configurable
203 changes: 117 additions & 86 deletions handler.js
View file
Original file line number Diff line number Diff line change
@@ -1,20 +1,20 @@
'use strict';
"use strict";

const AWS = require('aws-sdk');
const moment = require('moment');
const filter = require('lodash.filter');
const rp = require('request-promise');
const AWS = require("aws-sdk");
const moment = require("moment");
const filter = require("lodash.filter");
const rp = require("request-promise");

function postToSlack(text) {
if (typeof process.env.SLACK_WEBHOOK === 'undefined') {
if (typeof process.env.SLACK_WEBHOOK === "undefined") {
return Promise.resolve(text);
}

const options = {
method: 'POST',
method: "POST",
uri: process.env.SLACK_WEBHOOK,
body: { text },
json: true
json: true,
};
return rp(options);
}
Expand All @@ -23,11 +23,13 @@ function getAllImages(ecr, registryId, repoName) {
const params = {
registryId,
repositoryName: repoName,
maxResults: 100
maxResults: 100,
};

return ecr.describeImages(params).promise()
.then(data => Promise.resolve(data.imageDetails));
return ecr
.describeImages(params)
.promise()
.then((data) => Promise.resolve(data.imageDetails));
}

function buildReport(
Expand All @@ -41,118 +43,131 @@ function buildReport(
const untaggedRepoKeys = Object.keys(reposWithUntaggedImages);
const deletedDryRunRepoKeys = Object.keys(reposWithDeletedImagesDryRun);
const deletedRepoKeys = Object.keys(reposWithDeletedImages);
const failedToDeletedRepoKeys = Object.keys(reposWithImagesThatFailedToDelete);
const failedToDeletedRepoKeys = Object.keys(
reposWithImagesThatFailedToDelete
);

if (
reposNotFound.length === 0
&& untaggedRepoKeys.length === 0
&& deletedDryRunRepoKeys === 0
&& deletedRepoKeys.length === 0
&& failedToDeletedRepoKeys === 0
reposNotFound.length === 0 &&
untaggedRepoKeys.length === 0 &&
deletedDryRunRepoKeys === 0 &&
deletedRepoKeys.length === 0 &&
failedToDeletedRepoKeys === 0
) {
return 'Robin ran but there no vigilamnte justice was needed';
return "Robin ran but there no vigilamnte justice was needed";
}

const backticks = str => (`\`${str}\``);
const dryRunText = isDryRun ? ' [DRY RUN]' : '';
const backticks = (str) => `\`${str}\``;
const dryRunText = isDryRun ? " [DRY RUN]" : "";

let text = 'Robin has attempted to clean up the streets!';
let text = "Robin has attempted to clean up the streets!";

if (reposNotFound.length !== 0) {
text += '\n\n\n===================================================';
text += "\n\n\n===================================================";
text += `\nRepositories not found (${reposNotFound.length})${dryRunText}`;
text += '\n===================================================';
reposNotFound.forEach(repoName => {
text += "\n===================================================";
reposNotFound.forEach((repoName) => {
text += `\n${backticks(repoName)}`;
});
}

if (untaggedRepoKeys.length !== 0) {
text += '\n\n\n===================================================';
text += "\n\n\n===================================================";
text += `\nRepositories with untagged images (${untaggedRepoKeys.length})${dryRunText}`;
text += '\n===================================================';
untaggedRepoKeys.forEach(repoName => {
text += `\n${backticks(repoName)} - ${reposWithUntaggedImages[repoName]} image${reposWithUntaggedImages[repoName] > 1 ? 's' : ''}`;
text += "\n===================================================";
untaggedRepoKeys.forEach((repoName) => {
text += `\n${backticks(repoName)} - ${
reposWithUntaggedImages[repoName]
} image${reposWithUntaggedImages[repoName] > 1 ? "s" : ""}`;
});
}

if (isDryRun) {
text += '\n\n\n===================================================';
text += "\n\n\n===================================================";
text += `\nRepositories with images deleted (${deletedDryRunRepoKeys.length})${dryRunText}`;
text += '\n===================================================';
text += "\n===================================================";
if (deletedDryRunRepoKeys.length === 0) {
text += '\nNo images deleted';
text += "\nNo images deleted";
} else {
deletedDryRunRepoKeys.forEach(repoName => {
deletedDryRunRepoKeys.forEach((repoName) => {
// eslint-disable-next-line max-len
text += `\n${backticks(repoName)} (${reposWithDeletedImagesDryRun[repoName].length} tags): ${reposWithDeletedImagesDryRun[repoName].join(', ')}`;
text += `\n${backticks(repoName)} (${
reposWithDeletedImagesDryRun[repoName].length
} tags): ${reposWithDeletedImagesDryRun[repoName].join(", ")}`;
});
}
} else {
text += '\n\n\n===================================================';
text += "\n\n\n===================================================";
text += `\nRepositories with images deleted (${deletedRepoKeys.length})`;
text += '\n===================================================';
text += "\n===================================================";
if (deletedRepoKeys.length === 0) {
text += '\nNo images deleted';
text += "\nNo images deleted";
} else {
deletedRepoKeys.forEach(repoName => {
text += `\n${backticks(repoName)} (${reposWithDeletedImages[repoName].length} tags): ${reposWithDeletedImages[repoName].join(', ')}`;
deletedRepoKeys.forEach((repoName) => {
text += `\n${backticks(repoName)} (${
reposWithDeletedImages[repoName].length
} tags): ${reposWithDeletedImages[repoName].join(", ")}`;
});
}

if (failedToDeletedRepoKeys.length !== 0) {
text += '\n\n\n===================================================';
text += "\n\n\n===================================================";
text += `\nRepositories with images that failed deleted (${failedToDeletedRepoKeys.length})`;
text += '\n===================================================';
deletedRepoKeys.forEach(repoName => {
text += "\n===================================================";
deletedRepoKeys.forEach((repoName) => {
// eslint-disable-next-line max-len
text += `\n${backticks(repoName)} (${reposWithImagesThatFailedToDelete[repoName].length} tags): ${reposWithImagesThatFailedToDelete[repoName].join(', ')}`;
text += `\n${backticks(repoName)} (${
reposWithImagesThatFailedToDelete[repoName].length
} tags): ${reposWithImagesThatFailedToDelete[repoName].join(", ")}`;
});
}
}


return text;
}

module.exports.cleanupImages = (event, context, callback) => {
if (typeof process.env.REPO_NAMES === 'undefined') {
throw new Error("Can't start lambda: missing REPO_NAMES environment variable");
if (typeof process.env.REPO_NAMES === "undefined") {
throw new Error(
"Can't start lambda: missing REPO_NAMES environment variable"
);
}

if (typeof process.env.AWS_ACCOUNT_ID === 'undefined') {
throw new Error("Can't start lambda: missing AWS_ACCOUNT_ID environment variable");
if (typeof process.env.AWS_ACCOUNT_ID === "undefined") {
throw new Error(
"Can't start lambda: missing AWS_ACCOUNT_ID environment variable"
);
}

const repoNames = process.env.REPO_NAMES.split(',');
const repoNames = process.env.REPO_NAMES.split(",");
const registry = process.env.AWS_ACCOUNT_ID;

const ecrRegion = process.env.ECR_REGION || 'us-east-1';
const ecr = new AWS.ECR({ apiVersion: '2015-09-21', region: ecrRegion });
const ecrRegion = process.env.ECR_REGION || "us-east-1";
const ecr = new AWS.ECR({ apiVersion: "2015-09-21", region: ecrRegion });

const reposNotFound = [];
const reposWithUntaggedImages = {};
const reposWithDeletedImages = {};
const reposWithDeletedImagesDryRun = {};
const reposWithImagesThatFailedToDelete = {};

console.log('Robin is dealing out some of his own justice...');
console.log('Robin is using ECR Region: ', ecrRegion);
console.log("Robin is dealing out some of his own justice...");
console.log("Robin is using ECR Region: ", ecrRegion);

const isDryRun = process.env.DRY_RUN === 'true';
console.log('Robin is running in dry run mode: ', isDryRun);
const isDryRun = process.env.DRY_RUN === "true";
console.log("Robin is running in dry run mode: ", isDryRun);

const cutOffDate = moment().add(-30, 'd');
console.log('Using cut off date: ', cutOffDate);
const cutOffDate = moment().add(-30, "d");
console.log("Using cut off date: ", cutOffDate);

const promises = repoNames.map(repoName => (
const promises = repoNames.map((repoName) =>
getAllImages(ecr, registry, repoName)
.then(images => { // eslint-disable-line arrow-body-style
return filter(images, image => {

const isUntagged = typeof image.imageTags === 'undefined';
.then((images) => {
// eslint-disable-line arrow-body-style
return filter(images, (image) => {
const isUntagged = typeof image.imageTags === "undefined";
if (isUntagged) {
if (typeof reposWithUntaggedImages[repoName] !== 'number') {
if (typeof reposWithUntaggedImages[repoName] !== "number") {
reposWithUntaggedImages[repoName] = 1;
} else {
reposWithUntaggedImages[repoName]++;
Expand All @@ -162,22 +177,28 @@ module.exports.cleanupImages = (event, context, callback) => {
}

// filters out images that are 30 days old and don't contain the master tag
return !isUntagged
&& moment(image.imagePushedAt).isBefore(cutOffDate)
&& !image.imageTags.find(tag => tag.indexOf('master') > -1);
return (
!isUntagged &&
moment(image.imagePushedAt).isBefore(cutOffDate) &&
!image.imageTags.find(
(tag) => tag.indexOf("master") > -1 || tag.indexOf("main") > -1
)
);
});
})
.then(toDelete => {
.then((toDelete) => {
if (!toDelete || toDelete.length === 0) {
return Promise.resolve({ imageIds: 'none', failures: 'none' });
return Promise.resolve({ imageIds: "none", failures: "none" });
}

console.log('Images to delete: ', toDelete);
console.log("Images to delete: ", toDelete);

const convertedToDelete = toDelete.map(image => {
const convertedToDelete = toDelete.map((image) => {
if (isDryRun) {
image.imageTags.forEach(tag => {
if (typeof reposWithDeletedImagesDryRun[repoName] === 'undefined') {
image.imageTags.forEach((tag) => {
if (
typeof reposWithDeletedImagesDryRun[repoName] === "undefined"
) {
reposWithDeletedImagesDryRun[repoName] = [];
}
reposWithDeletedImagesDryRun[repoName].push(tag);
Expand All @@ -194,37 +215,47 @@ module.exports.cleanupImages = (event, context, callback) => {
const deleteParams = {
registryId: registry,
repositoryName: repoName,
imageIds: convertedToDelete
imageIds: convertedToDelete,
};

return ecr.batchDeleteImage(deleteParams).promise()
return ecr
.batchDeleteImage(deleteParams)
.promise()
.then(({ failures, imageIds }) => {
console.log('failures: ', failures);
console.log('imageIds: ', imageIds);
console.log("failures: ", failures);
console.log("imageIds: ", imageIds);

failures.forEach(({ imageId }) => {
if (typeof reposWithImagesThatFailedToDelete[repoName] === 'undefined') {
if (
typeof reposWithImagesThatFailedToDelete[repoName] ===
"undefined"
) {
reposWithImagesThatFailedToDelete[repoName] = [];
}
reposWithImagesThatFailedToDelete[repoName].push(imageId.imageTag);
reposWithImagesThatFailedToDelete[repoName].push(
imageId.imageTag
);
});

imageIds.forEach(({ imageTag }) => {
if (typeof reposWithDeletedImages[repoName] === 'undefined') {
if (typeof reposWithDeletedImages[repoName] === "undefined") {
reposWithDeletedImages[repoName] = [];
}
reposWithDeletedImages[repoName].push(imageTag);
});
});
})
.catch(err => {
if (err.code === 'RepositoryNotFoundException' && reposNotFound.indexOf(repoName) === -1) {
.catch((err) => {
if (
err.code === "RepositoryNotFoundException" &&
reposNotFound.indexOf(repoName) === -1
) {
reposNotFound.push(repoName);
}

console.log(err);
})
));
);

return Promise.all(promises)
.then(() => {
Expand All @@ -238,17 +269,17 @@ module.exports.cleanupImages = (event, context, callback) => {
);

// Log Results
console.log(reportText.replace(/`/g, '')); // strip backticks when logging to CloudWatch (backticks are for Slack!)
console.log(reportText.replace(/`/g, "")); // strip backticks when logging to CloudWatch (backticks are for Slack!)

return Promise.resolve(reportText);
})
.then(text => (
.then((text) =>
postToSlack(text) // Post results to Slack
))
)
.then(() => {
callback(null, { message: 'robin executed successfully!', event });
callback(null, { message: "robin executed successfully!", event });
})
.catch(err => {
.catch((err) => {
console.log(err); // an error occurred
callback(err);
return;
Expand Down
Loading

0 comments on commit bbb1091

Please sign in to comment.