Merge pull request #8 from atomist/delete-deleted-2

Add sync option

lib/publishToS3.ts

@@ -85,13 +85,20 @@ export interface PublishToS3Options {
      * provided, no externalUrl is provided in the goal result.
      */
     pathToIndex?: string;
+
+    /**
+     * If true, delete objects from S3 bucket that do not map to files
+     * in the repository being copied to the bucket.  If false, files
+     * from the repository are copied to the bucket but no existing
+     * objects in the bucket are deleted.
+     */
+    sync?: boolean;
 }
 
 /**
  * Get a goal that will publish (portions of) a project to S3.
  * If the project needs to be built or otherwise processed first, use
  * `.withProjectListeners` to get those prerequisite steps done.
- *
  */
 export class PublishToS3 extends GoalWithFulfillment {
 
@@ -109,8 +116,16 @@ export class PublishToS3 extends GoalWithFulfillment {
     }
 }
 
-function putObject(s3: S3, params: S3.Types.PutObjectRequest): () => Promise<S3.Types.PutObjectOutput> {
-    return promisify<S3.Types.PutObjectOutput>(cb => s3.putObject(params, cb));
+function putObject(s3: S3, params: S3.PutObjectRequest): () => Promise<S3.PutObjectOutput> {
+    return promisify<S3.PutObjectOutput>(cb => s3.putObject(params, cb));
+}
+
+function listObjects(s3: S3, params: S3.ListObjectsV2Request): () => Promise<S3.ListObjectsV2Output> {
+    return promisify<S3.ListObjectsV2Output>(cb => s3.listObjectsV2(params, cb));
+}
+
+function deleteObjects(s3: S3, params: S3.DeleteObjectsRequest): () => Promise<S3.DeleteObjectsOutput> {
+    return promisify<S3.DeleteObjectsOutput>(cb => s3.deleteObjects(params, cb));
 }
 
 export function executePublishToS3(params: PublishToS3Options): ExecuteGoal {
@@ -139,7 +154,8 @@ export function executePublishToS3(params: PublishToS3Options): ExecuteGoal {
                     inv.progressLog.write("URL: " + linkToIndex);
                 }
                 inv.progressLog.write(result.warnings.join("\n"));
-                inv.progressLog.write(`${result.fileCount} files uploaded to ${linkToIndex}`);
+                inv.progressLog.write(`${result.fileCount} files uploaded to ${params.bucketName}`);
+                inv.progressLog.write(`${result.deleted} objects deleted from ${params.bucketName}`);
 
                 if (result.warnings.length > 0) {
                     await inv.addressChannels(formatWarningMessage(linkToIndex, result.warnings, inv.id, inv.context));
@@ -164,12 +180,19 @@ function formatWarningMessage(url: string, warnings: string[], id: RepoRef, ctx:
     });
 }
 
-async function pushToS3(s3: S3, inv: ProjectAwareGoalInvocation, params: PublishToS3Options):
-    Promise<{ bucketUrl: string, warnings: string[], fileCount: number }> {
+interface PushToS3Result {
+    bucketUrl: string;
+    warnings: string[];
+    fileCount: number;
+    deleted: number;
+}
+
+async function pushToS3(s3: S3, inv: ProjectAwareGoalInvocation, params: PublishToS3Options): Promise<PushToS3Result> {
     const { bucketName, filesToPublish, pathTranslation, region } = params;
     const project = inv.project;
     const log = inv.progressLog;
     const warnings: string[] = [];
+    const keys: string[] = [];
     let fileCount = 0;
     await doWithFiles(project, filesToPublish, async file => {
         fileCount++;
@@ -184,6 +207,7 @@ async function pushToS3(s3: S3, inv: ProjectAwareGoalInvocation, params: Publish
                 Body: content,
                 ContentType: contentType,
             })();
+            keys.push(key);
             log.write(`Put '${file.path}' to 's3://${bucketName}/${key}'`);
         } catch (e) {
             const msg = `Failed to put '${file.path}' to 's3://${bucketName}/${key}': ${e.message}`;
@@ -192,9 +216,73 @@ async function pushToS3(s3: S3, inv: ProjectAwareGoalInvocation, params: Publish
         }
     });
 
+    let deleted = 0;
+    if (params.sync) {
+        let listObjectsResponse: S3.ListObjectsV2Output = {
+            IsTruncated: true,
+            NextContinuationToken: undefined,
+        };
+        const maxItems = 1000;
+        const deletedKeys: S3.ObjectIdentifier[] = [];
+        while (listObjectsResponse.IsTruncated) {
+            try {
+                listObjectsResponse = await listObjects(s3, {
+                    Bucket: bucketName,
+                    MaxKeys: maxItems,
+                    ContinuationToken: listObjectsResponse.NextContinuationToken,
+                })();
+                deletedKeys.push(...filterKeys(keys, listObjectsResponse.Contents));
+            } catch (e) {
+                const msg = `Failed to list objects in 's3://${bucketName}': ${e.message}`;
+                log.write(msg);
+                warnings.push(msg);
+                break;
+            }
+        }
+        for (let i = 0; i < deletedKeys.length; i += maxItems) {
+            const toDelete = deletedKeys.slice(i, i + maxItems);
+            try {
+                const deleteObjectsResult = await deleteObjects(s3, {
+                    Bucket: bucketName,
+                    Delete: {
+                        Objects: toDelete,
+                    },
+                })();
+                deleted += deleteObjectsResult.Deleted.length;
+                const deletedString = deleteObjectsResult.Deleted.map(o => o.Key).join(",");
+                log.write(`Deleted objects (${deletedString}) in 's3://${bucketName}'`);
+                if (deleteObjectsResult.Errors && deleteObjectsResult.Errors.length > 0) {
+                    deleteObjectsResult.Errors.forEach(e => {
+                        const msg = `Error deleting object '${e.Key}': ${e.Message}`;
+                        log.write(msg);
+                        warnings.push(msg);
+                    });
+                }
+            } catch (e) {
+                const keysString = toDelete.map(o => o.Key).join(",");
+                const msg = `Failed to delete objects (${keysString}) in 's3://${bucketName}': ${e.message}`;
+                log.write(msg);
+                warnings.push(msg);
+                break;
+            }
+        }
+    }
+
     return {
         bucketUrl: `http://${bucketName}.s3-website.${region}.amazonaws.com/`,
         warnings,
         fileCount,
+        deleted,
     };
 }
+
+/**
+ * Remove objects that either have no key or match a key in `keys`.
+ *
+ * @param keys Keys that should be removed from `objects`
+ * @param objects Array to filter
+ * @return Array of object identifiers
+ */
+export function filterKeys(keys: string[], objects: S3.Object[]): S3.ObjectIdentifier[] {
+    return objects.filter(o => o.Key && !keys.includes(o.Key)).map(o => ({ Key: o.Key }));
+}

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@atomist/sdm-pack-s3",
-  "version": "0.1.1",
+  "version": "0.2.0",
   "description": "SDM extension pack for publishing artifacts to AWS S3",
   "author": {
     "name": "Atomist",

test/publishToS3.test.ts

@@ -0,0 +1,89 @@
+/*
+ * Copyright © 2019 Atomist, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as assert from "power-assert";
+import { filterKeys } from "../lib/publishToS3";
+
+describe("publishToS3", () => {
+
+    describe("filterKeys", () => {
+
+        it("should filter out all objects", () => {
+            const k = ["Rage", "Against", "the", "Machine", "RageAgainstTheMachine/01-Bombtrack.mp3"];
+            const o = [
+                { Key: "Rage" },
+                { Key: "Machine" },
+                { Key: "RageAgainstTheMachine/01-Bombtrack.mp3" },
+            ];
+            const r = filterKeys(k, o);
+            assert.deepStrictEqual(r, []);
+        });
+
+        it("should filter out objects without keys", () => {
+            const k = ["Rage", "Against", "the", "Machine", "RageAgainstTheMachine/01-Bombtrack.mp3"];
+            const o = [
+                { Size: 1993 },
+                { ETag: "Rage" },
+                {
+                    Owner: {
+                        DisplayName: "Against",
+                        ID: "TheMachine",
+                    },
+                },
+            ];
+            const r = filterKeys(k, o);
+            assert.deepStrictEqual(r, []);
+        });
+
+        it("should filter objects that do not match keys", () => {
+            const k = ["Rage", "Against", "the", "Machine", "RageAgainstTheMachine/02-KillingInTheName.mp3"];
+            const o = [
+                { Key: "Rage" },
+                { Key: "Machine" },
+                { Key: "RageAgainstTheMachine/01-Bombtrack.mp3" },
+                { Key: "RageAgainstTheMachine/02-KillingInTheName.mp3" },
+                { Key: "RageAgainstTheMachine/03-TakeThePowerBack.mp3" },
+            ];
+            const r = filterKeys(k, o);
+            const e = [
+                { Key: "RageAgainstTheMachine/01-Bombtrack.mp3" },
+                { Key: "RageAgainstTheMachine/03-TakeThePowerBack.mp3" },
+            ];
+            assert.deepStrictEqual(r, e);
+        });
+
+        it("should only filter objects on keys", () => {
+            const k = ["Rage", "Against", "the", "Machine", "RageAgainstTheMachine/02-KillingInTheName.mp3"];
+            const o = [
+                { ETag: "Rage", Key: "AgainstThe" },
+                { Key: "The", StorageClass: "Machine" },
+                { Key: "RageAgainstTheMachine/01-Bombtrack.mp3" },
+                { Key: "RageAgainstTheMachine/02-KillingInTheName.mp3" },
+                { Key: "RageAgainstTheMachine/03-TakeThePowerBack.mp3" },
+            ];
+            const r = filterKeys(k, o);
+            const e = [
+                { Key: "AgainstThe" },
+                { Key: "The" },
+                { Key: "RageAgainstTheMachine/01-Bombtrack.mp3" },
+                { Key: "RageAgainstTheMachine/03-TakeThePowerBack.mp3" },
+            ];
+            assert.deepStrictEqual(r, e);
+        });
+
+    });
+
+});