From cd92a1fa9d0a1a45312686e6045aaa5550070c9f Mon Sep 17 00:00:00 2001
From: generalov <e.generalov@gmail.com>
Date: Tue, 17 Nov 2020 11:12:48 +0300
Subject: [PATCH] Task5 (#4)

* version: 1.0.5

* chore: import-service stub

* chore: prerequisites

* feat: create s3 bucket

* feat: importProductsFile

* chore: update policies

* feat: importFileParser

* fix: content type

* fix: naming

* test: importProductsFile

Co-authored-by: Evgeny Generalov <Evgeniy.Generalov@coxautoinc.com>
---
 package.json                                  |  9 +-
 packages/import-service/.env                  |  4 +
 packages/import-service/.gitignore            |  9 ++
 packages/import-service/babel.config.json     | 12 +++
 packages/import-service/package.json          | 30 +++++++
 packages/import-service/serverless.yml        | 62 ++++++++++++++
 .../src/handlers/importFileParser.js          | 76 +++++++++++++++++
 .../src/handlers/importProductsFile.js        | 51 ++++++++++++
 .../src/handlers/importProductsFile.spec.js   | 26 ++++++
 packages/import-service/webpack.config.js     | 28 +++++++
 terraform/main.tf                             | 14 ++++
 yarn.lock                                     | 82 ++++++++++++++++++-
 12 files changed, 399 insertions(+), 4 deletions(-)
 create mode 100644 packages/import-service/.env
 create mode 100644 packages/import-service/.gitignore
 create mode 100644 packages/import-service/babel.config.json
 create mode 100644 packages/import-service/package.json
 create mode 100644 packages/import-service/serverless.yml
 create mode 100644 packages/import-service/src/handlers/importFileParser.js
 create mode 100644 packages/import-service/src/handlers/importProductsFile.js
 create mode 100644 packages/import-service/src/handlers/importProductsFile.spec.js
 create mode 100644 packages/import-service/webpack.config.js

diff --git a/package.json b/package.json
index 326139e..b93cf75 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "nodejs-aws-be",
-  "version": "1.0.4",
+  "version": "1.0.5",
   "license": "MIT",
   "private": true,
   "workspaces": [
@@ -8,13 +8,16 @@
     "terraform"
   ],
   "devDependencies": {
+    "prettier": "^2.1.2",
     "serverless": "^2.8.0"
   },
   "scripts": {
     "tf": "yarn workspace terraform run terraform",
+    "start:product-service": "yarn workspace product-service sls offline",
+    "start:import-service": "yarn workspace import-service sls offline",
     "deploy:tf": "yarn tf apply -auto-approve",
-    "start": "yarn workspace product-service sls offline",
-    "deploy": "yarn workspace product-service sls deploy",
+    "deploy:product-service": "yarn workspace product-service sls deploy",
+    "deploy:import-service": "yarn workspace import-service sls deploy",
     "create:sls": "yarn sls create --template-path=$(pwd)/scripts/template-aws-nodejs"
   }
 }
diff --git a/packages/import-service/.env b/packages/import-service/.env
new file mode 100644
index 0000000..feea68c
--- /dev/null
+++ b/packages/import-service/.env
@@ -0,0 +1,4 @@
+IMPORT_S3_BUCKET=nodejs-aws-task5-csv
+IMPORT_S3_REGION=us-east-2
+IMPORT_S3_PARSED_PREFIX=parsed/
+IMPORT_S3_UPLOAD_PREFIX=uploaded/
diff --git a/packages/import-service/.gitignore b/packages/import-service/.gitignore
new file mode 100644
index 0000000..9837493
--- /dev/null
+++ b/packages/import-service/.gitignore
@@ -0,0 +1,9 @@
+# package directories
+node_modules
+jspm_packages
+
+# Serverless directories
+.serverless
+
+# Webpack directories
+.webpack
\ No newline at end of file
diff --git a/packages/import-service/babel.config.json b/packages/import-service/babel.config.json
new file mode 100644
index 0000000..41046be
--- /dev/null
+++ b/packages/import-service/babel.config.json
@@ -0,0 +1,12 @@
+{
+  "presets": [
+    [
+      "@babel/preset-env",
+      {
+        "targets": {
+          "node": "12"
+        }
+      }
+    ]
+  ]
+}
diff --git a/packages/import-service/package.json b/packages/import-service/package.json
new file mode 100644
index 0000000..2c4604b
--- /dev/null
+++ b/packages/import-service/package.json
@@ -0,0 +1,30 @@
+{
+  "name": "import-service",
+  "version": "1.0.0",
+  "description": "Serverless webpack example using ecma script",
+  "scripts": {
+    "test": "jest"
+  },
+  "devDependencies": {
+    "@babel/core": "^7.11.1",
+    "@babel/preset-env": "^7.11.0",
+    "@types/jest": "^26.0.15",
+    "babel-jest": "^26.6.1",
+    "babel-loader": "^8.1.0",
+    "babel-plugin-transform-runtime": "^6.23.0",
+    "babel-polyfill": "^6.23.0",
+    "babel-preset-env": "^1.6.0",
+    "jest": "^26.6.1",
+    "serverless-dotenv-plugin": "^3.1.0",
+    "serverless-offline": "^6.8.0",
+    "serverless-webpack": "^5.3.1",
+    "webpack": "^4.35.2"
+  },
+  "author": "The serverless webpack authors (https://github.com/elastic-coders/serverless-webpack)",
+  "license": "MIT",
+  "dependencies": {
+    "aws-sdk": "^2.792.0",
+    "csv-parser": "^2.3.3",
+    "http-errors": "^1.8.0"
+  }
+}
diff --git a/packages/import-service/serverless.yml b/packages/import-service/serverless.yml
new file mode 100644
index 0000000..5ee14b7
--- /dev/null
+++ b/packages/import-service/serverless.yml
@@ -0,0 +1,62 @@
+service:
+  name: import-service
+# app and org for use with dashboard.serverless.com
+#app: your-app-name
+#org: your-org-name
+frameworkVersion: '2'
+
+plugins:
+  - serverless-dotenv-plugin
+  - serverless-webpack
+  - serverless-offline
+
+custom:
+  dotenv:
+    required:
+      file: true
+  serverless-offline:
+    httpPort: 4000
+    lambdaPort: 4002
+    websocketPort: 4001
+  webpack:
+    packager: "yarn"
+
+provider:
+  name: aws
+  runtime: nodejs12.x
+  region: us-east-2
+  #  stage: dev
+
+  iamRoleStatements:
+    - Effect: "Allow"
+      Action: "s3:ListBucket"
+      Resource:
+        - "arn:aws:s3:::${env:IMPORT_S3_BUCKET}"
+    - Effect: "Allow"
+      Action: "s3:*"
+      Resource:
+        - "arn:aws:s3:::${env:IMPORT_S3_BUCKET}/*"
+
+functions:
+  importProductsFile:
+    handler: src/handlers/importProductsFile.handler
+    events:
+      - http:
+          method: get
+          path: import
+          cors: true
+          request:
+            parameters:
+              querystrings:
+                name: true
+                type: true
+
+  importFileParser:
+    handler: src/handlers/importFileParser.handler
+    events:
+      - s3:
+          bucket: "${env:IMPORT_S3_BUCKET}"
+          event: "s3:ObjectCreated:*"
+          rules:
+            - prefix: "${env:IMPORT_S3_UPLOAD_PREFIX}"
+          existing: true
\ No newline at end of file
diff --git a/packages/import-service/src/handlers/importFileParser.js b/packages/import-service/src/handlers/importFileParser.js
new file mode 100644
index 0000000..89db7c8
--- /dev/null
+++ b/packages/import-service/src/handlers/importFileParser.js
@@ -0,0 +1,76 @@
+import AWS from "aws-sdk";
+import csvParser from "csv-parser";
+import { Transform, pipeline as _pipeline } from "stream";
+import { promisify } from "util";
+import middy from "@middy/core";
+import middyRequestLogger from "middy-request-logger";
+
+const pipeline = promisify(_pipeline);
+
+const {
+  IMPORT_S3_BUCKET,
+  IMPORT_S3_UPLOAD_PREFIX,
+  IMPORT_S3_REGION,
+  IMPORT_S3_PARSED_PREFIX,
+} = process.env;
+
+export const handler = middy(importFileParser).use([middyRequestLogger()]);
+
+export async function importFileParser(event, context, callback) {
+  const s3 = new AWS.S3({ region: IMPORT_S3_REGION });
+
+  const tasks = event.Records.map(async (record) => {
+    const srcKey = record.s3.object.key;
+    const destKey = srcKey.replace(
+      IMPORT_S3_UPLOAD_PREFIX,
+      IMPORT_S3_PARSED_PREFIX
+    );
+
+    // parse CSV
+    const uploadedObject = s3.getObject({
+      Bucket: IMPORT_S3_BUCKET,
+      Key: srcKey,
+    });
+    await pipeline(
+      uploadedObject.createReadStream(),
+      csvParser(),
+      streamTap(console.log)
+    );
+
+    // move
+    await s3
+      .copyObject({
+        Bucket: IMPORT_S3_BUCKET,
+        CopySource: IMPORT_S3_BUCKET + "/" + srcKey,
+        Key: destKey,
+      })
+      .promise();
+    await s3
+      .deleteObject({
+        Bucket: IMPORT_S3_BUCKET,
+        Key: srcKey,
+      })
+      .promise();
+  });
+
+  const results = await Promise.allSettled(tasks);
+  const success = results.filter(({ status }) => status === "fulfilled");
+  console.log(
+    `${success.length} of ${results.length} files was copied successfully`
+  );
+}
+
+export function streamTap(fn) {
+  return new Transform({
+    objectMode: true,
+    transform: (data, encoding, done) => {
+      try {
+        fn({ data, encoding });
+      } catch (err) {
+        done(err);
+        return;
+      }
+      done(null, data);
+    },
+  });
+}
diff --git a/packages/import-service/src/handlers/importProductsFile.js b/packages/import-service/src/handlers/importProductsFile.js
new file mode 100644
index 0000000..e3ecf99
--- /dev/null
+++ b/packages/import-service/src/handlers/importProductsFile.js
@@ -0,0 +1,51 @@
+import AWS from "aws-sdk";
+import httpError from "http-errors";
+import middy from "@middy/core";
+import middyHttpCors from "@middy/http-cors";
+import middyErrorHandler from "middy-error-handler";
+import middyRequestLogger from "middy-request-logger";
+
+const {
+  IMPORT_S3_BUCKET,
+  IMPORT_S3_UPLOAD_PREFIX,
+  IMPORT_S3_REGION,
+} = process.env;
+const ALLOWED_CONTENT_TYPES = [
+  "text/csv",
+  "application/vnd.ms-excel",
+  "text/x-csv",
+];
+
+export const handler = middy(importProductsFile).use([
+  middyErrorHandler(),
+  middyRequestLogger(),
+  middyHttpCors(),
+]);
+
+export async function importProductsFile(event) {
+  const { name: fileName, type: fileType } = event.queryStringParameters;
+
+  if (!fileName) {
+    throw new httpError.BadRequest(`'name' should not be empty`);
+  }
+  if (
+    !fileType ||
+    !ALLOWED_CONTENT_TYPES.find((allowedType) => fileType.includes(allowedType))
+  ) {
+    throw new httpError.BadRequest(`Unsupported file type '${fileType}'`);
+  }
+
+  const s3 = new AWS.S3({ region: IMPORT_S3_REGION });
+  const uploadPath = IMPORT_S3_UPLOAD_PREFIX + fileName;
+  const url = await s3.getSignedUrlPromise("putObject", {
+    Bucket: IMPORT_S3_BUCKET,
+    Key: uploadPath,
+    Expires: 60,
+    ContentType: fileType,
+  });
+
+  return {
+    statusCode: 200,
+    body: JSON.stringify(url),
+  };
+}
diff --git a/packages/import-service/src/handlers/importProductsFile.spec.js b/packages/import-service/src/handlers/importProductsFile.spec.js
new file mode 100644
index 0000000..c94f8e6
--- /dev/null
+++ b/packages/import-service/src/handlers/importProductsFile.spec.js
@@ -0,0 +1,26 @@
+import { handler } from "./importProductsFile";
+import AWS from "aws-sdk";
+
+// aws-sdk-mock package does not support getSignedUrlPromise,
+// so let's use jest
+jest.mock("aws-sdk");
+jest.mock("middy-request-logger", () => () => ({
+  before: (handler, next) => next(),
+}));
+
+beforeEach(() => {
+  jest
+    .spyOn(AWS.S3.prototype, "getSignedUrlPromise")
+    .mockResolvedValue("http://aws");
+});
+
+test("importProductsFile ", async () => {
+  const event = {
+    queryStringParameters: { name: "products.csv", type: "text/csv" },
+  };
+
+  const resp = await handler(event);
+
+  expect(resp.statusCode).toBe(200);
+  expect(resp.body).toBe(JSON.stringify("http://aws"));
+});
diff --git a/packages/import-service/webpack.config.js b/packages/import-service/webpack.config.js
new file mode 100644
index 0000000..7e7beff
--- /dev/null
+++ b/packages/import-service/webpack.config.js
@@ -0,0 +1,28 @@
+const path = require("path");
+const slsw = require("serverless-webpack");
+
+module.exports = {
+  entry: slsw.lib.entries,
+  mode: slsw.lib.webpack.isLocal ? "development" : "production",
+  target: "node",
+  output: {
+    libraryTarget: "commonjs",
+    filename: "[name].js",
+    path: path.join(__dirname, ".webpack"),
+  },
+  module: {
+    rules: [
+      {
+        test: /\.js$/, // include .js files
+        enforce: "pre", // preload the jshint loader
+        exclude: /node_modules/, // exclude any and all files in the node_modules folder
+        include: __dirname,
+        use: [
+          {
+            loader: "babel-loader",
+          },
+        ],
+      },
+    ],
+  }
+};
diff --git a/terraform/main.tf b/terraform/main.tf
index 133e765..a747deb 100644
--- a/terraform/main.tf
+++ b/terraform/main.tf
@@ -72,3 +72,17 @@ resource "aws_security_group" "pg" {
     cidr_blocks = ["0.0.0.0/0"]
   }
 }
+
+// task 5
+resource "aws_s3_bucket" "task5_csv" {
+  bucket = "nodejs-aws-task5-csv"
+  acl    = "private"
+
+  cors_rule {
+    allowed_headers = ["*"]
+    allowed_methods = ["PUT"]
+    allowed_origins = ["*"]
+    expose_headers  = ["ETag"]
+    max_age_seconds = 3000
+  }
+}
diff --git a/yarn.lock b/yarn.lock
index 3291723..3fe2d47 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -1333,6 +1333,17 @@
     "@types/yargs" "^15.0.0"
     chalk "^4.0.0"
 
+"@jest/types@^26.6.2":
+  version "26.6.2"
+  resolved "https://registry.yarnpkg.com/@jest/types/-/types-26.6.2.tgz#bef5a532030e1d88a2f5a6d933f84e97226ed48e"
+  integrity sha512-fC6QCp7Sc5sX6g8Tvbmj4XUTbyrik0akgRy03yjXbQaBWWNWGE7SGtJk98m0N8nzegD/7SggrUlivxo5ax4KWQ==
+  dependencies:
+    "@types/istanbul-lib-coverage" "^2.0.0"
+    "@types/istanbul-reports" "^3.0.0"
+    "@types/node" "*"
+    "@types/yargs" "^15.0.0"
+    chalk "^4.0.0"
+
 "@kwsites/file-exists@^1.1.1":
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/@kwsites/file-exists/-/file-exists-1.1.1.tgz#ad1efcac13e1987d8dbaf235ef3be5b0d96faa99"
@@ -1791,6 +1802,14 @@
   dependencies:
     "@types/istanbul-lib-report" "*"
 
+"@types/jest@^26.0.15":
+  version "26.0.15"
+  resolved "https://registry.yarnpkg.com/@types/jest/-/jest-26.0.15.tgz#12e02c0372ad0548e07b9f4e19132b834cb1effe"
+  integrity sha512-s2VMReFXRg9XXxV+CW9e5Nz8fH2K1aEhwgjUqPPbQd7g95T0laAcvLv032EhFHIa5GHsZ8W7iJEQVaJq6k3Gog==
+  dependencies:
+    jest-diff "^26.0.0"
+    pretty-format "^26.0.0"
+
 "@types/json-schema@^7.0.5":
   version "7.0.6"
   resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.6.tgz#f4c7ec43e81b319a9815115031709f26987891f0"
@@ -2376,6 +2395,21 @@ aws-sdk@^2.781.0:
     uuid "3.3.2"
     xml2js "0.4.19"
 
+aws-sdk@^2.792.0:
+  version "2.792.0"
+  resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.792.0.tgz#d124a6074244a4675e0416887734e8f6934bdd30"
+  integrity sha512-h7oSlrCDtZkW5qNw/idKmMjjNJaaPlXFY+NbqtaTjejpCyVuIonUmFvm8GW16V58Avj/hujJfhpX9q0BMCg+VQ==
+  dependencies:
+    buffer "4.9.2"
+    events "1.1.1"
+    ieee754 "1.1.13"
+    jmespath "0.15.0"
+    querystring "0.2.0"
+    sax "1.2.1"
+    url "0.10.3"
+    uuid "3.3.2"
+    xml2js "0.4.19"
+
 aws-sign2@~0.7.0:
   version "0.7.0"
   resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8"
@@ -3958,6 +3992,14 @@ cssstyle@^2.2.0:
   dependencies:
     cssom "~0.3.6"
 
+csv-parser@^2.3.3:
+  version "2.3.3"
+  resolved "https://registry.yarnpkg.com/csv-parser/-/csv-parser-2.3.3.tgz#978120ca14f879a17a8b977d9448daa738a38f65"
+  integrity sha512-czcyxc4/3Tt63w0oiK1zsnRgRD4PkqWaRSJ6eef63xC0f+5LVLuGdSYEcJwGp2euPgRHx+jmlH2Lb49anb1CGQ==
+  dependencies:
+    minimist "^1.2.0"
+    through2 "^3.0.1"
+
 cuid@^2.1.8:
   version "2.1.8"
   resolved "https://registry.yarnpkg.com/cuid/-/cuid-2.1.8.tgz#cbb88f954171e0d5747606c0139fb65c5101eac0"
@@ -4224,6 +4266,11 @@ diff-sequences@^26.5.0:
   resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-26.5.0.tgz#ef766cf09d43ed40406611f11c6d8d9dd8b2fefd"
   integrity sha512-ZXx86srb/iYy6jG71k++wBN9P9J05UNQ5hQHQd9MtMPvcqXPx/vKU69jfHV637D00Q2gSgPk2D+jSx3l1lDW/Q==
 
+diff-sequences@^26.6.2:
+  version "26.6.2"
+  resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-26.6.2.tgz#48ba99157de1923412eed41db6b6d4aa9ca7c0b1"
+  integrity sha512-Mv/TDa3nZ9sbc5soK+OoA74BsS3mL37yixCvUAQkiuA4Wz6YtwP/K47n2rv2ovzHZvoiQeA5FTQOschKkEwB0Q==
+
 diff@^4.0.1:
   version "4.0.2"
   resolved "https://registry.yarnpkg.com/diff/-/diff-4.0.2.tgz#60f3aecb89d5fae520c11aa19efc2bb982aade7d"
@@ -6193,6 +6240,16 @@ jest-config@^26.6.1:
     micromatch "^4.0.2"
     pretty-format "^26.6.1"
 
+jest-diff@^26.0.0:
+  version "26.6.2"
+  resolved "https://registry.yarnpkg.com/jest-diff/-/jest-diff-26.6.2.tgz#1aa7468b52c3a68d7d5c5fdcdfcd5e49bd164394"
+  integrity sha512-6m+9Z3Gv9wN0WFVasqjCL/06+EFCMTqDEUl/b87HYK2rAPTyfz4ZIuSlPhY51PIQRWx5TaxeF1qmXKe9gfN3sA==
+  dependencies:
+    chalk "^4.0.0"
+    diff-sequences "^26.6.2"
+    jest-get-type "^26.3.0"
+    pretty-format "^26.6.2"
+
 jest-diff@^26.6.1:
   version "26.6.1"
   resolved "https://registry.yarnpkg.com/jest-diff/-/jest-diff-26.6.1.tgz#38aa194979f454619bb39bdee299fb64ede5300c"
@@ -8160,6 +8217,21 @@ prepend-http@^2.0.0:
   resolved "https://registry.yarnpkg.com/prepend-http/-/prepend-http-2.0.0.tgz#e92434bfa5ea8c19f41cdfd401d741a3c819d897"
   integrity sha1-6SQ0v6XqjBn0HN/UAddBo8gZ2Jc=
 
+prettier@^2.1.2:
+  version "2.1.2"
+  resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.1.2.tgz#3050700dae2e4c8b67c4c3f666cdb8af405e1ce5"
+  integrity sha512-16c7K+x4qVlJg9rEbXl7HEGmQyZlG4R9AgP+oHKRMsMsuk8s+ATStlf1NpDqyBI1HpVyfjLOeMhH2LvuNvV5Vg==
+
+pretty-format@^26.0.0, pretty-format@^26.6.2:
+  version "26.6.2"
+  resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-26.6.2.tgz#e35c2705f14cb7fe2fe94fa078345b444120fc93"
+  integrity sha512-7AeGuCYNGmycyQbCqd/3PWH4eOoX/OiCa0uphp57NVTeAGdJGaAliecxwBDHYQCIvrW7aDBZCYeNTP/WX69mkg==
+  dependencies:
+    "@jest/types" "^26.6.2"
+    ansi-regex "^5.0.0"
+    ansi-styles "^4.0.0"
+    react-is "^17.0.1"
+
 pretty-format@^26.6.1:
   version "26.6.1"
   resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-26.6.1.tgz#af9a2f63493a856acddeeb11ba6bcf61989660a8"
@@ -8419,7 +8491,7 @@ read-pkg@^5.2.0:
     string_decoder "~1.1.1"
     util-deprecate "~1.0.1"
 
-readable-stream@^3.0.0, readable-stream@^3.1.1, readable-stream@^3.4.0, readable-stream@^3.6.0:
+"readable-stream@2 || 3", readable-stream@^3.0.0, readable-stream@^3.1.1, readable-stream@^3.4.0, readable-stream@^3.6.0:
   version "3.6.0"
   resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198"
   integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==
@@ -9711,6 +9783,14 @@ through2@^2.0.0:
     readable-stream "~2.3.6"
     xtend "~4.0.1"
 
+through2@^3.0.1:
+  version "3.0.2"
+  resolved "https://registry.yarnpkg.com/through2/-/through2-3.0.2.tgz#99f88931cfc761ec7678b41d5d7336b5b6a07bf4"
+  integrity sha512-enaDQ4MUyP2W6ZyT6EsMzqBPZaM/avg8iuo+l2d3QCs0J+6RaqkHV/2/lOwDTueBHeJ/2LG9lrLW3d5rWPucuQ==
+  dependencies:
+    inherits "^2.0.4"
+    readable-stream "2 || 3"
+
 through@^2.3.6, through@^2.3.8:
   version "2.3.8"
   resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5"