feat: task 6

import-service/package.json

@@ -11,6 +11,7 @@
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.552.0",
+    "@aws-sdk/client-sqs": "^3.572.0",
     "@aws-sdk/s3-request-presigner": "^3.552.0",
     "@middy/core": "^3.4.0",
     "@middy/http-json-body-parser": "^3.4.0",
@@ -30,4 +31,4 @@
   },
   "author": "The serverless webpack authors (https://github.com/elastic-coders/serverless-webpack)",
   "license": "MIT"
-}
+}

import-service/serverless.ts

@@ -12,7 +12,6 @@ const serverlessConfiguration: AWS = {
     name: 'aws',
     runtime: 'nodejs20.x',
     region: 'eu-central-1',
-    profile: 'rootical',
     apiGateway: {
       minimumCompressionSize: 1024,
       shouldStartNameWithService: true,
@@ -27,10 +26,26 @@ const serverlessConfiguration: AWS = {
           {
             Effect: "Allow",
             Action: [
-              's3:*',
-              "logs:*"
+              "s3:PutObject",
+              "s3:PutObjectAcl",
+              "s3:GetObject",
+              "s3:GetObjectAcl",
+              "s3:DeleteObject",
+              "s3:CopyObject",
             ],
-            Resource: [`arn:aws:s3:::file-parcer-shop-stuff/*`, 'arn:aws:logs:*:*:*']
+            Resource: `arn:aws:s3:::file-parcer-shop-stuff`,
+          },
+          {
+            Effect: "Allow",
+            Action: [
+              "s3:*",
+            ],
+            Resource: `arn:aws:s3:::file-parcer-shop-stuff/*`,
+          },
+          {
+            Effect: "Allow",
+            Action: ["sqs:SendMessage", "sqs:GetQueueUrl", "sqs:ListQueues"],
+            Resource: `arn:aws:sqs:eu-central-1:434889505863:catalogItemsQueue`,
           },
         ],
       },

import-service/src/functions/importProductsFile/handler.ts

@@ -1,23 +1,64 @@
-import { formatJSONResponse } from '@libs/api-gateway';
-import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
-import { middyfy } from '@libs/lambda';
-import { PutObjectCommand, S3Client } from "@aws-sdk/client-s3";
-
-const importProductsFile = async (event) => {
-    const client = new S3Client({ region: 'eu-central-1'});
-
-    const { name } = event.queryStringParameters;
-    const command = new PutObjectCommand({
-        Bucket: 'file-parcer-shop-stuff',
-        Key: `uploaded/${name}`,
-    });
-
-    const signedUrl = await getSignedUrl(client, command, { expiresIn: 100000});
-
-    return formatJSONResponse({
-        body: signedUrl,
-        event,
-    });
+import type { S3Event } from 'aws-lambda';
+import parse from 'csv-parser';
+import { S3 } from '@aws-sdk/client-s3';
+import { SQSClient, SendMessageCommand } from '@aws-sdk/client-sqs';
+
+const REGION = 'eu-central-1';
+const s3 = new S3({
+    region: REGION,
+});
+const sqsClient = new SQSClient({ region: REGION });
+
+const importFileParser = async (
+    event: S3Event
+) => {
+    const  { s3: { object, bucket } } = event.Records[0];
+    const records = [];
+    try {
+        const stream = (await s3.getObject({
+            Bucket: bucket.name,
+            Key: object.key,
+        })).Body;
+
+        const parser = stream.pipe(parse());
+        for await (const record of parser) {
+            records.push(record);
+        }
+
+        console.log(records);
+
+        const parsedKey = `parsed/${object.key.split('/')[1]}`;
+
+        console.log(parsedKey);
+        console.log("start sending messages to the queue");
+        for (const record of records) {
+            console.log('sending record:');
+
+            const result = await sqsClient.send(
+                new SendMessageCommand({
+                    QueueUrl:
+                        `https://sqs.eu-central-1.amazonaws.com/434889505863/catalogItemsQueue`,
+                    MessageBody: JSON.stringify(record),
+                })
+            );
+            console.log('result', result);
+        }
+        console.log("complete sending messages to the queue");
+
+        await s3.copyObject({
+            Bucket: bucket.name,
+            CopySource: `${bucket.name}/${object.key}`,
+            Key: parsedKey
+        }).then(() => {
+            s3.deleteObject({
+                Bucket: bucket.name,
+                Key: object.key,
+            })
+        })
+    } catch (error) {
+        console.error('Error while downloading object from S3', error.message)
+        throw error
+    }
 };
 
-export const main = middyfy(importProductsFile);
+export const main = importFileParser;

import-service/src/functions/importProductsFile/index.ts

@@ -1,13 +1,18 @@
-import { handlerPath } from '@libs/handler-resolver';
+import { handlerPath } from "@libs/handler-resolver";
 
 export default {
     handler: `${handlerPath(__dirname)}/handler.main`,
     events: [
         {
-            http: {
-                method: 'get',
-                path: 'import',
-                cors: true,
+            s3: {
+                bucket: "file-parcer-shop-stuff",
+                event: "s3:ObjectCreated:*",
+                existing: true,
+                forceDeploy: true,
+                rules: [
+                    { prefix: 'uploads/' },
+                    { suffix: '.csv' }
+                ]
             },
         },
     ],

package.json

@@ -0,0 +1,5 @@
+{
+  "dependencies": {
+    "@aws-sdk/client-sqs": "^3.572.0"
+  }
+}