From d39a643405b0c4e4924d103e92222bddeac38ae6 Mon Sep 17 00:00:00 2001 From: Mugdhesh Pandkar <35412291+MugPand@users.noreply.github.com> Date: Fri, 26 Apr 2024 18:27:07 -0400 Subject: [PATCH] Feature 1144 model table (#1169) * added initial files * add updated user test changes * initial endpoints + tests * working initial endpoints * working updated endpoints * added delete_all & get_all endpoitns * add & update model test files * :art: Auto-generated directory tree for repository in Architecture.md --------- Co-authored-by: MugPand --- .github/Architecture.md | 296 +++++++++--------- .../functions/src/model/create_model.ts | 51 +++ .../functions/src/model/delete_all_model.ts | 78 +++++ .../functions/src/model/delete_model.ts | 43 +++ .../functions/src/model/get_all_model.ts | 49 +++ .../packages/functions/src/model/get_model.ts | 45 +++ .../src/model/tests/create_model.test.ts | 75 +++++ .../src/model/tests/delete_all_model.test.ts | 123 ++++++++ .../src/model/tests/delete_model.test.ts | 105 +++++++ .../src/model/tests/get_all_model.test.ts | 60 ++++ .../src/model/tests/get_model.test.ts | 73 +++++ serverless/stacks/AppStack.ts | 40 +++ 12 files changed, 890 insertions(+), 148 deletions(-) create mode 100644 serverless/packages/functions/src/model/create_model.ts create mode 100644 serverless/packages/functions/src/model/delete_all_model.ts create mode 100644 serverless/packages/functions/src/model/delete_model.ts create mode 100644 serverless/packages/functions/src/model/get_all_model.ts create mode 100644 serverless/packages/functions/src/model/get_model.ts create mode 100644 serverless/packages/functions/src/model/tests/create_model.test.ts create mode 100644 serverless/packages/functions/src/model/tests/delete_all_model.test.ts create mode 100644 serverless/packages/functions/src/model/tests/delete_model.test.ts create mode 100644 serverless/packages/functions/src/model/tests/get_all_model.test.ts create mode 100644 serverless/packages/functions/src/model/tests/get_model.test.ts diff --git a/.github/Architecture.md b/.github/Architecture.md index d1992046..c517edb9 100644 --- a/.github/Architecture.md +++ b/.github/Architecture.md @@ -7,228 +7,228 @@ | |- 📂 training: | | |- 📂 routes: | | | |- 📂 tabular: -| | | | |- 📜 schemas.py -| | | | |- 📜 __init__.py | | | | |- 📜 tabular.py -| | | |- 📂 datasets: -| | | | |- 📂 default: -| | | | | |- 📜 schemas.py -| | | | | |- 📜 __init__.py -| | | | | |- 📜 columns.py | | | | |- 📜 __init__.py +| | | | |- 📜 schemas.py | | | |- 📂 image: +| | | | |- 📜 __init__.py | | | | |- 📜 image.py | | | | |- 📜 schemas.py +| | | |- 📂 datasets: +| | | | |- 📂 default: +| | | | | |- 📜 columns.py +| | | | | |- 📜 __init__.py +| | | | | |- 📜 schemas.py | | | | |- 📜 __init__.py +| | | |- 📜 __init__.py | | | |- 📜 schemas.py +| | |- 📂 middleware: | | | |- 📜 __init__.py +| | | |- 📜 health_check_middleware.py | | |- 📂 core: +| | | |- 📜 trainer.py | | | |- 📜 criterion.py | | | |- 📜 dl_model.py : torch model based on user specifications from drag and drop | | | |- 📜 dataset.py : read in the dataset through URL or file upload | | | |- 📜 __init__.py | | | |- 📜 authenticator.py -| | | |- 📜 trainer.py | | | |- 📜 optimizer.py : what optimizer to use (ie: SGD or Adam for now) -| | |- 📂 middleware: -| | | |- 📜 __init__.py -| | | |- 📜 health_check_middleware.py +| | |- 📜 asgi.py | | |- 📜 settings.py -| | |- 📜 urls.py | | |- 📜 __init__.py | | |- 📜 wsgi.py -| | |- 📜 asgi.py +| | |- 📜 urls.py +| |- 📜 README.md | |- 📜 docker-compose.yml -| |- 📜 docker-compose.prod.yml +| |- 📜 cli.py | |- 📜 pyproject.toml -| |- 📜 README.md | |- 📜 poetry.lock -| |- 📜 cli.py -| |- 📜 environment.yml -| |- 📜 Dockerfile.prod +| |- 📜 pytest.ini | |- 📜 Dockerfile | |- 📜 manage.py -| |- 📜 pytest.ini +| |- 📜 environment.yml +| |- 📜 docker-compose.prod.yml +| |- 📜 Dockerfile.prod ``` ## Frontend Architecture ``` 📦 frontend -| |- 📂 layer_docs: -| | |- 📜 Linear.md : Doc for Linear layer -| | |- 📜 Softmax.md : Doc for Softmax layer -| | |- 📜 softmax_equation.png : PNG file of Softmax equation -| | |- 📜 ReLU.md : Doc for ReLU later | |- 📂 public: | | |- 📂 images: -| | | |- 📂 wiki_images: -| | | | |- 📜 maxpool2d.gif -| | | | |- 📜 conv2d.gif -| | | | |- 📜 softmax_equation.png : PNG file of Softmax equation -| | | | |- 📜 tanh_equation.png -| | | | |- 📜 dropout_diagram.png -| | | | |- 📜 batchnorm_diagram.png -| | | | |- 📜 tanh_plot.png -| | | | |- 📜 conv2d2.gif -| | | | |- 📜 sigmoid_equation.png -| | | | |- 📜 avgpool_maxpool.gif -| | | |- 📂 learn_mod_images: -| | | | |- 📜 lossExampleEquation.png -| | | | |- 📜 sigmoidactivation.png -| | | | |- 📜 neuralnet.png -| | | | |- 📜 binarystepactivation.png -| | | | |- 📜 lossExample.png -| | | | |- 📜 LeakyReLUactivation.png -| | | | |- 📜 lossExampleTable.png -| | | | |- 📜 tanhactivation.png -| | | | |- 📜 robotImage.jpg -| | | | |- 📜 ReLUactivation.png -| | | | |- 📜 neuron.png -| | | | |- 📜 neuronWithEquation.png -| | | | |- 📜 sigmoidfunction.png | | | |- 📂 logos: | | | | |- 📂 dlp_branding: -| | | | | |- 📜 dlp-logo.svg : DLP Logo, duplicate of files in public, but essential as the frontend can't read public | | | | | |- 📜 dlp-logo.png : DLP Logo, duplicate of files in public, but essential as the frontend can't read public -| | | | |- 📜 google.png -| | | | |- 📜 pytorch-logo.png +| | | | | |- 📜 dlp-logo.svg : DLP Logo, duplicate of files in public, but essential as the frontend can't read public +| | | | |- 📜 dsgt-logo-white-back.png | | | | |- 📜 python-logo.png -| | | | |- 📜 dsgt-logo-dark.png -| | | | |- 📜 aws-logo.png -| | | | |- 📜 github.png +| | | | |- 📜 google.png | | | | |- 📜 pandas-logo.png | | | | |- 📜 react-logo.png -| | | | |- 📜 dsgt-logo-white-back.png | | | | |- 📜 flask-logo.png +| | | | |- 📜 aws-logo.png +| | | | |- 📜 github.png +| | | | |- 📜 dsgt-logo-dark.png | | | | |- 📜 dsgt-logo-light.png +| | | | |- 📜 pytorch-logo.png +| | | |- 📂 learn_mod_images: +| | | | |- 📜 neuron.png +| | | | |- 📜 ReLUactivation.png +| | | | |- 📜 LeakyReLUactivation.png +| | | | |- 📜 lossExampleEquation.png +| | | | |- 📜 lossExampleTable.png +| | | | |- 📜 robotImage.jpg +| | | | |- 📜 neuralnet.png +| | | | |- 📜 sigmoidfunction.png +| | | | |- 📜 lossExample.png +| | | | |- 📜 tanhactivation.png +| | | | |- 📜 binarystepactivation.png +| | | | |- 📜 sigmoidactivation.png +| | | | |- 📜 neuronWithEquation.png +| | | |- 📂 wiki_images: +| | | | |- 📜 sigmoid_equation.png +| | | | |- 📜 conv2d.gif +| | | | |- 📜 conv2d2.gif +| | | | |- 📜 avgpool_maxpool.gif +| | | | |- 📜 softmax_equation.png : PNG file of Softmax equation +| | | | |- 📜 dropout_diagram.png +| | | | |- 📜 batchnorm_diagram.png +| | | | |- 📜 maxpool2d.gif +| | | | |- 📜 tanh_equation.png +| | | | |- 📜 tanh_plot.png | | | |- 📜 demo_video.gif : GIF tutorial of a simple classification training session -| | |- 📜 robots.txt +| | |- 📜 dlp-logo.ico : DLP Logo | | |- 📜 manifest.json : Default React file for choosing icon based on | | |- 📜 index.html : Base HTML file that will be initially rendered -| | |- 📜 dlp-logo.ico : DLP Logo +| | |- 📜 robots.txt +| |- 📂 layer_docs: +| | |- 📜 Softmax.md : Doc for Softmax layer +| | |- 📜 Linear.md : Doc for Linear layer +| | |- 📜 softmax_equation.png : PNG file of Softmax equation +| | |- 📜 ReLU.md : Doc for ReLU later | |- 📂 src: -| | |- 📂 pages: -| | | |- 📂 train: -| | | | |- 📜 [train_space_id].tsx -| | | | |- 📜 index.tsx -| | | |- 📜 dashboard.tsx -| | | |- 📜 learn.tsx -| | | |- 📜 settings.tsx -| | | |- 📜 about.tsx -| | | |- 📜 feedback.tsx -| | | |- 📜 wiki.tsx -| | | |- 📜 forgot.tsx -| | | |- 📜 _document.tsx -| | | |- 📜 LearnContent.tsx -| | | |- 📜 _app.tsx -| | | |- 📜 login.tsx +| | |- 📂 __tests__: +| | | |- 📂 common: +| | | | |- 📂 components: +| | | | | |- 📜 TitleText.test.tsx +| | |- 📂 backend_outputs: +| | | |- 📜 my_deep_learning_model.onnx : Last ONNX file output +| | | |- 📜 model.pkl +| | | |- 📜 model.pt : Last model.pt output | | |- 📂 common: -| | | |- 📂 utils: -| | | | |- 📜 dateFormat.ts -| | | | |- 📜 dndHelpers.ts -| | | | |- 📜 firebase.ts -| | | |- 📂 components: -| | | | |- 📜 Spacer.tsx -| | | | |- 📜 DlpTooltip.tsx -| | | | |- 📜 TitleText.tsx -| | | | |- 📜 EmailInput.tsx -| | | | |- 📜 HtmlTooltip.tsx -| | | | |- 📜 ClientOnlyPortal.tsx -| | | | |- 📜 Footer.tsx -| | | | |- 📜 NavBarMain.tsx | | | |- 📂 styles: | | | | |- 📜 Home.module.css | | | | |- 📜 globals.css | | | |- 📂 redux: | | | | |- 📜 hooks.ts | | | | |- 📜 store.ts +| | | | |- 📜 train.ts | | | | |- 📜 backendApi.ts | | | | |- 📜 userLogin.ts -| | | | |- 📜 train.ts +| | | |- 📂 utils: +| | | | |- 📜 dateFormat.ts +| | | | |- 📜 firebase.ts +| | | | |- 📜 dndHelpers.ts +| | | |- 📂 components: +| | | | |- 📜 EmailInput.tsx +| | | | |- 📜 HtmlTooltip.tsx +| | | | |- 📜 NavBarMain.tsx +| | | | |- 📜 Footer.tsx +| | | | |- 📜 DlpTooltip.tsx +| | | | |- 📜 ClientOnlyPortal.tsx +| | | | |- 📜 Spacer.tsx +| | | | |- 📜 TitleText.tsx | | |- 📂 features: +| | | |- 📂 LearnMod: +| | | | |- 📜 MCQuestion.tsx +| | | | |- 📜 ModulesSideBar.tsx +| | | | |- 📜 ImageComponent.tsx +| | | | |- 📜 ClassCard.tsx +| | | | |- 📜 FRQuestion.tsx +| | | | |- 📜 Exercise.tsx +| | | | |- 📜 LearningModulesContent.tsx +| | | |- 📂 OpenAi: +| | | | |- 📜 openAiUtils.ts +| | | |- 📂 Dashboard: +| | | | |- 📂 redux: +| | | | | |- 📜 dashboardApi.ts +| | | | |- 📂 components: +| | | | | |- 📜 TrainBarChart.tsx +| | | | | |- 📜 TrainDoughnutChart.tsx +| | | | | |- 📜 TrainDataGrid.tsx | | | |- 📂 Train: +| | | | |- 📂 redux: +| | | | | |- 📜 trainspaceSlice.ts +| | | | | |- 📜 trainspaceApi.ts +| | | | |- 📂 types: +| | | | | |- 📜 trainTypes.ts | | | | |- 📂 constants: | | | | | |- 📜 trainConstants.ts -| | | | |- 📂 components: -| | | | | |- 📜 CreateTrainspace.tsx -| | | | | |- 📜 TrainspaceLayout.tsx -| | | | | |- 📜 DatasetStepLayout.tsx | | | | |- 📂 features: -| | | | | |- 📂 Image: -| | | | | | |- 📂 constants: -| | | | | | | |- 📜 imageConstants.ts -| | | | | | |- 📂 components: -| | | | | | | |- 📜 ImageTrainspace.tsx -| | | | | | | |- 📜 ImageParametersStep.tsx -| | | | | | | |- 📜 ImageFlow.tsx -| | | | | | | |- 📜 ImageDatasetStep.tsx -| | | | | | | |- 📜 ImageReviewStep.tsx +| | | | | |- 📂 Tabular: | | | | | | |- 📂 redux: -| | | | | | | |- 📜 imageActions.ts -| | | | | | | |- 📜 imageApi.ts +| | | | | | | |- 📜 tabularActions.ts +| | | | | | | |- 📜 tabularApi.ts | | | | | | |- 📂 types: -| | | | | | | |- 📜 imageTypes.ts -| | | | | | |- 📜 index.ts -| | | | | |- 📂 Tabular: +| | | | | | | |- 📜 tabularTypes.ts | | | | | | |- 📂 constants: | | | | | | | |- 📜 tabularConstants.ts | | | | | | |- 📂 components: -| | | | | | | |- 📜 TabularDatasetStep.tsx -| | | | | | | |- 📜 TabularReviewStep.tsx -| | | | | | | |- 📜 TabularFlow.tsx | | | | | | | |- 📜 TabularTrainspace.tsx +| | | | | | | |- 📜 TabularReviewStep.tsx | | | | | | | |- 📜 TabularParametersStep.tsx +| | | | | | | |- 📜 TabularDatasetStep.tsx +| | | | | | | |- 📜 TabularFlow.tsx +| | | | | | |- 📜 index.ts +| | | | | |- 📂 Image: | | | | | | |- 📂 redux: -| | | | | | | |- 📜 tabularApi.ts -| | | | | | | |- 📜 tabularActions.ts +| | | | | | | |- 📜 imageApi.ts +| | | | | | | |- 📜 imageActions.ts | | | | | | |- 📂 types: -| | | | | | | |- 📜 tabularTypes.ts +| | | | | | | |- 📜 imageTypes.ts +| | | | | | |- 📂 constants: +| | | | | | | |- 📜 imageConstants.ts +| | | | | | |- 📂 components: +| | | | | | | |- 📜 ImageReviewStep.tsx +| | | | | | | |- 📜 ImageTrainspace.tsx +| | | | | | | |- 📜 ImageFlow.tsx +| | | | | | | |- 📜 ImageParametersStep.tsx +| | | | | | | |- 📜 ImageDatasetStep.tsx | | | | | | |- 📜 index.ts -| | | | |- 📂 redux: -| | | | | |- 📜 trainspaceApi.ts -| | | | | |- 📜 trainspaceSlice.ts -| | | | |- 📂 types: -| | | | | |- 📜 trainTypes.ts +| | | | |- 📂 components: +| | | | | |- 📜 CreateTrainspace.tsx +| | | | | |- 📜 DatasetStepLayout.tsx +| | | | | |- 📜 TrainspaceLayout.tsx | | | |- 📂 Feedback: | | | | |- 📂 redux: | | | | | |- 📜 feedbackApi.ts -| | | |- 📂 Dashboard: -| | | | |- 📂 components: -| | | | | |- 📜 TrainDoughnutChart.tsx -| | | | | |- 📜 TrainBarChart.tsx -| | | | | |- 📜 TrainDataGrid.tsx -| | | | |- 📂 redux: -| | | | | |- 📜 dashboardApi.ts -| | | |- 📂 LearnMod: -| | | | |- 📜 FRQuestion.tsx -| | | | |- 📜 MCQuestion.tsx -| | | | |- 📜 LearningModulesContent.tsx -| | | | |- 📜 Exercise.tsx -| | | | |- 📜 ClassCard.tsx -| | | | |- 📜 ImageComponent.tsx -| | | | |- 📜 ModulesSideBar.tsx -| | | |- 📂 OpenAi: -| | | | |- 📜 openAiUtils.ts -| | |- 📂 backend_outputs: -| | | |- 📜 my_deep_learning_model.onnx : Last ONNX file output -| | | |- 📜 model.pkl -| | | |- 📜 model.pt : Last model.pt output -| | |- 📂 __tests__: -| | | |- 📂 common: -| | | | |- 📂 components: -| | | | | |- 📜 TitleText.test.tsx -| | |- 📜 next-env.d.ts +| | |- 📂 pages: +| | | |- 📂 train: +| | | | |- 📜 [train_space_id].tsx +| | | | |- 📜 index.tsx +| | | |- 📜 _app.tsx +| | | |- 📜 forgot.tsx +| | | |- 📜 about.tsx +| | | |- 📜 settings.tsx +| | | |- 📜 _document.tsx +| | | |- 📜 feedback.tsx +| | | |- 📜 dashboard.tsx +| | | |- 📜 learn.tsx +| | | |- 📜 LearnContent.tsx +| | | |- 📜 login.tsx +| | | |- 📜 wiki.tsx +| | |- 📜 constants.ts | | |- 📜 iris.csv : Sample CSV data | | |- 📜 GlobalStyle.ts -| | |- 📜 constants.ts -| |- 📜 next-env.d.ts -| |- 📜 .eslintignore -| |- 📜 jest.config.ts +| | |- 📜 next-env.d.ts | |- 📜 pnpm-lock.yaml -| |- 📜 .eslintrc.json -| |- 📜 next.config.js | |- 📜 tsconfig.json | |- 📜 package.json +| |- 📜 .eslintrc.json +| |- 📜 next.config.js +| |- 📜 next-env.d.ts +| |- 📜 jest.config.ts +| |- 📜 .eslintignore ``` diff --git a/serverless/packages/functions/src/model/create_model.ts b/serverless/packages/functions/src/model/create_model.ts new file mode 100644 index 00000000..4d496261 --- /dev/null +++ b/serverless/packages/functions/src/model/create_model.ts @@ -0,0 +1,51 @@ +import { APIGatewayProxyEventV2 } from "aws-lambda"; +import parseJwt from "@dlp-sst-app/core/src/parseJwt"; +import { v4 as uuidv4 } from 'uuid'; +import { DynamoDBClient, PutItemCommand } from '@aws-sdk/client-dynamodb'; + +export async function handler(event : APIGatewayProxyEventV2) { + if (event) { + const user_id: string = parseJwt(event.headers.authorization ?? "")["user_id"]; + const model_id = uuidv4(); + + const client = new DynamoDBClient({}); + + const eventBody = JSON.parse(event.body? event.body : ""); + const putCommand: PutItemCommand = new PutItemCommand({ + TableName: "ModelTable", + Item: + { + user_id: {"S": user_id}, + model_id: {"S": model_id}, + name: {"S": eventBody['name']}, + model_structure: {"S": eventBody['model_structure']} + } + }); + + if (putCommand == null) + { + return { + statusCode: 400, + body: JSON.stringify({ message: "Invalid request body" }) + } + } + + const response = await client.send(putCommand); + + if (response.$metadata.httpStatusCode != 200) { + return { + statusCode: 500, + body: JSON.stringify({ message: "Internal server error."}) + }; + } + + return { + statusCode: 200, + body: JSON.stringify({ model_id: model_id, message: "Successfully created a new model."}) + }; + } + return { + statusCode: 404, + body: JSON.stringify({ message: "Not Found" }), + }; +}; \ No newline at end of file diff --git a/serverless/packages/functions/src/model/delete_all_model.ts b/serverless/packages/functions/src/model/delete_all_model.ts new file mode 100644 index 00000000..9ff62863 --- /dev/null +++ b/serverless/packages/functions/src/model/delete_all_model.ts @@ -0,0 +1,78 @@ +import { APIGatewayProxyEventV2 } from "aws-lambda"; +import parseJwt from "@dlp-sst-app/core/src/parseJwt"; +import { DynamoDBClient, QueryCommand, BatchExecuteStatementCommand, BatchStatementRequest } from '@aws-sdk/client-dynamodb'; + +export async function handler(event : APIGatewayProxyEventV2) { + if (event) { + const uid: string = parseJwt(event.headers.authorization ?? "")[ + "user_id" + ]; + const client = new DynamoDBClient({}); + + const foundModelIds: Array = [] + const deletedModelIds: Array = []; + let lastEvaluatedKey = undefined; + do { + const queryCommand: QueryCommand = new QueryCommand({ + TableName: "ModelTable", + IndexName: "user_id_index", + KeyConditionExpression: "user_id = :uid", + ExpressionAttributeValues: { + ":uid" : {"S": uid} + }, + Limit: 25, + ExclusiveStartKey: lastEvaluatedKey + }); + + const currentModelIds: Array = [] + const getResults = await client.send(queryCommand); + if (getResults["Count"] !== 0 && getResults['Items']) { + const page: Array = getResults['Items'].map(model => model['model_id'].S); + page.forEach(model_id => { + if (model_id) foundModelIds.push(model_id); + if (model_id) currentModelIds.push(model_id); + }); + } else { + return { + statusCode: 405, + body: JSON.stringify({ message: "no models deleted: none found associated with user"}) + } + } + + lastEvaluatedKey = getResults.LastEvaluatedKey; + + const statements: BatchStatementRequest[] = []; + for (const model_id of currentModelIds) { + statements.push( { + Statement: "DELETE FROM ModelTable where model_id=?", + Parameters: [{ "S": model_id.valueOf() }] + }); + deletedModelIds.push(model_id.valueOf()); + } + + const command = new BatchExecuteStatementCommand({ + Statements: statements + }); + + const response = await client.send(command); + + if (response.$metadata.httpStatusCode == undefined || response.$metadata.httpStatusCode != 200) + { + return { + statusCode: 404, + body: JSON.stringify({ message : "Delete operation failed" }) + } + } + } while (lastEvaluatedKey !== undefined); + if (deletedModelIds.length !== 0) { + return { + statusCode: 200, + body: JSON.stringify({ message: "Succesfully deleted models", model_ids : foundModelIds}) + }; + } + } + return { + statusCode: 400, + body: JSON.stringify({ message: "Event Not Found" }), + }; +}; \ No newline at end of file diff --git a/serverless/packages/functions/src/model/delete_model.ts b/serverless/packages/functions/src/model/delete_model.ts new file mode 100644 index 00000000..c5c0d619 --- /dev/null +++ b/serverless/packages/functions/src/model/delete_model.ts @@ -0,0 +1,43 @@ +import { APIGatewayProxyEventV2 } from "aws-lambda"; +import { DynamoDBClient, DeleteItemCommand } from '@aws-sdk/client-dynamodb'; + +export async function handler(event : APIGatewayProxyEventV2) { + let queryParams = null; + if (event && (queryParams = event['pathParameters']) != null) { + const model_id: string | undefined = queryParams['model_id']; + if (model_id == undefined) { + return { + statusCode: 401, + body: JSON.stringify({message: "Malformed request content - model ID missing."}) + } + } + + const client = new DynamoDBClient({}); + + const command = new DeleteItemCommand({ + TableName : "ModelTable", + Key : + { + model_id: {"S": model_id} + } + }); + + const response = await client.send(command); + + if (response.$metadata.httpStatusCode == undefined || response.$metadata.httpStatusCode != 200) + { + return { + statusCode: 404, + body: JSON.stringify({ message : "Delete operation failed" }) + } + } + return { + statusCode: 200, + body: "Successfully deleted model with id: " + model_id + } + } + return { + statusCode: 400, + body: JSON.stringify({ message : "Malformed request content" }), + }; +}; \ No newline at end of file diff --git a/serverless/packages/functions/src/model/get_all_model.ts b/serverless/packages/functions/src/model/get_all_model.ts new file mode 100644 index 00000000..8b43662f --- /dev/null +++ b/serverless/packages/functions/src/model/get_all_model.ts @@ -0,0 +1,49 @@ +import { APIGatewayProxyEventV2 } from "aws-lambda"; +import parseJwt from "@dlp-sst-app/core/src/parseJwt"; +import { DynamoDBClient, QueryCommand } from '@aws-sdk/client-dynamodb'; + +export async function handler(event : APIGatewayProxyEventV2) { + if (event) { + const uid: string = parseJwt(event.headers.authorization ?? "")[ + "user_id" + ]; + + const client = new DynamoDBClient({}); + const fetchedModelIds: Array = []; + let lastEvaluatedKey = undefined; + do { + const queryCommand: QueryCommand = new QueryCommand({ + TableName: "ModelTable", + IndexName: "user_id_index", + KeyConditionExpression: "user_id = :uid", + ExpressionAttributeValues: { + ":uid" : {"S": uid} + }, + ExclusiveStartKey: lastEvaluatedKey + }); + + const results = await client.send(queryCommand); + + if (results.Items && results.Count != 0) { + const page: Array = results['Items']?.map(model => model['model_id'].S); + page.forEach(model_id => { if (model_id) fetchedModelIds.push(model_id); }); + } else { + return { + statusCode: 404, + body: JSON.stringify({message: "no models associated with user"}) + } + } + lastEvaluatedKey = results.LastEvaluatedKey; + } while (lastEvaluatedKey !== undefined); + + return { + statusCode: 200, + body: JSON.stringify({ model_ids : fetchedModelIds}) + }; + } + + return { + statusCode: 400, + body: JSON.stringify({ message: "Not Found" }), + }; +}; \ No newline at end of file diff --git a/serverless/packages/functions/src/model/get_model.ts b/serverless/packages/functions/src/model/get_model.ts new file mode 100644 index 00000000..a82805f4 --- /dev/null +++ b/serverless/packages/functions/src/model/get_model.ts @@ -0,0 +1,45 @@ +import { APIGatewayProxyEventV2 } from "aws-lambda"; +import { DynamoDBClient, GetItemCommand } from '@aws-sdk/client-dynamodb'; + +export async function handler(event : APIGatewayProxyEventV2) { + let queryParams = null; + + if (event && ((queryParams = event['pathParameters']) != null)) + { + const model_id: string | undefined = queryParams['model_id']; + if (model_id == undefined) { + return { + statusCode: 401, + body: JSON.stringify({message: "Malformed request content - model ID missing."}) + } + } + + const client: DynamoDBClient = new DynamoDBClient({}); + + const command : GetItemCommand = new GetItemCommand({ + TableName : "ModelTable", + Key : + { + model_id : {"S": model_id} + } + }); + + const response = await client.send(command); + + if (!response.Item) + { + return { + statusCode: 404, + body: JSON.stringify({message: "Provided Model ID does not exist"}) + } + } + return { + statusCode: 200, + body: JSON.stringify({message: "Successfully retrieved Model data", model: response.Item}) + } + } + return { + statusCode: 400, + body: JSON.stringify({message: "Malformed request content"}) + }; +} \ No newline at end of file diff --git a/serverless/packages/functions/src/model/tests/create_model.test.ts b/serverless/packages/functions/src/model/tests/create_model.test.ts new file mode 100644 index 00000000..fab3cf65 --- /dev/null +++ b/serverless/packages/functions/src/model/tests/create_model.test.ts @@ -0,0 +1,75 @@ +import { APIGatewayProxyEventV2 } from "aws-lambda"; +import { beforeEach, expect, it, vi} from "vitest"; +import { DynamoDBClient, PutItemCommand} from '@aws-sdk/client-dynamodb'; +import { mockClient } from 'aws-sdk-client-mock'; +import { handler } from '../create_model'; + +//mocks parseJwt so that the call just returns whatever the input is +vi.mock('@dlp-sst-app/core/src/parseJwt', async () => { + return { + default: vi.fn().mockImplementation(input => input), + } +}) + +beforeEach(async () => { + ddbMock.reset(); +}) + +const ddbMock = mockClient(DynamoDBClient); + +it("test successful create model call", async () => { + ddbMock.on(PutItemCommand).resolves({ + $metadata: { + httpStatusCode: 200, + } + }) + // @ts-expect-error : error doesn't affect functionality. We don't need the rest of the event, and it's really long for no reason + const event: APIGatewayProxyEventV2 = { + headers: { + authorization: 'abcd', + + }, + body: '{\n' + + ' "user_id": "SOME USER ID",\n' + + ' "model_id": "SOME MODEL ID",\n' + + ' "name": "SOME NAME",\n' + + ' "model_structure": "SOME MODEL STRUCTURE"\n' + + '}', + } + const result = await handler(event); + expect(result.statusCode).toEqual(200); +}); + +it("test internal service error", async () => { + ddbMock.on(PutItemCommand).resolves({ + $metadata: { + httpStatusCode: 456, + } + }) + // @ts-expect-error : error doesn't affect functionality. We don't need the rest of the event, and it's really long for no reason + const event: APIGatewayProxyEventV2 = { + headers: { + authorization: 'abcd', + }, + body: '{\n' + + ' "user_id": "SOME USER ID",\n' + + ' "model_id": "SOME MODEL ID",\n' + + ' "name": "SOME NAME",\n' + + ' "model_structure": "SOME MODEL STRUCTURE"\n' + + '}', + } + + const result = await handler(event); + expect(result.statusCode).toEqual(500); + }); + +it("test undefined event", async () => { + ddbMock.on(PutItemCommand).resolves({ + $metadata: { + httpStatusCode: 400, + } + }) + // @ts-expect-error : we are trying to cause an error + const result = await handler(undefined); + expect(result.statusCode).toEqual(404); + }); \ No newline at end of file diff --git a/serverless/packages/functions/src/model/tests/delete_all_model.test.ts b/serverless/packages/functions/src/model/tests/delete_all_model.test.ts new file mode 100644 index 00000000..a2585c23 --- /dev/null +++ b/serverless/packages/functions/src/model/tests/delete_all_model.test.ts @@ -0,0 +1,123 @@ +import { APIGatewayProxyEventV2 } from "aws-lambda"; +import { beforeEach, expect, it, vi} from "vitest"; +import { DynamoDBClient, QueryCommand, BatchExecuteStatementCommand } from '@aws-sdk/client-dynamodb'; +import { mockClient } from 'aws-sdk-client-mock'; +import { handler } from '../delete_all_model'; + + +//mocks parseJwt so that the call just returns whatever the input is +vi.mock('@dlp-sst-app/core/src/parseJwt', async () => { + return { + default: (input: String) => ({ user_id: input }) + } +}); + +beforeEach(async () => { + ddbMock.reset(); +}); + +const ddbMock = mockClient(DynamoDBClient); + +it("test successful delete all model call", async () => { + ddbMock.on(QueryCommand).resolves({ + "Items": [{ + "model_id": { "S": "test id" }, + }], + "Count": 1 + }); + ddbMock.on(BatchExecuteStatementCommand).resolves({ + $metadata: { + httpStatusCode: 200, + } + }) + + // @ts-expect-error : error doesn't affect functionality. We don't need the rest of the event, and it's really long for no reason + const event: APIGatewayProxyEventV2 = { + headers: { + authorization: 'abcd', + }, + } + const result = await handler(event); + expect(result.statusCode).toEqual(200); +}); + + +it("test no batch delete response call", async () => { + ddbMock.on(QueryCommand).resolves({ + "Items": [{ + "user_id":{"S":"abcd"}, + "model_id": { "S": "test id" }, + }], + "Count": 4 + }); + ddbMock.on(BatchExecuteStatementCommand).resolves({ + $metadata: { + httpStatusCode: undefined, + } + }) + + // @ts-expect-error : error doesn't affect functionality. We don't need the rest of the event, and it's really long for no reason + const event: APIGatewayProxyEventV2 = { + headers: { + authorization: 'abcd', + }, + } + + const result = await handler(event); + expect(result.statusCode).toEqual(404); +}); + +it("test incorrect batch delete response failed call", async () => { + ddbMock.on(QueryCommand).resolves({ + "Items": [{ + "user_id":{"S":"abcd"}, + "model_id": { "S": "test id" }, + }], + "Count": 4 + }); + ddbMock.on(BatchExecuteStatementCommand).resolves({ + $metadata: { + httpStatusCode: 267, + } + }) + + // @ts-expect-error : error doesn't affect functionality. We don't need the rest of the event, and it's really long for no reason + const event: APIGatewayProxyEventV2 = { + headers: { + authorization: 'abcd', + }, + } + + const result = await handler(event); + expect(result.statusCode).toEqual(404); +}); + + +it("test delete all on no existing model call", async () => { + ddbMock.on(QueryCommand).resolves({ + "Items": [], + "Count": 0 + }); + ddbMock.on(BatchExecuteStatementCommand).resolves({ + $metadata: { + httpStatusCode: 200, + } + }) + + // @ts-expect-error : error doesn't affect functionality. We don't need the rest of the event, and it's really long for no reason + const event: APIGatewayProxyEventV2 = { + headers: { + authorization: 'abcd', + }, + } + + const result = await handler(event); + expect(result.statusCode).toEqual(405); +}); + +it("test malformed call", async () => { + + // @ts-expect-error : we are trying to cause an error + const result = await handler(undefined); + expect(result.statusCode).toEqual(400); +}); \ No newline at end of file diff --git a/serverless/packages/functions/src/model/tests/delete_model.test.ts b/serverless/packages/functions/src/model/tests/delete_model.test.ts new file mode 100644 index 00000000..e072e33c --- /dev/null +++ b/serverless/packages/functions/src/model/tests/delete_model.test.ts @@ -0,0 +1,105 @@ +import { APIGatewayProxyEventV2 } from "aws-lambda"; +import { beforeEach, expect, it, vi} from "vitest"; +import { DynamoDBClient, DeleteItemCommand } from '@aws-sdk/client-dynamodb'; +import { mockClient } from 'aws-sdk-client-mock'; +import { handler } from '../delete_model'; + +//mocks parseJwt so that the call just returns whatever the input is +vi.mock('@dlp-sst-app/core/src/parseJwt', async () => { + return { + default: vi.fn().mockImplementation(input => input), + } +}) + +beforeEach(async () => { + ddbMock.reset(); +}) + +const ddbMock = mockClient(DynamoDBClient); + +it("test successful delete model call", async () => { + ddbMock.on(DeleteItemCommand).resolves({ + $metadata: { + httpStatusCode: 200, + } + }) + // @ts-expect-error : error doesn't affect functionality. We don't need the rest of the event, and it's really long for no reason + const event: APIGatewayProxyEventV2 = { + headers: { + authorization: 'abcd', + }, + pathParameters: { + model_id: "model_id" + }, +} + + const result = await handler(event); + expect(result.statusCode).toEqual(200); +}); + + +it("test no response failed operation call", async () => { + ddbMock.on(DeleteItemCommand).resolves({ + $metadata: { + httpStatusCode: undefined, + } + }) + + // @ts-expect-error : error doesn't affect functionality. We don't need the rest of the event, and it's really long for no reason + const event: APIGatewayProxyEventV2 = { + headers: { + authorization: 'abcd', + }, + pathParameters: { + model_id: "model_id" + }, + } + + const result = await handler(event); + expect(result.statusCode).toEqual(404); +}); + + +it("test different status code failed operation call", async () => { + ddbMock.on(DeleteItemCommand).resolves({ + $metadata: { + httpStatusCode: 267, + } + }) + // @ts-expect-error : error doesn't affect functionality. We don't need the rest of the event, and it's really long for no reason + const event: APIGatewayProxyEventV2 = { + headers: { + authorization: 'abcd', + }, + pathParameters: { + model_id: "model_id" + }, + } + + const result = await handler(event); + expect(result.statusCode).toEqual(404); +}); + +it("test no model id given", async () => { + ddbMock.on(DeleteItemCommand).resolves({ + $metadata: { + httpStatusCode: 267, + } + }) + + // @ts-expect-error : error doesn't affect functionality. We don't need the rest of the event, and it's really long for no reason + const event: APIGatewayProxyEventV2 = { + pathParameters: { + } + } + + const result = await handler(event); + expect(result.statusCode).toEqual(401); +}); + + +it("test malformed call", async () => { + // @ts-expect-error : we are trying to cause an error + const result = await handler(undefined); + expect(result.statusCode).toEqual(400); +}); \ No newline at end of file diff --git a/serverless/packages/functions/src/model/tests/get_all_model.test.ts b/serverless/packages/functions/src/model/tests/get_all_model.test.ts new file mode 100644 index 00000000..993284a3 --- /dev/null +++ b/serverless/packages/functions/src/model/tests/get_all_model.test.ts @@ -0,0 +1,60 @@ +import { APIGatewayProxyEventV2 } from "aws-lambda"; +import { beforeEach, expect, it, vi} from "vitest"; +import { mockClient } from 'aws-sdk-client-mock'; +import { DynamoDBClient, QueryCommand } from '@aws-sdk/client-dynamodb'; +import { handler } from '../get_all_model'; + +//mocks parseJwt so that the call just returns whatever the input is +vi.mock('@dlp-sst-app/core/src/parseJwt', async () => { + return { + default: (input: String) => ({ user_id: input }) + } +}) + +beforeEach(async () => { + ddbMock.reset(); +}) + +const ddbMock = mockClient(DynamoDBClient); + +it("test successful get all model call", async () => { + ddbMock.on(QueryCommand).resolves({ + "Items": [{ + "user_id":{"S":"abcd"}, + "model_id": { "S": "test id" }, + }], + "Count": 4 + }); + // @ts-expect-error : error doesn't affect functionality. We don't need the rest of the event, and it's really long for no reason + const event: APIGatewayProxyEventV2 = { + headers: { + authorization: 'abcd', + }, + } + + const result = await handler(event); + + expect(result.statusCode).toEqual(200); +}); + +it("test no existing models for user id", async () => { + ddbMock.on(QueryCommand).resolves({ + Items: undefined + }) + + // @ts-expect-error : error doesn't affect functionality. We don't need the rest of the event, and it's really long for no reason + const event: APIGatewayProxyEventV2 = { + headers: { + authorization: 'abcd', + }, + } + + const result = await handler(event); + expect(result.statusCode).toEqual(404); +}); + +it("test malformed request", async () => { + // @ts-expect-error : we are trying to cause an error + const result = await handler(undefined); + expect(result.statusCode).toEqual(400); +}); \ No newline at end of file diff --git a/serverless/packages/functions/src/model/tests/get_model.test.ts b/serverless/packages/functions/src/model/tests/get_model.test.ts new file mode 100644 index 00000000..29a9a6ae --- /dev/null +++ b/serverless/packages/functions/src/model/tests/get_model.test.ts @@ -0,0 +1,73 @@ +import { APIGatewayProxyEventV2 } from "aws-lambda"; +import { beforeEach, expect, it, vi} from "vitest"; +import { DynamoDBClient, GetItemCommand } from '@aws-sdk/client-dynamodb'; +import { mockClient } from 'aws-sdk-client-mock'; +import { handler } from '../get_model'; + +//mocks parseJwt so that the call just returns whatever the input is +vi.mock('@dlp-sst-app/core/src/parseJwt', async () => { + return { + default: vi.fn().mockImplementation(input => input), + } +}) + +beforeEach(async () => { + ddbMock.reset(); +}) + +const ddbMock = mockClient(DynamoDBClient); + + +it("test successful get model call", async () => { + ddbMock.on(GetItemCommand).resolves({ + Item: { modelID: { S: 'sample model id' } } + }) + + // @ts-expect-error : error doesn't affect functionality. We don't need the rest of the event, and it's really long for no reason + const event: APIGatewayProxyEventV2 = { + pathParameters: { + model_id: "some model_id" + }, + } + + const result = await handler(event); + expect(result.statusCode).toEqual(200); +}); + + +it("test no existing model id", async () => { + ddbMock.on(GetItemCommand).resolves({ + Item: undefined + }) + + // @ts-expect-error : error doesn't affect functionality. We don't need the rest of the event, and it's really long for no reason + const event: APIGatewayProxyEventV2 = { + pathParameters: { + model_id: "some model_id" + }, + } + + const result = await handler(event); + expect(result.statusCode).toEqual(404); +}); + +it("test no model id given", async () => { + ddbMock.on(GetItemCommand).resolves({ + Item: { modelId: { S: 'sample model id' } } + }) + + // @ts-expect-error : error doesn't affect functionality. We don't need the rest of the event, and it's really long for no reason + const event: APIGatewayProxyEventV2 = { + pathParameters: { + } + } + + const result = await handler(event); + expect(result.statusCode).toEqual(401); +}); + +it("test malformed request", async () => { + // @ts-expect-error : we are trying to cause an error + const result = await handler(undefined); + expect(result.statusCode).toEqual(400); +}); \ No newline at end of file diff --git a/serverless/stacks/AppStack.ts b/serverless/stacks/AppStack.ts index b910bbde..38c1eef8 100644 --- a/serverless/stacks/AppStack.ts +++ b/serverless/stacks/AppStack.ts @@ -86,6 +86,36 @@ export function AppStack({ stack }: StackContext) { handler: "packages/functions/src/user/delete_user.handler", permissions: ["dynamodb:DeleteItem"] } + }, + "POST /model": { + function: { + handler: "packages/functions/src/model/create_model.handler", + permissions: ["dynamodb:PutItem"] + } + }, + "GET /model/{model_id}": { + function : { + handler: "packages/functions/src/model/get_model.handler", + permissions: ["dynamodb:GetItem"] + } + }, + "GET /model": { + function : { + handler: "packages/functions/src/model/get_all_model.handler", + permissions: ["dynamodb:Query"] + } + }, + "DELETE /model/{model_id}": { + function : { + handler: "packages/functions/src/model/delete_model.handler", + permissions: ["dynamodb:DeleteItem"] + } + }, + "DELETE /model": { + function : { + handler: "packages/functions/src/model/delete_all_model.handler", + permissions: ["dynamodb:PartiQLDelete", "dynamodb:Query"] + } } }, }); @@ -118,5 +148,15 @@ export function AppStack({ stack }: StackContext) { api.getFunction("GET /user")?.functionName ?? "", DeleteUserFunctionName: api.getFunction("DELETE /user")?.functionName ?? "", + CreateModelFunctionName: + api.getFunction("POST /model")?.functionName ?? "", + GetModelFunctionName: + api.getFunction("GET /model/{model_id}")?.functionName ?? "", + GetAllModelFunctionName: + api.getFunction("GET /model")?.functionName ?? "", + DeleteModelFunctionName: + api.getFunction("DELETE /model/{model_id}")?.functionName ?? "", + DeleteAllModelFunctionName: + api.getFunction("DELETE /model")?.functionName ?? "", }); }