1. A, C, & E: ClearImageErrors, ImageError for Unknown Serials

.github/workflows/test.yml

@@ -22,7 +22,7 @@ jobs:
           ref: ${{github.event.pull_request.head.sha || github.sha}}
 
       - name: Docker API Build
-        run: docker build -t api .
+        run: docker build -t test -f Dockerfile-test .
 
       - name: Docker API Test Run
-        run: docker run api
+        run: docker run test

Dockerfile

@@ -1,18 +1,5 @@
-FROM ubuntu:22.04
+FROM public.ecr.aws/lambda/nodejs:18
 
-ENV HOME=/home/animl
-WORKDIR $HOME
-COPY ./ $HOME/
-
-RUN apt-get update \
-    && DEBIAN_FRONTEND=noninteractive apt-get install -y curl git
-
-RUN export NODEV='18.16.0' \
-    && curl "https://nodejs.org/dist/v${NODEV}/node-v${NODEV}-linux-x64.tar.gz" | tar -xzv \
-    && cp ./node-v${NODEV}-linux-x64/bin/node /usr/bin/ \
-    && ./node-v${NODEV}-linux-x64/bin/npm install -g npm
+COPY . ${LAMBDA_TASK_ROOT}/
 
 RUN npm install
-
-CMD npm run lint && npm test
-

Dockerfile-test

@@ -0,0 +1,18 @@
+FROM ubuntu:22.04
+
+ENV HOME=/home/animl
+WORKDIR $HOME
+COPY ./ $HOME/
+
+RUN apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y curl git
+
+RUN export NODEV='18.16.0' \
+    && curl "https://nodejs.org/dist/v${NODEV}/node-v${NODEV}-linux-x64.tar.gz" | tar -xzv \
+    && cp ./node-v${NODEV}-linux-x64/bin/node /usr/bin/ \
+    && ./node-v${NODEV}-linux-x64/bin/npm install -g npm
+
+RUN npm install
+
+CMD npm run lint && npm test
+

package.json

@@ -1,5 +1,6 @@
 {
     "name": "animl-serverless",
+    "type": "module",
     "version": "1.0.0",
     "description": "Lambda-based graphQL API for camera trap data managment platform",
     "main": "src/handler.js",
@@ -48,7 +49,7 @@
         "lodash": "^4.17.20",
         "luxon": "^3.2.1",
         "mongo-cursor-pagination": "^8.1.2",
-        "mongodb-query-parser": "^2.4.3",
+        "mongodb-query-parser": "^3.0.0",
         "mongoose": "^7.0.0",
         "prompt": "^1.2.0",
         "stream-transform": "^3.2.0",

serverless.yml

@@ -1,7 +1,6 @@
 org: animl
 app: animl
 service: animl-api
-# variablesResolutionMode: 20210326
 
 plugins:
   - serverless-offline
@@ -12,14 +11,19 @@ package:
 
 provider:
   name: aws
-  runtime: nodejs18.x
   stage: ${opt:stage, 'dev'}
   region: us-west-2
   profile: animl
   environment:
     REGION: ${self:provider.region}
     STAGE: ${self:provider.stage}
     ACCOUNT: ${aws:accountId}
+
+  ecr:
+    images:
+      base:
+        path: .
+
   iam:
     role:
       statements:
@@ -90,7 +94,9 @@ provider:
 
 functions:
   graphql:
-    handler: src/api/handler.server
+    image:
+      name: base
+      command: src/api/handler.server
     events:
     - http:
         path: /
@@ -113,7 +119,9 @@ functions:
     memorySize: 3008
     timeout: 30
   inference:
-    handler: src/ml/handler.inference
+    image:
+      name: base
+      command: src/ml/handler.inference
     reservedConcurrency: 20
     events:
       - sqs:
@@ -125,11 +133,15 @@ functions:
           functionResponseType: ReportBatchItemFailures
     timeout: 120
   batchinference:
-    handler: src/ml/handler.inference
+    image:
+      name: base
+      command: src/ml/handler.inference
     reservedConcurrency: 8
     timeout: 120
   export:
-    handler: src/export/handler.export
+    image:
+      name: base
+      command: src/export/handler.handler
     reservedConcurrency: 10
     events:
       - sqs:

src/api/auth/authorization.js

@@ -1,4 +1,4 @@
-const jwt = require('jwt-simple');
+import jwt from 'jwt-simple';
 
 const BEARER_TOKEN_PATTERN = /^Bearer [-_=.0-9a-zA-Z]+$/i;
 
@@ -60,6 +60,6 @@ async function getUserInfo(req, config) {
 
 }
 
-module.exports = {
+export {
   getUserInfo
 };

src/api/auth/roles.js

@@ -10,7 +10,7 @@ const WRITE_DEPLOYMENTS_ROLES =          [MANAGER];
 const WRITE_AUTOMATION_RULES_ROLES =     [MANAGER];
 const WRITE_CAMERA_REGISTRATION_ROLES =  [MANAGER];
 
-module.exports = {
+export {
   EXPORT_DATA_ROLES,
   WRITE_OBJECTS_ROLES,
   WRITE_VIEWS_ROLES,

src/api/db/connect.js

@@ -1,13 +1,14 @@
-const mongoose = require('mongoose');
-const { ApolloError } = require('apollo-server-errors');
+import mongoose from 'mongoose';
+import  { ApolloError } from 'apollo-server-errors';
+
 // TODO: consider using multiple connections (one per model)
 // to reduce risk of slow trains
 // https://mongoosejs.com/docs/connections.html#multiple_connections
 // https://thecodebarbarian.com/slow-trains-in-mongodb-and-nodejs
 
 let cachedConnectionPromise = null;
 
-module.exports.connectToDatabase = async function connectToDb(config) {
+async function connectToDatabase(config) {
   if (!cachedConnectionPromise) {
     // If no connection promise is cached, create a new one.
     // We cache the promise instead of the connection itself to prevent race
@@ -26,4 +27,8 @@ module.exports.connectToDatabase = async function connectToDb(config) {
   } catch (err) {
     throw new ApolloError(err);
   }
+}
+
+export {
+  connectToDatabase
 };

src/api/db/models/Batch.js

@@ -1,15 +1,15 @@
-const { ApolloError, ForbiddenError } = require('apollo-server-errors');
-const MongoPaging = require('mongo-cursor-pagination');
-const { WRITE_IMAGES_ROLES } = require('../../auth/roles');
-const { randomUUID } = require('crypto');
-const S3 = require('@aws-sdk/client-s3');
-const SQS = require('@aws-sdk/client-sqs');
-const Lambda = require('@aws-sdk/client-lambda');
-const { getSignedUrl } = require('@aws-sdk/s3-request-presigner');
-const Batch = require('../schemas/Batch');
-const BatchError = require('../schemas/BatchError');
-const retry = require('async-retry');
-const utils = require('./utils');
+import { ApolloError, ForbiddenError } from 'apollo-server-errors';
+import MongoPaging from 'mongo-cursor-pagination';
+import { WRITE_IMAGES_ROLES } from '../../auth/roles.js';
+import { randomUUID } from 'node:crypto';
+import S3 from '@aws-sdk/client-s3';
+import SQS from '@aws-sdk/client-sqs';
+import Lambda from '@aws-sdk/client-lambda';
+import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
+import Batch from '../schemas/Batch.js';
+import BatchError from '../schemas/BatchError.js';
+import retry from 'async-retry';
+import { hasRole } from './utils.js';
 
 const generateBatchModel = ({ user } = {}) => ({
   queryByFilter: async (input) => {
@@ -90,22 +90,19 @@ const generateBatchModel = ({ user } = {}) => ({
   },
 
   get stopBatch() {
-    if (!utils.hasRole(user, WRITE_IMAGES_ROLES)) throw new ForbiddenError;
+    if (!hasRole(user, WRITE_IMAGES_ROLES)) throw new ForbiddenError;
 
     return async (input) => {
       const operation = async (input) => {
         return await retry(async (bail, attempt) => {
-          if (attempt > 1) {
-            console.log(`Retrying updateObject operation! Try #: ${attempt}`);
-          }
+          if (attempt > 1) console.log(`Retrying stopBatch operation! Try #: ${attempt}`);
+
           return await this.queryById(input._id);
         }, { retries: 2 });
       };
 
       try {
-        const batch = await operation({
-          _id: input.batch
-        });
+        const batch = await operation({ _id: input.batch });
         if (batch.processingEnd) throw new Error('Stack has already terminated');
 
         const lambda = new Lambda.LambdaClient({ region: process.env.REGION });
@@ -118,9 +115,44 @@ const generateBatchModel = ({ user } = {}) => ({
           })
         }));
 
-        return {
-          message: 'Batch Scheduled for Deletion'
-        };
+        return { message: 'Batch Scheduled for Deletion' };
+      } catch (err) {
+        console.error(err);
+        // if error is uncontrolled, throw new ApolloError
+        if (err instanceof ApolloError) throw err;
+        throw new ApolloError(err);
+      }
+    };
+  },
+
+  get redriveBatch() {
+    if (!hasRole(user, WRITE_IMAGES_ROLES)) throw new ForbiddenError;
+
+    return async (input) => {
+      const operation = async (input) => {
+        return await retry(async (bail, attempt) => {
+          if (attempt > 1) console.log(`Retrying redriveBatch operation! Try #: ${attempt}`);
+
+          // Ensure the batch actually exists
+          const batch = await this.queryById(input.batch);
+          if (batch.processingEnd) throw new Error('Stack has already terminated');
+
+          const sqs = new SQS.SQSClient({ region: process.env.REGION });
+
+          await sqs.send(new SQS.StartMessageMoveTaskCommand({
+            SourceArn: `arn:aws:sqs:${process.env.REGION}:${process.env.ACCOUNT}:animl-ingest-${process.env.STAGE}-${batch._id}-dlq`,
+            DestinationArn: `arn:aws:sqs:${process.env.REGION}:${process.env.ACCOUNT}:animl-ingest-${process.env.STAGE}-${batch._id}`
+          }));
+
+          return batch;
+
+        }, { retries: 2 });
+      };
+
+      try {
+        await operation(input);
+
+        return { message: 'Batch Redrive Initiated' };
       } catch (err) {
         console.error(err);
         // if error is uncontrolled, throw new ApolloError
@@ -131,14 +163,13 @@ const generateBatchModel = ({ user } = {}) => ({
   },
 
   get updateBatch() {
-    if (!utils.hasRole(user, WRITE_IMAGES_ROLES)) throw new ForbiddenError;
+    if (!hasRole(user, WRITE_IMAGES_ROLES)) throw new ForbiddenError;
 
     return async (input) => {
       const operation = async (input) => {
         return await retry(async (bail, attempt) => {
-          if (attempt > 1) {
-            console.log(`Retrying updateObject operation! Try #: ${attempt}`);
-          }
+          if (attempt > 1) console.log(`Retrying updateBatch operation! Try #: ${attempt}`);
+
           // find image, apply object updates, and save
           const batch = await this.queryById(input._id);
 
@@ -162,7 +193,7 @@ const generateBatchModel = ({ user } = {}) => ({
   },
 
   get createUpload() {
-    if (!utils.hasRole(user, WRITE_IMAGES_ROLES)) throw new ForbiddenError;
+    if (!hasRole(user, WRITE_IMAGES_ROLES)) throw new ForbiddenError;
 
     return async (input) => {
       const operation = async (input) => {
@@ -208,4 +239,4 @@ const generateBatchModel = ({ user } = {}) => ({
   }
 });
 
-module.exports = generateBatchModel;
+export default generateBatchModel;

src/api/db/models/BatchError.js

@@ -1,12 +1,12 @@
-const { ApolloError, ForbiddenError } = require('apollo-server-errors');
-const { WRITE_IMAGES_ROLES } = require('../../auth/roles');
-const BatchError = require('../schemas/BatchError');
-const retry = require('async-retry');
-const utils = require('./utils');
+import { ApolloError, ForbiddenError } from 'apollo-server-errors';
+import { WRITE_IMAGES_ROLES } from '../../auth/roles.js';
+import BatchError from '../schemas/BatchError.js';
+import retry from 'async-retry';
+import { hasRole } from './utils.js';
 
 const generateBatchErrorModel = ({ user } = {}) => ({
   get createError() {
-    if (!utils.hasRole(user, WRITE_IMAGES_ROLES)) throw new ForbiddenError;
+    if (!hasRole(user, WRITE_IMAGES_ROLES)) throw new ForbiddenError;
 
     return async (input) => {
       const operation = async (input) => {
@@ -35,7 +35,31 @@ const generateBatchErrorModel = ({ user } = {}) => ({
         throw new ApolloError(err);
       }
     };
+  },
+
+  get clearErrors() {
+    if (!hasRole(user, WRITE_IMAGES_ROLES)) throw new ForbiddenError;
+
+    return async (input) => {
+      const operation = async (input) => {
+        return await retry(async () => {
+          return await BatchError.deleteMany(input);
+        }, { retries: 2 });
+      };
+
+      try {
+        await operation({
+          batch: input.batch
+        });
+
+        return { message: 'Cleared' };
+      } catch (err) {
+        // if error is uncontrolled, throw new ApolloError
+        if (err instanceof ApolloError) throw err;
+        throw new ApolloError(err);
+      }
+    };
   }
 });
 
-module.exports = generateBatchErrorModel;
+export default generateBatchErrorModel;

src/api/db/models/Camera.js

@@ -1,13 +1,11 @@
-const { ApolloError, ForbiddenError } = require('apollo-server-errors');
-const { CameraRegistrationError } = require('../../errors');
-const WirelessCamera = require('../schemas/WirelessCamera');
-const retry = require('async-retry');
-const { WRITE_CAMERA_REGISTRATION_ROLES } = require('../../auth/roles');
-const { hasRole, idMatch } = require('./utils');
-
+import { ApolloError, ForbiddenError } from 'apollo-server-errors';
+import { CameraRegistrationError } from '../../errors.js';
+import WirelessCamera from '../schemas/WirelessCamera.js';
+import retry from 'async-retry';
+import { WRITE_CAMERA_REGISTRATION_ROLES } from '../../auth/roles.js';
+import { hasRole, idMatch } from './utils.js';
 
 const generateCameraModel = ({ user } = {}) => ({
-
   getWirelessCameras: async (_ids) => {
     const query = _ids ? { _id: { $in: _ids } } : {};
     // if user has curr_project, limit returned cameras to those that
@@ -220,4 +218,4 @@ const generateCameraModel = ({ user } = {}) => ({
 
 });
 
-module.exports = generateCameraModel;
+export default generateCameraModel;