Merge pull request #2707 from LiteFarmOrg/integration

Version 3.3.2

packages/api/package.json

@@ -1,6 +1,6 @@
 {
   "name": "litefarm-api",
-  "version": "3.3.1",
+  "version": "3.3.2",
   "description": "LiteFarm API server",
   "main": "./api/src/server.js",
   "type": "module",
@@ -54,7 +54,6 @@
     "aws-sdk": "^2.931.0",
     "axios": "^0.21.4",
     "bcryptjs": "^2.4.3",
-    "body-parser": "^1.19.0",
     "bull": "^3.22.9",
     "chai": "^4.2.0",
     "chai-http": "^4.3.0",

packages/api/src/controllers/sensorController.js

@@ -33,8 +33,9 @@ import {
   registerOrganizationWebhook,
   bulkSensorClaim,
   unclaimSensor,
+  ENSEMBLE_UNITS_MAPPING,
 } from '../util/ensemble.js';
-
+import { databaseUnit } from '../util/unit.js';
 import { sensorErrors, parseSensorCsv } from '../../../shared/validation/sensorCSV.js';
 import syncAsyncResponse from '../util/syncAsyncResponse.js';
 import knex from '../util/knex.js';
@@ -206,26 +207,32 @@ const sensorController = {
         );
 
         // Save sensors in database
-        const sensorLocations = await Promise.allSettled(
-          registeredSensors.map(async (sensor) => {
-            return await SensorModel.createSensor(
+        const sensorLocations = [];
+        for (const sensor of registeredSensors) {
+          try {
+            const value = await SensorModel.createSensor(
               sensor,
               farm_id,
               user_id,
               esids.includes(sensor.external_id) ? 1 : 0,
             );
-          }),
-        );
+            sensorLocations.push({ status: 'fulfilled', value });
+          } catch (error) {
+            sensorLocations.push({ status: 'rejected', reason: error });
+          }
+        }
 
         const successSensors = sensorLocations.reduce((prev, curr, idx) => {
           if (curr.status === 'fulfilled') {
             prev.push(curr.value);
           } else {
+            // These are sensors that were not saved to the database as locations
             errorSensors.push({
               row: data.findIndex((elem) => elem === registeredSensors[idx]) + 2,
-              column: 'External_ID',
               translation_key: sensorErrors.INTERNAL_ERROR,
-              variables: { sensorId: registeredSensors[idx].external_id },
+              variables: {
+                sensorId: registeredSensors[idx].external_id || registeredSensors[idx].name,
+              },
             });
           }
           return prev;
@@ -249,7 +256,7 @@ const sensorController = {
                   error_download: {
                     errors: errorSensors,
                     file_name: 'sensor-upload-outcomes.txt',
-                    success: successSensors.map((s) => s.sensor?.external_id), // Notification download needs an array of only ESIDs
+                    success: successSensors.map((s) => s.sensor?.external_id || s.name), // Notification download needs an array of only ESIDs
                     error_type: 'claim',
                   },
                 },
@@ -411,7 +418,7 @@ const sensorController = {
     }
     try {
       const infoBody = [];
-      const partnerId = parseInt(req.params.partner_id) || 1;
+      const partnerId = parseInt(req.params.partner_id);
       const farmId = req.params.farm_id || '';
       if (!farmId.length) return res.status(400).send('farm id not found');
       const {
@@ -440,11 +447,18 @@ const sensorController = {
             );
             continue;
           }
-          const unit = sensorInfo.unit;
+          // Reconcile incoming units with stored as units and conversion function keys
+          const system = ENSEMBLE_UNITS_MAPPING[sensorInfo.unit]?.system;
+          const unit = ENSEMBLE_UNITS_MAPPING[sensorInfo.unit]?.conversionKey;
+          const readingTypeStoredAsUnit = databaseUnit[readingType] ?? undefined;
+          const isStoredAsUnit = unit == readingTypeStoredAsUnit;
 
-          if (sensorInfo.values.length < sensorInfo.timestamps.length)
+          if (sensorInfo.values.length < sensorInfo.timestamps.length) {
             return res.status(400).send('sensor values and timestamps are not in sync');
-
+          }
+          if (!system || !unit || !readingTypeStoredAsUnit || !isStoredAsUnit) {
+            return res.status(400).send('provided units are not supported');
+          }
           for (let k = 0; k < sensorInfo.values.length; ++k) {
             infoBody.push({
               read_time: sensorInfo.timestamps[k] || '',
@@ -457,6 +471,7 @@ const sensorController = {
           }
         }
       }
+
       if (infoBody.length === 0) {
         return res.status(200).json({
           error: 'No records of sensor readings added to the Litefarm.',

packages/api/src/middleware/validation/deleteLocation.js

@@ -18,7 +18,7 @@ import { Model } from 'objection';
 import managementPlanModel from '../../models/managementPlanModel.js';
 
 async function validateLocationDependency(req, res, next) {
-  const location_id = req?.params?.location_id;
+  const location_id = req?.params?.location_id || req?.body?.location_id;
 
   const tasks = await Model.knex().raw(
     `
@@ -29,6 +29,7 @@ async function validateLocationDependency(req, res, next) {
         WHERE lt.location_id = :location_id
           AND t.complete_date is null
           AND t.abandon_date is null
+          AND t.deleted = false
         UNION
         SELECT DISTINCT mt.task_id, pmp.location_id
         FROM management_tasks mt
@@ -37,6 +38,7 @@ async function validateLocationDependency(req, res, next) {
         WHERE pmp.location_id = :location_id
           AND t.complete_date is null
           AND t.abandon_date is null
+          AND t.deleted = false
         UNION
         SELECT DISTINCT pt.task_id, pmp.location_id
         from plant_task pt
@@ -45,6 +47,7 @@ async function validateLocationDependency(req, res, next) {
         WHERE pmp.location_id = :location_id
           AND t.complete_date is null
           AND t.abandon_date is null
+          AND t.deleted = false
         UNION
         SELECT DISTINCT tt.task_id, pmp.location_id
         from transplant_task tt
@@ -53,6 +56,7 @@ async function validateLocationDependency(req, res, next) {
         WHERE pmp.location_id = :location_id
           AND t.complete_date is null
           AND t.abandon_date is null
+          AND t.deleted = false
     `,
     { location_id },
   );

packages/api/src/middleware/validation/validateWebhook.js

@@ -1,7 +1,7 @@
 async function validateRequest(req, res, next) {
   const partnerId = parseInt(req.params.partner_id);
   if (partnerId !== 1) {
-    next();
+    return res.status(400).send('Partner not registered with Litefarm');
   } else {
     const farmId = req.params.farm_id;
     const authKey = `${farmId}${process.env.SENSOR_SECRET}`;

packages/api/src/routes/sensorRoute.js

@@ -19,6 +19,7 @@ import multer from 'multer';
 import hasFarmAccess from '../middleware/acl/hasFarmAccess.js';
 import checkScope from '../middleware/acl/checkScope.js';
 import validateRequest from '../middleware/validation/validateWebhook.js';
+import validateLocationDependency from '../middleware/validation/deleteLocation.js';
 import SensorController from '../controllers/sensorController.js';
 
 const storage = multer.memoryStorage();
@@ -44,7 +45,13 @@ router.post(
 router.get('/:location_id/reading', SensorController.getAllReadingsByLocationId);
 router.get('/reading/farm/:farm_id', SensorController.getReadingsByFarmId);
 router.post('/reading/invalidate', SensorController.invalidateReadings);
-router.post('/unclaim', SensorController.retireSensor);
+router.post(
+  '/unclaim',
+  hasFarmAccess({ body: 'location_id' }),
+  checkScope(['delete:fields']),
+  validateLocationDependency,
+  SensorController.retireSensor,
+);
 router.get('/:location_id/reading_type', SensorController.getSensorReadingTypes);
 router.get('/farm/:farm_id/reading_type', SensorController.getAllSensorReadingTypes);
 router.get('/partner/:partner_id/brand_name', SensorController.getBrandName);

packages/api/src/server.js

@@ -23,7 +23,6 @@ import './dotenvConfig.js';
 // dotenv.config();
 // dotenv.config({ path: path.resolve(process.cwd(), '.env.local') });
 import express from 'express';
-import bodyParser from 'body-parser';
 const app = express();
 import expressOasGenerator from 'express-oas-generator';
 const environment = process.env.NODE_ENV || 'development';
@@ -196,9 +195,29 @@ app.set('json replacer', (key, value) => {
   return value;
 });
 
+// Apply default express.json() request size limit to all routes except sensor webhook
+const applyExpressJSON = (req, res, next) => {
+  if (req.path.startsWith('/sensor/reading/partner/1/farm/')) return next();
+
+  const jsonMiddleware = express.json({ limit: '100kB' });
+  jsonMiddleware(req, res, next);
+};
+
+// Refuse GET or DELETE requests with a request body
+const rejectBodyInGetAndDelete = (req, res, next) => {
+  if (
+    (req.method === 'DELETE' || req.method === 'GET') &&
+    req.body &&
+    Object.keys(req.body).length > 0
+  ) {
+    return res.sendStatus(400);
+  }
+  next();
+};
+
 app
-  .use(bodyParser.json())
-  .use(bodyParser.urlencoded({ extended: true }))
+  .use(applyExpressJSON)
+  .use(express.urlencoded({ extended: true }))
 
   // prevent CORS errors
   .use(cors())
@@ -211,14 +230,9 @@ app
     if (req.method === 'OPTIONS') {
       res.header('Access-Control-Allow-Methods', 'PUT, POST, PATCH, DELETE, GET');
       return res.status(200).json({});
-    } else if (
-      (req.method === 'DELETE' || req.method === 'GET') &&
-      Object.keys(req.body).length > 0
-    ) {
-      // TODO: Find new bugs caused by this change
-      return res.sendStatus(400);
     }
-    next();
+
+    rejectBodyInGetAndDelete(req, res, next);
   })
   .use(router)
   .set('json spaces', 2)
@@ -266,8 +280,10 @@ app
   .use('/product', productRoute)
   .use('/nomination', nominationRoutes)
   .use('/notification_user', notificationUserRoute)
-  .use('/time_notification', timeNotificationRoute)
-  .use('/sensor', sensorRoute);
+  .use('/time_notification', timeNotificationRoute);
+
+// Allow a 1MB limit on sensors to match incoming Ensemble data
+app.use('/sensor', express.json({ limit: '1MB' }), rejectBodyInGetAndDelete, sensorRoute);
 
 expressOasGenerator.handleRequests();