feat: pagination and filtering on all matches + refactoring

api/src/controllers/match.controller.js

@@ -1,95 +1,131 @@
 const { promisify } = require('util');
-const fs = require('fs');
-const sharp = require('sharp');
 const sizeOf = promisify(require('image-size'));
 const database = require('../util/db.util');
 const filesystem = require('../util/fs.util');
+const { tryParseJSON } = require('../util/validators.util');
 const { jwt } = require('../util/auth.util');
 const process = require('../util/process.util');
 const { AUTH, STORAGE } = require('../constants');
 const { BAD_REQUEST } = require('../constants/http-status');
 const DETECTORS = require('../constants/config').detectors();
 
-let matchProps = [];
-
 const format = async (matches) => {
   const token = AUTH && matches.length ? jwt.sign({ route: 'storage' }) : null;
-
   matches = await Promise.all(
     matches.map(async (obj) => {
       const { id, filename, event, response, isTrained } = obj;
       const { camera, type, zones, updatedAt } = JSON.parse(event);
-
       const key = `matches/${filename}`;
-
-      const output = {
+      const { width, height } = await sizeOf(`${STORAGE.PATH}/${key}`);
+      return {
         id,
         camera,
         type,
         zones,
         file: {
           key,
           filename,
+          width,
+          height,
         },
         isTrained: !!isTrained,
         response: JSON.parse(response),
         createdAt: obj.createdAt,
         updatedAt: updatedAt || null,
         token,
       };
-
-      const [matchProp] = matchProps.filter((prop) => prop.key === key);
-
-      if (matchProp) {
-        output.file = matchProp.file;
-      } else if (fs.existsSync(`${STORAGE.PATH}/${key}`)) {
-        const base64 = await sharp(`${STORAGE.PATH}/${key}`)
-          .jpeg({ quality: 70 })
-          .resize(500)
-          .withMetadata()
-          .toBuffer();
-        const { width, height } = await sizeOf(`${STORAGE.PATH}/${key}`);
-        output.file.base64 = base64.toString('base64');
-        output.file.width = width;
-        output.file.height = height;
-        // push sharp and sizeOf results to an array to search against
-        matchProps.unshift({ key, file: output.file });
-      }
-
-      return output;
     })
   );
-  matchProps = matchProps.slice(0, 500);
   return matches;
 };
 
 module.exports.get = async (req, res) => {
-  const { sinceId } = req.query;
+  const limit = 2;
+  const { sinceId, page } = req.query;
+  const filters = tryParseJSON(req.query.filters);
 
   const db = database.connect();
+
+  if (!filters || !Object.keys(filters).length) {
+    const [total] = db.prepare(`SELECT COUNT(*) count FROM match`).all();
+    const matches = db
+      .prepare(
+        `SELECT * FROM match
+          LEFT JOIN (SELECT filename as isTrained FROM train GROUP BY filename) train ON train.isTrained = match.filename
+          ORDER BY createdAt DESC
+          LIMIT ?,?`
+      )
+      .bind(limit * (page - 1), limit)
+      .all();
+
+    return res.send({ total: total.count, limit, matches: await format(matches) });
+  }
+
+  const filteredIds = db
+    .prepare(
+      `SELECT t.id, detector, value FROM (
+          SELECT match.id,  json_extract(value, '$.detector') detector, json_extract(value, '$.results') results
+          FROM match, json_each( match.response)
+          ) t, json_each(t.results)
+        WHERE json_extract(value, '$.name') IN (${database.params(filters.names)})
+        AND json_extract(value, '$.match') IN (${database.params(filters.matches)})
+        AND json_extract(value, '$.confidence') >= ?
+        AND json_extract(value, '$.box.width') >= ?
+        AND json_extract(value, '$.box.height') >= ?
+        AND detector IN (${database.params(filters.detectors)})
+        GROUP BY t.id`
+    )
+    .bind(
+      filters.names,
+      filters.matches.map((obj) => (obj === 'match' ? 1 : 0)),
+      filters.confidence,
+      filters.width,
+      filters.height,
+      filters.detectors
+    )
+    .all()
+    .map((obj) => obj.id);
+
+  const [total] = db
+    .prepare(
+      `SELECT COUNT(*) count FROM match
+      WHERE id IN (${database.params(filteredIds)})
+      AND id > ?
+      ORDER BY createdAt DESC`
+    )
+    .bind(filteredIds, sinceId || 0)
+    .all();
+
   const matches = db
     .prepare(
-      `
-        SELECT * FROM match
+      `SELECT * FROM match
         LEFT JOIN (SELECT filename as isTrained FROM train GROUP BY filename) train ON train.isTrained = match.filename
-        WHERE id > ?
-        GROUP BY match.id
-        ORDER BY createdAt DESC LIMIT 100
-        `
+        WHERE id IN (${database.params(filteredIds)})
+        AND id > ?
+        ORDER BY createdAt DESC
+        LIMIT ?,?`
     )
-    .bind(sinceId || 0)
+    .bind(filteredIds, sinceId || 0, limit * (page - 1), limit)
     .all();
 
-  res.send({ matches: await format(matches) });
+  res.send({ total: total.count, limit, matches: await format(matches) });
 };
 
 module.exports.delete = async (req, res) => {
-  const files = req.body;
-  files.forEach((file) => {
+  const ids = req.body;
+  if (ids.length) {
     const db = database.connect();
-    db.prepare('DELETE FROM match WHERE id = ?').run(file.id);
-    filesystem.delete(`${STORAGE.PATH}/${file.key}`);
-  });
+    const files = db
+      .prepare(`SELECT filename FROM match WHERE id IN (${database.params(ids)})`)
+      .bind(ids)
+      .all();
+
+    db.prepare(`DELETE FROM match WHERE id IN (${database.params(ids)})`).run(ids);
+
+    files.forEach(({ filename }) => {
+      filesystem.delete(`${STORAGE.PATH}/matches/${filename}`);
+    });
+  }
 
   res.send({ success: true });
 };
@@ -101,9 +137,7 @@ module.exports.reprocess = async (req, res) => {
   const db = database.connect();
   let [match] = db.prepare('SELECT * FROM match WHERE id = ?').bind(matchId).all();
 
-  if (!match) {
-    return res.status(BAD_REQUEST).error('No match found');
-  }
+  if (!match) return res.status(BAD_REQUEST).error('No match found');
 
   const results = await process.start({
     filename: match.filename,
@@ -122,7 +156,49 @@ module.exports.reprocess = async (req, res) => {
     )
     .bind(matchId)
     .all();
-  match = await format(match);
+  [match] = await format(match);
+
+  res.send(match);
+};
+
+module.exports.filters = async (req, res) => {
+  const db = database.connect();
+
+  const [total] = db.prepare('SELECT COUNT(*) count FROM match').all();
+
+  const detectors = db
+    .prepare(
+      `SELECT json_extract(value, '$.detector') name
+        FROM match, json_each( match.response)
+        GROUP BY name
+        ORDER BY name ASC`
+    )
+    .all()
+    .map((obj) => obj.name);
+
+  const names = db
+    .prepare(
+      `SELECT json_extract(value, '$.name') name FROM (
+          SELECT json_extract(value, '$.results') results
+          FROM match, json_each( match.response)
+          ) t, json_each(t.results)
+        GROUP BY name
+        ORDER BY name ASC`
+    )
+    .all()
+    .map((obj) => obj.name);
+
+  const matches = db
+    .prepare(
+      `SELECT IIF(json_extract(value, '$.match') == 1, 'match', 'miss') name FROM (
+          SELECT json_extract(value, '$.results') results
+          FROM match, json_each( match.response)
+          ) t, json_each(t.results)
+        GROUP BY name
+        ORDER BY name ASC`
+    )
+    .all()
+    .map((obj) => obj.name);
 
-  res.send(match[0]);
+  res.send({ total: total.count, detectors, names, matches });
 };

api/src/controllers/train.controller.js

@@ -1,5 +1,7 @@
+const { promisify } = require('util');
 const fs = require('fs');
-const sharp = require('sharp');
+const sizeOf = promisify(require('image-size'));
+const { getOrientation } = require('get-orientation');
 const time = require('../util/time.util');
 const database = require('../util/db.util');
 const train = require('../util/train.util');
@@ -43,28 +45,23 @@ module.exports.get = async (req, res) => {
       const { id, name, filename, results, createdAt } = obj;
 
       const key = `train/${name}/${filename}`;
+      const { width, height } = await sizeOf(`${STORAGE.PATH}/${key}`);
+      const orientation = await getOrientation(fs.readFileSync(`${STORAGE.PATH}/${key}`));
 
       const output = {
         id,
         name,
         file: {
           key,
           filename,
+          width: orientation === 6 ? height : width,
+          height: orientation === 6 ? width : height,
         },
         results,
         createdAt,
         token,
       };
 
-      if (fs.existsSync(`${STORAGE.PATH}/${key}`)) {
-        const base64 = await sharp(`${STORAGE.PATH}/${key}`)
-          .jpeg({ quality: 70 })
-          .resize(500)
-          .withMetadata()
-          .toBuffer();
-        output.file.base64 = base64.toString('base64');
-      }
-
       return output;
     })
   );
@@ -81,7 +78,7 @@ module.exports.delete = async (req, res) => {
 
 module.exports.add = async (req, res) => {
   const { name } = req.params;
-  const { urls, files: matchFiles } = req.body;
+  const { urls, ids } = req.body;
 
   let files = [];
 
@@ -95,15 +92,16 @@ module.exports.add = async (req, res) => {
         files.push({ name, filename });
       })
     );
-  } else if (matchFiles) {
-    for (let i = 0; i < matchFiles.length; i++) {
-      const filename = matchFiles[i];
+  } else if (ids) {
+    ids.forEach((id) => {
+      const db = database.connect();
+      const [match] = db.prepare('SELECT filename FROM match WHERE id = ?').bind(id).all();
       filesystem.copy(
-        `${STORAGE.PATH}/matches/${filename}`,
-        `${STORAGE.PATH}/train/${name}/${filename}`
+        `${STORAGE.PATH}/matches/${match.filename}`,
+        `${STORAGE.PATH}/train/${name}/${match.filename}`
       );
-      files.push({ name, filename });
-    }
+      files.push({ name, filename: match.filename });
+    });
   } else {
     files = (urls ? await filesystem.saveURLs(urls, `train/${name}`) : []).map((filename) => ({
       filename,

api/src/routes/match.routes.js

@@ -1,10 +1,14 @@
 const express = require('express');
-const { jwt } = require('../middlewares');
+const { jwt, expressValidator, validate } = require('../middlewares');
 const controller = require('../controllers/match.controller');
 
+const { query } = expressValidator;
 const router = express.Router();
 
-router.get('/', jwt, controller.get).delete('/', jwt, controller.delete);
+router
+  .get('/', jwt, validate([query('page').default(1).isInt()]), controller.get)
+  .delete('/', jwt, controller.delete);
 router.patch('/reprocess/:matchId', jwt, controller.reprocess);
+router.get('/filters', controller.filters);
 
 module.exports = router;

frontend/src/App.vue

@@ -1,9 +1,9 @@
 <template>
-  <div class="wrapper">
+  <div class="app-wrapper">
     <Toast position="bottom-left" />
     <ConfirmDialog />
-    <Toolbar />
-    <router-view />
+    <Toolbar ref="toolbar" />
+    <router-view :toolbarHeight="toolbarHeight" />
   </div>
 </template>
 
@@ -26,13 +26,19 @@ export default {
     ConfirmDialog,
     Toolbar,
   },
+  data: () => ({
+    toolbarHeight: null,
+  }),
   created() {
     this.checkLoginState();
     window.addEventListener('focus', this.checkLoginState);
     this.emitter.on('login', this.login);
     this.emitter.on('error', (error) => this.error(error));
     this.emitter.on('toast', (...args) => this.toast(...args));
   },
+  mounted() {
+    this.toolbarHeight = this.$refs.toolbar.getHeight();
+  },
   methods: {
     login() {
       if (this.$route.path !== '/login') {
@@ -164,8 +170,9 @@ body {
 
 <style scoped lang="scss">
 @import '@/assets/scss/_variables.scss';
-.wrapper {
+.app-wrapper {
   max-width: $max-width;
   margin: auto;
+  overflow: hidden;
 }
 </style>

frontend/src/assets/scss/_variables.scss

@@ -1,3 +1,2 @@
 $tool-bar-height: 32px;
-$match-header-height: 55px;
 $max-width: 1400px;