Skip to content

Commit

Permalink
feat(mqtt & api): total count for person, match, miss, and unknown (#223
Browse files Browse the repository at this point in the history 
)

also refactored how the values are calculated
  • Loading branch information
@jakowenko
jakowenko committed Oct 21, 2022
1 parent 759659c commit 2bf4406
Show file tree
Hide file tree
Showing 3 changed files with 87 additions and 80 deletions.
10 changes: 5 additions & 5 deletions api/src/controllers/recognize.controller.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ module.exports.start = async (req, res) => {
);
}

const { best, misses, unknown, results, attempts } = recognize.normalize(
const { best, misses, unknowns, results, attempts, counts } = recognize.normalize(
await Promise.all(promises)
);

Expand All @@ -160,28 +160,28 @@ module.exports.start = async (req, res) => {
attempts,
camera,
zones,
counts,
matches: best,
misses,
unknowns,
};
if (unknown && Object.keys(unknown).length) output.unknown = unknown;
if (AUTH) output.token = jwt.sign({ route: 'storage', expiresIn: TOKEN.IMAGE });

if (resultsOutput === 'all') output.results = results;

console.log(`done processing ${camera}: ${id} in ${duration} sec`);

const loggedOutput = JSON.parse(JSON.stringify(output));
['matches', 'misses'].forEach((type) =>
['matches', 'misses', 'unknowns'].forEach((type) =>
loggedOutput[type].forEach((result) => delete result.base64)
);
if (loggedOutput.unknown) delete loggedOutput.unknown.base64;
console.log(loggedOutput);

PROCESSING = false;

res.send(output);

recognize.save.latest(camera, best, misses, unknown);
recognize.save.latest(camera, best, misses, unknowns[0]);
mqtt.recognize(output);
notify.publish(output, camera, results);
if (event.type === 'frigate') frigate.subLabel(event.topic, id, best);
Expand Down
102 changes: 43 additions & 59 deletions api/src/util/mqtt.util.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ const { v4: uuidv4 } = require('uuid');
const axios = require('axios');
const mqtt = require('mqtt');
const fs = require('./fs.util');
const { contains } = require('./helpers.util');
const { jwt } = require('./auth.util');
const { AUTH, SERVER, MQTT, FRIGATE, CAMERAS, STORAGE, UI } = require('../constants')();
const config = require('../constants/config');
Expand Down Expand Up @@ -156,47 +155,51 @@ module.exports.subscribe = () => {
module.exports.recognize = (data) => {
try {
if (!MQTT || !MQTT.HOST) return;
const { matches, misses, unknown } = data;
const camera = data.camera.toLowerCase();
const hasUnknown = unknown && Object.keys(unknown).length;

const configData = JSON.parse(JSON.stringify(data));
delete configData.matches;
delete configData.unknown;
delete configData.results;
const baseData = JSON.parse(JSON.stringify(data));
const { id, duration, timestamp, attempts, zones, matches, misses, unknowns, counts } =
baseData;
const camera = baseData.camera.toLowerCase();

const payload = {
base: {
id,
duration,
timestamp,
attempts,
camera,
zones,
},
};
payload.unknown = { ...payload.base, unknown: unknowns[0], unknowns };
payload.match = { ...payload.base };
payload.camera = {
...payload.base,
matches,
misses,
unknowns,
counts,
};
payload.cameraReset = {
...payload.camera,
counts: {
person: 0,
match: 0,
miss: 0,
unknown: 0,
},
};

const messages = [];
const persons = [...new Set([...matches, ...misses].map(({ name }) => name))];
let personCount = persons.length ? persons.length : hasUnknown ? 1 : 0;
// check to see if unknown bounding box is contained within or contains any of the match bounding boxes
// if false, then add 1 to the person count
if (persons.length && hasUnknown) {
let unknownFoundInMatch = false;
matches.forEach((match) => {
if (contains(match.box, unknown.box) || contains(unknown.box, match.box))
unknownFoundInMatch = true;
});

let unknownFoundInMiss = false;
misses.forEach((miss) => {
if (contains(miss.box, unknown.box) || contains(unknown.box, miss.box))
unknownFoundInMiss = true;
});
if (!unknownFoundInMatch && !unknownFoundInMiss) personCount += 1;
}

messages.push({
topic: `${MQTT.TOPICS.CAMERAS}/${camera}/person`,
message: personCount.toString(),
message: counts.person.toString(),
});

if (hasUnknown) {
if (unknowns.length) {
messages.push({
topic: `${MQTT.TOPICS.MATCHES}/unknown`,
message: JSON.stringify({
...configData,
unknown,
}),
message: JSON.stringify(payload.unknown),
});

if (MQTT.TOPICS.HOMEASSISTANT) {
Expand All @@ -215,10 +218,7 @@ module.exports.recognize = (data) => {

messages.push({
topic: `${MQTT.TOPICS.HOMEASSISTANT}/sensor/double-take/unknown/state`,
message: JSON.stringify({
...configData,
unknown,
}),
message: JSON.stringify(payload.unknown),
});
}
}
Expand All @@ -230,7 +230,7 @@ module.exports.recognize = (data) => {
messages.push({
topic: `${MQTT.TOPICS.MATCHES}/${topic}`,
message: JSON.stringify({
...configData,
...payload.match,
match,
}),
});
Expand All @@ -252,22 +252,17 @@ module.exports.recognize = (data) => {
messages.push({
topic: `${MQTT.TOPICS.HOMEASSISTANT}/sensor/double-take/${topic}/state`,
message: JSON.stringify({
...configData,
...payload.match,
match,
}),
});
}
});

if (matches.length || misses.length || hasUnknown) {
if (matches.length || misses.length || unknowns.length) {
messages.push({
topic: `${MQTT.TOPICS.CAMERAS}/${camera}`,
message: JSON.stringify({
...configData,
matches,
misses,
unknown,
}),
message: JSON.stringify(payload.camera),
});

if (MQTT.TOPICS.HOMEASSISTANT) {
Expand All @@ -286,13 +281,7 @@ module.exports.recognize = (data) => {

messages.push({
topic: `${MQTT.TOPICS.HOMEASSISTANT}/sensor/double-take/${camera}/state`,
message: JSON.stringify({
...configData,
matches,
misses,
unknown,
personCount,
}),
message: JSON.stringify(payload.camera),
});
}
}
Expand All @@ -305,12 +294,7 @@ module.exports.recognize = (data) => {
if (MQTT.TOPICS.HOMEASSISTANT) {
this.publish({
topic: `${MQTT.TOPICS.HOMEASSISTANT}/sensor/double-take/${camera}/state`,
message: JSON.stringify({
...configData,
matches,
unknown,
personCount: 0,
}),
message: JSON.stringify(payload.cameraReset),
});
}
}, 30000);
Expand Down
55 changes: 39 additions & 16 deletions api/src/util/recognize.util.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,28 +39,33 @@ module.exports.save = {

module.exports.normalize = (results = []) => {
const best = { matches: [], misses: [] };
const tmp = { matches: {}, misses: {} };
let unknown = {};
const tmp = { matches: {}, misses: {}, counts: {} };
const unknowns = [];

let attempts = 0;
results.forEach((group) => {
attempts += group.attempts;
group.results.forEach((attempt) => {
const face = tmp.counts[attempt.detector];
if (face)
tmp.counts[attempt.detector] = {
count: face.count + attempt.results.length,
attempts: (face.attempts += 1),
};
else tmp.counts[attempt.detector] = { count: attempt.results.length, attempts: 1 };

const matches = attempt.results.filter((obj) => obj.match);
const misses = attempt.results.filter((obj) => !obj.match && obj.name !== 'unknown');
const unknowns = attempt.results.filter((obj) => obj.name === 'unknown');

unknowns.forEach((obj) => {
if (unknown.confidence === undefined || unknown.confidence < obj.confidence) {
unknown = {
...obj,
type: group.type,
duration: attempt.duration,
detector: attempt.detector,
filename: attempt.filename,
base64: attempt.base64 || null,
};
}
const tmpUnknowns = attempt.results.filter((obj) => obj.name === 'unknown');
tmpUnknowns.forEach((obj) => {
unknowns.push({
...obj,
type: group.type,
duration: attempt.duration,
detector: attempt.detector,
filename: attempt.filename,
base64: attempt.base64 || null,
});
});

matches.forEach((match) => {
Expand Down Expand Up @@ -100,5 +105,23 @@ module.exports.normalize = (results = []) => {
for (const value of Object.values(tmp.matches)) best.matches.push(value);
for (const value of Object.values(tmp.misses)) best.misses.push(value);

return { best: best.matches, misses: best.misses, results, attempts, unknown };
let personCount = 0;
for (const [, value] of Object.entries(tmp.counts))
personCount += Math.round(value.count / value.attempts);

const counts = {
person: personCount ? Math.round(personCount / Object.keys(tmp.counts).length) : 0,
match: best.matches.length,
miss: best.misses.length,
};
counts.unknown = counts.person - counts.match - counts.miss;

return {
best: best.matches,
misses: best.misses,
results,
attempts,
unknowns: unknowns.sort((a, b) => b.confidence - a.confidence),
counts,
};
};

0 comments on commit 2bf4406

Please sign in to comment.