Loads of improvements (#372)

* Disable unused ble functions to free up some memory
* Added Count of devices present in room
* Added known macs
* Reorg settings
* Improvements to ignoring (trailing spaces matched everything)
* Add battery percent to ATC sensors
* Remove sprite usage to clean up memory usage
* Initial support for samsung smart tag (may not actually work) (#351)
* DHT Sensor temp related changes (#385)
* Add interval, fix rounding
* Add batt for MACCHINA_A0
* Add running to macchina
* Add Garmin watch (Fenix 6) id
* Updgrade ArduinoJson

Co-authored-by: tr-v-r <83373608+tr-v-r@users.noreply.github.com>

espotabatch.py

@@ -4,10 +4,9 @@
 
 #this list contains array of esp32 clients,
 # and each client contains mDNS name and the path to .bin file
-#I only have 1 ESP so I duplicate mDNS entry for testing
 esps = [
   #mDNS name of ESP   #path to ".bin" file
-    ['192.168.128.114', 'macchina-a0'],
+    ['192.168.128.112', 'macchina-a0'],
     ['192.168.128.124', 'm5stickc'],
     ['192.168.128.64', 'm5atom-matrix'],
     ['192.168.128.84', 'm5atom-matrix']

lib/BleFingerprint/BleFingerprint.h

@@ -15,17 +15,18 @@
 #define NO_RSSI (-32768)
 
 #define ID_TYPE_TX_POW short(1)
+#define ID_TYPE_MISC_APPLE short(-5)
 
 #define ID_TYPE_MAC short(0)
 #define ID_TYPE_AD short(10)
 #define ID_TYPE_SD short(15)
 #define ID_TYPE_MD short(20)
-#define ID_TYPE_MISC_APPLE short(25)
 #define ID_TYPE_MISC short(30)
 #define ID_TYPE_NAME short(35)
 #define ID_TYPE_PUBLIC_MAC short(50)
 #define ID_TYPE_MSFT short(100)
 #define ID_TYPE_SONOS short(105)
+#define ID_TYPE_GARMIN short(107)
 #define ID_TYPE_MITHERM short(110)
 #define ID_TYPE_MIFIT short(115)
 #define ID_TYPE_EXPOSURE short(120)
@@ -34,13 +35,15 @@
 #define ID_TYPE_TILE short( 135)
 #define ID_TYPE_MEATER short(140)
 #define ID_TYPE_VANMOOF short(145)
+#define ID_TYPE_SMARTTAG short(146)
 #define ID_TYPE_APPLE_NEARBY short(150)
 #define ID_TYPE_QUERY_MODEL short(155)
 #define ID_TYPE_QUERY_NAME short(160)
 #define ID_TYPE_EBEACON short(165)
 #define ID_TYPE_ABEACON short(170)
 #define ID_TYPE_IBEACON short(175)
 #define ID_TYPE_RM_ASST short(180)
+#define ID_TYPE_KNOWN_MAC short(185)
 
 class BleFingerprintCollection;
 
@@ -56,13 +59,7 @@ class BleFingerprint
 
     bool query();
 
-    String getId()
-    {
-        if (!id.isEmpty() && idType > 10) return id;
-        if (macPublic) return getMac();
-        if (!id.isEmpty()) return id;
-        return getMac();
-    }
+    String getId() { return id; }
 
     bool setId(const String &newId, short int newIdType, const String &newName = "");
 
@@ -82,7 +79,9 @@ class BleFingerprint
 
     NimBLEAddress const getAddress() { return address; }
 
-    long getAge() const { return millis() - lastSeenMillis; };
+    unsigned long getMsSinceLastSeen() const { return millis() - lastSeenMillis; };
+
+    unsigned long getMsSinceFirstSeen() const { return millis() - firstSeenMillis; };
 
     bool getAdded() const { return added; };
 
@@ -92,26 +91,30 @@ class BleFingerprint
 
     bool getRmAsst() const { return rmAsst; };
 
-    int getSeenCount()
+    unsigned int getSeenCount()
     {
-        auto sc = seenCount;
-        seenCount = 0;
+        auto sc = seenCount - lastSeenCount;
+        lastSeenCount = seenCount;
         return sc;
     }
 
+    bool shouldCount();
+
 private:
 
     static bool shouldHide(const String &s);
 
-    bool hasValue = false, added = false, close = false, reported = false, macPublic = false, ignore = false, allowQuery = false, didQuery = false, rmAsst = false, hidden = false, connectable = false;
+    bool hasValue = false, added = false, close = false, reported = false, ignore = false, allowQuery = false, didQuery = false, rmAsst = false, hidden = false, connectable = false, countable = false, counting = false;
     NimBLEAddress address;
     String id, name, disc;
     short int idType = 0;
     int rssi = -100, calRssi = NO_RSSI, mdRssi = NO_RSSI, asRssi = NO_RSSI, newest = NO_RSSI, recent = NO_RSSI, oldest = NO_RSSI;
-    int qryAttempts = 0, seenCount = 1, qryDelayMillis = 0;
+    unsigned int qryAttempts = 0, qryDelayMillis = 0;
     float raw = 0, lastReported = 0, temp = 0, humidity = 0;
     unsigned long firstSeenMillis, lastSeenMillis = 0, lastReportedMillis = 0, lastQryMillis = 0;
+    unsigned long seenCount = 1, lastSeenCount = 0;
     uint16_t mv = 0;
+    uint8_t battery = 0xFF;
 
     Reading<Differential<float>> output;
 
@@ -122,11 +125,11 @@ class BleFingerprint
 
     void fingerprint(NimBLEAdvertisedDevice *advertisedDevice);
 
-    void fingerprintServiceAdvertisements(NimBLEAdvertisedDevice *advertisedDevice, size_t serviceAdvCount);
+    void fingerprintServiceAdvertisements(NimBLEAdvertisedDevice *advertisedDevice, size_t serviceAdvCount, bool haveTxPower, int8_t txPower);
 
-    void fingerprintServiceData(NimBLEAdvertisedDevice *advertisedDevice, size_t serviceDataCount);
+    void fingerprintServiceData(NimBLEAdvertisedDevice *advertisedDevice, size_t serviceDataCount, bool haveTxPower, int8_t txPower);
 
-    void fingerprintManufactureData(NimBLEAdvertisedDevice *advertisedDevice);
+    void fingerprintManufactureData(NimBLEAdvertisedDevice *advertisedDevice, bool haveTxPower, int8_t txPower);
 };
 
 #endif

lib/BleFingerprint/BleFingerprintCollection.cpp

@@ -8,7 +8,7 @@ void BleFingerprintCollection::cleanupOldFingerprints()
     auto it = fingerprints.begin();
     while (it != fingerprints.end())
     {
-        long age = (*it)->getAge();
+        auto age = (*it)->getMsSinceLastSeen();
         if (age > forgetMs)
         {
             GUI::removed((*it));
@@ -54,16 +54,21 @@ BleFingerprint *BleFingerprintCollection::getFingerprint(BLEAdvertisedDevice *ad
     return f;
 }
 
-std::list<BleFingerprint *> BleFingerprintCollection::getCopy()
+const std::list<BleFingerprint *> BleFingerprintCollection::getCopy()
 {
     if (xSemaphoreTake(fingerprintSemaphore, 1000) != pdTRUE)
         log_e("Couldn't take semaphore!");
     cleanupOldFingerprints();
     std::list<BleFingerprint *> copy(fingerprints);
     if (xSemaphoreGive(fingerprintSemaphore) != pdTRUE)
         log_e("Couldn't give semaphore!");
-    return copy;
+    return std::move(copy);
+}
+String BleFingerprintCollection::include{}, BleFingerprintCollection::exclude{}, BleFingerprintCollection::query{}, BleFingerprintCollection::knownMacs{}, BleFingerprintCollection::countIds{};
+float BleFingerprintCollection::skipDistance = 0.0f, BleFingerprintCollection::maxDistance = 0.0f, BleFingerprintCollection::absorption = 3.5f, BleFingerprintCollection::countEnter = 2, BleFingerprintCollection::countExit = 4;
+int BleFingerprintCollection::refRssi = 0, BleFingerprintCollection::forgetMs = 0, BleFingerprintCollection::skipMs = 0, BleFingerprintCollection::countMs = 10000;
+
+const std::list<BleFingerprint *>* const BleFingerprintCollection::getNative()
+{
+    return &fingerprints;
 }
-String BleFingerprintCollection::include{}, BleFingerprintCollection::exclude{}, BleFingerprintCollection::query{};
-float BleFingerprintCollection::skipDistance = 0.0f, BleFingerprintCollection::maxDistance = 0.0f, BleFingerprintCollection::absorption = 3.5f;
-int BleFingerprintCollection::refRssi = 0, BleFingerprintCollection::forgetMs = 0, BleFingerprintCollection::skipMs = 0;

lib/BleFingerprint/BleFingerprintCollection.h

@@ -18,12 +18,15 @@ class BleFingerprintCollection : public BLEAdvertisedDeviceCallbacks
     }
     BleFingerprint *getFingerprint(BLEAdvertisedDevice *advertisedDevice);
     void cleanupOldFingerprints();
-    std::list<BleFingerprint *> getCopy();
+    const std::list<BleFingerprint *>* const getNative();
+    const std::list<BleFingerprint *> getCopy();
     void setDisable(bool disable) { _disable = disable; }
-
-    static String include, exclude, query;
+    static String knownMacs, include, exclude, query;
     static float skipDistance, maxDistance, absorption;
     static int refRssi, forgetMs, skipMs;
+    static String countIds;
+    static float countEnter, countExit;
+    static int countMs;
 
 private:
     bool _disable = false;

lib/BleFingerprint/GUI.cpp

@@ -1,4 +1,7 @@
 #include "GUI.h"
+#ifdef M5STICK
+#include "tb_display.h"
+#endif
 
 #if defined M5STICK
 
@@ -57,8 +60,6 @@ void GUI::erased()
 
 void GUI::connecting()
 {
-    status("Connecting...");
-    connected(false, false);
 #ifdef LED_BUILTIN
     if (GUI::statusLed) digitalWrite(LED_BUILTIN, !digitalRead(LED_BUILTIN));
 #endif
@@ -85,45 +86,55 @@ void GUI::connected(bool wifi = false, bool mqtt = false)
 void GUI::added(BleFingerprint *f)
 {
     if (f->getIgnore()) return;
-    Serial.printf("%d New %s | MAC: %s, ID: %-60s %s\n", xPortGetCoreID(), f->getRmAsst() ? "R" : (f->getAllowQuery() ? "Q" : " "), f->getMac().c_str(), f->getId().c_str(), f->getDiscriminator().c_str());
+    Serial.printf("%u New %s | MAC: %s, ID: %-60s %s\n", xPortGetCoreID(), f->getRmAsst() ? "R" : (f->getAllowQuery() ? "Q" : " "), f->getMac().c_str(), f->getId().c_str(), f->getDiscriminator().c_str());
 }
 
  void GUI::removed(BleFingerprint *f)
 {
     if (f->getIgnore() || !f->getAdded()) return;
-    Serial.printf("\u001b[38;5;236m%d Del   | MAC: %s, ID: %-60s %s\u001b[0m\n", xPortGetCoreID(), f->getMac().c_str(), f->getId().c_str(), f->getDiscriminator().c_str());
+    Serial.printf("\u001b[38;5;236m%u Del   | MAC: %s, ID: %-60s %s\u001b[0m\n", xPortGetCoreID(), f->getMac().c_str(), f->getId().c_str(), f->getDiscriminator().c_str());
+}
+
+void GUI::plusOne(BleFingerprint *f)
+{
+    Serial.printf("\u001b[36m%u C# +1 | MAC: %s, ID: %-60s (%.2fm) %lums\u001b[0m\n", xPortGetCoreID(), f->getMac().c_str(), f->getId().c_str(), f->getDistance(), f->getMsSinceLastSeen());
+}
+
+void GUI::minusOne(BleFingerprint *f)
+{
+    Serial.printf("\u001b[35m%u C# -1 | MAC: %s, ID: %-60s (%.2fm) %lums\u001b[0m\n", xPortGetCoreID(), f->getMac().c_str(), f->getId().c_str(), f->getDistance(), f->getMsSinceLastSeen());
 }
 
  void GUI::close(BleFingerprint *f)
 {
     if (f->getIgnore()) return;
-    Serial.printf("\u001b[32m%d Close | MAC: %s, ID: %-60s (%.2fm) %ddBm\u001b[0m\n", xPortGetCoreID(), f->getMac().c_str(), f->getId().c_str(), f->getDistance(), f->getNewestRssi());
+    Serial.printf("\u001b[32m%u Close | MAC: %s, ID: %-60s (%.2fm) %ddBm\u001b[0m\n", xPortGetCoreID(), f->getMac().c_str(), f->getId().c_str(), f->getDistance(), f->getNewestRssi());
     status("C: %s", f->getId().c_str());
 }
 
  void GUI::left(BleFingerprint *f)
 {
     if (f->getIgnore()) return;
-    Serial.printf("\u001b[33m%d Left  | MAC: %s, ID: %-60s (%.2fm) %ddBm\u001b[0m\n", xPortGetCoreID(), f->getMac().c_str(), f->getId().c_str(), f->getDistance(), f->getNewestRssi());
+    Serial.printf("\u001b[33m%u Left  | MAC: %s, ID: %-60s (%.2fm) %ddBm\u001b[0m\n", xPortGetCoreID(), f->getMac().c_str(), f->getId().c_str(), f->getDistance(), f->getNewestRssi());
     status("L: %s", f->getId().c_str());
 }
 
  void GUI::status(const char *format, ...)
 {
     begin();
 #ifdef M5STICK
-    sprite.fillSprite(TFT_BLACK);
-    sprite.setTextDatum(MC_DATUM);
-
     char *message;
     va_list args;
     va_start(args, format);
     vasprintf(&message, format, args);
     va_end(args);
+    tb_display_print_String(message);
+    tb_display_print_String("\n");
 #ifdef PLUS
-    sprite.drawString(message, sprite.width() / 2, sprite.height() / 2, 4);
+    //drawString(message, sprite.width() / 2, sprite.height() / 2, 4);
 #else
-    sprite.drawString(message, sprite.width() / 2, sprite.height() / 2, 1);
+
+    //sprite.drawString(message, sprite.width() / 2, sprite.height() / 2, 1);
 #endif
     free(message);
     dirty = true;
@@ -143,29 +154,15 @@ void GUI::added(BleFingerprint *f)
     {
 #ifdef M5STICK
         M5.begin(true, true, false);
-        M5.Lcd.setRotation(3);
-        sprite.createSprite(M5.Lcd.width(), M5.Lcd.height());
-        sprite.setSwapBytes(true);
+        M5.Axp.ScreenBreath(12);
+        tb_display_init(3);
 #elif defined M5ATOM
         M5.begin(false, false, true);
-        M5.dis.drawpix(0, CRGB(64, 0, 0));
 #endif
         GUI::init = true;
     }
 }
 
- void GUI::blit()
-{
-    begin();
-#ifdef M5STICK
-    if (dirty)
-    {
-        sprite.pushSprite(0, 0);
-        M5.Axp.ScreenBreath(12);
-        dirty = false;
-    }
-#endif
-}
 
  void GUI::updateStart()
 {
@@ -193,5 +190,4 @@ bool GUI::statusLed=false;
 
 #ifdef M5STICK
 bool GUI::dirty = false;
-TFT_eSprite GUI::sprite(&M5.Lcd);
 #endif

lib/BleFingerprint/GUI.h

@@ -40,10 +40,12 @@ class GUI
     static void connected(bool wifi, bool mqtt);
 
     static void status(const char *message, ...);
-    static void blit();
 
     static bool statusLed;
 
+    static void plusOne(BleFingerprint *f);
+    static void minusOne(BleFingerprint *f);
+
 private:
     static void begin();
 

lib/BleFingerprint/strings.cpp

@@ -74,3 +74,49 @@ std::string hexStr(const std::string& s)
 {
     return hexStr(s.c_str(), s.length());
 }
+
+std::string hexStr(const uint8_t *&s, unsigned int len)
+{
+    return hexStr(reinterpret_cast<const char *>(s), len);
+}
+
+std::string hexStrRev(const char *data, unsigned int len)
+{
+    constexpr char hexmap[] = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'};
+
+    std::string s(len * 2, ' ');
+    for (int i = 0; i < len; ++i)
+    {
+        s[len - (2 * i + 1)] = hexmap[(data[i] & 0xF0) >> 4];
+        s[len - (2 * i + 2)] = hexmap[data[i] & 0x0F];
+    }
+    return s;
+}
+
+std::string hexStrRev(const uint8_t *&s, unsigned int len)
+{
+    return hexStrRev(reinterpret_cast<const char *>(s), len);
+}
+
+std::string hexStrRev(const std::string &s)
+{
+    return hexStrRev(s.c_str(), s.length());
+}
+
+bool prefixExists(const String& prefixes, const String& s)
+{
+    unsigned int start = 0;
+    unsigned int space;
+
+    while ((space = prefixes.indexOf(" ", start)) != -1)
+    {
+        if (space > start)
+        {
+            auto sub = prefixes.substring(start, space);
+            if (s.indexOf(sub) != -1) return true;
+        }
+        start = space + 1;
+    }
+    auto sub = prefixes.substring(start);
+    return !sub.isEmpty() && s.indexOf(sub) != -1;
+}

lib/BleFingerprint/strings.h

@@ -16,6 +16,11 @@ std::string slugify(const std::string& text);
 String slugify(const String& text);
 std::string kebabify(const std::string& text);
 String kebabify(const String& text);
+std::string hexStr(const uint8_t *data, int len);
 std::string hexStr(const char *data, int len);
 std::string hexStr(const std::string& s);
+std::string hexStrRev(const uint8_t *data, int len);
+std::string hexStrRev(const char *data, int len);
+std::string hexStrRev(const std::string &s);
+bool prefixExists(const String& prefixes, const String& s);
 #endif

lib/BleFingerprint/util.h

@@ -7,6 +7,7 @@
 static BLEUUID eddystoneUUID((uint16_t)0xFEAA);
 static BLEUUID tileUUID((uint16_t)0xFEED);
 static BLEUUID exposureUUID((uint16_t)0xFD6F);
+static BLEUUID smartTagUUID((uint16_t)0xFD5A);
 static BLEUUID sonosUUID((uint16_t)0xFE07);
 static BLEUUID itagUUID((uint16_t)0xffe0);
 static BLEUUID miThermUUID(uint16_t(0x181A));