Skip to content

Commit

Permalink
Merge pull request #872 from ericyeargan/server-v3-metrics-filter-v2
Browse files Browse the repository at this point in the history
Implement Server v3 metrics filtering
  • Loading branch information
dexterbg committed Apr 15, 2023
2 parents bdb1d3a + 88a6ea7 commit fbbf3c0
Show file tree
Hide file tree
Showing 12 changed files with 394 additions and 122 deletions.
25 changes: 25 additions & 0 deletions docs/source/userguide/components.rst
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,31 @@ OVMS Server v3

The OVMS Server v3 protocol is MQTT. This is an industry standard protocol, and means an OVMS v3 module can communicate with any standard MQTT server. While this is the future of OVMS, support for this is experimental at the moment and production users should use OVMS Server v2 protocol.

By default, OMVS Server v3 transmits all metrics. This can be adjusted by modifying the ``server.v3`` ``metrics.include`` and ``metrics.exclude`` configuration parameters.

The ``metrics.include`` and ``metrics.exclude`` parameters should contain a comma separate list of metric names with optional leading or trailing wildcards.

Metrics names matching the include list and **not matching** the exclude list will be transmitted. Note that if ``metrics.include`` is not defined or empty, all metric names will be included.

^^^^^^^^
Examples
^^^^^^^^

Only send the ``s.v3.connected`` and ``v.b.soc`` metrics::

OVMS# config set server.v3 metrics.include s.v3.connected,v.b.soc
OVMS# config rm server.v3 metrics.exclude

Send all metrics except ``v.b.soc``::

OVMS# config rm server.v3 metrics.include
OVMS# config set server.v3 metrics.exclude v.b.soc

Send all ``v.b`` metrics except ``v.b.soc``::

OVMS# config set server.v3 metrics.include v.b.*
OVMS# config set server.v3 metrics.exclude v.b.soc

-------------------------------
Upgrading from OVMS v1/v2 to v3
-------------------------------
Expand Down
4 changes: 4 additions & 0 deletions vehicle/OVMS.V3/changes.txt
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
Open Vehicle Monitor System v3 - Change log

????-??-?? ??? ??????? OTA release
- OVMS Server v3 metrics filtering
New configs:
[server.v3] metrics.include -- Comma-separated list of metric names (with possible wildcard) matching metrics to send
[server.v3] metrics.exclude -- Comma-separated list of metric names (with possible wildcard) matching metrics to not send
- Renault Zoe Phase 2: Initial support
- Improved output of bms shell command for narrow windows.
New commands:
Expand Down
117 changes: 5 additions & 112 deletions vehicle/OVMS.V3/components/can/src/canlog.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -338,6 +338,7 @@ std::string canlogconnection::GetStats()
////////////////////////////////////////////////////////////////////////

canlog::canlog(const char* type, std::string format, canformat::canformat_serve_mode_t mode)
: m_events_filters(TAG), m_metrics_filters(TAG)
{
m_type = type;
m_format = format;
Expand Down Expand Up @@ -438,83 +439,6 @@ void canlog::RxTask(void *context)
}
}

/**
* Parse a comma-separated list of filters, and assign them to a member of the class.
* We have 3 kind of comparisons, and an unlimited list of filters.
*
* A filter can be:
* - A "startsWith" comparison - when ending with '*',
* - An "endsWith" comparison - when starting with '*',
* - Invalid (and skipped) if empty, or with a '*' in any other position than beginning or end,
* - A "string equal" comparison for all other cases
*/
static void LoadFilters(conn_filters_arr_t &member, const std::string &value)
{
// Empty all previously defined filters, for all operators
for (int i=0; i<COUNT_OF_OPERATORS; i++)
{
member[i].clear();
}

if (!value.empty())
{
std::stringstream stream (value);
std::string item;
unsigned char comparison_operator;

// Comma-separated list
while (getline (stream, item, ','))
{
trim(item); // Removing leading and trailing spaces

if (item.empty())
{
ESP_LOGW(TAG, "LoadFilters: skipping empty value in the filter list");
continue;
}

// Check if there is a wildcard ('*') in any other place than first or last position
size_t wildcard_position = item.find('*', 1);
if ((wildcard_position != std::string::npos) && (wildcard_position != item.size()-1))
{
ESP_LOGW(TAG, "LoadFilters: skipping incorrect value (%s) in the filter list (wildcard in wrong position)", item.c_str());
continue;
}

// Depending on the presence and position of the wildcard, push the filter
// in the proper vector (without the wildcard)
if (item.front() == '*')
{
comparison_operator = OPERATOR_ENDSWITH;
item.erase(0, 1);
}
else if (item.back() == '*')
{
comparison_operator = OPERATOR_STARTSWITH;
item.pop_back();
}
else
{
comparison_operator = OPERATOR_EQUALS;
}
member[comparison_operator].push_back(item);
}
}

// for (int i=0; i<COUNT_OF_OPERATORS; i++)
// {
// ESP_LOGI(TAG, "LoadFilters: filters for operator %d:", i);
// for (std::vector<std::string>::iterator it=member[i].begin(); it!=member[i].end(); ++it)
// {
// ESP_LOGI(TAG, "LoadFilters: filter value '%s'", it->c_str());
// }
// if (member[i].begin() == member[i].end())
// {
// ESP_LOGI(TAG, "LoadFilters: (empty filter list)");
// }
// }
}

/**
* Load, or reload, the configuration of events and metrics filters.
*
Expand All @@ -529,15 +453,15 @@ void canlog::LoadConfig()
if (str_hash != m_events_filters_hash)
{
m_events_filters_hash = str_hash;
LoadFilters(m_events_filters, list_of_events_filters);
m_events_filters.LoadFilters(list_of_events_filters);
MyCan.LogInfo(NULL, CAN_LogInfo_Config, ("Events filters: " + list_of_events_filters).c_str());
}
std::string list_of_metrics_filters = MyConfig.GetParamValue(CAN_PARAM, "log.metrics_filters");
str_hash = std::hash<std::string>{}(list_of_metrics_filters);
if (str_hash != m_metrics_filters_hash)
{
m_metrics_filters_hash = str_hash;
LoadFilters(m_metrics_filters, list_of_metrics_filters);
m_metrics_filters.LoadFilters(list_of_metrics_filters);
MyCan.LogInfo(NULL, CAN_LogInfo_Config, ("Metrics filters: " + list_of_metrics_filters).c_str());
}
}
Expand All @@ -559,49 +483,18 @@ void canlog::UpdatedConfig(std::string event, void* data)
LoadConfig();
}

/**
* Check if a value matches in a list of filters.
*
* Match can be a:
* - startsWith match,
* - endsWith match,
* - equality match.
*/
static bool CheckFilter(conn_filters_arr_t &member, const std::string &value)
{
for (int i=0; i<COUNT_OF_OPERATORS; i++)
{
for (std::vector<std::string>::iterator it=member[i].begin(); it!=member[i].end(); ++it)
{
if ((i == OPERATOR_STARTSWITH) && (startsWith(value, *it)))
{
return true;
}
else if ((i == OPERATOR_ENDSWITH) && (endsWith(value, *it)))
{
return true;
}
else if ((i == OPERATOR_EQUALS) && (value == *it))
{
return true;
}
}
}
return false;
}

void canlog::EventListener(std::string event, void* data)
{
// Log vehicle custom (x…) & framework events:
if (CheckFilter(m_events_filters, event))
if (m_events_filters.CheckFilter(event))
LogInfo(NULL, CAN_LogInfo_Event, event.c_str());
}

void canlog::MetricListener(OvmsMetric* metric)
{
std::string name = metric->m_name;
// Log metrics (in JSON for later parsing):
if (CheckFilter(m_metrics_filters, name))
if (m_metrics_filters.CheckFilter(name))
{
std::string metric_text = "{ ";
metric_text += "\"name\": \"" + json_encode(name) + "\", ";
Expand Down
11 changes: 3 additions & 8 deletions vehicle/OVMS.V3/components/can/src/canlog.h
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
#include "ovms_netmanager.h"
#endif //#ifdef CONFIG_OVMS_SC_GPL_MONGOOSE
#include "ovms_metrics.h"
#include "id_filter.h"

/**
* canlog is the general interface and base implementation for all can loggers.
Expand Down Expand Up @@ -96,12 +97,6 @@ class canlogconnection: public InternalRamAllocated
uint32_t m_filtercount;
};

#define OPERATOR_STARTSWITH 0
#define OPERATOR_ENDSWITH 1
#define OPERATOR_EQUALS 2
#define COUNT_OF_OPERATORS 3
typedef std::array<std::vector<std::string>, COUNT_OF_OPERATORS> conn_filters_arr_t;

class canlog : public InternalRamAllocated
{
public:
Expand Down Expand Up @@ -165,9 +160,9 @@ class canlog : public InternalRamAllocated
virtual void LoadConfig();

protected:
conn_filters_arr_t m_events_filters;
IdFilter m_events_filters;
size_t m_events_filters_hash = 0;
conn_filters_arr_t m_metrics_filters;
IdFilter m_metrics_filters;
size_t m_metrics_filters_hash = 0;
};

Expand Down
4 changes: 4 additions & 0 deletions vehicle/OVMS.V3/components/id_filter/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
idf_component_register(SRCS "src/id_filter.cpp src/id_include_exclude_filter.cpp"
INCLUDE_DIRS src
PRIV_REQUIRES "main"
WHOLE_ARCHIVE)
12 changes: 12 additions & 0 deletions vehicle/OVMS.V3/components/id_filter/component.mk
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
#
# Main component makefile.
#
# This Makefile can be left empty. By default, it will take the sources in the
# src/ directory, compile them and link them into lib(subdirectory_name).a
# in the build directory. This behaviour is entirely configurable,
# please read the ESP-IDF documents if you need to do this.
#

COMPONENT_SRCDIRS := src
COMPONENT_ADD_INCLUDEDIRS := src
COMPONENT_ADD_LDFLAGS = -Wl,--whole-archive -l$(COMPONENT_NAME) -Wl,--no-whole-archive
135 changes: 135 additions & 0 deletions vehicle/OVMS.V3/components/id_filter/src/id_filter.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
/*
; Project: Open Vehicle Monitor System
; Date: 8th April 2023
;
; Changes:
; 1.0 Initial release
;
; Permission is hereby granted, free of charge, to any person obtaining a copy
; of this software and associated documentation files (the "Software"), to deal
; in the Software without restriction, including without limitation the rights
; to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
; copies of the Software, and to permit persons to whom the Software is
; furnished to do so, subject to the following conditions:
;
; The above copyright notice and this permission notice shall be included in
; all copies or substantial portions of the Software.
;
; THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
; IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
; FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
; AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
; LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
; OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
; THE SOFTWARE.
*/

#include "id_filter.h"

#include <sstream>
#include <esp_log.h>

#include "ovms_utils.h"


IdFilter::IdFilter(const char* log_tag)
: m_log_tag(log_tag)
{
}

void IdFilter::LoadFilters(const std::string &value)
{
// Empty all previously defined filters, for all operators
for (int i=0; i<COUNT_OF_OPERATORS; i++)
{
m_entries[i].clear();
}
m_entry_count = 0;

if (!value.empty())
{
std::stringstream stream (value);
std::string item;
unsigned char comparison_operator;

// Comma-separated list
while (getline (stream, item, ','))
{
trim(item); // Removing leading and trailing spaces

if (item.empty())
{
ESP_LOGW(m_log_tag, "LoadFilters: skipping empty value in the filter list");
continue;
}

// Check if there is a wildcard ('*') in any other place than first or last position
size_t wildcard_position = item.find('*', 1);
if ((wildcard_position != std::string::npos) && (wildcard_position != item.size()-1))
{
ESP_LOGW(m_log_tag, "LoadFilters: skipping incorrect value (%s) in the filter list (wildcard in wrong position)", item.c_str());
continue;
}

// Depending on the presence and position of the wildcard, push the filter
// in the proper vector (without the wildcard)
if (item.front() == '*')
{
comparison_operator = OPERATOR_ENDSWITH;
item.erase(0, 1);
}
else if (item.back() == '*')
{
comparison_operator = OPERATOR_STARTSWITH;
item.pop_back();
}
else
{
comparison_operator = OPERATOR_EQUALS;
}
m_entries[comparison_operator].push_back(item);
m_entry_count++;
}
}

// for (int i=0; i<COUNT_OF_OPERATORS; i++)
// {
// ESP_LOGI(m_log_tag, "LoadFilters: filters for operator %d:", i);
// for (std::vector<std::string>::iterator it=m_entries[i].begin(); it!=m_entries[i].end(); ++it)
// {
// ESP_LOGI(m_log_tag, "LoadFilters: filter value '%s'", it->c_str());
// }
// if (member[i].begin() == m_entries[i].end())
// {
// ESP_LOGI(m_log_tag, "LoadFilters: (empty filter list)");
// }
// }
}

size_t IdFilter::EntryCount() const
{
return m_entry_count;
}

bool IdFilter::CheckFilter(const std::string &value) const
{
for (int i=0; i<COUNT_OF_OPERATORS; i++)
{
for (std::vector<std::string>::const_iterator it=m_entries[i].begin(); it!=m_entries[i].end(); ++it)
{
if ((i == OPERATOR_STARTSWITH) && (startsWith(value, *it)))
{
return true;
}
else if ((i == OPERATOR_ENDSWITH) && (endsWith(value, *it)))
{
return true;
}
else if ((i == OPERATOR_EQUALS) && (value == *it))
{
return true;
}
}
}
return false;
}

0 comments on commit fbbf3c0

Please sign in to comment.