From d37c0d10e58768da870a5c21b653d1e1dd71f41c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 1 Aug 2023 16:31:35 +0200 Subject: [PATCH 01/16] Add regex filtering support for domains on the Query Log (new config option webserver.api.excludeRegex) Signed-off-by: DL6ER --- src/api/docs/content/specs/config.yaml | 5 + src/api/queries.c | 138 ++++++++++++++++++++----- src/config/config.c | 7 ++ src/config/config.h | 1 + test/pihole.toml | 7 ++ 5 files changed, 132 insertions(+), 26 deletions(-) diff --git a/src/api/docs/content/specs/config.yaml b/src/api/docs/content/specs/config.yaml index 5c7ee3ed9..c68c06ada 100644 --- a/src/api/docs/content/specs/config.yaml +++ b/src/api/docs/content/specs/config.yaml @@ -397,6 +397,10 @@ components: type: array items: type: string + excludeRegex: + type: array + items: + type: string maxHistory: type: integer allow_destructive: @@ -658,6 +662,7 @@ components: totp_secret: '' excludeClients: [ '1.2.3.4', 'localhost', 'fe80::345' ] excludeDomains: [ 'google.de', 'pi-hole.net' ] + excludeRegex: [ '\.fishy-domain\.com$' ] maxHistory: 86400 allow_destructive: true temp: diff --git a/src/api/queries.c b/src/api/queries.c index 9bb6646de..3d8f15e65 100644 --- a/src/api/queries.c +++ b/src/api/queries.c @@ -19,7 +19,7 @@ #include "database/aliasclients.h" // get_memdb() #include "database/query-table.h" - +#include "regex.h" // dbopen(false, ), dbclose() #include "database/common.h" @@ -340,6 +340,48 @@ int api_queries(struct ftl_conn *api) add_querystr_string(api, querystr, "q.dnssec=", ":dnssec", &where); } + // Regex filtering? + const size_t regex_filters = cJSON_GetArraySize(config.webserver.api.excludeRegex.v.json); + regex_t *regex = NULL; + if(regex_filters > 0) + { + // Allocate memory for regex array + regex = calloc(regex_filters, sizeof(regex_t)); + if(regex == NULL) + { + return send_json_error(api, 500, + "internal_error", + "Internal server error, failed to allocate memory", + NULL); + } + + // Compile regexes + for(size_t i = 0; i < regex_filters; i++) + { + // Iterate over regexes + cJSON *filter = NULL; + cJSON_ArrayForEach(filter, config.webserver.api.excludeRegex.v.json) + { + // Skip non-string, invalid and empty values + if(!cJSON_IsString(filter) || filter->valuestring == NULL || strlen(filter->valuestring) == 0) + continue; + + // Compile regex + int rc = regcomp(®ex[i], filter->valuestring, REG_EXTENDED); + if(rc != 0) + { + // Failed to compile regex + char errbuf[1024]; + regerror(rc, ®ex[i], errbuf, sizeof(errbuf)); + return send_json_error(api, 400, + "bad_request", + "Failed to compile regex", + errbuf); + } + } + } + } + // Get connection to in-memory database sqlite3 *db = get_memdb(); @@ -596,7 +638,7 @@ int api_queries(struct ftl_conn *api) log_debug(DEBUG_API, " with cursor: %lu, start: %u, length: %d", cursor, start, length); cJSON *queries = JSON_NEW_ARRAY(); - unsigned int added = 0, recordsCounted = 0; + unsigned int added = 0, recordsCounted = 0, regex_skipped = 0; sqlite3_int64 firstID = -1, id = -1; bool skipTheRest = false; while((rc = sqlite3_step(read_stmt)) == SQLITE_ROW) @@ -608,26 +650,6 @@ int api_queries(struct ftl_conn *api) if(skipTheRest) continue; - // Check if we have reached the limit - if(added >= (unsigned int)length) - { - if(filtering) - { - // We are filtering, so we have to continue to - // step over the remaining rows to get the - // correct number of total records - skipTheRest = true; - continue; - } - else - { - // We are not filtering, so we can stop here - // The total number of records is the number - // of records in the database - break; - } - } - // Get ID of query from database id = sqlite3_column_int64(read_stmt, 0); // q.id @@ -635,6 +657,33 @@ int api_queries(struct ftl_conn *api) if(firstID == -1) firstID = id; + // Apply possible regex filters to Query Log domains + const char *domain = (const char*)sqlite3_column_text(read_stmt, 4); // d.domain + if(regex_filters > 0) + { + bool match = false; + // Iterate over all regex filters + for(size_t i = 0; i < regex_filters; i++) + { + // Check if the domain matches the regex + if(regexec(®ex[i], domain, 0, NULL, 0) == 0) + { + // Domain matches, so we can stop + // iterating here + match = true; + break; + } + } + if(match) + { + // Domain matches, we skip it and adjust the + // counter + recordsCounted--; + regex_skipped++; + continue; + } + } + // Server-side pagination if((unsigned long)id > cursor) { @@ -652,7 +701,37 @@ int api_queries(struct ftl_conn *api) // everything. // Skip everything AFTER we added the requested number // of queries if length is > 0. - break; + continue; + } + + // Check if we have reached the limit + if(added >= (unsigned int)length) + { + if(filtering) + { + // We are filtering, so we have to continue to + // step over the remaining rows to get the + // correct number of total records + if(regex_filters == 0) + { + skipTheRest = true; + continue; + } + } + else + { + // We are not filtering, so we can stop here + // The total number of records is the number + // of records in the database + if(regex_filters == 0) + break; + else + // We are regex filtering, so we have to + // continue to step over the remaining + // rows to get the correct number of + // total records + continue; + } } // Build item object @@ -669,7 +748,7 @@ int api_queries(struct ftl_conn *api) JSON_COPY_STR_TO_OBJECT(item, "type", get_query_type_str(query.type, &query, buffer)); JSON_REF_STR_IN_OBJECT(item, "status", get_query_status_str(query.status)); JSON_REF_STR_IN_OBJECT(item, "dnssec", get_query_dnssec_str(query.dnssec)); - JSON_COPY_STR_TO_OBJECT(item, "domain", sqlite3_column_text(read_stmt, 4)); // d.domain + JSON_COPY_STR_TO_OBJECT(item, "domain", domain); if(sqlite3_column_type(read_stmt, 5) == SQLITE_TEXT && sqlite3_column_bytes(read_stmt, 5) > 0) @@ -735,7 +814,8 @@ int api_queries(struct ftl_conn *api) added++; } - log_debug(DEBUG_API, "Sending %u of %lu in memory and %lu on disk queries", added, mem_dbnum, disk_dbnum); + log_debug(DEBUG_API, "Sending %u of %lu in memory and %lu on disk queries (skipped %u because of regex filters)", + added, mem_dbnum, disk_dbnum, regex_skipped); cJSON *json = JSON_NEW_OBJECT(); JSON_ADD_ITEM_TO_OBJECT(json, "queries", queries); @@ -758,7 +838,7 @@ int api_queries(struct ftl_conn *api) // DataTables specific properties const unsigned long recordsTotal = disk ? disk_dbnum : mem_dbnum; JSON_ADD_NUMBER_TO_OBJECT(json, "recordsTotal", recordsTotal); - JSON_ADD_NUMBER_TO_OBJECT(json, "recordsFiltered", filtering ? recordsCounted : recordsTotal); + JSON_ADD_NUMBER_TO_OBJECT(json, "recordsFiltered", filtering || regex_filters > 0 ? recordsCounted : recordsTotal); JSON_ADD_NUMBER_TO_OBJECT(json, "draw", draw); // Finalize statements @@ -772,5 +852,11 @@ int api_queries(struct ftl_conn *api) message); } + // Free regex memory if allocated + if(regex_filters > 0) + { + free(regex); + } + JSON_SEND_OBJECT(json); } diff --git a/src/config/config.c b/src/config/config.c index 002b04d23..db00e43f2 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -934,6 +934,13 @@ void initConfig(struct config *conf) conf->webserver.api.excludeDomains.t = CONF_JSON_STRING_ARRAY; conf->webserver.api.excludeDomains.d.json = cJSON_CreateArray(); + conf->webserver.api.excludeRegex.k = "webserver.api.excludeRegex"; + conf->webserver.api.excludeRegex.h = "Array of regular expressions to be excluded from certain API responses\n Example: [ \"(^|\\.)\\.google\\.de$\", \"\\.pi-hole\\.net$\" ]"; + conf->webserver.api.excludeRegex.a = cJSON_CreateStringReference("array of regular expressions"); + conf->webserver.api.excludeRegex.t = CONF_JSON_STRING_ARRAY; + conf->webserver.api.excludeRegex.f = FLAG_RESTART_DNSMASQ | FLAG_ADVANCED_SETTING; + conf->webserver.api.excludeRegex.d.json = cJSON_CreateArray(); + conf->webserver.api.maxHistory.k = "webserver.api.maxHistory"; conf->webserver.api.maxHistory.h = "How much history should be imported from the database and returned by the API [seconds]? (max 24*60*60 = 86400)"; conf->webserver.api.maxHistory.t = CONF_UINT; diff --git a/src/config/config.h b/src/config/config.h index f9d765b89..e3a2fbda5 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -224,6 +224,7 @@ struct config { struct conf_item totp_secret; // This is a write-only item struct conf_item excludeClients; struct conf_item excludeDomains; + struct conf_item excludeRegex; struct conf_item maxHistory; struct conf_item allow_destructive; struct { diff --git a/test/pihole.toml b/test/pihole.toml index 878750f07..8e061ff00 100644 --- a/test/pihole.toml +++ b/test/pihole.toml @@ -622,6 +622,13 @@ # array of IP addresses and/or hostnames excludeDomains = [] + # Array of regular expressions to be excluded from certain API responses + # Example: [ "(^|\.)\.google\.de$", "\.pi-hole\.net$" ] + # + # Possible values are: + # array of regular expressions + excludeRegex = [] + # How much history should be imported from the database [seconds]? (max 24*60*60 = # 86400) maxHistory = 86400 From c89a2396bed0c3ea70596ba85155eff84c222281 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 4 Nov 2023 07:59:21 +0100 Subject: [PATCH 02/16] Backslashs need to be escaped to avoid invalid escape sequences in the TOML file Signed-off-by: DL6ER --- src/config/config.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config/config.c b/src/config/config.c index b58c797ee..f04fa10b4 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -950,7 +950,7 @@ void initConfig(struct config *conf) conf->webserver.api.excludeDomains.d.json = cJSON_CreateArray(); conf->webserver.api.excludeRegex.k = "webserver.api.excludeRegex"; - conf->webserver.api.excludeRegex.h = "Array of regular expressions to be excluded from certain API responses\n Example: [ \"(^|\\.)\\.google\\.de$\", \"\\.pi-hole\\.net$\" ]"; + conf->webserver.api.excludeRegex.h = "Array of regular expressions to be excluded from certain API responses. Note that backslashes \"\\\" need to be escaped, i.e. \"\\\\\" in this setting\n Example: [ \"(^|\\\\.)\\\\.google\\\\.de$\", \"\\\\.pi-hole\\\\.net$\" ]"; conf->webserver.api.excludeRegex.a = cJSON_CreateStringReference("array of regular expressions"); conf->webserver.api.excludeRegex.t = CONF_JSON_STRING_ARRAY; conf->webserver.api.excludeRegex.f = FLAG_RESTART_FTL | FLAG_ADVANCED_SETTING; From 23d116caac1872c76e02e5dbfe6f44e5a142b1a5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 10 Jan 2024 21:44:51 +0100 Subject: [PATCH 03/16] Regex filtering is filtering: We need to do full counting to get the correct number of rows Signed-off-by: DL6ER --- src/api/queries.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/api/queries.c b/src/api/queries.c index d29484ea7..9cb14c403 100644 --- a/src/api/queries.c +++ b/src/api/queries.c @@ -438,6 +438,11 @@ int api_queries(struct ftl_conn *api) } } + // We use this boolean to memorize if we are filtering at all. It is used + // later to decide if we can short-circuit the query counting for + // performance reasons. + bool filtering = false; + // Regex filtering? const int regex_filters = cJSON_GetArraySize(config.webserver.api.excludeRegex.v.json); regex_t *regex = NULL; @@ -478,6 +483,10 @@ int api_queries(struct ftl_conn *api) } } } + + // We are filtering, so we have to continue to step over the + // remaining rows to get the correct number of total records + filtering = true; } // Finish preparing query string @@ -504,10 +513,6 @@ int api_queries(struct ftl_conn *api) sqlite3_errstr(rc)); } - // We use this boolean to memorize if we are filtering at all. It is used - // later to decide if we can short-circuit the query counting for - // performance reasons. - bool filtering = false; // Bind items to prepared statement if(api->request->query_string != NULL) { From cff605b14eab7c43a97587622e75b15416bed239 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 10 Jan 2024 21:49:54 +0100 Subject: [PATCH 04/16] Further simplify skipping logic Signed-off-by: DL6ER --- src/api/queries.c | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/src/api/queries.c b/src/api/queries.c index 9cb14c403..2b4d870e3 100644 --- a/src/api/queries.c +++ b/src/api/queries.c @@ -838,25 +838,15 @@ int api_queries(struct ftl_conn *api) // We are filtering, so we have to continue to // step over the remaining rows to get the // correct number of total records - if(regex_filters == 0) - { - skipTheRest = true; - continue; - } + skipTheRest = true; + continue; } else { // We are not filtering, so we can stop here // The total number of records is the number // of records in the database - if(regex_filters == 0) - break; - else - // We are regex filtering, so we have to - // continue to step over the remaining - // rows to get the correct number of - // total records - continue; + break; } } From 0eb1aaa8f4d98b86a2f3d4bb4076f10fc5059601 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 11 Jan 2024 17:35:47 +0100 Subject: [PATCH 05/16] Extend webserver.api.excludeClients and webserver.api.excludeDomains to the Query Log Signed-off-by: DL6ER --- src/api/queries.c | 135 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 130 insertions(+), 5 deletions(-) diff --git a/src/api/queries.c b/src/api/queries.c index 2b4d870e3..0c53edcde 100644 --- a/src/api/queries.c +++ b/src/api/queries.c @@ -489,6 +489,70 @@ int api_queries(struct ftl_conn *api) filtering = true; } + const int client_filters = cJSON_GetArraySize(config.webserver.api.excludeClients.v.json); + char **filter_clients = NULL; + if(client_filters > 0) + { + // Allocate memory for regex array + filter_clients = calloc(client_filters, sizeof(char*)); + if(filter_clients == NULL) + { + return send_json_error(api, 500, + "internal_error", + "Internal server error, failed to allocate memory", + NULL); + } + + // Iterate over regexes + cJSON *filter = NULL; + unsigned int i = 0; + cJSON_ArrayForEach(filter, config.webserver.api.excludeClients.v.json) + { + // Skip non-string, invalid and empty values + if(!cJSON_IsString(filter) || filter->valuestring == NULL || strlen(filter->valuestring) == 0) + continue; + + // Copy string reference + filter_clients[i++] = filter->valuestring; + } + + // We are filtering, so we have to continue to step over the + // remaining rows to get the correct number of total records + filtering = true; + } + + const int domain_filters = cJSON_GetArraySize(config.webserver.api.excludeDomains.v.json); + char **filter_domains = NULL; + if(domain_filters > 0) + { + // Allocate memory for regex array + filter_domains = calloc(domain_filters, sizeof(char*)); + if(filter_domains == NULL) + { + return send_json_error(api, 500, + "internal_error", + "Internal server error, failed to allocate memory", + NULL); + } + + // Iterate over regexes + cJSON *filter = NULL; + unsigned int i = 0; + cJSON_ArrayForEach(filter, config.webserver.api.excludeDomains.v.json) + { + // Skip non-string, invalid and empty values + if(!cJSON_IsString(filter) || filter->valuestring == NULL || strlen(filter->valuestring) == 0) + continue; + + // Copy string reference + filter_domains[i++] = filter->valuestring; + } + + // We are filtering, so we have to continue to step over the + // remaining rows to get the correct number of total records + filtering = true; + } + // Finish preparing query string querystr_finish(querystr, sort_col, sort_dir); @@ -792,6 +856,61 @@ int api_queries(struct ftl_conn *api) } } + // Apply possible client filters to Query Log clients + const char *client_ip = (const char*)sqlite3_column_text(read_stmt, 10); // c.ip + const char *client_name = NULL; + if(sqlite3_column_type(read_stmt, 11) == SQLITE_TEXT && sqlite3_column_bytes(read_stmt, 11) > 0) + client_name = (const char*)sqlite3_column_text(read_stmt, 11); // c.name + if(client_filters > 0) + { + bool match = false; + // Iterate over all client filters + for(int i = 0; i < client_filters; i++) + { + // Check if the client matches the filter + if(strcasecmp(filter_clients[i], client_ip) == 0 || + (client_name != NULL && strcasecmp(filter_clients[i], client_name) == 0)) + { + // Client matches, so we can stop + // iterating here + match = true; + break; + } + } + if(match) + { + // Client matches, we skip it and adjust the + // counter + recordsCounted--; + continue; + } + } + + // Apply possible domain filters to Query Log domains + if(domain_filters > 0) + { + bool match = false; + // Iterate over all domain filters + for(int i = 0; i < domain_filters; i++) + { + // Check if the domain matches the filter + if(strcasecmp(filter_domains[i], domain) == 0) + { + // Domain matches, so we can stop + // iterating here + match = true; + break; + } + } + if(match) + { + // Domain matches, we skip it and adjust the + // counter + recordsCounted--; + continue; + } + } + // Skip all records once we have enough (but still count them) if(skipTheRest) continue; @@ -878,11 +997,9 @@ int api_queries(struct ftl_conn *api) JSON_ADD_ITEM_TO_OBJECT(item, "reply", reply); cJSON *client = JSON_NEW_OBJECT(); - JSON_COPY_STR_TO_OBJECT(client, "ip", sqlite3_column_text(read_stmt, 10)); // c.ip - - if(sqlite3_column_type(read_stmt, 11) == SQLITE_TEXT && - sqlite3_column_bytes(read_stmt, 11) > 0) - JSON_COPY_STR_TO_OBJECT(client, "name", sqlite3_column_text(read_stmt, 11)); // c.name + JSON_COPY_STR_TO_OBJECT(client, "ip", client_ip); + if(client_name != NULL) + JSON_COPY_STR_TO_OBJECT(client, "name", client_name); else JSON_ADD_NULL_TO_OBJECT(client, "name"); JSON_ADD_ITEM_TO_OBJECT(item, "client", client); @@ -971,5 +1088,13 @@ int api_queries(struct ftl_conn *api) free(regex); } + // Free client memory if allocated + if(client_filters > 0) + free(filter_clients); + + // Free domain memory if allocated + if(domain_filters > 0) + free(filter_domains); + JSON_SEND_OBJECT(json); } From ded6692fec41261203b6a535d77e52ae84bb7ca2 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 11 Jan 2024 17:44:54 +0100 Subject: [PATCH 06/16] Clarify which API endpoints are affected by the exclusion settings Signed-off-by: DL6ER --- src/config/config.c | 6 +++--- test/pihole.toml | 12 +++++++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/config/config.c b/src/config/config.c index dae1f1cc8..35987bb05 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -956,19 +956,19 @@ void initConfig(struct config *conf) conf->webserver.api.app_pwhash.d.s = (char*)""; conf->webserver.api.excludeClients.k = "webserver.api.excludeClients"; - conf->webserver.api.excludeClients.h = "Array of clients to be excluded from certain API responses\n Example: [ \"192.168.2.56\", \"fe80::341\", \"localhost\" ]"; + conf->webserver.api.excludeClients.h = "Array of clients to be excluded from certain API responses:\n - Query Log (/api/queries)\n - Top Clients (/api/stats/top_clients)\n - Client activity over time (/api/history/clients)\n Example: [ \"192.168.2.56\", \"fe80::341\", \"localhost\" ]"; conf->webserver.api.excludeClients.a = cJSON_CreateStringReference("array of IP addresses and/or hostnames"); conf->webserver.api.excludeClients.t = CONF_JSON_STRING_ARRAY; conf->webserver.api.excludeClients.d.json = cJSON_CreateArray(); conf->webserver.api.excludeDomains.k = "webserver.api.excludeDomains"; - conf->webserver.api.excludeDomains.h = "Array of domains to be excluded from certain API responses\n Example: [ \"google.de\", \"pi-hole.net\" ]"; + conf->webserver.api.excludeDomains.h = "Array of domains to be excluded from certain API responses:\n - Query Log (/api/queries)\n - Top Clients (/api/stats/top_domains)\n Example: [ \"google.de\", \"pi-hole.net\" ]"; conf->webserver.api.excludeDomains.a = cJSON_CreateStringReference("array of domains"); conf->webserver.api.excludeDomains.t = CONF_JSON_STRING_ARRAY; conf->webserver.api.excludeDomains.d.json = cJSON_CreateArray(); conf->webserver.api.excludeRegex.k = "webserver.api.excludeRegex"; - conf->webserver.api.excludeRegex.h = "Array of regular expressions to be excluded from certain API responses. Note that backslashes \"\\\" need to be escaped, i.e. \"\\\\\" in this setting\n Example: [ \"(^|\\\\.)\\\\.google\\\\.de$\", \"\\\\.pi-hole\\\\.net$\" ]"; + conf->webserver.api.excludeRegex.h = "Array of regular expressions to be excluded from the Query Log. Note that backslashes \"\\\" need to be escaped, i.e. \"\\\\\" in this setting\n Example: [ \"(^|\\\\.)\\\\.google\\\\.de$\", \"\\\\.pi-hole\\\\.net$\" ]"; conf->webserver.api.excludeRegex.a = cJSON_CreateStringReference("array of regular expressions"); conf->webserver.api.excludeRegex.t = CONF_JSON_STRING_ARRAY; conf->webserver.api.excludeRegex.f = FLAG_RESTART_FTL | FLAG_ADVANCED_SETTING; diff --git a/test/pihole.toml b/test/pihole.toml index ef5099733..ef2230149 100644 --- a/test/pihole.toml +++ b/test/pihole.toml @@ -673,7 +673,10 @@ # app_pwhash = "" - # Array of clients to be excluded from certain API responses + # Array of clients to be excluded from certain API responses: + # - Query Log (/api/queries) + # - Top Clients (/api/stats/top_clients) + # - Client activity over time (/api/history/clients) # Example: [ "192.168.2.56", "fe80::341", "localhost" ] # # Possible values are: @@ -682,14 +685,17 @@ "1.2.3.4" ] ### CHANGED, default = [] - # Array of domains to be excluded from certain API responses + # Array of domains to be excluded from certain API responses: + # - Query Log (/api/queries) + # - Top Clients (/api/stats/top_domains) # Example: [ "google.de", "pi-hole.net" ] # # Possible values are: # array of domains excludeDomains = [] - # Array of regular expressions to be excluded from certain API responses + # Array of regular expressions to be excluded from the Query Log. Note that backslashes + # "\" need to be escaped, i.e. "\\" in this setting # Example: [ "(^|\.)\.google\.de$", "\.pi-hole\.net$" ] # # Possible values are: From 4024af9cddd9e0091088a000d6004916fb265a97 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 11 Jan 2024 17:50:21 +0100 Subject: [PATCH 07/16] Only compare against valid filter strings Signed-off-by: DL6ER --- src/api/queries.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/api/queries.c b/src/api/queries.c index 0c53edcde..f2ba04c08 100644 --- a/src/api/queries.c +++ b/src/api/queries.c @@ -867,6 +867,10 @@ int api_queries(struct ftl_conn *api) // Iterate over all client filters for(int i = 0; i < client_filters; i++) { + // Only compare against valid filter strings + if(filter_clients[i] == NULL) + continue; + // Check if the client matches the filter if(strcasecmp(filter_clients[i], client_ip) == 0 || (client_name != NULL && strcasecmp(filter_clients[i], client_name) == 0)) @@ -893,6 +897,10 @@ int api_queries(struct ftl_conn *api) // Iterate over all domain filters for(int i = 0; i < domain_filters; i++) { + // Only compare against valid filter strings + if(filter_domains[i] == NULL) + continue; + // Check if the domain matches the filter if(strcasecmp(filter_domains[i], domain) == 0) { From aa8285846ba6d9bf7971774e0d0b3ceef991f5d0 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 13 Jan 2024 07:59:18 +0100 Subject: [PATCH 08/16] Remove excludeClients from Client activity over time (/api/history/clients) Signed-off-by: DL6ER --- src/api/history.c | 24 ++++++------------------ src/config/config.c | 2 +- test/pihole.toml | 1 - 3 files changed, 7 insertions(+), 20 deletions(-) diff --git a/src/api/history.c b/src/api/history.c index 8a33d7cdb..9e95197d2 100644 --- a/src/api/history.c +++ b/src/api/history.c @@ -72,25 +72,13 @@ int api_history_clients(struct ftl_conn *api) // if skipclient[i] == true then this client should be hidden from // returned data. We initialize it with false bool *skipclient = calloc(counters->clients, sizeof(bool)); - - unsigned int excludeClients = cJSON_GetArraySize(config.webserver.api.excludeClients.v.json); - if(excludeClients > 0) + if(skipclient == NULL) { - for(int clientID = 0; clientID < counters->clients; clientID++) - { - // Get client pointer - const clientsData* client = getClient(clientID, true); - if(client == NULL) - continue; - // Check if this client should be skipped - for(unsigned int i = 0; i < excludeClients; i++) - { - cJSON *item = cJSON_GetArrayItem(config.webserver.api.excludeClients.v.json, i); - if(strcmp(getstr(client->ippos), item->valuestring) == 0 || - strcmp(getstr(client->namepos), item->valuestring) == 0) - skipclient[clientID] = true; - } - } + unlock_shm(); + return send_json_error(api, 500, + "internal_error", + "Failed to allocate memory for skipclient array", + NULL); } // Also skip clients included in others (in alias-clients) diff --git a/src/config/config.c b/src/config/config.c index 35987bb05..2858b27fe 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -956,7 +956,7 @@ void initConfig(struct config *conf) conf->webserver.api.app_pwhash.d.s = (char*)""; conf->webserver.api.excludeClients.k = "webserver.api.excludeClients"; - conf->webserver.api.excludeClients.h = "Array of clients to be excluded from certain API responses:\n - Query Log (/api/queries)\n - Top Clients (/api/stats/top_clients)\n - Client activity over time (/api/history/clients)\n Example: [ \"192.168.2.56\", \"fe80::341\", \"localhost\" ]"; + conf->webserver.api.excludeClients.h = "Array of clients to be excluded from certain API responses:\n - Query Log (/api/queries)\n - Top Clients (/api/stats/top_clients)\n Example: [ \"192.168.2.56\", \"fe80::341\", \"localhost\" ]"; conf->webserver.api.excludeClients.a = cJSON_CreateStringReference("array of IP addresses and/or hostnames"); conf->webserver.api.excludeClients.t = CONF_JSON_STRING_ARRAY; conf->webserver.api.excludeClients.d.json = cJSON_CreateArray(); diff --git a/test/pihole.toml b/test/pihole.toml index ef2230149..3ef5ae5e5 100644 --- a/test/pihole.toml +++ b/test/pihole.toml @@ -676,7 +676,6 @@ # Array of clients to be excluded from certain API responses: # - Query Log (/api/queries) # - Top Clients (/api/stats/top_clients) - # - Client activity over time (/api/history/clients) # Example: [ "192.168.2.56", "fe80::341", "localhost" ] # # Possible values are: From e9e43094bf0f0fa8c00413f1faa07f19b99d844d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 13 Jan 2024 08:20:53 +0100 Subject: [PATCH 09/16] Remove webserver.api.excludeRegex and instead allow regex to be used in the existing excludeDomains and excludeClients Signed-off-by: DL6ER --- src/api/docs/content/specs/config.yaml | 9 +- src/api/queries.c | 185 +++++++++--------------- src/api/stats.c | 190 ++++++++++++++++++++----- src/config/config.c | 11 +- src/config/config.h | 1 - test/pihole.toml | 8 -- 6 files changed, 232 insertions(+), 172 deletions(-) diff --git a/src/api/docs/content/specs/config.yaml b/src/api/docs/content/specs/config.yaml index 42121ae9a..cfacd2d75 100644 --- a/src/api/docs/content/specs/config.yaml +++ b/src/api/docs/content/specs/config.yaml @@ -418,10 +418,6 @@ components: type: array items: type: string - excludeRegex: - type: array - items: - type: string maxHistory: type: integer allow_destructive: @@ -695,9 +691,8 @@ components: pwhash: '' totp_secret: '' app_pwhash: '' - excludeClients: [ '1.2.3.4', 'localhost', 'fe80::345' ] - excludeDomains: [ 'google.de', 'pi-hole.net' ] - excludeRegex: [ '\.fishy-domain\.com$' ] + excludeClients: [ '1\.2\.3\.4', 'localhost', 'fe80::345' ] + excludeDomains: [ 'google\\.de', 'pi-hole\.net' ] maxHistory: 86400 allow_destructive: true temp: diff --git a/src/api/queries.c b/src/api/queries.c index f2ba04c08..2af4f79df 100644 --- a/src/api/queries.c +++ b/src/api/queries.c @@ -444,42 +444,44 @@ int api_queries(struct ftl_conn *api) bool filtering = false; // Regex filtering? - const int regex_filters = cJSON_GetArraySize(config.webserver.api.excludeRegex.v.json); - regex_t *regex = NULL; - if(regex_filters > 0) + const int N_regex_domains = cJSON_GetArraySize(config.webserver.api.excludeDomains.v.json); + regex_t *regex_domains = NULL; + if(N_regex_domains > 0) { // Allocate memory for regex array - regex = calloc(regex_filters, sizeof(regex_t)); - if(regex == NULL) + regex_domains = calloc(N_regex_domains, sizeof(regex_t)); + if(regex_domains == NULL) { return send_json_error(api, 500, "internal_error", - "Internal server error, failed to allocate memory", + "Internal server error, failed to allocate memory for domain regex array", NULL); } // Compile regexes - for(int i = 0; i < regex_filters; i++) + for(int i = 0; i < N_regex_domains; i++) { // Iterate over regexes cJSON *filter = NULL; - cJSON_ArrayForEach(filter, config.webserver.api.excludeRegex.v.json) + cJSON_ArrayForEach(filter, config.webserver.api.excludeDomains.v.json) { // Skip non-string, invalid and empty values if(!cJSON_IsString(filter) || filter->valuestring == NULL || strlen(filter->valuestring) == 0) continue; // Compile regex - int rc = regcomp(®ex[i], filter->valuestring, REG_EXTENDED); + int rc = regcomp(®ex_domains[i], filter->valuestring, REG_EXTENDED); if(rc != 0) { // Failed to compile regex char errbuf[1024]; - regerror(rc, ®ex[i], errbuf, sizeof(errbuf)); + regerror(rc, ®ex_domains[i], errbuf, sizeof(errbuf)); + log_err("Failed to compile domain regex \"%s\": %s", + filter->valuestring, errbuf); return send_json_error(api, 400, - "bad_request", - "Failed to compile regex", - errbuf); + "bad_request", + "Failed to compile domain regex", + filter->valuestring); } } } @@ -489,63 +491,46 @@ int api_queries(struct ftl_conn *api) filtering = true; } - const int client_filters = cJSON_GetArraySize(config.webserver.api.excludeClients.v.json); - char **filter_clients = NULL; - if(client_filters > 0) + const int N_regex_clients = cJSON_GetArraySize(config.webserver.api.excludeClients.v.json); + regex_t *regex_clients = NULL; + if(N_regex_clients > 0) { // Allocate memory for regex array - filter_clients = calloc(client_filters, sizeof(char*)); - if(filter_clients == NULL) + regex_clients = calloc(N_regex_clients, sizeof(regex_t)); + if(regex_clients == NULL) { return send_json_error(api, 500, "internal_error", - "Internal server error, failed to allocate memory", + "Internal server error, failed to allocate memory for client regex array", NULL); } - // Iterate over regexes - cJSON *filter = NULL; - unsigned int i = 0; - cJSON_ArrayForEach(filter, config.webserver.api.excludeClients.v.json) - { - // Skip non-string, invalid and empty values - if(!cJSON_IsString(filter) || filter->valuestring == NULL || strlen(filter->valuestring) == 0) - continue; - - // Copy string reference - filter_clients[i++] = filter->valuestring; - } - - // We are filtering, so we have to continue to step over the - // remaining rows to get the correct number of total records - filtering = true; - } - - const int domain_filters = cJSON_GetArraySize(config.webserver.api.excludeDomains.v.json); - char **filter_domains = NULL; - if(domain_filters > 0) - { - // Allocate memory for regex array - filter_domains = calloc(domain_filters, sizeof(char*)); - if(filter_domains == NULL) - { - return send_json_error(api, 500, - "internal_error", - "Internal server error, failed to allocate memory", - NULL); - } - - // Iterate over regexes - cJSON *filter = NULL; - unsigned int i = 0; - cJSON_ArrayForEach(filter, config.webserver.api.excludeDomains.v.json) + // Compile regexes + for(int i = 0; i < N_regex_clients; i++) { - // Skip non-string, invalid and empty values - if(!cJSON_IsString(filter) || filter->valuestring == NULL || strlen(filter->valuestring) == 0) - continue; + // Iterate over regexes + cJSON *filter = NULL; + cJSON_ArrayForEach(filter, config.webserver.api.excludeClients.v.json) + { + // Skip non-string, invalid and empty values + if(!cJSON_IsString(filter) || filter->valuestring == NULL || strlen(filter->valuestring) == 0) + continue; - // Copy string reference - filter_domains[i++] = filter->valuestring; + // Compile regex + int rc = regcomp(®ex_clients[i], filter->valuestring, REG_EXTENDED); + if(rc != 0) + { + // Failed to compile regex + char errbuf[1024]; + regerror(rc, ®ex_clients[i], errbuf, sizeof(errbuf)); + log_err("Failed to compile client regex \"%s\": %s", + filter->valuestring, errbuf); + return send_json_error(api, 400, + "bad_request", + "Failed to compile client regex", + filter->valuestring); + } + } } // We are filtering, so we have to continue to step over the @@ -831,17 +816,16 @@ int api_queries(struct ftl_conn *api) // Apply possible regex filters to Query Log domains const char *domain = (const char*)sqlite3_column_text(read_stmt, 4); // d.domain - if(regex_filters > 0) + if(N_regex_domains > 0) { bool match = false; // Iterate over all regex filters - for(int i = 0; i < regex_filters; i++) + for(int i = 0; i < N_regex_domains; i++) { // Check if the domain matches the regex - if(regexec(®ex[i], domain, 0, NULL, 0) == 0) + if(regexec(®ex_domains[i], domain, 0, NULL, 0) == 0) { - // Domain matches, so we can stop - // iterating here + // Domain matches match = true; break; } @@ -861,51 +845,22 @@ int api_queries(struct ftl_conn *api) const char *client_name = NULL; if(sqlite3_column_type(read_stmt, 11) == SQLITE_TEXT && sqlite3_column_bytes(read_stmt, 11) > 0) client_name = (const char*)sqlite3_column_text(read_stmt, 11); // c.name - if(client_filters > 0) + if(N_regex_clients > 0) { bool match = false; - // Iterate over all client filters - for(int i = 0; i < client_filters; i++) + // Iterate over all regex filters + for(int i = 0; i < N_regex_clients; i++) { - // Only compare against valid filter strings - if(filter_clients[i] == NULL) - continue; - - // Check if the client matches the filter - if(strcasecmp(filter_clients[i], client_ip) == 0 || - (client_name != NULL && strcasecmp(filter_clients[i], client_name) == 0)) + // Check if the domain matches the regex + if(regexec(®ex_clients[i], client_ip, 0, NULL, 0) == 0) { - // Client matches, so we can stop - // iterating here + // Client IP matches match = true; break; } - } - if(match) - { - // Client matches, we skip it and adjust the - // counter - recordsCounted--; - continue; - } - } - - // Apply possible domain filters to Query Log domains - if(domain_filters > 0) - { - bool match = false; - // Iterate over all domain filters - for(int i = 0; i < domain_filters; i++) - { - // Only compare against valid filter strings - if(filter_domains[i] == NULL) - continue; - - // Check if the domain matches the filter - if(strcasecmp(filter_domains[i], domain) == 0) + else if(client_name != NULL && regexec(®ex_clients[i], client_name, 0, NULL, 0) == 0) { - // Domain matches, so we can stop - // iterating here + // Client name matches match = true; break; } @@ -915,6 +870,7 @@ int api_queries(struct ftl_conn *api) // Domain matches, we skip it and adjust the // counter recordsCounted--; + regex_skipped++; continue; } } @@ -1079,30 +1035,31 @@ int api_queries(struct ftl_conn *api) // DataTables specific properties const unsigned long recordsTotal = disk ? disk_dbnum : mem_dbnum; JSON_ADD_NUMBER_TO_OBJECT(json, "recordsTotal", recordsTotal); - JSON_ADD_NUMBER_TO_OBJECT(json, "recordsFiltered", filtering || regex_filters > 0 ? recordsCounted : recordsTotal); + JSON_ADD_NUMBER_TO_OBJECT(json, "recordsFiltered", filtering ? recordsCounted : recordsTotal); JSON_ADD_NUMBER_TO_OBJECT(json, "draw", draw); // Finalize statements sqlite3_finalize(read_stmt); // Free regex memory if allocated - if(regex_filters > 0) + if(N_regex_domains > 0) { // Free individual regexes - for(int i = 0; i < regex_filters; i++) - regfree(®ex[i]); + for(int i = 0; i < N_regex_domains; i++) + regfree(®ex_domains[i]); // Free array of regex pointers - free(regex); + free(regex_domains); } + if(N_regex_clients > 0) + { + // Free individual regexes + for(int i = 0; i < N_regex_clients; i++) + regfree(®ex_clients[i]); - // Free client memory if allocated - if(client_filters > 0) - free(filter_clients); - - // Free domain memory if allocated - if(domain_filters > 0) - free(filter_domains); + // Free array of regex pointers + free(regex_clients); + } JSON_SEND_OBJECT(json); } diff --git a/src/api/stats.c b/src/api/stats.c index 6d099f029..4fb0d2892 100644 --- a/src/api/stats.c +++ b/src/api/stats.c @@ -198,15 +198,15 @@ int api_stats_top_domains(struct ftl_conn *api) qsort(temparray, added_domains, sizeof(int[2]), cmpdesc); // Get filter - const char* filter = read_setupVarsconf("API_QUERY_LOG_SHOW"); + const char* log_show = read_setupVarsconf("API_QUERY_LOG_SHOW"); bool showpermitted = true, showblocked = true; - if(filter != NULL) + if(log_show != NULL) { - if((strcmp(filter, "permittedonly")) == 0) + if((strcmp(log_show, "permittedonly")) == 0) showblocked = false; - else if((strcmp(filter, "blockedonly")) == 0) + else if((strcmp(log_show, "blockedonly")) == 0) showpermitted = false; - else if((strcmp(filter, "nothing")) == 0) + else if((strcmp(log_show, "nothing")) == 0) { showpermitted = false; showblocked = false; @@ -215,7 +215,48 @@ int api_stats_top_domains(struct ftl_conn *api) clearSetupVarsArray(); // Get domains which the user doesn't want to see - unsigned int excludeDomains = cJSON_GetArraySize(config.webserver.api.excludeDomains.v.json); + const int N_regex_domains = cJSON_GetArraySize(config.webserver.api.excludeDomains.v.json); + regex_t *regex_domains = NULL; + if(N_regex_domains > 0) + { + // Allocate memory for regex array + regex_domains = calloc(N_regex_domains, sizeof(regex_t)); + if(regex_domains == NULL) + { + return send_json_error(api, 500, + "internal_error", + "Internal server error, failed to allocate memory for client regex array", + NULL); + } + + // Compile regexes + for(int i = 0; i < N_regex_domains; i++) + { + // Iterate over regexes + cJSON *filter = NULL; + cJSON_ArrayForEach(filter, config.webserver.api.excludeDomains.v.json) + { + // Skip non-string, invalid and empty values + if(!cJSON_IsString(filter) || filter->valuestring == NULL || strlen(filter->valuestring) == 0) + continue; + + // Compile regex + int rc = regcomp(®ex_domains[i], filter->valuestring, REG_EXTENDED); + if(rc != 0) + { + // Failed to compile regex + char errbuf[1024]; + regerror(rc, ®ex_domains[i], errbuf, sizeof(errbuf)); + log_err("Failed to compile domain regex \"%s\": %s", + filter->valuestring, errbuf); + return send_json_error(api, 400, + "bad_request", + "Failed to compile domain regex", + filter->valuestring); + } + } + } + } int n = 0; cJSON *top_domains = JSON_NEW_ARRAY(); @@ -228,22 +269,31 @@ int api_stats_top_domains(struct ftl_conn *api) if(domain == NULL) continue; - // Skip this domain if there is a filter on it + // Get domain name + const char *domain_name = getstr(domain->domainpos); + + // Hidden domain, probably due to privacy level. Skip this in the top lists + if(strcmp(domain_name, HIDDEN_DOMAIN) == 0) + continue; + + // Skip this client if there is a filter on it bool skip_domain = false; - for(unsigned int j = 0; j < excludeDomains; j++) + if(N_regex_domains > 0) { - cJSON *item = cJSON_GetArrayItem(config.webserver.api.excludeDomains.v.json, j); - if(strcmp(getstr(domain->domainpos), item->valuestring) == 0) + // Iterate over all regex filters + for(int j = 0; j < N_regex_domains; j++) { - skip_domain = true; - break; + // Check if the domain matches the regex + if(regexec(®ex_domains[j], domain_name, 0, NULL, 0) == 0) + { + // Domain matches + skip_domain = true; + break; + } } } - if(skip_domain) - continue; - // Hidden domain, probably due to privacy level. Skip this in the top lists - if(strcmp(getstr(domain->domainpos), HIDDEN_DOMAIN) == 0) + if(skip_domain) continue; int domain_count = -1; @@ -260,7 +310,7 @@ int api_stats_top_domains(struct ftl_conn *api) if(domain_count > -1) { cJSON *domain_item = JSON_NEW_OBJECT(); - JSON_REF_STR_IN_OBJECT(domain_item, "domain", getstr(domain->domainpos)); + JSON_REF_STR_IN_OBJECT(domain_item, "domain", domain_name); JSON_ADD_NUMBER_TO_OBJECT(domain_item, "count", domain_count); JSON_ADD_ITEM_TO_ARRAY(top_domains, domain_item); } @@ -271,6 +321,17 @@ int api_stats_top_domains(struct ftl_conn *api) } free(temparray); + // Free regexes + if(N_regex_domains > 0) + { + // Free individual regexes + for(int i = 0; i < N_regex_domains; i++) + regfree(®ex_domains[i]); + + // Free array of regex pointers + free(regex_domains); + } + cJSON *json = JSON_NEW_OBJECT(); JSON_ADD_ITEM_TO_OBJECT(json, "domains", top_domains); @@ -339,11 +400,52 @@ int api_stats_top_clients(struct ftl_conn *api) qsort(temparray, clients, sizeof(int[2]), cmpdesc); // Get clients which the user doesn't want to see - unsigned int excludeClients = cJSON_GetArraySize(config.webserver.api.excludeClients.v.json); + const int N_regex_clients = cJSON_GetArraySize(config.webserver.api.excludeClients.v.json); + regex_t *regex_clients = NULL; + if(N_regex_clients > 0) + { + // Allocate memory for regex array + regex_clients = calloc(N_regex_clients, sizeof(regex_t)); + if(regex_clients == NULL) + { + return send_json_error(api, 500, + "internal_error", + "Internal server error, failed to allocate memory for client regex array", + NULL); + } + + // Compile regexes + for(int i = 0; i < N_regex_clients; i++) + { + // Iterate over regexes + cJSON *filter = NULL; + cJSON_ArrayForEach(filter, config.webserver.api.excludeClients.v.json) + { + // Skip non-string, invalid and empty values + if(!cJSON_IsString(filter) || filter->valuestring == NULL || strlen(filter->valuestring) == 0) + continue; + + // Compile regex + int rc = regcomp(®ex_clients[i], filter->valuestring, REG_EXTENDED); + if(rc != 0) + { + // Failed to compile regex + char errbuf[1024]; + regerror(rc, ®ex_clients[i], errbuf, sizeof(errbuf)); + log_err("Failed to compile client regex \"%s\": %s", + filter->valuestring, errbuf); + return send_json_error(api, 400, + "bad_request", + "Failed to compile client regex", + filter->valuestring); + } + } + } + } int n = 0; cJSON *top_clients = JSON_NEW_ARRAY(); - for(int i=0; i < clients; i++) + for(int i = 0; i < clients; i++) { // Get sorted indices and counter values (may be either total or blocked count) const int clientID = temparray[2*i + 0]; @@ -353,29 +455,40 @@ int api_stats_top_clients(struct ftl_conn *api) if(client == NULL) continue; + // Get IP and host name of client + const char *client_ip = getstr(client->ippos); + const char *client_name = getstr(client->namepos); + + // Hidden client, probably due to privacy level. Skip this in the top lists + if(strcmp(client_ip, HIDDEN_CLIENT) == 0) + continue; + // Skip this client if there is a filter on it bool skip_client = false; - for(unsigned int j = 0; j < excludeClients; j++) + if(N_regex_clients > 0) { - cJSON *item = cJSON_GetArrayItem(config.webserver.api.excludeClients.v.json, j); - if(strcmp(getstr(client->ippos), item->valuestring) == 0 || - strcmp(getstr(client->namepos), item->valuestring) == 0) + // Iterate over all regex filters + for(int j = 0; j < N_regex_clients; j++) { - skip_client = true; - break; + // Check if the domain matches the regex + if(regexec(®ex_clients[j], client_ip, 0, NULL, 0) == 0) + { + // Client IP matches + skip_client = true; + break; + } + else if(client_name != NULL && regexec(®ex_clients[j], client_name, 0, NULL, 0) == 0) + { + // Client name matches + skip_client = true; + break; + } } } - if(skip_client) - continue; - // Hidden client, probably due to privacy level. Skip this in the top lists - if(strcmp(getstr(client->ippos), HIDDEN_CLIENT) == 0) + if(skip_client) continue; - // Get client IP and name - const char *client_ip = getstr(client->ippos); - const char *client_name = getstr(client->namepos); - // Return this client if the client made at least one query // within the most recent 24 hours if(client_count > 0) @@ -394,6 +507,17 @@ int api_stats_top_clients(struct ftl_conn *api) // Free temporary array free(temparray); + // Free regexes + if(N_regex_clients > 0) + { + // Free individual regexes + for(int i = 0; i < N_regex_clients; i++) + regfree(®ex_clients[i]); + + // Free array of regex pointers + free(regex_clients); + } + cJSON *json = JSON_NEW_OBJECT(); JSON_ADD_ITEM_TO_OBJECT(json, "clients", top_clients); diff --git a/src/config/config.c b/src/config/config.c index 2858b27fe..83747c031 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -956,24 +956,17 @@ void initConfig(struct config *conf) conf->webserver.api.app_pwhash.d.s = (char*)""; conf->webserver.api.excludeClients.k = "webserver.api.excludeClients"; - conf->webserver.api.excludeClients.h = "Array of clients to be excluded from certain API responses:\n - Query Log (/api/queries)\n - Top Clients (/api/stats/top_clients)\n Example: [ \"192.168.2.56\", \"fe80::341\", \"localhost\" ]"; + conf->webserver.api.excludeClients.h = "Array of clients to be excluded from certain API responses (regex):\n - Query Log (/api/queries)\n - Top Clients (/api/stats/top_clients)\n Note that backslashes \"\\\" need to be escaped, i.e. \"\\\\\" in this setting\n Example: [ \"(^|\\\\.)\\\\.google\\\\.de$\", \"\\\\.pi-hole\\\\.net$\" ]\n\n Example: [ \"192.168.2.56\", \"fe80::341\", \"localhost\" ]"; conf->webserver.api.excludeClients.a = cJSON_CreateStringReference("array of IP addresses and/or hostnames"); conf->webserver.api.excludeClients.t = CONF_JSON_STRING_ARRAY; conf->webserver.api.excludeClients.d.json = cJSON_CreateArray(); conf->webserver.api.excludeDomains.k = "webserver.api.excludeDomains"; - conf->webserver.api.excludeDomains.h = "Array of domains to be excluded from certain API responses:\n - Query Log (/api/queries)\n - Top Clients (/api/stats/top_domains)\n Example: [ \"google.de\", \"pi-hole.net\" ]"; + conf->webserver.api.excludeDomains.h = "Array of domains to be excluded from certain API responses (regex):\n - Query Log (/api/queries)\n - Top Clients (/api/stats/top_domains)\n Note that backslashes \"\\\" need to be escaped, i.e. \"\\\\\" in this setting\n Example: [ \"(^|\\\\.)\\\\.google\\\\.de$\", \"\\\\.pi-hole\\\\.net$\" ]\n\n Example: [ \"google.de\", \"pi-hole.net\" ]"; conf->webserver.api.excludeDomains.a = cJSON_CreateStringReference("array of domains"); conf->webserver.api.excludeDomains.t = CONF_JSON_STRING_ARRAY; conf->webserver.api.excludeDomains.d.json = cJSON_CreateArray(); - conf->webserver.api.excludeRegex.k = "webserver.api.excludeRegex"; - conf->webserver.api.excludeRegex.h = "Array of regular expressions to be excluded from the Query Log. Note that backslashes \"\\\" need to be escaped, i.e. \"\\\\\" in this setting\n Example: [ \"(^|\\\\.)\\\\.google\\\\.de$\", \"\\\\.pi-hole\\\\.net$\" ]"; - conf->webserver.api.excludeRegex.a = cJSON_CreateStringReference("array of regular expressions"); - conf->webserver.api.excludeRegex.t = CONF_JSON_STRING_ARRAY; - conf->webserver.api.excludeRegex.f = FLAG_RESTART_FTL | FLAG_ADVANCED_SETTING; - conf->webserver.api.excludeRegex.d.json = cJSON_CreateArray(); - conf->webserver.api.maxHistory.k = "webserver.api.maxHistory"; conf->webserver.api.maxHistory.h = "How much history should be imported from the database and returned by the API [seconds]? (max 24*60*60 = 86400)"; conf->webserver.api.maxHistory.t = CONF_UINT; diff --git a/src/config/config.h b/src/config/config.h index 9cf0f4130..f6f4baf08 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -244,7 +244,6 @@ struct config { struct conf_item app_pwhash; struct conf_item excludeClients; struct conf_item excludeDomains; - struct conf_item excludeRegex; struct conf_item maxHistory; struct conf_item allow_destructive; struct { diff --git a/test/pihole.toml b/test/pihole.toml index 3ef5ae5e5..2a595d11e 100644 --- a/test/pihole.toml +++ b/test/pihole.toml @@ -693,14 +693,6 @@ # array of domains excludeDomains = [] - # Array of regular expressions to be excluded from the Query Log. Note that backslashes - # "\" need to be escaped, i.e. "\\" in this setting - # Example: [ "(^|\.)\.google\.de$", "\.pi-hole\.net$" ] - # - # Possible values are: - # array of regular expressions - excludeRegex = [] - # How much history should be imported from the database [seconds]? (max 24*60*60 = # 86400) maxHistory = 86400 From b7f49e9636d1c384793d92c64384164a302f7402 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 13 Jan 2024 08:24:55 +0100 Subject: [PATCH 10/16] Add Pi-hole v5 -> v6 regex migration for webserver.api.exclude{Domains,Clients} Signed-off-by: DL6ER --- src/config/setupVars.c | 75 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 66 insertions(+), 9 deletions(-) diff --git a/src/config/setupVars.c b/src/config/setupVars.c index 3cdb66204..c50e5b4c4 100644 --- a/src/config/setupVars.c +++ b/src/config/setupVars.c @@ -120,7 +120,7 @@ static void get_conf_bool_from_setupVars(const char *key, struct conf_item *conf key, conf_item->k, conf_item->v.b ? "true" : "false"); } -static void get_conf_string_array_from_setupVars(const char *key, struct conf_item *conf_item) +static void get_conf_string_array_from_setupVars(const char *key, struct conf_item *conf_item, const bool convert_to_regex) { // Verify we are allowed to use this function if(conf_item->t != CONF_JSON_STRING_ARRAY) @@ -137,12 +137,69 @@ static void get_conf_string_array_from_setupVars(const char *key, struct conf_it getSetupVarsArray(array); for (unsigned int i = 0; i < setupVarsElements; ++i) { - // Add string to our JSON array - cJSON *item = cJSON_CreateString(setupVarsArray[i]); - cJSON_AddItemToArray(conf_item->v.json, item); - - log_debug(DEBUG_CONFIG, "setupVars.conf:%s -> Setting %s[%u] = %s\n", - key, conf_item->k, i, item->valuestring); + // Convert to regex if requested + if(convert_to_regex) + { + // Convert to regex by adding ^ and $ to the string and replacing . with \. + // We need to allocate memory for this + char *regex = calloc(2*strlen(setupVarsArray[i]), sizeof(char)); + if(regex == NULL) + { + log_warn("get_conf_string_array_from_setupVars(%s) failed: Could not allocate memory for regex", key); + continue; + } + + // Copy string + strcpy(regex, setupVarsArray[i]); + + // Replace . with \. + char *p = regex; + while(*p) + { + if(*p == '.') + { + // Move the rest of the string one character to the right + memmove(p + 1, p, strlen(p) + 1); + // Insert the escape character + *p = '\\'; + // Skip the escape character + p++; + } + p++; + } + + // Add ^ and $ to the string + char *regex2 = calloc(strlen(regex) + 3, sizeof(char)); + if(regex2 == NULL) + { + log_warn("get_conf_string_array_from_setupVars(%s) failed: Could not allocate memory for regex2", key); + free(regex); + continue; + } + sprintf(regex2, "^%s$", regex); + + // Free memory + free(regex); + + // Add string to our JSON array + cJSON *item = cJSON_CreateString(regex2); + cJSON_AddItemToArray(conf_item->v.json, item); + + log_debug(DEBUG_CONFIG, "setupVars.conf:%s -> Setting %s[%u] = %s\n", + key, conf_item->k, i, item->valuestring); + + // Free memory + free(regex2); + } + else + { + // Add string to our JSON array + cJSON *item = cJSON_CreateString(setupVarsArray[i]); + cJSON_AddItemToArray(conf_item->v.json, item); + + log_debug(DEBUG_CONFIG, "setupVars.conf:%s -> Setting %s[%u] = %s\n", + key, conf_item->k, i, item->valuestring); + } } } @@ -384,10 +441,10 @@ void importsetupVarsConf(void) get_conf_bool_from_setupVars("BLOCKING_ENABLED", &config.dns.blocking.active); // Get clients which the user doesn't want to see - get_conf_string_array_from_setupVars("API_EXCLUDE_CLIENTS", &config.webserver.api.excludeClients); + get_conf_string_array_from_setupVars("API_EXCLUDE_CLIENTS", &config.webserver.api.excludeClients, true); // Get domains which the user doesn't want to see - get_conf_string_array_from_setupVars("API_EXCLUDE_DOMAINS", &config.webserver.api.excludeDomains); + get_conf_string_array_from_setupVars("API_EXCLUDE_DOMAINS", &config.webserver.api.excludeDomains, true); // Try to obtain temperature hot value get_conf_temp_limit_from_setupVars(); From e35aa780301f22589e7f1e0cbf6d9f3fbc764336 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 13 Jan 2024 10:08:43 +0100 Subject: [PATCH 11/16] Only free API data when the API was started Signed-off-by: DL6ER --- src/api/auth.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/api/auth.c b/src/api/auth.c index d85811c97..c6f969d47 100644 --- a/src/api/auth.c +++ b/src/api/auth.c @@ -56,6 +56,9 @@ void init_api(void) void free_api(void) { + if(auth_data == NULL) + return; + // Store sessions in database backup_db_sessions(auth_data, max_sessions); max_sessions = 0; From 55339f01b5acfde5184d8d444b11207603f36174 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 13 Jan 2024 10:17:58 +0100 Subject: [PATCH 12/16] Adjust webserver.api.exclude{Clients,Domains} description Signed-off-by: DL6ER --- src/config/config.c | 8 ++++---- test/pihole.toml | 19 ++++++++++++------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/src/config/config.c b/src/config/config.c index 83747c031..a0452c57d 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -956,14 +956,14 @@ void initConfig(struct config *conf) conf->webserver.api.app_pwhash.d.s = (char*)""; conf->webserver.api.excludeClients.k = "webserver.api.excludeClients"; - conf->webserver.api.excludeClients.h = "Array of clients to be excluded from certain API responses (regex):\n - Query Log (/api/queries)\n - Top Clients (/api/stats/top_clients)\n Note that backslashes \"\\\" need to be escaped, i.e. \"\\\\\" in this setting\n Example: [ \"(^|\\\\.)\\\\.google\\\\.de$\", \"\\\\.pi-hole\\\\.net$\" ]\n\n Example: [ \"192.168.2.56\", \"fe80::341\", \"localhost\" ]"; - conf->webserver.api.excludeClients.a = cJSON_CreateStringReference("array of IP addresses and/or hostnames"); + conf->webserver.api.excludeClients.h = "Array of clients to be excluded from certain API responses (regex):\n - Query Log (/api/queries)\n - Top Clients (/api/stats/top_clients)\n This setting accepts both IP addresses (IPv4 and IPv6) as well as hostnames.\n Note that backslashes \"\\\" need to be escaped, i.e. \"\\\\\" in this setting\n\n Example: [ \"^192\\\\.168\\\\.2\\\\.56$\", \"^fe80::341:[0-9a-f]*$\", \"^localhost$\" ]"; + conf->webserver.api.excludeClients.a = cJSON_CreateStringReference("array of regular expressions describing clients"); conf->webserver.api.excludeClients.t = CONF_JSON_STRING_ARRAY; conf->webserver.api.excludeClients.d.json = cJSON_CreateArray(); conf->webserver.api.excludeDomains.k = "webserver.api.excludeDomains"; - conf->webserver.api.excludeDomains.h = "Array of domains to be excluded from certain API responses (regex):\n - Query Log (/api/queries)\n - Top Clients (/api/stats/top_domains)\n Note that backslashes \"\\\" need to be escaped, i.e. \"\\\\\" in this setting\n Example: [ \"(^|\\\\.)\\\\.google\\\\.de$\", \"\\\\.pi-hole\\\\.net$\" ]\n\n Example: [ \"google.de\", \"pi-hole.net\" ]"; - conf->webserver.api.excludeDomains.a = cJSON_CreateStringReference("array of domains"); + conf->webserver.api.excludeDomains.h = "Array of domains to be excluded from certain API responses (regex):\n - Query Log (/api/queries)\n - Top Clients (/api/stats/top_domains)\n Note that backslashes \"\\\" need to be escaped, i.e. \"\\\\\" in this setting\n\n Example: [ \"(^|\\\\.)\\\\.google\\\\.de$\", \"\\\\.pi-hole\\\\.net$\" ]"; + conf->webserver.api.excludeDomains.a = cJSON_CreateStringReference("array of regular expressions describing domains"); conf->webserver.api.excludeDomains.t = CONF_JSON_STRING_ARRAY; conf->webserver.api.excludeDomains.d.json = cJSON_CreateArray(); diff --git a/test/pihole.toml b/test/pihole.toml index 2a595d11e..25793c943 100644 --- a/test/pihole.toml +++ b/test/pihole.toml @@ -673,24 +673,29 @@ # app_pwhash = "" - # Array of clients to be excluded from certain API responses: + # Array of clients to be excluded from certain API responses (regex): # - Query Log (/api/queries) # - Top Clients (/api/stats/top_clients) - # Example: [ "192.168.2.56", "fe80::341", "localhost" ] + # This setting accepts both IP addresses (IPv4 and IPv6) as well as hostnames. + # Note that backslashes "\" need to be escaped, i.e. "\\" in this setting + # + # Example: [ "^192\\.168\\.2\\.56$", "^fe80::341:[0-9a-f]*$", "^localhost$" ] # # Possible values are: - # array of IP addresses and/or hostnames + # array of regular expressions describing clients excludeClients = [ - "1.2.3.4" + "^1\\.2\\.3\\.4$" ] ### CHANGED, default = [] - # Array of domains to be excluded from certain API responses: + # Array of domains to be excluded from certain API responses (regex): # - Query Log (/api/queries) # - Top Clients (/api/stats/top_domains) - # Example: [ "google.de", "pi-hole.net" ] + # Note that backslashes "\" need to be escaped, i.e. "\\" in this setting + # + # Example: [ "(^|\\.)\\.google\\.de$", "\\.pi-hole\\.net$" ] # # Possible values are: - # array of domains + # array of regular expressions describing domains excludeDomains = [] # How much history should be imported from the database [seconds]? (max 24*60*60 = From 5c4355f1b1c9fc2875280d12cd5d044c2b437991 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 16 Jan 2024 22:35:53 +0100 Subject: [PATCH 13/16] Compile exclude regexes only once, not N^2 times Signed-off-by: DL6ER --- src/api/queries.c | 88 +++++++++++++++++++++++++---------------------- src/api/stats.c | 88 +++++++++++++++++++++++++---------------------- 2 files changed, 92 insertions(+), 84 deletions(-) diff --git a/src/api/queries.c b/src/api/queries.c index 2af4f79df..44b94a07d 100644 --- a/src/api/queries.c +++ b/src/api/queries.c @@ -459,31 +459,33 @@ int api_queries(struct ftl_conn *api) } // Compile regexes - for(int i = 0; i < N_regex_domains; i++) + unsigned int i = 0; + cJSON *filter = NULL; + cJSON_ArrayForEach(filter, config.webserver.api.excludeDomains.v.json) { - // Iterate over regexes - cJSON *filter = NULL; - cJSON_ArrayForEach(filter, config.webserver.api.excludeDomains.v.json) + // Skip non-string, invalid and empty values + if(!cJSON_IsString(filter) || filter->valuestring == NULL || strlen(filter->valuestring) == 0) { - // Skip non-string, invalid and empty values - if(!cJSON_IsString(filter) || filter->valuestring == NULL || strlen(filter->valuestring) == 0) - continue; + log_warn("Skipping invalid regex at webserver.api.excludeDomains.%u", i); + continue; + } - // Compile regex - int rc = regcomp(®ex_domains[i], filter->valuestring, REG_EXTENDED); - if(rc != 0) - { - // Failed to compile regex - char errbuf[1024]; - regerror(rc, ®ex_domains[i], errbuf, sizeof(errbuf)); - log_err("Failed to compile domain regex \"%s\": %s", - filter->valuestring, errbuf); - return send_json_error(api, 400, - "bad_request", - "Failed to compile domain regex", - filter->valuestring); - } + // Compile regex + int rc = regcomp(®ex_domains[i], filter->valuestring, REG_EXTENDED); + if(rc != 0) + { + // Failed to compile regex + char errbuf[1024]; + regerror(rc, ®ex_domains[i], errbuf, sizeof(errbuf)); + log_err("Failed to compile domain regex \"%s\": %s", + filter->valuestring, errbuf); + return send_json_error(api, 400, + "bad_request", + "Failed to compile domain regex", + filter->valuestring); } + + i++; } // We are filtering, so we have to continue to step over the @@ -506,31 +508,33 @@ int api_queries(struct ftl_conn *api) } // Compile regexes - for(int i = 0; i < N_regex_clients; i++) + unsigned int i = 0; + cJSON *filter = NULL; + cJSON_ArrayForEach(filter, config.webserver.api.excludeClients.v.json) { - // Iterate over regexes - cJSON *filter = NULL; - cJSON_ArrayForEach(filter, config.webserver.api.excludeClients.v.json) + // Skip non-string, invalid and empty values + if(!cJSON_IsString(filter) || filter->valuestring == NULL || strlen(filter->valuestring) == 0) { - // Skip non-string, invalid and empty values - if(!cJSON_IsString(filter) || filter->valuestring == NULL || strlen(filter->valuestring) == 0) - continue; + log_warn("Skipping invalid regex at webserver.api.excludeClients.%u", i); + continue; + } - // Compile regex - int rc = regcomp(®ex_clients[i], filter->valuestring, REG_EXTENDED); - if(rc != 0) - { - // Failed to compile regex - char errbuf[1024]; - regerror(rc, ®ex_clients[i], errbuf, sizeof(errbuf)); - log_err("Failed to compile client regex \"%s\": %s", - filter->valuestring, errbuf); - return send_json_error(api, 400, - "bad_request", - "Failed to compile client regex", - filter->valuestring); - } + // Compile regex + int rc = regcomp(®ex_clients[i], filter->valuestring, REG_EXTENDED); + if(rc != 0) + { + // Failed to compile regex + char errbuf[1024]; + regerror(rc, ®ex_clients[i], errbuf, sizeof(errbuf)); + log_err("Failed to compile client regex \"%s\": %s", + filter->valuestring, errbuf); + return send_json_error(api, 400, + "bad_request", + "Failed to compile client regex", + filter->valuestring); } + + i++; } // We are filtering, so we have to continue to step over the diff --git a/src/api/stats.c b/src/api/stats.c index 4fb0d2892..0fcab1b7b 100644 --- a/src/api/stats.c +++ b/src/api/stats.c @@ -230,31 +230,33 @@ int api_stats_top_domains(struct ftl_conn *api) } // Compile regexes - for(int i = 0; i < N_regex_domains; i++) + unsigned int i = 0; + cJSON *filter = NULL; + cJSON_ArrayForEach(filter, config.webserver.api.excludeDomains.v.json) { - // Iterate over regexes - cJSON *filter = NULL; - cJSON_ArrayForEach(filter, config.webserver.api.excludeDomains.v.json) + // Skip non-string, invalid and empty values + if(!cJSON_IsString(filter) || filter->valuestring == NULL || strlen(filter->valuestring) == 0) { - // Skip non-string, invalid and empty values - if(!cJSON_IsString(filter) || filter->valuestring == NULL || strlen(filter->valuestring) == 0) - continue; + log_warn("Skipping invalid regex at webserver.api.excludeDomains.%u", i); + continue; + } - // Compile regex - int rc = regcomp(®ex_domains[i], filter->valuestring, REG_EXTENDED); - if(rc != 0) - { - // Failed to compile regex - char errbuf[1024]; - regerror(rc, ®ex_domains[i], errbuf, sizeof(errbuf)); - log_err("Failed to compile domain regex \"%s\": %s", - filter->valuestring, errbuf); - return send_json_error(api, 400, - "bad_request", - "Failed to compile domain regex", - filter->valuestring); - } + // Compile regex + int rc = regcomp(®ex_domains[i], filter->valuestring, REG_EXTENDED); + if(rc != 0) + { + // Failed to compile regex + char errbuf[1024]; + regerror(rc, ®ex_domains[i], errbuf, sizeof(errbuf)); + log_err("Failed to compile domain regex \"%s\": %s", + filter->valuestring, errbuf); + return send_json_error(api, 400, + "bad_request", + "Failed to compile domain regex", + filter->valuestring); } + + i++; } } @@ -415,31 +417,33 @@ int api_stats_top_clients(struct ftl_conn *api) } // Compile regexes - for(int i = 0; i < N_regex_clients; i++) + unsigned int i = 0; + cJSON *filter = NULL; + cJSON_ArrayForEach(filter, config.webserver.api.excludeClients.v.json) { - // Iterate over regexes - cJSON *filter = NULL; - cJSON_ArrayForEach(filter, config.webserver.api.excludeClients.v.json) + // Skip non-string, invalid and empty values + if(!cJSON_IsString(filter) || filter->valuestring == NULL || strlen(filter->valuestring) == 0) { - // Skip non-string, invalid and empty values - if(!cJSON_IsString(filter) || filter->valuestring == NULL || strlen(filter->valuestring) == 0) - continue; + log_warn("Skipping invalid regex at webserver.api.excludeClients.%u", i); + continue; + } - // Compile regex - int rc = regcomp(®ex_clients[i], filter->valuestring, REG_EXTENDED); - if(rc != 0) - { - // Failed to compile regex - char errbuf[1024]; - regerror(rc, ®ex_clients[i], errbuf, sizeof(errbuf)); - log_err("Failed to compile client regex \"%s\": %s", - filter->valuestring, errbuf); - return send_json_error(api, 400, - "bad_request", - "Failed to compile client regex", - filter->valuestring); - } + // Compile regex + int rc = regcomp(®ex_clients[i], filter->valuestring, REG_EXTENDED); + if(rc != 0) + { + // Failed to compile regex + char errbuf[1024]; + regerror(rc, ®ex_clients[i], errbuf, sizeof(errbuf)); + log_err("Failed to compile client regex \"%s\": %s", + filter->valuestring, errbuf); + return send_json_error(api, 400, + "bad_request", + "Failed to compile client regex", + filter->valuestring); } + + i++; } } From 24b6df4cb49a06a49fdf048d0ff55f55e9ea6bd1 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 20 Jan 2024 09:33:06 +0100 Subject: [PATCH 14/16] Reduce code duplication by factoring out filter regex compilation Signed-off-by: DL6ER --- src/api/api.h | 3 + src/api/queries.c | 166 +++++++++++++++++++--------------------------- src/api/stats.c | 102 ++++------------------------ 3 files changed, 83 insertions(+), 188 deletions(-) diff --git a/src/api/api.h b/src/api/api.h index 97907b2ca..e8e57964b 100644 --- a/src/api/api.h +++ b/src/api/api.h @@ -15,6 +15,8 @@ // type cJSON #include "webserver/cJSON/cJSON.h" #include "webserver/http-common.h" +// regex_t +#include "regex_r.h" // Common definitions #define LOCALHOSTv4 "127.0.0.1" @@ -43,6 +45,7 @@ int api_history_database_clients(struct ftl_conn *api); // Query methods int api_queries(struct ftl_conn *api); int api_queries_suggestions(struct ftl_conn *api); +bool compile_filter_regex(struct ftl_conn *api, const char *path, cJSON *json, regex_t **regex, unsigned int *N_regex); // Statistics methods (database) int api_stats_database_top_items(struct ftl_conn *api); diff --git a/src/api/queries.c b/src/api/queries.c index c4d6bf45b..c7ceedb32 100644 --- a/src/api/queries.c +++ b/src/api/queries.c @@ -19,7 +19,6 @@ #include "database/aliasclients.h" // get_memdb() #include "database/query-table.h" -#include "regex.h" // dbopen(false, ), dbclose() #include "database/common.h" @@ -444,103 +443,19 @@ int api_queries(struct ftl_conn *api) bool filtering = false; // Regex filtering? - const int N_regex_domains = cJSON_GetArraySize(config.webserver.api.excludeDomains.v.json); regex_t *regex_domains = NULL; - if(N_regex_domains > 0) - { - // Allocate memory for regex array - regex_domains = calloc(N_regex_domains, sizeof(regex_t)); - if(regex_domains == NULL) - { - return send_json_error(api, 500, - "internal_error", - "Internal server error, failed to allocate memory for domain regex array", - NULL); - } - - // Compile regexes - unsigned int i = 0; - cJSON *filter = NULL; - cJSON_ArrayForEach(filter, config.webserver.api.excludeDomains.v.json) - { - // Skip non-string, invalid and empty values - if(!cJSON_IsString(filter) || filter->valuestring == NULL || strlen(filter->valuestring) == 0) - { - log_warn("Skipping invalid regex at webserver.api.excludeDomains.%u", i); - continue; - } - - // Compile regex - int rc = regcomp(®ex_domains[i], filter->valuestring, REG_EXTENDED); - if(rc != 0) - { - // Failed to compile regex - char errbuf[1024] = { 0 }; - regerror(rc, ®ex_domains[i], errbuf, sizeof(errbuf)); - log_err("Failed to compile domain regex \"%s\": %s", - filter->valuestring, errbuf); - return send_json_error(api, 400, - "bad_request", - "Failed to compile domain regex", - filter->valuestring); - } - - i++; - } - - // We are filtering, so we have to continue to step over the - // remaining rows to get the correct number of total records + unsigned int N_regex_domains = 0; + if(compile_filter_regex(api, "webserver.api.excludeDomains", + config.webserver.api.excludeDomains.v.json, + ®ex_domains, &N_regex_domains)) filtering = true; - } - const int N_regex_clients = cJSON_GetArraySize(config.webserver.api.excludeClients.v.json); regex_t *regex_clients = NULL; - if(N_regex_clients > 0) - { - // Allocate memory for regex array - regex_clients = calloc(N_regex_clients, sizeof(regex_t)); - if(regex_clients == NULL) - { - return send_json_error(api, 500, - "internal_error", - "Internal server error, failed to allocate memory for client regex array", - NULL); - } - - // Compile regexes - unsigned int i = 0; - cJSON *filter = NULL; - cJSON_ArrayForEach(filter, config.webserver.api.excludeClients.v.json) - { - // Skip non-string, invalid and empty values - if(!cJSON_IsString(filter) || filter->valuestring == NULL || strlen(filter->valuestring) == 0) - { - log_warn("Skipping invalid regex at webserver.api.excludeClients.%u", i); - continue; - } - - // Compile regex - int rc = regcomp(®ex_clients[i], filter->valuestring, REG_EXTENDED); - if(rc != 0) - { - // Failed to compile regex - char errbuf[1024] = { 0 }; - regerror(rc, ®ex_clients[i], errbuf, sizeof(errbuf)); - log_err("Failed to compile client regex \"%s\": %s", - filter->valuestring, errbuf); - return send_json_error(api, 400, - "bad_request", - "Failed to compile client regex", - filter->valuestring); - } - - i++; - } - - // We are filtering, so we have to continue to step over the - // remaining rows to get the correct number of total records + unsigned int N_regex_clients = 0; + if(compile_filter_regex(api, "webserver.api.excludeClients", + config.webserver.api.excludeClients.v.json, + ®ex_clients, &N_regex_clients)) filtering = true; - } // Finish preparing query string querystr_finish(querystr, sort_col, sort_dir); @@ -824,7 +739,7 @@ int api_queries(struct ftl_conn *api) { bool match = false; // Iterate over all regex filters - for(int i = 0; i < N_regex_domains; i++) + for(unsigned int i = 0; i < N_regex_domains; i++) { // Check if the domain matches the regex if(regexec(®ex_domains[i], domain, 0, NULL, 0) == 0) @@ -853,7 +768,7 @@ int api_queries(struct ftl_conn *api) { bool match = false; // Iterate over all regex filters - for(int i = 0; i < N_regex_clients; i++) + for(unsigned int i = 0; i < N_regex_clients; i++) { // Check if the domain matches the regex if(regexec(®ex_clients[i], client_ip, 0, NULL, 0) == 0) @@ -1049,7 +964,7 @@ int api_queries(struct ftl_conn *api) if(N_regex_domains > 0) { // Free individual regexes - for(int i = 0; i < N_regex_domains; i++) + for(unsigned int i = 0; i < N_regex_domains; i++) regfree(®ex_domains[i]); // Free array of regex pointers @@ -1058,12 +973,67 @@ int api_queries(struct ftl_conn *api) if(N_regex_clients > 0) { // Free individual regexes - for(int i = 0; i < N_regex_clients; i++) + for(unsigned int i = 0; i < N_regex_clients; i++) regfree(®ex_clients[i]); - // Free array of regex pointers + // Free array of regex po^inters free(regex_clients); } JSON_SEND_OBJECT(json); } + +bool compile_filter_regex(struct ftl_conn *api, const char *path, cJSON *json, regex_t **regex, unsigned int *N_regex) +{ + + const int N = cJSON_GetArraySize(json); + if(N < 1) + return false; + + // Set number of regexes (positive = unsigned integer) + *N_regex = N; + + // Allocate memory for regex array + *regex = calloc(N, sizeof(regex_t)); + if(*regex == NULL) + { + return send_json_error(api, 500, + "internal_error", + "Internal server error, failed to allocate memory for regex array", + NULL); + } + + // Compile regexes + unsigned int i = 0; + cJSON *filter = NULL; + cJSON_ArrayForEach(filter, json) + { + // Skip non-string, invalid and empty values + if(!cJSON_IsString(filter) || filter->valuestring == NULL || strlen(filter->valuestring) == 0) + { + log_warn("Skipping invalid regex at %s.%u", path, i); + continue; + } + + // Compile regex + int rc = regcomp(regex[i], filter->valuestring, REG_EXTENDED); + if(rc != 0) + { + // Failed to compile regex + char errbuf[1024] = { 0 }; + regerror(rc, regex[i], errbuf, sizeof(errbuf)); + log_err("Failed to compile regex \"%s\": %s", + filter->valuestring, errbuf); + return send_json_error(api, 400, + "bad_request", + "Failed to compile regex", + filter->valuestring); + } + + i++; + } + + // We are filtering, so we have to continue to step over the + // remaining rows to get the correct number of total records + return true; +} \ No newline at end of file diff --git a/src/api/stats.c b/src/api/stats.c index ff1618f19..339fb72eb 100644 --- a/src/api/stats.c +++ b/src/api/stats.c @@ -216,50 +216,11 @@ int api_stats_top_domains(struct ftl_conn *api) clearSetupVarsArray(); // Get domains which the user doesn't want to see - const int N_regex_domains = cJSON_GetArraySize(config.webserver.api.excludeDomains.v.json); regex_t *regex_domains = NULL; - if(N_regex_domains > 0) - { - // Allocate memory for regex array - regex_domains = calloc(N_regex_domains, sizeof(regex_t)); - if(regex_domains == NULL) - { - return send_json_error(api, 500, - "internal_error", - "Internal server error, failed to allocate memory for client regex array", - NULL); - } - - // Compile regexes - unsigned int i = 0; - cJSON *filter = NULL; - cJSON_ArrayForEach(filter, config.webserver.api.excludeDomains.v.json) - { - // Skip non-string, invalid and empty values - if(!cJSON_IsString(filter) || filter->valuestring == NULL || strlen(filter->valuestring) == 0) - { - log_warn("Skipping invalid regex at webserver.api.excludeDomains.%u", i); - continue; - } - - // Compile regex - int rc = regcomp(®ex_domains[i], filter->valuestring, REG_EXTENDED); - if(rc != 0) - { - // Failed to compile regex - char errbuf[1024] = { 0 }; - regerror(rc, ®ex_domains[i], errbuf, sizeof(errbuf)); - log_err("Failed to compile domain regex \"%s\": %s", - filter->valuestring, errbuf); - return send_json_error(api, 400, - "bad_request", - "Failed to compile domain regex", - filter->valuestring); - } - - i++; - } - } + unsigned int N_regex_domains = 0; + compile_filter_regex(api, "webserver.api.excludeDomains", + config.webserver.api.excludeDomains.v.json, + ®ex_domains, &N_regex_domains); int n = 0; cJSON *top_domains = JSON_NEW_ARRAY(); @@ -284,7 +245,7 @@ int api_stats_top_domains(struct ftl_conn *api) if(N_regex_domains > 0) { // Iterate over all regex filters - for(int j = 0; j < N_regex_domains; j++) + for(unsigned int j = 0; j < N_regex_domains; j++) { // Check if the domain matches the regex if(regexec(®ex_domains[j], domain_name, 0, NULL, 0) == 0) @@ -328,7 +289,7 @@ int api_stats_top_domains(struct ftl_conn *api) if(N_regex_domains > 0) { // Free individual regexes - for(int i = 0; i < N_regex_domains; i++) + for(unsigned int i = 0; i < N_regex_domains; i++) regfree(®ex_domains[i]); // Free array of regex pointers @@ -403,50 +364,11 @@ int api_stats_top_clients(struct ftl_conn *api) qsort(temparray, clients, sizeof(int[2]), cmpdesc); // Get clients which the user doesn't want to see - const int N_regex_clients = cJSON_GetArraySize(config.webserver.api.excludeClients.v.json); regex_t *regex_clients = NULL; - if(N_regex_clients > 0) - { - // Allocate memory for regex array - regex_clients = calloc(N_regex_clients, sizeof(regex_t)); - if(regex_clients == NULL) - { - return send_json_error(api, 500, - "internal_error", - "Internal server error, failed to allocate memory for client regex array", - NULL); - } - - // Compile regexes - unsigned int i = 0; - cJSON *filter = NULL; - cJSON_ArrayForEach(filter, config.webserver.api.excludeClients.v.json) - { - // Skip non-string, invalid and empty values - if(!cJSON_IsString(filter) || filter->valuestring == NULL || strlen(filter->valuestring) == 0) - { - log_warn("Skipping invalid regex at webserver.api.excludeClients.%u", i); - continue; - } - - // Compile regex - int rc = regcomp(®ex_clients[i], filter->valuestring, REG_EXTENDED); - if(rc != 0) - { - // Failed to compile regex - char errbuf[1024] = { 0 }; - regerror(rc, ®ex_clients[i], errbuf, sizeof(errbuf)); - log_err("Failed to compile client regex \"%s\": %s", - filter->valuestring, errbuf); - return send_json_error(api, 400, - "bad_request", - "Failed to compile client regex", - filter->valuestring); - } - - i++; - } - } + unsigned int N_regex_clients = 0; + compile_filter_regex(api, "webserver.api.excludeClients", + config.webserver.api.excludeClients.v.json, + ®ex_clients, &N_regex_clients); int n = 0; cJSON *top_clients = JSON_NEW_ARRAY(); @@ -473,7 +395,7 @@ int api_stats_top_clients(struct ftl_conn *api) if(N_regex_clients > 0) { // Iterate over all regex filters - for(int j = 0; j < N_regex_clients; j++) + for(unsigned int j = 0; j < N_regex_clients; j++) { // Check if the domain matches the regex if(regexec(®ex_clients[j], client_ip, 0, NULL, 0) == 0) @@ -516,7 +438,7 @@ int api_stats_top_clients(struct ftl_conn *api) if(N_regex_clients > 0) { // Free individual regexes - for(int i = 0; i < N_regex_clients; i++) + for(unsigned int i = 0; i < N_regex_clients; i++) regfree(®ex_clients[i]); // Free array of regex pointers From bafbc780edbe6b35e88c919068021d90af1b8a00 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 20 Jan 2024 09:38:08 +0100 Subject: [PATCH 15/16] Apply review comments Signed-off-by: DL6ER --- src/api/queries.c | 4 +- src/config/setupVars.c | 115 ++++++++++++++++++++--------------------- 2 files changed, 59 insertions(+), 60 deletions(-) diff --git a/src/api/queries.c b/src/api/queries.c index c7ceedb32..fb06cd627 100644 --- a/src/api/queries.c +++ b/src/api/queries.c @@ -733,7 +733,7 @@ int api_queries(struct ftl_conn *api) // Increase number of records from the database recordsCounted++; - // Apply possible regex filters to Query Log domains + // Apply possible domain regex filters to Query Log const char *domain = (const char*)sqlite3_column_text(read_stmt, 4); // d.domain if(N_regex_domains > 0) { @@ -759,7 +759,7 @@ int api_queries(struct ftl_conn *api) } } - // Apply possible client filters to Query Log clients + // Apply possible client regex filters to Query Log const char *client_ip = (const char*)sqlite3_column_text(read_stmt, 10); // c.ip const char *client_name = NULL; if(sqlite3_column_type(read_stmt, 11) == SQLITE_TEXT && sqlite3_column_bytes(read_stmt, 11) > 0) diff --git a/src/config/setupVars.c b/src/config/setupVars.c index c50e5b4c4..59b3224d2 100644 --- a/src/config/setupVars.c +++ b/src/config/setupVars.c @@ -120,7 +120,7 @@ static void get_conf_bool_from_setupVars(const char *key, struct conf_item *conf key, conf_item->k, conf_item->v.b ? "true" : "false"); } -static void get_conf_string_array_from_setupVars(const char *key, struct conf_item *conf_item, const bool convert_to_regex) +static void get_conf_string_array_from_setupVars_regex(const char *key, struct conf_item *conf_item) { // Verify we are allowed to use this function if(conf_item->t != CONF_JSON_STRING_ARRAY) @@ -137,69 +137,56 @@ static void get_conf_string_array_from_setupVars(const char *key, struct conf_it getSetupVarsArray(array); for (unsigned int i = 0; i < setupVarsElements; ++i) { - // Convert to regex if requested - if(convert_to_regex) + // Convert to regex by adding ^ and $ to the string and replacing . with \. + // We need to allocate memory for this + char *regex = calloc(2*strlen(setupVarsArray[i]), sizeof(char)); + if(regex == NULL) { - // Convert to regex by adding ^ and $ to the string and replacing . with \. - // We need to allocate memory for this - char *regex = calloc(2*strlen(setupVarsArray[i]), sizeof(char)); - if(regex == NULL) - { - log_warn("get_conf_string_array_from_setupVars(%s) failed: Could not allocate memory for regex", key); - continue; - } + log_warn("get_conf_string_array_from_setupVars(%s) failed: Could not allocate memory for regex", key); + continue; + } - // Copy string - strcpy(regex, setupVarsArray[i]); + // Copy string + strcpy(regex, setupVarsArray[i]); - // Replace . with \. - char *p = regex; - while(*p) + // Replace . with \. + char *p = regex; + while(*p) + { + if(*p == '.') { - if(*p == '.') - { - // Move the rest of the string one character to the right - memmove(p + 1, p, strlen(p) + 1); - // Insert the escape character - *p = '\\'; - // Skip the escape character - p++; - } + // Move the rest of the string one character to the right + memmove(p + 1, p, strlen(p) + 1); + // Insert the escape character + *p = '\\'; + // Skip the escape character p++; } + p++; + } - // Add ^ and $ to the string - char *regex2 = calloc(strlen(regex) + 3, sizeof(char)); - if(regex2 == NULL) - { - log_warn("get_conf_string_array_from_setupVars(%s) failed: Could not allocate memory for regex2", key); - free(regex); - continue; - } - sprintf(regex2, "^%s$", regex); - - // Free memory + // Add ^ and $ to the string + char *regex2 = calloc(strlen(regex) + 3, sizeof(char)); + if(regex2 == NULL) + { + log_warn("get_conf_string_array_from_setupVars(%s) failed: Could not allocate memory for regex2", key); free(regex); + continue; + } + sprintf(regex2, "^%s$", regex); - // Add string to our JSON array - cJSON *item = cJSON_CreateString(regex2); - cJSON_AddItemToArray(conf_item->v.json, item); + // Free memory + free(regex); - log_debug(DEBUG_CONFIG, "setupVars.conf:%s -> Setting %s[%u] = %s\n", - key, conf_item->k, i, item->valuestring); + // Add string to our JSON array + cJSON *item = cJSON_CreateString(regex2); + cJSON_AddItemToArray(conf_item->v.json, item); - // Free memory - free(regex2); - } - else - { - // Add string to our JSON array - cJSON *item = cJSON_CreateString(setupVarsArray[i]); - cJSON_AddItemToArray(conf_item->v.json, item); + log_debug(DEBUG_CONFIG, "setupVars.conf:%s -> Setting %s[%u] = %s\n", + key, conf_item->k, i, item->valuestring); - log_debug(DEBUG_CONFIG, "setupVars.conf:%s -> Setting %s[%u] = %s\n", - key, conf_item->k, i, item->valuestring); - } + // Free memory + free(regex2); } } @@ -441,10 +428,10 @@ void importsetupVarsConf(void) get_conf_bool_from_setupVars("BLOCKING_ENABLED", &config.dns.blocking.active); // Get clients which the user doesn't want to see - get_conf_string_array_from_setupVars("API_EXCLUDE_CLIENTS", &config.webserver.api.excludeClients, true); + get_conf_string_array_from_setupVars_regex("API_EXCLUDE_CLIENTS", &config.webserver.api.excludeClients); // Get domains which the user doesn't want to see - get_conf_string_array_from_setupVars("API_EXCLUDE_DOMAINS", &config.webserver.api.excludeDomains, true); + get_conf_string_array_from_setupVars_regex("API_EXCLUDE_DOMAINS", &config.webserver.api.excludeDomains); // Try to obtain temperature hot value get_conf_temp_limit_from_setupVars(); @@ -640,15 +627,27 @@ void getSetupVarsArray(const char * input) /* split string and append tokens to 'res' */ while (p) { - setupVarsArray = realloc(setupVarsArray, sizeof(char*) * ++setupVarsElements); - if(setupVarsArray == NULL) return; + char **tmp = realloc(setupVarsArray, sizeof(char*) * ++setupVarsElements); + if(tmp == NULL) + { + free(setupVarsArray); + setupVarsArray = NULL; + return; + } + setupVarsArray = tmp; setupVarsArray[setupVarsElements-1] = p; p = strtok(NULL, ","); } /* realloc one extra element for the last NULL */ - setupVarsArray = realloc(setupVarsArray, sizeof(char*) * (setupVarsElements+1)); - if(setupVarsArray == NULL) return; + char **tmp = realloc(setupVarsArray, sizeof(char*) * (setupVarsElements+1)); + if(tmp == NULL) + { + free(setupVarsArray); + setupVarsArray = NULL; + return; + } + setupVarsArray = tmp; setupVarsArray[setupVarsElements] = NULL; } From 619a8b1cf44f6699f05ff4d08555f020b5fc303d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 20 Jan 2024 12:44:03 +0100 Subject: [PATCH 16/16] Fix pointer magic going wrong Signed-off-by: DL6ER --- src/api/queries.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api/queries.c b/src/api/queries.c index fb06cd627..2431d5663 100644 --- a/src/api/queries.c +++ b/src/api/queries.c @@ -1016,12 +1016,12 @@ bool compile_filter_regex(struct ftl_conn *api, const char *path, cJSON *json, r } // Compile regex - int rc = regcomp(regex[i], filter->valuestring, REG_EXTENDED); + int rc = regcomp(&(*regex)[i], filter->valuestring, REG_EXTENDED); if(rc != 0) { // Failed to compile regex char errbuf[1024] = { 0 }; - regerror(rc, regex[i], errbuf, sizeof(errbuf)); + regerror(rc, &(*regex)[i], errbuf, sizeof(errbuf)); log_err("Failed to compile regex \"%s\": %s", filter->valuestring, errbuf); return send_json_error(api, 400,