Use database table for domain auditlog

src/api/api.c

@@ -20,6 +20,8 @@
 #include "config.h"
 #include "database/common.h"
 #include "database/query-table.h"
+// in_auditlist()
+#include "database/gravity-db.h"
 #include "overTime.h"
 #include "api.h"
 #include "version.h"
@@ -290,9 +292,13 @@ void getTopDomains(const char *client_message, const int *sock)
 		if(excludedomains != NULL && insetupVarsArray(getstr(domain->domainpos)))
 			continue;
 
-		// Skip this domain if already included in audit
-		if(audit && countlineswith(getstr(domain->domainpos), FTLfiles.auditlist) > 0)
+		// Skip this domain if already audited
+		if(audit && in_auditlist(getstr(domain->domainpos)) > 0)
+		{
+			if(config.debug & DEBUG_API)
+				logg("API: %s has been audited.", getstr(domain->domainpos));
 			continue;
+		}
 
 		// Hidden domain, probably due to privacy level. Skip this in the top lists
 		if(strcmp(getstr(domain->domainpos), HIDDEN_DOMAIN) == 0)

src/config.c

@@ -313,9 +313,6 @@ void read_FTLconf(void)
 	// SETUPVARSFILE
 	getpath(fp, "SETUPVARSFILE", "/etc/pihole/setupVars.conf", &FTLfiles.setupVars);
 
-	// AUDITLISTFILE
-	getpath(fp, "AUDITLISTFILE", "/etc/pihole/auditlog.list", &FTLfiles.auditlist);
-
 	// MACVENDORDB
 	getpath(fp, "MACVENDORDB", "/etc/pihole/macvendor.db", &FTLfiles.macvendor_db);
 

src/database/gravity-db.c

@@ -18,8 +18,9 @@
 
 // Private variables
 static sqlite3 *gravity_db = NULL;
-static sqlite3_stmt* stmt = NULL;
+static sqlite3_stmt* table_stmt = NULL;
 static sqlite3_stmt* whitelist_stmt = NULL;
+static sqlite3_stmt* auditlist_stmt = NULL;
 bool gravity_database_avail = false;
 
 // Prototypes from functions in dnsmasq's source
@@ -37,7 +38,7 @@ bool gravityDB_open(void)
 	}
 
 	int rc = sqlite3_open_v2(FTLfiles.gravity_db, &gravity_db, SQLITE_OPEN_READONLY, NULL);
-	if( rc )
+	if( rc != SQLITE_OK )
 	{
 		logg("gravityDB_open() - SQL error (%i): %s", rc, sqlite3_errmsg(gravity_db));
 		gravityDB_close();
@@ -63,9 +64,18 @@ bool gravityDB_open(void)
 	// returns true as soon as it sees the first row from the query inside
 	// of EXISTS().
 	rc = sqlite3_prepare_v2(gravity_db, "SELECT EXISTS(SELECT domain from vw_whitelist WHERE domain = ?);", -1, &whitelist_stmt, NULL);
-	if( rc )
+	if( rc != SQLITE_OK )
 	{
-		logg("gravityDB_open(\"SELECT EXISTS(...)\") - SQL error prepare (%i): %s", rc, sqlite3_errmsg(gravity_db));
+		logg("gravityDB_open(\"SELECT EXISTS(... vw_whitelist ...)\") - SQL error prepare (%i): %s", rc, sqlite3_errmsg(gravity_db));
+		gravityDB_close();
+		return false;
+	}
+
+	// Prepare audit statement
+	rc = sqlite3_prepare_v2(gravity_db, "SELECT EXISTS(SELECT domain from domain_audit WHERE domain = ?);", -1, &auditlist_stmt, NULL);
+	if( rc != SQLITE_OK )
+	{
+		logg("gravityDB_open(\"SELECT EXISTS(... domain_audit ...)\") - SQL error prepare (%i): %s", rc, sqlite3_errmsg(gravity_db));
 		gravityDB_close();
 		return false;
 	}
@@ -87,6 +97,9 @@ void gravityDB_close(void)
 	// Finalize whitelist scanning statement
 	sqlite3_finalize(whitelist_stmt);
 
+	// Finalize audit scanning statement
+	sqlite3_finalize(auditlist_stmt);
+
 	// Close table
 	sqlite3_close(gravity_db);
 	gravity_database_avail = false;
@@ -122,7 +135,7 @@ bool gravityDB_getTable(const unsigned char list)
 	}
 
 	// Prepare SQLite3 statement
-	int rc = sqlite3_prepare_v2(gravity_db, querystr, -1, &stmt, NULL);
+	int rc = sqlite3_prepare_v2(gravity_db, querystr, -1, &table_stmt, NULL);
 	if( rc )
 	{
 		logg("readGravity(%s) - SQL error prepare (%i): %s", querystr, rc, sqlite3_errmsg(gravity_db));
@@ -144,12 +157,12 @@ bool gravityDB_getTable(const unsigned char list)
 inline const char* gravityDB_getDomain(void)
 {
 	// Perform step
-	const int rc = sqlite3_step(stmt);
+	const int rc = sqlite3_step(table_stmt);
 
 	// Valid row
 	if(rc == SQLITE_ROW)
 	{
-		const char* domain = (char*)sqlite3_column_text(stmt, 0);
+		const char* domain = (char*)sqlite3_column_text(table_stmt, 0);
 		return domain;
 	}
 
@@ -174,7 +187,7 @@ void gravityDB_finalizeTable(void)
 		return;
 
 	// Finalize statement
-	sqlite3_finalize(stmt);
+	sqlite3_finalize(table_stmt);
 }
 
 // Get number of domains in a specified table of the gravity database
@@ -210,87 +223,97 @@ int gravityDB_count(const unsigned char list)
 	}
 
 	// Prepare query
-	int rc = sqlite3_prepare_v2(gravity_db, querystr, -1, &stmt, NULL);
+	int rc = sqlite3_prepare_v2(gravity_db, querystr, -1, &table_stmt, NULL);
 	if( rc ){
 		logg("gravityDB_count(%s) - SQL error prepare (%i): %s", querystr, rc, sqlite3_errmsg(gravity_db));
-		sqlite3_finalize(stmt);
+		sqlite3_finalize(table_stmt);
 		gravityDB_close();
 		return DB_FAILED;
 	}
 
 	// Perform query
-	rc = sqlite3_step(stmt);
+	rc = sqlite3_step(table_stmt);
 	if( rc != SQLITE_ROW ){
 		logg("gravityDB_count(%s) - SQL error step (%i): %s", querystr, rc, sqlite3_errmsg(gravity_db));
-		sqlite3_finalize(stmt);
+		sqlite3_finalize(table_stmt);
 		gravityDB_close();
 		return DB_FAILED;
 	}
 
 	// Get result when there was no error
-	const int result = sqlite3_column_int(stmt, 0);
+	const int result = sqlite3_column_int(table_stmt, 0);
 
 	// Finalize statement
 	gravityDB_finalizeTable();
 
 	return result;
 }
 
-bool in_whitelist(const char *domain)
+static bool domain_in_list(const char *domain, sqlite3_stmt* stmt)
 {
 	int retval;
 	// Bind domain to prepared statement
 	// SQLITE_STATIC: Use the string without first duplicating it internally.
 	// We can do this as domain has dynamic scope that exceeds that of the binding.
-	if((retval = sqlite3_bind_text(whitelist_stmt, 1, domain, -1, SQLITE_STATIC)) != SQLITE_OK)
+	if((retval = sqlite3_bind_text(stmt, 1, domain, -1, SQLITE_STATIC)) != SQLITE_OK)
 	{
-		logg("in_whitelist(\"%s\"): Failed to bind domain (error %d) - %s",
+		logg("domain_in_list(\"%s\"): Failed to bind domain (error %d) - %s",
 		     domain, retval, sqlite3_errmsg(gravity_db));
-		sqlite3_reset(whitelist_stmt);
+		sqlite3_reset(stmt);
 		return false;
 	}
 
 	// Perform step
-	retval = sqlite3_step(whitelist_stmt);
+	retval = sqlite3_step(stmt);
 	if(retval == SQLITE_BUSY)
 	{
 		// Database is busy
-		logg("in_whitelist(\"%s\"): Database is busy, assuming domain is NOT whitelisted",
+		logg("domain_in_list(\"%s\"): Database is busy, assuming domain is NOT on list",
 		     domain);
-		sqlite3_reset(whitelist_stmt);
-		sqlite3_clear_bindings(whitelist_stmt);
+		sqlite3_reset(stmt);
+		sqlite3_clear_bindings(stmt);
 		return false;
 	}
 	else if(retval != SQLITE_ROW)
 	{
 		// Any return code that is neither SQLITE_BUSY not SQLITE_ROW
 		// is a real error we should log
-		logg("in_whitelist(\"%s\"): Failed to perform step (error %d) - %s",
+		logg("domain_in_list(\"%s\"): Failed to perform step (error %d) - %s",
 		     domain, retval, sqlite3_errmsg(gravity_db));
-		sqlite3_reset(whitelist_stmt);
-		sqlite3_clear_bindings(whitelist_stmt);
+		sqlite3_reset(stmt);
+		sqlite3_clear_bindings(stmt);
 		return false;
 	}
 
 	// Get result of query "SELECT EXISTS(...)"
-	const int result = sqlite3_column_int(whitelist_stmt, 0);
+	const int result = sqlite3_column_int(stmt, 0);
 
 	if(config.debug & DEBUG_DATABASE)
-		logg("in_whitelist(\"%s\"): %d", domain, result);
+		logg("domain_in_list(\"%s\"): %d", domain, result);
 
 	// The sqlite3_reset() function is called to reset a prepared
 	// statement object back to its initial state, ready to be
 	// re-executed. Note: Any SQL statement variables that had values
 	// bound to them using the sqlite3_bind_*() API retain their values.
-	sqlite3_reset(whitelist_stmt);
+	sqlite3_reset(stmt);
 
 	// Contrary to the intuition of many, sqlite3_reset() does not reset
 	// the bindings on a prepared statement. Use this routine to reset
 	// all host parameters to NULL.
-	sqlite3_clear_bindings(whitelist_stmt);
+	sqlite3_clear_bindings(stmt);
 
 	// Return result.
 	// SELECT EXISTS(...) either returns 0 (false) or 1 (true).
 	return result == 1;
 }
 
+bool in_whitelist(const char *domain)
+{
+	return domain_in_list(domain, whitelist_stmt);
+}
+
+bool in_auditlist(const char *domain)
+{
+	return domain_in_list(domain, auditlist_stmt);
+}
+

src/database/gravity-db.h

@@ -17,5 +17,6 @@ const char* gravityDB_getDomain(void);
 void gravityDB_finalizeTable(void);
 int gravityDB_count(unsigned char list);
 bool in_whitelist(const char *domain);
+bool in_auditlist(const char *domain);
 
 #endif //GRAVITY_H

src/files.c

@@ -15,98 +15,6 @@
 #include "setupVars.h"
 #include "log.h"
 
-char ** wildcarddomains = NULL;
-
-int countlines(const char* fname)
-{
-	FILE *fp;
-	int ch = 0, lines = 0, chars = 0;
-
-	if((fp = fopen(fname, "r")) == NULL) {
-		return -1;
-	}
-
-	while ((ch = fgetc(fp)) != EOF)
-	{
-		chars++;
-		if (ch=='\n')
-		{
-			// Add one to the lines counter
-			++lines;
-			// Reset chars counter
-			chars = 0;
-		}
-	}
-
-	// Add one more line if there were characters at the
-	// last line of the file even without a final "\n"
-	if(chars > 0)
-		++lines;
-
-	// Close the file
-	fclose(fp);
-
-	return lines;
-}
-
-int countlineswith(const char* str, const char* fname)
-{
-	FILE *fp;
-	int found = 0;
-	char *buffer = NULL;
-	size_t size = 0;
-
-	if((fp = fopen(fname, "r")) == NULL) {
-		return -1;
-	}
-
-	// Search through file
-	// getline reads a string from the specified file up to either a
-	// newline character or EOF
-	while(getline(&buffer, &size, fp) != -1)
-	{
-		// Strip potential newline character at the end of line we just read
-		if(buffer[strlen(buffer)-1] == '\n')
-			buffer[strlen(buffer)-1] = '\0';
-
-		// Search for exact match
-		if(strcmp(buffer, str) == 0)
-		{
-			found++;
-			continue;
-		}
-
-		// If line starts with *, search for partial match of
-		// needle "buffer+1" in haystack "str"
-		if(buffer[0] == '*')
-		{
-			char * buf = strstr(str, buffer+1);
-			// The  strstr() function finds the first occurrence of
-			// the substring buffer+1 in the string str.
-			// These functions return a pointer to the beginning of
-			// the located substring, or NULL if the substring is not
-			// found. Hence, we compare the length of the substring to
-			// the wildcard entry to rule out the possiblity that
-			// there is anything behind the wildcard. This avoids that given
-			// "*example.com" "example.com.xxxxx" would also match.
-			if(buf != NULL && strlen(buf) == strlen(buffer+1))
-				found++;
-		}
-	}
-
-	// Free allocated memory
-	if(buffer != NULL)
-	{
-		free(buffer);
-		buffer = NULL;
-	}
-
-	// Close the file
-	fclose(fp);
-
-	return found;
-}
-
 // chmod_file() changes the file mode bits of a given file (relative
 // to the directory file descriptor) according to mode. mode is an
 // octal number representing the bit pattern for the new mode bits

src/files.h

@@ -10,8 +10,6 @@
 #ifndef FILE_H
 #define FILE_H
 
-int countlines(const char* fname);
-int countlineswith(const char* str, const char* fname);
 bool chmod_file(const char *filename, const mode_t mode);
 bool file_exists(const char *filename);
 long int get_FTL_db_filesize(void);

test/gravity.db.sql

@@ -81,13 +81,21 @@ CREATE TABLE gravity
 (
 	domain TEXT PRIMARY KEY
 );
+
+CREATE TABLE domain_audit
+(
+	id INTEGER PRIMARY KEY AUTOINCREMENT,
+	domain TEXT UNIQUE NOT NULL,
+	date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int))
+);
+
 CREATE TABLE info
 (
 	property TEXT PRIMARY KEY,
 	value TEXT NOT NULL
 );
 
-INSERT INTO info VALUES("version","1");
+INSERT INTO info VALUES("version","2");
 
 CREATE VIEW vw_gravity AS SELECT domain
     FROM gravity
@@ -151,4 +159,6 @@ INSERT INTO "group" VALUES(1,0,'Test group','A disabled test group');
 INSERT INTO blacklist VALUES(2,'blacklisted-group-disabled.com',1,1559928803,1559928803,'Entry disabled by a group');
 INSERT INTO blacklist_by_group VALUES(2,1);
 
+INSERT INTO domain_audit VALUES(1,'google.com',1559928803);
+
 COMMIT;

test/test_suite.bats

@@ -151,6 +151,12 @@
   [[ ${lines[4]} == "" ]]
 }
 
+@test "Domain auditing, approved domains are not shown" {
+  run bash -c 'echo ">top-domains for audit >quit" | nc -v 127.0.0.1 4711'
+  printf "%s\n" "${lines[@]}"
+  [[ ${lines[@]} != *"google.com"* ]]
+}
+
 @test "Forward Destinations" {
   run bash -c 'echo ">forward-dest >quit" | nc -v 127.0.0.1 4711'
   printf "%s\n" "${lines[@]}"