Merge simple domain format into FTLDNS beta branch

FTL.h

@@ -85,9 +85,9 @@ typedef struct {
 typedef struct {
 	const char* log;
 	const char* preEventHorizon;
-	const char* numBlocked;
 	const char* whitelist;
 	const char* blacklist;
+	const char* gravity;
 	const char* setupVars;
 	const char* wildcards;
 	const char* auditlist;
@@ -132,6 +132,7 @@ typedef struct {
 	int maxlogage;
 	int privacylevel;
 	bool ignore_localhost;
+	unsigned char blockingmode;
 } ConfigStruct;
 
 // Dynamic structs
@@ -203,15 +204,16 @@ typedef struct {
 } memoryStruct;
 
 // Prepare timers, used mainly for debugging purposes
-#define NUMTIMERS 3
-enum { DATABASE_WRITE_TIMER, EXIT_TIMER, GC_TIMER };
+#define NUMTIMERS 4
+enum { DATABASE_WRITE_TIMER, EXIT_TIMER, GC_TIMER, LISTS_TIMER };
 
 enum { QUERIES, FORWARDED, CLIENTS, DOMAINS, OVERTIME, WILDCARD };
 enum { DNSSEC_UNSPECIFIED, DNSSEC_SECURE, DNSSEC_INSECURE, DNSSEC_BOGUS, DNSSEC_ABANDONED, DNSSEC_UNKNOWN };
 enum { QUERY_UNKNOWN, QUERY_GRAVITY, QUERY_FORWARDED, QUERY_CACHE, QUERY_WILDCARD, QUERY_BLACKLIST };
 enum { TYPE_A = 1, TYPE_AAAA, TYPE_ANY, TYPE_SRV, TYPE_SOA, TYPE_PTR, TYPE_TXT, TYPE_MAX };
 enum { REPLY_UNKNOWN, REPLY_NODATA, REPLY_NXDOMAIN, REPLY_CNAME, REPLY_IP };
 enum { PRIVACY_SHOW_ALL = 0, PRIVACY_HIDE_DOMAINS, PRIVACY_HIDE_DOMAINS_CLIENTS, PRIVACY_MAXIMUM };
+enum { MODE_IP, MODE_NX };
 
 // Used to check memory integrity in various structs
 #define MAGICBYTE 0x57

README.md

@@ -168,6 +168,7 @@ Possible settings (**the option shown first is the default**):
 - `FTLPORT=4711` (On which port should FTL be listening?)
 - `PRIVACYLEVEL=0` (Which privacy level is used? Can be 0 (permissive) to 3 (very restrictive), see below)
 - `IGNORE_LOCALHOST=no|yes` (Should `FTL` ignore queries coming from the local machine?)
+- `BLOCKINGMODE=IP|NXDOMAIN` (Should `FTL` reply queries to blocked domains with IPs or NXDOMAIN?)
 
 ### Privacy levels
 Specifies if we want to anonymize the DNS queries somehow, available options are:

config.c

@@ -187,6 +187,27 @@ void read_FTLconf(void)
 	else
 		logg("   IGNORE_LOCALHOST: Show queries from localhost");
 
+	// BLOCKINGMODE
+	// defaults to: MODE_IP
+	config.blockingmode = MODE_IP;
+	buffer = parse_FTLconf(fp, "BLOCKINGMODE");
+
+	if(buffer != NULL)
+	{
+		if(strcasecmp(buffer, "NXDOMAIN") == 0)
+			config.blockingmode = MODE_NX;
+	}
+
+	switch(config.blockingmode)
+	{
+		case MODE_NX:
+			logg("   BLOCKINGMODE: NXDOMAIN for blocked domains");
+			break;
+		default:
+			logg("   BLOCKINGMODE: Pi-hole's IP for blocked domains");
+			break;
+	}
+
 	logg("Finished config file parsing");
 
 	// Release memory

dnsmasq/cache.c

@@ -75,7 +75,7 @@ static const struct {
 static void cache_free(struct crec *crecp);
 static void cache_unlink(struct crec *crecp);
 static void cache_link(struct crec *crecp);
-static void rehash(int size);
+void rehash(int size);
 static void cache_hash(struct crec *crecp);
 
 static unsigned int next_uid(void)
@@ -118,7 +118,7 @@ void cache_init(void)
    but if the hosts file(s) are big (some people have 50000 ad-block entries), the table
    will be much too small, so the hosts reading code calls rehash every 1000 addresses, to
    expand the table. */
-static void rehash(int size)
+void rehash(int size)
 {
   struct crec **new, **old, *p, *tmp;
   int i, new_size, old_size;
@@ -796,8 +796,8 @@ static void add_hosts_cname(struct crec *target)
       }
 }
 
-static void add_hosts_entry(struct crec *cache, struct all_addr *addr, int addrlen,
-			    unsigned int index, struct crec **rhash, int hashsz)
+void add_hosts_entry(struct crec *cache, struct all_addr *addr, int addrlen,
+		     unsigned int index, struct crec **rhash, int hashsz)
 {
   struct crec *lookup = cache_find_by_name(NULL, cache_get_name(cache), 0, cache->flags & (F_IPV4 | F_IPV6));
   int i, nameexists = 0;
@@ -933,6 +933,9 @@ int read_hostsfile(char *filename, unsigned int index, int cache_size, struct cr
 
   eatspace(f);
 
+  name_count = FTL_listsfile(filename, index, f, cache_size, rhash, hashsz);
+  addr_count = cache_size - name_count;
+
   while ((atnl = gettok(f, token)) != EOF)
     {
       lineno++;

dnsmasq/rfc1035.c

@@ -1695,7 +1695,7 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen,
 			      nxdomain = 1;
 			    if (!dryrun)
 			    {
-			      log_query(crecp->flags, name, NULL, NULL);
+			      log_query(crecp->flags, name, NULL, record_source(crecp->uid));
 			      FTL_cache(crecp->flags, name, NULL, NULL, daemon->log_display_id);
 			    }
 			  }

dnsmasq_interface.c

@@ -500,6 +500,8 @@ void FTL_cache(unsigned int flags, char *name, struct all_addr *addr, char *arg,
 				requesttype = QUERY_GRAVITY;
 			else if(arg != NULL && strstr(arg, "/black.list") != NULL)
 				requesttype = QUERY_BLACKLIST;
+			else if(flags & F_NXDOMAIN)
+				requesttype = QUERY_GRAVITY;
 			else // local.list, hostname.list, /etc/hosts and others
 				requesttype = QUERY_CACHE;
 		}
@@ -768,3 +770,142 @@ unsigned long converttimeval(struct timeval time)
 	// of 10*milliseconds
 	return time.tv_sec*10000 + time.tv_usec/100;
 }
+
+// Routine that handles simple lists format for both gravity.list and black.list
+void add_hosts_entry(struct crec *cache, struct all_addr *addr, int addrlen, unsigned int index, struct crec **rhash, int hashsz);
+void rehash(int size);
+int FTL_listsfile(char* filename, unsigned int index, FILE *f, int cache_size, struct crec **rhash, int hashsz)
+{
+	int name_count = cache_size;
+	int added = 0;
+	size_t size = 0;
+	char *buffer = NULL, *a = NULL;
+	struct all_addr addr4, addr6;
+	bool has_IPv4 = false, has_IPv6 = false;
+
+	// Handle only gravity.list and black.list
+	// Skip all other files (they are interpreted in the usual format)
+	if(strcmp(filename, files.gravity) != 0 &&
+	   strcmp(filename, files.blacklist) != 0)
+		return cache_size;
+
+	// Start timer for list analysis
+	timer_start(LISTS_TIMER);
+
+	// Read IPv4 address for host entries from setupVars.conf
+	char *IPv4addr = read_setupVarsconf("IPV4_ADDRESS");
+	if(IPv4addr != NULL)
+	{
+		// Strip off everything at the end of the IP (CIDR might be there)
+		a=IPv4addr; for(;*a;a++) if(*a == '/') *a = 0;
+		// Prepare IPv4 address for records
+		if(inet_pton(AF_INET, IPv4addr, &addr4) > 0)
+			has_IPv4 = true;
+	}
+	clearSetupVarsArray(); // will free/invalidate IPv4addr
+
+	// Read IPv6 address for host entries from setupVars.conf
+	char *IPv6addr = read_setupVarsconf("IPV6_ADDRESS");
+	if(IPv6addr != NULL)
+	{
+		// Strip off everything at the end of the IP (CIDR might be there)
+		a=IPv6addr; for(;*a;a++) if(*a == '/') *a = 0;
+		// Prepare IPv6 address for records
+		if(inet_pton(AF_INET6, IPv6addr, &addr6) > 1)
+			has_IPv6 = true;
+	}
+	clearSetupVarsArray(); // will free/invalidate IPv6addr
+
+	// If no IPv4 address was found but user wants us to server NXDOMAIN
+	// we have to mock an IP record (which won't do anything in the end)
+	if(!has_IPv4 && config.blockingmode == MODE_NX)
+	{
+		if(inet_pton(AF_INET, "127.0.0.1", &addr4) > 0)
+			has_IPv4 = true;
+	}
+
+	// If we have neither a valid IPv4 nor a valid IPv6, then we cannot add any entries here
+	if(!has_IPv4 && !has_IPv6)
+	{
+		logg("ERROR: found neither a valid IPV4_ADDRESS nor IPV6_ADDRESS in setupVars.conf");
+		return cache_size;
+	}
+
+	// Walk file line by line
+	bool firstline = true;
+	while(getline(&buffer, &size, f) != -1)
+	{
+		char *domain = buffer;
+		// Skip hashed out lines
+		while (*domain == '#')
+			continue;
+
+		// Filter leading dots or spaces
+		while (*domain == '.' || *domain == ' ') domain++;
+
+		// Check for spaces or tabs
+		// If found, then this list is still in HOSTS format and we
+		// don't analyze it here.
+		if(firstline &&
+		   (strstr(domain, " ") != NULL || strstr(domain, "\t") != NULL))
+		{
+			// Reset file pointer back to beginning of the list
+			rewind(f);
+			logg("File %s is in HOSTS format, please run pihole -g!", filename);
+			return name_count;
+		}
+		firstline = false;
+
+		// Skip empty lines
+		if(strlen(domain) == 0)
+			continue;
+
+		// Strip newline character at the end of line we just read
+		if(domain[strlen(domain)-1] == '\n')
+			domain[strlen(domain)-1] = '\0';
+
+		// As of here we assume the entry to be valid
+		// Rehash every 1000 valid names
+		if(rhash && ((name_count - cache_size) > 1000))
+		{
+			rehash(name_count);
+			cache_size = name_count;
+		}
+
+		struct crec *cache4,*cache6;
+		// Add IPv4 record
+		if(has_IPv4 &&
+		   (cache4 = malloc(sizeof(struct crec) + strlen(domain)+1-SMALLDNAME)))
+		{
+			strcpy(cache4->name.sname, domain);
+			cache4->flags = F_HOSTS | F_IMMORTAL | F_FORWARD | F_REVERSE | F_IPV4;
+			if(config.blockingmode == MODE_NX) cache4->flags |= F_NEG | F_NXDOMAIN;
+			cache4->ttd = daemon->local_ttl;
+			add_hosts_entry(cache4, &addr4, INADDRSZ, index, rhash, hashsz);
+			name_count++;
+		}
+		// Add IPv6 record only if we respond with an IP address to blocked domains
+		if(has_IPv6 && config.blockingmode == MODE_IP &&
+		   (cache6 = malloc(sizeof(struct crec) + strlen(domain)+1-SMALLDNAME)))
+		{
+			strcpy(cache6->name.sname, domain);
+			cache6->flags = F_HOSTS | F_IMMORTAL | F_FORWARD | F_REVERSE | F_IPV6;
+			cache6->ttd = daemon->local_ttl;
+			add_hosts_entry(cache6, &addr6, IN6ADDRSZ, index, rhash, hashsz);
+			name_count++;
+		}
+		// Count added domain
+		added++;
+	}
+
+	// Free allocated memory
+	if(buffer != NULL)
+	{
+		free(buffer);
+		buffer = NULL;
+	}
+
+	logg("%s: parsed %i domains (took %.1f ms)", filename, added, timer_elapsed_msec(LISTS_TIMER));
+	counters.gravity += added;
+	return name_count;
+}

dnsmasq_interface.h

@@ -18,3 +18,4 @@ void FTL_dnsmasq_reload(void);
 void FTL_fork_and_bind_sockets(void);
 
 void FTL_forwarding_failed(struct server *server);
+int FTL_listsfile(char* filename, unsigned int index, FILE *f, int cache_size, struct crec **rhash, int hashsz);

grep.c

@@ -57,24 +57,7 @@ int readnumberfromfile(const char* fname)
 
 void readGravityFiles(void)
 {
-	// Get number of domains being blocked
-	int gravity = readnumberfromfile(files.numBlocked);
-
-	if(gravity < 0)
-	{
-		logg("WARN: failed to read %s", files.numBlocked);
-		// Fallback method is counting number of lines in preEventHorizon
-		gravity = countlines(files.preEventHorizon);
-		if(gravity < 0)
-		{
-			logg("Error: failed to read %s", files.preEventHorizon);
-			gravity = 0;
-		}
-	}
-
-	logg("Gravity list entries: %i", gravity);
-	counters.gravity = gravity;
-
+	// Get number of domains being blocked by wildcards
 	readWildcardsList();
 	if(counters.wildcarddomains > 0)
 	{

memory.c

@@ -22,9 +22,9 @@ FTLFileNamesStruct FTLfiles = {
 logFileNamesStruct files = {
 	"/var/log/pihole.log",
 	"/etc/pihole/list.preEventHorizon",
-	"/etc/pihole/numBlocked",
 	"/etc/pihole/whitelist.txt",
-	"/etc/pihole/blacklist.txt",
+	"/etc/pihole/black.list",
+	"/etc/pihole/gravity.list",
 	"/etc/pihole/setupVars.conf",
 	"/etc/dnsmasq.d/03-pihole-wildcard.conf",
 	"/etc/pihole/auditlog.list",