Add cache redis support

README.md

@@ -27,7 +27,7 @@ first step of surfing the Internet.
     + Custom IP network
 + Minimum TTL modification
 + Hosts (Both IPv4 and IPv6 are supported and IPs will be returned in random order. If you want to use regex match, please understand regex first)
-+ Cache with ECS
++ Cache with ECS and Redis(Persistence) support
 + DNS over HTTP server support
 
 ### Dispatch process
@@ -114,6 +114,8 @@ hostsFile:
 minimumTTL: 0
 domainTTLFile: ./domain_ttl_sample
 cacheSize: 0
+cacheRedisUrl: redis://localhost:6379/0
+cacheRedisConnectionPoolSize: 10 
 rejectQType:
   - 255
 ```
@@ -122,7 +124,7 @@ Tips:
 
 + bindAddress: Specifying only port (e.g. `:53`) will let overture listen on all available addresses (both IPv4 and
 IPv6). Overture will handle both TCP and UDP requests. Literal IPv6 addresses are enclosed in square brackets (e.g. `[2001:4860:4860::8888]:53`)
-+ debugHTTPAddress: Specifying an HTTP port for debugging (**`5555` is the default port but it is also acknowledged as the android wifi adb listener port**), currently used to dump DNS cache, and the request url is `/cache`, available query argument is `nobody`(boolean)
++ debugHTTPAddress: Specifying an HTTP port for debugging (**`5555` is the default port despite it is also acknowledged as the android Wi-Fi adb listener port**), currently used to dump DNS cache, and the request url is `/cache`, available query argument is `nobody`(boolean)
 
     * true(default): only get the cache size;
 
@@ -172,7 +174,7 @@ IPv6). Overture will handle both TCP and UDP requests. Literal IPv6 addresses ar
           }
         }
         ```
-+ dohEnabled(Experimental): Enable DNS over HTTP server using `DebugHTTPAddress` above with url path `/dns-query`. DNS over HTTPS server can be easily achieved helping by another web server software like caddy or nginx.
++ dohEnabled: Enable DNS over HTTP server using `DebugHTTPAddress` above with url path `/dns-query`. DNS over HTTPS server can be easily achieved helping by another web server software like caddy or nginx. (Experimental)
 + primaryDNS/alternativeDNS:
     + name: This field is only used for logging.
     + address: Same rule as BindAddress.
@@ -193,13 +195,14 @@ IPv6). Overture will handle both TCP and UDP requests. Literal IPv6 addresses ar
 + ipv6UseAlternativeDNS: For to redirect IPv6 DNS queries to alternative DNS servers.
 + alternativeDNSConcurrent: Query the primaryDNS and alternativeDNS at the same time.
 + whenPrimaryDNSAnswerNoneUse: If the response of PrimaryDNS exists and there is no `ANSWER SECTION` in it, the final chosen DNS upstream should be defined here. (There is no `AAAA` record for most domains right now) 
-+ *File: Absolute path like `/path/to/file` is allowed. For Windows users, please use properly escaped path like
++ *File: Both relative like `./file` or absolute path like `/path/to/file` are supported. Especially, for Windows users, please use properly escaped path like
   `C:\\path\\to\\file.txt` in the configuration.
 + domainFile.Matcher: Matching policy and implementation, including "full-list", "full-map", "regex-list", "mix-list", "suffix-tree" and "final". Default value is "full-map".
 + hostsFile.Finder: Finder policy and implementation, including "full-map", "regex-list". Default value is "full-map".
 + domainTTLFile: Regex match only for now;
 + minimumTTL: Set the minimum TTL value (in seconds) in order to improve caching efficiency, use `0` to disable.
 + cacheSize: The number of query record to cache, use `0` to disable.
++ cacheRedisUrl, cacheRedisConnectionPoolSize: Use redis cache instead of local cache. (Experimental)
 + rejectQType: Reject query with specific DNS record types, check [List of DNS record types](https://en.wikipedia.org/wiki/List_of_DNS_record_types) for details.
 
 #### Domain file example (full match)
@@ -289,13 +292,10 @@ www.qq.com.     43  IN  A   14.17.42.40
 
 ## Acknowledgements
 
-+ Dependencies:
-    + [dns](https://github.com/miekg/dns): BSD-3-Clause
-    + [logrus](https://github.com/Sirupsen/logrus): MIT
-+ Code reference:
-    + [skydns](https://github.com/skynetservices/skydns): MIT
-    + [go-dnsmasq](https://github.com/janeczku/go-dnsmasq):  MIT
-+ Contributors: https://github.com/shawn1m/overture/graphs/contributors
++ [dns](https://github.com/miekg/dns): BSD-3-Clause
++ [skydns](https://github.com/skynetservices/skydns): MIT
++ [go-dnsmasq](https://github.com/janeczku/go-dnsmasq):  MIT
++ [All Contributors](https://github.com/shawn1m/overture/graphs/contributors)
 
 ## License
 

config.sample.yml

@@ -38,5 +38,7 @@ hostsFile:
 minimumTTL: 0
 domainTTLFile: ./domain_ttl_sample
 cacheSize: 0
+cacheRedisUrl: redis://localhost:6379/0
+cacheRedisConnectionPoolSize: 10
 rejectQType:
   - 255

config.test.yml

@@ -38,5 +38,7 @@ hostsFile:
 minimumTTL: 86400
 domainTTLFile: ./domain_ttl_sample
 cacheSize: 10000
+cacheRedisUrl: redis://localhost:6379/0
+cacheRedisConnectionPoolSize: 10
 rejectQType:
   - 255

core/cache/cache.go

@@ -8,10 +8,13 @@ package cache
 // Cache that holds RRs.
 
 import (
+	"context"
+	"encoding/json"
 	"fmt"
 	"sync"
 	"time"
 
+	"github.com/go-redis/redis/v8"
 	"github.com/miekg/dns"
 	log "github.com/sirupsen/logrus"
 )
@@ -23,29 +26,71 @@ type elem struct {
 	msg        *dns.Msg
 }
 
+type elemData struct {
+	Expiration time.Time
+	Msg        []byte // dns.Msg cannot be converted to the json format successfully thus using its pack() method instead
+}
+
+func (e *elem) MarshalBinary() (data []byte, err error) {
+	msgBytes, _ := e.msg.Pack()
+	ed := elemData{e.expiration, msgBytes}
+	return json.Marshal(ed)
+}
+
+func (e *elem) UnmarshalBinary(data []byte) error {
+	var ed elemData
+	err := json.Unmarshal(data, &ed)
+	if err != nil {
+		return err
+	}
+	e.expiration = ed.Expiration
+	e.msg = &dns.Msg{}
+	return e.msg.Unpack(ed.Msg)
+}
+
 // Cache is a cache that holds on the a number of RRs or DNS messages. The cache
 // eviction is randomized.
 type Cache struct {
 	sync.RWMutex
 
-	capacity int
-	table    map[string]*elem
+	capacity    int
+	table       map[string]*elem
+	redisClient *redis.Client
 }
 
 // New returns a new cache with the capacity and the ttl specified.
-func New(capacity int) *Cache {
+func New(capacity int, redisUrl string, cacheRedisConnectionPoolSize int) *Cache {
 	if capacity <= 0 {
 		return nil
 	}
 	c := new(Cache)
 	c.table = make(map[string]*elem)
 	c.capacity = capacity
+
+	opt, err := redis.ParseURL(redisUrl)
+	if err != nil {
+		if redisUrl != "" {
+			log.Error("redisUrl error ", redisUrl, err)
+		}
+	} else {
+		if cacheRedisConnectionPoolSize > 0 {
+			opt.PoolSize = cacheRedisConnectionPoolSize
+		} else {
+			log.Warn("cacheRedisConnectionPoolSize is ignored", cacheRedisConnectionPoolSize)
+		}
+		c.redisClient = redis.NewClient(opt)
+		log.Info("Cache redis connected! ", c.redisClient.String())
+	}
+
 	return c
 }
 
 func (c *Cache) Capacity() int { return c.capacity }
 
 func (c *Cache) Remove(s string) {
+	if c.redisClient != nil {
+		return
+	}
 	c.Lock()
 	delete(c.table, s)
 	c.Unlock()
@@ -74,30 +119,86 @@ func (c *Cache) InsertMessage(s string, m *dns.Msg, mTTL uint32) {
 	if c.capacity <= 0 || m == nil {
 		return
 	}
+	var err error
+	if c.redisClient == nil {
+		c.InsertMessageToLocal(s, m, mTTL)
+	} else {
+		err = c.InsertMessageToRedis(s, m, mTTL)
+	}
+	if err!=nil{
+		log.Warnf("Insert cache failed: %s", s, err)
+	}else {
+		log.Debugf("Cached: %s", s)
+	}
+}
+
+func (c *Cache) InsertMessageToRedis(s string, m *dns.Msg, mTTL uint32) error{
+
+	ttlDuration := convertToTTLDuration(m, mTTL)
+	if _, ok := c.table[s]; !ok {
+		e := &elem{time.Now().Add(ttlDuration), m.Copy()}
+		cmd := c.redisClient.Set(context.TODO(), s, e, ttlDuration)
+		if cmd.Err() != nil {
+			log.Warn("Redis set for cache failed!", cmd.Err())
+			return cmd.Err()
+		}
+	}
+	return nil
+
+}
+func (c *Cache) InsertMessageToLocal(s string, m *dns.Msg, mTTL uint32) {
 
 	c.Lock()
+	ttlDuration := convertToTTLDuration(m, mTTL)
+	if _, ok := c.table[s]; !ok {
+		e := &elem{time.Now().Add(ttlDuration), m.Copy()}
+		c.table[s] = e
+	}
+
+	c.EvictRandom()
+	c.Unlock()
+}
+
+func convertToTTLDuration(m *dns.Msg, mTTL uint32) time.Duration {
 	var ttl uint32
 	if len(m.Answer) == 0 {
 		ttl = mTTL
 	} else {
 		ttl = m.Answer[0].Header().Ttl
 	}
-	ttlDuration := time.Duration(ttl) * time.Second
-	if _, ok := c.table[s]; !ok {
-		c.table[s] = &elem{time.Now().Add(ttlDuration), m.Copy()}
-	}
-	log.Debugf("Cached: %s", s)
-	c.EvictRandom()
-	c.Unlock()
+	return time.Duration(ttl) * time.Second
 }
 
 // Search returns a dns.Msg, the expiration time and a boolean indicating if we found something
 // in the cache.
-// todo: use finder implementation
 func (c *Cache) Search(s string) (*dns.Msg, time.Time, bool) {
 	if c.capacity <= 0 {
 		return nil, time.Time{}, false
 	}
+	if c.redisClient == nil {
+		return c.SearchFromLocal(s)
+	} else {
+		return c.SearchFromRedis(s)
+	}
+}
+
+func (c *Cache) SearchFromRedis(s string) (*dns.Msg, time.Time, bool) {
+	var e elem
+	err := c.redisClient.Get(context.TODO(), s).Scan(&e)
+	if err != nil {
+		if err.Error() == "redis: nil" {
+			log.Debug("Redis get return nil for ", s, err)
+		} else {
+			log.Warn("Redis get return nil for ", s, err)
+		}
+		return nil, time.Time{}, false
+	}
+	return e.msg, e.expiration, true
+
+}
+
+// todo: use finder implementation
+func (c *Cache) SearchFromLocal(s string) (*dns.Msg, time.Time, bool) {
 	c.RLock()
 	if e, ok := c.table[s]; ok {
 		e1 := e.msg.Copy()
@@ -113,8 +214,8 @@ func Key(q dns.Question, ednsIP string) string {
 	return fmt.Sprintf("%s %d %s", q.Name, q.Qtype, ednsIP)
 }
 
-// Hit returns a dns message from the cache. If the message's TTL is expired nil
-// is returned and the message is removed from the cache.
+// Hit returns a dns message from the cache. If the message's TTL is expired, nil
+// will be returned and the message is removed from the cache.
 func (c *Cache) Hit(key string, msgid uint16) *dns.Msg {
 	m, exp, hit := c.Search(key)
 	if hit {
@@ -135,7 +236,7 @@ func (c *Cache) Hit(key string, msgid uint16) *dns.Msg {
 	return nil
 }
 
-// Dump returns all dns cache information, for dubugging
+// Dump returns all local dns cache information for debugging
 func (c *Cache) Dump(nobody bool) (rs map[string][]string, l int) {
 	if c.capacity <= 0 {
 		return

core/config/config.go

@@ -56,10 +56,12 @@ type Config struct {
 		HostsFile string `yaml:"hostsFile"`
 		Finder    string `yaml:"finder"`
 	} `yaml:"hostsFile"`
-	MinimumTTL    int      `yaml:"minimumTTL"`
-	DomainTTLFile string   `yaml:"domainTTLFile"`
-	CacheSize     int      `yaml:"cacheSize"`
-	RejectQType   []uint16 `yaml:"rejectQType"`
+	MinimumTTL                   int      `yaml:"minimumTTL"`
+	DomainTTLFile                string   `yaml:"domainTTLFile"`
+	CacheSize                    int      `yaml:"cacheSize"`
+	CacheRedisUrl                string   `yaml:"cacheRedisUrl"`
+	CacheRedisConnectionPoolSize int      `yaml:"cacheRedisConnectionPoolSize"`
+	RejectQType                  []uint16 `yaml:"rejectQType"`
 
 	DomainTTLMap            map[string]uint32
 	DomainPrimaryList       matcher.Matcher
@@ -89,7 +91,7 @@ func NewConfig(configFile string) *Config {
 		log.Info("Minimum TTL is disabled")
 	}
 
-	config.Cache = cache.New(config.CacheSize)
+	config.Cache = cache.New(config.CacheSize, config.CacheRedisUrl, config.CacheRedisConnectionPoolSize)
 	if config.CacheSize > 0 {
 		log.Infof("CacheSize is %d", config.CacheSize)
 	} else {

go.mod

@@ -4,10 +4,10 @@ go 1.12
 
 require (
 	github.com/coredns/coredns v1.8.0
+	github.com/go-redis/redis/v8 v8.4.4
 	github.com/miekg/dns v1.1.34
 	github.com/silenceper/pool v0.0.0-20191105065223-1f4530b6ba17
 	github.com/sirupsen/logrus v1.6.0
-	golang.org/x/net v0.0.0-20200707034311-ab3426394381
-	golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 // indirect
+	golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb
 	gopkg.in/yaml.v2 v2.3.0
 )