RF: block lists to check and test resolvers

checks/block_lists.go

@@ -0,0 +1,128 @@
+package checks
+
+import (
+	"context"
+	"net"
+	"slices"
+	"sort"
+	"sync"
+	"time"
+
+	"github.com/xray-web/web-check-api/checks/clients/ip"
+)
+
+type dnsServer struct {
+	Name string
+	IP   string
+}
+
+var DNS_SERVERS = []dnsServer{
+	{Name: "AdGuard", IP: "176.103.130.130"},
+	{Name: "AdGuard Family", IP: "176.103.130.132"},
+	{Name: "CleanBrowsing Adult", IP: "185.228.168.10"},
+	{Name: "CleanBrowsing Family", IP: "185.228.168.168"},
+	{Name: "CleanBrowsing Security", IP: "185.228.168.9"},
+	{Name: "CloudFlare", IP: "1.1.1.1"},
+	{Name: "CloudFlare Family", IP: "1.1.1.3"},
+	{Name: "Comodo Secure", IP: "8.26.56.26"},
+	{Name: "Google DNS", IP: "8.8.8.8"},
+	{Name: "Neustar Family", IP: "156.154.70.3"},
+	{Name: "Neustar Protection", IP: "156.154.70.2"},
+	{Name: "Norton Family", IP: "199.85.126.20"},
+	{Name: "OpenDNS", IP: "208.67.222.222"},
+	{Name: "OpenDNS Family", IP: "208.67.222.123"},
+	{Name: "Quad9", IP: "9.9.9.9"},
+	{Name: "Yandex Family", IP: "77.88.8.7"},
+	{Name: "Yandex Safe", IP: "77.88.8.88"},
+}
+
+var knownBlockIPs = []string{
+	"146.112.61.106",
+	"185.228.168.10",
+	"8.26.56.26",
+	"9.9.9.9",
+	"208.69.38.170",
+	"208.69.39.170",
+	"208.67.222.222",
+	"208.67.222.123",
+	"199.85.126.10",
+	"199.85.126.20",
+	"156.154.70.22",
+	"77.88.8.7",
+	"77.88.8.8",
+	"::1",
+	"2a02:6b8::feed:0ff",
+	"2a02:6b8::feed:bad",
+	"2a02:6b8::feed:a11",
+	"2620:119:35::35",
+	"2620:119:53::53",
+	"2606:4700:4700::1111",
+	"2606:4700:4700::1001",
+	"2001:4860:4860::8888",
+	"2a0d:2a00:1::",
+	"2a0d:2a00:2::",
+}
+
+type Blocklist struct {
+	Server    string `json:"server"`
+	ServerIP  string `json:"serverIp"`
+	IsBlocked bool   `json:"isBlocked"`
+}
+
+type BlockList struct {
+	lookup ip.DNSLookup
+}
+
+func NewBlockList(lookup ip.DNSLookup) *BlockList {
+	return &BlockList{lookup: lookup}
+}
+
+func (b *BlockList) domainBlocked(ctx context.Context, domain, serverIP string) bool {
+	ips, err := b.lookup.DNSLookupIP(ctx, "ip4", domain, serverIP)
+	if err != nil {
+		// if there's an error, consider it not blocked
+		// TODO: return more detailed errors for each server
+		return false
+	}
+
+	return slices.ContainsFunc(ips, func(ip net.IP) bool {
+		return slices.Contains(knownBlockIPs, ip.String())
+	})
+}
+
+func (b *BlockList) BlockedServers(ctx context.Context, domain string) []Blocklist {
+	var lock sync.Mutex
+	var wg sync.WaitGroup
+	limit := make(chan struct{}, 5)
+
+	var results []Blocklist
+
+	ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+	defer cancel()
+
+	for _, server := range DNS_SERVERS {
+		wg.Add(1)
+		go func(server dnsServer) {
+			limit <- struct{}{}
+			defer func() {
+				<-limit
+				wg.Done()
+			}()
+
+			isBlocked := b.domainBlocked(ctx, domain, server.IP)
+			lock.Lock()
+			defer lock.Unlock()
+			results = append(results, Blocklist{
+				Server:    server.Name,
+				ServerIP:  server.IP,
+				IsBlocked: isBlocked,
+			})
+		}(server)
+	}
+	wg.Wait()
+
+	sort.Slice(results, func(i, j int) bool {
+		return results[i].Server < results[j].Server
+	})
+	return results
+}

checks/block_lists_test.go

@@ -0,0 +1,24 @@
+package checks
+
+import (
+	"context"
+	"net"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/xray-web/web-check-api/checks/clients/ip"
+)
+
+func TestBlockList(t *testing.T) {
+	t.Parallel()
+
+	t.Run("blocked IP", func(t *testing.T) {
+		t.Parallel()
+
+		dnsLookup := ip.DNSLookupFunc(func(ctx context.Context, network, host, dns string) ([]net.IP, error) {
+			return []net.IP{net.ParseIP("146.112.61.106")}, nil
+		})
+		list := NewBlockList(dnsLookup).BlockedServers(context.Background(), "example.com")
+		assert.Contains(t, list, Blocklist{Server: "AdGuard", ServerIP: "176.103.130.130", IsBlocked: true})
+	})
+}

checks/checks.go

@@ -4,10 +4,12 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/xray-web/web-check-api/checks/clients/ip"
 	"github.com/xray-web/web-check-api/checks/store/legacyrank"
 )
 
 type Checks struct {
+	BlockList   *BlockList
 	Carbon      *Carbon
 	Headers     *Headers
 	IpAddress   *Ip
@@ -23,8 +25,9 @@ func NewChecks() *Checks {
 		Timeout: 5 * time.Second,
 	}
 	return &Checks{
+		BlockList:   NewBlockList(&ip.NetDNSLookup{}),
 		Carbon:      NewCarbon(client),
-		Headers:    NewHeaders(client),
+		Headers:     NewHeaders(client),
 		IpAddress:   NewIp(NewNetIp()),
 		LegacyRank:  NewLegacyRank(legacyrank.NewInMemoryStore()),
 		LinkedPages: NewLinkedPages(client),

checks/clients/ip/ip.go

@@ -0,0 +1,54 @@
+package ip
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"time"
+)
+
+type Lookup interface {
+	LookupIP(ctx context.Context, network string, host string) ([]net.IP, error)
+}
+
+type LookupFunc func(ctx context.Context, network string, host string) ([]net.IP, error)
+
+func (fn LookupFunc) LookupIP(ctx context.Context, network string, host string) ([]net.IP, error) {
+	return fn(ctx, network, host)
+}
+
+// NetLookup is a client for looking up IP addresses using a net.Resolver.
+type NetLookup struct{}
+
+func (l *NetLookup) LookupIP(ctx context.Context, network string, host string) ([]net.IP, error) {
+	netResolver := &net.Resolver{
+		PreferGo: true,
+	}
+	return netResolver.LookupIP(ctx, network, host)
+}
+
+type DNSLookup interface {
+	DNSLookupIP(ctx context.Context, network, host, dns string) ([]net.IP, error)
+}
+
+type DNSLookupFunc func(ctx context.Context, network, host, dns string) ([]net.IP, error)
+
+func (fn DNSLookupFunc) DNSLookupIP(ctx context.Context, network, host, dns string) ([]net.IP, error) {
+	return fn(ctx, network, host, dns)
+}
+
+// DNSLookup is a client for looking up IP addresses with a custom DNS server.
+type NetDNSLookup struct{}
+
+func (l *NetDNSLookup) DNSLookupIP(ctx context.Context, network, host, dns string) ([]net.IP, error) {
+	netResolver := &net.Resolver{
+		PreferGo: true,
+		Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
+			d := net.Dialer{
+				Timeout: 3 * time.Second,
+			}
+			return d.DialContext(ctx, network, fmt.Sprintf("%s:%d", dns, 53))
+		},
+	}
+	return netResolver.LookupIP(ctx, network, host)
+}

codecov.yml

@@ -0,0 +1,2 @@
+ignore:
+  - checks/clients/ip/ip.go  # this contains go std lib code wrapped for interfaces, not worth testing

handlers/block_lists.go

@@ -1,143 +1,19 @@
 package handlers
 
 import (
-	"context"
-	"encoding/json"
-	"net"
 	"net/http"
-	"slices"
-	"sort"
-	"sync"
-	"time"
-)
-
-type dnsServer struct {
-	Name string
-	IP   string
-}
-
-var DNS_SERVERS = []dnsServer{
-	{Name: "AdGuard", IP: "176.103.130.130"},
-	{Name: "AdGuard Family", IP: "176.103.130.132"},
-	{Name: "CleanBrowsing Adult", IP: "185.228.168.10"},
-	{Name: "CleanBrowsing Family", IP: "185.228.168.168"},
-	{Name: "CleanBrowsing Security", IP: "185.228.168.9"},
-	{Name: "CloudFlare", IP: "1.1.1.1"},
-	{Name: "CloudFlare Family", IP: "1.1.1.3"},
-	{Name: "Comodo Secure", IP: "8.26.56.26"},
-	{Name: "Google DNS", IP: "8.8.8.8"},
-	{Name: "Neustar Family", IP: "156.154.70.3"},
-	{Name: "Neustar Protection", IP: "156.154.70.2"},
-	{Name: "Norton Family", IP: "199.85.126.20"},
-	{Name: "OpenDNS", IP: "208.67.222.222"},
-	{Name: "OpenDNS Family", IP: "208.67.222.123"},
-	{Name: "Quad9", IP: "9.9.9.9"},
-	{Name: "Yandex Family", IP: "77.88.8.7"},
-	{Name: "Yandex Safe", IP: "77.88.8.88"},
-}
-
-var knownBlockIPs = []string{
-	"146.112.61.106",
-	"185.228.168.10",
-	"8.26.56.26",
-	"9.9.9.9",
-	"208.69.38.170",
-	"208.69.39.170",
-	"208.67.222.222",
-	"208.67.222.123",
-	"199.85.126.10",
-	"199.85.126.20",
-	"156.154.70.22",
-	"77.88.8.7",
-	"77.88.8.8",
-	"::1",
-	"2a02:6b8::feed:0ff",
-	"2a02:6b8::feed:bad",
-	"2a02:6b8::feed:a11",
-	"2620:119:35::35",
-	"2620:119:53::53",
-	"2606:4700:4700::1111",
-	"2606:4700:4700::1001",
-	"2001:4860:4860::8888",
-	"2a0d:2a00:1::",
-	"2a0d:2a00:2::",
-}
-
-type Blocklist struct {
-	Server    string `json:"server"`
-	ServerIP  string `json:"serverIp"`
-	IsBlocked bool   `json:"isBlocked"`
-}
-
-func isDomainBlocked(domain, serverIP string) bool {
-	resolver := &net.Resolver{
-		PreferGo: true,
-		Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
-			d := net.Dialer{
-				Timeout: time.Second * 3,
-			}
-			return d.DialContext(ctx, network, serverIP+":53")
-		},
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
-	defer cancel()
 
-	ips, err := resolver.LookupIP(ctx, "ip4", domain)
-	if err != nil {
-		// if there's an error, consider it not blocked
-		return false
-	}
-
-	return slices.ContainsFunc(ips, func(ip net.IP) bool {
-		return slices.Contains(knownBlockIPs, ip.String())
-	})
-}
-
-func checkDomainAgainstDNSServers(domain string) []Blocklist {
-	var lock sync.Mutex
-	var wg sync.WaitGroup
-	limit := make(chan struct{}, 5)
-
-	var results []Blocklist
-
-	for _, server := range DNS_SERVERS {
-		wg.Add(1)
-		go func(server dnsServer) {
-			limit <- struct{}{}
-			defer func() {
-				<-limit
-				wg.Done()
-			}()
-
-			isBlocked := isDomainBlocked(domain, server.IP)
-			lock.Lock()
-			defer lock.Unlock()
-			results = append(results, Blocklist{
-				Server:    server.Name,
-				ServerIP:  server.IP,
-				IsBlocked: isBlocked,
-			})
-		}(server)
-	}
-	wg.Wait()
-
-	sort.Slice(results, func(i, j int) bool {
-		return results[i].Server > results[j].Server
-	})
-	return results
-}
+	"github.com/xray-web/web-check-api/checks"
+)
 
-func HandleBlockLists() http.Handler {
-	type Response struct {
-		BlockLists []Blocklist `json:"blocklists"`
-	}
+func HandleBlockLists(b *checks.BlockList) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		rawURL, err := extractURL(r)
 		if err != nil {
 			JSONError(w, ErrMissingURLParameter, http.StatusBadRequest)
 			return
 		}
-		json.NewEncoder(w).Encode(Response{BlockLists: checkDomainAgainstDNSServers(rawURL.Hostname())})
+		list := b.BlockedServers(r.Context(), rawURL.Hostname())
+		JSON(w, list, http.StatusOK)
 	})
 }