Update filterlists to use new format

Also, add database reset flag
safing · Nov 6, 2020 · 563bff1 · 563bff1
1 parent d5c00e3
commit 563bff1
Show file tree

Hide file tree

Showing 3 changed files with 34 additions and 6 deletions.
diff --git a/intel/filterlists/cache_version.go b/intel/filterlists/cache_version.go
@@ -4,15 +4,20 @@ import (
 	"fmt"
 	"sync"
 
+	"github.com/safing/portbase/database"
+
 	"github.com/hashicorp/go-version"
 	"github.com/safing/portbase/database/record"
 )
 
+const resetVersion = "v0.6.0"
+
 type cacheVersionRecord struct {
 	record.Base
 	sync.Mutex
 
 	Version string
+	Reset   string
 }
 
 // getCacheDatabaseVersion reads and returns the cache
@@ -37,6 +42,10 @@ func getCacheDatabaseVersion() (*version.Version, error) {
 		}
 	}
 
+	if verRecord.Reset != resetVersion {
+		return nil, database.ErrNotFound
+	}
+
 	ver, err := version.NewSemver(verRecord.Version)
 	if err != nil {
 		return nil, err
@@ -50,6 +59,7 @@ func getCacheDatabaseVersion() (*version.Version, error) {
 func setCacheDatabaseVersion(ver string) error {
 	verRecord := &cacheVersionRecord{
 		Version: ver,
+		Reset:   resetVersion,
 	}
 
 	verRecord.SetKey(filterListCacheVersionKey)

diff --git a/intel/filterlists/database.go b/intel/filterlists/database.go
@@ -200,14 +200,14 @@ func processEntry(ctx context.Context, filter *scopedBloom, entry *listEntry, re
 	normalizeEntry(entry)
 
 	// Only add the entry to the bloom filter if it has any sources.
-	if len(entry.Sources) > 0 {
+	if len(entry.Resources) > 0 {
 		filter.add(entry.Type, entry.Entity)
 	}
 
 	r := &entityRecord{
 		Value:     entry.Entity,
 		Type:      entry.Type,
-		Sources:   entry.Sources,
+		Sources:   entry.getSources(),
 		UpdatedAt: time.Now().Unix(),
 	}
 

diff --git a/intel/filterlists/decoder.go b/intel/filterlists/decoder.go
@@ -8,13 +8,31 @@ import (
 	"io"
 
 	"github.com/safing/portbase/formats/dsd"
+	"github.com/safing/portbase/utils"
 )
 
 type listEntry struct {
-	Entity    string   `json:"entity"`
-	Sources   []string `json:"sources"`
-	Whitelist bool     `json:"whitelist"`
-	Type      string   `json:"type"`
+	Type      string          `json:"type"`
+	Entity    string          `json:"entity"`
+	Whitelist bool            `json:"whitelist"`
+	Resources []entryResource `json:"resources"`
+}
+
+type entryResource struct {
+	SourceID   string `json:"sourceID"`
+	ResourceID string `json:"resourceID"`
+}
+
+func (entry *listEntry) getSources() (sourceIDs []string) {
+	sourceIDs = make([]string, 0, len(entry.Resources))
+
+	for _, resource := range entry.Resources {
+		if !utils.StringInSlice(sourceIDs, resource.SourceID) {
+			sourceIDs = append(sourceIDs, resource.SourceID)
+		}
+	}
+
+	return
 }
 
 // decodeFile decodes a DSDL filterlists file and sends decoded entities to