Skip to content

Commit

Permalink
Merge: * DNS filters: optimize filter update
Browse files Browse the repository at this point in the history 
Close #1463

Squashed commit of the following:

commit d5bdc939a2ae9f6d1ae879e4225b1dce09657b92
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Mon Mar 16 16:39:17 2020 +0300

    minor

commit e15b56a0d9db182f9d30b434584018cb1bf038d5
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Thu Mar 12 14:39:07 2020 +0300

    minor

commit 77bf59ca6e556b75af48c5987866af6d5025dae8
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Thu Mar 12 14:30:04 2020 +0300

    minor

commit e19c13f82dd408ed638bd4b68d21cdfebbdf782f
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Thu Mar 12 14:24:50 2020 +0300

    minor

commit 9113c6dae6263aa7ee6e4295c2b60dd3083e2bf0
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Thu Mar 12 14:02:06 2020 +0300

    minor

commit 70283e329e32def3375e893f806a2a02d8ca9f57
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Thu Mar 12 13:35:11 2020 +0300

    logical module Filtering

commit 837a255c6a04941e9fc007a56d71faf4c4213257
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Thu Mar 12 13:11:37 2020 +0300

    minor

commit 1853ed2b57a86dd49508023f47218219399b4fe5
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Thu Mar 12 12:59:28 2020 +0300

    refactor

commit 1ba3cc53c76255439fe54693b40ee9665fdc15e4
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Wed Mar 11 20:12:53 2020 +0300

    * filters: optimize update procedure
  • Loading branch information
@szolin
szolin committed Mar 17, 2020
1 parent a93652b commit 646725e
Show file tree
Hide file tree
Showing 8 changed files with 225 additions and 199 deletions.
7 changes: 0 additions & 7 deletions home/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -339,12 +339,5 @@ func writeAllConfigs() error {
return err
}

userFilter := userFilter()
err = userFilter.save()
if err != nil {
log.Error("Couldn't save the user filter: %s", err)
return err
}

return nil
}
1 change: 0 additions & 1 deletion home/control.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,6 @@ func registerControlHandlers() {

httpRegister("GET", "/control/profile", handleGetProfile)

RegisterFilteringHandlers()
RegisterTLSHandlers()
RegisterBlockedServicesHandlers()
RegisterAuthHandlers()
Expand Down
74 changes: 29 additions & 45 deletions home/control_filtering.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ type filterAddJSON struct {
Whitelist bool `json:"whitelist"`
}

func handleFilteringAddURL(w http.ResponseWriter, r *http.Request) {
func (f *Filtering) handleFilteringAddURL(w http.ResponseWriter, r *http.Request) {
fj := filterAddJSON{}
err := json.NewDecoder(r.Body).Decode(&fj)
if err != nil {
Expand All @@ -53,52 +53,41 @@ func handleFilteringAddURL(w http.ResponseWriter, r *http.Request) {
}

// Set necessary properties
f := filter{
filt := filter{
Enabled: true,
URL: fj.URL,
Name: fj.Name,
white: fj.Whitelist,
}
f.ID = assignUniqueFilterID()
filt.ID = assignUniqueFilterID()

// Download the filter contents
ok, err := f.update()
ok, err := f.update(&filt)
if err != nil {
httpError(w, http.StatusBadRequest, "Couldn't fetch filter from url %s: %s", f.URL, err)
return
}
if f.RulesCount == 0 {
httpError(w, http.StatusBadRequest, "Filter at the url %s has no rules (maybe it points to blank page?)", f.URL)
httpError(w, http.StatusBadRequest, "Couldn't fetch filter from url %s: %s", filt.URL, err)
return
}
if !ok {
httpError(w, http.StatusBadRequest, "Filter at the url %s is invalid (maybe it points to blank page?)", f.URL)
return
}

// Save the filter contents
err = f.save()
if err != nil {
httpError(w, http.StatusBadRequest, "Failed to save filter %d due to %s", f.ID, err)
httpError(w, http.StatusBadRequest, "Filter at the url %s is invalid (maybe it points to blank page?)", filt.URL)
return
}

// URL is deemed valid, append it to filters, update config, write new filter file and tell dns to reload it
if !filterAdd(f) {
httpError(w, http.StatusBadRequest, "Filter URL already added -- %s", f.URL)
if !filterAdd(filt) {
httpError(w, http.StatusBadRequest, "Filter URL already added -- %s", filt.URL)
return
}

onConfigModified()
enableFilters(true)

_, err = fmt.Fprintf(w, "OK %d rules\n", f.RulesCount)
_, err = fmt.Fprintf(w, "OK %d rules\n", filt.RulesCount)
if err != nil {
httpError(w, http.StatusInternalServerError, "Couldn't write body: %s", err)
}
}

func handleFilteringRemoveURL(w http.ResponseWriter, r *http.Request) {
func (f *Filtering) handleFilteringRemoveURL(w http.ResponseWriter, r *http.Request) {

type request struct {
URL string `json:"url"`
Expand Down Expand Up @@ -156,7 +145,7 @@ type filterURLReq struct {
Data filterURLJSON `json:"data"`
}

func handleFilteringSetURL(w http.ResponseWriter, r *http.Request) {
func (f *Filtering) handleFilteringSetURL(w http.ResponseWriter, r *http.Request) {
fj := filterURLReq{}
err := json.NewDecoder(r.Body).Decode(&fj)
if err != nil {
Expand All @@ -169,12 +158,12 @@ func handleFilteringSetURL(w http.ResponseWriter, r *http.Request) {
return
}

f := filter{
filt := filter{
Enabled: fj.Data.Enabled,
Name: fj.Data.Name,
URL: fj.Data.URL,
}
status := filterSetProperties(fj.URL, f, fj.Whitelist)
status := f.filterSetProperties(fj.URL, filt, fj.Whitelist)
if (status & statusFound) == 0 {
http.Error(w, "URL doesn't exist", http.StatusBadRequest)
return
Expand All @@ -196,7 +185,7 @@ func handleFilteringSetURL(w http.ResponseWriter, r *http.Request) {
if fj.Whitelist {
flags = FilterRefreshAllowlists
}
nUpdated, _ := refreshFilters(flags, true)
nUpdated, _ := f.refreshFilters(flags, true)
// if at least 1 filter has been updated, refreshFilters() restarts the filtering automatically
// if not - we restart the filtering ourselves
restart = false
Expand All @@ -209,7 +198,7 @@ func handleFilteringSetURL(w http.ResponseWriter, r *http.Request) {
}
}

func handleFilteringSetRules(w http.ResponseWriter, r *http.Request) {
func (f *Filtering) handleFilteringSetRules(w http.ResponseWriter, r *http.Request) {
body, err := ioutil.ReadAll(r.Body)
if err != nil {
httpError(w, http.StatusBadRequest, "Failed to read request body: %s", err)
Expand All @@ -218,15 +207,10 @@ func handleFilteringSetRules(w http.ResponseWriter, r *http.Request) {

config.UserRules = strings.Split(string(body), "\n")
onConfigModified()
userFilter := userFilter()
err = userFilter.save()
if err != nil {
log.Error("Couldn't save the user filter: %s", err)
}
enableFilters(true)
}

func handleFilteringRefresh(w http.ResponseWriter, r *http.Request) {
func (f *Filtering) handleFilteringRefresh(w http.ResponseWriter, r *http.Request) {
type Req struct {
White bool `json:"whitelist"`
}
Expand All @@ -248,7 +232,7 @@ func handleFilteringRefresh(w http.ResponseWriter, r *http.Request) {
if req.White {
flags = FilterRefreshAllowlists
}
resp.Updated, err = refreshFilters(flags|FilterRefreshForce, false)
resp.Updated, err = f.refreshFilters(flags|FilterRefreshForce, false)
Context.controlLock.Lock()
if err != nil {
httpError(w, http.StatusInternalServerError, "%s", err)
Expand Down Expand Up @@ -298,7 +282,7 @@ func filterToJSON(f filter) filterJSON {
}

// Get filtering configuration
func handleFilteringStatus(w http.ResponseWriter, r *http.Request) {
func (f *Filtering) handleFilteringStatus(w http.ResponseWriter, r *http.Request) {
resp := filteringConfig{}
config.RLock()
resp.Enabled = config.DNS.FilteringEnabled
Expand Down Expand Up @@ -327,7 +311,7 @@ func handleFilteringStatus(w http.ResponseWriter, r *http.Request) {
}

// Set filtering configuration
func handleFilteringConfig(w http.ResponseWriter, r *http.Request) {
func (f *Filtering) handleFilteringConfig(w http.ResponseWriter, r *http.Request) {
req := filteringConfig{}
err := json.NewDecoder(r.Body).Decode(&req)
if err != nil {
Expand Down Expand Up @@ -359,7 +343,7 @@ type checkHostResp struct {
IPList []net.IP `json:"ip_addrs"` // list of IP addresses
}

func handleCheckHost(w http.ResponseWriter, r *http.Request) {
func (f *Filtering) handleCheckHost(w http.ResponseWriter, r *http.Request) {
q := r.URL.Query()
host := q.Get("name")

Expand Down Expand Up @@ -389,15 +373,15 @@ func handleCheckHost(w http.ResponseWriter, r *http.Request) {
}

// RegisterFilteringHandlers - register handlers
func RegisterFilteringHandlers() {
httpRegister("GET", "/control/filtering/status", handleFilteringStatus)
httpRegister("POST", "/control/filtering/config", handleFilteringConfig)
httpRegister("POST", "/control/filtering/add_url", handleFilteringAddURL)
httpRegister("POST", "/control/filtering/remove_url", handleFilteringRemoveURL)
httpRegister("POST", "/control/filtering/set_url", handleFilteringSetURL)
httpRegister("POST", "/control/filtering/refresh", handleFilteringRefresh)
httpRegister("POST", "/control/filtering/set_rules", handleFilteringSetRules)
httpRegister("GET", "/control/filtering/check_host", handleCheckHost)
func (f *Filtering) RegisterFilteringHandlers() {
httpRegister("GET", "/control/filtering/status", f.handleFilteringStatus)
httpRegister("POST", "/control/filtering/config", f.handleFilteringConfig)
httpRegister("POST", "/control/filtering/add_url", f.handleFilteringAddURL)
httpRegister("POST", "/control/filtering/remove_url", f.handleFilteringRemoveURL)
httpRegister("POST", "/control/filtering/set_url", f.handleFilteringSetURL)
httpRegister("POST", "/control/filtering/refresh", f.handleFilteringRefresh)
httpRegister("POST", "/control/filtering/set_rules", f.handleFilteringSetRules)
httpRegister("GET", "/control/filtering/check_host", f.handleCheckHost)
}

func checkFiltersUpdateIntervalHours(i uint32) bool {
Expand Down
6 changes: 4 additions & 2 deletions home/dns.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ func initDNSServer() error {
Context.rdns = InitRDNS(Context.dnsServer, &Context.clients)
Context.whois = initWhois(&Context.clients)

initFiltering()
Context.filters.Init()
return nil
}

Expand Down Expand Up @@ -230,7 +230,7 @@ func startDNSServer() error {
}

Context.dnsFilter.Start()
startFiltering()
Context.filters.Start()
Context.stats.Start()
Context.queryLog.Start()

Expand Down Expand Up @@ -300,5 +300,7 @@ func closeDNSServer() {
Context.auth = nil
}

Context.filters.Close()

log.Debug("Closed all DNS modules")
}
Loading

0 comments on commit 646725e

Please sign in to comment.