From 607077ca8141f732d4678955c62cc2345958766b Mon Sep 17 00:00:00 2001 From: Ainar Garipov Date: Thu, 20 Oct 2022 18:07:09 +0300 Subject: [PATCH] all: use hostlists registry --- .github/ISSUE_TEMPLATE/config.yml | 5 + client/src/helpers/filters/filters.json | 268 ++++++++++++++---------- scripts/README.md | 20 ++ scripts/make/go-lint.sh | 2 +- scripts/vetted-filters/main.go | 150 +++++++++++++ 5 files changed, 337 insertions(+), 108 deletions(-) create mode 100644 scripts/vetted-filters/main.go diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 921e07cc408..c80d6d66ac4 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -6,6 +6,11 @@ website 'name': 'AdGuard filters issues' 'url': 'https://link.adtidy.org/forward.html?action=report&app=home&from=github' + - 'about': > + Please send requests for addition to the vetted filtering lists to the + Hostlists Registry repository. + 'name': 'AdGuard Hostlists Registry' + 'url': 'https://github.com/AdguardTeam/HostlistsRegistry' - 'about': > Please use GitHub Discussions for questions 'name': 'Q&A Discussions' diff --git a/client/src/helpers/filters/filters.json b/client/src/helpers/filters/filters.json index 63fe0995eed..3bef7420457 100644 --- a/client/src/helpers/filters/filters.json +++ b/client/src/helpers/filters/filters.json @@ -4,159 +4,213 @@ "name": "filter_category_general", "description": "filter_category_general_desc" }, - "security": { - "name": "filter_category_security", - "description": "filter_category_security_desc" + "other": { + "name": "filter_category_other", + "description": "filter_category_other_desc" }, "regional": { "name": "filter_category_regional", "description": "filter_category_regional_desc" }, - "other": { - "name": "filter_category_other", - "description": "filter_category_other_desc" + "security": { + "name": "filter_category_security", + "description": "filter_category_security_desc" } }, "filters": { - "adguard-dns-filter": { + "1hosts_lite": { + "name": "1Hosts (Lite)", + "categoryId": "general", + "homepage": "https://badmojr.github.io/1Hosts/", + "source": "https://badmojr.gitlab.io/1hosts/Lite/adblock.txt" + }, + "CHN_adrules": { + "name": "CHN: AdRules DNS List", + "categoryId": "regional", + "homepage": "https://github.com/Cats-Team/AdRules", + "source": "https://raw.githubusercontent.com/Cats-Team/AdRules/main/dns.txt" + }, + "CHN_anti_ad": { + "name": "CHN: anti-AD", + "categoryId": "regional", + "homepage": "https://anti-ad.net/", + "source": "https://anti-ad.net/easylist.txt" + }, + "IDN_abpindo": { + "name": "IDN: ABPindo", + "categoryId": "regional", + "homepage": "https://github.com/ABPindo/indonesianadblockrules", + "source": "https://raw.githubusercontent.com/ABPindo/indonesianadblockrules/master/subscriptions/aghome.txt" + }, + "IRN_unwanted_iranian_domains": { + "name": "IRN: PersianBlocker list", + "categoryId": "regional", + "homepage": "https://github.com/MasterKia/PersianBlocker", + "source": "https://raw.githubusercontent.com/MasterKia/PersianBlocker/main/PersianBlockerHosts.txt" + }, + "ITA_filtri_dns": { + "name": "ITA: Filtri-DNS", + "categoryId": "regional", + "homepage": "https://filtri-dns.ga/", + "source": "https://filtri-dns.ga/filtri.txt" + }, + "KOR_list_kr": { + "name": "KOR: List-KR DNS", + "categoryId": "regional", + "homepage": "https://github.com/List-KR/List-KR", + "source": "https://github.com/List-KR/List-KR" + }, + "KOR_youslist": { + "name": "KOR: YousList", + "categoryId": "regional", + "homepage": "https://github.com/yous/YousList", + "source": "https://raw.githubusercontent.com/yous/YousList/master/hosts.txt" + }, + "MKD_macedonian_pi_hole_blocklist": { + "name": "MKD: Macedonian Pi-hole Blocklist", + "categoryId": "regional", + "homepage": "https://github.com/cchevy/macedonian-pi-hole-blocklist", + "source": "https://raw.githubusercontent.com/cchevy/macedonian-pi-hole-blocklist/master/hosts.txt" + }, + "NOR_dandelion_sprouts_anti_malware_list": { + "name": "NOR: Dandelion Sprouts nordiske filtre", + "categoryId": "regional", + "homepage": "https://github.com/DandelionSprout/adfilt", + "source": "https://raw.githubusercontent.com/DandelionSprout/adfilt/master/NorwegianExperimentalList%20alternate%20versions/NordicFiltersAdGuardHome.txt" + }, + "POL_polish_filters_for_pi_hole": { + "name": "POL: Polish filters for Pi hole", + "categoryId": "regional", + "homepage": "https://www.certyficate.it/", + "source": "https://raw.githubusercontent.com/MajkiIT/polish-ads-filter/master/polish-pihole-filters/hostfile.txt" + }, + "SWE_frellwit_swedish_hosts_file": { + "name": "SWE: Frellwit's Swedish Hosts File", + "categoryId": "regional", + "homepage": "https://github.com/lassekongo83/Frellwits-filter-lists/", + "source": "https://raw.githubusercontent.com/lassekongo83/Frellwits-filter-lists/master/Frellwits-Swedish-Hosts-File.txt" + }, + "TUR_turk_adlist": { + "name": "TUR: turk-adlist", + "categoryId": "regional", + "homepage": "https://github.com/bkrucarci/turk-adlist", + "source": "https://raw.githubusercontent.com/bkrucarci/turk-adlist/master/hosts" + }, + "VNM_abpvn": { + "name": "VNM: ABPVN List", + "categoryId": "regional", + "homepage": "http://abpvn.com/", + "source": "https://abpvn.com/android/abpvn.txt" + }, + "adguard_dns_filter": { "name": "AdGuard DNS filter", "categoryId": "general", "homepage": "https://github.com/AdguardTeam/AdGuardSDNSFilter", "source": "https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt" }, - "adaway-default-blocklist": { + "adway_default_blocklist": { "name": "AdAway Default Blocklist", "categoryId": "general", "homepage": "https://github.com/AdAway/adaway.github.io/", "source": "https://adaway.org/hosts.txt" }, - "peter-lowe-list": { - "name": "Peter Lowe's List", - "categoryId": "general", - "homepage": "https://pgl.yoyo.org/adservers/", - "source": "https://pgl.yoyo.org/adservers/serverlist.php?hostformat=adblockplus&showintro=1&mimetype=plaintext" + "curben_phishing_filter": { + "name": "Phishing URL Blocklist (PhishTank and OpenPhish)", + "categoryId": "security", + "homepage": "https://gitlab.com/malware-filter/phishing-filter", + "source": "https://malware-filter.gitlab.io/malware-filter/phishing-filter-agh.txt" }, - "dan-pollock-list": { + "dan_pollocks_list": { "name": "Dan Pollock's List", "categoryId": "general", "homepage": "https://someonewhocares.org/", "source": "https://someonewhocares.org/hosts/zero/hosts" }, - "oisd": { - "name": "OISD Blocklist Basic", - "categoryId": "general", - "homepage": "https://oisd.nl/", - "source": "https://abp.oisd.nl/basic/" + "dandelion_sprouts_anti_malware_list": { + "name": "Dandelion Sprout's Anti-Malware List", + "categoryId": "security", + "homepage": "https://github.com/DandelionSprout/adfilt", + "source": "https://raw.githubusercontent.com/DandelionSprout/adfilt/master/Alternate%20versions%20Anti-Malware%20List/AntiMalwareAdGuardHome.txt" }, - "game-console-adblock-list": { - "name": "Game Console Adblock List", - "categoryId": "general", + "dandelion_sprouts_game_console_adblock_list": { + "name": "Dandelion Sprout's Game Console Adblock List", + "categoryId": "other", "homepage": "https://github.com/DandelionSprout/adfilt", "source": "https://raw.githubusercontent.com/DandelionSprout/adfilt/master/GameConsoleAdblockList.txt" }, - "perflyst-dandelion-sprout-smart-tv-blocklist-for-adguard-home": { - "name": "Perflyst and Dandelion Sprout's Smart-TV Blocklist", + "energized_spark": { + "name": "Energized Spark", "categoryId": "general", - "homepage": "https://github.com/Perflyst/PiHoleBlocklist", - "source": "https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/SmartTV-AGH.txt" + "homepage": "https://energized.pro/", + "source": "https://block.energized.pro/spark/formats/filter" }, - "nocoin-filter-list": { + "nocoin_filter_list": { "name": "NoCoin Filter List", "categoryId": "security", "homepage": "https://github.com/hoshsadiq/adblock-nocoin-list/", "source": "https://raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/master/hosts.txt" }, - "the-big-list-of-hacked-malware-web-sites": { - "name": "The Big List of Hacked Malware Web Sites", - "categoryId": "security", - "homepage": "https://github.com/mitchellkrogza/The-Big-List-of-Hacked-Malware-Web-Sites", - "source": "https://raw.githubusercontent.com/mitchellkrogza/The-Big-List-of-Hacked-Malware-Web-Sites/master/hosts" + "notracking_hosts_blocklists": { + "name": "The NoTracking blocklist", + "categoryId": "general", + "homepage": "https://github.com/notracking/hosts-blocklists", + "source": "https://raw.githubusercontent.com/notracking/hosts-blocklists/master/adblock/adblock.txt" + }, + "oisd_basic": { + "name": "OISD Blocklist Basic", + "categoryId": "general", + "homepage": "https://oisd.nl/", + "source": "https://abp.oisd.nl/basic/" + }, + "oisd_full": { + "name": "OISD Blocklist Full", + "categoryId": "general", + "homepage": "https://oisd.nl/", + "source": "https://abp.oisd.nl/" + }, + "perflyst_dandelion_sprout_smart_tv_blocklist_for_adguard_home": { + "name": "Perflyst and Dandelion Sprout's Smart-TV Blocklist", + "categoryId": "other", + "homepage": "https://github.com/Perflyst/PiHoleBlocklist", + "source": "https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/SmartTV-AGH.txt" + }, + "peter_lowe_list": { + "name": "Peter Lowe's Blocklist", + "categoryId": "general", + "homepage": "https://pgl.yoyo.org/adservers/", + "source": "https://pgl.yoyo.org/adservers/serverlist.php?hostformat=adblockplus\u0026showintro=1\u0026mimetype=plaintext" }, - "scam-blocklist-by-durable-napkin": { + "scam_blocklist_by_durablenapkin": { "name": "Scam Blocklist by DurableNapkin", "categoryId": "security", "homepage": "https://github.com/durablenapkin/scamblocklist", "source": "https://raw.githubusercontent.com/durablenapkin/scamblocklist/master/adguard.txt" }, - "urlhaus-filter-online": { - "name": "Online Malicious URL Blocklist", - "categoryId": "security", - "homepage": "https://gitlab.com/malware-filter/urlhaus-filter", - "source": "https://malware-filter.gitlab.io/malware-filter/urlhaus-filter-agh-online.txt" - }, - "dandelion-sprouts-anti-malware-list": { - "name": "Dandelion Sprout's Anti-Malware List", + "staklerware_indicators_list": { + "name": "Stalkerware Indicators List", "categoryId": "security", - "homepage": "https://github.com/DandelionSprout/adfilt", - "source": "https://raw.githubusercontent.com/DandelionSprout/adfilt/master/Alternate%20versions%20Anti-Malware%20List/AntiMalwareAdGuardHome.txt" - }, - "NOR-dandelion-sprouts-nordiske-filtre": { - "name": "NOR: Dandelion Sprouts nordiske filtre", - "categoryId": "regional", - "homepage": "https://github.com/DandelionSprout/adfilt", - "source": "https://raw.githubusercontent.com/DandelionSprout/adfilt/master/NorwegianExperimentalList%20alternate%20versions/NordicFiltersAdGuardHome.txt" - }, - "POL-polish-filters-for-pihole": { - "name": "POL: Polish filters for Pi hole", - "categoryId": "regional", - "homepage": "https://www.certyficate.it/", - "source": "https://raw.githubusercontent.com/MajkiIT/polish-ads-filter/master/polish-pihole-filters/hostfile.txt" + "homepage": "https://github.com/AssoEchap/stalkerware-indicators", + "source": "https://raw.githubusercontent.com/AssoEchap/stalkerware-indicators/master/generated/hosts" }, - "KOR-youslist": { - "name": "KOR: YousList", - "categoryId": "regional", - "homepage": "https://github.com/yous/YousList", - "source": "https://raw.githubusercontent.com/yous/YousList/master/hosts.txt" - }, - "VNM-abpvn-list": { - "name": "VNM: ABPVN List", - "categoryId": "regional", - "homepage": "http://abpvn.com/", - "source": "https://abpvn.com/android/abpvn.txt" - }, - "SWE-frellwit-swedish-hosts-file": { - "name": "SWE: Frellwit's Swedish Hosts File", - "categoryId": "regional", - "homepage": "https://github.com/lassekongo83/Frellwits-filter-lists/", - "source": "https://raw.githubusercontent.com/lassekongo83/Frellwits-filter-lists/master/Frellwits-Swedish-Hosts-File.txt" - }, - "ITA-filtri-dns": { - "name": "ITA: Filtri-DNS", - "categoryId": "regional", - "homepage": "https://filtri-dns.ga/", - "source": "https://filtri-dns.ga/filtri.txt" - }, - "IRN-unwanted-iranian-domains": { - "name": "IRN: Unwanted Iranian domains", - "categoryId": "regional", - "homepage": "https://github.com/DRSDavidSoft/additional-hosts", - "source": "https://raw.githubusercontent.com/DRSDavidSoft/additional-hosts/master/domains/blacklist/unwanted-iranian.txt" - }, - "MKD-macedonian-pi-hole-blocklist": { - "name": "MKD: Macedonian Pi-hole Blocklist", - "categoryId": "regional", - "homepage": "https://github.com/cchevy/macedonian-pi-hole-blocklist", - "source": "https://raw.githubusercontent.com/cchevy/macedonian-pi-hole-blocklist/master/hosts.txt" - }, - "CHN-anti-ad" : { - "name": "CHN: anti-AD", - "categoryId": "regional", - "homepage": "https://anti-ad.net/", - "source": "https://anti-ad.net/easylist.txt" + "steven_blacks_list": { + "name": "Steven Black's List", + "categoryId": "general", + "homepage": "https://github.com/StevenBlack/hosts", + "source": "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts" }, - "IDN-abpindo": { - "name": "IDN: ABPindo", - "categoryId": "regional", - "homepage": "https://github.com/ABPindo/indonesianadblockrules/", - "source": "https://raw.githubusercontent.com/ABPindo/indonesianadblockrules/master/subscriptions/abpindo.txt" + "the_big_list_of_hacked_malware_web_sites": { + "name": "The Big List of Hacked Malware Web Sites", + "categoryId": "security", + "homepage": "https://github.com/mitchellkrogza/The-Big-List-of-Hacked-Malware-Web-Sites", + "source": "https://raw.githubusercontent.com/mitchellkrogza/The-Big-List-of-Hacked-Malware-Web-Sites/master/hosts" }, - "NLD-Easylist": { - "name": "NLD: Easylist", - "categoryId": "regional", - "homepage": "https://forums.lanik.us/viewforum.php?f=100", - "source": "https://easylist-downloads.adblockplus.org/easylistdutch.txt" + "urlhaus_filter_online": { + "name": "Malicious URL Blocklist (URLHaus)", + "categoryId": "security", + "homepage": "https://gitlab.com/malware-filter/urlhaus-filter", + "source": "https://malware-filter.gitlab.io/malware-filter/urlhaus-filter-agh.txt" }, - "windows-spy-blocker" : { + "windowsspyblocker_hosts_spy_rules": { "name": "WindowsSpyBlocker - Hosts spy rules", "categoryId": "other", "homepage": "https://github.com/crazy-max/WindowsSpyBlocker", diff --git a/scripts/README.md b/scripts/README.md index 2ec087e070e..3c9026b1dc9 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -205,3 +205,23 @@ code from [the repo][companiesrepo]. ``` [companiesrepo]: https://github.com/AdguardTeam/companiesdb + + + +## `vetted-filters/`: Vetted Filters Updater + +A simple script that downloads and updates the vetted filtering list data from +AdGuard's [Hostlists Registry][reg]. + +Optional environment: + + * `URL`: the URL of the index file. By default it's + `https://adguardteam.github.io/HostlistsRegistry/assets/filters.json`. + + ### Usage + +```sh +go run ./scripts/vetted-filters/main.go +``` + +[reg]: https://github.com/AdguardTeam/HostlistsRegistry diff --git a/scripts/make/go-lint.sh b/scripts/make/go-lint.sh index 8c462d5bad8..31aba8c34bc 100644 --- a/scripts/make/go-lint.sh +++ b/scripts/make/go-lint.sh @@ -229,7 +229,7 @@ gocyclo --over 13 ./internal/dhcpd ./internal/filtering/ ./internal/home/ gocyclo --over 10 ./internal/aghio/ ./internal/aghnet/ ./internal/aghos/\ ./internal/aghtest/ ./internal/dnsforward/ ./internal/stats/\ ./internal/tools/ ./internal/updater/ ./internal/next/ ./internal/version/\ - ./main.go + ./scripts/vetted-filters/ ./main.go ineffassign ./... diff --git a/scripts/vetted-filters/main.go b/scripts/vetted-filters/main.go new file mode 100644 index 00000000000..9d6e5cc0a00 --- /dev/null +++ b/scripts/vetted-filters/main.go @@ -0,0 +1,150 @@ +// vetted-filters fetches the most recent Hostlists Registry index and +// transforms the filters from it to AdGuard Home's format. +package main + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + "os" + "time" + + "github.com/AdguardTeam/golibs/log" + "github.com/google/renameio/maybe" +) + +func main() { + urlStr := "https://adguardteam.github.io/HostlistsRegistry/assets/filters.json" + if v, ok := os.LookupEnv("URL"); ok { + urlStr = v + } + + // Validate the URL. + _, err := url.Parse(urlStr) + check(err) + + c := &http.Client{ + Timeout: 10 * time.Second, + } + + resp, err := c.Get(urlStr) + check(err) + defer log.OnCloserError(resp.Body, log.ERROR) + + if resp.StatusCode != http.StatusOK { + panic(fmt.Errorf("expected code %d, got %d", http.StatusOK, resp.StatusCode)) + } + + hlFlt := &hlFilters{} + err = json.NewDecoder(resp.Body).Decode(hlFlt) + check(err) + + aghFlt := &aghFilters{ + Categories: map[string]*aghFiltersCategory{ + "general": { + Name: "filter_category_general", + Description: "filter_category_general_desc", + }, + "other": { + Name: "filter_category_other", + Description: "filter_category_other_desc", + }, + "regional": { + Name: "filter_category_regional", + Description: "filter_category_regional_desc", + }, + "security": { + Name: "filter_category_security", + Description: "filter_category_security_desc", + }, + }, + Filters: map[string]*aghFiltersFilter{}, + } + + for i, f := range hlFlt.Filters { + id := f.FilterID + cat := f.category() + if cat == "" { + log.Info("warning: filter %s at index %d does not have a fitting category", id, i) + } + + aghFlt.Filters[id] = &aghFiltersFilter{ + Name: f.Name, + CategoryID: cat, + Homepage: f.Homepage, + Source: f.SourceURL, + } + } + + b, err := json.MarshalIndent(aghFlt, "", " ") + check(err) + + b = append(b, '\n') + + err = maybe.WriteFile("client/src/helpers/filters/filters.json", b, 0o644) + check(err) +} + +// check is a simple error-checking helper for scripts. +func check(err error) { + if err != nil { + panic(err) + } +} + +// hlFilters is the JSON structure for the Hostlists Registry rule list index. +type hlFilters struct { + Filters []*hlFiltersFilter `json:"filters"` +} + +// hlFiltersFilter is the JSON structure for a filter in the Hostlists Registry. +type hlFiltersFilter struct { + FilterID string `json:"filterId"` + Name string `json:"name"` + Homepage string `json:"homepage"` + SourceURL string `json:"sourceUrl"` + Tags []string `json:"tags"` +} + +// category returns the AdGuard Home category for this filter. If there is no +// fitting category, cat is empty. +func (f *hlFiltersFilter) category() (cat string) { + for _, t := range f.Tags { + switch t { + case "purpose:general": + return "general" + case "purpose:other": + return "other" + case "purpose:regional": + return "regional" + case "purpose:security": + return "security" + } + } + + return "" +} + +// aghFilters is the JSON structure for AdGuard Home's list of vetted filtering +// rule list in file client/src/helpers/filters/filters.json. +type aghFilters struct { + Categories map[string]*aghFiltersCategory `json:"categories"` + Filters map[string]*aghFiltersFilter `json:"filters"` +} + +// aghFiltersCategory is the JSON structure for a category in the vetted +// filtering rule list file. +type aghFiltersCategory struct { + Name string `json:"name"` + Description string `json:"description"` +} + +// aghFiltersFilter is the JSON structure for a filter in the vetted filtering +// rule list file. +type aghFiltersFilter struct { + Name string `json:"name"` + CategoryID string `json:"categoryId"` + Homepage string `json:"homepage"` + Source string `json:"source"` +}