add root domain version of gfwlist & chinalist

close #44

scripts/_post_/03-root-domain.sh

@@ -0,0 +1,46 @@
+#!/bin/bash -e
+set -o pipefail
+
+CUR_DIR=$(pwd)
+TMP_DIR=$(mktemp -d /tmp/root-domain.XXXXXX)
+
+SRC_FILE_1="$CUR_DIR/dist/gfwlist/gfwlist-root.txt"
+SRC_FILE_2="$CUR_DIR/dist/chinalist/chinalist-root.txt"
+DEST_FILE_1="dist/gfwlist/gfwlist-root.txt"
+DEST_FILE_2="dist/chinalist/chinalist-root.txt"
+
+fetch_src() {
+  cd $TMP_DIR
+
+  cp $SRC_FILE_1 .
+  cp $SRC_FILE_2 .
+
+  cd $CUR_DIR
+}
+
+gen_list() {
+  cd $TMP_DIR
+
+  # find intersection set
+  grep -Fx -f gfwlist-root.txt chinalist-root.txt > common.tmp
+  # gfwlist-root purify
+  grep -Fxv -f common.tmp gfwlist-root.txt > gfwlist-root.new
+  # chinalist-root purify
+  grep -Fxv -f common.tmp chinalist-root.txt > chinalist-root.new
+
+  cd $CUR_DIR
+}
+
+copy_dest() {
+  install -D -m 644 $TMP_DIR/gfwlist-root.new $DEST_FILE_1
+  install -D -m 644 $TMP_DIR/chinalist-root.new $DEST_FILE_2
+}
+
+clean_up() {
+  rm -r $TMP_DIR
+}
+
+fetch_src
+gen_list
+copy_dest
+clean_up

scripts/chinalist/chinalist.sh

@@ -7,17 +7,20 @@ TMP_DIR=$(mktemp -d /tmp/chinalist.XXXXXX)
 SRC_URL_1="https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/apple.china.conf"
 SRC_URL_2="https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/google.china.conf"
 SRC_URL_3="https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/accelerated-domains.china.conf"
-SRC_FILE="$CUR_DIR/dist/toplist/toplist.txt"
+SRC_FILE_1="$CUR_DIR/dist/toplist/toplist.txt"
+SRC_FILE_2="$CUR_DIR/dist/tldlist/tldlist.txt"
 DEST_FILE_1="dist/chinalist/chinalist.txt"
 DEST_FILE_2="dist/chinalist/chinalist-lite.txt"
+DEST_FILE_3="dist/chinalist/chinalist-root.txt"
 
 fetch_src() {
   cd $TMP_DIR
 
   curl -sSL $SRC_URL_1 -o apple.conf
   curl -sSL $SRC_URL_2 -o google.conf
   curl -sSL $SRC_URL_3 -o china.conf
-  cp $SRC_FILE .
+  cp $SRC_FILE_1 .
+  cp $SRC_FILE_2 .
 
   cd $CUR_DIR
 }
@@ -43,15 +46,22 @@ gen_list() {
   grep -Fxv -f toplist.txt chinalist.tmp > chinalist_tail.tmp
   # merge to chinalist
   cat chinalist_head.tmp chinalist_tail.tmp > chinalist.txt
+
   # lite version
   cat chinalist_head.tmp > chinalist-lite.txt
 
+  # root domain version
+  local tlds_regex=$(cat tldlist.txt | tr '\n' '|' | sed '$ s/.$//')
+  local root_domain_regex="([^\.]+)\.($tlds_regex)$"
+  grep -Po $root_domain_regex chinalist.txt | awk '!x[$0]++' > chinalist-root.txt
+
   cd $CUR_DIR
 }
 
 copy_dest() {
   install -D -m 644 $TMP_DIR/chinalist.txt $DEST_FILE_1
   install -D -m 644 $TMP_DIR/chinalist-lite.txt $DEST_FILE_2
+  install -D -m 644 $TMP_DIR/chinalist-root.txt $DEST_FILE_3
 }
 
 clean_up() {

scripts/gfwlist/gfwlist.sh

@@ -6,15 +6,18 @@ TMP_DIR=$(mktemp -d /tmp/gfwlist.XXXXXX)
 
 SRC_URL_1="https://raw.githubusercontent.com/gfwlist/gfwlist/master/gfwlist.txt"
 SRC_URL_2="https://raw.githubusercontent.com/pexcn/gfwlist-extras/master/gfwlist-extras.txt"
-SRC_FILE="$CUR_DIR/dist/toplist/toplist.txt"
-DEST_FILE="dist/gfwlist/gfwlist.txt"
+SRC_FILE_1="$CUR_DIR/dist/toplist/toplist.txt"
+SRC_FILE_2="$CUR_DIR/dist/tldlist/tldlist.txt"
+DEST_FILE_1="dist/gfwlist/gfwlist.txt"
+DEST_FILE_2="dist/gfwlist/gfwlist-root.txt"
 
 fetch_src() {
   cd $TMP_DIR
 
   curl -sSL $SRC_URL_1 | base64 -d > gfwlist-plain.txt
   curl -sSL $SRC_URL_2 -o gfwlist-extras.txt
-  cp $SRC_FILE .
+  cp $SRC_FILE_1 .
+  cp $SRC_FILE_2 .
 
   cd $CUR_DIR
 }
@@ -47,11 +50,17 @@ gen_list() {
   # merge to gfwlist
   cat gfwlist_head.tmp gfwlist_tail.tmp > gfwlist.txt
 
+  # root domain version
+  local tlds_regex=$(cat tldlist.txt | tr '\n' '|' | sed '$ s/.$//')
+  local root_domain_regex="([^\.]+)\.($tlds_regex)$"
+  grep -Po $root_domain_regex gfwlist.txt | awk '!x[$0]++' > gfwlist-root.txt
+
   cd $CUR_DIR
 }
 
 copy_dest() {
-  install -D -m 644 $TMP_DIR/gfwlist.txt $DEST_FILE
+  install -D -m 644 $TMP_DIR/gfwlist.txt $DEST_FILE_1
+  install -D -m 644 $TMP_DIR/gfwlist-root.txt $DEST_FILE_2
 }
 
 clean_up() {