[Utils] makeseeds script update

Remove old generate-seeds.py script/dir
Update makeseeds for use with PIVX

makeseeds can now pull data from a DNS seeder directly to generate
the list of valid fallback nodes.

GitHub-Pull: bitcoin#167
Rebased-From: a283d96

contrib/seeds/README.md

@@ -1,8 +1,19 @@
-### Seeds ###
+# Seeds
 
 Utility to generate the seeds.txt list that is compiled into the client
-(see [src/chainparamsseeds.h](/src/chainparamsseeds.h) and [share/seeds](/share/seeds)).
+(see [src/chainparamsseeds.h](/src/chainparamsseeds.h) and other utilities in [contrib/seeds](/contrib/seeds)).
 
-The 512 seeds compiled into the 0.10 release were created from sipa's DNS seed data, like this:
+Be sure to update `PATTERN_AGENT` in `makeseeds.py` to include the current version,
+and remove old versions as necessary.
 
-	curl -s http://bitcoin.sipa.be/seeds.txt | makeseeds.py
+The seeds compiled into the release are created from fuzzbawls' DNS seed data, like this:
+
+    curl -s http://seeder.fuzzbawls.pw/pivx-mainnet.txt > seeds_main.txt
+    python3 makeseeds.py < seeds_main.txt > nodes_main.txt
+    python3 generate-seeds.py . > ../../src/chainparamsseeds.h
+
+## Dependencies
+
+Ubuntu:
+
+    sudo apt-get install python3-dnspython

share/seeds/generate-seeds.py → contrib/seeds/generate-seeds.py

@@ -1,5 +1,5 @@
-#!/usr/bin/python
-# Copyright (c) 2014 Wladmir J. van der Laan
+#!/usr/bin/env python3
+# Copyright (c) 2014-2017 Wladimir J. van der Laan
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 '''
@@ -11,7 +11,7 @@
     nodes_main.txt
     nodes_test.txt
 
-These files must consist of lines in the format 
+These files must consist of lines in the format
 
     <ip>
     <ip>:<port>
@@ -31,7 +31,7 @@
 
 These should be pasted into `src/chainparamsseeds.h`.
 '''
-from __future__ import print_function, division
+
 from base64 import b32decode
 from binascii import a2b_hex
 import sys, os
@@ -77,6 +77,9 @@ def parse_spec(s, defaultport):
     if match: # ipv6
         host = match.group(1)
         port = match.group(2)
+    elif s.count(':') > 1: # ipv6, no port
+        host = s
+        port = ''
     else:
         (host,_,port) = s.partition(':')
 
@@ -117,8 +120,8 @@ def main():
     g.write('#ifndef BITCOIN_CHAINPARAMSSEEDS_H\n')
     g.write('#define BITCOIN_CHAINPARAMSSEEDS_H\n')
     g.write('/**\n')
-    g.write(' * List of fixed seed nodes for the pivx network\n')
-    g.write(' * AUTOGENERATED by share/seeds/generate-seeds.py\n')
+    g.write(' * List of fixed seed nodes for the bitcoin network\n')
+    g.write(' * AUTOGENERATED by contrib/seeds/generate-seeds.py\n')
     g.write(' *\n')
     g.write(' * Each line contains a 16-byte IPv6 address and a port.\n')
     g.write(' * IPv4 as well as onion addresses are wrapped inside a IPv6 address accordingly.\n')
@@ -129,7 +132,6 @@ def main():
     with open(os.path.join(indir,'nodes_test.txt'),'r') as f:
         process_nodes(g, f, 'pnSeed6_test', 51474)
     g.write('#endif // BITCOIN_CHAINPARAMSSEEDS_H\n')
-            
+
 if __name__ == '__main__':
     main()
-

contrib/seeds/makeseeds.py

@@ -1,4 +1,7 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
+# Copyright (c) 2013-2017 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
 #
 # Generate seeds.txt from Pieter's DNS seeder
 #
@@ -7,41 +10,61 @@
 
 MAX_SEEDS_PER_ASN=2
 
-MIN_BLOCKS = 200000
+MIN_BLOCKS = 615801
 
 # These are hosts that have been observed to be behaving strangely (e.g.
 # aggressively connecting to every node).
-SUSPICIOUS_HOSTS = set([
-    "130.211.129.106", "178.63.107.226",
-    "83.81.130.26", "88.198.17.7", "148.251.238.178", "176.9.46.6",
-    "54.173.72.127", "54.174.10.182", "54.183.64.54", "54.194.231.211",
-    "54.66.214.167", "54.66.220.137", "54.67.33.14", "54.77.251.214",
-    "54.94.195.96", "54.94.200.247"
-])
+SUSPICIOUS_HOSTS = {
+    ""
+}
 
 import re
 import sys
 import dns.resolver
+import collections
 
-PATTERN_IPV4 = re.compile(r"^((\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})):51472$")
-PATTERN_AGENT = re.compile(r"^(\/Satoshi:0.8.6\/|\/Satoshi:0.9.(2|3)\/|\/Core:0.1(0|1|2).\d{1,2}.\d{1,2}\/)$")
+PATTERN_IPV4 = re.compile(r"^((\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})):(\d+)$")
+PATTERN_IPV6 = re.compile(r"^\[([0-9a-z:]+)\]:(\d+)$")
+PATTERN_ONION = re.compile(r"^([abcdefghijklmnopqrstuvwxyz234567]{16}\.onion):(\d+)$")
+PATTERN_AGENT = re.compile(r"^(/PIVXCore:2.2.(0|1|99)/)$")
 
 def parseline(line):
     sline = line.split()
     if len(sline) < 11:
        return None
-    # Match only IPv4
     m = PATTERN_IPV4.match(sline[0])
+    sortkey = None
+    ip = None
     if m is None:
-        return None
-    # Do IPv4 sanity check
-    ip = 0
-    for i in range(0,4):
-        if int(m.group(i+2)) < 0 or int(m.group(i+2)) > 255:
+        m = PATTERN_IPV6.match(sline[0])
+        if m is None:
+            m = PATTERN_ONION.match(sline[0])
+            if m is None:
+                return None
+            else:
+                net = 'onion'
+                ipstr = sortkey = m.group(1)
+                port = int(m.group(2))
+        else:
+            net = 'ipv6'
+            if m.group(1) in ['::']: # Not interested in localhost
+                return None
+            ipstr = m.group(1)
+            sortkey = ipstr # XXX parse IPv6 into number, could use name_to_ipv6 from generate-seeds
+            port = int(m.group(2))
+    else:
+        # Do IPv4 sanity check
+        ip = 0
+        for i in range(0,4):
+            if int(m.group(i+2)) < 0 or int(m.group(i+2)) > 255:
+                return None
+            ip = ip + (int(m.group(i+2)) << (8*(3-i)))
+        if ip == 0:
             return None
-        ip = ip + (int(m.group(i+2)) << (8*(3-i)))
-    if ip == 0:
-        return None
+        net = 'ipv4'
+        sortkey = ip
+        ipstr = m.group(1)
+        port = int(m.group(6))
     # Skip bad results.
     if sline[1] == 0:
         return None
@@ -52,28 +75,47 @@ def parseline(line):
     # Extract protocol version.
     version = int(sline[10])
     # Extract user agent.
-    agent = sline[11][1:-1]
+    if len(sline) > 11:
+        agent = sline[11][1:] + sline[12][:-1]
+    else:
+        agent = sline[11][1:-1]
     # Extract service flags.
     service = int(sline[9], 16)
     # Extract blocks.
     blocks = int(sline[8])
     # Construct result.
     return {
-        'ip': m.group(1),
+        'net': net,
+        'ip': ipstr,
+        'port': port,
         'ipnum': ip,
         'uptime': uptime30,
         'lastsuccess': lastsuccess,
         'version': version,
         'agent': agent,
         'service': service,
         'blocks': blocks,
+        'sortkey': sortkey,
     }
 
+def filtermultiport(ips):
+    '''Filter out hosts with more nodes per IP'''
+    hist = collections.defaultdict(list)
+    for ip in ips:
+        hist[ip['sortkey']].append(ip)
+    return [value[0] for (key,value) in list(hist.items()) if len(value)==1]
+
 # Based on Greg Maxwell's seed_filter.py
 def filterbyasn(ips, max_per_asn, max_total):
+    # Sift out ips by type
+    ips_ipv4 = [ip for ip in ips if ip['net'] == 'ipv4']
+    ips_ipv6 = [ip for ip in ips if ip['net'] == 'ipv6']
+    ips_onion = [ip for ip in ips if ip['net'] == 'onion']
+
+    # Filter IPv4 by ASN
     result = []
     asn_count = {}
-    for ip in ips:
+    for ip in ips_ipv4:
         if len(result) == max_total:
             break
         try:
@@ -86,13 +128,19 @@ def filterbyasn(ips, max_per_asn, max_total):
             result.append(ip)
         except:
             sys.stderr.write('ERR: Could not resolve ASN for "' + ip['ip'] + '"\n')
+
+    # TODO: filter IPv6 by ASN
+
+    # Add back non-IPv4
+    result.extend(ips_ipv6)
+    result.extend(ips_onion)
     return result
 
 def main():
     lines = sys.stdin.readlines()
     ips = [parseline(line) for line in lines]
 
-    # Skip entries with valid IPv4 address.
+    # Skip entries with valid address.
     ips = [ip for ip in ips if ip is not None]
     # Skip entries from suspicious hosts.
     ips = [ip for ip in ips if ip['ip'] not in SUSPICIOUS_HOSTS]
@@ -103,16 +151,21 @@ def main():
     # Require at least 50% 30-day uptime.
     ips = [ip for ip in ips if ip['uptime'] > 50]
     # Require a known and recent user agent.
-    ips = [ip for ip in ips if PATTERN_AGENT.match(ip['agent'])]
+    ips = [ip for ip in ips if PATTERN_AGENT.match(re.sub(' ', '-', ip['agent']))]
     # Sort by availability (and use last success as tie breaker)
     ips.sort(key=lambda x: (x['uptime'], x['lastsuccess'], x['ip']), reverse=True)
+    # Filter out hosts with multiple bitcoin ports, these are likely abusive
+    ips = filtermultiport(ips)
     # Look up ASNs and limit results, both per ASN and globally.
     ips = filterbyasn(ips, MAX_SEEDS_PER_ASN, NSEEDS)
     # Sort the results by IP address (for deterministic output).
-    ips.sort(key=lambda x: (x['ipnum']))
+    ips.sort(key=lambda x: (x['net'], x['sortkey']))
 
     for ip in ips:
-        print ip['ip']
+        if ip['net'] == 'ipv6':
+            print('[%s]:%i' % (ip['ip'], ip['port']))
+        else:
+            print('%s:%i' % (ip['ip'], ip['port']))
 
 if __name__ == '__main__':
     main()