Review diff for addrindex in 0.11

README.md

@@ -34,7 +34,7 @@ development team members simply pulls it.
 
 If it is a *more complicated or potentially controversial* change, then the patch
 submitter will be asked to start a discussion (if they haven't already) on the
-[mailing list](http://sourceforge.net/mailarchive/forum.php?forum_name=bitcoin-development).
+[mailing list](https://lists.linuxfoundation.org/mailman/listinfo/bitcoin-dev)
 
 The patch will be accepted if there is broad consensus that it is a good thing.
 Developers should expect to rework and resubmit patches if the code doesn't

configure.ac

@@ -438,7 +438,7 @@ if test x$TARGET_OS = xdarwin; then
   AX_CHECK_LINK_FLAG([[-Wl,-dead_strip]], [LDFLAGS="$LDFLAGS -Wl,-dead_strip"])
 fi
 
-AC_CHECK_HEADERS([endian.h byteswap.h stdio.h stdlib.h unistd.h strings.h sys/types.h sys/stat.h sys/select.h sys/prctl.h])
+AC_CHECK_HEADERS([endian.h sys/endian.h byteswap.h stdio.h stdlib.h unistd.h strings.h sys/types.h sys/stat.h sys/select.h sys/prctl.h])
 AC_SEARCH_LIBS([getaddrinfo_a], [anl], [AC_DEFINE(HAVE_GETADDRINFO_A, 1, [Define this symbol if you have getaddrinfo_a])])
 AC_SEARCH_LIBS([inet_pton], [nsl resolv], [AC_DEFINE(HAVE_INET_PTON, 1, [Define this symbol if you have inet_pton])])
 
@@ -447,6 +447,8 @@ AC_CHECK_DECLS([strnlen])
 AC_CHECK_DECLS([le16toh, le32toh, le64toh, htole16, htole32, htole64, be16toh, be32toh, be64toh, htobe16, htobe32, htobe64],,,
 		[#if HAVE_ENDIAN_H
                  #include <endian.h>
+                 #elif HAVE_SYS_ENDIAN_H
+                 #include <sys/endian.h>
                  #endif])
 
 AC_CHECK_DECLS([bswap_16, bswap_32, bswap_64],,,
@@ -677,6 +679,14 @@ else
   fi
 fi
 
+AC_CHECK_LIB([crypto],[RAND_egd],[],[
+  AC_ARG_WITH([libressl],
+    [AS_HELP_STRING([--with-libressl],[Build with system LibreSSL (default is no; DANGEROUS; NOT SUPPORTED)])],
+    [AC_MSG_WARN([Detected LibreSSL: This is NOT supported, and may break consensus compatibility!])],
+    [AC_MSG_ERROR([Detected LibreSSL: This is NOT supported, and may break consensus compatibility!])]
+  )
+])
+
 CFLAGS_TEMP="$CFLAGS"
 LIBS_TEMP="$LIBS"
 CFLAGS="$CFLAGS $SSL_CFLAGS $CRYPTO_CFLAGS"

contrib/debian/copyright

@@ -2,8 +2,7 @@ Format: http://svn.debian.org/wsvn/dep/web/deps/dep5.mdwn?rev=174
 Upstream-Name: Bitcoin
 Upstream-Contact: Satoshi Nakamoto <satoshin@gmx.com>
  irc://#bitcoin@freenode.net
-Source: http://sourceforge.net/projects/bitcoin/files/
- https://github.com/bitcoin/bitcoin
+Source: https://github.com/bitcoin/bitcoin
 
 Files: *
 Copyright: 2009-2012, Bitcoin Core Developers

contrib/gitian-descriptors/gitian-osx-signer.yml

@@ -8,10 +8,11 @@ packages:
 - "libc6:i386"
 - "faketime"
 reference_datetime: "2015-06-01 00:00:00"
-remotes: []
+remotes:
+- "url": "https://github.com/bitcoin/bitcoin-detached-sigs.git"
+  "dir": "signature"
 files:
 - "bitcoin-osx-unsigned.tar.gz"
-- "signature.tar.gz"
 script: |
   WRAP_DIR=$HOME/wrapped
   mkdir -p ${WRAP_DIR}
@@ -32,6 +33,6 @@ script: |
   SIGNED=bitcoin-osx-signed.dmg
 
   tar -xf ${UNSIGNED}
-  ./detached-sig-apply.sh ${UNSIGNED} signature.tar.gz
+  ./detached-sig-apply.sh ${UNSIGNED} signature/osx
   ${WRAP_DIR}/genisoimage -no-cache-inodes -D -l -probe -V "Bitcoin-Core" -no-pad -r -apple -o uncompressed.dmg signed-app
   ${WRAP_DIR}/dmg dmg uncompressed.dmg ${OUTDIR}/${SIGNED}

contrib/gitian-descriptors/gitian-win-signer.yml

@@ -0,0 +1,39 @@
+---
+name: "bitcoin-win-signer"
+suites:
+- "precise"
+architectures:
+- "amd64"
+packages:
+- "libssl-dev"
+- "autoconf"
+reference_datetime: "2015-06-01 00:00:00"
+remotes:
+- "url": "https://github.com/bitcoin/bitcoin-detached-sigs.git"
+  "dir": "signature"
+files:
+- "osslsigncode-1.7.1.tar.gz"
+- "osslsigncode-Backports-to-1.7.1.patch"
+- "bitcoin-win-unsigned.tar.gz"
+script: |
+  BUILD_DIR=`pwd`
+  SIGDIR=${BUILD_DIR}/signature/win
+  UNSIGNED_DIR=${BUILD_DIR}/unsigned
+
+  echo "f9a8cdb38b9c309326764ebc937cba1523a3a751a7ab05df3ecc99d18ae466c9  osslsigncode-1.7.1.tar.gz" | sha256sum -c
+  echo "a8c4e9cafba922f89de0df1f2152e7be286aba73f78505169bc351a7938dd911  osslsigncode-Backports-to-1.7.1.patch" | sha256sum -c
+
+  mkdir -p ${UNSIGNED_DIR}
+  tar -C ${UNSIGNED_DIR} -xf bitcoin-win-unsigned.tar.gz
+
+  tar xf osslsigncode-1.7.1.tar.gz
+  cd osslsigncode-1.7.1
+  patch -p1 < ${BUILD_DIR}/osslsigncode-Backports-to-1.7.1.patch
+
+  ./configure --without-gsf --without-curl --disable-dependency-tracking
+  make
+  find ${UNSIGNED_DIR} -name "*-unsigned.exe" | while read i; do
+    INFILE="`basename "${i}"`"
+    OUTFILE="`echo "${INFILE}" | sed s/-unsigned//`"
+    ./osslsigncode attach-signature -in "${i}" -out "${OUTDIR}/${OUTFILE}" -sigin "${SIGDIR}/${INFILE}.pem"
+  done

contrib/gitian-descriptors/gitian-win.yml

@@ -84,6 +84,8 @@ script: |
   pushd temp
   tar xf ../$SOURCEDIST
   find bitcoin-* | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ../$SOURCEDIST
+  mkdir -p $OUTDIR/src
+  cp ../$SOURCEDIST $OUTDIR/src
   popd
 
   ORIGPATH="$PATH"
@@ -109,7 +111,8 @@ script: |
     find ${DISTNAME} -type f | sort | zip -X@ ${OUTDIR}/${DISTNAME}-${i}.zip
     cd ../..
   done
-  mkdir -p $OUTDIR/src
-  mv $SOURCEDIST $OUTDIR/src
+  cd $OUTDIR
+  rename 's/-setup\.exe$/-setup-unsigned.exe/' *-setup.exe
+  find . -name "*-setup-unsigned.exe" | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz
   mv ${OUTDIR}/${DISTNAME}-x86_64-*.zip ${OUTDIR}/${DISTNAME}-win64.zip
   mv ${OUTDIR}/${DISTNAME}-i686-*.zip ${OUTDIR}/${DISTNAME}-win32.zip

contrib/gitian-downloader/linux-download-config

@@ -3,7 +3,7 @@ name: bitcoin
 urls:
 - http://bitcoin.org/bitcoin-latest-linux-gitian.zip
 rss:
-- url: http://sourceforge.net/api/file/index/project-id/244765/mtime/desc/limit/100/rss
+- url: 
   xpath: //item/link/text()
   pattern: bitcoin-\d+.\d+.\d+-linux-gitian.zip
 signers:

contrib/gitian-downloader/win32-download-config

@@ -3,7 +3,7 @@ name: bitcoin
 urls:
 - http://bitcoin.org/bitcoin-latest-win32-gitian.zip
 rss:
-- url: http://sourceforge.net/api/file/index/project-id/244765/mtime/desc/limit/100/rss
+- url: 
   xpath: //item/link/text()
   pattern: bitcoin-\d+.\d+.\d+-win32-gitian.zip
 signers:

contrib/macdeploy/detached-sig-apply.sh

@@ -20,7 +20,7 @@ fi
 
 rm -rf ${TEMPDIR} && mkdir -p ${TEMPDIR}
 tar -C ${TEMPDIR} -xf ${UNSIGNED}
-tar -C ${TEMPDIR} -xf ${SIGNATURE}
+cp -rf "${SIGNATURE}"/* ${TEMPDIR}
 
 if [ -z "${PAGESTUFF}" ]; then
   PAGESTUFF=${TEMPDIR}/pagestuff

contrib/macdeploy/detached-sig-create.sh

@@ -7,6 +7,7 @@ CODESIGN=codesign
 TEMPDIR=sign.temp
 TEMPLIST=${TEMPDIR}/signatures.txt
 OUT=signature.tar.gz
+OUTROOT=osx
 
 if [ ! -n "$1" ]; then
   echo "usage: $0 <codesign args>"
@@ -23,7 +24,7 @@ grep -v CodeResources < "${TEMPLIST}" | while read i; do
   TARGETFILE="${BUNDLE}/`echo "${i}" | sed "s|.*${BUNDLE}/||"`"
   SIZE=`pagestuff "$i" -p | tail -2 | grep size | sed 's/[^0-9]*//g'`
   OFFSET=`pagestuff "$i" -p | tail -2 | grep offset | sed 's/[^0-9]*//g'`
-  SIGNFILE="${TEMPDIR}/${TARGETFILE}.sign"
+  SIGNFILE="${TEMPDIR}/${OUTROOT}/${TARGETFILE}.sign"
   DIRNAME="`dirname "${SIGNFILE}"`"
   mkdir -p "${DIRNAME}"
   echo "Adding detached signature for: ${TARGETFILE}. Size: ${SIZE}. Offset: ${OFFSET}"
@@ -32,7 +33,7 @@ done
 
 grep CodeResources < "${TEMPLIST}" | while read i; do
   TARGETFILE="${BUNDLE}/`echo "${i}" | sed "s|.*${BUNDLE}/||"`"
-  RESOURCE="${TEMPDIR}/${TARGETFILE}"
+  RESOURCE="${TEMPDIR}/${OUTROOT}/${TARGETFILE}"
   DIRNAME="`dirname "${RESOURCE}"`"
   mkdir -p "${DIRNAME}"
   echo "Adding resource for: "${TARGETFILE}""

contrib/seeds/README.md

@@ -1,7 +1,7 @@
 ### Seeds ###
 
 Utility to generate the seeds.txt list that is compiled into the client
-(see [src/chainparamsseeds.h](/src/chainparamsseeds.h) and [share/seeds](/share/seeds)).
+(see [src/chainparamsseeds.h](/src/chainparamsseeds.h) and other utilities in [contrib/seeds](/contrib/seeds)).
 
 The 512 seeds compiled into the 0.10 release were created from sipa's DNS seed data, like this:
 

share/seeds/generate-seeds.py → contrib/seeds/generate-seeds.py

@@ -77,6 +77,9 @@ def parse_spec(s, defaultport):
     if match: # ipv6
         host = match.group(1)
         port = match.group(2)
+    elif s.count(':') > 1: # ipv6, no port
+        host = s
+        port = ''
     else:
         (host,_,port) = s.partition(':')
 
@@ -118,7 +121,7 @@ def main():
     g.write('#define BITCOIN_CHAINPARAMSSEEDS_H\n')
     g.write('/**\n')
     g.write(' * List of fixed seed nodes for the bitcoin network\n')
-    g.write(' * AUTOGENERATED by share/seeds/generate-seeds.py\n')
+    g.write(' * AUTOGENERATED by contrib/seeds/generate-seeds.py\n')
     g.write(' *\n')
     g.write(' * Each line contains a 16-byte IPv6 address and a port.\n')
     g.write(' * IPv4 as well as onion addresses are wrapped inside a IPv6 address accordingly.\n')

contrib/seeds/makeseeds.py

@@ -22,26 +22,50 @@
 import re
 import sys
 import dns.resolver
+import collections
 
-PATTERN_IPV4 = re.compile(r"^((\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})):8333$")
-PATTERN_AGENT = re.compile(r"^(\/Satoshi:0.8.6\/|\/Satoshi:0.9.(2|3)\/|\/Satoshi:0.10.\d{1,2}\/)$")
+PATTERN_IPV4 = re.compile(r"^((\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})):(\d+)$")
+PATTERN_IPV6 = re.compile(r"^\[([0-9a-z:]+)\]:(\d+)$")
+PATTERN_ONION = re.compile(r"^([abcdefghijklmnopqrstuvwxyz234567]{16}\.onion):(\d+)$")
+PATTERN_AGENT = re.compile(r"^(\/Satoshi:0\.8\.6\/|\/Satoshi:0\.9\.(2|3|4|5)\/|\/Satoshi:0\.10\.\d{1,2}\/|\/Satoshi:0\.11\.\d{1,2}\/)$")
 
 def parseline(line):
     sline = line.split()
     if len(sline) < 11:
        return None
-    # Match only IPv4
     m = PATTERN_IPV4.match(sline[0])
+    sortkey = None
+    ip = None
     if m is None:
-        return None
-    # Do IPv4 sanity check
-    ip = 0
-    for i in range(0,4):
-        if int(m.group(i+2)) < 0 or int(m.group(i+2)) > 255:
+        m = PATTERN_IPV6.match(sline[0])
+        if m is None:
+            m = PATTERN_ONION.match(sline[0])
+            if m is None:
+                return None
+            else:
+                net = 'onion'
+                ipstr = sortkey = m.group(1)
+                port = int(m.group(2))
+        else:
+            net = 'ipv6'
+            if m.group(1) in ['::']: # Not interested in localhost
+                return None
+            ipstr = m.group(1)
+            sortkey = ipstr # XXX parse IPv6 into number, could use name_to_ipv6 from generate-seeds
+            port = int(m.group(2))
+    else:
+        # Do IPv4 sanity check
+        ip = 0
+        for i in range(0,4):
+            if int(m.group(i+2)) < 0 or int(m.group(i+2)) > 255:
+                return None
+            ip = ip + (int(m.group(i+2)) << (8*(3-i)))
+        if ip == 0:
             return None
-        ip = ip + (int(m.group(i+2)) << (8*(3-i)))
-    if ip == 0:
-        return None
+        net = 'ipv4'
+        sortkey = ip
+        ipstr = m.group(1)
+        port = int(m.group(6))
     # Skip bad results.
     if sline[1] == 0:
         return None
@@ -59,21 +83,37 @@ def parseline(line):
     blocks = int(sline[8])
     # Construct result.
     return {
-        'ip': m.group(1),
+        'net': net,
+        'ip': ipstr,
+        'port': port,
         'ipnum': ip,
         'uptime': uptime30,
         'lastsuccess': lastsuccess,
         'version': version,
         'agent': agent,
         'service': service,
         'blocks': blocks,
+        'sortkey': sortkey,
     }
 
+def filtermultiport(ips):
+    '''Filter out hosts with more nodes per IP'''
+    hist = collections.defaultdict(list)
+    for ip in ips:
+        hist[ip['sortkey']].append(ip)
+    return [value[0] for (key,value) in hist.items() if len(value)==1]
+
 # Based on Greg Maxwell's seed_filter.py
 def filterbyasn(ips, max_per_asn, max_total):
+    # Sift out ips by type
+    ips_ipv4 = [ip for ip in ips if ip['net'] == 'ipv4']
+    ips_ipv6 = [ip for ip in ips if ip['net'] == 'ipv6']
+    ips_onion = [ip for ip in ips if ip['net'] == 'onion']
+
+    # Filter IPv4 by ASN
     result = []
     asn_count = {}
-    for ip in ips:
+    for ip in ips_ipv4:
         if len(result) == max_total:
             break
         try:
@@ -86,13 +126,19 @@ def filterbyasn(ips, max_per_asn, max_total):
             result.append(ip)
         except:
             sys.stderr.write('ERR: Could not resolve ASN for "' + ip['ip'] + '"\n')
+
+    # TODO: filter IPv6 by ASN
+
+    # Add back non-IPv4
+    result.extend(ips_ipv6)
+    result.extend(ips_onion)
     return result
 
 def main():
     lines = sys.stdin.readlines()
     ips = [parseline(line) for line in lines]
 
-    # Skip entries with valid IPv4 address.
+    # Skip entries with valid address.
     ips = [ip for ip in ips if ip is not None]
     # Skip entries from suspicious hosts.
     ips = [ip for ip in ips if ip['ip'] not in SUSPICIOUS_HOSTS]
@@ -106,13 +152,18 @@ def main():
     ips = [ip for ip in ips if PATTERN_AGENT.match(ip['agent'])]
     # Sort by availability (and use last success as tie breaker)
     ips.sort(key=lambda x: (x['uptime'], x['lastsuccess'], x['ip']), reverse=True)
+    # Filter out hosts with multiple bitcoin ports, these are likely abusive
+    ips = filtermultiport(ips)
     # Look up ASNs and limit results, both per ASN and globally.
     ips = filterbyasn(ips, MAX_SEEDS_PER_ASN, NSEEDS)
     # Sort the results by IP address (for deterministic output).
-    ips.sort(key=lambda x: (x['ipnum']))
+    ips.sort(key=lambda x: (x['net'], x['sortkey']))
 
     for ip in ips:
-        print ip['ip']
+        if ip['net'] == 'ipv6':
+            print '[%s]:%i' % (ip['ip'], ip['port'])
+        else:
+            print '%s:%i' % (ip['ip'], ip['port'])
 
 if __name__ == '__main__':
     main()