Skip to content
Permalink
Browse files

Port BridgeDB to python 3

Converted our codebase to python 3...

  https://trac.torproject.org/projects/tor/ticket/30946

Tests pass with Python 3.5.2, but hasn't been ran within a production
environment.
  • Loading branch information
atagar committed Jan 21, 2020
2 parents 5bded87 + 61711c8 commit 57d25390e7fa3cdd41bf05f010061b3e951c048d
Showing with 539 additions and 560 deletions.
  1. +8 −13 bridgedb/Bridges.py
  2. +18 −18 bridgedb/Stability.py
  3. +14 −8 bridgedb/Storage.py
  4. +1 −1 bridgedb/bridgerequest.py
  5. +8 −8 bridgedb/bridges.py
  6. +15 −12 bridgedb/captcha.py
  7. +1 −1 bridgedb/configure.py
  8. +19 −26 bridgedb/crypto.py
  9. +2 −2 bridgedb/distribute.py
  10. +0 −6 bridgedb/distributors/email/__init__.py
  11. +28 −30 bridgedb/distributors/email/autoresponder.py
  12. +2 −4 bridgedb/distributors/email/dkim.py
  13. +17 −20 bridgedb/distributors/email/server.py
  14. +1 −0 bridgedb/distributors/email/templates.py
  15. +3 −2 bridgedb/distributors/https/distributor.py
  16. +18 −18 bridgedb/distributors/https/server.py
  17. +5 −5 bridgedb/distributors/moat/server.py
  18. +3 −2 bridgedb/filters.py
  19. +6 −6 bridgedb/metrics.py
  20. +7 −6 bridgedb/parse/addr.py
  21. +4 −38 bridgedb/parse/descriptors.py
  22. +3 −4 bridgedb/parse/headers.py
  23. +1 −1 bridgedb/parse/nickname.py
  24. +3 −3 bridgedb/parse/options.py
  25. +10 −5 bridgedb/persistent.py
  26. +9 −9 bridgedb/proxy.py
  27. +3 −3 bridgedb/qrcodes.py
  28. +5 −4 bridgedb/schedule.py
  29. +1 −1 bridgedb/test/deprecated.py
  30. +5 −5 bridgedb/test/email_helpers.py
  31. +3 −3 bridgedb/test/https_helpers.py
  32. +18 −18 bridgedb/test/legacy_Tests.py
  33. +3 −3 bridgedb/test/moat_helpers.py
  34. +5 −5 bridgedb/test/test_Bridges.py
  35. +2 −2 bridgedb/test/test_bridgerequest.py
  36. +38 −38 bridgedb/test/test_bridges.py
  37. +15 −15 bridgedb/test/test_captcha.py
  38. +9 −8 bridgedb/test/test_crypto.py
  39. +1 −1 bridgedb/test/test_distributors_moat_request.py
  40. +2 −2 bridgedb/test/test_distributors_moat_server.py
  41. +4 −4 bridgedb/test/test_email_autoresponder.py
  42. +4 −4 bridgedb/test/test_email_distributor.py
  43. +4 −4 bridgedb/test/test_email_dkim.py
  44. +22 −17 bridgedb/test/test_email_server.py
  45. +1 −1 bridgedb/test/test_email_templates.py
  46. +2 −2 bridgedb/test/test_geo.py
  47. +1 −1 bridgedb/test/test_https.py
  48. +5 −5 bridgedb/test/test_https_distributor.py
  49. +1 −1 bridgedb/test/test_https_request.py
  50. +25 −5 bridgedb/test/test_https_server.py
  51. +1 −1 bridgedb/test/test_main.py
  52. +2 −2 bridgedb/test/test_metrics.py
  53. +7 −7 bridgedb/test/test_parse_addr.py
  54. +56 −56 bridgedb/test/test_parse_descriptors.py
  55. +9 −9 bridgedb/test/test_persistent.py
  56. +10 −11 bridgedb/test/test_persistentSaveAndLoad.py
  57. +9 −9 bridgedb/test/test_proxy.py
  58. +1 −1 bridgedb/test/test_schedule.py
  59. +4 −4 bridgedb/test/test_smtp.py
  60. +8 −4 bridgedb/test/test_translations.py
  61. +2 −11 bridgedb/test/test_txrecaptcha.py
  62. +1 −1 bridgedb/test/test_util.py
  63. +4 −4 bridgedb/test/util.py
  64. +15 −8 bridgedb/translations.py
  65. +13 −18 bridgedb/txrecaptcha.py
  66. +10 −13 bridgedb/util.py
  67. +2 −1 requirements.txt
@@ -10,6 +10,7 @@
them into hashrings for distributors.
"""

import binascii
import bisect
import logging
import re
@@ -28,12 +29,6 @@
from bridgedb.parse.fingerprint import toHex
from bridgedb.safelog import logSafely

try:
from cStringIO import StringIO
except ImportError:
from io import StringIO


ID_LEN = 20 # XXX Only used in commented out line in Storage.py
DIGEST_LEN = 20
PORTSPEC_LEN = 16
@@ -335,7 +330,7 @@ def getBridges(self, pos, N=1, filterBySubnet=False):
else:
logging.debug(
"Got duplicate bridge %r in main hashring for position %r."
% (logSafely(k.encode('hex')), pos.encode('hex')))
% (logSafely(binascii.hexlify(k).decode('utf-8')), binascii.hexlify(pos).decode('utf-8')))
keys.sort()

if filterBySubnet:
@@ -361,7 +356,7 @@ def getBridgeByID(self, fp):

def dumpAssignments(self, f, description=""):
logging.info("Dumping bridge assignments for %s..." % self.name)
for b in self.bridges.itervalues():
for b in self.bridges.values():
desc = [ description ]
for tp,val,_,subring in self.subrings:
if subring.getBridgeByID(b.identity):
@@ -380,7 +375,7 @@ def __init__(self, key, rings):
def insert(self, bridge):
# Grab the first 4 bytes
digest = self.hmac(bridge.identity)
pos = long( digest[:8], 16 )
pos = int( digest[:8], 16 )
which = pos % len(self.rings)
self.rings[which].insert(bridge)

@@ -405,7 +400,7 @@ def dumpAssignments(self, filename, description=""):
description is ``"IPv6 obfs2 bridges"`` the line would read:
``"IPv6 obfs2 bridges ring=3"``.
"""
for index, ring in zip(xrange(len(self.rings)), self.rings):
for index, ring in zip(range(len(self.rings)), self.rings):
ring.dumpAssignments(filename, "%s ring=%s" % (description, index))


@@ -544,7 +539,7 @@ def insert(self, bridge):
logging.info("Current rings: %s" % " ".join(self.ringsByName))

def dumpAssignments(self, f, description=""):
for name,ring in self.ringsByName.iteritems():
for name,ring in self.ringsByName.items():
ring.dumpAssignments(f, "%s %s" % (description, name))


@@ -633,8 +628,8 @@ def extractFilterNames(self, ringname):
"""
filterNames = []

for filterName in [x.func_name for x in list(ringname)]:
# Using `assignBridgesToSubring.func_name` gives us a messy
for filterName in [x.__name__ for x in list(ringname)]:
# Using `assignBridgesToSubring.__name__` gives us a messy
# string which includes all parameters and memory addresses. Get
# rid of this by partitioning at the first `(`:
realFilterName = filterName.partition('(')[0]
@@ -33,7 +33,7 @@

# tunables
weighting_factor = float(19)/float(20)
discountIntervalMillis = long(60*60*12*1000)
discountIntervalMillis = 60*60*12*1000


class BridgeHistory(object):
@@ -76,15 +76,15 @@ def __init__(self, fingerprint, ip, port,
self.fingerprint = fingerprint
self.ip = ip
self.port = port
self.weightedUptime = long(weightedUptime)
self.weightedTime = long(weightedTime)
self.weightedRunLength = long(weightedRunLength)
self.weightedUptime = int(weightedUptime)
self.weightedTime = int(weightedTime)
self.weightedRunLength = int(weightedRunLength)
self.totalRunWeights = float(totalRunWeights)
self.lastSeenWithDifferentAddressAndPort = \
long(lastSeenWithDifferentAddressAndPort)
self.lastSeenWithThisAddressAndPort = long(lastSeenWithThisAddressAndPort)
self.lastDiscountedHistoryValues = long(lastDiscountedHistoryValues)
self.lastUpdatedWeightedTime = long(lastUpdatedWeightedTime)
int(lastSeenWithDifferentAddressAndPort)
self.lastSeenWithThisAddressAndPort = int(lastSeenWithThisAddressAndPort)
self.lastDiscountedHistoryValues = int(lastDiscountedHistoryValues)
self.lastUpdatedWeightedTime = int(lastUpdatedWeightedTime)

def discountWeightedFractionalUptimeAndWeightedTime(self, discountUntilMillis):
""" discount weighted times """
@@ -111,8 +111,8 @@ def numDiscountRounds(self, discountUntilMillis):
@property
def weightedFractionalUptime(self):
"""Weighted Fractional Uptime"""
if self.weightedTime <0.0001: return long(0)
return long(10000) * self.weightedUptime / self.weightedTime
if self.weightedTime <0.0001: return 0
return 10000 * self.weightedUptime / self.weightedTime

@property
def tosa(self):
@@ -127,7 +127,7 @@ def familiar(self):
more recently than it, or if it has been around for a Weighted Time of 8 days.
"""
# if this bridge has been around longer than 8 days
if self.weightedTime >= long(8 * 24 * 60 * 60):
if self.weightedTime >= 8 * 24 * 60 * 60:
return True

# return True if self.weightedTime is greater than the weightedTime
@@ -146,10 +146,10 @@ def wmtbac(self):
"""Weighted Mean Time Between Address Change"""
totalRunLength = self.weightedRunLength + \
((self.lastSeenWithThisAddressAndPort -
self.lastSeenWithDifferentAddressAndPort) / long(1000))
self.lastSeenWithDifferentAddressAndPort) / 1000)

totalWeights = self.totalRunWeights + 1.0
if totalWeights < 0.0001: return long(0)
if totalWeights < 0.0001: return 0
assert(isinstance(long,totalRunLength))
assert(isinstance(long,totalWeights))
return totalRunlength / totalWeights
@@ -159,9 +159,9 @@ def addOrUpdateBridgeHistory(bridge, timestamp):
bhe = db.getBridgeHistory(bridge.fingerprint)
if not bhe:
# This is the first status, assume 60 minutes.
secondsSinceLastStatusPublication = long(60*60)
lastSeenWithDifferentAddressAndPort = timestamp * long(1000)
lastSeenWithThisAddressAndPort = timestamp * long(1000)
secondsSinceLastStatusPublication = 60*60
lastSeenWithDifferentAddressAndPort = timestamp * 1000
lastSeenWithThisAddressAndPort = timestamp * 1000

bhe = BridgeHistory(
bridge.fingerprint, bridge.address, bridge.orPort,
@@ -179,9 +179,9 @@ def addOrUpdateBridgeHistory(bridge, timestamp):
# Calculate the seconds since the last parsed status. If this is
# the first status or we haven't seen a status for more than 60
# minutes, assume 60 minutes.
statusPublicationMillis = long(timestamp * 1000)
statusPublicationMillis = timestamp * 1000
if (statusPublicationMillis - bhe.lastSeenWithThisAddressAndPort) > 60*60*1000:
secondsSinceLastStatusPublication = long(60*60)
secondsSinceLastStatusPublication = 60*60
logging.debug("Capping secondsSinceLastStatusPublication to 1 hour")
# otherwise, roll with it
else:
@@ -8,7 +8,6 @@
import sqlite3
import time
import hashlib
from contextlib import GeneratorContextManager
from functools import wraps
from ipaddr import IPAddress
import sys
@@ -212,7 +211,7 @@ def cleanEmailedBridges(self, expireBefore):
cur.execute("DELETE FROM EmailedBridges WHERE when_mailed < ?", (t,))

def getEmailTime(self, addr):
addr = hashlib.sha1(addr).hexdigest()
addr = hashlib.sha1(addr.encode('utf-8')).hexdigest()
cur = self._cur
cur.execute("SELECT when_mailed FROM EmailedBridges WHERE email = ?", (addr,))
v = cur.fetchone()
@@ -221,7 +220,7 @@ def getEmailTime(self, addr):
return strToTime(v[0])

def setEmailTime(self, addr, whenMailed):
addr = hashlib.sha1(addr).hexdigest()
addr = hashlib.sha1(addr.encode('utf-8')).hexdigest()
cur = self._cur
t = timeToStr(whenMailed)
cur.execute("INSERT OR REPLACE INTO EmailedBridges "
@@ -262,7 +261,7 @@ def updateDistributorForHexKey(self, distributor, hex_key):
(distributor, hex_key))

def getWarnedEmail(self, addr):
addr = hashlib.sha1(addr).hexdigest()
addr = hashlib.sha1(addr.encode('utf-8')).hexdigest()
cur = self._cur
cur.execute("SELECT * FROM WarnedEmails WHERE email = ?", (addr,))
v = cur.fetchone()
@@ -271,7 +270,7 @@ def getWarnedEmail(self, addr):
return True

def setWarnedEmail(self, addr, warned=True, whenWarned=time.time()):
addr = hashlib.sha1(addr).hexdigest()
addr = hashlib.sha1(addr.encode('utf-8')).hexdigest()
t = timeToStr(whenWarned)
cur = self._cur
if warned == True:
@@ -345,11 +344,18 @@ def openDatabase(sqlite_file):
return conn


class DBGeneratorContextManager(GeneratorContextManager):
class DBGeneratorContextManager(object):
"""Helper for @contextmanager decorator.
Overload __exit__() so we can call the generator many times
"""

def __init__(self, gen):
self.gen = gen

def __enter__(self):
return next(self.gen)

def __exit__(self, type, value, traceback):
"""Handle exiting a with statement block
@@ -362,7 +368,7 @@ def __exit__(self, type, value, traceback):
"""
if type is None:
try:
self.gen.next()
next(self.gen)
except StopIteration:
return
return
@@ -374,7 +380,7 @@ def __exit__(self, type, value, traceback):
try:
self.gen.throw(type, value, traceback)
raise RuntimeError("generator didn't stop after throw()")
except StopIteration, exc:
except StopIteration as exc:
# Suppress the exception *unless* it's the same exception that
# was passed to throw(). This prevents a StopIteration
# raised inside the "with" statement from being suppressed
@@ -164,7 +164,7 @@ def getHashringPlacement(self, key, client=None):
# Get an HMAC with the key of the client identifier:
digest = getHMACFunc(key)(client)
# Take the lower 8 bytes of the digest and convert to a long:
position = long(digest[:8], 16)
position = int(digest[:8], 16)
return position

def isValid(self, valid=None):
@@ -240,7 +240,7 @@ def identity(self, value):
:param str value: The binary-encoded SHA-1 hash digest of the public
half of this Bridge's identity key.
"""
self.fingerprint = toHex(value)
self.fingerprint = toHex(value).decode('utf-8')

@identity.deleter
def identity(self):
@@ -744,7 +744,7 @@ def _backwardsCompatible(self, nickname=None, address=None, orPort=None,
if not fingerprint:
if not len(idDigest) == 20:
raise TypeError("Bridge with invalid ID")
self.fingerprint = toHex(idDigest)
self.fingerprint = toHex(idDigest).decode('utf-8')
elif fingerprint:
if not isValidFingerprint(fingerprint):
raise TypeError("Bridge with invalid fingerprint (%r)"
@@ -1038,7 +1038,7 @@ def __str__(self):
if safelog.safe_logging:
prefix = '$$'
if fingerprint:
fingerprint = hashlib.sha1(fingerprint).hexdigest().upper()
fingerprint = hashlib.sha1(fingerprint.encode('utf-8')).hexdigest().upper()

if not fingerprint:
fingerprint = '0' * 40
@@ -1178,7 +1178,7 @@ def _getTransportForRequest(self, bridgeRequest):
# their ``methodname`` matches the requested transport:
transports = filter(lambda pt: pt.methodname == desired, self.transports)
# Filter again for whichever of IPv4 or IPv6 was requested:
transports = filter(lambda pt: pt.address.version == ipVersion, transports)
transports = list(filter(lambda pt: pt.address.version == ipVersion, transports))

if not transports:
raise PluggableTransportUnavailable(
@@ -1378,7 +1378,7 @@ def _addBlockByKey(self, key, countryCode):
:meth:`_getBlockKey`.
:param str countryCode: A two-character country code specifier.
"""
if self._blockedIn.has_key(key):
if key in self._blockedIn:
self._blockedIn[key].append(countryCode.lower())
else:
self._blockedIn[key] = [countryCode.lower(),]
@@ -1666,7 +1666,7 @@ def _verifyExtraInfoSignature(self, descriptor):
logging.info("Verifying extrainfo signature for %s..." % self)

# Get the bytes of the descriptor signature without the headers:
document, signature = descriptor.get_bytes().split(TOR_BEGIN_SIGNATURE)
document, signature = str(descriptor).split(TOR_BEGIN_SIGNATURE)
signature = signature.replace(TOR_END_SIGNATURE, '')
signature = signature.replace('\n', '')
signature = signature.strip()
@@ -1710,8 +1710,8 @@ def _verifyExtraInfoSignature(self, descriptor):

# This is the hexadecimal SHA-1 hash digest of the descriptor document
# as it was signed:
signedDigest = codecs.encode(unpadded, 'hex_codec')
actualDigest = hashlib.sha1(document).hexdigest()
signedDigest = codecs.encode(unpadded, 'hex_codec').decode('utf-8')
actualDigest = hashlib.sha1(document.encode('utf-8')).hexdigest()

except Exception as error:
logging.debug("Error verifying extrainfo signature: %s" % error)

0 comments on commit 57d2539

Please sign in to comment.
You can’t perform that action at this time.