Skip to content
Permalink
Browse files

Fix broken download of Tor exit relays.

The periodic download of Tor exit relays was broken for a number of
reasons.  Here's what we're doing to fix this issue:

1. We're moving the proxies variable outside of state.  The problem is
   that the state object is written to disk and reloaded every 30
   minutes (a cron job is triggering this reload by running
   ~/bridgedb-admin/reload-bridgedb).  The reload causes state.proxies
   to be at a different memory address than before the reload, which
   breaks the looping call that fetches new exit relays every three
   hours.  This looping call expects to write exit relays to the same
   memory address each time, but after BridgeDB's first reload, the
   memory address changed, so exit relays are no longer updated.
   There's no need to keep our proxies in BridgeDB's state.  We fetch
   them continuously anyway, and also right after BridgeDB starts.

2. We're adding the method replaceExitRelays().  Once we have a new
   batch of exit relay addresses, this method allows us to completely
   overwrite the past batch.

3. We're adding the argument "setStdout=False" to the call to
   startLogging() because otherwise we're missing the download script's
   output.
  • Loading branch information
NullHypothesis committed Aug 12, 2019
1 parent 081ff36 commit 0d5ed52e5906260e142ef9cfa12752810fd1ffa2
Showing with 17 additions and 7 deletions.
  1. +3 −4 bridgedb/main.py
  2. +13 −2 bridgedb/proxy.py
  3. +1 −1 scripts/get-tor-exits
@@ -359,7 +359,6 @@ def run(options, reactor=reactor):
moatDistributor = None

# Save our state
state.proxies = proxies
state.key = key
state.save()

@@ -411,13 +410,13 @@ def reload(inThread=True): # pragma: no cover
logging.info("Reloading the list of open proxies...")
for proxyfile in cfg.PROXY_LIST_FILES:
logging.info("Loading proxies from: %s" % proxyfile)
proxy.loadProxiesFromFile(proxyfile, state.proxies, removeStale=True)
proxy.loadProxiesFromFile(proxyfile, proxies, removeStale=True)

logging.info("Reparsing bridge descriptors...")
(hashring,
emailDistributorTmp,
ipDistributorTmp,
moatDistributorTmp) = createBridgeRings(cfg, state.proxies, key)
moatDistributorTmp) = createBridgeRings(cfg, proxies, key)
logging.info("Bridges loaded: %d" % len(hashring))

# Initialize our DB.
@@ -483,7 +482,7 @@ def reload(inThread=True): # pragma: no cover
if config.TASKS['GET_TOR_EXIT_LIST']:
tasks['GET_TOR_EXIT_LIST'] = task.LoopingCall(
proxy.downloadTorExits,
state.proxies,
proxies,
config.SERVER_PUBLIC_EXTERNAL_IP)

if config.TASKS.get('DELETE_UNPARSEABLE_DESCRIPTORS'):
@@ -51,7 +51,7 @@ def downloadTorExits(proxyList, ipaddress, port=443, protocol=None):
"""
proto = ExitListProtocol() if protocol is None else protocol()
args = [proto.script, '--stdout', '-a', ipaddress, '-p', str(port)]
proto.deferred.addCallback(proxyList.addExitRelays)
proto.deferred.addCallback(proxyList.replaceExitRelays)
proto.deferred.addErrback(logging.exception)
transport = reactor.spawnProcess(proto, proto.script, args=args, env={})
return proto.deferred
@@ -76,7 +76,7 @@ def loadProxiesFromFile(filename, proxySet=None, removeStale=False):
:returns: A list of all the proxies listed in the **files* (regardless of
whether they were added or removed).
"""
logging.info("Reloading proxy lists...")
logging.info("Reloading proxy lists from file %s" % filename)

addresses = []

@@ -256,6 +256,17 @@ def addExitRelays(self, relays):
logging.info("Loading exit relays into proxy list...")
[self.add(x, self._exitTag) for x in relays]

def replaceExitRelays(self, relays):
existingExitRelays = self.getAllWithTag(self._exitTag)
logging.debug("Replacing %d existing with %d new exit relays." %
(len(existingExitRelays), len(relays)))

for relay in existingExitRelays:
self.discard(relay)

self.addExitRelays(relays)


def getTag(self, ip):
"""Get the tag for an **ip** in this ``ProxySet``, if available.
@@ -36,7 +36,7 @@ from twisted.internet.error import DNSLookupError
from twisted.internet.error import TimeoutError


log.startLogging(sys.stderr)
log.startLogging(sys.stderr, setStdout=False)


def backupFile(filename):

0 comments on commit 0d5ed52

Please sign in to comment.
You can’t perform that action at this time.