Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Merge pull request #1026 from tahoe-lafs/3657.util-python-3
Port the rest of allmydata.util to Python 3

Fixes ticket:3657
  • Loading branch information
itamarst committed Apr 6, 2021
2 parents 4965ecc + 203d539 commit 1336d15
Show file tree
Hide file tree
Showing 12 changed files with 227 additions and 407 deletions.
Empty file added newsfragments/3657.minor
Empty file.
8 changes: 5 additions & 3 deletions src/allmydata/introducer/common.py
Expand Up @@ -11,9 +11,11 @@
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401

import re

from foolscap.furl import decode_furl
from allmydata.crypto.util import remove_prefix
from allmydata.crypto import ed25519
from allmydata.util import base32, rrefutil, jsonbytes as json
from allmydata.util import base32, jsonbytes as json


def get_tubid_string_from_ann(ann):
Expand Down Expand Up @@ -123,10 +125,10 @@ def __init__(self, when, index, canary, ann_d):
self.service_name = ann_d["service-name"]
self.version = ann_d.get("my-version", "")
self.nickname = ann_d.get("nickname", u"")
(service_name, key_s) = index
(_, key_s) = index
self.serverid = key_s
furl = ann_d.get("anonymous-storage-FURL")
if furl:
self.connection_hints = rrefutil.connection_hints_for_furl(furl)
_, self.connection_hints, _ = decode_furl(furl)
else:
self.connection_hints = []
14 changes: 12 additions & 2 deletions src/allmydata/introducer/server.py
Expand Up @@ -24,11 +24,12 @@
from zope.interface import implementer
from twisted.application import service
from twisted.internet import defer
from twisted.internet.address import IPv4Address
from twisted.python.failure import Failure
from foolscap.api import Referenceable
import allmydata
from allmydata import node
from allmydata.util import log, rrefutil, dictutil
from allmydata.util import log, dictutil
from allmydata.util.i2p_provider import create as create_i2p_provider
from allmydata.util.tor_provider import create as create_tor_provider
from allmydata.introducer.interfaces import \
Expand Down Expand Up @@ -148,6 +149,15 @@ def init_web(self, webport):
ws = IntroducerWebishServer(self, webport, nodeurl_path, staticdir)
ws.setServiceParent(self)


def stringify_remote_address(rref):
remote = rref.getPeer()
if isinstance(remote, IPv4Address):
return "%s:%d" % (remote.host, remote.port)
# loopback is a non-IPv4Address
return str(remote)


@implementer(RIIntroducerPublisherAndSubscriberService_v2)
class IntroducerService(service.MultiService, Referenceable):
name = "introducer"
Expand Down Expand Up @@ -216,7 +226,7 @@ def get_subscribers(self):
# tubid will be None. Also, subscribers do not tell us which
# pubkey they use; only publishers do that.
tubid = rref.getRemoteTubID() or "?"
remote_address = rrefutil.stringify_remote_address(rref)
remote_address = stringify_remote_address(rref)
# these three assume subscriber_info["version"]==0, but
# should tolerate other versions
nickname = subscriber_info.get("nickname", u"?")
Expand Down
84 changes: 84 additions & 0 deletions src/allmydata/test/test_consumer.py
@@ -0,0 +1,84 @@
"""
Tests for allmydata.util.consumer.
Ported to Python 3.
"""

from __future__ import unicode_literals
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401

from zope.interface import implementer
from twisted.trial.unittest import TestCase
from twisted.internet.interfaces import IPushProducer, IPullProducer

from allmydata.util.consumer import MemoryConsumer


@implementer(IPushProducer)
@implementer(IPullProducer)
class Producer(object):
"""Can be used as either streaming or non-streaming producer.
If used as streaming, the test should call iterate() manually.
"""

def __init__(self, consumer, data):
self.data = data
self.consumer = consumer
self.done = False

def resumeProducing(self):
"""Kick off streaming."""
self.iterate()

def iterate(self):
"""Do another iteration of writing."""
if self.done:
raise RuntimeError(
"There's a bug somewhere, shouldn't iterate after being done"
)
if self.data:
self.consumer.write(self.data.pop(0))
else:
self.done = True
self.consumer.unregisterProducer()


class MemoryConsumerTests(TestCase):
"""Tests for MemoryConsumer."""

def test_push_producer(self):
"""
A MemoryConsumer accumulates all data sent by a streaming producer.
"""
consumer = MemoryConsumer()
producer = Producer(consumer, [b"abc", b"def", b"ghi"])
consumer.registerProducer(producer, True)
self.assertEqual(consumer.chunks, [b"abc"])
producer.iterate()
producer.iterate()
self.assertEqual(consumer.chunks, [b"abc", b"def", b"ghi"])
self.assertEqual(consumer.done, False)
producer.iterate()
self.assertEqual(consumer.chunks, [b"abc", b"def", b"ghi"])
self.assertEqual(consumer.done, True)

def test_pull_producer(self):
"""
A MemoryConsumer accumulates all data sent by a non-streaming producer.
"""
consumer = MemoryConsumer()
producer = Producer(consumer, [b"abc", b"def", b"ghi"])
consumer.registerProducer(producer, False)
self.assertEqual(consumer.chunks, [b"abc", b"def", b"ghi"])
self.assertEqual(consumer.done, True)


# download_to_data() is effectively tested by some of the filenode tests, e.g.
# test_immutable.py.
46 changes: 44 additions & 2 deletions src/allmydata/test/test_util.py
Expand Up @@ -17,15 +17,17 @@
import json

from twisted.trial import unittest
from foolscap.api import Violation, RemoteException

from allmydata.util import idlib, mathutil
from allmydata.util import fileutil
from allmydata.util import jsonbytes
from allmydata.util import pollmixin
from allmydata.util import yamlutil
from allmydata.util import rrefutil
from allmydata.util.fileutil import EncryptedTemporaryFile
from allmydata.test.common_util import ReallyEqualMixin

from .no_network import fireNow, LocalWrapper

if six.PY3:
long = int
Expand Down Expand Up @@ -480,7 +482,12 @@ def __eq__(self, other):

class YAML(unittest.TestCase):
def test_convert(self):
data = yaml.safe_dump(["str", u"unicode", u"\u1234nicode"])
"""
Unicode and (ASCII) native strings get roundtripped to Unicode strings.
"""
data = yaml.safe_dump(
[six.ensure_str("str"), u"unicode", u"\u1234nicode"]
)
back = yamlutil.safe_load(data)
self.assertIsInstance(back[0], str)
self.assertIsInstance(back[1], str)
Expand Down Expand Up @@ -521,3 +528,38 @@ def test_dumps_bytes(self):
encoded = jsonbytes.dumps_bytes(x)
self.assertIsInstance(encoded, bytes)
self.assertEqual(json.loads(encoded, encoding="utf-8"), x)


class FakeGetVersion(object):
"""Emulate an object with a get_version."""

def __init__(self, result):
self.result = result

def remote_get_version(self):
if isinstance(self.result, Exception):
raise self.result
return self.result


class RrefUtilTests(unittest.TestCase):
"""Tests for rrefutil."""

def test_version_returned(self):
"""If get_version() succeeded, it is set on the rref."""
rref = LocalWrapper(FakeGetVersion(12345), fireNow)
result = self.successResultOf(
rrefutil.add_version_to_remote_reference(rref, "default")
)
self.assertEqual(result.version, 12345)
self.assertIdentical(result, rref)

def test_exceptions(self):
"""If get_version() failed, default version is set on the rref."""
for exception in (Violation(), RemoteException(ValueError())):
rref = LocalWrapper(FakeGetVersion(exception), fireNow)
result = self.successResultOf(
rrefutil.add_version_to_remote_reference(rref, "Default")
)
self.assertEqual(result.version, "Default")
self.assertIdentical(result, rref)
5 changes: 5 additions & 0 deletions src/allmydata/util/_python3.py
Expand Up @@ -115,6 +115,8 @@
"allmydata.util.base62",
"allmydata.util.configutil",
"allmydata.util.connection_status",
"allmydata.util.consumer",
"allmydata.util.dbutil",
"allmydata.util.deferredutil",
"allmydata.util.dictutil",
"allmydata.util.eliotutil",
Expand All @@ -135,10 +137,12 @@
"allmydata.util.observer",
"allmydata.util.pipeline",
"allmydata.util.pollmixin",
"allmydata.util.rrefutil",
"allmydata.util.spans",
"allmydata.util.statistics",
"allmydata.util.time_format",
"allmydata.util.tor_provider",
"allmydata.util.yamlutil",
"allmydata.web",
"allmydata.web.check_results",
"allmydata.web.common",
Expand Down Expand Up @@ -191,6 +195,7 @@
"allmydata.test.test_configutil",
"allmydata.test.test_connections",
"allmydata.test.test_connection_status",
"allmydata.test.test_consumer",
"allmydata.test.test_crawler",
"allmydata.test.test_crypto",

Expand Down
16 changes: 14 additions & 2 deletions src/allmydata/util/consumer.py
@@ -1,11 +1,22 @@

"""This file defines a basic download-to-memory consumer, suitable for use in
"""
This file defines a basic download-to-memory consumer, suitable for use in
a filenode's read() method. See download_to_data() for an example of its use.
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401

from zope.interface import implementer
from twisted.internet.interfaces import IConsumer


@implementer(IConsumer)
class MemoryConsumer(object):

Expand All @@ -28,6 +39,7 @@ def write(self, data):
def unregisterProducer(self):
self.done = True


def download_to_data(n, offset=0, size=None):
"""
Return Deferred that fires with results of reading from the given filenode.
Expand Down
26 changes: 17 additions & 9 deletions src/allmydata/util/dbutil.py
@@ -1,9 +1,23 @@
"""
SQLite3 utilities.
Test coverage currently provided by test_backupdb.py.
Ported to Python 3.
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401

import os, sys

import sqlite3
from sqlite3 import IntegrityError
[IntegrityError]


class DBError(Exception):
Expand All @@ -12,7 +26,7 @@ class DBError(Exception):

def get_db(dbfile, stderr=sys.stderr,
create_version=(None, None), updaters={}, just_create=False, dbname="db",
journal_mode=None, synchronous=None):
):
"""Open or create the given db file. The parent directory must exist.
create_version=(SCHEMA, VERNUM), and SCHEMA must have a 'version' table.
Updaters is a {newver: commands} mapping, where e.g. updaters[2] is used
Expand All @@ -32,12 +46,6 @@ def get_db(dbfile, stderr=sys.stderr,
# The default is unspecified according to <http://www.sqlite.org/foreignkeys.html#fk_enable>.
c.execute("PRAGMA foreign_keys = ON;")

if journal_mode is not None:
c.execute("PRAGMA journal_mode = %s;" % (journal_mode,))

if synchronous is not None:
c.execute("PRAGMA synchronous = %s;" % (synchronous,))

if must_create:
c.executescript(schema)
c.execute("INSERT INTO version (version) VALUES (?)", (target_version,))
Expand Down
35 changes: 12 additions & 23 deletions src/allmydata/util/rrefutil.py
@@ -1,6 +1,16 @@
"""
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

from twisted.internet import address
from foolscap.api import Violation, RemoteException, SturdyRef
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401

from foolscap.api import Violation, RemoteException


def add_version_to_remote_reference(rref, default):
Expand All @@ -18,24 +28,3 @@ def _no_get_version(f):
return rref
d.addCallbacks(_got_version, _no_get_version)
return d


def connection_hints_for_furl(furl):
hints = []
for h in SturdyRef(furl).locationHints:
# Foolscap-0.2.5 and earlier used strings in .locationHints, 0.2.6
# through 0.6.4 used tuples of ("ipv4",host,port), 0.6.5 through
# 0.8.0 used tuples of ("tcp",host,port), and >=0.9.0 uses strings
# again. Tolerate them all.
if isinstance(h, tuple):
hints.append(":".join([str(s) for s in h]))
else:
hints.append(h)
return hints

def stringify_remote_address(rref):
remote = rref.getPeer()
if isinstance(remote, address.IPv4Address):
return "%s:%d" % (remote.host, remote.port)
# loopback is a non-IPv4Address
return str(remote)

0 comments on commit 1336d15

Please sign in to comment.