Skip to content

Commit

Permalink
Bringing local cluster.py closer to upstream.
Browse files Browse the repository at this point in the history 
This is so that a simple diff is possible.
  • Loading branch information
@dhermes
dhermes committed Dec 23, 2015
1 parent b2741d6 commit 8c11909
Show file tree
Hide file tree
Showing 4 changed files with 127 additions and 131 deletions.
56 changes: 1 addition & 55 deletions gcloud_bigtable/_helpers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

"""Utility methods for gcloud_bigtable.
"""Utility methods for gcloud Bigtable.
Primarily includes helpers for dealing with low-level
protobuf objects.
Expand All @@ -27,23 +27,9 @@

from grpc.beta import implementations

from gcloud_bigtable._generated import (
bigtable_cluster_service_messages_pb2 as messages_pb2)
from gcloud_bigtable._generated import duration_pb2


_TYPE_URL_BASE = 'type.googleapis.com/google.bigtable.'
_ADMIN_TYPE_URL_BASE = _TYPE_URL_BASE + 'admin.cluster.v1.'
_CLUSTER_TYPE_URL = _ADMIN_TYPE_URL_BASE + 'Cluster'
_CLUSTER_CREATE_METADATA = _ADMIN_TYPE_URL_BASE + 'CreateClusterMetadata'
_TYPE_URL_MAP = {
_CLUSTER_CREATE_METADATA: messages_pb2.CreateClusterMetadata,
_ADMIN_TYPE_URL_BASE + 'UndeleteClusterMetadata': (
messages_pb2.UndeleteClusterMetadata),
_ADMIN_TYPE_URL_BASE + 'UpdateClusterMetadata': (
messages_pb2.UpdateClusterMetadata),
}

EPOCH = datetime.datetime.utcfromtimestamp(0).replace(tzinfo=pytz.utc)
# See https://gist.github.com/dhermes/bbc5b7be1932bfffae77
# for appropriate values on other systems.
Expand Down Expand Up @@ -82,46 +68,6 @@ class AuthInfo(object):
ROOT_CERTIFICATES = None


def _pb_timestamp_to_datetime(timestamp):
"""Convert a Timestamp protobuf to a datetime object.
:type timestamp: :class:`._generated.timestamp_pb2.Timestamp`
:param timestamp: A Google returned timestamp protobuf.
:rtype: :class:`datetime.datetime`
:returns: A UTC datetime object converted from a protobuf timestamp.
"""
return (
EPOCH +
datetime.timedelta(
seconds=timestamp.seconds,
microseconds=(timestamp.nanos / 1000.0),
)
)


def _parse_pb_any_to_native(any_val, expected_type=None):
"""Convert a serialized "google.protobuf.Any" value to actual type.
:type any_val: :class:`gcloud_bigtable._generated.any_pb2.Any`
:param any_val: A serialized protobuf value container.
:type expected_type: str
:param expected_type: (Optional) The type URL we expect ``any_val``
to have.
:rtype: object
:returns: The de-serialized object.
:raises: :class:`ValueError <exceptions.ValueError>` if the
``expected_type`` does not match the ``type_url`` on the input.
"""
if expected_type is not None and expected_type != any_val.type_url:
raise ValueError('Expected type: %s, Received: %s' % (
expected_type, any_val.type_url))
container_class = _TYPE_URL_MAP[any_val.type_url]
return container_class.FromString(any_val.value)


def _timedelta_to_duration_pb(timedelta_val):
"""Convert a Python timedelta object to a duration protobuf.
Expand Down
54 changes: 52 additions & 2 deletions gcloud_bigtable/cluster.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
"""User friendly container for Google Cloud Bigtable Cluster."""


import datetime
import re

from gcloud_bigtable._generated import bigtable_cluster_data_pb2 as data_pb2
Expand All @@ -23,8 +24,7 @@
from gcloud_bigtable._generated import (
bigtable_table_service_messages_pb2 as table_messages_pb2)
from gcloud_bigtable._generated import operations_pb2
from gcloud_bigtable._helpers import _parse_pb_any_to_native
from gcloud_bigtable._helpers import _pb_timestamp_to_datetime
from gcloud_bigtable._helpers import EPOCH as _EPOCH
from gcloud_bigtable.table import Table


Expand All @@ -35,6 +35,16 @@
r'clusters/([a-z][-a-z0-9]*)/operations/'
r'(?P<operation_id>\d+)$')
_DEFAULT_SERVE_NODES = 3
_TYPE_URL_BASE = 'type.googleapis.com/google.bigtable.'
_ADMIN_TYPE_URL_BASE = _TYPE_URL_BASE + 'admin.cluster.v1.'
_CLUSTER_CREATE_METADATA = _ADMIN_TYPE_URL_BASE + 'CreateClusterMetadata'
_UPDATE_CREATE_METADATA = _ADMIN_TYPE_URL_BASE + 'UpdateClusterMetadata'
_UNDELETE_CREATE_METADATA = _ADMIN_TYPE_URL_BASE + 'UndeleteClusterMetadata'
_TYPE_URL_MAP = {
_CLUSTER_CREATE_METADATA: messages_pb2.CreateClusterMetadata,
_UPDATE_CREATE_METADATA: messages_pb2.UpdateClusterMetadata,
_UNDELETE_CREATE_METADATA: messages_pb2.UndeleteClusterMetadata,
}


def _get_pb_property_value(message_pb, property_name):
Expand Down Expand Up @@ -82,6 +92,46 @@ def _prepare_create_request(cluster):
)


def _pb_timestamp_to_datetime(timestamp):
"""Convert a Timestamp protobuf to a datetime object.
:type timestamp: :class:`._generated.timestamp_pb2.Timestamp`
:param timestamp: A Google returned timestamp protobuf.
:rtype: :class:`datetime.datetime`
:returns: A UTC datetime object converted from a protobuf timestamp.
"""
return (
_EPOCH +
datetime.timedelta(
seconds=timestamp.seconds,
microseconds=(timestamp.nanos / 1000.0),
)
)


def _parse_pb_any_to_native(any_val, expected_type=None):
"""Convert a serialized "google.protobuf.Any" value to actual type.
:type any_val: :class:`gcloud.bigtable._generated.any_pb2.Any`
:param any_val: A serialized protobuf value container.
:type expected_type: str
:param expected_type: (Optional) The type URL we expect ``any_val``
to have.
:rtype: object
:returns: The de-serialized object.
:raises: :class:`ValueError <exceptions.ValueError>` if the
``expected_type`` does not match the ``type_url`` on the input.
"""
if expected_type is not None and expected_type != any_val.type_url:
raise ValueError('Expected type: %s, Received: %s' % (
expected_type, any_val.type_url))
container_class = _TYPE_URL_MAP[any_val.type_url]
return container_class.FromString(any_val.value)


def _process_operation(operation_pb):
"""Processes a create protobuf response.
Expand Down
74 changes: 0 additions & 74 deletions gcloud_bigtable/test__helpers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,80 +73,6 @@ class _ReturnVal(object):
self.assertEqual(scoped_creds._called, [('get_access_token', (), {})])


class Test__pb_timestamp_to_datetime(unittest2.TestCase):

def _callFUT(self, timestamp):
from gcloud_bigtable._helpers import _pb_timestamp_to_datetime
return _pb_timestamp_to_datetime(timestamp)

def test_it(self):
import datetime
import pytz
from gcloud_bigtable._generated.timestamp_pb2 import Timestamp

# Epoch is midnight on January 1, 1970 ...
dt_stamp = datetime.datetime(1970, month=1, day=1, hour=0,
minute=1, second=1, microsecond=1234,
tzinfo=pytz.utc)
# ... so 1 minute and 1 second after is 61 seconds and 1234
# microseconds is 1234000 nanoseconds.
timestamp = Timestamp(seconds=61, nanos=1234000)
self.assertEqual(self._callFUT(timestamp), dt_stamp)


class Test__parse_pb_any_to_native(unittest2.TestCase):

def _callFUT(self, any_val, expected_type=None):
from gcloud_bigtable._helpers import _parse_pb_any_to_native
return _parse_pb_any_to_native(any_val, expected_type=expected_type)

def test_it(self):
from gcloud_bigtable._generated import any_pb2
from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2
from gcloud_bigtable._testing import _Monkey
from gcloud_bigtable import _helpers as MUT

type_url = 'type.googleapis.com/' + data_pb2._CELL.full_name
fake_type_url_map = {type_url: data_pb2.Cell}

cell = data_pb2.Cell(
timestamp_micros=0,
value=b'foobar',
)
any_val = any_pb2.Any(
type_url=type_url,
value=cell.SerializeToString(),
)
with _Monkey(MUT, _TYPE_URL_MAP=fake_type_url_map):
result = self._callFUT(any_val)

self.assertEqual(result, cell)

def test_unknown_type_url(self):
from gcloud_bigtable._generated import any_pb2
from gcloud_bigtable._testing import _Monkey
from gcloud_bigtable import _helpers as MUT

fake_type_url_map = {}
any_val = any_pb2.Any()
with _Monkey(MUT, _TYPE_URL_MAP=fake_type_url_map):
with self.assertRaises(KeyError):
self._callFUT(any_val)

def test_disagreeing_type_url(self):
from gcloud_bigtable._generated import any_pb2
from gcloud_bigtable._testing import _Monkey
from gcloud_bigtable import _helpers as MUT

type_url1 = 'foo'
type_url2 = 'bar'
fake_type_url_map = {type_url1: None}
any_val = any_pb2.Any(type_url=type_url2)
with _Monkey(MUT, _TYPE_URL_MAP=fake_type_url_map):
with self.assertRaises(ValueError):
self._callFUT(any_val, expected_type=type_url1)


class Test__timedelta_to_duration_pb(unittest2.TestCase):

def _callFUT(self, timedelta_val):
Expand Down
74 changes: 74 additions & 0 deletions gcloud_bigtable/test_cluster.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -665,6 +665,80 @@ def test_list_tables_failure_name_bad_before(self):
self._list_tables_helper(table_id, table_name=bad_table_name)


class Test__pb_timestamp_to_datetime(unittest2.TestCase):

def _callFUT(self, timestamp):
from gcloud_bigtable.cluster import _pb_timestamp_to_datetime
return _pb_timestamp_to_datetime(timestamp)

def test_it(self):
import datetime
import pytz
from gcloud_bigtable._generated.timestamp_pb2 import Timestamp

# Epoch is midnight on January 1, 1970 ...
dt_stamp = datetime.datetime(1970, month=1, day=1, hour=0,
minute=1, second=1, microsecond=1234,
tzinfo=pytz.utc)
# ... so 1 minute and 1 second after is 61 seconds and 1234
# microseconds is 1234000 nanoseconds.
timestamp = Timestamp(seconds=61, nanos=1234000)
self.assertEqual(self._callFUT(timestamp), dt_stamp)


class Test__parse_pb_any_to_native(unittest2.TestCase):

def _callFUT(self, any_val, expected_type=None):
from gcloud_bigtable.cluster import _parse_pb_any_to_native
return _parse_pb_any_to_native(any_val, expected_type=expected_type)

def test_it(self):
from gcloud_bigtable._generated import any_pb2
from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2
from gcloud_bigtable._testing import _Monkey
from gcloud_bigtable import cluster as MUT

type_url = 'type.googleapis.com/' + data_pb2._CELL.full_name
fake_type_url_map = {type_url: data_pb2.Cell}

cell = data_pb2.Cell(
timestamp_micros=0,
value=b'foobar',
)
any_val = any_pb2.Any(
type_url=type_url,
value=cell.SerializeToString(),
)
with _Monkey(MUT, _TYPE_URL_MAP=fake_type_url_map):
result = self._callFUT(any_val)

self.assertEqual(result, cell)

def test_unknown_type_url(self):
from gcloud_bigtable._generated import any_pb2
from gcloud_bigtable._testing import _Monkey
from gcloud_bigtable import cluster as MUT

fake_type_url_map = {}
any_val = any_pb2.Any()
with _Monkey(MUT, _TYPE_URL_MAP=fake_type_url_map):
with self.assertRaises(KeyError):
self._callFUT(any_val)

def test_disagreeing_type_url(self):
from gcloud_bigtable._generated import any_pb2
from gcloud_bigtable._testing import _Monkey
from gcloud_bigtable import cluster as MUT

type_url1 = 'foo'
type_url2 = 'bar'
fake_type_url_map = {type_url1: None}
any_val = any_pb2.Any(type_url=type_url2)
with _Monkey(MUT, _TYPE_URL_MAP=fake_type_url_map):
with self.assertRaises(ValueError):
self._callFUT(any_val, expected_type=type_url1)


class _Client(object):

cluster_stub = None
Expand Down

0 comments on commit 8c11909

Please sign in to comment.