Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature: Add cuboids_from_id functionality #28

Open
wants to merge 6 commits into
base: integration
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -69,4 +69,3 @@ c_lib/c_version/makefile

# Vim files
*.swp

39 changes: 39 additions & 0 deletions spdb/spatialdb/object.py
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,26 @@ def get_ids_in_region(
"""
raise NotImplemented

@abstractmethod
def get_cuboids_from_id(self, resource, resolution, id, version=0):
"""
Method to get cuboids that belong to a particular ID.

Args:
resource (project.BossResource): Data model info based on the request or target resource
resolution (int): the resolution level
id (int): id of interest.
version (optional[int]): Reserved for future use. Defaults to 0

Returns:
(dict) : Extents of cuboids containing ID in XYZ order. Example output:
{'cuboids': [
[(0, 512), (0, 512), (0,16)],
[(512, 1024), (0, 512), (16,32)]
]}
"""
raise NotImplemented


class AWSObjectStore(ObjectStore):
def __init__(self, conf):
Expand Down Expand Up @@ -829,6 +849,25 @@ def get_ids_in_region(
ids_as_str = ['%d' % n for n in id_set]

return { 'ids': ids_as_str }

def get_cuboids_from_id(self, resource, resolution, id, version=0):
"""
Method to get cuboids that belong to a particular ID.

Args:
resource (project.BossResource): Data model info based on the request or target resource
resolution (int): the resolution level
id (int): id of interest.
version (optional[int]): Reserved for future use. Defaults to 0

Returns:
(dict) : Extents of cuboids containing ID in XYZ order. Example output:
{'cuboids': [
[(0, 512), (0, 512), (0,16)],
[(512, 1024), (0, 512), (16,32)]
]}
"""
return self.obj_ind.get_cuboids_from_id(resource, resolution, id, version)

def _get_ids_from_cutout(
self, cutout_fcn, resource, resolution, corner, extent,
Expand Down
39 changes: 39 additions & 0 deletions spdb/spatialdb/object_indices.py
Original file line number Diff line number Diff line change
Expand Up @@ -1072,6 +1072,45 @@ def update_last_partition_key(self, key, chunk_num, version=0):
raise SpdbError("Failed to update {} for ID: {}".format(LAST_PARTITION_KEY, key),
ErrorCodes.OBJECT_STORE_ERROR)

def get_cuboids_from_id(self, resource, resolution, id, version=0):
"""
Get a list of cuboid bounds that contains the object labeled with id.

Args:
resource (project.BossResource): Data model info based on the request or target resource
resolution (int): the resolution level
id (uint64|string): object's id
version (optional[int]): Version - reserved for future use.
Returns:
(dict|None) : Extents of cuboids in XYZ containing ID or None if the id is not found.

Raises:
(SpdbError): Can't talk to id index database or database corrupt.
"""

cuboid_list = []

[x_cube_dim, y_cube_dim, z_cube_dim] = CUBOIDSIZE[resolution]
obj_keys = self.get_cuboids(resource, resolution, id)

if len(obj_keys) == 0:
return None

for key in obj_keys:
key_parts = AWSObjectStore.get_object_key_parts(key)
xyz = MortonXYZ(key_parts.morton_id)
x = xyz[0] * x_cube_dim
y = xyz[1] * y_cube_dim
z = xyz[2] * z_cube_dim
cuboid_list.append([
(x, x+x_cube_dim),
(y, y+y_cube_dim,
(z, z+z_cube_dim))
])

return {
'cuboids': cuboid_list
}


# Import statement at the bottom to avoid problems with circular imports.
Expand Down
16 changes: 16 additions & 0 deletions spdb/spatialdb/spatialdb.py
Original file line number Diff line number Diff line change
Expand Up @@ -964,4 +964,20 @@ def reserve_ids(self, resource, num_ids, version=0):
"""
return self.objectio.reserve_ids(resource, num_ids, version)

def get_cuboids_from_id(self, resource, resolution, id, version=0):
"""Method to get cuboid indicies for a particular ID

Args:
resource (spdb.project.resource.BossResource): Data model info based on the request or target resource.
resolution (int): the resolution level
id (uint64|string): object's id
version (optional[int]): Defaults to zero, reserved for future use.

Returns:
(dict) : Extents of cuboids containing ID in XYZ order. Example output:
{'cuboids': [
[(0, 512), (0, 512), (0,16)],
[(512, 1024), (0, 512), (16,32)]
]}
"""
return self.objectio.get_cuboids_from_id(resource, resolution, int(id), version)
44 changes: 44 additions & 0 deletions spdb/spatialdb/test/int_test_object_indices.py
Original file line number Diff line number Diff line change
Expand Up @@ -395,6 +395,50 @@ def test_get_loose_bounding_box(self):
}
self.assertEqual(expected, actual)

@unittest.skip('Skipping - currently indexing disabled')
def test_cuboids_from_id(self):
id = 33333
resolution = 0
time_sample = 0
version = 0

[x_cube_dim, y_cube_dim, z_cube_dim] = CUBOIDSIZE[resolution]

pos0 = [4, 4, 4]
pos1 = [2, 1, 3]

bytes0 = np.zeros(10, dtype='uint64')
bytes0[1] = id
morton_id0 = XYZMorton(pos0)
key0 = AWSObjectStore.generate_object_key(
self.resource, resolution, time_sample, morton_id0)

bytes1 = np.zeros(4, dtype='uint64')
bytes1[0] = id # Pre-existing id.
morton_id1 = XYZMorton(pos1)
key1 = AWSObjectStore.generate_object_key(
self.resource, resolution, time_sample, morton_id1)

self.obj_ind.update_id_indices(
self.resource, resolution, [key0, key1], [bytes0, bytes1], version)

actual = self.obj_ind.cuboids_from_id(self.resource, resolution, id)
expected = {
'cuboids': [
[
(pos0[0]*x_cube_dim, (pos0[0]+1)*x_cube_dim),
(pos0[1]*y_cube_dim, (pos0[1]+1)*y_cube_dim),
(pos0[2]*z_cube_dim, (pos0[2]+1)*z_cube_dim)
],
[
(pos1[0]*x_cube_dim, (pos1[0]+1)*x_cube_dim),
(pos1[1]*y_cube_dim, (pos1[1]+1)*y_cube_dim),
(pos1[2]*z_cube_dim, (pos1[2]+1)*z_cube_dim)
]
]
}
self.assertEqual(expected, actual)

def test_reserve_id_init(self):
start_id = self.obj_ind.reserve_ids(self.resource, 10)
self.assertEqual(start_id, 1)
Expand Down
55 changes: 55 additions & 0 deletions spdb/spatialdb/test/test_object_indices.py
Original file line number Diff line number Diff line change
Expand Up @@ -943,6 +943,61 @@ def test_write_cuboid_dynamo_no_revision_id(self):
self.obj_ind.write_cuboid_dynamo(
morton, chan_key, rev_id, lookup_key, version)

def test_cuboids_from_id(self):
resolution = 0
time_sample = 0

[x_cube_dim, y_cube_dim, z_cube_dim] = CUBOIDSIZE[resolution]

pos0 = [4, 4, 4]
pos1 = [2, 1, 3]

mort0 = XYZMorton(pos0)
mort1 = XYZMorton(pos1)


key0 = AWSObjectStore.generate_object_key(self.resource, resolution, time_sample, mort0)
key1 = AWSObjectStore.generate_object_key(self.resource, resolution, time_sample, mort1)

id = 2234

with patch.object(self.obj_ind, 'get_cuboids') as fake_get_cuboids:
fake_get_cuboids.return_value = [key0, key1]

# Method under test.
actual = self.obj_ind.cuboids_from_id(self.resource, resolution, id)

expected = {
'cuboids': [
[
(pos0[0]*x_cube_dim, (pos0[0]+1)*x_cube_dim),
(pos0[1]*y_cube_dim, (pos0[1]+1)*y_cube_dim),
(pos0[2]*z_cube_dim, (pos0[2]+1)*z_cube_dim)
],
[
(pos1[0]*x_cube_dim, (pos1[0]+1)*x_cube_dim),
(pos1[1]*y_cube_dim, (pos1[1]+1)*y_cube_dim),
(pos1[2]*z_cube_dim, (pos1[2]+1)*z_cube_dim)
]
]
}
self.assertEqual(expected, actual)

def test_cuboids_from_id_not_found(self):
"""Make sure None returned if id is not in channel."""
resolution = 0
time_sample = 0
id = 2234

with patch.object(self.obj_ind, 'get_cuboids') as fake_get_cuboids:
fake_get_cuboids.return_value = []

actual = self.obj_ind.cuboids_from_id(
self.resource, resolution, id)

expected = None
self.assertEqual(expected, actual)


class TestObjectIndices(ObjectIndicesTestMixin, unittest.TestCase):
@classmethod
Expand Down