From f7bb4761bb7906b7dee3e6ad53ea6ed7c7c56ce6 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Fri, 15 Jun 2018 14:19:44 +0300 Subject: [PATCH 1/9] More renaming --- README.md | 6 +- docs/data-model-queries.md | 36 +-- docs/imports.md | 6 +- docs/server-interaction.md | 10 +- integration_tests/test_client_it.py | 184 ++++++++------- pilosa/client.py | 83 +++---- pilosa/exceptions.py | 4 +- pilosa/orm.py | 333 ++++++++++------------------ pilosa/validator.py | 6 +- tests/test_client.py | 13 +- tests/test_orm.py | 191 +++++++--------- tests/test_validator.py | 8 +- 12 files changed, 379 insertions(+), 501 deletions(-) diff --git a/README.md b/README.md index 08a2d7b..d426ae4 100644 --- a/README.md +++ b/README.md @@ -44,10 +44,10 @@ schema = client.schema() # Create an Index object myindex = schema.index("myindex") -# Create a Frame object -myframe = myindex.frame("myframe") +# Create a Field object +myframe = myindex.field("myframe") -# make sure the index and frame exists on the server +# make sure the index and field exists on the server client.sync_schema(schema) # Send a SetBit query. PilosaError is thrown if execution of the query fails. diff --git a/docs/data-model-queries.md b/docs/data-model-queries.md index ee82d6e..23170cc 100644 --- a/docs/data-model-queries.md +++ b/docs/data-model-queries.md @@ -2,7 +2,7 @@ ## Indexes and Frames -*Index* and *frame*s are the main data models of Pilosa. You can check the [Pilosa documentation](https://www.pilosa.com/docs) for more detail about the data model. +*Index* and *field*s are the main data models of Pilosa. You can check the [Pilosa documentation](https://www.pilosa.com/docs) for more detail about the data model. `schema.index` method is used to create an index object. Note that this does not create an index on the server; the index object simply defines the schema. @@ -11,26 +11,26 @@ schema = pilosa.Schema() repository = schema.index("repository") ``` -Frames are created with a call to `index.frame` method: +Frames are created with a call to `index.field` method: ```python -stargazer = repository.frame("stargazer") +stargazer = repository.field("stargazer") ``` -Similar to index objects, you can pass custom options to the `index.frame` method: +Similar to index objects, you can pass custom options to the `index.field` method: ```python -stargazer = repository.frame("stargazer", time_quantum=pilosa.TimeQuantum.YEAR_MONTH_DAY) +stargazer = repository.field("stargazer", time_quantum=pilosa.TimeQuantum.YEAR_MONTH_DAY) ``` ## Queries -Once you have indexes and frame objects created, you can create queries for them. Some of the queries work on the columns; corresponding methods are attached to the index. Other queries work on rows, with related methods attached to frames. +Once you have indexes and field objects created, you can create queries for them. Some of the queries work on the columns; corresponding methods are attached to the index. Other queries work on rows, with related methods attached to frames. -For instance, `Bitmap` queries work on rows; use a frame object to create those queries: +For instance, `Bitmap` queries work on rows; use a field object to create those queries: ```python -bitmap_query = stargazer.bitmap(1) # corresponds to PQL: Bitmap(frame='stargazer', row=1) +bitmap_query = stargazer.bitmap(1) # corresponds to PQL: Bitmap(field='stargazer', row=1) ``` `Union` queries work on columns; use the index object to create them: @@ -47,25 +47,25 @@ query = repository.batch_query( repository.union(stargazer.bitmap(100), stargazer.bitmap(5))) ``` -The recommended way of creating query objects is, using dedicated methods attached to index and frame objects. But sometimes it would be desirable to send raw queries to Pilosa. You can use the `index.raw_query` method for that. Note that, query string is not validated before sending to the server: +The recommended way of creating query objects is, using dedicated methods attached to index and field objects. But sometimes it would be desirable to send raw queries to Pilosa. You can use the `index.raw_query` method for that. Note that, query string is not validated before sending to the server: ```python -query = repository.raw_query("Bitmap(frame='stargazer', row=5)") +query = repository.raw_query("Bitmap(field='stargazer', row=5)") ``` This client supports [Range encoded fields](https://www.pilosa.com/docs/latest/query-language/#range-bsi). Read [Range Encoded Bitmaps](https://www.pilosa.com/blog/range-encoded-bitmaps/) blog post for more information about the BSI implementation of range encoding in Pilosa. -In order to use range encoded fields, a frame should be created with one or more integer fields. Each field should have their minimums and maximums set. Here's how you would do that using this library: +In order to use range encoded fields, a field should be created with one or more integer fields. Each field should have their minimums and maximums set. Here's how you would do that using this library: ```python index = schema.index("animals") -frame = index.frame("traits", fields=[pilosa.IntField.int("captivity", min=0, max=956)]) +field = index.field("traits", fields=[pilosa.IntField.int("captivity", min=0, max=956)]) client.sync_schema(schema) ``` -If the frame with the necessary field already exists on the server, you don't need to create the field instance, `client.syncSchema(schema)` would load that to `schema`. You can then add some data: +If the field with the necessary field already exists on the server, you don't need to create the field instance, `client.syncSchema(schema)` would load that to `schema`. You can then add some data: ```python # Add the captivity values to the field. -captivity = frame.field("captivity") +captivity = field.field("captivity") data = [3, 392, 47, 956, 219, 14, 47, 504, 21, 0, 123, 318] query = index.batch_query() for i, x in enumerate(data): @@ -89,11 +89,11 @@ It's possible to pass a bitmap query to `sum`, so only columns where a row is se ```python # Let's run a few setbit queries first client.query(index.batch_query( - frame.setbit(42, 1), - frame.setbit(42, 6) + field.setbit(42, 1), + field.setbit(42, 6) )) # Query for the total number of animals in captivity where row 42 is set -response = client.query(captivity.sum(frame.bitmap(42))) +response = client.query(captivity.sum(field.bitmap(42))) print(response.result.value) ``` @@ -110,7 +110,7 @@ Index: * `set_column_attrs(self, column_id, attrs)` * `xor(self, *bitmaps)` -Frame: +Field: * `bitmap(self, row_id)` * `setbit(self, row_id, column_id, timestamp=None)` diff --git a/docs/imports.md b/docs/imports.md index daec003..3469414 100644 --- a/docs/imports.md +++ b/docs/imports.md @@ -13,7 +13,7 @@ ROW_ID,COLUMN_ID,TIMESTAMP ``` Note that, each line corresponds to a single bit and the lines end with a new line (`\n` or `\r\n`). -The target index and frame must have been created before hand. +The target index and field must have been created before hand. Here's some sample code: ```python @@ -37,7 +37,7 @@ reader = csv_bit_reader(StringIO(text)) client = pilosa.Client() schema = client.schema() index = schema.index("sample-index") -frame = index.frame("sample-frame", time_quantum=pilosa.TimeQuantum.YEAR_MONTH_DAY_HOUR) +field = index.field("sample-field", time_quantum=pilosa.TimeQuantum.YEAR_MONTH_DAY_HOUR) client.sync_schema(schema) -client.import_frame(frame, reader) +client.import_field(field, reader) ``` diff --git a/docs/server-interaction.md b/docs/server-interaction.md index 6ece81f..cba4324 100644 --- a/docs/server-interaction.md +++ b/docs/server-interaction.md @@ -72,25 +72,25 @@ client = pilosa.Client(cluster, Once you create a client, you can create indexes, frames and start sending queries. -Here is how you would create a index and frame: +Here is how you would create a index and field: ```python schema = client.schema() index = schema.index("repository") -frame = index.frame("stargazer") +field = index.field("stargazer") client.sync_schema(schema) ``` You can send queries to a Pilosa server using the `query` method of client objects: ```python -response = client.query(frame.bitmap(5)) +response = client.query(field.bitmap(5)) ``` `query` method accepts optional `columns` argument: ```python -response = client.query(frame.bitmap(5), +response = client.query(field.bitmap(5), columns=True # return column data in the response ) ``` @@ -102,7 +102,7 @@ When a query is sent to a Pilosa server, the server either fulfills the query or A `QueryResponse` object may contain zero or more results of `QueryResult` type. You can access all results using the `results` property of `QueryResponse` (which returns a list of `QueryResult` objects) or you can use the `result` property (which returns either the first result or `None` if there are no results): ```python -response = client.query(frame.bitmap(5)) +response = client.query(field.bitmap(5)) # check that there's a result and act on it result = response.result diff --git a/integration_tests/test_client_it.py b/integration_tests/test_client_it.py index 10ba5e1..4588656 100644 --- a/integration_tests/test_client_it.py +++ b/integration_tests/test_client_it.py @@ -42,7 +42,7 @@ from pilosa.client import Client, URI, Cluster, PilosaServerError from pilosa.exceptions import PilosaError -from pilosa.orm import Index, TimeQuantum, Schema, IntField, CacheType +from pilosa.orm import Index, TimeQuantum, Schema, CacheType from pilosa.imports import csv_bit_reader @@ -54,13 +54,13 @@ def setUp(self): schema = Schema() self.index = schema.index(self.random_index_name()) client = self.get_client() - self.index.frame("another-frame") - self.index.frame("test") - self.index.frame("count-test") - self.index.frame("topn_test") + self.index.field("another-field") + self.index.field("test") + self.index.field("count-test") + self.index.field("topn_test") self.col_index = schema.index(self.index.name + "-opts") - self.frame = self.col_index.frame("collab") + self.field = self.col_index.field("collab") client.sync_schema(schema) def tearDown(self): @@ -68,63 +68,63 @@ def tearDown(self): client.delete_index(self.index) client.delete_index(self.col_index) - def test_create_frame_with_time_quantum(self): - frame = self.index.frame("frame-with-timequantum", time_quantum=TimeQuantum.YEAR) + def test_create_field_with_time_quantum(self): + field = self.index.field("field-with-timequantum", time_quantum=TimeQuantum.YEAR) client = self.get_client() - client.ensure_frame(frame) + client.ensure_field(field) schema = client.schema() - # Check the frame time quantum + # Check the field time quantum index = schema._indexes[self.index.name] - frame = index._frames["frame-with-timequantum"] - self.assertEquals(TimeQuantum.YEAR.value, frame.time_quantum.value) + field = index._fields["field-with-timequantum"] + self.assertEquals(TimeQuantum.YEAR.value, field.time_quantum.value) def test_query(self): client = self.get_client() - frame = self.index.frame("query-test") - client.ensure_frame(frame) - response = client.query(frame.setbit(555, 10)) + field = self.index.field("query-test") + client.ensure_field(field) + response = client.query(field.setbit(555, 10)) self.assertTrue(response.result is not None) def test_query_with_columns(self): client = self.get_client() - frame = self.index.frame("query-test") - client.ensure_frame(frame) - client.query(frame.setbit(100, 1000)) + field = self.index.field("query-test") + client.ensure_field(field) + client.query(field.setbit(100, 1000)) column_attrs = {"name": "bombo"} client.query(self.index.set_column_attrs(1000, column_attrs)) - response = client.query(frame.bitmap(100), columns=True) + response = client.query(field.bitmap(100), columns=True) self.assertTrue(response is not None) self.assertEquals(1000, response.column.id) self.assertEquals({"name": "bombo"}, response.column.attributes) - response = client.query(frame.bitmap(300)) + response = client.query(field.bitmap(300)) self.assertTrue(response.column is None) def test_failed_connection(self): client = Client("http://non-existent-sub.pilosa.com:22222") - self.assertRaises(PilosaError, client.query, self.frame.setbit(15, 10)) + self.assertRaises(PilosaError, client.query, self.field.setbit(15, 10)) def test_parse_error(self): client = self.get_client() - q = self.index.raw_query("SetBit(id=5, frame=\"test\", col_id:=10)") + q = self.index.raw_query("SetBit(id=5, field=\"test\", col_id:=10)") self.assertRaises(PilosaError, client.query, q) def test_orm_count(self): client = self.get_client() - count_frame = self.index.frame("count-test") - client.ensure_frame(count_frame) + count_field = self.index.field("count-test") + client.ensure_field(count_field) qry = self.index.batch_query( - count_frame.setbit(10, 20), - count_frame.setbit(10, 21), - count_frame.setbit(15, 25)) + count_field.setbit(10, 20), + count_field.setbit(10, 21), + count_field.setbit(15, 25)) client.query(qry) - response = client.query(self.index.count(count_frame.bitmap(10))) + response = client.query(self.index.count(count_field.bitmap(10))) self.assertEquals(2, response.result.count) def test_new_orm(self): client = self.get_client() - client.query(self.frame.setbit(10, 20)) - response1 = client.query(self.frame.bitmap(10)) + client.query(self.field.setbit(10, 20)) + response1 = client.query(self.field.bitmap(10)) self.assertEquals(0, len(response1.columns)) bitmap1 = response1.result.bitmap self.assertEquals(0, len(bitmap1.attributes)) @@ -133,7 +133,7 @@ def test_new_orm(self): column_attrs = {"name": "bombo"} client.query(self.col_index.set_column_attrs(20, column_attrs)) - response2 = client.query(self.frame.bitmap(10), columns=True) + response2 = client.query(self.field.bitmap(10), columns=True) column = response2.column self.assertTrue(column is not None) self.assertEquals(20, column.id) @@ -144,8 +144,8 @@ def test_new_orm(self): "height": 1.81, "name": "Mr. Pi" } - client.query(self.frame.set_row_attrs(10, bitmap_attrs)) - response3 = client.query(self.frame.bitmap(10)) + client.query(self.field.set_row_attrs(10, bitmap_attrs)) + response3 = client.query(self.field.bitmap(10)) bitmap = response3.result.bitmap self.assertEquals(1, len(bitmap.bits)) self.assertEquals(4, len(bitmap.attributes)) @@ -156,16 +156,16 @@ def test_new_orm(self): def test_topn(self): client = self.get_client() - frame = self.index.frame("topn_test") + field = self.index.field("topn_test") client.query(self.index.batch_query( - frame.setbit(10, 5), - frame.setbit(10, 10), - frame.setbit(10, 15), - frame.setbit(20, 5), - frame.setbit(30, 5))) + field.setbit(10, 5), + field.setbit(10, 10), + field.setbit(10, 15), + field.setbit(20, 5), + field.setbit(30, 5))) # XXX: The following is required to make this test pass. See: https://github.com/pilosa/pilosa/issues/625 client.http_request("POST", "/recalculate-caches") - response4 = client.query(frame.topn(2)) + response4 = client.query(field.topn(2)) items = response4.result.count_items self.assertEquals(2, len(items)) item = items[0] @@ -176,23 +176,23 @@ def test_ensure_index_exists(self): client = self.get_client() index = Index(self.index.name + "-ensure") client.ensure_index(index) - client.create_frame(index.frame("frm")) + client.create_field(index.field("frm")) client.ensure_index(index) client.delete_index(index) - def test_delete_frame(self): + def test_delete_field(self): client = self.get_client() - frame = self.index.frame("to-delete") - client.ensure_frame(frame) - client.delete_frame(frame) + field = self.index.field("to-delete") + client.ensure_field(field) + client.delete_field(field) # the following should succeed - client.create_frame(frame) + client.create_field(field) - def test_frame_for_nonexisting_index(self): + def test_field_for_nonexisting_index(self): client = self.get_client() index = Index("non-existing-database") - frame = index.frame("frm") - self.assertRaises(PilosaServerError, client.create_frame, frame) + field = index.field("frm") + self.assertRaises(PilosaServerError, client.create_field, field) def test_csv_import(self): client = self.get_client() @@ -203,13 +203,13 @@ def test_csv_import(self): 7, 1 """ reader = csv_bit_reader(StringIO(text)) - frame = self.index.frame("importframe") - client.ensure_frame(frame) - client.import_frame(frame, reader) + field = self.index.field("importfield") + client.ensure_field(field) + client.import_field(field, reader) bq = self.index.batch_query( - frame.bitmap(2), - frame.bitmap(7), - frame.bitmap(10), + field.bitmap(2), + field.bitmap(7), + field.bitmap(10), ) response = client.query(bq) target = [3, 1, 5] @@ -227,52 +227,48 @@ def test_csv_import2(self): reader = csv_bit_reader(StringIO(text)) client = self.get_client() schema = client.schema() - frame = schema.index(self.index.name).frame("importframe", time_quantum=TimeQuantum.YEAR_MONTH_DAY_HOUR) + field = schema.index(self.index.name).field("importfield", time_quantum=TimeQuantum.YEAR_MONTH_DAY_HOUR) client.sync_schema(schema) - client.import_frame(frame, reader) + client.import_field(field, reader) def test_schema(self): client = self.get_client() schema = client.schema() self.assertGreaterEqual(len(schema._indexes), 1) - self.assertGreaterEqual(len(list(schema._indexes.values())[0]._frames), 1) - frame = self.index.frame("schema-test-frame", + self.assertGreaterEqual(len(list(schema._indexes.values())[0]._fields), 1) + field = self.index.field("schema-test-field", cache_type=CacheType.LRU, - cache_size=9999, - inverse_enabled=True, - time_quantum=TimeQuantum.YEAR_MONTH_DAY) - client.ensure_frame(frame) + cache_size=9999) + client.ensure_field(field) schema = client.schema() - f = schema._indexes[self.index.name]._frames["schema-test-frame"] + f = schema._indexes[self.index.name]._fields["schema-test-field"] self.assertEquals(CacheType.LRU, f.cache_type) self.assertEquals(9999, f.cache_size) - self.assertEquals(True, f.inverse_enabled) - self.assertEquals(TimeQuantum.YEAR_MONTH_DAY, f.time_quantum) def test_sync(self): client = self.get_client() remote_index = Index("remote-index-1") - remote_frame = remote_index.frame("remote-frame-1") + remote_field = remote_index.field("remote-field-1") schema1 = Schema() index11 = schema1.index("diff-index1") - index11.frame("frame1-1") - index11.frame("frame1-2") + index11.field("field1-1") + index11.field("field1-2") index12 = schema1.index("diff-index2") - index12.frame("frame2-1") + index12.field("field2-1") schema1.index(remote_index.name) try: client.ensure_index(remote_index) - client.ensure_frame(remote_frame) + client.ensure_field(remote_field) client.sync_schema(schema1) # check that the schema was created schema2 = client.schema() self.assertTrue("remote-index-1" in schema2._indexes) - self.assertTrue("remote-frame-1" in schema2.index("remote-index-1")._frames) + self.assertTrue("remote-field-1" in schema2.index("remote-index-1")._fields) self.assertTrue("diff-index1" in schema2._indexes) - self.assertTrue("frame1-1" in schema2.index("diff-index1")._frames) - self.assertTrue("frame1-2" in schema2.index("diff-index1")._frames) + self.assertTrue("field1-1" in schema2.index("diff-index1")._fields) + self.assertTrue("field1-2" in schema2.index("diff-index1")._fields) self.assertTrue("diff-index2" in schema2._indexes) - self.assertTrue("frame2-1" in schema2.index("diff-index2")._frames) + self.assertTrue("field2-1" in schema2.index("diff-index2")._fields) finally: try: client.delete_index(remote_index) @@ -284,49 +280,47 @@ def test_sync(self): def test_failover_fail(self): uris = [URI.address("nonexistent%s" % i) for i in range(20)] client = Client(Cluster(*uris)) - self.assertRaises(PilosaError, client.query, self.frame.bitmap(5)) + self.assertRaises(PilosaError, client.query, self.field.bitmap(5)) - def test_range_frame(self): + def test_range_field(self): client = self.get_client() - frame = self.col_index.frame("rangeframe", fields=[IntField.int("foo", 10, 20)]) - client.ensure_frame(frame) - foo = frame.field("foo") + field = self.col_index.field("rangefield", int_min=10, int_max=20) + client.ensure_field(field) client.query(self.col_index.batch_query( - frame.setbit(1, 10), - frame.setbit(1, 100), - foo.set_value(10, 11), - foo.set_value(100, 15), + field.setbit(1, 10), + field.setbit(1, 100), + field.set_value(10, 11), )) - response = client.query(foo.sum(frame.bitmap(1))) - self.assertEquals(26, response.result.value) - self.assertEquals(2, response.result.count) + response = client.query(field.sum(field.bitmap(1))) + self.assertEquals(11, response.result.value) + self.assertEquals(1, response.result.count) - response = client.query(foo.min(frame.bitmap(1))) + response = client.query(field.min(field.bitmap(1))) self.assertEquals(11, response.result.value) self.assertEquals(1, response.result.count) - response = client.query(foo.max(frame.bitmap(1))) - self.assertEquals(15, response.result.value) + response = client.query(field.max(field.bitmap(1))) + self.assertEquals(11, response.result.value) self.assertEquals(1, response.result.count) - response = client.query(foo.lt(15)) + response = client.query(field.lt(15)) self.assertEquals(1, len(response.results)) self.assertEquals(10, response.result.bitmap.bits[0]) def test_exclude_attrs_bits(self): client = self.get_client() client.query(self.col_index.batch_query( - self.frame.setbit(1, 100), - self.frame.set_row_attrs(1, {"foo": "bar"}) + self.field.setbit(1, 100), + self.field.set_row_attrs(1, {"foo": "bar"}) )) # test exclude bits. - response = client.query(self.frame.bitmap(1), exclude_bits=True) + response = client.query(self.field.bitmap(1), exclude_bits=True) self.assertEquals(0, len(response.result.bitmap.bits)) self.assertEquals(1, len(response.result.bitmap.attributes)) # test exclude attributes. - response = client.query(self.frame.bitmap(1), exclude_attrs=True) + response = client.query(self.field.bitmap(1), exclude_attrs=True) self.assertEquals(1, len(response.result.bitmap.bits)) self.assertEquals(0, len(response.result.bitmap.attributes)) diff --git a/pilosa/client.py b/pilosa/client.py index ccfb22c..ab84b04 100644 --- a/pilosa/client.py +++ b/pilosa/client.py @@ -39,7 +39,7 @@ import urllib3 -from .exceptions import PilosaError, PilosaURIError, IndexExistsError, FrameExistsError +from .exceptions import PilosaError, PilosaURIError, IndexExistsError, FieldExistsError from .imports import batch_bits from .internal import public_pb2 as internal from .orm import TimeQuantum, Schema, CacheType @@ -67,7 +67,7 @@ class Client(object): # Create an Index instance index = pilosa.Index("repository") - stargazer = index.frame("stargazer") + stargazer = index.field("stargazer") response = client.query(stargazer.bitmap(5)) # Act on the result @@ -145,29 +145,29 @@ def delete_index(self, index): path = "/index/%s" % index.name self.__http_request("DELETE", path) - def create_frame(self, frame): - """Creates a frame on the server using the given Frame object. + def create_field(self, field): + """Creates a field on the server using the given Field object. - :param pilosa.Frame frame: - :raises pilosa.FrameExistsError: if there already is a frame with the given name + :param pilosa.Field field: + :raises pilosa.FieldExistsError: if there already is a field with the given name """ - data = frame._get_options_string() - path = "/index/%s/frame/%s" % (frame.index.name, frame.name) + data = field._get_options_string() + path = "/index/%s/field/%s" % (field.index.name, field.name) try: self.__http_request("POST", path, data=data) except PilosaServerError as e: if e.response.status == 409: - raise FrameExistsError + raise FieldExistsError raise - def delete_frame(self, frame): - """Deletes the given frame on the server. + def delete_field(self, field): + """Deletes the given field on the server. - :param pilosa.Frame frame: - :raises pilosa.PilosaError: if the frame does not exist + :param pilosa.Field field: + :raises pilosa.PilosaError: if the field does not exist """ - path = "/index/%s/frame/%s" % (frame.index.name, frame.name) + path = "/index/%s/field/%s" % (field.index.name, field.name) self.__http_request("DELETE", path) def ensure_index(self, index): @@ -180,14 +180,14 @@ def ensure_index(self, index): except IndexExistsError: pass - def ensure_frame(self, frame): - """Creates a frame on the server if it does not exist. + def ensure_field(self, field): + """Creates a field on the server if it does not exist. - :param pilosa.Frame frame: + :param pilosa.Field field: """ try: - self.create_frame(frame) - except FrameExistsError: + self.create_field(field) + except FieldExistsError: pass def _read_schema(self): @@ -198,9 +198,9 @@ def schema(self): schema = Schema() for index_info in self._read_schema(): index = schema.index(index_info["name"]) - for frame_info in index_info.get("frames") or []: - options = decode_frame_meta_options(frame_info) - index.frame(frame_info["name"], **options) + for field_info in index_info.get("fields") or []: + options = decode_field_meta_options(field_info) + index.field(field_info["name"], **options) return schema @@ -209,12 +209,12 @@ def sync_schema(self, schema): # find out local - remote schema diff_schema = schema._diff(server_schema) - # create indexes and frames which doesn't exist on the server side + # create indexes and fields which doesn't exist on the server side for index_name, index in diff_schema._indexes.items(): if index_name not in server_schema._indexes: self.ensure_index(index) - for frame_name, frame in index._frames.items(): - self.ensure_frame(frame) + for field_name, field in index._fields.items(): + self.ensure_field(field) # find out remote - local schema diff_schema = server_schema._diff(schema) @@ -223,21 +223,21 @@ def sync_schema(self, schema): if local_index is None: schema._indexes[index_name] = index else: - for frame_name, frame in index._frames.items(): - local_index._frames[frame_name] = frame + for field_name, field in index._fields.items(): + local_index._fields[field_name] = field - def import_frame(self, frame, bit_reader, batch_size=100000): - """Imports a frame using the given bit reader + def import_field(self, field, bit_reader, batch_size=100000): + """Imports a field using the given bit reader - :param frame: + :param field: :param bit_reader: :param batch_size: """ - index_name = frame.index.name - frame_name = frame.name + index_name = field.index.name + field_name = field.name import_bits = self._import_bits for slice, bits in batch_bits(bit_reader, batch_size): - import_bits(index_name, frame_name, slice, bits) + import_bits(index_name, field_name, slice, bits) def http_request(self, method, path, data=None, headers=None): """Sends an HTTP request to the Pilosa server @@ -253,7 +253,7 @@ def http_request(self, method, path, data=None, headers=None): """ return self.__http_request(method, path, data=data, headers=headers) - def _import_bits(self, index_name, frame_name, slice, bits): + def _import_bits(self, index_name, field_name, slice, bits): # sort by row_id then by column_id bits.sort(key=lambda bit: (bit.row_id, bit.column_id)) nodes = self._fetch_fragment_nodes(index_name, slice) @@ -269,7 +269,7 @@ def _import_bits(self, index_name, frame_name, slice, bits): client_params[k] = v for node in nodes: client = Client(URI.address(node.url), **client_params) - client._import_node(_ImportRequest(index_name, frame_name, slice, bits)) + client._import_node(_ImportRequest(index_name, field_name, slice, bits)) def _fetch_fragment_nodes(self, index_name, slice): path = "/fragment/nodes?slice=%d&index=%s" % (slice, index_name) @@ -338,13 +338,14 @@ def __connect(self): self.__client = client -def decode_frame_meta_options(frame_info): - meta = frame_info.get("options", {}) +def decode_field_meta_options(field_info): + meta = field_info.get("options", {}) return { "cache_size": meta.get("cacheSize", 50000), "cache_type": CacheType(meta.get("cacheType", "")), - "inverse_enabled": meta.get("inverseEnabled", False), "time_quantum": TimeQuantum(meta.get("timeQuantum", "")), + "int_min": meta.get("min", 0), + "int_max": meta.get("max", 0), } @@ -510,16 +511,16 @@ def to_protobuf(self, return_bytearray=_IS_PY2): class _ImportRequest: - def __init__(self, index_name, frame_name, slice, bits): + def __init__(self, index_name, field_name, slice, bits): self.index_name = index_name - self.frame_name = frame_name + self.field_name = field_name self.slice = slice self.bits = bits def to_protobuf(self, return_bytearray=_IS_PY2): import_request = internal.ImportRequest() import_request.Index = self.index_name - import_request.Frame = self.frame_name + import_request.Frame = self.field_name import_request.Slice = self.slice row_ids = import_request.RowIDs column_ids = import_request.ColumnIDs diff --git a/pilosa/exceptions.py b/pilosa/exceptions.py index e7e49bc..21b42f6 100644 --- a/pilosa/exceptions.py +++ b/pilosa/exceptions.py @@ -31,7 +31,7 @@ # DAMAGE. # -__all__ = ("PilosaError", "ValidationError", "PilosaURIError", "IndexExistsError", "FrameExistsError") +__all__ = ("PilosaError", "ValidationError", "PilosaURIError", "IndexExistsError", "FieldExistsError") class PilosaError(Exception): @@ -50,6 +50,6 @@ class IndexExistsError(PilosaError): pass -class FrameExistsError(PilosaError): +class FieldExistsError(PilosaError): pass diff --git a/pilosa/orm.py b/pilosa/orm.py index 366ad19..95f10f7 100644 --- a/pilosa/orm.py +++ b/pilosa/orm.py @@ -34,10 +34,10 @@ import json from .exceptions import PilosaError, ValidationError -from .validator import validate_index_name, validate_frame_name, validate_label, validate_key +from .validator import validate_index_name, validate_field_name, validate_label, validate_key __all__ = ("TimeQuantum", "CacheType", "Schema", "Index", "PQLQuery", - "PQLBatchQuery", "IntField", "RangeField", "Frame") + "PQLBatchQuery", "Field") _TIME_FORMAT = "%Y-%m-%dT%H:%M" @@ -47,7 +47,7 @@ class TimeQuantum: - """Valid time quantum values for frames having support for that. + """Valid time quantum values for fields having support for that. * See: `Data Model `_ """ @@ -150,14 +150,14 @@ def _diff(self, other): # if the index doesn't exist in the other schema, simply copy it result._indexes[index_name] = index.copy() else: - # the index exists in the other schema; check the frames - result_index = index.copy(frames=False) - for frame_name, frame in index._frames.items(): - # if the frame doesn't exist in the other scheme, copy it - if frame_name not in result_index._frames: - result_index._frames[frame_name] = frame.copy() + # the index exists in the other schema; check the fields + result_index = index.copy(fields=False) + for field_name, field in index._fields.items(): + # if the field doesn't exist in the other scheme, copy it + if field_name not in result_index._fields: + result_index._fields[field_name] = field.copy() # check whether we modified result index - if len(result_index._frames) > 0: + if len(result_index._fields) > 0: result._indexes[index_name] = result_index return result @@ -177,7 +177,7 @@ class Index: def __init__(self, name): validate_index_name(name) self.name = name - self._frames = {} + self._fields = {} def __eq__(self, other): if id(self) == id(other): @@ -185,7 +185,7 @@ def __eq__(self, other): if not isinstance(other, self.__class__): return False return self._meta_eq(other) and \ - self._frames == other._frames + self._fields == other._fields def __ne__(self, other): return not self.__eq__(other) @@ -193,31 +193,32 @@ def __ne__(self, other): def _meta_eq(self, other): return self.name == other.name - def copy(self, frames=True): + def copy(self, fields=True): index = Index(self.name) - if frames: - index._frames = dict((name, frame.copy()) for name, frame in self._frames.items()) + if fields: + index._fields = dict((name, field.copy()) for name, field in self._fields.items()) return index - def frame(self, name, time_quantum=TimeQuantum.NONE, - inverse_enabled=False, cache_type=CacheType.DEFAULT, cache_size=0, fields=None): - """Creates a frame object with the specified name and defaults. + def field(self, name, time_quantum=TimeQuantum.NONE, + cache_type=CacheType.DEFAULT, cache_size=0, + int_min=0, int_max=0): + """Creates a field object with the specified name and defaults. - :param str name: frame name - :param pilosa.TimeQuantum time_quantum: Sets the time quantum for the frame. If a Frame has a time quantum, then Views are generated for each of the defined time segments. - :param bool inverse_enabled: + :param str name: field name + :param pilosa.TimeQuantum time_quantum: Sets the time quantum for the field. If a Field has a time quantum, then Views are generated for each of the defined time segments. :param pilosa.CacheType cache_type: ``CacheType.DEFAULT``, ``CacheType.LRU`` or ``CacheType.RANKED`` :param int cache_size: Values greater than 0 sets the cache size. Otherwise uses the default cache size - :param list(IntField) fields: List of ``IntField`` objects. E.g.: ``[IntField.int("rate", 0, 100)]`` - :return: Pilosa frame - :rtype: pilosa.Frame - """ - frame = self._frames.get(name) - if frame is None: - frame = Frame(self, name, time_quantum, - inverse_enabled, cache_type, cache_size, fields or []) - self._frames[name] = frame - return frame + :param int int_min: Minimum for the integer field + :param int int_max: Maximum for the integer field + :return: Pilosa field + :rtype: pilosa.Field + """ + field = self._fields.get(name) + if field is None: + field = Field(self, name, time_quantum, + cache_type, cache_size, int_min, int_max) + self._fields[name] = field + return field def raw_query(self, query): """Creates a raw query. @@ -331,29 +332,31 @@ def _bitmap_op(self, name, bitmaps): return PQLQuery(u"%s(%s)" % (name, u", ".join(b.serialize() for b in bitmaps)), self) -class Frame: - """Frames are used to segment and define different functional characteristics within your entire index. +class Field: + """Fields are used to segment and define different functional characteristics within your entire index. - You can think of a Frame as a table-like data partition within your Index. - Row-level attributes are namespaced at the Frame level. + You can think of a Field as a table-like data partition within your Index. + Row-level attributes are namespaced at the Field level. - Do not create a Frame object directly. Instead, use ``pilosa.Index.frame`` method. + Do not create a Field object directly. Instead, use ``pilosa.Index.field`` method. * See `Data Model `_ * See `Query Language `_ """ - def __init__(self, index, name, time_quantum, inverse_enabled, - cache_type, cache_size, fields): - validate_frame_name(name) + def __init__(self, index, name, time_quantum, + cache_type, cache_size, int_min, int_max): + validate_field_name(name) + if int_max < int_min: + raise ValidationError("Max should be greater than min for int fields") + self.index = index self.name = name self.time_quantum = time_quantum - self.inverse_enabled = inverse_enabled self.cache_type = cache_type self.cache_size = cache_size - self.fields = fields - self.range_fields = {} + self.int_min = int_min + self.int_max = int_max def __eq__(self, other): if id(self) == id(other): @@ -361,22 +364,20 @@ def __eq__(self, other): if not isinstance(other, self.__class__): return False - # Note that we skip comparing the frames of the indexes by using index._meta_eq + # Note that we skip comparing the fields of the indexes by using index._meta_eq # in order to avoid a call cycle return self.name == other.name and \ self.index._meta_eq(other.index) and \ self.time_quantum == other.time_quantum and \ - self.inverse_enabled == other.inverse_enabled and \ self.cache_type == other.cache_type and \ - self.cache_size == other.cache_size and \ - self.fields == other.fields + self.cache_size == other.cache_size def __ne__(self, other): return not self.__eq__(other) def copy(self): - return Frame(self.index, self.name, self.time_quantum, - self.inverse_enabled, self.cache_type, self.cache_size, self.fields) + return Field(self.index, self.name, self.time_quantum, + self.cache_type, self.cache_size, self.int_min, self.int_max) def bitmap(self, row_idkey): """Creates a Bitmap query. @@ -390,30 +391,14 @@ def bitmap(self, row_idkey): :rtype: pilosa.PQLBitmapQuery """ fmt = id_key_format("Row ID/Key", row_idkey, - u"Bitmap(row=%s, frame='%s')", - u"Bitmap(row='%s', frame='%s')") + u"Bitmap(row=%s, field='%s')", + u"Bitmap(row='%s', field='%s')") return PQLQuery(fmt % (row_idkey, self.name), self.index) - def inverse_bitmap(self, column_idkey): - """Creates a Bitmap query. - - ``Bitmap`` retrieves the indices of all the set bits in a row or column based on whether the row label or column label is given in the query. It also retrieves any attributes set on that row or column. - - This variant of Bitmap query uses the column label. - - :param int column_idkey: - :return: Pilosa bitmap query - :rtype: pilosa.PQLBitmapQuery - """ - fmt = id_key_format("Column", column_idkey, - u"Bitmap(col=%s, frame='%s')", - u"Bitmap(col='%s', frame='%s')") - return PQLQuery(fmt % (column_idkey, self.name), self.index) - def setbit(self, row_idkey, column_idkey, timestamp=None): """Creates a SetBit query. - ``SetBit`` assigns a value of 1 to a bit in the binary matrix, thus associating the given row in the given frame with the given column. + ``SetBit`` assigns a value of 1 to a bit in the binary matrix, thus associating the given row in the given field with the given column. :param int row_idkey: :param int column_idkey: @@ -422,9 +407,9 @@ def setbit(self, row_idkey, column_idkey, timestamp=None): :rtype: pilosa.PQLQuery """ if isinstance(row_idkey, int) and isinstance(column_idkey, int): - fmt = u"SetBit(row=%s, frame='%s', col=%s%s)" + fmt = u"SetBit(row=%s, field='%s', col=%s%s)" elif isinstance(row_idkey, _basestring) and isinstance(column_idkey, _basestring): - fmt = u"SetBit(row='%s', frame='%s', col='%s'%s)" + fmt = u"SetBit(row='%s', field='%s', col='%s'%s)" validate_key(row_idkey) validate_key(column_idkey) else: @@ -435,7 +420,7 @@ def setbit(self, row_idkey, column_idkey, timestamp=None): def clearbit(self, row_idkey, column_idkey): """Creates a ClearBit query. - ``ClearBit`` assigns a value of 0 to a bit in the binary matrix, thus disassociating the given row in the given frame from the given column. + ``ClearBit`` assigns a value of 0 to a bit in the binary matrix, thus disassociating the given row in the given field from the given column. :param int row_idkey: :param int column_idkey: @@ -443,9 +428,9 @@ def clearbit(self, row_idkey, column_idkey): :rtype: pilosa.PQLQuery """ if isinstance(row_idkey, int) and isinstance(column_idkey, int): - fmt = u"ClearBit(row=%s, frame='%s', col=%s)" + fmt = u"ClearBit(row=%s, field='%s', col=%s)" elif isinstance(row_idkey, _basestring) and isinstance(column_idkey, _basestring): - fmt = u"ClearBit(row='%s', frame='%s', col='%s')" + fmt = u"ClearBit(row='%s', field='%s', col='%s')" validate_key(row_idkey) validate_key(column_idkey) else: @@ -455,7 +440,7 @@ def clearbit(self, row_idkey, column_idkey): def topn(self, n, bitmap=None, field="", *values): """Creates a TopN query. - ``TopN`` returns the id and count of the top n bitmaps (by count of bits) in the frame. + ``TopN`` returns the id and count of the top n bitmaps (by count of bits) in the field. * see: `TopN Query `_ @@ -464,26 +449,7 @@ def topn(self, n, bitmap=None, field="", *values): :param field str field: field name :param object values: filter values to be matched against the field """ - return self._topn(n, bitmap, field, False, *values) - - def inverse_topn(self, n, bitmap=None, field="", *values): - """Creates a TopN query. - - ``TopN`` returns the id and count of the top n bitmaps (by count of bits) in the frame. - - This version sets `inverse=true`. - - * see: `TopN Query `_ - - :param int n: number of items to return - :param pilosa.PQLBitmapQuery bitmap: a PQL Bitmap query - :param field str field: field name - :param object values: filter values to be matched against the field - """ - return self._topn(n, bitmap, field, True, *values) - - def _topn(self, n, bitmap=None, field="", inverse=False, *values): - parts = ["frame='%s'" % self.name, "n=%d" % n, "inverse=%s" % ('true' if inverse else 'false')] + parts = ["field='%s'" % self.name, "n=%d" % n] if bitmap: parts.insert(0, bitmap.serialize()) if field: @@ -504,33 +470,18 @@ def range(self, row_idkey, start, end): :param datetime.datetime start: start timestamp :param datetime.datetime end: end timestamp """ - return self._range("row", row_idkey, start, end) - - def inverse_range(self, column_id, start, end): - """Creates a Range query. - - Similar to ``Bitmap``, but only returns bits which were set with timestamps between the given start and end timestamps. - - - :param int column_id: - :param datetime.datetime start: start timestamp - :param datetime.datetime end: end timestamp - """ - return self._range("col", column_id, start, end) - - def _range(self, label, row_idkey, start, end): fmt = id_key_format("Row", row_idkey, - u"Range(%s=%s, frame='%s', start='%s', end='%s')", - u"Range(%s='%s', frame='%s', start='%s', end='%s')") + u"Range(row=%s, field='%s', start='%s', end='%s')", + u"Range(row='%s', field='%s', start='%s', end='%s')") start_str = start.strftime(_TIME_FORMAT) end_str = end.strftime(_TIME_FORMAT) - return PQLQuery(fmt % (label, row_idkey, self.name, start_str, end_str), + return PQLQuery(fmt % (row_idkey, self.name, start_str, end_str), self.index) def set_row_attrs(self, row_idkey, attrs): """Creates a SetRowAttrs query. - ``SetRowAttrs`` associates arbitrary key/value pairs with a row in a frame. + ``SetRowAttrs`` associates arbitrary key/value pairs with a row in a field. Following object types are accepted: @@ -545,101 +496,11 @@ def set_row_attrs(self, row_idkey, attrs): :rtype: pilosa.PQLQuery """ fmt = id_key_format("Row", row_idkey, - u"SetRowAttrs(row=%s, frame='%s', %s)", - u"SetRowAttrs(row='%s', frame='%s', %s)") + u"SetRowAttrs(row=%s, field='%s', %s)", + u"SetRowAttrs(row='%s', field='%s', %s)") attrs_str = _create_attributes_str(attrs) return PQLQuery(fmt % (row_idkey, self.name, attrs_str), self.index) - def field(self, name): - """Returns a _RangeField object with the given name. - - :param name: field name - :return: _RangeField object - :rtype: RangeField - """ - field = self.range_fields.get(name) - if not field: - validate_label(name) - field = RangeField(self, name) - self.range_fields[name] = field - return field - - def _get_options_string(self): - data = {} - if self.inverse_enabled: - data["inverseEnabled"] = True - if self.time_quantum != TimeQuantum.NONE: - data["timeQuantum"] = str(self.time_quantum) - if self.cache_type != CacheType.DEFAULT: - data["cacheType"] = str(self.cache_type) - if self.cache_size > 0: - data["cacheSize"] = self.cache_size - if self.fields: - data["rangeEnabled"] = True - data["fields"] = [f.attrs for f in self.fields] - return json.dumps({"options": data}, sort_keys=True) - - -class PQLQuery: - - def __init__(self, pql, index): - self.pql = pql - self.index = index - - def serialize(self): - return self.pql - - -def _create_attributes_str(attrs): - kvs = [] - try: - for k, v in attrs.items(): - # TODO: make key use its own validator - validate_label(k) - kvs.append("%s=%s" % (k, json.dumps(v))) - return ", ".join(sorted(kvs)) - except TypeError: - raise PilosaError("Error while converting values") - - -class PQLBatchQuery: - - def __init__(self, index): - self.index = index - self.queries = [] - - def add(self, *queries): - self.queries.extend(queries) - - def serialize(self): - return u''.join(q.serialize() for q in self.queries) - - -class IntField: - - def __init__(self, attrs): - self.attrs = attrs - - @classmethod - def int(cls, name, min=0, max=100): - validate_label(name) - if max <= min: - raise ValidationError("Max should be greater than min for int fields") - return cls({ - "name": name, - "type": "int", - "min": min, - "max": max - }) - - -class RangeField: - - def __init__(self, frame, name): - self.frame_name = frame.name - self.name = name - self.index = frame.index - def lt(self, n): """Creates a Range query with less than (<) condition. @@ -700,7 +561,7 @@ def not_null(self): :return: a PQL query :rtype: PQLQuery """ - q = u"Range(frame='%s', %s != null)" % (self.frame_name, self.name) + q = u"Range(%s != null)" % self.name return PQLQuery(q, self.index) def between(self, a, b): @@ -711,7 +572,7 @@ def between(self, a, b): :return: a PQL query :rtype: PQLQuery """ - q = u"Range(frame='%s', %s >< [%d,%d])" % (self.frame_name, self.name, a, b) + q = u"Range(%s >< [%d,%d])" % (self.name, a, b) return PQLQuery(q, self.index) def sum(self, bitmap=None): @@ -742,26 +603,76 @@ def max(self, bitmap=None): return self._value_query("Max", bitmap) def set_value(self, column_id, value): - """Creates a SetFieldValue query. + """Creates a SetValue query. :param column_id: column ID :param value: the value to assign to the field :return: a PQL query :rtype: PQLQuery """ - q = u"SetFieldValue(frame='%s', col=%d, %s=%d)" % \ - (self.frame_name, column_id, self.name, value) + q = u"SetValue(col=%d, %s=%d)" % (column_id, self.name, value) return PQLQuery(q, self.index) def _binary_operation(self, op, n): - q = u"Range(frame='%s', %s %s %d)" % (self.frame_name, self.name, op, n) + q = u"Range(%s %s %d)" % (self.name, op, n) return PQLQuery(q, self.index) def _value_query(self, op, bitmap): bitmap_str = "%s, " % bitmap.serialize() if bitmap else "" - q = u"%s(%sframe='%s', field='%s')" % (op, bitmap_str, self.frame_name, self.name) + q = u"%s(%sfield='%s')" % (op, bitmap_str, self.name) return PQLQuery(q, self.index) + def _get_options_string(self): + data = {} + if self.time_quantum != TimeQuantum.NONE: + data["type"] = "time" + data["timeQuantum"] = str(self.time_quantum) + elif self.int_min != 0 or self.int_max != 0: + data["type"] = "int" + data["min"] = self.int_min + data["max"] = self.int_max + else: + data["type"] = "set" + if self.cache_type != CacheType.DEFAULT: + data["cacheType"] = str(self.cache_type) + if self.cache_size > 0: + data["cacheSize"] = self.cache_size + return json.dumps({"options": data}, sort_keys=True) + + +class PQLQuery: + + def __init__(self, pql, index): + self.pql = pql + self.index = index + + def serialize(self): + return self.pql + + +def _create_attributes_str(attrs): + kvs = [] + try: + for k, v in attrs.items(): + # TODO: make key use its own validator + validate_label(k) + kvs.append("%s=%s" % (k, json.dumps(v))) + return ", ".join(sorted(kvs)) + except TypeError: + raise PilosaError("Error while converting values") + + +class PQLBatchQuery: + + def __init__(self, index): + self.index = index + self.queries = [] + + def add(self, *queries): + self.queries.extend(queries) + + def serialize(self): + return u''.join(q.serialize() for q in self.queries) def id_key_format(name, id_key, id_fmt, key_fmt): diff --git a/pilosa/validator.py b/pilosa/validator.py index fda7fac..78e18fa 100644 --- a/pilosa/validator.py +++ b/pilosa/validator.py @@ -36,7 +36,7 @@ from pilosa.exceptions import ValidationError __all__ = ("valid_index_name", "validate_index_name", "valid_frame_name", - "validate_frame_name", "valid_label", "validate_label") + "validate_field_name", "valid_label", "validate_label") __INDEX_NAME = re.compile(r"^[a-z][a-z0-9_-]*$") @@ -72,9 +72,9 @@ def valid_key(key): return bool(__KEY.match(key)) -def validate_frame_name(frame_name): +def validate_field_name(frame_name): if not valid_frame_name(frame_name): - raise ValidationError("Invalid frame name: %s" % frame_name) + raise ValidationError("Invalid field name: %s" % frame_name) def valid_label(label): diff --git a/tests/test_client.py b/tests/test_client.py index 9d4150c..eb9c690 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -37,7 +37,7 @@ import pilosa.internal.public_pb2 as internal from pilosa import TimeQuantum, CacheType from pilosa.client import Client, URI, Cluster, _QueryRequest, \ - decode_frame_meta_options, _ImportRequest, _Node + decode_field_meta_options, _ImportRequest, _Node from pilosa.exceptions import PilosaURIError, PilosaError from pilosa.imports import Bit @@ -61,12 +61,13 @@ def test_create_client(self): def test_decode_frame_meta_options(self): frame_info = {} - options = decode_frame_meta_options(frame_info) + options = decode_field_meta_options(frame_info) target = { "cache_size": 50000, "cache_type": CacheType.DEFAULT, - "inverse_enabled": False, - "time_quantum": TimeQuantum.NONE + "time_quantum": TimeQuantum.NONE, + "int_min": 0, + "int_max": 0, } self.assertEquals(target, options) @@ -219,12 +220,12 @@ def test_cluster_reset(self): class QueryRequestTestCase(unittest.TestCase): def test_serialize(self): - qr = _QueryRequest("Bitmap(frame='foo', id=1)", columns=True) + qr = _QueryRequest("Bitmap(field='foo', id=1)", columns=True) bin = qr.to_protobuf(False) # do not return a bytearray self.assertIsNotNone(bin) qr = internal.QueryRequest() qr.ParseFromString(bin) - self.assertEquals("Bitmap(frame='foo', id=1)", qr.Query) + self.assertEquals("Bitmap(field='foo', id=1)", qr.Query) self.assertEquals(True, qr.ColumnAttrs) diff --git a/tests/test_orm.py b/tests/test_orm.py index 62c5814..1fa4a51 100644 --- a/tests/test_orm.py +++ b/tests/test_orm.py @@ -34,14 +34,14 @@ import unittest from datetime import datetime -from pilosa import PilosaError, Index, TimeQuantum, CacheType, IntField, ValidationError +from pilosa import PilosaError, Index, TimeQuantum, CacheType, ValidationError from pilosa.orm import Schema schema = Schema() sampleIndex = schema.index("sample-db") -sampleFrame = sampleIndex.frame("sample-frame") +sampleFrame = sampleIndex.field("sample-field") projectIndex = schema.index("project-db") -collabFrame = projectIndex.frame("collaboration") +collabFrame = projectIndex.field("collaboration") class SchemaTestCase(unittest.TestCase): @@ -49,21 +49,21 @@ class SchemaTestCase(unittest.TestCase): def test_diff(self): schema1 = Schema() index11 = schema1.index("diff-index1") - index11.frame("frame1-1") - index11.frame("frame1-2") + index11.field("field1-1") + index11.field("field1-2") index12 = schema1.index("diff-index2") - index12.frame("frame2-1") + index12.field("field2-1") schema2 = Schema() index21 = schema2.index("diff-index1") - index21.frame("another-frame") + index21.field("another-field") target_diff12 = Schema() target_index1 = target_diff12.index("diff-index1") - target_index1.frame("frame1-1") - target_index1.frame("frame1-2") + target_index1.field("field1-1") + target_index1.field("field1-2") target_index2 = target_diff12.index("diff-index2") - target_index2.frame("frame2-1") + target_index2.field("field2-1") diff12 = schema1._diff(schema2) self.assertEqual(target_diff12, diff12) @@ -109,17 +109,17 @@ def test_union(self): q1 = sampleIndex.union(b1, b2) self.assertEquals( - "Union(Bitmap(row=10, frame='sample-frame'), Bitmap(row=20, frame='sample-frame'))", + "Union(Bitmap(row=10, field='sample-field'), Bitmap(row=20, field='sample-field'))", q1.serialize()) q2 = sampleIndex.union(b1, b2, b3) self.assertEquals( - "Union(Bitmap(row=10, frame='sample-frame'), Bitmap(row=20, frame='sample-frame'), Bitmap(row=42, frame='sample-frame'))", + "Union(Bitmap(row=10, field='sample-field'), Bitmap(row=20, field='sample-field'), Bitmap(row=42, field='sample-field'))", q2.serialize()) q3 = sampleIndex.union(b1, b4) self.assertEquals( - "Union(Bitmap(row=10, frame='sample-frame'), Bitmap(row=2, frame='collaboration'))", + "Union(Bitmap(row=10, field='sample-field'), Bitmap(row=2, field='collaboration'))", q3.serialize()) def test_intersect(self): @@ -130,17 +130,17 @@ def test_intersect(self): q1 = sampleIndex.intersect(b1, b2) self.assertEquals( - "Intersect(Bitmap(row=10, frame='sample-frame'), Bitmap(row=20, frame='sample-frame'))", + "Intersect(Bitmap(row=10, field='sample-field'), Bitmap(row=20, field='sample-field'))", q1.serialize()) q2 = sampleIndex.intersect(b1, b2, b3) self.assertEquals( - "Intersect(Bitmap(row=10, frame='sample-frame'), Bitmap(row=20, frame='sample-frame'), Bitmap(row=42, frame='sample-frame'))", + "Intersect(Bitmap(row=10, field='sample-field'), Bitmap(row=20, field='sample-field'), Bitmap(row=42, field='sample-field'))", q2.serialize()) q3 = sampleIndex.intersect(b1, b4) self.assertEquals( - "Intersect(Bitmap(row=10, frame='sample-frame'), Bitmap(row=2, frame='collaboration'))", + "Intersect(Bitmap(row=10, field='sample-field'), Bitmap(row=2, field='collaboration'))", q3.serialize()) def test_difference(self): @@ -151,17 +151,17 @@ def test_difference(self): q1 = sampleIndex.difference(b1, b2) self.assertEquals( - "Difference(Bitmap(row=10, frame='sample-frame'), Bitmap(row=20, frame='sample-frame'))", + "Difference(Bitmap(row=10, field='sample-field'), Bitmap(row=20, field='sample-field'))", q1.serialize()) q2 = sampleIndex.difference(b1, b2, b3) self.assertEquals( - "Difference(Bitmap(row=10, frame='sample-frame'), Bitmap(row=20, frame='sample-frame'), Bitmap(row=42, frame='sample-frame'))", + "Difference(Bitmap(row=10, field='sample-field'), Bitmap(row=20, field='sample-field'), Bitmap(row=42, field='sample-field'))", q2.serialize()) q3 = sampleIndex.difference(b1, b4) self.assertEquals( - "Difference(Bitmap(row=10, frame='sample-frame'), Bitmap(row=2, frame='collaboration'))", + "Difference(Bitmap(row=10, field='sample-field'), Bitmap(row=2, field='collaboration'))", q3.serialize()) def test_xor(self): @@ -170,7 +170,7 @@ def test_xor(self): q1 = sampleIndex.xor(b1, b2) self.assertEquals( - "Xor(Bitmap(row=10, frame='sample-frame'), Bitmap(row=20, frame='sample-frame'))", + "Xor(Bitmap(row=10, field='sample-field'), Bitmap(row=20, field='sample-field'))", q1.serialize()) def test_union0(self): @@ -179,7 +179,7 @@ def test_union0(self): def test_union1(self): q = sampleIndex.union(sampleFrame.bitmap(10)) - self.assertEquals("Union(Bitmap(row=10, frame='sample-frame'))", q.serialize()) + self.assertEquals("Union(Bitmap(row=10, field='sample-field'))", q.serialize()) def test_intersect_invalid_bitmap_count_fails(self): self.assertRaises(PilosaError, projectIndex.intersect) @@ -194,7 +194,7 @@ def test_count(self): b = collabFrame.bitmap(42) q = projectIndex.count(b) self.assertEquals( - "Count(Bitmap(row=42, frame='collaboration'))", + "Count(Bitmap(row=42, field='collaboration'))", q.serialize()) def test_set_column_attributes(self): @@ -217,12 +217,12 @@ def test_set_column_attributes_invalid_values(self): class FrameTestCase(unittest.TestCase): - def test_create_frame(self): + def test_create_field(self): db = Index("foo") - frame = db.frame("sample-frame") - self.assertEqual(db, frame.index) - self.assertEqual("sample-frame", frame.name) - self.assertEqual(TimeQuantum.NONE, frame.time_quantum) + field = db.field("sample-field") + self.assertEqual(db, field.index) + self.assertEqual("sample-field", field.name) + self.assertEqual(TimeQuantum.NONE, field.time_quantum) def test_same_equals(self): self.assertEqual(sampleFrame, sampleFrame) @@ -232,50 +232,39 @@ def test_other_class_not_equals(self): self.assertNotEqual(sampleFrame, schema) def test_bitmap(self): - qry1 = sampleFrame.bitmap(5) + qry1 = collabFrame.bitmap(5) self.assertEquals( - "Bitmap(row=5, frame='sample-frame')", + "Bitmap(row=5, field='collaboration')", qry1.serialize()) qry2 = collabFrame.bitmap(10) self.assertEquals( - "Bitmap(row=10, frame='collaboration')", + "Bitmap(row=10, field='collaboration')", qry2.serialize()) - qry3 = sampleFrame.bitmap("b7feb014-8ea7-49a8-9cd8-19709161ab63") + qry3 = collabFrame.bitmap("b7feb014-8ea7-49a8-9cd8-19709161ab63") self.assertEquals( - "Bitmap(row='b7feb014-8ea7-49a8-9cd8-19709161ab63', frame='sample-frame')", + "Bitmap(row='b7feb014-8ea7-49a8-9cd8-19709161ab63', field='collaboration')", qry3.serialize()) def test_bitmap_with_invalid_id_type(self): self.assertRaises(ValidationError, sampleFrame.bitmap, {}) - def test_inverse_bitmap(self): - f1 = projectIndex.frame("f1-inversable", inverse_enabled=True) - qry = f1.inverse_bitmap(5) - self.assertEquals( - "Bitmap(col=5, frame='f1-inversable')", - qry.serialize()) - qry2 = f1.inverse_bitmap("b7feb014-8ea7-49a8-9cd8-19709161ab63") - self.assertEquals( - "Bitmap(col='b7feb014-8ea7-49a8-9cd8-19709161ab63', frame='f1-inversable')", - qry2.serialize()) - def test_setbit(self): - qry1 = sampleFrame.setbit(5, 10) + qry1 = collabFrame.setbit(5, 10) self.assertEquals( - "SetBit(row=5, frame='sample-frame', col=10)", + "SetBit(row=5, field='collaboration', col=10)", qry1.serialize()) qry2 = collabFrame.setbit(10, 20) self.assertEquals( - "SetBit(row=10, frame='collaboration', col=20)", + "SetBit(row=10, field='collaboration', col=20)", qry2.serialize()) - qry3 = sampleFrame.setbit("b7feb014-8ea7-49a8-9cd8-19709161ab63", "some_id") + qry3 = collabFrame.setbit("b7feb014-8ea7-49a8-9cd8-19709161ab63", "some_id") self.assertEquals( - "SetBit(row='b7feb014-8ea7-49a8-9cd8-19709161ab63', frame='sample-frame', col='some_id')", + "SetBit(row='b7feb014-8ea7-49a8-9cd8-19709161ab63', field='collaboration', col='some_id')", qry3.serialize()) def test_setbit_with_invalid_id_type(self): @@ -287,24 +276,24 @@ def test_setbit_with_timestamp(self): timestamp = datetime(2017, 4, 24, 12, 14) qry = collabFrame.setbit(10, 20, timestamp) self.assertEquals( - "SetBit(row=10, frame='collaboration', col=20, timestamp='2017-04-24T12:14')", + "SetBit(row=10, field='collaboration', col=20, timestamp='2017-04-24T12:14')", qry.serialize() ) def test_clearbit(self): - qry1 = sampleFrame.clearbit(5, 10) + qry1 = collabFrame.clearbit(5, 10) self.assertEquals( - "ClearBit(row=5, frame='sample-frame', col=10)", + "ClearBit(row=5, field='collaboration', col=10)", qry1.serialize()) qry2 = collabFrame.clearbit(10, 20) self.assertEquals( - "ClearBit(row=10, frame='collaboration', col=20)", + "ClearBit(row=10, field='collaboration', col=20)", qry2.serialize()) - qry3 = sampleFrame.clearbit("b7feb014-8ea7-49a8-9cd8-19709161ab63", "some_id") + qry3 = collabFrame.clearbit("b7feb014-8ea7-49a8-9cd8-19709161ab63", "some_id") self.assertEquals( - "ClearBit(row='b7feb014-8ea7-49a8-9cd8-19709161ab63', frame='sample-frame', col='some_id')", + "ClearBit(row='b7feb014-8ea7-49a8-9cd8-19709161ab63', field='collaboration', col='some_id')", qry3.serialize()) def test_clearbit_with_invalid_id_type(self): @@ -313,43 +302,33 @@ def test_clearbit_with_invalid_id_type(self): self.assertRaises(ValidationError, sampleFrame.clearbit, 1, "zero") def test_topn(self): - q1 = sampleFrame.topn(27) + q1 = collabFrame.topn(27) self.assertEquals( - "TopN(frame='sample-frame', n=27, inverse=false)", + "TopN(field='collaboration', n=27)", q1.serialize()) - q2 = sampleFrame.topn(10, collabFrame.bitmap(3)) + q2 = collabFrame.topn(10, collabFrame.bitmap(3)) self.assertEquals( - u"TopN(Bitmap(row=3, frame='collaboration'), frame='sample-frame', n=10, inverse=false)", + u"TopN(Bitmap(row=3, field='collaboration'), field='collaboration', n=10)", q2.serialize()) q3 = sampleFrame.topn(12, collabFrame.bitmap(7), "category", 80, 81) self.assertEquals( - "TopN(Bitmap(row=7, frame='collaboration'), frame='sample-frame', n=12, inverse=false, field='category', filters=[80,81])", + "TopN(Bitmap(row=7, field='collaboration'), field='sample-field', n=12, field='category', filters=[80,81])", q3.serialize()) - q4 = sampleFrame.inverse_topn(12, collabFrame.bitmap(7), "category", 80, 81) - self.assertEquals( - "TopN(Bitmap(row=7, frame='collaboration'), frame='sample-frame', n=12, inverse=true, field='category', filters=[80,81])", - q4.serialize()) - def test_range(self): start = datetime(1970, 1, 1, 0, 0) end = datetime(2000, 2, 2, 3, 4) q1 = collabFrame.range(10, start, end) self.assertEquals( - "Range(row=10, frame='collaboration', start='1970-01-01T00:00', end='2000-02-02T03:04')", + "Range(row=10, field='collaboration', start='1970-01-01T00:00', end='2000-02-02T03:04')", q1.serialize()) - q2 = collabFrame.inverse_range(10, start, end) - self.assertEquals( - "Range(col=10, frame='collaboration', start='1970-01-01T00:00', end='2000-02-02T03:04')", - q2.serialize()) - q3 = sampleFrame.range("b7feb014-8ea7-49a8-9cd8-19709161ab63", start, end) self.assertEquals( - u"Range(row='b7feb014-8ea7-49a8-9cd8-19709161ab63', frame='sample-frame', start='1970-01-01T00:00', end='2000-02-02T03:04')", + u"Range(row='b7feb014-8ea7-49a8-9cd8-19709161ab63', field='sample-field', start='1970-01-01T00:00', end='2000-02-02T03:04')", q3.serialize()) def test_set_row_attributes(self): @@ -359,88 +338,80 @@ def test_set_row_attributes(self): } q = collabFrame.set_row_attrs(5, attrs_map) self.assertEquals( - "SetRowAttrs(row=5, frame='collaboration', active=true, quote=\"\\\"Don't worry, be happy\\\"\")", + "SetRowAttrs(row=5, field='collaboration', active=true, quote=\"\\\"Don't worry, be happy\\\"\")", q.serialize()) - def test_field(self): - # only a single instance of a field should exist - field1 = sampleFrame.field("the-field") - field2 = sampleFrame.field("the-field") - self.assertTrue(id(field1) == id(field2)) - def test_field_lt(self): - q = sampleFrame.field("foo").lt(10) + q = collabFrame.lt(10) self.assertEquals( - "Range(frame='sample-frame', foo < 10)", + "Range(collaboration < 10)", q.serialize()) def test_field_lte(self): - q = sampleFrame.field("foo").lte(10) + q = collabFrame.lte(10) self.assertEquals( - "Range(frame='sample-frame', foo <= 10)", + "Range(collaboration <= 10)", q.serialize()) def test_field_gt(self): - q = sampleFrame.field("foo").gt(10) + q = collabFrame.gt(10) self.assertEquals( - "Range(frame='sample-frame', foo > 10)", + "Range(collaboration > 10)", q.serialize()) def test_field_gte(self): - q = sampleFrame.field("foo").gte(10) + q = collabFrame.gte(10) self.assertEquals( - "Range(frame='sample-frame', foo >= 10)", + "Range(collaboration >= 10)", q.serialize()) def test_field_equals(self): - q = sampleFrame.field("foo").equals(10) + q = collabFrame.equals(10) self.assertEquals( - "Range(frame='sample-frame', foo == 10)", + "Range(collaboration == 10)", q.serialize()) def test_field_not_equals(self): - q = sampleFrame.field("foo").not_equals(10) + q = collabFrame.not_equals(10) self.assertEquals( - "Range(frame='sample-frame', foo != 10)", + "Range(collaboration != 10)", q.serialize()) def test_field_not_null(self): - q = sampleFrame.field("foo").not_null() + q = collabFrame.not_null() self.assertEquals( - "Range(frame='sample-frame', foo != null)", + "Range(collaboration != null)", q.serialize()) def test_field_between(self): - q = sampleFrame.field("foo").between(10, 20) + q = collabFrame.between(10, 20) self.assertEquals( - "Range(frame='sample-frame', foo >< [10,20])", + "Range(collaboration >< [10,20])", q.serialize()) - def test_field_set_value(self): - q = sampleFrame.field("foo").set_value(10, 20) + def test_field_sum(self): + q = collabFrame.sum(collabFrame.bitmap(10)) self.assertEquals( - "SetFieldValue(frame='sample-frame', col=10, foo=20)", + "Sum(Bitmap(row=10, field='collaboration'), field='collaboration')", q.serialize()) - - def test_field_sum(self): - q = sampleFrame.field("foo").sum(sampleFrame.bitmap(10)) + q = collabFrame.sum() self.assertEquals( - "Sum(Bitmap(row=10, frame='sample-frame'), frame='sample-frame', field='foo')", + "Sum(field='collaboration')", q.serialize()) - q = sampleFrame.field("foo").sum() + + def test_field_set_value(self): + q = collabFrame.set_value(10, 20) self.assertEquals( - "Sum(frame='sample-frame', field='foo')", + "SetValue(col=10, collaboration=20)", q.serialize()) def test_get_options_string(self): - frame = sampleIndex.frame("stargazer_id", + field = sampleIndex.field("stargazer_id", time_quantum=TimeQuantum.DAY_HOUR, - inverse_enabled=True, cache_type=CacheType.RANKED, - cache_size=1000, - fields=[IntField.int("foo"), IntField.int("bar", min=-1, max=1)]) - target = '{"options": {"cacheSize": 1000, "cacheType": "ranked", "fields": [{"max": 100, "min": 0, "name": "foo", "type": "int"}, {"max": 1, "min": -1, "name": "bar", "type": "int"}], "inverseEnabled": true, "rangeEnabled": true, "timeQuantum": "DH"}}' - self.assertEquals(target, frame._get_options_string()) + cache_size=1000) + target = '{"options": {"timeQuantum": "DH", "type": "time"}}' + self.assertEquals(target, field._get_options_string()) class TimeQuantumTestCase(unittest.TestCase): @@ -470,4 +441,4 @@ def test_equality(self): class RangeFieldTestCase(unittest.TestCase): def test_min_greater_equals_max_fails(self): - self.assertRaises(ValidationError, IntField.int, "foo", min=10, max=9) \ No newline at end of file + self.assertRaises(ValidationError, sampleIndex.field, "intminmax", int_min=10, int_max=9) \ No newline at end of file diff --git a/tests/test_validator.py b/tests/test_validator.py index fec7f3a..70ede24 100644 --- a/tests/test_validator.py +++ b/tests/test_validator.py @@ -36,7 +36,7 @@ import unittest from pilosa.exceptions import ValidationError -from pilosa.validator import validate_frame_name, validate_index_name, validate_label, validate_key +from pilosa.validator import validate_field_name, validate_index_name, validate_label, validate_key class ValidatorTestCase(unittest.TestCase): @@ -100,15 +100,15 @@ def test_invalid_index_name_fails(self): def test_validate_valid_frame_name(self): for name in self.VALID_FRAME_NAMES: - validate_frame_name(name) + validate_field_name(name) def test_invalid_frame_name_fails(self): for name in self.INVALID_FRAME_NAMES: try: - validate_frame_name(name) + validate_field_name(name) except ValidationError: continue - self.fail("Frame name validation should have failed for: " + name) + self.fail("Field name validation should have failed for: " + name) def test_validate_valid_label(self): for label in self.VALID_LABELS: From d6d9e575763d653831e2cb645048a0a2e7d1a024 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Mon, 18 Jun 2018 17:06:13 +0300 Subject: [PATCH 2/9] More renaming --- CHANGELOG.md | 2 +- README.md | 16 ++-- doc/index.rst | 30 +++---- docs/data-model-queries.md | 38 ++++----- docs/imports.md | 6 +- docs/server-interaction.md | 16 ++-- integration_tests/test_client_it.py | 64 +++++++------- pilosa/client.py | 48 +++++------ pilosa/imports.py | 14 ++-- pilosa/internal/public.proto | 15 ++-- pilosa/internal/public_pb2.py | 69 +++++++-------- pilosa/orm.py | 126 ++++++++++++++-------------- pilosa/response.py | 28 +++---- pilosa/validator.py | 18 ++-- tests/test_client.py | 14 ++-- tests/test_imports.py | 15 ++-- tests/test_orm.py | 112 ++++++++++++------------- tests/test_response.py | 6 +- tests/test_validator.py | 14 ++-- 19 files changed, 321 insertions(+), 330 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a00ce8f..814ee34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,7 +28,7 @@ * **v0.7.0** (2017-10-04): * Added support for creating range encoded frames. * Added `Xor` call. - * Added support for excluding bits or attributes from bitmap calls. In order to exclude bits, call `setExcludeBits(true)` in your `QueryOptions.Builder`. In order to exclude attributes, call `setExcludeAttributes(true)`. + * Added support for excluding bits or attributes from bitmap calls. * Added range field operations. * Customizable CSV timestamp format (Contributed by @lachlanorr). * **Deprecation** Row and column labels are deprecated, and will be removed in a future release of this library. Do not use `column_label` field when creating `Index` objects and do not use `row_label` field when creating `Frame` objects for new code. See: https://github.com/pilosa/pilosa/issues/752 for more info. diff --git a/README.md b/README.md index d426ae4..0f557c6 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ -Python client for Pilosa high performance distributed bitmap index. +Python client for Pilosa high performance distributed row index. ## What's New? @@ -45,30 +45,30 @@ schema = client.schema() myindex = schema.index("myindex") # Create a Field object -myframe = myindex.field("myframe") +myfield = myindex.field("myfield") # make sure the index and field exists on the server client.sync_schema(schema) # Send a SetBit query. PilosaError is thrown if execution of the query fails. -client.query(myframe.setbit(5, 42)) +client.query(myfield.setbit(5, 42)) # Send a Bitmap query. PilosaError is thrown if execution of the query fails. -response = client.query(myframe.bitmap(5)) +response = client.query(myfield.row(5)) # Get the result result = response.result # Act on the result if result: - bits = result.bitmap.bits - print("Got bits: ", bits) + columns = result.row.columns + print("Got columns: ", columns) # You can batch queries to improve throughput response = client.query( myindex.batch_query( - myframe.bitmap(5), - myframe.bitmap(10), + myfield.row(5), + myfield.row(10), ) ) for result in response.results: diff --git a/doc/index.rst b/doc/index.rst index bab6c7a..d5367a5 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -6,7 +6,7 @@ Welcome to Python Client for Pilosa's documentation! ==================================================== -Python client for `Pilosa `_ high performance distributed bitmap index. +Python client for `Pilosa `_ high performance distributed row index. .. toctree:: @@ -43,37 +43,37 @@ at ``localhost:10101`` (the default): # Create the default client client = pilosa.Client() - # Create an Index object - myindex = pilosa.Index("myindex") + # Retrieve the schema + schema = client.schema() - # Make sure the index exists on the server - client.ensure_index(myindex) + # Create an Index object + myindex = schema.index("myindex") - # Create a Frame object - myframe = myindex.frame("myframe") + # Create a Field object + myfield = myindex.field("myfield") - # Make sure the frame exists on the server - client.ensure_frame(myframe) + # make sure the index and field exists on the server + client.sync_schema(schema) # Send a SetBit query. PilosaError is thrown if execution of the query fails. - client.query(myframe.setbit(5, 42)) + client.query(myfield.setbit(5, 42)) # Send a Bitmap query. PilosaError is thrown if execution of the query fails. - response = client.query(myframe.bitmap(5)) + response = client.query(myfield.row(5)) # Get the result result = response.result # Act on the result if result: - bits = result.bitmap.bits - print("Got bits: ", bits) + columns = result.row.columns + print("Got columns: ", columns) # You can batch queries to improve throughput response = client.query( myindex.batch_query( - myframe.bitmap(5), - myframe.bitmap(10), + myfield.row(5), + myfield.row(10), ) ) for result in response.results: diff --git a/docs/data-model-queries.md b/docs/data-model-queries.md index 23170cc..197594c 100644 --- a/docs/data-model-queries.md +++ b/docs/data-model-queries.md @@ -1,6 +1,6 @@ # Data Model and Queries -## Indexes and Frames +## Indexes and Fields *Index* and *field*s are the main data models of Pilosa. You can check the [Pilosa documentation](https://www.pilosa.com/docs) for more detail about the data model. @@ -11,7 +11,7 @@ schema = pilosa.Schema() repository = schema.index("repository") ``` -Frames are created with a call to `index.field` method: +Fields are created with a call to `index.field` method: ```python stargazer = repository.field("stargazer") @@ -25,12 +25,12 @@ stargazer = repository.field("stargazer", time_quantum=pilosa.TimeQuantum.YEAR_M ## Queries -Once you have indexes and field objects created, you can create queries for them. Some of the queries work on the columns; corresponding methods are attached to the index. Other queries work on rows, with related methods attached to frames. +Once you have indexes and field objects created, you can create queries for them. Some of the queries work on the columns; corresponding methods are attached to the index. Other queries work on rows, with related methods attached to fields. For instance, `Bitmap` queries work on rows; use a field object to create those queries: ```python -bitmap_query = stargazer.bitmap(1) # corresponds to PQL: Bitmap(field='stargazer', row=1) +bitmap_query = stargazer.row(1) # corresponds to PQL: Bitmap(field='stargazer', row=1) ``` `Union` queries work on columns; use the index object to create them: @@ -43,8 +43,8 @@ In order to increase throughput, you may want to batch queries sent to the Pilos ```python query = repository.batch_query( - stargazer.bitmap(1), - repository.union(stargazer.bitmap(100), stargazer.bitmap(5))) + stargazer.row(1), + repository.union(stargazer.row(100), stargazer.row(5))) ``` The recommended way of creating query objects is, using dedicated methods attached to index and field objects. But sometimes it would be desirable to send raw queries to Pilosa. You can use the `index.raw_query` method for that. Note that, query string is not validated before sending to the server: @@ -70,7 +70,7 @@ data = [3, 392, 47, 956, 219, 14, 47, 504, 21, 0, 123, 318] query = index.batch_query() for i, x in enumerate(data): column = i + 1 - query.add(captivity.set_value(column, x)) + query.add(captivity.setvalue(column, x)) client.query(query) ``` @@ -78,14 +78,14 @@ Let's write a range query: ```python # Query for all animals with more than 100 specimens response = client.query(captivity.gt(100)) -print(response.result.bitmap.bits) +print(response.result.row.columns) # Query for the total number of animals in captivity response = client.query(captivity.sum()) print(response.result.value) ``` -It's possible to pass a bitmap query to `sum`, so only columns where a row is set are filtered in: +It's possible to pass a row query to `sum`, so only columns where a row is set are filtered in: ```python # Let's run a few setbit queries first client.query(index.batch_query( @@ -93,7 +93,7 @@ client.query(index.batch_query( field.setbit(42, 6) )) # Query for the total number of animals in captivity where row 42 is set -response = client.query(captivity.sum(field.bitmap(42))) +response = client.query(captivity.sum(field.row(42))) print(response.result.value) ``` @@ -106,22 +106,22 @@ Index: * `union(self, *bitmaps)` * `intersect(self, *bitmaps)` * `difference(self, *bitmaps)` -* `count(self, bitmap)` +* `count(self, row)` * `set_column_attrs(self, column_id, attrs)` * `xor(self, *bitmaps)` Field: -* `bitmap(self, row_id)` +* `row(self, row_id)` * `setbit(self, row_id, column_id, timestamp=None)` * `clearbit(self, row_id, column_id)` -* `topn(self, n, bitmap=None, field="", *values)` +* `topn(self, n, row=None, field="", *values)` * `range(self, row_id, start, end)` * `set_row_attrs(self, row_id, attrs)` * (**deprecated**) `inverse_bitmap(self, column_id)` -* (**deprecated**) `inverse_topn(self, n, bitmap=None, field="", *values)` +* (**deprecated**) `inverse_topn(self, n, row=None, field="", *values)` * (**deprecated**) `inverse_range(self, column_id, start, end)` -* (**deprecated**) `sum(self, bitmap, field)` +* (**deprecated**) `sum(self, row, field)` * (**deprecated**) `set_field_value(self, column_id, field, value)` Field: @@ -131,7 +131,7 @@ Field: * `gt(self, n)` * `gte(self, n)` * `between(self, a, b)` -* `sum(self, bitmap=None)` -* `min(self, bitmap=None)` -* `max(self, bitmap=None)` -* `set_value(self, column_id, value)` +* `sum(self, row=None)` +* `min(self, row=None)` +* `max(self, row=None)` +* `setvalue(self, column_id, value)` diff --git a/docs/imports.md b/docs/imports.md index 3469414..e3cd629 100644 --- a/docs/imports.md +++ b/docs/imports.md @@ -2,7 +2,7 @@ If you have large amounts of data, it is more efficient to import it to Pilosa instead of several `SetBit` queries. -This library supports importing bits in the CSV (comma separated values) format: +This library supports importing columns in the CSV (comma separated values) format: ``` ROW_ID,COLUMN_ID ``` @@ -18,7 +18,7 @@ The target index and field must have been created before hand. Here's some sample code: ```python import pilosa -from pilosa.imports import csv_bit_reader +from pilosa.imports import csv_column_reader try: # python 2.7 and 3 @@ -33,7 +33,7 @@ text = u""" 3,41,683793385 10,10485760,683793385 """ -reader = csv_bit_reader(StringIO(text)) +reader = csv_column_reader(StringIO(text)) client = pilosa.Client() schema = client.schema() index = schema.index("sample-index") diff --git a/docs/server-interaction.md b/docs/server-interaction.md index cba4324..c536705 100644 --- a/docs/server-interaction.md +++ b/docs/server-interaction.md @@ -70,7 +70,7 @@ client = pilosa.Client(cluster, ) ``` -Once you create a client, you can create indexes, frames and start sending queries. +Once you create a client, you can create indexes, fields and start sending queries. Here is how you would create a index and field: @@ -84,13 +84,13 @@ client.sync_schema(schema) You can send queries to a Pilosa server using the `query` method of client objects: ```python -response = client.query(field.bitmap(5)) +response = client.query(field.row(5)) ``` `query` method accepts optional `columns` argument: ```python -response = client.query(field.bitmap(5), +response = client.query(field.row(5), columns=True # return column data in the response ) ``` @@ -102,7 +102,7 @@ When a query is sent to a Pilosa server, the server either fulfills the query or A `QueryResponse` object may contain zero or more results of `QueryResult` type. You can access all results using the `results` property of `QueryResponse` (which returns a list of `QueryResult` objects) or you can use the `result` property (which returns either the first result or `None` if there are no results): ```python -response = client.query(field.bitmap(5)) +response = client.query(field.row(5)) # check that there's a result and act on it result = response.result @@ -129,7 +129,7 @@ for column in response.columns: `QueryResult` objects contain: -* `bitmap` property to retrieve a bitmap result, +* `row` property to retrieve a row result, * `count_items` property to retrieve column count per row ID entries returned from `topn` queries, * `count` attribute to retrieve the number of rows per the given row ID returned from `count` queries. * `value` attribute to retrieve the result of `Min`, `Max` or `Sum` queries. @@ -137,9 +137,9 @@ for column in response.columns: ```python result = response.result -bitmap = result.bitmap -bits = bitmap.bits -attributes = bitmap.attributes +row = result.row +columns = row.columns +attributes = row.attributes count_items = result.count_items diff --git a/integration_tests/test_client_it.py b/integration_tests/test_client_it.py index 4588656..5288cd1 100644 --- a/integration_tests/test_client_it.py +++ b/integration_tests/test_client_it.py @@ -43,7 +43,7 @@ from pilosa.client import Client, URI, Cluster, PilosaServerError from pilosa.exceptions import PilosaError from pilosa.orm import Index, TimeQuantum, Schema, CacheType -from pilosa.imports import csv_bit_reader +from pilosa.imports import csv_column_reader class ClientIT(unittest.TestCase): @@ -92,12 +92,12 @@ def test_query_with_columns(self): client.query(field.setbit(100, 1000)) column_attrs = {"name": "bombo"} client.query(self.index.set_column_attrs(1000, column_attrs)) - response = client.query(field.bitmap(100), columns=True) + response = client.query(field.row(100), column_attrs=True) self.assertTrue(response is not None) self.assertEquals(1000, response.column.id) self.assertEquals({"name": "bombo"}, response.column.attributes) - response = client.query(field.bitmap(300)) + response = client.query(field.row(300)) self.assertTrue(response.column is None) def test_failed_connection(self): @@ -118,22 +118,22 @@ def test_orm_count(self): count_field.setbit(10, 21), count_field.setbit(15, 25)) client.query(qry) - response = client.query(self.index.count(count_field.bitmap(10))) + response = client.query(self.index.count(count_field.row(10))) self.assertEquals(2, response.result.count) def test_new_orm(self): client = self.get_client() client.query(self.field.setbit(10, 20)) - response1 = client.query(self.field.bitmap(10)) + response1 = client.query(self.field.row(10)) self.assertEquals(0, len(response1.columns)) - bitmap1 = response1.result.bitmap + bitmap1 = response1.result.row self.assertEquals(0, len(bitmap1.attributes)) - self.assertEquals(1, len(bitmap1.bits)) - self.assertEquals(20, bitmap1.bits[0]) + self.assertEquals(1, len(bitmap1.columns)) + self.assertEquals(20, bitmap1.columns[0]) column_attrs = {"name": "bombo"} client.query(self.col_index.set_column_attrs(20, column_attrs)) - response2 = client.query(self.field.bitmap(10), columns=True) + response2 = client.query(self.field.row(10), column_attrs=True) column = response2.column self.assertTrue(column is not None) self.assertEquals(20, column.id) @@ -145,9 +145,9 @@ def test_new_orm(self): "name": "Mr. Pi" } client.query(self.field.set_row_attrs(10, bitmap_attrs)) - response3 = client.query(self.field.bitmap(10)) - bitmap = response3.result.bitmap - self.assertEquals(1, len(bitmap.bits)) + response3 = client.query(self.field.row(10)) + bitmap = response3.result.row + self.assertEquals(1, len(bitmap.columns)) self.assertEquals(4, len(bitmap.attributes)) self.assertEquals(True, bitmap.attributes["active"]) self.assertEquals(5, bitmap.attributes["unsigned"]) @@ -202,19 +202,19 @@ def test_csv_import(self): 2, 3 7, 1 """ - reader = csv_bit_reader(StringIO(text)) + reader = csv_column_reader(StringIO(text)) field = self.index.field("importfield") client.ensure_field(field) client.import_field(field, reader) bq = self.index.batch_query( - field.bitmap(2), - field.bitmap(7), - field.bitmap(10), + field.row(2), + field.row(7), + field.row(10), ) response = client.query(bq) target = [3, 1, 5] self.assertEqual(3, len(response.results)) - self.assertEqual(target, [result.bitmap.bits[0] for result in response.results]) + self.assertEqual(target, [result.row.columns[0] for result in response.results]) def test_csv_import2(self): # Checks against encoding errors on Python 2.x @@ -224,7 +224,7 @@ def test_csv_import2(self): 3,41,683793385 10,10485760,683793385 """ - reader = csv_bit_reader(StringIO(text)) + reader = csv_column_reader(StringIO(text)) client = self.get_client() schema = client.schema() field = schema.index(self.index.name).field("importfield", time_quantum=TimeQuantum.YEAR_MONTH_DAY_HOUR) @@ -280,7 +280,7 @@ def test_sync(self): def test_failover_fail(self): uris = [URI.address("nonexistent%s" % i) for i in range(20)] client = Client(Cluster(*uris)) - self.assertRaises(PilosaError, client.query, self.field.bitmap(5)) + self.assertRaises(PilosaError, client.query, self.field.row(5)) def test_range_field(self): client = self.get_client() @@ -289,40 +289,40 @@ def test_range_field(self): client.query(self.col_index.batch_query( field.setbit(1, 10), field.setbit(1, 100), - field.set_value(10, 11), + field.setvalue(10, 11), )) - response = client.query(field.sum(field.bitmap(1))) + response = client.query(field.sum(field.row(1))) self.assertEquals(11, response.result.value) self.assertEquals(1, response.result.count) - response = client.query(field.min(field.bitmap(1))) + response = client.query(field.min(field.row(1))) self.assertEquals(11, response.result.value) self.assertEquals(1, response.result.count) - response = client.query(field.max(field.bitmap(1))) + response = client.query(field.max(field.row(1))) self.assertEquals(11, response.result.value) self.assertEquals(1, response.result.count) response = client.query(field.lt(15)) self.assertEquals(1, len(response.results)) - self.assertEquals(10, response.result.bitmap.bits[0]) + self.assertEquals(10, response.result.row.columns[0]) - def test_exclude_attrs_bits(self): + def test_exclude_attrs_columns(self): client = self.get_client() client.query(self.col_index.batch_query( self.field.setbit(1, 100), self.field.set_row_attrs(1, {"foo": "bar"}) )) - # test exclude bits. - response = client.query(self.field.bitmap(1), exclude_bits=True) - self.assertEquals(0, len(response.result.bitmap.bits)) - self.assertEquals(1, len(response.result.bitmap.attributes)) + # test exclude columns. + response = client.query(self.field.row(1), exclude_columns=True) + self.assertEquals(0, len(response.result.row.columns)) + self.assertEquals(1, len(response.result.row.attributes)) # test exclude attributes. - response = client.query(self.field.bitmap(1), exclude_attrs=True) - self.assertEquals(1, len(response.result.bitmap.bits)) - self.assertEquals(0, len(response.result.bitmap.attributes)) + response = client.query(self.field.row(1), exclude_attrs=True) + self.assertEquals(1, len(response.result.row.columns)) + self.assertEquals(0, len(response.result.row.attributes)) def test_http_request(self): self.get_client().http_request("GET", "/status") diff --git a/pilosa/client.py b/pilosa/client.py index ab84b04..648ebb1 100644 --- a/pilosa/client.py +++ b/pilosa/client.py @@ -40,7 +40,7 @@ import urllib3 from .exceptions import PilosaError, PilosaURIError, IndexExistsError, FieldExistsError -from .imports import batch_bits +from .imports import batch_columns from .internal import public_pb2 as internal from .orm import TimeQuantum, Schema, CacheType from .response import QueryResponse @@ -68,7 +68,7 @@ class Client(object): index = pilosa.Index("repository") stargazer = index.field("stargazer") - response = client.query(stargazer.bitmap(5)) + response = client.query(stargazer.row(5)) # Act on the result print(response.result) @@ -104,17 +104,17 @@ def __init__(self, cluster_or_uri=None, connect_timeout=30000, socket_timeout=30 self.__client = None self.logger = logging.getLogger("pilosa") - def query(self, query, columns=False, exclude_bits=False, exclude_attrs=False): + def query(self, query, column_attrs=False, exclude_columns=False, exclude_attrs=False): """Runs the given query against the server with the given options. :param pilosa.PqlQuery query: a PqlQuery object with a non-null index - :param bool columns: Enables returning column data from bitmap queries - :param bool exclude_bits: Disables returning bits from bitmap queries - :param bool exclude_attrs: Disables returning attributes from bitmap queries + :param bool column_attrs: Enables returning column data from row queries + :param bool exclude_columns: Disables returning columns from row queries + :param bool exclude_attrs: Disables returning attributes from row queries :return: Pilosa response :rtype: pilosa.Response """ - request = _QueryRequest(query.serialize(), columns=columns, exclude_bits=exclude_bits, exclude_attrs=exclude_attrs) + request = _QueryRequest(query.serialize(), column_attrs=column_attrs, exclude_columns=exclude_columns, exclude_row_attrs=exclude_attrs) path = "/index/%s/query" % query.index.name try: response = self.__http_request("POST", path, data=request.to_protobuf()) @@ -235,9 +235,9 @@ def import_field(self, field, bit_reader, batch_size=100000): """ index_name = field.index.name field_name = field.name - import_bits = self._import_bits - for slice, bits in batch_bits(bit_reader, batch_size): - import_bits(index_name, field_name, slice, bits) + import_columns = self._import_columns + for slice, columns in batch_columns(bit_reader, batch_size): + import_columns(index_name, field_name, slice, columns) def http_request(self, method, path, data=None, headers=None): """Sends an HTTP request to the Pilosa server @@ -253,9 +253,9 @@ def http_request(self, method, path, data=None, headers=None): """ return self.__http_request(method, path, data=data, headers=headers) - def _import_bits(self, index_name, field_name, slice, bits): + def _import_columns(self, index_name, field_name, slice, columns): # sort by row_id then by column_id - bits.sort(key=lambda bit: (bit.row_id, bit.column_id)) + columns.sort(key=lambda bit: (bit.row_id, bit.column_id)) nodes = self._fetch_fragment_nodes(index_name, slice) # copy client params client_params = {} @@ -269,7 +269,7 @@ def _import_bits(self, index_name, field_name, slice, bits): client_params[k] = v for node in nodes: client = Client(URI.address(node.url), **client_params) - client._import_node(_ImportRequest(index_name, field_name, slice, bits)) + client._import_node(_ImportRequest(index_name, field_name, slice, columns)) def _fetch_fragment_nodes(self, index_name, slice): path = "/fragment/nodes?slice=%d&index=%s" % (slice, index_name) @@ -492,18 +492,18 @@ def _reset(self): class _QueryRequest: - def __init__(self, query, columns=False, exclude_bits=False, exclude_attrs=False): + def __init__(self, query, column_attrs=False, exclude_columns=False, exclude_row_attrs=False): self.query = query - self.columns = columns - self.exclude_bits = exclude_bits - self.exclude_attrs = exclude_attrs + self.column_attrs = column_attrs + self.exclude_columns = exclude_columns + self.exclude_row_attrs = exclude_row_attrs def to_protobuf(self, return_bytearray=_IS_PY2): qr = internal.QueryRequest() qr.Query = self.query - qr.ColumnAttrs = self.columns - qr.ExcludeBits = self.exclude_bits - qr.ExcludeAttrs = self.exclude_attrs + qr.ColumnAttrs = self.column_attrs + qr.ExcludeColumns = self.exclude_columns + qr.ExcludeRowAttrs = self.exclude_row_attrs if return_bytearray: return bytearray(qr.SerializeToString()) return qr.SerializeToString() @@ -511,21 +511,21 @@ def to_protobuf(self, return_bytearray=_IS_PY2): class _ImportRequest: - def __init__(self, index_name, field_name, slice, bits): + def __init__(self, index_name, field_name, slice, columns): self.index_name = index_name self.field_name = field_name self.slice = slice - self.bits = bits + self.columns = columns def to_protobuf(self, return_bytearray=_IS_PY2): import_request = internal.ImportRequest() import_request.Index = self.index_name - import_request.Frame = self.field_name + import_request.Field = self.field_name import_request.Slice = self.slice row_ids = import_request.RowIDs column_ids = import_request.ColumnIDs timestamps = import_request.Timestamps - for bit in self.bits: + for bit in self.columns: row_ids.append(bit.row_id) column_ids.append(bit.column_id) timestamps.append(bit.timestamp) diff --git a/pilosa/imports.py b/pilosa/imports.py index f3a7e7c..c80c0ae 100644 --- a/pilosa/imports.py +++ b/pilosa/imports.py @@ -36,14 +36,14 @@ from pilosa.exceptions import PilosaError -__all__ = ("Bit", "csv_bit_reader") +__all__ = ("Columns", "csv_column_reader") -Bit = namedtuple("Bit", "row_id column_id timestamp") +Columns = namedtuple("Column", "row_id column_id timestamp") -def csv_bit_reader(file_obj, timefunc=int): +def csv_column_reader(file_obj, timefunc=int): """ - Reads bits from the given file-like object. + Reads columns from the given file-like object. Each line of the file-like object should correspond to a single bit and must be in the following form: rowID,columnID[,timestamp] @@ -59,12 +59,12 @@ def csv_bit_reader(file_obj, timefunc=int): parts = line.split(",") if len(parts) == 2: try: - bit = Bit(row_id=int(parts[0]), column_id=int(parts[1]), timestamp=0) + bit = Columns(row_id=int(parts[0]), column_id=int(parts[1]), timestamp=0) except ValueError: raise PilosaError("Invalid CSV line: %s", line) elif len(parts) == 3: try: - bit = Bit(row_id=int(parts[0]), column_id=int(parts[1]), timestamp=timefunc(parts[2])) + bit = Columns(row_id=int(parts[0]), column_id=int(parts[1]), timestamp=timefunc(parts[2])) except ValueError: raise PilosaError("Invalid CSV line: %s", line) else: @@ -72,7 +72,7 @@ def csv_bit_reader(file_obj, timefunc=int): yield bit -def batch_bits(reader, batch_size): +def batch_columns(reader, batch_size): slice_width = 1048576 while 1: batch = list(itertools.islice(reader, batch_size)) diff --git a/pilosa/internal/public.proto b/pilosa/internal/public.proto index 026a374..cd34b88 100644 --- a/pilosa/internal/public.proto +++ b/pilosa/internal/public.proto @@ -2,8 +2,8 @@ syntax = "proto3"; package internal; -message Bitmap { - repeated uint64 Bits = 1; +message Row { + repeated uint64 Columns = 1; repeated string Keys = 3; repeated Attr Attrs = 2; } @@ -49,8 +49,8 @@ message QueryRequest { repeated uint64 Slices = 2; bool ColumnAttrs = 3; bool Remote = 5; - bool ExcludeAttrs = 6; - bool ExcludeBits = 7; + bool ExcludeRowAttrs = 6; + bool ExcludeColumns = 7; } message QueryResponse { @@ -61,7 +61,7 @@ message QueryResponse { message QueryResult { uint32 Type = 6; - Bitmap Bitmap = 1; + Row Row = 1; uint64 N = 2; repeated Pair Pairs = 3; ValCount ValCount = 5; @@ -70,7 +70,7 @@ message QueryResult { message ImportRequest { string Index = 1; - string Frame = 2; + string Field = 2; uint64 Slice = 3; repeated uint64 RowIDs = 4; repeated uint64 ColumnIDs = 5; @@ -81,9 +81,8 @@ message ImportRequest { message ImportValueRequest { string Index = 1; - string Frame = 2; + string Field = 2; uint64 Slice = 3; - string Field = 4; repeated uint64 ColumnIDs = 5; repeated string ColumnKeys = 7; repeated int64 Values = 6; diff --git a/pilosa/internal/public_pb2.py b/pilosa/internal/public_pb2.py index 1c865d7..faec967 100644 --- a/pilosa/internal/public_pb2.py +++ b/pilosa/internal/public_pb2.py @@ -19,36 +19,36 @@ name='public.proto', package='internal', syntax='proto3', - serialized_pb=_b('\n\x0cpublic.proto\x12\x08internal\"C\n\x06\x42itmap\x12\x0c\n\x04\x42its\x18\x01 \x03(\x04\x12\x0c\n\x04Keys\x18\x03 \x03(\t\x12\x1d\n\x05\x41ttrs\x18\x02 \x03(\x0b\x32\x0e.internal.Attr\".\n\x04Pair\x12\n\n\x02ID\x18\x01 \x01(\x04\x12\x0b\n\x03Key\x18\x03 \x01(\t\x12\r\n\x05\x43ount\x18\x02 \x01(\x04\"&\n\x08ValCount\x12\x0b\n\x03Val\x18\x01 \x01(\x03\x12\r\n\x05\x43ount\x18\x02 \x01(\x03\"9\n\x03\x42it\x12\r\n\x05RowID\x18\x01 \x01(\x04\x12\x10\n\x08\x43olumnID\x18\x02 \x01(\x04\x12\x11\n\tTimestamp\x18\x03 \x01(\x03\"G\n\rColumnAttrSet\x12\n\n\x02ID\x18\x01 \x01(\x04\x12\x0b\n\x03Key\x18\x03 \x01(\t\x12\x1d\n\x05\x41ttrs\x18\x02 \x03(\x0b\x32\x0e.internal.Attr\"o\n\x04\x41ttr\x12\x0b\n\x03Key\x18\x01 \x01(\t\x12\x0c\n\x04Type\x18\x02 \x01(\x04\x12\x13\n\x0bStringValue\x18\x03 \x01(\t\x12\x10\n\x08IntValue\x18\x04 \x01(\x03\x12\x11\n\tBoolValue\x18\x05 \x01(\x08\x12\x12\n\nFloatValue\x18\x06 \x01(\x01\"(\n\x07\x41ttrMap\x12\x1d\n\x05\x41ttrs\x18\x01 \x03(\x0b\x32\x0e.internal.Attr\"}\n\x0cQueryRequest\x12\r\n\x05Query\x18\x01 \x01(\t\x12\x0e\n\x06Slices\x18\x02 \x03(\x04\x12\x13\n\x0b\x43olumnAttrs\x18\x03 \x01(\x08\x12\x0e\n\x06Remote\x18\x05 \x01(\x08\x12\x14\n\x0c\x45xcludeAttrs\x18\x06 \x01(\x08\x12\x13\n\x0b\x45xcludeBits\x18\x07 \x01(\x08\"u\n\rQueryResponse\x12\x0b\n\x03\x45rr\x18\x01 \x01(\t\x12&\n\x07Results\x18\x02 \x03(\x0b\x32\x15.internal.QueryResult\x12/\n\x0e\x43olumnAttrSets\x18\x03 \x03(\x0b\x32\x17.internal.ColumnAttrSet\"\x9e\x01\n\x0bQueryResult\x12\x0c\n\x04Type\x18\x06 \x01(\r\x12 \n\x06\x42itmap\x18\x01 \x01(\x0b\x32\x10.internal.Bitmap\x12\t\n\x01N\x18\x02 \x01(\x04\x12\x1d\n\x05Pairs\x18\x03 \x03(\x0b\x32\x0e.internal.Pair\x12$\n\x08ValCount\x18\x05 \x01(\x0b\x32\x12.internal.ValCount\x12\x0f\n\x07\x43hanged\x18\x04 \x01(\x08\"\x98\x01\n\rImportRequest\x12\r\n\x05Index\x18\x01 \x01(\t\x12\r\n\x05\x46rame\x18\x02 \x01(\t\x12\r\n\x05Slice\x18\x03 \x01(\x04\x12\x0e\n\x06RowIDs\x18\x04 \x03(\x04\x12\x11\n\tColumnIDs\x18\x05 \x03(\x04\x12\x0f\n\x07RowKeys\x18\x07 \x03(\t\x12\x12\n\nColumnKeys\x18\x08 \x03(\t\x12\x12\n\nTimestamps\x18\x06 \x03(\x03\"\x87\x01\n\x12ImportValueRequest\x12\r\n\x05Index\x18\x01 \x01(\t\x12\r\n\x05\x46rame\x18\x02 \x01(\t\x12\r\n\x05Slice\x18\x03 \x01(\x04\x12\r\n\x05\x46ield\x18\x04 \x01(\t\x12\x11\n\tColumnIDs\x18\x05 \x03(\x04\x12\x12\n\nColumnKeys\x18\x07 \x03(\t\x12\x0e\n\x06Values\x18\x06 \x03(\x03\x62\x06proto3') + serialized_pb=_b('\n\x0cpublic.proto\x12\x08internal\"C\n\x03Row\x12\x0f\n\x07\x43olumns\x18\x01 \x03(\x04\x12\x0c\n\x04Keys\x18\x03 \x03(\t\x12\x1d\n\x05\x41ttrs\x18\x02 \x03(\x0b\x32\x0e.internal.Attr\".\n\x04Pair\x12\n\n\x02ID\x18\x01 \x01(\x04\x12\x0b\n\x03Key\x18\x03 \x01(\t\x12\r\n\x05\x43ount\x18\x02 \x01(\x04\"&\n\x08ValCount\x12\x0b\n\x03Val\x18\x01 \x01(\x03\x12\r\n\x05\x43ount\x18\x02 \x01(\x03\"9\n\x03\x42it\x12\r\n\x05RowID\x18\x01 \x01(\x04\x12\x10\n\x08\x43olumnID\x18\x02 \x01(\x04\x12\x11\n\tTimestamp\x18\x03 \x01(\x03\"G\n\rColumnAttrSet\x12\n\n\x02ID\x18\x01 \x01(\x04\x12\x0b\n\x03Key\x18\x03 \x01(\t\x12\x1d\n\x05\x41ttrs\x18\x02 \x03(\x0b\x32\x0e.internal.Attr\"o\n\x04\x41ttr\x12\x0b\n\x03Key\x18\x01 \x01(\t\x12\x0c\n\x04Type\x18\x02 \x01(\x04\x12\x13\n\x0bStringValue\x18\x03 \x01(\t\x12\x10\n\x08IntValue\x18\x04 \x01(\x03\x12\x11\n\tBoolValue\x18\x05 \x01(\x08\x12\x12\n\nFloatValue\x18\x06 \x01(\x01\"(\n\x07\x41ttrMap\x12\x1d\n\x05\x41ttrs\x18\x01 \x03(\x0b\x32\x0e.internal.Attr\"\x83\x01\n\x0cQueryRequest\x12\r\n\x05Query\x18\x01 \x01(\t\x12\x0e\n\x06Slices\x18\x02 \x03(\x04\x12\x13\n\x0b\x43olumnAttrs\x18\x03 \x01(\x08\x12\x0e\n\x06Remote\x18\x05 \x01(\x08\x12\x17\n\x0f\x45xcludeRowAttrs\x18\x06 \x01(\x08\x12\x16\n\x0e\x45xcludeColumns\x18\x07 \x01(\x08\"u\n\rQueryResponse\x12\x0b\n\x03\x45rr\x18\x01 \x01(\t\x12&\n\x07Results\x18\x02 \x03(\x0b\x32\x15.internal.QueryResult\x12/\n\x0e\x43olumnAttrSets\x18\x03 \x03(\x0b\x32\x17.internal.ColumnAttrSet\"\x98\x01\n\x0bQueryResult\x12\x0c\n\x04Type\x18\x06 \x01(\r\x12\x1a\n\x03Row\x18\x01 \x01(\x0b\x32\r.internal.Row\x12\t\n\x01N\x18\x02 \x01(\x04\x12\x1d\n\x05Pairs\x18\x03 \x03(\x0b\x32\x0e.internal.Pair\x12$\n\x08ValCount\x18\x05 \x01(\x0b\x32\x12.internal.ValCount\x12\x0f\n\x07\x43hanged\x18\x04 \x01(\x08\"\x98\x01\n\rImportRequest\x12\r\n\x05Index\x18\x01 \x01(\t\x12\r\n\x05\x46ield\x18\x02 \x01(\t\x12\r\n\x05Slice\x18\x03 \x01(\x04\x12\x0e\n\x06RowIDs\x18\x04 \x03(\x04\x12\x11\n\tColumnIDs\x18\x05 \x03(\x04\x12\x0f\n\x07RowKeys\x18\x07 \x03(\t\x12\x12\n\nColumnKeys\x18\x08 \x03(\t\x12\x12\n\nTimestamps\x18\x06 \x03(\x03\"x\n\x12ImportValueRequest\x12\r\n\x05Index\x18\x01 \x01(\t\x12\r\n\x05\x46ield\x18\x02 \x01(\t\x12\r\n\x05Slice\x18\x03 \x01(\x04\x12\x11\n\tColumnIDs\x18\x05 \x03(\x04\x12\x12\n\nColumnKeys\x18\x07 \x03(\t\x12\x0e\n\x06Values\x18\x06 \x03(\x03\x62\x06proto3') ) _sym_db.RegisterFileDescriptor(DESCRIPTOR) -_BITMAP = _descriptor.Descriptor( - name='Bitmap', - full_name='internal.Bitmap', +_ROW = _descriptor.Descriptor( + name='Row', + full_name='internal.Row', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( - name='Bits', full_name='internal.Bitmap.Bits', index=0, + name='Columns', full_name='internal.Row.Columns', index=0, number=1, type=4, cpp_type=4, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='Keys', full_name='internal.Bitmap.Keys', index=1, + name='Keys', full_name='internal.Row.Keys', index=1, number=3, type=9, cpp_type=9, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='Attrs', full_name='internal.Bitmap.Attrs', index=2, + name='Attrs', full_name='internal.Row.Attrs', index=2, number=2, type=11, cpp_type=10, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, @@ -377,14 +377,14 @@ is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='ExcludeAttrs', full_name='internal.QueryRequest.ExcludeAttrs', index=4, + name='ExcludeRowAttrs', full_name='internal.QueryRequest.ExcludeRowAttrs', index=4, number=6, type=8, cpp_type=7, label=1, has_default_value=False, default_value=False, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='ExcludeBits', full_name='internal.QueryRequest.ExcludeBits', index=5, + name='ExcludeColumns', full_name='internal.QueryRequest.ExcludeColumns', index=5, number=7, type=8, cpp_type=7, label=1, has_default_value=False, default_value=False, message_type=None, enum_type=None, containing_type=None, @@ -402,8 +402,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=470, - serialized_end=595, + serialized_start=471, + serialized_end=602, ) @@ -447,8 +447,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=597, - serialized_end=714, + serialized_start=604, + serialized_end=721, ) @@ -467,7 +467,7 @@ is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='Bitmap', full_name='internal.QueryResult.Bitmap', index=1, + name='Row', full_name='internal.QueryResult.Row', index=1, number=1, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, @@ -513,8 +513,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=717, - serialized_end=875, + serialized_start=724, + serialized_end=876, ) @@ -533,7 +533,7 @@ is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='Frame', full_name='internal.ImportRequest.Frame', index=1, + name='Field', full_name='internal.ImportRequest.Field', index=1, number=2, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, @@ -593,8 +593,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=878, - serialized_end=1030, + serialized_start=879, + serialized_end=1031, ) @@ -613,7 +613,7 @@ is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='Frame', full_name='internal.ImportValueRequest.Frame', index=1, + name='Field', full_name='internal.ImportValueRequest.Field', index=1, number=2, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, @@ -627,28 +627,21 @@ is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='Field', full_name='internal.ImportValueRequest.Field', index=3, - number=4, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='ColumnIDs', full_name='internal.ImportValueRequest.ColumnIDs', index=4, + name='ColumnIDs', full_name='internal.ImportValueRequest.ColumnIDs', index=3, number=5, type=4, cpp_type=4, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='ColumnKeys', full_name='internal.ImportValueRequest.ColumnKeys', index=5, + name='ColumnKeys', full_name='internal.ImportValueRequest.ColumnKeys', index=4, number=7, type=9, cpp_type=9, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='Values', full_name='internal.ImportValueRequest.Values', index=6, + name='Values', full_name='internal.ImportValueRequest.Values', index=5, number=6, type=3, cpp_type=2, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, @@ -667,18 +660,18 @@ oneofs=[ ], serialized_start=1033, - serialized_end=1168, + serialized_end=1153, ) -_BITMAP.fields_by_name['Attrs'].message_type = _ATTR +_ROW.fields_by_name['Attrs'].message_type = _ATTR _COLUMNATTRSET.fields_by_name['Attrs'].message_type = _ATTR _ATTRMAP.fields_by_name['Attrs'].message_type = _ATTR _QUERYRESPONSE.fields_by_name['Results'].message_type = _QUERYRESULT _QUERYRESPONSE.fields_by_name['ColumnAttrSets'].message_type = _COLUMNATTRSET -_QUERYRESULT.fields_by_name['Bitmap'].message_type = _BITMAP +_QUERYRESULT.fields_by_name['Row'].message_type = _ROW _QUERYRESULT.fields_by_name['Pairs'].message_type = _PAIR _QUERYRESULT.fields_by_name['ValCount'].message_type = _VALCOUNT -DESCRIPTOR.message_types_by_name['Bitmap'] = _BITMAP +DESCRIPTOR.message_types_by_name['Row'] = _ROW DESCRIPTOR.message_types_by_name['Pair'] = _PAIR DESCRIPTOR.message_types_by_name['ValCount'] = _VALCOUNT DESCRIPTOR.message_types_by_name['Bit'] = _BIT @@ -691,12 +684,12 @@ DESCRIPTOR.message_types_by_name['ImportRequest'] = _IMPORTREQUEST DESCRIPTOR.message_types_by_name['ImportValueRequest'] = _IMPORTVALUEREQUEST -Bitmap = _reflection.GeneratedProtocolMessageType('Bitmap', (_message.Message,), dict( - DESCRIPTOR = _BITMAP, +Row = _reflection.GeneratedProtocolMessageType('Row', (_message.Message,), dict( + DESCRIPTOR = _ROW, __module__ = 'public_pb2' - # @@protoc_insertion_point(class_scope:internal.Bitmap) + # @@protoc_insertion_point(class_scope:internal.Row) )) -_sym_db.RegisterMessage(Bitmap) +_sym_db.RegisterMessage(Row) Pair = _reflection.GeneratedProtocolMessageType('Pair', (_message.Message,), dict( DESCRIPTOR = _PAIR, diff --git a/pilosa/orm.py b/pilosa/orm.py index 95f10f7..7e3b24e 100644 --- a/pilosa/orm.py +++ b/pilosa/orm.py @@ -242,68 +242,68 @@ def batch_query(self, *queries): q.add(*queries) return q - def union(self, *bitmaps): + def union(self, *rows): """Creates a ``Union`` query. - ``Union`` performs a logical OR on the results of each BITMAP_CALL query passed to it. + ``Union`` performs a logical OR on the results of each ROW_CALL query passed to it. - :param pilosa.PQLBitmapQuery bitmaps: 0 or more bitmap queries to union - :return: Pilosa bitmap query - :rtype: pilosa.PQLBitmapQuery + :param pilosa.PQLQuery rows: 0 or more row queries to union + :return: Pilosa row query + :rtype: pilosa.PQLQuery """ - return self._bitmap_op("Union", bitmaps) + return self._row_op("Union", rows) - def intersect(self, *bitmaps): + def intersect(self, *rows): """Creates an ``Intersect`` query. ``Intersect`` performs a logical AND on the results of each BITMAP_CALL query passed to it. - :param pilosa.PQLBitmapQuery bitmaps: 1 or more bitmap queries to intersect - :return: Pilosa bitmap query - :rtype: pilosa.PQLBitmapQuery - :raise PilosaError: if the number of bitmaps is less than 1 + :param pilosa.PQLQuery rows: 1 or more row queries to intersect + :return: Pilosa row query + :rtype: pilosa.PQLQuery + :raise PilosaError: if the number of rows is less than 1 """ - if len(bitmaps) < 1: - raise PilosaError("Number of bitmap queries should be greater than or equal to 1") - return self._bitmap_op("Intersect", bitmaps) + if len(rows) < 1: + raise PilosaError("Number of row queries should be greater than or equal to 1") + return self._row_op("Intersect", rows) - def difference(self, *bitmaps): + def difference(self, *rows): """Creates a ``Difference`` query. - ``Difference`` returns all of the bits from the first BITMAP_CALL argument passed to it, - without the bits from each subsequent BITMAP_CALL. + ``Difference`` returns all of the columns from the first ROW_CALL argument passed to it, + without the columns from each subsequent ROW_CALL. - :param pilosa.PQLBitmapQuery bitmaps: 1 or more bitmap queries to differentiate - :return: Pilosa bitmap query - :rtype: pilosa.PQLBitmapQuery - :raise PilosaError: if the number of bitmaps is less than 1 + :param pilosa.PQLQuery rows: 1 or more row queries to differentiate + :return: Pilosa row query + :rtype: pilosa.PQLQuery + :raise PilosaError: if the number of rows is less than 1 """ - if len(bitmaps) < 1: - raise PilosaError("Number of bitmap queries should be greater than or equal to 1") - return self._bitmap_op("Difference", bitmaps) + if len(rows) < 1: + raise PilosaError("Number of row queries should be greater than or equal to 1") + return self._row_op("Difference", rows) - def xor(self, *bitmaps): + def xor(self, *rows): """Creates a ``Xor`` query. - :param pilosa.PQLBitmapQuery bitmaps: 2 or more bitmap queries to xor - :return: Pilosa bitmap query - :rtype: pilosa.PQLBitmapQuery - :raise PilosaError: if the number of bitmaps is less than 2 + :param pilosa.PQLQuery rows: 2 or more row queries to xor + :return: Pilosa row query + :rtype: pilosa.PQLQuery + :raise PilosaError: if the number of rows is less than 2 """ - if len(bitmaps) < 2: - raise PilosaError("Number of bitmap queries should be greater than or equal to 2") - return self._bitmap_op("Xor", bitmaps) + if len(rows) < 2: + raise PilosaError("Number of row queries should be greater than or equal to 2") + return self._row_op("Xor", rows) - def count(self, bitmap): + def count(self, row): """Creates a Count query. - ``Count`` returns the number of set bits in the BITMAP_CALL passed in. + ``Count`` returns the number of set columns in the ROW_CALL passed in. - :param pilosa.PQLQuery bitmap: the bitmap query + :param pilosa.PQLQuery row: the row query :return: Pilosa query :rtype: pilosa.PQLQuery """ - return PQLQuery(u"Count(%s)" % bitmap.serialize(), self) + return PQLQuery(u"Count(%s)" % row.serialize(), self) def set_column_attrs(self, column_idkey, attrs): """Creates a SetColumnAttrs query. @@ -328,8 +328,8 @@ def set_column_attrs(self, column_idkey, attrs): attrs_str = _create_attributes_str(attrs) return PQLQuery(fmt % (column_idkey, attrs_str), self) - def _bitmap_op(self, name, bitmaps): - return PQLQuery(u"%s(%s)" % (name, u", ".join(b.serialize() for b in bitmaps)), self) + def _row_op(self, name, rows): + return PQLQuery(u"%s(%s)" % (name, u", ".join(b.serialize() for b in rows)), self) class Field: @@ -379,16 +379,16 @@ def copy(self): return Field(self.index, self.name, self.time_quantum, self.cache_type, self.cache_size, self.int_min, self.int_max) - def bitmap(self, row_idkey): - """Creates a Bitmap query. + def row(self, row_idkey): + """Creates a Row query. - Bitmap retrieves the indices of all the set bits in a row or column based on whether the row label or column label is given in the query. It also retrieves any attributes set on that row or column. + Row retrieves the indices of all the set columns in a row or column based on whether the row label or column label is given in the query. It also retrieves any attributes set on that row or column. - This variant of Bitmap query uses the row label. + This variant of Row query uses the row label. :param int row_idkey: - :return: Pilosa bitmap query - :rtype: pilosa.PQLBitmapQuery + :return: Pilosa row query + :rtype: pilosa.PQLQuery """ fmt = id_key_format("Row ID/Key", row_idkey, u"Bitmap(row=%s, field='%s')", @@ -437,21 +437,21 @@ def clearbit(self, row_idkey, column_idkey): raise ValidationError("Both Row and Columns must be integers or strings") return PQLQuery(fmt % (row_idkey, self.name, column_idkey), self.index) - def topn(self, n, bitmap=None, field="", *values): + def topn(self, n, row=None, field="", *values): """Creates a TopN query. - ``TopN`` returns the id and count of the top n bitmaps (by count of bits) in the field. + ``TopN`` returns the id and count of the top n rows (by count of columns) in the field. * see: `TopN Query `_ :param int n: number of items to return - :param pilosa.PQLBitmapQuery bitmap: a PQL Bitmap query + :param pilosa.PQLQuery row: a PQL Row query :param field str field: field name :param object values: filter values to be matched against the field """ parts = ["field='%s'" % self.name, "n=%d" % n] - if bitmap: - parts.insert(0, bitmap.serialize()) + if row: + parts.insert(0, row.serialize()) if field: validate_label(field) values_str = json.dumps(values, separators=(',', ': ')) @@ -462,7 +462,7 @@ def topn(self, n, bitmap=None, field="", *values): def range(self, row_idkey, start, end): """Creates a Range query. - Similar to ``Bitmap``, but only returns bits which were set with timestamps between the given start and end timestamps. + Similar to ``Row``, but only returns columns which were set with timestamps between the given start and end timestamps. * see: `Range Query `_ @@ -575,34 +575,34 @@ def between(self, a, b): q = u"Range(%s >< [%d,%d])" % (self.name, a, b) return PQLQuery(q, self.index) - def sum(self, bitmap=None): + def sum(self, row=None): """Creates a Sum query. - :param bitmap: The bitmap query to use. + :param row: The row query to use. :return: a PQL query :rtype: PQLQuery """ - return self._value_query("Sum", bitmap) + return self._value_query("Sum", row) - def min(self, bitmap=None): + def min(self, row=None): """Creates a Min query. - :param bitmap: The bitmap query to use. + :param row: The row query to use. :return: a PQL query :rtype: PQLQuery """ - return self._value_query("Min", bitmap) + return self._value_query("Min", row) - def max(self, bitmap=None): + def max(self, row=None): """Creates a Max query. - :param bitmap: The bitmap query to use. + :param row: The row query to use. :return: a PQL query :rtype: PQLQuery """ - return self._value_query("Max", bitmap) + return self._value_query("Max", row) - def set_value(self, column_id, value): + def setvalue(self, column_id, value): """Creates a SetValue query. :param column_id: column ID @@ -617,9 +617,9 @@ def _binary_operation(self, op, n): q = u"Range(%s %s %d)" % (self.name, op, n) return PQLQuery(q, self.index) - def _value_query(self, op, bitmap): - bitmap_str = "%s, " % bitmap.serialize() if bitmap else "" - q = u"%s(%sfield='%s')" % (op, bitmap_str, self.name) + def _value_query(self, op, row): + row_str = "%s, " % row.serialize() if row else "" + q = u"%s(%sfield='%s')" % (op, row_str, self.name) return PQLQuery(q, self.index) def _get_options_string(self): diff --git a/pilosa/response.py b/pilosa/response.py index 69017d6..a69feb0 100644 --- a/pilosa/response.py +++ b/pilosa/response.py @@ -35,27 +35,27 @@ from .internal import public_pb2 as internal -__all__ = ("BitmapResult", "CountResultItem", "QueryResult", "ColumnItem", "QueryResponse") +__all__ = ("RowResult", "CountResultItem", "QueryResult", "ColumnItem", "QueryResponse") -QUERYRESULT_NONE, QUERYRESULT_BITMAP, QUERYRESULT_PAIRS, \ - QUERYRESULT_VAL_COUNT, QUERYRESULT_INT, QUERYRESULT_BOOL = range(6) +QUERYRESULT_NONE, QUERYRESULT_ROW, QUERYRESULT_PAIRS, \ +QUERYRESULT_VAL_COUNT, QUERYRESULT_INT, QUERYRESULT_BOOL = range(6) -class BitmapResult: - """Represents a result from ``Bitmap``, ``Union``, ``Intersect``, ``Difference`` and ``Range`` PQL calls. +class RowResult: + """Represents a result from ``Row``, ``Union``, ``Intersect``, ``Difference`` and ``Range`` PQL calls. * See `Query Language `_ """ - def __init__(self, bits=None, keys=None, attributes=None): - self.bits = bits or [] + def __init__(self, columns=None, keys=None, attributes=None): + self.columns = columns or [] self.keys = keys or [] self.attributes = attributes or {} @classmethod def from_internal(cls, obj): - return cls(list(obj.Bits), obj.Keys, _convert_protobuf_attrs_to_dict(obj.Attrs)) + return cls(list(obj.Columns), obj.Keys, _convert_protobuf_attrs_to_dict(obj.Attrs)) class CountResultItem: @@ -76,8 +76,8 @@ class QueryResult: * See `Query Language `_ """ - def __init__(self, bitmap=None, count_items=None, count=0, value=0, changed=False): - self.bitmap = bitmap or BitmapResult() + def __init__(self, row=None, count_items=None, count=0, value=0, changed=False): + self.row = row or RowResult() self.count_items = count_items or [] self.count = count self.value = value @@ -85,14 +85,14 @@ def __init__(self, bitmap=None, count_items=None, count=0, value=0, changed=Fals @classmethod def from_internal(cls, obj): - bitmap = None + row = None count_items = [] count = 0 value = 0 changed = False - if obj.Type == QUERYRESULT_BITMAP: - bitmap = BitmapResult.from_internal(obj.Bitmap) + if obj.Type == QUERYRESULT_ROW: + row = RowResult.from_internal(obj.Row) elif obj.Type == QUERYRESULT_PAIRS: for pair in obj.Pairs: count_items.append(CountResultItem(pair.ID, pair.Key, pair.Count)) @@ -108,7 +108,7 @@ def from_internal(cls, obj): else: raise PilosaError("Unknown type: %s" % obj.Type) - return cls(bitmap, count_items, count, value, changed) + return cls(row, count_items, count, value, changed) class ColumnItem: diff --git a/pilosa/validator.py b/pilosa/validator.py index 78e18fa..500d4e2 100644 --- a/pilosa/validator.py +++ b/pilosa/validator.py @@ -35,16 +35,16 @@ from pilosa.exceptions import ValidationError -__all__ = ("valid_index_name", "validate_index_name", "valid_frame_name", +__all__ = ("valid_index_name", "validate_index_name", "valid_field_name", "validate_field_name", "valid_label", "validate_label") __INDEX_NAME = re.compile(r"^[a-z][a-z0-9_-]*$") -__FRAME_NAME = re.compile(r"^[a-z][a-z0-9_-]*$") +__FIELD_NAME = re.compile(r"^[a-z][a-z0-9_-]*$") __LABEL = re.compile(r"^[a-zA-Z][a-zA-Z0-9_-]*$") __KEY = re.compile(r"^[A-Za-z0-9_{}+/=.~%:-]*$") __MAX_INDEX_NAME = 64 -__MAX_FRAME_NAME = 64 +__MAX_FIELD_NAME = 64 __MAX_LABEL = 64 __MAX_KEY = 64 @@ -60,10 +60,10 @@ def validate_index_name(index_name): raise ValidationError("Invalid index name: %s" % index_name) -def valid_frame_name(frame_name): - if len(frame_name) > __MAX_FRAME_NAME: +def valid_field_name(field_name): + if len(field_name) > __MAX_FIELD_NAME: return False - return bool(__FRAME_NAME.match(frame_name)) + return bool(__FIELD_NAME.match(field_name)) def valid_key(key): @@ -72,9 +72,9 @@ def valid_key(key): return bool(__KEY.match(key)) -def validate_field_name(frame_name): - if not valid_frame_name(frame_name): - raise ValidationError("Invalid field name: %s" % frame_name) +def validate_field_name(field_name): + if not valid_field_name(field_name): + raise ValidationError("Invalid field name: %s" % field_name) def valid_label(label): diff --git a/tests/test_client.py b/tests/test_client.py index eb9c690..32a6965 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -39,7 +39,7 @@ from pilosa.client import Client, URI, Cluster, _QueryRequest, \ decode_field_meta_options, _ImportRequest, _Node from pilosa.exceptions import PilosaURIError, PilosaError -from pilosa.imports import Bit +from pilosa.imports import Columns logger = logging.getLogger(__name__) @@ -59,9 +59,9 @@ def test_create_client(self): # create with invalid type self.assertRaises(PilosaError, Client, 15000) - def test_decode_frame_meta_options(self): - frame_info = {} - options = decode_field_meta_options(frame_info) + def test_decode_field_meta_options(self): + field_info = {} + options = decode_field_meta_options(field_info) target = { "cache_size": 50000, "cache_type": CacheType.DEFAULT, @@ -220,7 +220,7 @@ def test_cluster_reset(self): class QueryRequestTestCase(unittest.TestCase): def test_serialize(self): - qr = _QueryRequest("Bitmap(field='foo', id=1)", columns=True) + qr = _QueryRequest("Bitmap(field='foo', id=1)", column_attrs=True) bin = qr.to_protobuf(False) # do not return a bytearray self.assertIsNotNone(bin) qr = internal.QueryRequest() @@ -232,13 +232,13 @@ def test_serialize(self): class ImportRequestTestCase(unittest.TestCase): def test_serialize(self): - ir = _ImportRequest("foo", "bar", 0, [Bit(row_id=1, column_id=2, timestamp=3)]) + ir = _ImportRequest("foo", "bar", 0, [Columns(row_id=1, column_id=2, timestamp=3)]) bin = ir.to_protobuf(False) self.assertIsNotNone(bin) ir = internal.ImportRequest() ir.ParseFromString(bin) self.assertEquals("foo", ir.Index) - self.assertEquals("bar", ir.Frame) + self.assertEquals("bar", ir.Field) self.assertEquals([1], ir.RowIDs) self.assertEquals([2], ir.ColumnIDs) self.assertEquals([3], ir.Timestamps) diff --git a/tests/test_imports.py b/tests/test_imports.py index 9935d1c..f448365 100644 --- a/tests/test_imports.py +++ b/tests/test_imports.py @@ -31,13 +31,12 @@ # DAMAGE. # -import unittest - -import datetime import calendar +import datetime +import unittest from pilosa.exceptions import PilosaError -from pilosa.imports import csv_bit_reader, batch_bits +from pilosa.imports import csv_column_reader, batch_columns try: from io import StringIO @@ -48,13 +47,13 @@ class ImportsTestCase(unittest.TestCase): def test_csvbititerator(self): - reader = csv_bit_reader(StringIO(u""" + reader = csv_column_reader(StringIO(u""" 1,10,683793200 5,20,683793300 3,41,683793385 10,10485760,683793385 """)) - slice_bit_groups = list(batch_bits(reader, 2)) + slice_bit_groups = list(batch_columns(reader, 2)) self.assertEqual(3, len(slice_bit_groups)) slice1, batch1 = slice_bit_groups[0] @@ -82,7 +81,7 @@ def test_invalid_input(self): ] for text in invalid_inputs: - reader = csv_bit_reader(StringIO(text)) + reader = csv_column_reader(StringIO(text)) self.assertRaises(PilosaError, list, reader) def test_csvbititerator_customtimefunc(self): @@ -100,7 +99,7 @@ def timefunc_utcstr(timeval): dt = dt.replace(tzinfo=UtcTzinfo()) return calendar.timegm(dt.timetuple()) - reader = csv_bit_reader(StringIO(u""" + reader = csv_column_reader(StringIO(u""" 1,10,1991-09-02T06:33:20 5,20,1991-09-02T06:35:00 3,41,1991-09-02T06:36:25 diff --git a/tests/test_orm.py b/tests/test_orm.py index 1fa4a51..0faf3d3 100644 --- a/tests/test_orm.py +++ b/tests/test_orm.py @@ -39,9 +39,9 @@ schema = Schema() sampleIndex = schema.index("sample-db") -sampleFrame = sampleIndex.field("sample-field") +sampleField = sampleIndex.field("sample-field") projectIndex = schema.index("project-db") -collabFrame = projectIndex.field("collaboration") +collabField = projectIndex.field("collaboration") class SchemaTestCase(unittest.TestCase): @@ -102,10 +102,10 @@ def test_raw_query(self): q.serialize()) def test_union(self): - b1 = sampleFrame.bitmap(10) - b2 = sampleFrame.bitmap(20) - b3 = sampleFrame.bitmap(42) - b4 = collabFrame.bitmap(2) + b1 = sampleField.row(10) + b2 = sampleField.row(20) + b3 = sampleField.row(42) + b4 = collabField.row(2) q1 = sampleIndex.union(b1, b2) self.assertEquals( @@ -123,10 +123,10 @@ def test_union(self): q3.serialize()) def test_intersect(self): - b1 = sampleFrame.bitmap(10) - b2 = sampleFrame.bitmap(20) - b3 = sampleFrame.bitmap(42) - b4 = collabFrame.bitmap(2) + b1 = sampleField.row(10) + b2 = sampleField.row(20) + b3 = sampleField.row(42) + b4 = collabField.row(2) q1 = sampleIndex.intersect(b1, b2) self.assertEquals( @@ -144,10 +144,10 @@ def test_intersect(self): q3.serialize()) def test_difference(self): - b1 = sampleFrame.bitmap(10) - b2 = sampleFrame.bitmap(20) - b3 = sampleFrame.bitmap(42) - b4 = collabFrame.bitmap(2) + b1 = sampleField.row(10) + b2 = sampleField.row(20) + b3 = sampleField.row(42) + b4 = collabField.row(2) q1 = sampleIndex.difference(b1, b2) self.assertEquals( @@ -165,8 +165,8 @@ def test_difference(self): q3.serialize()) def test_xor(self): - b1 = sampleFrame.bitmap(10) - b2 = sampleFrame.bitmap(20) + b1 = sampleField.row(10) + b2 = sampleField.row(20) q1 = sampleIndex.xor(b1, b2) self.assertEquals( @@ -178,7 +178,7 @@ def test_union0(self): self.assertEquals("Union()", q.serialize()) def test_union1(self): - q = sampleIndex.union(sampleFrame.bitmap(10)) + q = sampleIndex.union(sampleField.row(10)) self.assertEquals("Union(Bitmap(row=10, field='sample-field'))", q.serialize()) def test_intersect_invalid_bitmap_count_fails(self): @@ -188,10 +188,10 @@ def test_difference_invalid_bitmap_count_fails(self): self.assertRaises(PilosaError, projectIndex.difference) def test_xor_invalid_bitmap_count_fails(self): - self.assertRaises(PilosaError, projectIndex.xor, sampleFrame.bitmap(10)) + self.assertRaises(PilosaError, projectIndex.xor, sampleField.row(10)) def test_count(self): - b = collabFrame.bitmap(42) + b = collabField.row(42) q = projectIndex.count(b) self.assertEquals( "Count(Bitmap(row=42, field='collaboration'))", @@ -215,7 +215,7 @@ def test_set_column_attributes_invalid_values(self): self.assertRaises(PilosaError, projectIndex.set_column_attrs, 5, attrs_map) -class FrameTestCase(unittest.TestCase): +class FieldTestCase(unittest.TestCase): def test_create_field(self): db = Index("foo") @@ -225,94 +225,94 @@ def test_create_field(self): self.assertEqual(TimeQuantum.NONE, field.time_quantum) def test_same_equals(self): - self.assertEqual(sampleFrame, sampleFrame) + self.assertEqual(sampleField, sampleField) def test_other_class_not_equals(self): schema = Schema() - self.assertNotEqual(sampleFrame, schema) + self.assertNotEqual(sampleField, schema) def test_bitmap(self): - qry1 = collabFrame.bitmap(5) + qry1 = collabField.row(5) self.assertEquals( "Bitmap(row=5, field='collaboration')", qry1.serialize()) - qry2 = collabFrame.bitmap(10) + qry2 = collabField.row(10) self.assertEquals( "Bitmap(row=10, field='collaboration')", qry2.serialize()) - qry3 = collabFrame.bitmap("b7feb014-8ea7-49a8-9cd8-19709161ab63") + qry3 = collabField.row("b7feb014-8ea7-49a8-9cd8-19709161ab63") self.assertEquals( "Bitmap(row='b7feb014-8ea7-49a8-9cd8-19709161ab63', field='collaboration')", qry3.serialize()) def test_bitmap_with_invalid_id_type(self): - self.assertRaises(ValidationError, sampleFrame.bitmap, {}) + self.assertRaises(ValidationError, sampleField.row, {}) def test_setbit(self): - qry1 = collabFrame.setbit(5, 10) + qry1 = collabField.setbit(5, 10) self.assertEquals( "SetBit(row=5, field='collaboration', col=10)", qry1.serialize()) - qry2 = collabFrame.setbit(10, 20) + qry2 = collabField.setbit(10, 20) self.assertEquals( "SetBit(row=10, field='collaboration', col=20)", qry2.serialize()) - qry3 = collabFrame.setbit("b7feb014-8ea7-49a8-9cd8-19709161ab63", "some_id") + qry3 = collabField.setbit("b7feb014-8ea7-49a8-9cd8-19709161ab63", "some_id") self.assertEquals( "SetBit(row='b7feb014-8ea7-49a8-9cd8-19709161ab63', field='collaboration', col='some_id')", qry3.serialize()) def test_setbit_with_invalid_id_type(self): - self.assertRaises(ValidationError, sampleFrame.setbit, {}, 1) - self.assertRaises(ValidationError, sampleFrame.setbit, 1, {}) - self.assertRaises(ValidationError, sampleFrame.setbit, 1, "zero") + self.assertRaises(ValidationError, sampleField.setbit, {}, 1) + self.assertRaises(ValidationError, sampleField.setbit, 1, {}) + self.assertRaises(ValidationError, sampleField.setbit, 1, "zero") def test_setbit_with_timestamp(self): timestamp = datetime(2017, 4, 24, 12, 14) - qry = collabFrame.setbit(10, 20, timestamp) + qry = collabField.setbit(10, 20, timestamp) self.assertEquals( "SetBit(row=10, field='collaboration', col=20, timestamp='2017-04-24T12:14')", qry.serialize() ) def test_clearbit(self): - qry1 = collabFrame.clearbit(5, 10) + qry1 = collabField.clearbit(5, 10) self.assertEquals( "ClearBit(row=5, field='collaboration', col=10)", qry1.serialize()) - qry2 = collabFrame.clearbit(10, 20) + qry2 = collabField.clearbit(10, 20) self.assertEquals( "ClearBit(row=10, field='collaboration', col=20)", qry2.serialize()) - qry3 = collabFrame.clearbit("b7feb014-8ea7-49a8-9cd8-19709161ab63", "some_id") + qry3 = collabField.clearbit("b7feb014-8ea7-49a8-9cd8-19709161ab63", "some_id") self.assertEquals( "ClearBit(row='b7feb014-8ea7-49a8-9cd8-19709161ab63', field='collaboration', col='some_id')", qry3.serialize()) def test_clearbit_with_invalid_id_type(self): - self.assertRaises(ValidationError, sampleFrame.clearbit, {}, 1) - self.assertRaises(ValidationError, sampleFrame.clearbit, 1, {}) - self.assertRaises(ValidationError, sampleFrame.clearbit, 1, "zero") + self.assertRaises(ValidationError, sampleField.clearbit, {}, 1) + self.assertRaises(ValidationError, sampleField.clearbit, 1, {}) + self.assertRaises(ValidationError, sampleField.clearbit, 1, "zero") def test_topn(self): - q1 = collabFrame.topn(27) + q1 = collabField.topn(27) self.assertEquals( "TopN(field='collaboration', n=27)", q1.serialize()) - q2 = collabFrame.topn(10, collabFrame.bitmap(3)) + q2 = collabField.topn(10, collabField.row(3)) self.assertEquals( u"TopN(Bitmap(row=3, field='collaboration'), field='collaboration', n=10)", q2.serialize()) - q3 = sampleFrame.topn(12, collabFrame.bitmap(7), "category", 80, 81) + q3 = sampleField.topn(12, collabField.row(7), "category", 80, 81) self.assertEquals( "TopN(Bitmap(row=7, field='collaboration'), field='sample-field', n=12, field='category', filters=[80,81])", q3.serialize()) @@ -321,12 +321,12 @@ def test_range(self): start = datetime(1970, 1, 1, 0, 0) end = datetime(2000, 2, 2, 3, 4) - q1 = collabFrame.range(10, start, end) + q1 = collabField.range(10, start, end) self.assertEquals( "Range(row=10, field='collaboration', start='1970-01-01T00:00', end='2000-02-02T03:04')", q1.serialize()) - q3 = sampleFrame.range("b7feb014-8ea7-49a8-9cd8-19709161ab63", start, end) + q3 = sampleField.range("b7feb014-8ea7-49a8-9cd8-19709161ab63", start, end) self.assertEquals( u"Range(row='b7feb014-8ea7-49a8-9cd8-19709161ab63', field='sample-field', start='1970-01-01T00:00', end='2000-02-02T03:04')", q3.serialize()) @@ -336,71 +336,71 @@ def test_set_row_attributes(self): "quote": '''"Don't worry, be happy"''', "active": True } - q = collabFrame.set_row_attrs(5, attrs_map) + q = collabField.set_row_attrs(5, attrs_map) self.assertEquals( "SetRowAttrs(row=5, field='collaboration', active=true, quote=\"\\\"Don't worry, be happy\\\"\")", q.serialize()) def test_field_lt(self): - q = collabFrame.lt(10) + q = collabField.lt(10) self.assertEquals( "Range(collaboration < 10)", q.serialize()) def test_field_lte(self): - q = collabFrame.lte(10) + q = collabField.lte(10) self.assertEquals( "Range(collaboration <= 10)", q.serialize()) def test_field_gt(self): - q = collabFrame.gt(10) + q = collabField.gt(10) self.assertEquals( "Range(collaboration > 10)", q.serialize()) def test_field_gte(self): - q = collabFrame.gte(10) + q = collabField.gte(10) self.assertEquals( "Range(collaboration >= 10)", q.serialize()) def test_field_equals(self): - q = collabFrame.equals(10) + q = collabField.equals(10) self.assertEquals( "Range(collaboration == 10)", q.serialize()) def test_field_not_equals(self): - q = collabFrame.not_equals(10) + q = collabField.not_equals(10) self.assertEquals( "Range(collaboration != 10)", q.serialize()) def test_field_not_null(self): - q = collabFrame.not_null() + q = collabField.not_null() self.assertEquals( "Range(collaboration != null)", q.serialize()) def test_field_between(self): - q = collabFrame.between(10, 20) + q = collabField.between(10, 20) self.assertEquals( "Range(collaboration >< [10,20])", q.serialize()) def test_field_sum(self): - q = collabFrame.sum(collabFrame.bitmap(10)) + q = collabField.sum(collabField.row(10)) self.assertEquals( "Sum(Bitmap(row=10, field='collaboration'), field='collaboration')", q.serialize()) - q = collabFrame.sum() + q = collabField.sum() self.assertEquals( "Sum(field='collaboration')", q.serialize()) def test_field_set_value(self): - q = collabFrame.set_value(10, 20) + q = collabField.setvalue(10, 20) self.assertEquals( "SetValue(col=10, collaboration=20)", q.serialize()) diff --git a/tests/test_response.py b/tests/test_response.py index fcb67de..3b1beca 100644 --- a/tests/test_response.py +++ b/tests/test_response.py @@ -35,7 +35,7 @@ from pilosa.exceptions import PilosaError from pilosa.internal import public_pb2 as internal -from pilosa.response import QueryResponse, QUERYRESULT_BITMAP +from pilosa.response import QueryResponse, QUERYRESULT_ROW class QueryResultTestCase(unittest.TestCase): @@ -43,8 +43,8 @@ class QueryResultTestCase(unittest.TestCase): def test_invalid_attr_type(self): qr = internal.QueryResponse() result1 = qr.Results.add() - result1.Type = QUERYRESULT_BITMAP - attr = result1.Bitmap.Attrs.add() + result1.Type = QUERYRESULT_ROW + attr = result1.Row.Attrs.add() attr.Key = "foo" attr.StringValue = "bar" attr.Type = 0 diff --git a/tests/test_validator.py b/tests/test_validator.py index 70ede24..8156724 100644 --- a/tests/test_validator.py +++ b/tests/test_validator.py @@ -51,12 +51,12 @@ class ValidatorTestCase(unittest.TestCase): "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1" ] - VALID_FRAME_NAMES = [ + VALID_FIELD_NAMES = [ "a", "ab", "ab1", "b-c", "d_e", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" ] - INVALID_FRAME_NAMES = [ + INVALID_FIELD_NAMES = [ "", "'", "^", "/", "\\", "A", "*", "a:b", "valid?no", "yüce", "_", "-", ".data", "d.e", "1", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1" ] @@ -98,12 +98,12 @@ def test_invalid_index_name_fails(self): continue self.fail("Index name validation should have failed for: " + name) - def test_validate_valid_frame_name(self): - for name in self.VALID_FRAME_NAMES: + def test_validate_valid_field_name(self): + for name in self.VALID_FIELD_NAMES: validate_field_name(name) - def test_invalid_frame_name_fails(self): - for name in self.INVALID_FRAME_NAMES: + def test_invalid_field_name_fails(self): + for name in self.INVALID_FIELD_NAMES: try: validate_field_name(name) except ValidationError: @@ -114,7 +114,7 @@ def test_validate_valid_label(self): for label in self.VALID_LABELS: validate_label(label) - def test_validate_invalid_frame_name_fails(self): + def test_validate_invalid_field_name_fails(self): for label in self.INVALID_LABELS: try: validate_label(label) From 9a5574275283f9148034b48c9e1a43244fb7db5c Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Mon, 18 Jun 2018 17:22:46 +0300 Subject: [PATCH 3/9] More renaming --- docs/data-model-queries.md | 22 +++++++--------------- integration_tests/test_client_it.py | 26 +++++++++++++------------- pilosa/orm.py | 2 +- tests/test_client.py | 4 ++-- tests/test_orm.py | 10 +++++----- 5 files changed, 28 insertions(+), 36 deletions(-) diff --git a/docs/data-model-queries.md b/docs/data-model-queries.md index 197594c..14d4026 100644 --- a/docs/data-model-queries.md +++ b/docs/data-model-queries.md @@ -27,16 +27,16 @@ stargazer = repository.field("stargazer", time_quantum=pilosa.TimeQuantum.YEAR_M Once you have indexes and field objects created, you can create queries for them. Some of the queries work on the columns; corresponding methods are attached to the index. Other queries work on rows, with related methods attached to fields. -For instance, `Bitmap` queries work on rows; use a field object to create those queries: +For instance, `Row` queries work on rows; use a field object to create those queries: ```python -bitmap_query = stargazer.row(1) # corresponds to PQL: Bitmap(field='stargazer', row=1) +row_query = stargazer.row(1) # corresponds to PQL: Bitmap(field='stargazer', row=1) ``` `Union` queries work on columns; use the index object to create them: ```python -query = repository.union(bitmap_query1, bitmap_query2) +query = repository.union(row_query1, row_query2) ``` In order to increase throughput, you may want to batch queries sent to the Pilosa server. The `index.batch_query` method is used for that purpose: @@ -103,12 +103,12 @@ Please check [Pilosa documentation](https://www.pilosa.com/docs) for PQL details Index: -* `union(self, *bitmaps)` -* `intersect(self, *bitmaps)` -* `difference(self, *bitmaps)` +* `union(self, *rows)` +* `intersect(self, *rows)` +* `difference(self, *rows)` * `count(self, row)` * `set_column_attrs(self, column_id, attrs)` -* `xor(self, *bitmaps)` +* `xor(self, *rows)` Field: @@ -118,14 +118,6 @@ Field: * `topn(self, n, row=None, field="", *values)` * `range(self, row_id, start, end)` * `set_row_attrs(self, row_id, attrs)` -* (**deprecated**) `inverse_bitmap(self, column_id)` -* (**deprecated**) `inverse_topn(self, n, row=None, field="", *values)` -* (**deprecated**) `inverse_range(self, column_id, start, end)` -* (**deprecated**) `sum(self, row, field)` -* (**deprecated**) `set_field_value(self, column_id, field, value)` - -Field: - * `lt(self, n)` * `lte(self, n)` * `gt(self, n)` diff --git a/integration_tests/test_client_it.py b/integration_tests/test_client_it.py index 5288cd1..4d2642f 100644 --- a/integration_tests/test_client_it.py +++ b/integration_tests/test_client_it.py @@ -126,10 +126,10 @@ def test_new_orm(self): client.query(self.field.setbit(10, 20)) response1 = client.query(self.field.row(10)) self.assertEquals(0, len(response1.columns)) - bitmap1 = response1.result.row - self.assertEquals(0, len(bitmap1.attributes)) - self.assertEquals(1, len(bitmap1.columns)) - self.assertEquals(20, bitmap1.columns[0]) + row1 = response1.result.row + self.assertEquals(0, len(row1.attributes)) + self.assertEquals(1, len(row1.columns)) + self.assertEquals(20, row1.columns[0]) column_attrs = {"name": "bombo"} client.query(self.col_index.set_column_attrs(20, column_attrs)) @@ -138,21 +138,21 @@ def test_new_orm(self): self.assertTrue(column is not None) self.assertEquals(20, column.id) - bitmap_attrs = { + row_attrs = { "active": True, "unsigned": 5, "height": 1.81, "name": "Mr. Pi" } - client.query(self.field.set_row_attrs(10, bitmap_attrs)) + client.query(self.field.set_row_attrs(10, row_attrs)) response3 = client.query(self.field.row(10)) - bitmap = response3.result.row - self.assertEquals(1, len(bitmap.columns)) - self.assertEquals(4, len(bitmap.attributes)) - self.assertEquals(True, bitmap.attributes["active"]) - self.assertEquals(5, bitmap.attributes["unsigned"]) - self.assertEquals(1.81, bitmap.attributes["height"]) - self.assertEquals("Mr. Pi", bitmap.attributes["name"]) + row = response3.result.row + self.assertEquals(1, len(row.columns)) + self.assertEquals(4, len(row.attributes)) + self.assertEquals(True, row.attributes["active"]) + self.assertEquals(5, row.attributes["unsigned"]) + self.assertEquals(1.81, row.attributes["height"]) + self.assertEquals("Mr. Pi", row.attributes["name"]) def test_topn(self): client = self.get_client() diff --git a/pilosa/orm.py b/pilosa/orm.py index 7e3b24e..67d7647 100644 --- a/pilosa/orm.py +++ b/pilosa/orm.py @@ -256,7 +256,7 @@ def union(self, *rows): def intersect(self, *rows): """Creates an ``Intersect`` query. - ``Intersect`` performs a logical AND on the results of each BITMAP_CALL query passed to it. + ``Intersect`` performs a logical AND on the results of each ROW_CALL query passed to it. :param pilosa.PQLQuery rows: 1 or more row queries to intersect :return: Pilosa row query diff --git a/tests/test_client.py b/tests/test_client.py index 32a6965..b1acba8 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -220,12 +220,12 @@ def test_cluster_reset(self): class QueryRequestTestCase(unittest.TestCase): def test_serialize(self): - qr = _QueryRequest("Bitmap(field='foo', id=1)", column_attrs=True) + qr = _QueryRequest("Row(field='foo', id=1)", column_attrs=True) bin = qr.to_protobuf(False) # do not return a bytearray self.assertIsNotNone(bin) qr = internal.QueryRequest() qr.ParseFromString(bin) - self.assertEquals("Bitmap(field='foo', id=1)", qr.Query) + self.assertEquals("Row(field='foo', id=1)", qr.Query) self.assertEquals(True, qr.ColumnAttrs) diff --git a/tests/test_orm.py b/tests/test_orm.py index 0faf3d3..9174c48 100644 --- a/tests/test_orm.py +++ b/tests/test_orm.py @@ -181,13 +181,13 @@ def test_union1(self): q = sampleIndex.union(sampleField.row(10)) self.assertEquals("Union(Bitmap(row=10, field='sample-field'))", q.serialize()) - def test_intersect_invalid_bitmap_count_fails(self): + def test_intersect_invalid_row_count_fails(self): self.assertRaises(PilosaError, projectIndex.intersect) - def test_difference_invalid_bitmap_count_fails(self): + def test_difference_invalid_row_count_fails(self): self.assertRaises(PilosaError, projectIndex.difference) - def test_xor_invalid_bitmap_count_fails(self): + def test_xor_invalid_row_count_fails(self): self.assertRaises(PilosaError, projectIndex.xor, sampleField.row(10)) def test_count(self): @@ -231,7 +231,7 @@ def test_other_class_not_equals(self): schema = Schema() self.assertNotEqual(sampleField, schema) - def test_bitmap(self): + def test_row(self): qry1 = collabField.row(5) self.assertEquals( "Bitmap(row=5, field='collaboration')", @@ -247,7 +247,7 @@ def test_bitmap(self): "Bitmap(row='b7feb014-8ea7-49a8-9cd8-19709161ab63', field='collaboration')", qry3.serialize()) - def test_bitmap_with_invalid_id_type(self): + def test_row_with_invalid_id_type(self): self.assertRaises(ValidationError, sampleField.row, {}) def test_setbit(self): From 9762b240730d5a5c345083fee86aca0995ef14ed Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Fri, 22 Jun 2018 23:14:53 +0300 Subject: [PATCH 4/9] do not send protobuf accept/content-type headers by default --- pilosa/client.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/pilosa/client.py b/pilosa/client.py index 648ebb1..f4093c3 100644 --- a/pilosa/client.py +++ b/pilosa/client.py @@ -117,7 +117,11 @@ def query(self, query, column_attrs=False, exclude_columns=False, exclude_attrs= request = _QueryRequest(query.serialize(), column_attrs=column_attrs, exclude_columns=exclude_columns, exclude_row_attrs=exclude_attrs) path = "/index/%s/query" % query.index.name try: - response = self.__http_request("POST", path, data=request.to_protobuf()) + headers = { + 'Content-Type': 'application/x-protobuf', + 'Accept': 'application/x-protobuf', + } + response = self.__http_request("POST", path, data=request.to_protobuf(), headers=headers) return QueryResponse._from_protobuf(response.data) except PilosaServerError as e: raise PilosaError(e.content) @@ -284,7 +288,11 @@ def _fetch_fragment_nodes(self, index_name, slice): def _import_node(self, import_request): data = import_request.to_protobuf() - self.__http_request("POST", "/import", data=data) + headers = { + 'Content-Type': 'application/x-protobuf', + 'Accept': 'application/x-protobuf', + } + self.__http_request("POST", "/import", data=data, headers=headers) def __http_request(self, method, path, data=None, headers=None): if not self.__client: @@ -316,8 +324,6 @@ def __get_address(self): def __connect(self): num_pools = float(self.pool_size_total) / self.pool_size_per_route headers = { - 'Content-Type': 'application/x-protobuf', - 'Accept': 'application/x-protobuf', 'User-Agent': 'python-pilosa/%s' % VERSION, } From 67db274db835d5f332e01618f5adb670b75baa69 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Tue, 26 Jun 2018 16:44:22 +0300 Subject: [PATCH 5/9] More develop updates --- README.md | 2 +- doc/index.rst | 2 +- docs/data-model-queries.md | 10 ++-- integration_tests/test_client_it.py | 62 ++++++++++++--------- pilosa/orm.py | 35 ++++++------ tests/test_orm.py | 83 ++++++++++++++--------------- 6 files changed, 104 insertions(+), 90 deletions(-) diff --git a/README.md b/README.md index 0f557c6..86676c2 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ myfield = myindex.field("myfield") client.sync_schema(schema) # Send a SetBit query. PilosaError is thrown if execution of the query fails. -client.query(myfield.setbit(5, 42)) +client.query(myfield.set(5, 42)) # Send a Bitmap query. PilosaError is thrown if execution of the query fails. response = client.query(myfield.row(5)) diff --git a/doc/index.rst b/doc/index.rst index d5367a5..33f4e66 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -56,7 +56,7 @@ at ``localhost:10101`` (the default): client.sync_schema(schema) # Send a SetBit query. PilosaError is thrown if execution of the query fails. - client.query(myfield.setbit(5, 42)) + client.query(myfield.set(5, 42)) # Send a Bitmap query. PilosaError is thrown if execution of the query fails. response = client.query(myfield.row(5)) diff --git a/docs/data-model-queries.md b/docs/data-model-queries.md index 14d4026..ddf33ae 100644 --- a/docs/data-model-queries.md +++ b/docs/data-model-queries.md @@ -87,10 +87,10 @@ print(response.result.value) It's possible to pass a row query to `sum`, so only columns where a row is set are filtered in: ```python -# Let's run a few setbit queries first +# Let's run a few set queries first client.query(index.batch_query( - field.setbit(42, 1), - field.setbit(42, 6) + field.set(42, 1), + field.set(42, 6) )) # Query for the total number of animals in captivity where row 42 is set response = client.query(captivity.sum(field.row(42))) @@ -113,8 +113,8 @@ Index: Field: * `row(self, row_id)` -* `setbit(self, row_id, column_id, timestamp=None)` -* `clearbit(self, row_id, column_id)` +* `set(self, row_id, column_id, timestamp=None)` +* `clear(self, row_id, column_id)` * `topn(self, n, row=None, field="", *values)` * `range(self, row_id, start, end)` * `set_row_attrs(self, row_id, attrs)` diff --git a/integration_tests/test_client_it.py b/integration_tests/test_client_it.py index 4d2642f..91903da 100644 --- a/integration_tests/test_client_it.py +++ b/integration_tests/test_client_it.py @@ -82,14 +82,14 @@ def test_query(self): client = self.get_client() field = self.index.field("query-test") client.ensure_field(field) - response = client.query(field.setbit(555, 10)) + response = client.query(field.set(555, 10)) self.assertTrue(response.result is not None) def test_query_with_columns(self): client = self.get_client() field = self.index.field("query-test") client.ensure_field(field) - client.query(field.setbit(100, 1000)) + client.query(field.set(100, 1000)) column_attrs = {"name": "bombo"} client.query(self.index.set_column_attrs(1000, column_attrs)) response = client.query(field.row(100), column_attrs=True) @@ -102,7 +102,7 @@ def test_query_with_columns(self): def test_failed_connection(self): client = Client("http://non-existent-sub.pilosa.com:22222") - self.assertRaises(PilosaError, client.query, self.field.setbit(15, 10)) + self.assertRaises(PilosaError, client.query, self.field.set(15, 10)) def test_parse_error(self): client = self.get_client() @@ -114,27 +114,28 @@ def test_orm_count(self): count_field = self.index.field("count-test") client.ensure_field(count_field) qry = self.index.batch_query( - count_field.setbit(10, 20), - count_field.setbit(10, 21), - count_field.setbit(15, 25)) + count_field.set(10, 20), + count_field.set(10, 21), + count_field.set(15, 25)) client.query(qry) response = client.query(self.index.count(count_field.row(10))) self.assertEquals(2, response.result.count) def test_new_orm(self): client = self.get_client() - client.query(self.field.setbit(10, 20)) - response1 = client.query(self.field.row(10)) - self.assertEquals(0, len(response1.columns)) - row1 = response1.result.row + response1 = client.query(self.field.set(10, 20)) + self.assertTrue(response1.result.changed) + response2 = client.query(self.field.row(10)) + self.assertEquals(0, len(response2.columns)) + row1 = response2.result.row self.assertEquals(0, len(row1.attributes)) self.assertEquals(1, len(row1.columns)) self.assertEquals(20, row1.columns[0]) column_attrs = {"name": "bombo"} client.query(self.col_index.set_column_attrs(20, column_attrs)) - response2 = client.query(self.field.row(10), column_attrs=True) - column = response2.column + response3 = client.query(self.field.row(10), column_attrs=True) + column = response3.column self.assertTrue(column is not None) self.assertEquals(20, column.id) @@ -145,8 +146,8 @@ def test_new_orm(self): "name": "Mr. Pi" } client.query(self.field.set_row_attrs(10, row_attrs)) - response3 = client.query(self.field.row(10)) - row = response3.result.row + response4 = client.query(self.field.row(10)) + row = response4.result.row self.assertEquals(1, len(row.columns)) self.assertEquals(4, len(row.attributes)) self.assertEquals(True, row.attributes["active"]) @@ -154,24 +155,37 @@ def test_new_orm(self): self.assertEquals(1.81, row.attributes["height"]) self.assertEquals("Mr. Pi", row.attributes["name"]) + response5 = client.query(self.field.clear(10, 20)) + self.assertTrue(response5.result.changed) + response6 = client.query(self.field.row(10)) + row = response6.result.row + self.assertEquals(0, len(row.columns)) + def test_topn(self): client = self.get_client() field = self.index.field("topn_test") client.query(self.index.batch_query( - field.setbit(10, 5), - field.setbit(10, 10), - field.setbit(10, 15), - field.setbit(20, 5), - field.setbit(30, 5))) + field.set(10, 5), + field.set(10, 10), + field.set(10, 15), + field.set(20, 5), + field.set(30, 5))) # XXX: The following is required to make this test pass. See: https://github.com/pilosa/pilosa/issues/625 client.http_request("POST", "/recalculate-caches") - response4 = client.query(field.topn(2)) - items = response4.result.count_items + response = client.query(field.topn(2)) + items = response.result.count_items self.assertEquals(2, len(items)) item = items[0] self.assertEquals(10, item.id) self.assertEquals(3, item.count) + response = client.query(field.topn(5, row=field.row(10))) + items = response.result.count_items + self.assertEquals(3, len(items)) + item = items[0] + self.assertEquals(3, item.count) + + def test_ensure_index_exists(self): client = self.get_client() index = Index(self.index.name + "-ensure") @@ -287,8 +301,8 @@ def test_range_field(self): field = self.col_index.field("rangefield", int_min=10, int_max=20) client.ensure_field(field) client.query(self.col_index.batch_query( - field.setbit(1, 10), - field.setbit(1, 100), + field.set(1, 10), + field.set(1, 100), field.setvalue(10, 11), )) response = client.query(field.sum(field.row(1))) @@ -310,7 +324,7 @@ def test_range_field(self): def test_exclude_attrs_columns(self): client = self.get_client() client.query(self.col_index.batch_query( - self.field.setbit(1, 100), + self.field.set(1, 100), self.field.set_row_attrs(1, {"foo": "bar"}) )) diff --git a/pilosa/orm.py b/pilosa/orm.py index 67d7647..f1e134e 100644 --- a/pilosa/orm.py +++ b/pilosa/orm.py @@ -323,8 +323,8 @@ def set_column_attrs(self, column_idkey, attrs): :rtype: pilosa.PQLQuery """ fmt = id_key_format("Column", column_idkey, - u"SetColumnAttrs(col=%s, %s)", - u"SetColumnAttrs(col='%s, %s)") + u"SetColumnAttrs(%s, %s)", + u"SetColumnAttrs('%s, %s)") attrs_str = _create_attributes_str(attrs) return PQLQuery(fmt % (column_idkey, attrs_str), self) @@ -391,11 +391,11 @@ def row(self, row_idkey): :rtype: pilosa.PQLQuery """ fmt = id_key_format("Row ID/Key", row_idkey, - u"Bitmap(row=%s, field='%s')", - u"Bitmap(row='%s', field='%s')") - return PQLQuery(fmt % (row_idkey, self.name), self.index) + u"Row(%s, field='%s')", + u"Row(row='%s', field='%s')") + return PQLQuery(u"Row(%s=%s)"% (self.name, row_idkey), self.index) - def setbit(self, row_idkey, column_idkey, timestamp=None): + def set(self, row_idkey, column_idkey, timestamp=None): """Creates a SetBit query. ``SetBit`` assigns a value of 1 to a bit in the binary matrix, thus associating the given row in the given field with the given column. @@ -407,17 +407,17 @@ def setbit(self, row_idkey, column_idkey, timestamp=None): :rtype: pilosa.PQLQuery """ if isinstance(row_idkey, int) and isinstance(column_idkey, int): - fmt = u"SetBit(row=%s, field='%s', col=%s%s)" + fmt = u"Set(%s, %s=%s%s)" elif isinstance(row_idkey, _basestring) and isinstance(column_idkey, _basestring): - fmt = u"SetBit(row='%s', field='%s', col='%s'%s)" + fmt = u"Set(%s, %s='%s'%s)" validate_key(row_idkey) validate_key(column_idkey) else: raise ValidationError("Both Row and Columns must be integers or strings") ts = ", timestamp='%s'" % timestamp.strftime(_TIME_FORMAT) if timestamp else '' - return PQLQuery(fmt % (row_idkey, self.name, column_idkey, ts), self.index) + return PQLQuery(fmt % (column_idkey, self.name, row_idkey, ts), self.index) - def clearbit(self, row_idkey, column_idkey): + def clear(self, row_idkey, column_idkey): """Creates a ClearBit query. ``ClearBit`` assigns a value of 0 to a bit in the binary matrix, thus disassociating the given row in the given field from the given column. @@ -428,14 +428,14 @@ def clearbit(self, row_idkey, column_idkey): :rtype: pilosa.PQLQuery """ if isinstance(row_idkey, int) and isinstance(column_idkey, int): - fmt = u"ClearBit(row=%s, field='%s', col=%s)" + fmt = u"Clear(%s, %s=%s)" elif isinstance(row_idkey, _basestring) and isinstance(column_idkey, _basestring): - fmt = u"ClearBit(row='%s', field='%s', col='%s')" + fmt = u"Clear('%s', %s='%s')" validate_key(row_idkey) validate_key(column_idkey) else: raise ValidationError("Both Row and Columns must be integers or strings") - return PQLQuery(fmt % (row_idkey, self.name, column_idkey), self.index) + return PQLQuery(fmt % (column_idkey, self.name, row_idkey), self.index) def topn(self, n, row=None, field="", *values): """Creates a TopN query. @@ -449,9 +449,10 @@ def topn(self, n, row=None, field="", *values): :param field str field: field name :param object values: filter values to be matched against the field """ - parts = ["field='%s'" % self.name, "n=%d" % n] + parts = [self.name] if row: - parts.insert(0, row.serialize()) + parts.append(row.serialize()) + parts.append("n=%d" % n) if field: validate_label(field) values_str = json.dumps(values, separators=(',', ': ')) @@ -496,10 +497,10 @@ def set_row_attrs(self, row_idkey, attrs): :rtype: pilosa.PQLQuery """ fmt = id_key_format("Row", row_idkey, - u"SetRowAttrs(row=%s, field='%s', %s)", + u"SetRowAttrs(%s, %s, %s)", u"SetRowAttrs(row='%s', field='%s', %s)") attrs_str = _create_attributes_str(attrs) - return PQLQuery(fmt % (row_idkey, self.name, attrs_str), self.index) + return PQLQuery(fmt % (self.name, row_idkey, attrs_str), self.index) def lt(self, n): """Creates a Range query with less than (<) condition. diff --git a/tests/test_orm.py b/tests/test_orm.py index 9174c48..82b664d 100644 --- a/tests/test_orm.py +++ b/tests/test_orm.py @@ -109,17 +109,17 @@ def test_union(self): q1 = sampleIndex.union(b1, b2) self.assertEquals( - "Union(Bitmap(row=10, field='sample-field'), Bitmap(row=20, field='sample-field'))", + "Union(Row(sample-field=10), Row(sample-field=20))", q1.serialize()) q2 = sampleIndex.union(b1, b2, b3) self.assertEquals( - "Union(Bitmap(row=10, field='sample-field'), Bitmap(row=20, field='sample-field'), Bitmap(row=42, field='sample-field'))", + "Union(Row(sample-field=10), Row(sample-field=20), Row(sample-field=42))", q2.serialize()) q3 = sampleIndex.union(b1, b4) self.assertEquals( - "Union(Bitmap(row=10, field='sample-field'), Bitmap(row=2, field='collaboration'))", + "Union(Row(sample-field=10), Row(collaboration=2))", q3.serialize()) def test_intersect(self): @@ -130,17 +130,17 @@ def test_intersect(self): q1 = sampleIndex.intersect(b1, b2) self.assertEquals( - "Intersect(Bitmap(row=10, field='sample-field'), Bitmap(row=20, field='sample-field'))", + "Intersect(Row(sample-field=10), Row(sample-field=20))", q1.serialize()) q2 = sampleIndex.intersect(b1, b2, b3) self.assertEquals( - "Intersect(Bitmap(row=10, field='sample-field'), Bitmap(row=20, field='sample-field'), Bitmap(row=42, field='sample-field'))", + "Intersect(Row(sample-field=10), Row(sample-field=20), Row(sample-field=42))", q2.serialize()) q3 = sampleIndex.intersect(b1, b4) self.assertEquals( - "Intersect(Bitmap(row=10, field='sample-field'), Bitmap(row=2, field='collaboration'))", + "Intersect(Row(sample-field=10), Row(collaboration=2))", q3.serialize()) def test_difference(self): @@ -151,17 +151,17 @@ def test_difference(self): q1 = sampleIndex.difference(b1, b2) self.assertEquals( - "Difference(Bitmap(row=10, field='sample-field'), Bitmap(row=20, field='sample-field'))", + "Difference(Row(sample-field=10), Row(sample-field=20))", q1.serialize()) q2 = sampleIndex.difference(b1, b2, b3) self.assertEquals( - "Difference(Bitmap(row=10, field='sample-field'), Bitmap(row=20, field='sample-field'), Bitmap(row=42, field='sample-field'))", + "Difference(Row(sample-field=10), Row(sample-field=20), Row(sample-field=42))", q2.serialize()) q3 = sampleIndex.difference(b1, b4) self.assertEquals( - "Difference(Bitmap(row=10, field='sample-field'), Bitmap(row=2, field='collaboration'))", + "Difference(Row(sample-field=10), Row(collaboration=2))", q3.serialize()) def test_xor(self): @@ -170,7 +170,7 @@ def test_xor(self): q1 = sampleIndex.xor(b1, b2) self.assertEquals( - "Xor(Bitmap(row=10, field='sample-field'), Bitmap(row=20, field='sample-field'))", + "Xor(Row(sample-field=10), Row(sample-field=20))", q1.serialize()) def test_union0(self): @@ -179,7 +179,7 @@ def test_union0(self): def test_union1(self): q = sampleIndex.union(sampleField.row(10)) - self.assertEquals("Union(Bitmap(row=10, field='sample-field'))", q.serialize()) + self.assertEquals("Union(Row(sample-field=10))", q.serialize()) def test_intersect_invalid_row_count_fails(self): self.assertRaises(PilosaError, projectIndex.intersect) @@ -194,7 +194,7 @@ def test_count(self): b = collabField.row(42) q = projectIndex.count(b) self.assertEquals( - "Count(Bitmap(row=42, field='collaboration'))", + "Count(Row(collaboration=42))", q.serialize()) def test_set_column_attributes(self): @@ -204,7 +204,7 @@ def test_set_column_attributes(self): } q = projectIndex.set_column_attrs(5, attrs_map) self.assertEquals( - u"SetColumnAttrs(col=5, happy=true, quote=\"\\\"Don't worry, be happy\\\"\")", + u"SetColumnAttrs(5, happy=true, quote=\"\\\"Don't worry, be happy\\\"\")", q.serialize()) def test_set_column_attributes_invalid_values(self): @@ -234,87 +234,86 @@ def test_other_class_not_equals(self): def test_row(self): qry1 = collabField.row(5) self.assertEquals( - "Bitmap(row=5, field='collaboration')", + "Row(collaboration=5)", qry1.serialize()) qry2 = collabField.row(10) self.assertEquals( - "Bitmap(row=10, field='collaboration')", + "Row(collaboration=10)", qry2.serialize()) qry3 = collabField.row("b7feb014-8ea7-49a8-9cd8-19709161ab63") self.assertEquals( - "Bitmap(row='b7feb014-8ea7-49a8-9cd8-19709161ab63', field='collaboration')", + "Row(collaboration=b7feb014-8ea7-49a8-9cd8-19709161ab63)", qry3.serialize()) def test_row_with_invalid_id_type(self): self.assertRaises(ValidationError, sampleField.row, {}) def test_setbit(self): - qry1 = collabField.setbit(5, 10) + qry1 = collabField.set(5, 10) self.assertEquals( - "SetBit(row=5, field='collaboration', col=10)", + u"Set(10, collaboration=5)", qry1.serialize()) - qry2 = collabField.setbit(10, 20) + qry2 = collabField.set(10, 20) self.assertEquals( - "SetBit(row=10, field='collaboration', col=20)", - + u"Set(20, collaboration=10)", qry2.serialize()) - qry3 = collabField.setbit("b7feb014-8ea7-49a8-9cd8-19709161ab63", "some_id") + qry3 = collabField.set("b7feb014-8ea7-49a8-9cd8-19709161ab63", "some_id") self.assertEquals( - "SetBit(row='b7feb014-8ea7-49a8-9cd8-19709161ab63', field='collaboration', col='some_id')", + u"Set(some_id, collaboration='b7feb014-8ea7-49a8-9cd8-19709161ab63')", qry3.serialize()) def test_setbit_with_invalid_id_type(self): - self.assertRaises(ValidationError, sampleField.setbit, {}, 1) - self.assertRaises(ValidationError, sampleField.setbit, 1, {}) - self.assertRaises(ValidationError, sampleField.setbit, 1, "zero") + self.assertRaises(ValidationError, sampleField.set, {}, 1) + self.assertRaises(ValidationError, sampleField.set, 1, {}) + self.assertRaises(ValidationError, sampleField.set, 1, "zero") def test_setbit_with_timestamp(self): timestamp = datetime(2017, 4, 24, 12, 14) - qry = collabField.setbit(10, 20, timestamp) + qry = collabField.set(10, 20, timestamp) self.assertEquals( - "SetBit(row=10, field='collaboration', col=20, timestamp='2017-04-24T12:14')", + u"Set(20, collaboration=10, timestamp='2017-04-24T12:14')", qry.serialize() ) def test_clearbit(self): - qry1 = collabField.clearbit(5, 10) + qry1 = collabField.clear(5, 10) self.assertEquals( - "ClearBit(row=5, field='collaboration', col=10)", + "Clear(10, collaboration=5)", qry1.serialize()) - qry2 = collabField.clearbit(10, 20) + qry2 = collabField.clear(10, 20) self.assertEquals( - "ClearBit(row=10, field='collaboration', col=20)", + "Clear(20, collaboration=10)", qry2.serialize()) - qry3 = collabField.clearbit("b7feb014-8ea7-49a8-9cd8-19709161ab63", "some_id") + qry3 = collabField.clear("b7feb014-8ea7-49a8-9cd8-19709161ab63", "some_id") self.assertEquals( - "ClearBit(row='b7feb014-8ea7-49a8-9cd8-19709161ab63', field='collaboration', col='some_id')", + "Clear('some_id', collaboration='b7feb014-8ea7-49a8-9cd8-19709161ab63')", qry3.serialize()) def test_clearbit_with_invalid_id_type(self): - self.assertRaises(ValidationError, sampleField.clearbit, {}, 1) - self.assertRaises(ValidationError, sampleField.clearbit, 1, {}) - self.assertRaises(ValidationError, sampleField.clearbit, 1, "zero") + self.assertRaises(ValidationError, sampleField.clear, {}, 1) + self.assertRaises(ValidationError, sampleField.clear, 1, {}) + self.assertRaises(ValidationError, sampleField.clear, 1, "zero") def test_topn(self): q1 = collabField.topn(27) self.assertEquals( - "TopN(field='collaboration', n=27)", + "TopN(collaboration, n=27)", q1.serialize()) q2 = collabField.topn(10, collabField.row(3)) self.assertEquals( - u"TopN(Bitmap(row=3, field='collaboration'), field='collaboration', n=10)", + u"TopN(collaboration, Row(collaboration=3), n=10)", q2.serialize()) q3 = sampleField.topn(12, collabField.row(7), "category", 80, 81) self.assertEquals( - "TopN(Bitmap(row=7, field='collaboration'), field='sample-field', n=12, field='category', filters=[80,81])", + "TopN(sample-field, Row(collaboration=7), n=12, field='category', filters=[80,81])", q3.serialize()) def test_range(self): @@ -338,7 +337,7 @@ def test_set_row_attributes(self): } q = collabField.set_row_attrs(5, attrs_map) self.assertEquals( - "SetRowAttrs(row=5, field='collaboration', active=true, quote=\"\\\"Don't worry, be happy\\\"\")", + u'SetRowAttrs(collaboration, 5, active=true, quote="\\"Don\'t worry, be happy\\"")', q.serialize()) def test_field_lt(self): @@ -392,7 +391,7 @@ def test_field_between(self): def test_field_sum(self): q = collabField.sum(collabField.row(10)) self.assertEquals( - "Sum(Bitmap(row=10, field='collaboration'), field='collaboration')", + "Sum(Row(collaboration=10), field='collaboration')", q.serialize()) q = collabField.sum() self.assertEquals( From 150d1765da0dc109c255e16dd870b4bdbefb230f Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Tue, 26 Jun 2018 22:06:49 +0300 Subject: [PATCH 6/9] Quote string key for Row call --- pilosa/orm.py | 6 +++--- tests/test_orm.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pilosa/orm.py b/pilosa/orm.py index f1e134e..7fff3cb 100644 --- a/pilosa/orm.py +++ b/pilosa/orm.py @@ -391,9 +391,9 @@ def row(self, row_idkey): :rtype: pilosa.PQLQuery """ fmt = id_key_format("Row ID/Key", row_idkey, - u"Row(%s, field='%s')", - u"Row(row='%s', field='%s')") - return PQLQuery(u"Row(%s=%s)"% (self.name, row_idkey), self.index) + u"Row(%s=%s)", + u"Row(%s='%s')") + return PQLQuery(fmt % (self.name, row_idkey), self.index) def set(self, row_idkey, column_idkey, timestamp=None): """Creates a SetBit query. diff --git a/tests/test_orm.py b/tests/test_orm.py index 82b664d..cd8636a 100644 --- a/tests/test_orm.py +++ b/tests/test_orm.py @@ -244,7 +244,7 @@ def test_row(self): qry3 = collabField.row("b7feb014-8ea7-49a8-9cd8-19709161ab63") self.assertEquals( - "Row(collaboration=b7feb014-8ea7-49a8-9cd8-19709161ab63)", + "Row(collaboration='b7feb014-8ea7-49a8-9cd8-19709161ab63')", qry3.serialize()) def test_row_with_invalid_id_type(self): From 572b7e8404e21f2b86905402ad60f9f982a0c4c0 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Wed, 27 Jun 2018 00:49:21 +0300 Subject: [PATCH 7/9] Fixed the range call --- integration_tests/test_client_it.py | 15 ++++++++++++- pilosa/orm.py | 10 ++++----- tests/test_orm.py | 34 ++++++++++------------------- 3 files changed, 31 insertions(+), 28 deletions(-) diff --git a/integration_tests/test_client_it.py b/integration_tests/test_client_it.py index 91903da..876e3b8 100644 --- a/integration_tests/test_client_it.py +++ b/integration_tests/test_client_it.py @@ -185,7 +185,6 @@ def test_topn(self): item = items[0] self.assertEquals(3, item.count) - def test_ensure_index_exists(self): client = self.get_client() index = Index(self.index.name + "-ensure") @@ -296,6 +295,20 @@ def test_failover_fail(self): client = Client(Cluster(*uris)) self.assertRaises(PilosaError, client.query, self.field.row(5)) + def test_range(self): + from datetime import datetime + client = self.get_client() + field = self.col_index.field("test-range-field", time_quantum=TimeQuantum.MONTH_DAY_HOUR) + client.ensure_field(field) + client.query(self.col_index.batch_query( + field.set(10, 100, timestamp=datetime(2017, 1, 1, 0, 0)), + field.set(10, 100, timestamp=datetime(2018, 1, 1, 0, 0)), + field.set(10, 100, timestamp=datetime(2019, 1, 1, 0, 0)), + )) + response = client.query(field.range(10, start=datetime(2017, 5, 1, 0, 0), end=datetime(2018, 5, 1, 0, 0))) + self.assertEqual([100], response.result.row.columns) + + def test_range_field(self): client = self.get_client() field = self.col_index.field("rangefield", int_min=10, int_max=20) diff --git a/pilosa/orm.py b/pilosa/orm.py index 7fff3cb..23ac79f 100644 --- a/pilosa/orm.py +++ b/pilosa/orm.py @@ -414,7 +414,7 @@ def set(self, row_idkey, column_idkey, timestamp=None): validate_key(column_idkey) else: raise ValidationError("Both Row and Columns must be integers or strings") - ts = ", timestamp='%s'" % timestamp.strftime(_TIME_FORMAT) if timestamp else '' + ts = ", %s" % timestamp.strftime(_TIME_FORMAT) if timestamp else '' return PQLQuery(fmt % (column_idkey, self.name, row_idkey, ts), self.index) def clear(self, row_idkey, column_idkey): @@ -472,11 +472,11 @@ def range(self, row_idkey, start, end): :param datetime.datetime end: end timestamp """ fmt = id_key_format("Row", row_idkey, - u"Range(row=%s, field='%s', start='%s', end='%s')", - u"Range(row='%s', field='%s', start='%s', end='%s')") + u"Range(%s=%s, %s, %s)", + u"Range(%s='%s', %s, %s)") start_str = start.strftime(_TIME_FORMAT) end_str = end.strftime(_TIME_FORMAT) - return PQLQuery(fmt % (row_idkey, self.name, start_str, end_str), + return PQLQuery(fmt % (self.name, row_idkey, start_str, end_str), self.index) def set_row_attrs(self, row_idkey, attrs): @@ -498,7 +498,7 @@ def set_row_attrs(self, row_idkey, attrs): """ fmt = id_key_format("Row", row_idkey, u"SetRowAttrs(%s, %s, %s)", - u"SetRowAttrs(row='%s', field='%s', %s)") + u"SetRowAttrs('%s', '%s', %s)") attrs_str = _create_attributes_str(attrs) return PQLQuery(fmt % (self.name, row_idkey, attrs_str), self.index) diff --git a/tests/test_orm.py b/tests/test_orm.py index cd8636a..1d7e2b9 100644 --- a/tests/test_orm.py +++ b/tests/test_orm.py @@ -237,15 +237,10 @@ def test_row(self): "Row(collaboration=5)", qry1.serialize()) - qry2 = collabField.row(10) - self.assertEquals( - "Row(collaboration=10)", - qry2.serialize()) - - qry3 = collabField.row("b7feb014-8ea7-49a8-9cd8-19709161ab63") + qry2 = collabField.row("b7feb014-8ea7-49a8-9cd8-19709161ab63") self.assertEquals( "Row(collaboration='b7feb014-8ea7-49a8-9cd8-19709161ab63')", - qry3.serialize()) + qry2.serialize()) def test_row_with_invalid_id_type(self): self.assertRaises(ValidationError, sampleField.row, {}) @@ -256,15 +251,10 @@ def test_setbit(self): u"Set(10, collaboration=5)", qry1.serialize()) - qry2 = collabField.set(10, 20) - self.assertEquals( - u"Set(20, collaboration=10)", - qry2.serialize()) - - qry3 = collabField.set("b7feb014-8ea7-49a8-9cd8-19709161ab63", "some_id") + qry2 = collabField.set("b7feb014-8ea7-49a8-9cd8-19709161ab63", "some_id") self.assertEquals( u"Set(some_id, collaboration='b7feb014-8ea7-49a8-9cd8-19709161ab63')", - qry3.serialize()) + qry2.serialize()) def test_setbit_with_invalid_id_type(self): self.assertRaises(ValidationError, sampleField.set, {}, 1) @@ -275,11 +265,11 @@ def test_setbit_with_timestamp(self): timestamp = datetime(2017, 4, 24, 12, 14) qry = collabField.set(10, 20, timestamp) self.assertEquals( - u"Set(20, collaboration=10, timestamp='2017-04-24T12:14')", + u"Set(20, collaboration=10, 2017-04-24T12:14)", qry.serialize() ) - def test_clearbit(self): + def test_clear(self): qry1 = collabField.clear(5, 10) self.assertEquals( "Clear(10, collaboration=5)", @@ -295,7 +285,7 @@ def test_clearbit(self): "Clear('some_id', collaboration='b7feb014-8ea7-49a8-9cd8-19709161ab63')", qry3.serialize()) - def test_clearbit_with_invalid_id_type(self): + def test_clear_with_invalid_id_type(self): self.assertRaises(ValidationError, sampleField.clear, {}, 1) self.assertRaises(ValidationError, sampleField.clear, 1, {}) self.assertRaises(ValidationError, sampleField.clear, 1, "zero") @@ -303,7 +293,7 @@ def test_clearbit_with_invalid_id_type(self): def test_topn(self): q1 = collabField.topn(27) self.assertEquals( - "TopN(collaboration, n=27)", + u"TopN(collaboration, n=27)", q1.serialize()) q2 = collabField.topn(10, collabField.row(3)) @@ -313,7 +303,7 @@ def test_topn(self): q3 = sampleField.topn(12, collabField.row(7), "category", 80, 81) self.assertEquals( - "TopN(sample-field, Row(collaboration=7), n=12, field='category', filters=[80,81])", + u"TopN(sample-field, Row(collaboration=7), n=12, field='category', filters=[80,81])", q3.serialize()) def test_range(self): @@ -322,12 +312,12 @@ def test_range(self): q1 = collabField.range(10, start, end) self.assertEquals( - "Range(row=10, field='collaboration', start='1970-01-01T00:00', end='2000-02-02T03:04')", + "Range(collaboration=10, 1970-01-01T00:00, 2000-02-02T03:04)", q1.serialize()) - q3 = sampleField.range("b7feb014-8ea7-49a8-9cd8-19709161ab63", start, end) + q3 = collabField.range("b7feb014-8ea7-49a8-9cd8-19709161ab63", start, end) self.assertEquals( - u"Range(row='b7feb014-8ea7-49a8-9cd8-19709161ab63', field='sample-field', start='1970-01-01T00:00', end='2000-02-02T03:04')", + u"Range(collaboration='b7feb014-8ea7-49a8-9cd8-19709161ab63', 1970-01-01T00:00, 2000-02-02T03:04)", q3.serialize()) def test_set_row_attributes(self): From 61dec0f15d1b0e2c5958983dbac716151d6a9f2c Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Wed, 27 Jun 2018 17:22:28 +0300 Subject: [PATCH 8/9] trivial --- integration_tests/test_client_it.py | 1 - tests/test_orm.py | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/integration_tests/test_client_it.py b/integration_tests/test_client_it.py index 876e3b8..d094c80 100644 --- a/integration_tests/test_client_it.py +++ b/integration_tests/test_client_it.py @@ -308,7 +308,6 @@ def test_range(self): response = client.query(field.range(10, start=datetime(2017, 5, 1, 0, 0), end=datetime(2018, 5, 1, 0, 0))) self.assertEqual([100], response.result.row.columns) - def test_range_field(self): client = self.get_client() field = self.col_index.field("rangefield", int_min=10, int_max=20) diff --git a/tests/test_orm.py b/tests/test_orm.py index 1d7e2b9..8d3849e 100644 --- a/tests/test_orm.py +++ b/tests/test_orm.py @@ -245,7 +245,7 @@ def test_row(self): def test_row_with_invalid_id_type(self): self.assertRaises(ValidationError, sampleField.row, {}) - def test_setbit(self): + def test_set(self): qry1 = collabField.set(5, 10) self.assertEquals( u"Set(10, collaboration=5)", @@ -256,12 +256,12 @@ def test_setbit(self): u"Set(some_id, collaboration='b7feb014-8ea7-49a8-9cd8-19709161ab63')", qry2.serialize()) - def test_setbit_with_invalid_id_type(self): + def test_set_with_invalid_id_type(self): self.assertRaises(ValidationError, sampleField.set, {}, 1) self.assertRaises(ValidationError, sampleField.set, 1, {}) self.assertRaises(ValidationError, sampleField.set, 1, "zero") - def test_setbit_with_timestamp(self): + def test_set_with_timestamp(self): timestamp = datetime(2017, 4, 24, 12, 14) qry = collabField.set(10, 20, timestamp) self.assertEquals( @@ -312,7 +312,7 @@ def test_range(self): q1 = collabField.range(10, start, end) self.assertEquals( - "Range(collaboration=10, 1970-01-01T00:00, 2000-02-02T03:04)", + u"Range(collaboration=10, 1970-01-01T00:00, 2000-02-02T03:04)", q1.serialize()) q3 = collabField.range("b7feb014-8ea7-49a8-9cd8-19709161ab63", start, end) From 06ef069efcf30ae072e60f1b85bd6b21fa8648a9 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Fri, 29 Jun 2018 00:02:16 +0300 Subject: [PATCH 9/9] Slice -> shard; added clean target to the Makefile --- Makefile | 5 ++++- pilosa/client.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index ed3520f..cb3fc8d 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ SRC_DIR = pilosa/internal DST_DIR = pilosa/internal -.PHONY: cover doc generate test test-all +.PHONY: build clean cover doc generate test test-all release upload cover: py.test --cov=pilosa tests integration_tests @@ -25,3 +25,6 @@ upload: twine upload dist/* release: build upload + +clean: + rm -rf build dist pilosa.egg-info \ No newline at end of file diff --git a/pilosa/client.py b/pilosa/client.py index f4093c3..4b5b971 100644 --- a/pilosa/client.py +++ b/pilosa/client.py @@ -276,7 +276,7 @@ def _import_columns(self, index_name, field_name, slice, columns): client._import_node(_ImportRequest(index_name, field_name, slice, columns)) def _fetch_fragment_nodes(self, index_name, slice): - path = "/fragment/nodes?slice=%d&index=%s" % (slice, index_name) + path = "/fragment/nodes?shard=%d&index=%s" % (slice, index_name) response = self.__http_request("GET", path) content = response.data.decode("utf-8") node_dicts = json.loads(content)