Skip to content
This repository was archived by the owner on Sep 28, 2022. It is now read-only.
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,7 @@ group: deprecated-2017Q2
before_install:
- openssl req -x509 -newkey rsa:4096 -keyout test.pilosa.local.key -out test.pilosa.local.crt -days 3650 -nodes -subj "/C=US/ST=Texas/L=Austin/O=Pilosa/OU=Com/CN=test.pilosa.local"
- wget https://s3.amazonaws.com/build.pilosa.com/pilosa-master-linux-amd64.tar.gz && tar xf pilosa-master-linux-amd64.tar.gz
- wget https://s3.amazonaws.com/build.pilosa.com/pilosa-cluster-resize-linux-amd64.tar.gz && tar xf pilosa-cluster-resize-linux-amd64.tar.gz
- ./pilosa-cluster-resize-linux-amd64/pilosa server --metric.diagnostics=false -b https://:30101 -d cluster_resize_data --cluster.disabled --tls.skip-verify --tls.certificate test.pilosa.local.crt --tls.key test.pilosa.local.key &
- ./pilosa-master-linux-amd64/pilosa server --metric.diagnostics=false -b https://:30101 -d https_data --cluster.disabled --tls.skip-verify --tls.certificate test.pilosa.local.crt --tls.key test.pilosa.local.key &
- ./pilosa-master-linux-amd64/pilosa server --metric.diagnostics=false -d http_data &
install: "pip install -r requirements/main.txt -r requirements/test.txt coveralls"
script:
Expand Down
9 changes: 1 addition & 8 deletions integration_tests/test_client_it.py
Original file line number Diff line number Diff line change
Expand Up @@ -324,14 +324,10 @@ def test_exclude_attrs_bits(self):
def test_http_request(self):
self.get_client().http_request("GET", "/status")

def test_version(self):
version = self.get_client()._server_version()
self.assertTrue(version)

def test_create_index_fail(self):
server = MockServer(404)
with server:
client = Client(server.uri, skip_version_check=True)
client = Client(server.uri)
self.assertRaises(PilosaServerError, client.create_index, self.index)


Expand All @@ -346,9 +342,6 @@ def get_client(cls):
server_address = os.environ.get("PILOSA_BIND", "")
if not server_address:
server_address = "http://:10101"
if os.environ.get("LEGACY_MODE_OFF", "") == "true":
return Client(server_address, tls_skip_verify=True, skip_version_check=True, legacy_mode=False)

return Client(server_address, tls_skip_verify=True)


Expand Down
65 changes: 2 additions & 63 deletions pilosa/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,6 @@
__all__ = ("Client", "Cluster", "URI")

_MAX_HOSTS = 10
_PILOSA_MIN_VERSION = ">=0.9.0"
_IS_PY2 = sys.version_info.major == 2


Expand Down Expand Up @@ -82,8 +81,7 @@ class Client(object):

def __init__(self, cluster_or_uri=None, connect_timeout=30000, socket_timeout=300000,
pool_size_per_route=10, pool_size_total=100, retry_count=3,
tls_skip_verify=False, tls_ca_certificate_path="", skip_version_check=False,
legacy_mode=False):
tls_skip_verify=False, tls_ca_certificate_path=""):
if cluster_or_uri is None:
self.cluster = Cluster(URI())
elif isinstance(cluster_or_uri, Cluster):
Expand All @@ -102,8 +100,6 @@ def __init__(self, cluster_or_uri=None, connect_timeout=30000, socket_timeout=30
self.retry_count = retry_count
self.tls_skip_verify = tls_skip_verify
self.tls_ca_certificate_path = tls_ca_certificate_path
self.skip_version_check = skip_version_check
self.legacy_mode = legacy_mode
self.__current_host = None
self.__client = None
self.logger = logging.getLogger("pilosa")
Expand Down Expand Up @@ -198,25 +194,7 @@ def _read_schema(self):
response = self.__http_request("GET", "/schema")
return json.loads(response.data.decode('utf-8')).get("indexes") or []

def _read_schema_legacy(self):
response = self.__http_request("GET", "/status")
return json.loads(response.data.decode('utf-8'))["status"]

def _schema_legacy(self):
status = self._read_schema_legacy()
nodes = status.get("Nodes")
schema = Schema()
for index_info in nodes[0].get("Indexes", []):
index = schema.index(index_info["Name"])
for frame_info in index_info.get("Frames", []):
options = decode_frame_meta_options_legacy(frame_info)
index.frame(frame_info["Name"], **options)

return schema

def schema(self):
if self.legacy_mode:
return self._schema_legacy()
schema = Schema()
for index_info in self._read_schema():
index = schema.index(index_info["name"])
Expand Down Expand Up @@ -300,41 +278,14 @@ def _fetch_fragment_nodes(self, index_name, slice):
node_dicts = json.loads(content)
nodes = []
for node_dict in node_dicts:
# NOTE: Legacy Pilosa < 0.9 doesn't have uri field
node_dict = node_dict["uri"] if "uri" in node_dict else node_dict
node_dict = node_dict["uri"]
nodes.append(_Node(node_dict["scheme"], node_dict["host"], node_dict.get("port", "")))
return nodes

def _import_node(self, import_request):
data = import_request.to_protobuf()
self.__http_request("POST", "/import", data=data)

def _server_version(self):
path = "/version"
response = self.__http_request("GET", path)
content = json.loads(response.data.decode("utf-8"))
return content.get("version", "")

def _check_server_version(self, version):
import semver
if not version:
self.logger.warning("Pilosa server version is not available")
return False
if version.startswith("v"):
version = version[1:]
self.logger.info("Pilosa server version: %s", version)
try:
if not semver.match(version, _PILOSA_MIN_VERSION):
self.logger.warning("Pilosa server's version is %s, "
"does not meet the minimum required for this version of the client: %s",
version, _PILOSA_MIN_VERSION)
return False
return True
except ValueError:
self.logger.warning("Invalid Pilosa server version: %s or minimum server version: %s",
version, _PILOSA_MIN_VERSION)
return False

def __http_request(self, method, path, data=None, headers=None):
if not self.__client:
self.__connect()
Expand Down Expand Up @@ -385,9 +336,6 @@ def __connect(self):

client = urllib3.PoolManager(**client_options)
self.__client = client
if not self.skip_version_check:
ok = self._check_server_version(self._server_version())
self.legacy_mode = not ok


def decode_frame_meta_options(frame_info):
Expand All @@ -399,15 +347,6 @@ def decode_frame_meta_options(frame_info):
"time_quantum": TimeQuantum(meta.get("timeQuantum", "")),
}

def decode_frame_meta_options_legacy(frame_info):
meta = frame_info.get("Meta", {})
return {
"cache_size": meta.get("CacheSize", 50000),
"cache_type": CacheType(meta.get("CacheType", "")),
"inverse_enabled": meta.get("InverseEnabled", False),
"time_quantum": TimeQuantum(meta.get("TimeQuantum", "")),
}


class URI:
"""Represents a Pilosa URI
Expand Down
42 changes: 21 additions & 21 deletions pilosa/orm.py
Original file line number Diff line number Diff line change
Expand Up @@ -321,9 +321,9 @@ def set_column_attrs(self, column_idkey, attrs):
:return: Pilosa query
:rtype: pilosa.PQLQuery
"""
fmt = id_key_format("Column ID", column_idkey,
u"SetColumnAttrs(columnID=%s, %s)",
u"SetColumnAttrs(columnID='%s, %s)")
fmt = id_key_format("Column", column_idkey,
u"SetColumnAttrs(col=%s, %s)",
u"SetColumnAttrs(col='%s, %s)")
attrs_str = _create_attributes_str(attrs)
return PQLQuery(fmt % (column_idkey, attrs_str), self)

Expand Down Expand Up @@ -390,8 +390,8 @@ def bitmap(self, row_idkey):
:rtype: pilosa.PQLBitmapQuery
"""
fmt = id_key_format("Row ID/Key", row_idkey,
u"Bitmap(rowID=%s, frame='%s')",
u"Bitmap(rowID='%s', frame='%s')")
u"Bitmap(row=%s, frame='%s')",
u"Bitmap(row='%s', frame='%s')")
return PQLQuery(fmt % (row_idkey, self.name), self.index)

def inverse_bitmap(self, column_idkey):
Expand All @@ -405,9 +405,9 @@ def inverse_bitmap(self, column_idkey):
:return: Pilosa bitmap query
:rtype: pilosa.PQLBitmapQuery
"""
fmt = id_key_format("Column ID/Key", column_idkey,
u"Bitmap(columnID=%s, frame='%s')",
u"Bitmap(columnID='%s', frame='%s')")
fmt = id_key_format("Column", column_idkey,
u"Bitmap(col=%s, frame='%s')",
u"Bitmap(col='%s', frame='%s')")
return PQLQuery(fmt % (column_idkey, self.name), self.index)

def setbit(self, row_idkey, column_idkey, timestamp=None):
Expand All @@ -422,13 +422,13 @@ def setbit(self, row_idkey, column_idkey, timestamp=None):
:rtype: pilosa.PQLQuery
"""
if isinstance(row_idkey, int) and isinstance(column_idkey, int):
fmt = u"SetBit(rowID=%s, frame='%s', columnID=%s%s)"
fmt = u"SetBit(row=%s, frame='%s', col=%s%s)"
elif isinstance(row_idkey, _basestring) and isinstance(column_idkey, _basestring):
fmt = u"SetBit(rowID='%s', frame='%s', columnID='%s'%s)"
fmt = u"SetBit(row='%s', frame='%s', col='%s'%s)"
validate_key(row_idkey)
validate_key(column_idkey)
else:
raise ValidationError("Both Row and Column ID/Keys must be integers or strings")
raise ValidationError("Both Row and Columns must be integers or strings")
ts = ", timestamp='%s'" % timestamp.strftime(_TIME_FORMAT) if timestamp else ''
return PQLQuery(fmt % (row_idkey, self.name, column_idkey, ts), self.index)

Expand All @@ -443,13 +443,13 @@ def clearbit(self, row_idkey, column_idkey):
:rtype: pilosa.PQLQuery
"""
if isinstance(row_idkey, int) and isinstance(column_idkey, int):
fmt = u"ClearBit(rowID=%s, frame='%s', columnID=%s)"
fmt = u"ClearBit(row=%s, frame='%s', col=%s)"
elif isinstance(row_idkey, _basestring) and isinstance(column_idkey, _basestring):
fmt = u"ClearBit(rowID='%s', frame='%s', columnID='%s')"
fmt = u"ClearBit(row='%s', frame='%s', col='%s')"
validate_key(row_idkey)
validate_key(column_idkey)
else:
raise ValidationError("Both Row and Column ID/Keys must be integers or strings")
raise ValidationError("Both Row and Columns must be integers or strings")
return PQLQuery(fmt % (row_idkey, self.name, column_idkey), self.index)

def topn(self, n, bitmap=None, field="", *values):
Expand Down Expand Up @@ -504,7 +504,7 @@ def range(self, row_idkey, start, end):
:param datetime.datetime start: start timestamp
:param datetime.datetime end: end timestamp
"""
return self._range("rowID", row_idkey, start, end)
return self._range("row", row_idkey, start, end)

def inverse_range(self, column_id, start, end):
"""Creates a Range query.
Expand All @@ -516,10 +516,10 @@ def inverse_range(self, column_id, start, end):
:param datetime.datetime start: start timestamp
:param datetime.datetime end: end timestamp
"""
return self._range("columnID", column_id, start, end)
return self._range("col", column_id, start, end)

def _range(self, label, row_idkey, start, end):
fmt = id_key_format("Row ID", row_idkey,
fmt = id_key_format("Row", row_idkey,
u"Range(%s=%s, frame='%s', start='%s', end='%s')",
u"Range(%s='%s', frame='%s', start='%s', end='%s')")
start_str = start.strftime(_TIME_FORMAT)
Expand All @@ -544,9 +544,9 @@ def set_row_attrs(self, row_idkey, attrs):
:return: Pilosa query
:rtype: pilosa.PQLQuery
"""
fmt = id_key_format("Row ID", row_idkey,
u"SetRowAttrs(rowID=%s, frame='%s', %s)",
u"SetRowAttrs(rowID='%s', frame='%s', %s)")
fmt = id_key_format("Row", row_idkey,
u"SetRowAttrs(row=%s, frame='%s', %s)",
u"SetRowAttrs(row='%s', frame='%s', %s)")
attrs_str = _create_attributes_str(attrs)
return PQLQuery(fmt % (row_idkey, self.name, attrs_str), self.index)

Expand Down Expand Up @@ -733,7 +733,7 @@ def set_value(self, column_id, value):
:return: a PQL query
:rtype: PQLQuery
"""
q = u"SetFieldValue(frame='%s', columnID=%d, %s=%d)" % \
q = u"SetFieldValue(frame='%s', col=%d, %s=%d)" % \
(self.frame_name, column_id, self.name, value)
return PQLQuery(q, self.index)

Expand Down
11 changes: 0 additions & 11 deletions tests/test_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -156,17 +156,6 @@ def test_repr(self):
uri = URI.address("https://pilosa.com:1337")
self.assertEquals("<URI https://pilosa.com:1337>", repr(uri))

def test_check_server_version(self):
# Mostly for coverage
client = Client()
# Falsy version
client._check_server_version("")
# Invalid version
client._check_server_version("v0")
client._check_server_version("v0.8.0-127-g2772daa")
client._check_server_version("v0.9.0-127-g2772daa")
client._check_server_version("v0.9.0")

def compare(self, uri, scheme, host, port):
self.assertEquals(scheme, uri.scheme)
self.assertEquals(host, uri.host)
Expand Down
Loading