Permalink
Browse files

hypertable: use SerializedCellsReader, ci search

  • Loading branch information...
slackhappy committed May 15, 2012
1 parent afcd5cb commit e09578f769f7888f33dc603fcf3c405afcde8b3f
@@ -19,6 +19,7 @@
from graphite.account.models import Profile
from graphite.hypertable_client import HyperTablePool, addPrefix, removePrefix
+from graphite.metrics.hypertable_search import HyperStore
from graphite.util import getProfile, getProfileByUsername, defaultUser, json
from graphite.logger import log
import hashlib
@@ -59,14 +60,8 @@ def searchHypertable(request):
if not query:
return HttpResponse("")
- query = 'SELECT * FROM search WHERE ROW REGEXP ".*%s.*"' % (query)
+ result_string = ','.join(HyperStore().search(query))
- metrics = []
- def processResult(key, family, column, val, ts):
- metrics.append(removePrefix(key))
-
- HyperTablePool.doQuery(query, processResult)
- result_string = ','.join(metrics)
return HttpResponse(result_string, mimetype='text/plain')
@@ -3,6 +3,7 @@
from django.conf import settings
from graphite.logger import log
import hypertable.thriftclient
+import libHyperPython
import threading
import re
import time
@@ -47,11 +48,16 @@ def doScan(self, spec, table, cb):
scanner = conn.scanner_open(namespace, table, spec)
while True:
- row_data = conn.scanner_get_cells_as_arrays(scanner)
- if(len(row_data) == 0):
+ buf = conn.scanner_get_cells_serialized(scanner)
+ if (len(buf) <= 1):
break
- for key, family, column, val, ts in row_data:
- cb(key, family, column, val, ts)
+ scr = libHyperPython.SerializedCellsReader(buf, len(buf))
+ while scr.has_next():
+ cb( scr.row(),
+ scr.column_family(),
+ scr.column_qualifier(),
+ scr.value()[0:scr.value_len()],
+ scr.timestamp())
conn.close_scanner(scanner)
self.releaseConn(conn)
@@ -62,18 +68,25 @@ def doScan(self, spec, table, cb):
def doQuery(self, query, cb):
with self.semaphore:
+ start = time.time()
conn = self.getConn()
namespace = conn.namespace_open('monitor')
results = conn.hql_exec2(namespace, query, 0, 1)
-
while True:
- row_data = conn.next_row_as_arrays(results.scanner)
- if not row_data:
+ buf = conn.scanner_get_cells_serialized(results.scanner)
+ if (len(buf) <= 1):
break
- for key, family, column, val, ts in row_data:
- cb(key, family, column, val, ts)
+ scr = libHyperPython.SerializedCellsReader(buf, len(buf))
+ while scr.has_next():
+ cb( scr.row(),
+ scr.column_family(),
+ scr.column_qualifier(),
+ scr.value()[0:scr.value_len()],
+ scr.timestamp())
conn.close_scanner(results.scanner)
self.releaseConn(conn)
+ log.info(query)
+ log.info('fetch time: %s' % (time.time() - start))
HyperTablePool = ConnectionPool(20)
@@ -30,6 +30,11 @@ def find(self, pathExpr):
else:
return [removePrefix(pathExpr)]
+ def search(self, query):
+ qre = '(?i)%s' % re.sub('\*', '.*', re.sub('\.', '\.', query))
+ return [removePrefix(p) for p in self.findByRegex(qre)]
+
+
def findByRegex(self, regex):
where = 'ROW REGEXP "%s"' % regex
return self.findHelper(where)
@@ -77,4 +82,4 @@ def processResult(key, family, column, val, ts):
HyperTablePool.doQuery(query, processResult)
return nodes
-hypertable_searcher = HyperTableIndexSearcher()
+hypertable_searcher = HyperTableIndexSearcher()

0 comments on commit e09578f

Please sign in to comment.