Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Merge pull request #3 from slackhappy/new_hypertable_api

hypertable: use SerializedCellsReader, ci search
  • Loading branch information...
commit 645c5a9cfd26355ed7d8435a21d265b1bb8e4353 2 parents 395dfd3 + e09578f
@slackhappy slackhappy authored
View
9 webapp/graphite/browser/views.py
@@ -19,6 +19,7 @@
from graphite.account.models import Profile
from graphite.hypertable_client import HyperTablePool, addPrefix, removePrefix
+from graphite.metrics.hypertable_search import HyperStore
from graphite.util import getProfile, getProfileByUsername, defaultUser, json
from graphite.logger import log
import hashlib
@@ -59,14 +60,8 @@ def searchHypertable(request):
if not query:
return HttpResponse("")
- query = 'SELECT * FROM search WHERE ROW REGEXP ".*%s.*"' % (query)
+ result_string = ','.join(HyperStore().search(query))
- metrics = []
- def processResult(key, family, column, val, ts):
- metrics.append(removePrefix(key))
-
- HyperTablePool.doQuery(query, processResult)
- result_string = ','.join(metrics)
return HttpResponse(result_string, mimetype='text/plain')
View
31 webapp/graphite/hypertable_client.py
@@ -3,6 +3,7 @@
from django.conf import settings
from graphite.logger import log
import hypertable.thriftclient
+import libHyperPython
import threading
import re
import time
@@ -47,11 +48,16 @@ def doScan(self, spec, table, cb):
scanner = conn.scanner_open(namespace, table, spec)
while True:
- row_data = conn.scanner_get_cells_as_arrays(scanner)
- if(len(row_data) == 0):
+ buf = conn.scanner_get_cells_serialized(scanner)
+ if (len(buf) <= 1):
break
- for key, family, column, val, ts in row_data:
- cb(key, family, column, val, ts)
+ scr = libHyperPython.SerializedCellsReader(buf, len(buf))
+ while scr.has_next():
+ cb( scr.row(),
+ scr.column_family(),
+ scr.column_qualifier(),
+ scr.value()[0:scr.value_len()],
+ scr.timestamp())
conn.close_scanner(scanner)
self.releaseConn(conn)
@@ -62,18 +68,25 @@ def doScan(self, spec, table, cb):
def doQuery(self, query, cb):
with self.semaphore:
+ start = time.time()
conn = self.getConn()
namespace = conn.namespace_open('monitor')
results = conn.hql_exec2(namespace, query, 0, 1)
-
while True:
- row_data = conn.next_row_as_arrays(results.scanner)
- if not row_data:
+ buf = conn.scanner_get_cells_serialized(results.scanner)
+ if (len(buf) <= 1):
break
- for key, family, column, val, ts in row_data:
- cb(key, family, column, val, ts)
+ scr = libHyperPython.SerializedCellsReader(buf, len(buf))
+ while scr.has_next():
+ cb( scr.row(),
+ scr.column_family(),
+ scr.column_qualifier(),
+ scr.value()[0:scr.value_len()],
+ scr.timestamp())
conn.close_scanner(results.scanner)
self.releaseConn(conn)
+ log.info(query)
+ log.info('fetch time: %s' % (time.time() - start))
HyperTablePool = ConnectionPool(20)
View
7 webapp/graphite/metrics/hypertable_search.py
@@ -30,6 +30,11 @@ def find(self, pathExpr):
else:
return [removePrefix(pathExpr)]
+ def search(self, query):
+ qre = '(?i)%s' % re.sub('\*', '.*', re.sub('\.', '\.', query))
+ return [removePrefix(p) for p in self.findByRegex(qre)]
+
+
def findByRegex(self, regex):
where = 'ROW REGEXP "%s"' % regex
return self.findHelper(where)
@@ -77,4 +82,4 @@ def processResult(key, family, column, val, ts):
HyperTablePool.doQuery(query, processResult)
return nodes
-hypertable_searcher = HyperTableIndexSearcher()
+hypertable_searcher = HyperTableIndexSearcher()
Please sign in to comment.
Something went wrong with that request. Please try again.