Skip to content

Commit

Permalink
add hbase adapter
Browse files Browse the repository at this point in the history
  • Loading branch information
GreatYYX committed Apr 11, 2018
1 parent 7255bc4 commit 6a57c4b
Show file tree
Hide file tree
Showing 2 changed files with 53 additions and 0 deletions.
1 change: 1 addition & 0 deletions rltk/io/adapter/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,4 @@
from rltk.io.adapter.memory_adapter import MemoryAdapter
from rltk.io.adapter.dbm_adapter import DBMAdapter
from rltk.io.adapter.redis_adapter import RedisAdapter
from rltk.io.adapter.hbase_adapter import HBaseAdapter
52 changes: 52 additions & 0 deletions rltk/io/adapter/hbase_adapter.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
import happybase

from rltk.record import Record
from rltk.io.adapter import KeyValueAdapter
from rltk.io.serializer import Serializer, PickleSerializer


class HBaseAdapter(KeyValueAdapter):
def __init__(self, host, table, serializer: Serializer=None, key_format='{record_id}', **kwargs):
if not serializer:
serializer = PickleSerializer()
self._conn = happybase.Connection(host=host, **kwargs)
self._serializer = serializer
self._key_format = key_format
self._family_name = 'rltk'
self._col_name = 'obj'
if bytes(table, 'utf-8') not in self._conn.tables():
self._create_table(table)
self._table = self._conn.table(table)

try:
self._get_key('test_id')
except:
raise ValueError('Invalid key_format.')

def _get_key(self, record_id):
return bytes(self._key_format.format(record_id=record_id), 'utf-8')

def __del__(self):
try:
self._conn.close()
except:
pass

def _create_table(self, table_name):
self._conn.create_table(table_name, {bytes(self._family_name, 'utf-8'):dict()})

def get(self, record_id) -> Record:
col = bytes('{}:{}'.format(self._family_name, self._col_name), 'utf-8')
return self._serializer.loads(self._table.rows(self._get_key(record_id))[col])

def set(self, record_id, record: Record):
col = bytes('{}:{}'.format(self._family_name, self._col_name), 'utf-8')
return self._table.put(self._get_key(record_id), {col: self._serializer.dumps(record)})

def __iter__(self):
return self.__next__()

def __next__(self):
col = bytes('{}:{}'.format(self._family_name, self._col_name), 'utf-8')
for key, data in self._table.scan(self._get_key('*')):
yield self._serializer.loads(data[col])

0 comments on commit 6a57c4b

Please sign in to comment.