Skip to content

Commit

Permalink
add loader for Hbase based on storage
Browse files Browse the repository at this point in the history
  • Loading branch information
Damien Hardy committed Apr 25, 2012
1 parent 420f74f commit bd24600
Show file tree
Hide file tree
Showing 4 changed files with 87 additions and 1 deletion.
3 changes: 3 additions & 0 deletions hbase-start.sh
@@ -0,0 +1,3 @@
java -Xmx1000m -ea -XX:+UseConcMarkSweepGC -XX:+CMSIncrementalMode -Dhbase.log.dir=/usr/lib/hbase/bin/../logs -Dhbase.log.file=hbase.log -Dhbase.home.dir=/usr/lib/hbase/bin/.. -Dhbase.id.str= -Dhbase.root.logger=INFO,console -Djava.library.path=/usr/lib/hadoop-0.20/lib/native/Linux-amd64-64:/usr/lib/hbase/bin/../lib/native/Linux-amd64-64 -classpath `for i in /usr/lib/hbase/*.jar /usr/lib/hbase/lib/*.jar; do printf '%s:' $i; done` org.apache.hadoop.hbase.master.HMaster start >/dev/null 2>&1 &

java -Xmx1000m -ea -XX:+UseConcMarkSweepGC -XX:+CMSIncrementalMode -Dhbase.zookeeper.quorum=localhost:2181 -classpath `for i in /usr/lib/hbase/*.jar /usr/lib/hbase/lib/*.jar; do printf '%s:' $i; done` org.apache.hadoop.hbase.thrift.ThriftServer start >/dev/null 2>&1 &
2 changes: 1 addition & 1 deletion thumbor_hbase/__init__.py
Expand Up @@ -14,4 +14,4 @@
Config.define('HBASE_STORAGE_TABLE', 'thumbor')
Config.define('HBASE_STORAGE_FAMILY', 'images')

__version__ = "0.4"
__version__ = "0.5"
19 changes: 19 additions & 0 deletions thumbor_hbase/loader.py
@@ -0,0 +1,19 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-

# thumbor imaging service
# https://github.com/globocom/thumbor/wiki

# Licensed under the MIT license:
# http://www.opensource.org/licenses/mit-license
# Copyright (c) 2011 globo.com timehome@corp.globo.com

from thumbor_hbase.storage import Storage
from thumbor.context import Context
from thumbor.config import Config

import pprint

def load(context, path, callback):
storage = Storage(context)
callback(storage.get(path))
64 changes: 64 additions & 0 deletions vows/loader_vows.py
@@ -0,0 +1,64 @@
#se!/usr/bin/python
# -*- coding: utf-8 -*-

# thumbor imaging service
# https://github.com/globocom/thumbor/wiki

# Licensed under the MIT license:
# http://www.opensource.org/licenses/mit-license
# Copyright (c) 2011 globo.com timehome@corp.globo.com

from thrift.transport.TSocket import TSocket
from thrift.transport.TTransport import TBufferedTransport
from thrift.protocol import TBinaryProtocol
from hbase import Hbase, ttypes


from pyvows import Vows, expect
from thumbor_hbase.storage import Storage
import thumbor_hbase.loader as loader
from thumbor.context import Context
from thumbor.config import Config
from fixtures.storage_fixture import IMAGE_URL, IMAGE_BYTES, get_server

class HbaseDBContext(Vows.Context):
def setup(self):
transport = TBufferedTransport(TSocket(host='localhost', port=9090))
transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(transport)
self.connection = Hbase.Client(protocol)
self.table='thumbor-test'
self.family='images:'

columns = []
col = ttypes.ColumnDescriptor()
col.name = self.family
col.maxVersions = 1
columns.append(col)
try:
self.connection.disableTable(self.table)
self.connection.deleteTable(self.table)
except ttypes.IOError:
pass
self.connection.createTable(self.table, columns)


@Vows.batch
class HbaseLoaderVows(HbaseDBContext):
class CanLoadImage(Vows.Context):
@Vows.async_topic
def topic(self,callback ):
config = Config(HBASE_STORAGE_TABLE=self.parent.table,HBASE_STORAGE_SERVER_PORT=9090)
context = Context(config=config, server=get_server('ACME-SEC'))
storage = Storage(context)

storage.put(IMAGE_URL % '1', IMAGE_BYTES)
return loader.load(context, IMAGE_URL % '1', callback)

def should_not_be_null(self, topic):
expect(topic).not_to_be_null()
expect(topic).not_to_be_an_error()

def should_have_proper_bytes(self, topic):
expect(topic[0]).to_equal(IMAGE_BYTES)

0 comments on commit bd24600

Please sign in to comment.