Permalink
Browse files

add loader for Hbase based on storage

  • Loading branch information...
1 parent 420f74f commit bd246004b5fcb5eb50c53450eaade333902b8064 Damien Hardy committed Apr 25, 2012
Showing with 87 additions and 1 deletion.
  1. +3 −0 hbase-start.sh
  2. +1 −1 thumbor_hbase/__init__.py
  3. +19 −0 thumbor_hbase/loader.py
  4. +64 −0 vows/loader_vows.py
View
@@ -0,0 +1,3 @@
+java -Xmx1000m -ea -XX:+UseConcMarkSweepGC -XX:+CMSIncrementalMode -Dhbase.log.dir=/usr/lib/hbase/bin/../logs -Dhbase.log.file=hbase.log -Dhbase.home.dir=/usr/lib/hbase/bin/.. -Dhbase.id.str= -Dhbase.root.logger=INFO,console -Djava.library.path=/usr/lib/hadoop-0.20/lib/native/Linux-amd64-64:/usr/lib/hbase/bin/../lib/native/Linux-amd64-64 -classpath `for i in /usr/lib/hbase/*.jar /usr/lib/hbase/lib/*.jar; do printf '%s:' $i; done` org.apache.hadoop.hbase.master.HMaster start >/dev/null 2>&1 &
+
+java -Xmx1000m -ea -XX:+UseConcMarkSweepGC -XX:+CMSIncrementalMode -Dhbase.zookeeper.quorum=localhost:2181 -classpath `for i in /usr/lib/hbase/*.jar /usr/lib/hbase/lib/*.jar; do printf '%s:' $i; done` org.apache.hadoop.hbase.thrift.ThriftServer start >/dev/null 2>&1 &
@@ -14,4 +14,4 @@
Config.define('HBASE_STORAGE_TABLE', 'thumbor')
Config.define('HBASE_STORAGE_FAMILY', 'images')
-__version__ = "0.4"
+__version__ = "0.5"
View
@@ -0,0 +1,19 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# thumbor imaging service
+# https://github.com/globocom/thumbor/wiki
+
+# Licensed under the MIT license:
+# http://www.opensource.org/licenses/mit-license
+# Copyright (c) 2011 globo.com timehome@corp.globo.com
+
+from thumbor_hbase.storage import Storage
+from thumbor.context import Context
+from thumbor.config import Config
+
+import pprint
+
+def load(context, path, callback):
+ storage = Storage(context)
+ callback(storage.get(path))
View
@@ -0,0 +1,64 @@
+#se!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# thumbor imaging service
+# https://github.com/globocom/thumbor/wiki
+
+# Licensed under the MIT license:
+# http://www.opensource.org/licenses/mit-license
+# Copyright (c) 2011 globo.com timehome@corp.globo.com
+
+from thrift.transport.TSocket import TSocket
+from thrift.transport.TTransport import TBufferedTransport
+from thrift.protocol import TBinaryProtocol
+from hbase import Hbase, ttypes
+
+
+from pyvows import Vows, expect
+from thumbor_hbase.storage import Storage
+import thumbor_hbase.loader as loader
+from thumbor.context import Context
+from thumbor.config import Config
+from fixtures.storage_fixture import IMAGE_URL, IMAGE_BYTES, get_server
+
+class HbaseDBContext(Vows.Context):
+ def setup(self):
+ transport = TBufferedTransport(TSocket(host='localhost', port=9090))
+ transport.open()
+ protocol = TBinaryProtocol.TBinaryProtocol(transport)
+ self.connection = Hbase.Client(protocol)
+ self.table='thumbor-test'
+ self.family='images:'
+
+ columns = []
+ col = ttypes.ColumnDescriptor()
+ col.name = self.family
+ col.maxVersions = 1
+ columns.append(col)
+ try:
+ self.connection.disableTable(self.table)
+ self.connection.deleteTable(self.table)
+ except ttypes.IOError:
+ pass
+ self.connection.createTable(self.table, columns)
+
+
+@Vows.batch
+class HbaseLoaderVows(HbaseDBContext):
+ class CanLoadImage(Vows.Context):
+ @Vows.async_topic
+ def topic(self,callback ):
+ config = Config(HBASE_STORAGE_TABLE=self.parent.table,HBASE_STORAGE_SERVER_PORT=9090)
+ context = Context(config=config, server=get_server('ACME-SEC'))
+ storage = Storage(context)
+
+ storage.put(IMAGE_URL % '1', IMAGE_BYTES)
+ return loader.load(context, IMAGE_URL % '1', callback)
+
+ def should_not_be_null(self, topic):
+ expect(topic).not_to_be_null()
+ expect(topic).not_to_be_an_error()
+
+ def should_have_proper_bytes(self, topic):
+ expect(topic[0]).to_equal(IMAGE_BYTES)
+

0 comments on commit bd24600

Please sign in to comment.