Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
add loader for Hbase based on storage
- Loading branch information
Damien Hardy
committed
Apr 25, 2012
1 parent
420f74f
commit bd24600
Showing
4 changed files
with
87 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
java -Xmx1000m -ea -XX:+UseConcMarkSweepGC -XX:+CMSIncrementalMode -Dhbase.log.dir=/usr/lib/hbase/bin/../logs -Dhbase.log.file=hbase.log -Dhbase.home.dir=/usr/lib/hbase/bin/.. -Dhbase.id.str= -Dhbase.root.logger=INFO,console -Djava.library.path=/usr/lib/hadoop-0.20/lib/native/Linux-amd64-64:/usr/lib/hbase/bin/../lib/native/Linux-amd64-64 -classpath `for i in /usr/lib/hbase/*.jar /usr/lib/hbase/lib/*.jar; do printf '%s:' $i; done` org.apache.hadoop.hbase.master.HMaster start >/dev/null 2>&1 & | ||
|
||
java -Xmx1000m -ea -XX:+UseConcMarkSweepGC -XX:+CMSIncrementalMode -Dhbase.zookeeper.quorum=localhost:2181 -classpath `for i in /usr/lib/hbase/*.jar /usr/lib/hbase/lib/*.jar; do printf '%s:' $i; done` org.apache.hadoop.hbase.thrift.ThriftServer start >/dev/null 2>&1 & |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,19 @@ | ||
#!/usr/bin/python | ||
# -*- coding: utf-8 -*- | ||
|
||
# thumbor imaging service | ||
# https://github.com/globocom/thumbor/wiki | ||
|
||
# Licensed under the MIT license: | ||
# http://www.opensource.org/licenses/mit-license | ||
# Copyright (c) 2011 globo.com timehome@corp.globo.com | ||
|
||
from thumbor_hbase.storage import Storage | ||
from thumbor.context import Context | ||
from thumbor.config import Config | ||
|
||
import pprint | ||
|
||
def load(context, path, callback): | ||
storage = Storage(context) | ||
callback(storage.get(path)) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,64 @@ | ||
#se!/usr/bin/python | ||
# -*- coding: utf-8 -*- | ||
|
||
# thumbor imaging service | ||
# https://github.com/globocom/thumbor/wiki | ||
|
||
# Licensed under the MIT license: | ||
# http://www.opensource.org/licenses/mit-license | ||
# Copyright (c) 2011 globo.com timehome@corp.globo.com | ||
|
||
from thrift.transport.TSocket import TSocket | ||
from thrift.transport.TTransport import TBufferedTransport | ||
from thrift.protocol import TBinaryProtocol | ||
from hbase import Hbase, ttypes | ||
|
||
|
||
from pyvows import Vows, expect | ||
from thumbor_hbase.storage import Storage | ||
import thumbor_hbase.loader as loader | ||
from thumbor.context import Context | ||
from thumbor.config import Config | ||
from fixtures.storage_fixture import IMAGE_URL, IMAGE_BYTES, get_server | ||
|
||
class HbaseDBContext(Vows.Context): | ||
def setup(self): | ||
transport = TBufferedTransport(TSocket(host='localhost', port=9090)) | ||
transport.open() | ||
protocol = TBinaryProtocol.TBinaryProtocol(transport) | ||
self.connection = Hbase.Client(protocol) | ||
self.table='thumbor-test' | ||
self.family='images:' | ||
|
||
columns = [] | ||
col = ttypes.ColumnDescriptor() | ||
col.name = self.family | ||
col.maxVersions = 1 | ||
columns.append(col) | ||
try: | ||
self.connection.disableTable(self.table) | ||
self.connection.deleteTable(self.table) | ||
except ttypes.IOError: | ||
pass | ||
self.connection.createTable(self.table, columns) | ||
|
||
|
||
@Vows.batch | ||
class HbaseLoaderVows(HbaseDBContext): | ||
class CanLoadImage(Vows.Context): | ||
@Vows.async_topic | ||
def topic(self,callback ): | ||
config = Config(HBASE_STORAGE_TABLE=self.parent.table,HBASE_STORAGE_SERVER_PORT=9090) | ||
context = Context(config=config, server=get_server('ACME-SEC')) | ||
storage = Storage(context) | ||
|
||
storage.put(IMAGE_URL % '1', IMAGE_BYTES) | ||
return loader.load(context, IMAGE_URL % '1', callback) | ||
|
||
def should_not_be_null(self, topic): | ||
expect(topic).not_to_be_null() | ||
expect(topic).not_to_be_an_error() | ||
|
||
def should_have_proper_bytes(self, topic): | ||
expect(topic[0]).to_equal(IMAGE_BYTES) | ||
|