Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

adjust dir structure

  • Loading branch information...
commit a9d5cb4a7e8e9f4080723c5fa38c1815424a3900 1 parent 7ed097e
@davies davies authored
Showing with 2 additions and 611 deletions.
  1. +2 −2 Makefile.am
  2. 0  README → README.md
  3. +0 −157 python/dbserver.py
  4. +0 −16 python/setup.py
  5. +0 −36 python/status.py
  6. +0 −209 python/store.pyx
  7. 0  { → src}/ae_epoll.c
  8. 0  { → src}/ae_kqueue.c
  9. 0  { → src}/ae_select.c
  10. 0  { → src}/beansdb.c
  11. 0  { → src}/beansdb.h
  12. 0  { → src}/bitcask.c
  13. 0  { → src}/bitcask.h
  14. 0  { → src}/clock_gettime_stub.c
  15. 0  { → src}/codec.c
  16. 0  { → src}/codec.h
  17. 0  { → src}/crc32.c
  18. 0  { → src}/diskmgr.c
  19. 0  { → src}/diskmgr.h
  20. 0  { → src}/fnv1a.h
  21. 0  { → src}/hint.c
  22. 0  { → src}/hint.h
  23. 0  { → src}/hstore.c
  24. 0  { → src}/hstore.h
  25. 0  { → src}/htree.c
  26. 0  { → src}/htree.h
  27. 0  { → src}/item.c
  28. 0  { → src}/quicklz.c
  29. 0  { → src}/quicklz.h
  30. 0  { → src}/record.c
  31. 0  { → src}/record.h
  32. 0  { → src}/stats.c
  33. 0  { → src}/thread.c
  34. +0 −5 t/test.sh
  35. +0 −42 t/test_htree.c
  36. +0 −21 t/test_store.c
  37. +0 −2  tools/Makefile.am
  38. +0 −121 tools/convert.c
View
4 Makefile.am
@@ -1,9 +1,9 @@
bin_PROGRAMS = beansdb
-beansdb_SOURCES = beansdb.c item.c fnv1a.h beansdb.h thread.c htree.h htree.c hint.h hint.c record.h record.c codec.h codec.c bitcask.h bitcask.c hstore.h hstore.c quicklz.h quicklz.c diskmgr.h diskmgr.c
+beansdb_SOURCES = src/beansdb.c src/item.c src/fnv1a.h src/beansdb.h src/thread.c src/htree.h src/htree.c src/hint.h src/hint.c src/record.h src/record.c src/codec.h src/codec.c src/bitcask.h src/bitcask.c src/hstore.h src/hstore.c src/quicklz.h src/quicklz.c src/diskmgr.h src/diskmgr.c
beansdb_CPPFLAGS = -DNDEBUG
SUBDIRS = doc
-EXTRA_DIST = python crc32.c clock_gettime_stub.c ae_epoll.c ae_kqueue.c ae_select.c CREDITS AUTHORS LICENSE
+EXTRA_DIST = python src/crc32.c src/clock_gettime_stub.c src/ae_epoll.c src/ae_kqueue.c src/ae_select.c CREDITS AUTHORS LICENSE
dist-hook:
rm -rf $(distdir)/*/.svn/
View
0  README → README.md
File renamed without changes
View
157 python/dbserver.py
@@ -1,157 +0,0 @@
-#!/usr/bin/env python
-
-import time, math
-import logging
-from eventlet import api, util, tpool
-#from tcstore import TCStore, MultiTCStore
-from store import HStore
-
-quit = False
-
-def flush(store):
- while not quit:
- store.flush(30)
- api.sleep(1)
-
-def handler(store, sock, reader, writer):
-
- def writeline(line):
- writer.write(line)
- writer.write('\r\n')
-
- while not quit:
- # pass through every non-eof line
- x = reader.readline()
- if not x: break
-
- args = x.split()
- cmd = args[0]
- #print args
-
- st = time.time()
- if cmd == 'get':
- def do():
- for key in args[1:]:
- v = store.get(key)
- if v is not None:
- writeline("VALUE %s %d %d" % (key, 0, len(v)))
- writeline(v)
- del v
- writeline('END')
- t = time.time() - st
- if t > 0.1:
- print t, args
- writer.flush()
- tpool.execute(do)
-
- elif cmd == 'set':
- def do():
- key, flag, rev, bytes = args[1:5]
- flag, rev, bytes = int(flag), int(rev), int(bytes)
-
- buf = reader.read(bytes)
- while len(buf) < bytes:
- buf += reader.read(bytes - len(buf))
- reader.read(2)
- if store.set(key, buf, rev):
- writeline('STORED')
- else:
- writeline('NOT_STORED')
- del buf
- t = time.time() - st
- if t > 0.1:
- print t, args
- writer.flush()
- tpool.execute(do)
-
- elif cmd == 'delete':
- def do():
- key = args[1]
- v = store.delete(key)
- noreply = len(args) > 3 and int(args[3]) or False
- if not noreply:
- writeline(v and 'DELETED' or 'NOT_FOUND')
- t = time.time() - st
- if t > 0.1:
- print t, args
- writer.flush()
- tpool.execute(do)
-
- elif cmd == 'stat':
- writeline('END')
- writer.flush()
-
- elif cmd == 'quit':
- break
-
- else:
- writeline('CLIENT_ERROR')
-
- api.sleep()
-
- reader.close()
- writer.close()
- sock.close()
-
-
-def main():
- from optparse import OptionParser
- parser = OptionParser()
- parser.add_option("-l", "--listen", dest="host", default="0.0.0.0",
- help="the ip interface to bind")
- parser.add_option("-p", "--port", default=7902, type=int,
- help="which port to listen")
-# parser.add_option("-d", "--daemon", action="store_true",
-# help="run in daemon", default=False)
- parser.add_option("-H", "--home", default="beansdb",
- help="the database path")
- parser.add_option("-c", "--count", default=16, type=int,
- help="number of db file, power of 16")
- parser.add_option("-s", "--start", default=0, type=int,
- help="start index of db file")
- parser.add_option("-e", "--end", default=-1, type=int,
- help="last end of db file, -1 means no limit")
- parser.add_option("-n", "--limit", default=100, type=int,
- help="diffs limit to do db scan")
- parser.add_option("-t", "--threads", type=int, default=20,
- help="number of IO threads")
-
-
- (options, args) = parser.parse_args()
-
- store = (HStore(options.home,
- int(math.log(options.count, 16)),
- options.start, options.end))
- #store.check(options.limit, nonblocking=True)
- api.spawn(tpool.execute, store.check, options.limit) # check in thread pool
- api.spawn(tpool.execute, flush, store)
-
- print "server listening on %s:%s" % (options.host, options.port)
- server = api.tcp_listener((options.host, options.port))
- util.set_reuse_addr(server)
-
- while True:
- try:
- new_sock, address = server.accept()
- except KeyboardInterrupt:
- break
- api.spawn(handler, store, new_sock,
- new_sock.makefile('r'), new_sock.makefile('w'))
-
- global quit
- quit = True
-
- print 'close listener ...'
- server.close()
-
- print 'stop checker thread ...'
- store.stop_check()
-
- print 'stop worker threads ...'
- tpool.killall()
-
- print 'close store...'
- store.close()
-
-if __name__ == '__main__':
- main()
View
16 python/setup.py
@@ -1,16 +0,0 @@
-from setuptools import setup, Extension
-from Cython.Distutils import build_ext
-
-setup(
- name="beansdb",
- cmdclass={'build_ext': build_ext},
- ext_modules=[
- Extension("store", ["store.pyx", "../htree.c", "../hstore.c"],
- libraries=["tokyocabinet"],
- )],
- packages=["beansdb"],
- test_suite="test",
- install_requires=['cython'],
- zip_safe=False,
- version="0.3.0",
-)
View
36 python/status.py
@@ -1,36 +0,0 @@
-#!/usr/bin/python
-
-import os,os.path
-import time
-import sys
-import math
-
-from dbclient import Beansdb, db
-
-def stat(s):
- st = {}
- for i in range(16):
- for d,h,c in [line.split(' ') for line in (s.get('@%x'%i) or '').split('\n') if line]:
- if not d.endswith('/') or len(d) != 2: continue
- st[i*16+int(d[0],16)] = int(c)
- return st
-
-def status(db):
- st = {}
- for s in db.servers.values():
- st[str(s)] = stat(s)
- print '\t',s,
- print
- scale = 256/db.buckets_count
- for b in range(db.buckets_count):
- print '%x' % b,
- for s in db.servers.values():
- print '\t',
- if s in db.buckets[b]:
- print '\x1b[01;32m%8d\x1b[0m' % sum(st[str(s)].get(b*scale+i,0) for i in range(scale)),
- else:
- print '%8d' % sum(st[str(s)].get(b*scale,0) for i in range(scale)),
- print
-
-if __name__ == '__main__':
- status(db)
View
209 python/store.pyx
@@ -1,209 +0,0 @@
-"""
- Beansdb - A high available distributed key-value storage system:
-
- http://beansdb.googlecode.com
-
- Copyright 2009 Douban Inc. All rights reserved.
-
- Use and distribution licensed under the BSD license. See
- the LICENSE file for full text.
-"""
-
-__authors__ = ["Davies Liu <davies.liu@gmail.com>"]
-__version__ = "0.3.0"
-
-import os
-
-cdef extern from "Python.h":
- object PyString_FromString(char *s)
- object PyString_FromStringAndSize(char *s, Py_ssize_t len)
- ctypedef struct PyThreadState:
- pass
- PyThreadState *PyEval_SaveThread()
- void PyEval_RestoreThread(PyThreadState *_save)
-
-cdef extern from "stdlib.h":
- void free(void *ptr)
-
-cdef extern from "stdint.h":
- ctypedef unsigned int uint32_t
-
-cdef extern from "stdbool.h":
- ctypedef int bool
-
-cdef extern from "../htree.h":
- ctypedef struct C_HTree "HTree":
- pass
- C_HTree* ht_open(char *path, int depth) except NULL
- void ht_close(C_HTree *tree)
- void ht_clear(C_HTree *tree)
- void ht_flush(C_HTree *tree)
- void ht_add(C_HTree *tree, char *name,
- int ver, uint32_t hash, bool update)
- void ht_remove(C_HTree *tree, char *name, bool update)
- uint32_t ht_get_hash(C_HTree *tree, char *key, int *count)
- char* ht_list(C_HTree *tree, char *dir) except NULL
-
-cdef extern from "../hstore.h":
- ctypedef struct C_HStore "HStore":
- pass
- C_HStore* hs_open(char* path, int height, int start, int end)
- void hs_clear(C_HStore *store)
- void hs_check(C_HStore *store, int scan_limit)
- void hs_stop_check(C_HStore *store)
- void hs_flush(C_HStore *store, int limit)
- void hs_close(C_HStore *store)
- bool hs_set(C_HStore *store, char *key, char* value,
- int vlen, int ver, uint32_t flag)
- char *hs_get(C_HStore *store, char *key, int *vlen, uint32_t *flag)
- bool hs_delete(C_HStore *store, char *key)
-
-
-cdef class HTree(object):
-
- cdef C_HTree *tree
- cdef readonly object path
- cdef readonly int depth
-
- def __init__(self, char* path, int depth):
- self.path = path
- self.depth = depth
- self.tree = ht_open(path, depth)
-
- if not self.tree:
- raise Exception, 'open %s failed' % path
-
- def close(self):
- if self.tree:
- ht_close(self.tree)
- self.tree = NULL
-
- def flush(self):
- ht_flush(self.tree)
-
- def clear(self):
- ht_clear(self.tree)
-
- def add(self, char* key, int ver, int hash=0, bool update=1):
- ht_add(self.tree, key, ver, hash, update)
-
- def remove(self, char* key, bool update=0):
- ht_remove(self.tree, key, update)
-
- def get_hash(self, object key):
- cdef int count = 0
- cdef int hash
- hash = ht_get_hash(self.tree, key, &count)
- return (hash, count)
-
- def list(self, object dir):
- cdef char *rs = ht_list(self.tree, dir)
- cdef object r = PyString_FromString(rs)
- free(rs)
- return r
-
- def __hash__(self):
- cdef int count = 0
- return ht_get_hash(self.tree, "@", &count)
-
- def __len__(self):
- cdef int count = 0
- ht_get_hash(self.tree, "@", &count)
- return count
-
-
-cdef class HStore(object):
-
- cdef C_HStore *store
- cdef readonly object path
- cdef readonly int height
-
- def __init__(self, char* path, int height=0, int start=0, int end=-1):
- cdef PyThreadState *_save
- self.path = path
- self.height = height
- if not os.path.exists(path):
- os.makedirs(path)
- if height > 1:
- end = 1 << (height * 4) if end == -1 else end
- for i in range(start/16, end/16):
- d = os.path.join(path, ("%%0%dx" % (height - 1)) % i)
- if not os.path.exists(d):
- os.makedirs(d)
-
- self.store = hs_open(path, height, start, end)
- if not self.store:
- raise Exception, 'open %s failed' % path
-
- def check(self, int limit=0):
- cdef PyThreadState *_save
- if self.store:
- _save = PyEval_SaveThread()
- hs_check(self.store, limit)
- PyEval_RestoreThread(_save)
-
- def stop_check(self):
- cdef PyThreadState *_save
- if self.store:
- _save = PyEval_SaveThread()
- hs_stop_check(self.store)
- PyEval_RestoreThread(_save)
-
- def flush(self, int limit=64):
- cdef PyThreadState *_save
- if self.store:
- _save = PyEval_SaveThread()
- hs_flush(self.store, limit)
- PyEval_RestoreThread(_save)
-
- def clear(self):
- if self.store:
- hs_clear(self.store)
-
- def close(self):
- cdef PyThreadState *_save
- if self.store:
- _save = PyEval_SaveThread()
- hs_close(self.store)
- PyEval_RestoreThread(_save)
- self.store = NULL
-
- def set(self, key, value, int ver=0, int flag=0):
- cdef PyThreadState *_save
- cdef bool r
- if self.store:
- _save = PyEval_SaveThread()
- r = hs_set(self.store, key, value, len(value), ver, flag)
- PyEval_RestoreThread(_save)
- return r
-
- def get(self, key):
-
- if not self.store:
- return
- cdef PyThreadState *_save
- cdef int len = 0
- cdef uint32_t flag = 0
- cdef char *value
-
- _save = PyEval_SaveThread()
- try:
- value = hs_get(self.store, key, &len, &flag)
- finally:
- PyEval_RestoreThread(_save)
-
- if value:
- r = PyString_FromStringAndSize(value, len)
- free(value)
- return r
-
- def delete(self, char *key):
-
- cdef PyThreadState *_save
- cdef bool r
-
- if self.store:
- _save = PyEval_SaveThread()
- r = hs_delete(self.store, key)
- PyEval_RestoreThread(_save)
- return r
View
0  ae_epoll.c → src/ae_epoll.c
File renamed without changes
View
0  ae_kqueue.c → src/ae_kqueue.c
File renamed without changes
View
0  ae_select.c → src/ae_select.c
File renamed without changes
View
0  beansdb.c → src/beansdb.c
File renamed without changes
View
0  beansdb.h → src/beansdb.h
File renamed without changes
View
0  bitcask.c → src/bitcask.c
File renamed without changes
View
0  bitcask.h → src/bitcask.h
File renamed without changes
View
0  clock_gettime_stub.c → src/clock_gettime_stub.c
File renamed without changes
View
0  codec.c → src/codec.c
File renamed without changes
View
0  codec.h → src/codec.h
File renamed without changes
View
0  crc32.c → src/crc32.c
File renamed without changes
View
0  diskmgr.c → src/diskmgr.c
File renamed without changes
View
0  diskmgr.h → src/diskmgr.h
File renamed without changes
View
0  fnv1a.h → src/fnv1a.h
File renamed without changes
View
0  hint.c → src/hint.c
File renamed without changes
View
0  hint.h → src/hint.h
File renamed without changes
View
0  hstore.c → src/hstore.c
File renamed without changes
View
0  hstore.h → src/hstore.h
File renamed without changes
View
0  htree.c → src/htree.c
File renamed without changes
View
0  htree.h → src/htree.h
File renamed without changes
View
0  item.c → src/item.c
File renamed without changes
View
0  quicklz.c → src/quicklz.c
File renamed without changes
View
0  quicklz.h → src/quicklz.h
File renamed without changes
View
0  record.c → src/record.c
File renamed without changes
View
0  record.h → src/record.h
File renamed without changes
View
0  stats.c → src/stats.c
File renamed without changes
View
0  thread.c → src/thread.c
File renamed without changes
View
5 t/test.sh
@@ -1,5 +0,0 @@
-#!/bin/bash
-
-gcc -std=c99 -O2 ../htree.c test_htree.c -ltokyocabinet -pg && time ./a.out
-#gcc -std=c99 htree.c hstore.c test_store.c -ltokyocabinet && ./a.out
-#gcc -std=c99 htree.c hstore.c test_store.c -ltokyocabinet && ./a.out
View
42 t/test_htree.c
@@ -1,42 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <stdint.h>
-#include <stdbool.h>
-#include "../htree.h"
-
-int main(int argc, char** argv)
-{
- HTree *t = ht_open("*", 0);
- printf("hash %d\n", ht_get_hash(t, "@", NULL));
- printf("%s\n", ht_list(t,""));
- ht_clear(t);
- //printf("hash %d\n", ht_get_hash(t, "@", NULL));
- //printf("%s\n", ht_list(t,""));
-
- int i=0;
- char buf[100];
- for (int k=0;k<1;k++){
- for (i=0;i<200000;i++){
- sprintf(buf, "/photo/photo/%d.jpg", i);
- //printf(buf);
- ht_add(t, buf, 1, 3, 0);
- }
- printf("add complete\n");
- for (i=0;i<1000;i++){
- sprintf(buf, "/photo/photo/%d.jpg", i);
- /*sprintf(buf, "/photo/photo/xxxxxxxxxxxxxxxxxxxxxxfile%d", i);*/
- //ht_remove(t, buf, 0);
- }
- printf("remove complete\n");
- }
- //remove_from_htree(buf);
- printf("update complete\n");
- // print_tree(&tree);
-
- printf("hash %d\n", ht_get_hash(t, "@", NULL));
- printf("%s\n", ht_list(t,""));
-
- ht_close(t);
- return 0;
-}
View
21 t/test_store.c
@@ -1,21 +0,0 @@
-#include <stdio.h>
-#include <string.h>
-#include <stdbool.h>
-#include <assert.h>
-
-#include "hstore.h"
-
-int main(int argc, char** argv)
-{
- HStore *store = hs_open("testdb", 2);
- assert(store);
- hs_set(store, "hello", "hello", 5, 1);
- return 0;
- int n;
- printf("%s\n", hs_get(store, "@", &n));
- char *r = hs_get(store, "hello", &n);
- printf("%d %d\n", n, 0);
- return 0;
- hs_close(store);
- return 0;
-}
View
2  tools/Makefile.am
@@ -1,2 +0,0 @@
-bin_PROGRAMS = convert
-convert_SOURCES = convert.c
View
121 tools/convert.c
@@ -1,121 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <stdint.h>
-#include <stdbool.h>
-#include <stdio.h>
-#include <stdbool.h>
-#include <stdint.h>
-#include <tchdb.h>
-#include "../fnv1a.h"
-#include "../htree.c"
-#include "../hstore.c"
-
-typedef struct t_old_meta {
- int32_t version;
- uint32_t flags;
- int32_t length;
-} OldMeta;
-
-static TCHDB *my_open(const char *path, int mode)
-{
- TCHDB *db = tchdbnew();
- if (!db){
- printf("create %s failed\n", path);
- exit(1);
- }
- tchdbtune(db, 1000000, 6, -1, HDBTDEFLATE);
- if(!tchdbopen(db, path, mode)){
- printf("open %s failed %d\n", path, tchdbecode(db));
- // exit(1);
- }
- return db;
-}
-
-int main(int argc, char** argv)
-{
- int i ;
- char buf[255];
- char *src = argv[1];
- char *dst = argv[2];
- printf("from %s to %s\n", src, dst);
- /*HStore *store = hs_open(dst, true, 2, 0, -1);
- if (!store){
- exit(1);
- }*/
- TCHDB **out = (TCHDB**) malloc(sizeof(TCHDB*) * 256);
- memset(out, 0, sizeof(TCHDB*) * 256);
- for (i=0; i<256; i++) {
- sprintf(buf, "%s/%02x.tch", dst, i);
- printf("open %s\n", buf);
- out[i] = my_open(buf, HDBOCREAT | HDBOREADER | HDBOWRITER| HDBONOLCK);
- if(!out[i]) {
- exit(1);
- }
- }
-
- for (i=63; i<64; i++) {
- sprintf(buf, "%s/%02x.tch", src, i);
- printf("start to scan %s\n", buf);
- TCHDB* db = my_open(buf, HDBOREADER);
- if(!db) continue;
- sprintf(buf, "%s/.%02x.tch", src, i);
- TCHDB* index = my_open(buf, HDBOREADER);
- if(!index) continue;
- tchdbiterinit(db);
- TCXSTR *xkey = tcxstrnew(), *xvalue = tcxstrnew();
- int c = 0;
- time_t now = time(0);
- while (tchdbiternext3(db, xkey, xvalue)){
- void *key = (void*)tcxstrptr(xkey);
- void *value = (void*)tcxstrptr(xvalue);
- int nkey = tcxstrsize(xkey);
- int nvalue = tcxstrsize(xvalue);
- int ver = 1, flag = 0, vlen;
-
- OldMeta *old = (OldMeta*)tchdbget2(index, key);
- if (old){
- ver = old->version;
- flag = old->flags;
- free(old);
- }else{
- printf("no index: %s\n", key);
- }
- //hs_set(store, key, value, nvalue, ver, flag);
- char *v = malloc(nvalue + sizeof(Meta));
- memcpy(v, value, nvalue);
- Meta *meta = (Meta*)(v + nvalue);
- meta->version = ver;
- meta->flag = flag;
- meta->hash = gen_hash(v, nvalue);
- meta->modified = now;
-
- unsigned int h = fnv1a(key, nkey);
- h = (h >> 24) & 0xff;
- if (out[h])
- tchdbput(out[h], key, nkey, v, nvalue + sizeof(Meta));
- free(v);
-
- tcxstrclear(xkey);
- tcxstrclear(xvalue);
-
- c ++;
- if (c % 10000 == 0){
- //hs_flush(store, 100);
- printf("%d\n", c);
- }
- }
- tcxstrdel(xkey);
- tcxstrdel(xvalue);
- tchdbclose(index);
- tchdbclose(db);
- }
- for (i=0; i<256; i++){
- if(out[i])
- tchdbclose(out[i]);
- }
- //hs_close(store);
-
- return 0;
-}
-
Please sign in to comment.
Something went wrong with that request. Please try again.