Permalink
Browse files

unibase with both key-value store and bigtable

  • Loading branch information...
haifengl committed Nov 27, 2017
1 parent 719cd7e commit c0bd92a56be5473e0ced303b69cff87b7078d8d7
@@ -28,9 +28,6 @@ trait BigTableDatabase[+T <: BigTable] extends AutoCloseable {
*/
def apply(name: String): T
/** Returns the list of tables. */
def tables: Set[String]
/** Creates a table.
* @param name the name of table.
* @param families the column families in the table. A column family name
@@ -64,6 +61,9 @@ trait BigTableDatabase[+T <: BigTable] extends AutoCloseable {
*/
def drop(name: String): Unit
/** Returns the list of tables. */
def tables: Set[String]
/** Tests if a table exists.
* @param name the name of table.
*/
@@ -22,11 +22,11 @@ import java.util.Properties
*
* @author Haifeng Li
*/
trait KeyValueStore extends AutoCloseable {
trait KeyValueStore[+T <: Keyspace] extends AutoCloseable {
/** Returns a keyspace/table.
* @param name the name of keyspace.
*/
def apply(name: String): Keyspace
def apply(name: String): T
/** Creates a keyspace/table.
* @param name the name of keyspace.
@@ -45,4 +45,12 @@ trait KeyValueStore extends AutoCloseable {
* @param name the name of keyspace.
*/
def drop(name: String): Unit
/** Returns the list of tables. */
def tables: Set[String]
/** Tests if a table exists.
* @param name the name of table.
*/
def exists(name: String): Boolean
}
@@ -31,7 +31,7 @@ import unicorn.kv.KeyValueStore
*
* @author Haifeng Li
*/
class RocksDB(val path: String) extends KeyValueStore {
class RocksDB(val path: String) extends KeyValueStore[Rockspace] {
val dir = new File(path)
require(dir.exists, s"Directory $path doesn't exist")
@@ -72,6 +72,14 @@ class RocksDB(val path: String) extends KeyValueStore {
})
}
override def tables: Set[String] = {
new File(path).list().toSet
}
override def exists(name: String): Boolean = {
new File(s"$path/$name").exists()
}
def compact(name: String): Unit = {
org.rocksdb.RocksDB.open(s"$path/$name").compactRange
}
@@ -18,6 +18,7 @@ package unicorn.unibase
import unicorn.bigtable.{BigTable, BigTableDatabase}
import unicorn.json._
import unicorn.kv.{Keyspace, KeyValue, KeyValueStore}
/** A Cabinet is a database of documents. A collection of documents can be
* stored as a Drawer or Table. Drawer uses a compact storage format but
@@ -34,15 +35,6 @@ trait Cabinet {
*/
def documents(name: String): Documents
/*
/** Returns a read only graph, which doesn't need an ID
* generator. This is sufficient for graph traversal and analytics.
*
* @param name The name of graph table.
*/
def graph(name: String): Graph
*/
/** Creates a document collection.
* @param name the name of document collection.
* @param key the document field(s) used as row key in BigTable.
@@ -51,13 +43,6 @@ trait Cabinet {
*/
def createDocuments(name: String, key: RowKey = PrimitiveRowKey(DefaultRowKeyField)): Unit
/*
/** Creates a graph table.
* @param name the name of graph table.
*/
def createGraph(name: String): Unit
*/
/** Drops a table. */
def drop(name: String): Unit
@@ -70,16 +55,99 @@ trait Cabinet {
def tables: Set[String]
}
object Cabinet {
def apply[T <: Keyspace](db: KeyValueStore[T]): Cabinet = {
new KeyValueCabinet[T](db)
}
def apply[T <: BigTable](db: BigTableDatabase[T]): Cabinet = {
new BigTableCabinet[T](db)
}
}
class KeyValueCabinet[+T <: Keyspace](db: KeyValueStore[T]) extends Cabinet {
private lazy val metaTable = {
if (!db.exists(MetaTableName)) {
db.create(MetaTableName)
}
new KeyValueDocuments(db(MetaTableName), RowKey("table"))
}
/** Returns a document collection.
* @param name The name of document collection.
*/
override def documents(name: String): Documents = {
val meta = metaTable(name)
if (meta.isEmpty)
throw new IllegalArgumentException(s"$name metadata doesn't exist")
if (meta.get.`type`.toString != TABLE_TYPE_DOCUMENTS)
throw new IllegalArgumentException(s"$name is not a drawer")
val rowkey = meta.map(TableMeta.rowkey(_)).get
new KeyValueDocuments(db(name), rowkey)
}
/** Creates a document collection.
* @param name the name of document collection.
* @param key the document field(s) used as row key in BigTable.
* If not specified, the "_id" field is used as the
* document key as in MongoDB.
*/
override def createDocuments(name: String, key: RowKey = PrimitiveRowKey(DefaultRowKeyField)): Unit = {
db.create(name)
val meta = TableMeta(name, TABLE_TYPE_DOCUMENTS, key)
metaTable.upsert(meta)
}
/*
/** Creates a graph table.
* @param name the name of graph table.
*/
def createGraph(name: String): Unit = {
val vertexKeyTable = graphVertexKeyTable(name)
require(!db.tableExists(vertexKeyTable), s"Vertex key table $vertexKeyTable already exists")
val table = db.createTable(name,
GraphVertexColumnFamily,
GraphInEdgeColumnFamily,
GraphOutEdgeColumnFamily)
table.close
val keyTable = db.createTable(vertexKeyTable, GraphVertexColumnFamily)
keyTable.close
}
*/
/** Drops a table. All column families in the table will be dropped. */
override def drop(name: String): Unit = {
db.drop(name)
metaTable.delete(name)
}
/** Tests if a BigTable exists.
* @param name the name of table.
*/
override def exists(name: String): Boolean = {
db.exists(name)
}
/** Returns the list of BigTables. */
override def tables: Set[String] = {
db.tables
}
}
class BigTableCabinet[+T <: BigTable](db: BigTableDatabase[T]) extends Cabinet {
private lazy val metaTable = {
if (!db.exists(MetaTableName)) {
db.create(MetaTableName, DocumentColumnFamily)
val metaTable = db(MetaTableName)
metaTable.close
}
new Table(db(MetaTableName), RowKey("table"))
new BigTableDocuments(db(MetaTableName), RowKey("table"))
}
/** Returns a document collection.
@@ -90,7 +158,7 @@ class BigTableCabinet[+T <: BigTable](db: BigTableDatabase[T]) extends Cabinet {
if (meta.isEmpty)
throw new IllegalArgumentException(s"$name metadata doesn't exist")
if (meta.get.`type`.toString != TABLE_TYPE_DRAWER)
if (meta.get.`type`.toString != TABLE_TYPE_DOCUMENTS)
throw new IllegalArgumentException(s"$name is not a drawer")
val rowkey = meta.map(TableMeta.rowkey(_)).get
@@ -130,7 +198,7 @@ class BigTableCabinet[+T <: BigTable](db: BigTableDatabase[T]) extends Cabinet {
override def createDocuments(name: String, key: RowKey = PrimitiveRowKey(DefaultRowKeyField)): Unit = {
db.create(name, DocumentColumnFamily)
val meta = TableMeta(name, TABLE_TYPE_DRAWER, key)
val meta = TableMeta(name, TABLE_TYPE_DOCUMENTS, key)
metaTable.upsert(meta)
}
@@ -198,12 +266,6 @@ class BigTableCabinet[+T <: BigTable](db: BigTableDatabase[T]) extends Cabinet {
}
}
object Cabinet {
def apply[T <: BigTable](db: BigTableDatabase[T]): Cabinet = {
new BigTableCabinet[T](db)
}
}
private[unicorn] object TableMeta {
/** Creates JsObject of table meta data.
*
@@ -23,7 +23,7 @@ import unicorn.json._
*
* @author Haifeng Li
*/
trait FindOps extends ScanOps {
trait FindOps {
val index: Seq[Index]
/** Searches the table.
@@ -27,23 +27,22 @@ import unicorn.kv._
trait KeyspaceScanOps {
val table: OrderedKeyspace
val rowkey: RowKey
val serializer = new JsonSerializer()
/** Scan the whole table. */
def scan: Iterator[JsObject] = {
scan(table.scan)
def scan: Iterator[KeyValue] = {
table.scan
}
/** Scan the the rows whose key starts with the given prefix. */
def scan(prefix: Key): Iterator[JsObject] = {
scan(table.scan(rowkey(prefix)))
def scan(prefix: Key): Iterator[KeyValue] = {
table.scan(rowkey(prefix))
}
/** Scan the the rows in the given range.
* @param start row to start scanner at or after (inclusive)
* @param end row to stop scanner before (exclusive)
*/
def scan(start: Key, end: Key): Iterator[JsObject] = {
def scan(start: Key, end: Key): Iterator[KeyValue] = {
val startKey = rowkey(start)
val endKey = rowkey(end)
@@ -52,13 +51,9 @@ trait KeyspaceScanOps {
throw new IllegalArgumentException("Start and end keys are the same")
if (c < 0)
scan(table.scan(startKey, endKey))
table.scan(startKey, endKey)
else
scan(table.scan(endKey, startKey))
}
private def scan(it: Iterator[KeyValue]): Iterator[JsObject] = {
it.map { kv => serializer.deserialize(kv.value).asInstanceOf[JsObject] }
table.scan(endKey, startKey)
}
}
@@ -71,20 +66,20 @@ trait BigTableScanOps {
val rowkey: RowKey
/** Scan the whole table. */
def scan(fields: Seq[String]): Iterator[JsObject] = {
scan(table.scan(DocumentColumnFamily, fields))
def scan(fields: Seq[String]): RowIterator = {
table.scan(DocumentColumnFamily, fields)
}
/** Scan the the rows whose key starts with the given prefix. */
def scan(prefix: Key, fields: String*): Iterator[JsObject] = {
scan(table.scanPrefix(rowkey(prefix), DocumentColumnFamily, fields))
def scan(prefix: Key, fields: String*): RowIterator = {
table.scanPrefix(rowkey(prefix), DocumentColumnFamily, fields)
}
/** Scan the the rows in the given range.
* @param start row to start scanner at or after (inclusive)
* @param end row to stop scanner before (exclusive)
*/
def scan(start: Key, end: Key, fields: String*): Iterator[JsObject] = {
def scan(start: Key, end: Key, fields: String*): RowIterator = {
val startKey = rowkey(start)
val endKey = rowkey(end)
@@ -93,18 +88,9 @@ trait BigTableScanOps {
throw new IllegalArgumentException("Start and end keys are the same")
if (c < 0)
scan(table.scan(startKey, endKey, DocumentColumnFamily, fields))
else
scan(table.scan(endKey, startKey, DocumentColumnFamily, fields))
}
private def scan(rows: RowIterator): Iterator[JsObject] = {
if (this.isInstanceOf[Table])
new TableIterator(rows)
else if (this.isInstanceOf[Documents])
new SimpleDocumentIterator(rows)
table.scan(startKey, endKey, DocumentColumnFamily, fields)
else
throw new IllegalStateException("Unsupported Scan table type: " + getClass)
table.scan(endKey, startKey, DocumentColumnFamily, fields)
}
}
Oops, something went wrong.

0 comments on commit c0bd92a

Please sign in to comment.