Merge pull request #9 from bauman/dan

[DDB-15] [DDB-16]

.github/workflows/build-and-test-u20.yml

@@ -18,7 +18,7 @@ jobs:
           sudo rm -rf /usr/include/discodb.h /usr/include/ddb*.h /usr/lib/libdiscodb.so
       - name: install c dependency
         run:
-          sudo apt-get install -y wget libcmph-dev cmake make build-essential;
+          sudo apt-get install -y wget libcmph-dev cmake make build-essential luajit;
           wget http://pkgs.bauman.space/discodb/sample.ddb -O /tmp/sample.ddb;
           wget http://pkgs.bauman.space/discodb/animals.ddb -O /tmp/animals.ddb;
           wget http://pkgs.bauman.space/discodb/cjk.ddb -O /tmp/cjk.ddb;
@@ -49,3 +49,7 @@ jobs:
           find . -name "*.so" | xargs -P1 -n1 ldd ;
           pip3 install --force-reinstall dist/discodb*.whl ;
           python3 util/test_merge.py ;
+      - name: verify lua library
+        run:
+          cd lua;
+          luajit discotest.lua ;

doc/capi.rst

@@ -5,3 +5,82 @@ capi -- C library to read/write DiscoDBs
 ====================================================================
 
 The C library is the native control application for reading and writing DiscoDBs
+
+
+Examples
+========
+
+Opening a discodb
+------------------
+Likely want to check a few things when opening a ddb file.
+
+
+.. code-block:: c
+
+    static struct ddb *open_discodb(const char *file)
+    {
+        struct ddb *db;
+        int fd;
+        if (!(db = ddb_new())){
+            fprintf(stderr, "Couldn't initialize discodb: Out of memory\n");
+            exit(1);
+        }
+        if ((fd = open(file, O_RDONLY)) == -1){
+            fprintf(stderr, "Couldn't open discodb %s\n", file);
+            exit(1);
+        }
+        if (ddb_load(db, fd)){
+            const char *err;
+            ddb_error(db, &err);
+            fprintf(stderr, "Invalid discodb in %s: %s\n", file, err);
+            exit(1);
+        }
+        return db;
+    }
+
+
+
+Adding key/values to a ddb
+--------------------------
+
+You'll need to create a :code:`ddb_entry` from memory available on the stack or heap.  During the :code:`ddb_cons_add` call, the entries will be duplicated into the discodb and the original stack/heap memory can be reused or freed.
+
+
+.. code-block:: c
+
+    uint64_t flags = 0;
+    flags |= getenv("DONT_COMPRESS") ? DDB_OPT_DISABLE_COMPRESSION: 0;
+    flags |= getenv("UNIQUE_ITEMS") ? DDB_OPT_UNIQUE_ITEMS: 0;
+    flags |= DDB_OPT_UNIQUE_ITEMS;
+
+
+    char key[MAX_KV_SIZE] = "Key\0";
+    char val[MAX_KV_SIZE] = "Value\0";
+    struct ddb_entry key_e = {.data = key, .length = strlen(key)};
+    struct ddb_entry val_e = {.data = val, .length = strlen(val)};
+
+
+    if (ddb_cons_add(db, &key_e, &val_e)){
+        fprintf(stderr, "Adding '%s':'%s' failed\n", key, val);
+        exit(1);
+    }
+
+
+    if (!(data = ddb_cons_finalize(db, &size, flags))){
+        fprintf(stderr, "Packing the index failed\n");
+        exit(1);
+    }
+
+    ddb_cons_free(db);
+
+    if (!(out = fopen("/path/to/ddb", "w"))){
+        fprintf(stderr, "Opening file %s failed\n", ddbfile);
+        exit(1);
+    }
+    if (!fwrite(data, size, 1, out)){
+        fprintf(stderr, "Writing file %s failed\n", ddbfile);
+        exit(1);
+    }
+    fflush(out);
+    fclose(out);
+    free(data);

doc/cnf.rst

@@ -1,7 +1,22 @@
 
 .. cnf:
 
-cnf -- Query Language
+cnf -- Conjunctive Normal Form (used by Query Language)
 ====================================================================
 
-Both the Python API and C API will interpret CNF based queries to access values
+Both the Python API and C API will interpret CNF based queries into a normalized form for queries.
+
+
+
+.. code-block:: python
+
+
+    >>> Q.parse('~(B | C)') == Q.parse('~B & ~C')
+    True
+    >>> Q.parse('(A & B) | C') == Q.parse('(A | C) & (B | C)')
+    True
+    >>> Q.parse('A & (B | (D & E))') == Q.parse('A & (B | D) & (B | E)')
+    True
+    >>> Q.parse(str(Q.parse('a | b | c & d | e'))) == Q.parse('a | b | c & d | e')
+    True
+

doc/index.rst

@@ -49,20 +49,26 @@ indexing system, `discodex`_.
 Install
 -------
 
-DiscoDB does not depend on Disco in any way, although it is a core
-component in `discodex`_. You can use it without Disco as a general-purpose
-scalable, immutable datastructure for Python. To install only DiscoDB without
-rest of Disco, clone Disco, and run::
+.. code-block:: bash
 
-    make install-discodb
+    mkdir cmake-release && cd cmake-release;
+    cmake -DCMAKE_BUILD_TYPE=Release ..;
+    make;
+    make test;
+    sudo install libdiscodb.so /usr/lib/libdiscodb.so ;
+    sudo install ../src/*.h /usr/include/. ;
 
-or if you just want to build it locally::
+to install the python wrapper
 
-    cd contrib/discodb
+.. code-block:: bash
+
+    cd python
     python setup.py build
 
+
 DiscoDB requires `CMPH library v0.9 or newer
-<http://cmph.sourceforge.net/>`_ (``libcmph-dev`` in Debian).
+<http://cmph.sourceforge.net/>`_ (``libcmph-dev`` in Debian) (``cmph-dev`` for headers and ``libcmph0`` for libs on ubuntu) (``cmph-dev`` for headers ``libcmph`` for libs on alpine).
+
 
 Example
 -------
@@ -71,17 +77,17 @@ Here is a simple example that builds a simple discodb and queries it::
 
     from discodb import DiscoDB, Q
 
-    data = {'mammals': ['cow', 'dog', 'cat', 'whale'],
-            'pets': ['dog', 'cat', 'goldfish'],
-            'aquatic': ['goldfish', 'whale']}
+    data = {b'mammals': [b'cow', b'dog', b'cat', b'whale'],
+            b'pets': [b'dog', b'cat', b'goldfish'],
+            b'aquatic': [b'goldfish', b'whale']}
 
     db = DiscoDB(data) # create an immutable discodb object
 
-    print list(db.keys()) # => mammals, aquatic, pets
-    print list(db['pets']) # => dog, cat, goldfish
-    print list(db.query(Q.parse('mammals & aquatic'))) # => whale
-    print list(db.query(Q.parse('pets & ~aquatic'))) # => dog, cat
-    print list(db.query(Q.parse('pets | aquatic'))) # => dog, cat, whale, goldfish
+    print(list(db.keys())) # => mammals, aquatic, pets
+    print(list(db[b'pets'])) # => dog, cat, goldfish
+    print(list(db.query(Q.parse('mammals & aquatic')))) # => whale
+    print(list(db.query(Q.parse('pets & ~aquatic')))) # => dog, cat
+    print(list(db.query(Q.parse('pets | aquatic')))) # => dog, cat, whale, goldfish
 
     db.dump(file('animals.db', 'w')) # dump discodb to a file
 
@@ -146,11 +152,13 @@ See Also
    :caption: Contents:
 
    index
-   structure
    pythonapi
+   capi
+   luaapi
    cnf
    query
-   capi
+   structure
+
 
 
 .. _conjunctive normal form: http://en.wikipedia.org/wiki/Conjunctive_normal_form

doc/luaapi.rst

@@ -0,0 +1,82 @@
+
+.. luaapi:
+
+luaapi -- Lua library to read/write DiscoDBs
+====================================================================
+
+The lua API links against libdiscodb at runtime and provides a thin wrapper to the functionality.
+
+This api is currently **read only**
+
+The following examples demonstrate a prebuilt ddb obtained using
+
+.. code-block:: bash
+
+    wget http://pkgs.bauman.space/discodb/myths.ddb -O /tmp/myths.ddb`
+
+
+
+
+Examples
+========
+
+Basic discodb wrapper for Lua.
+
+libdiscodb.so should be in :code:`LD_LIBRARY_PATH`
+
+Open and list keys
+------------------
+
+.. code-block:: lua
+
+    local ffi = require("ffi")
+    local ddblib = ffi.load("libdiscodb.so", true)  -- Load lib into global
+    local discodb = require("discodb")
+    local db = discodb.open("/tmp/myths.ddb")
+    local iter = db:keys()
+    for i = 1,#iter do
+        print(i, iter:next())
+    end
+
+
+Open and get individual key
+---------------------------
+
+.. code-block:: lua
+
+    local ffi = require("ffi")
+    local ddblib = ffi.load("libdiscodb.so", true)  -- Load lib into global
+    local discodb = require("discodb")
+    local db = discodb.open("/tmp/myths.ddb")
+    local iter = db:get("Major")
+    for i = 1,#iter do
+        print(i, iter:next())
+    end
+
+
+
+Check Key Exists Before Accessing
+---------------------------------
+Depends on the control flow of choice.
+
+Either handle the error of attempting to access :code:`next()` on a nil iter
+or check iter is not nil before calling :code:`next()` on it.
+
+.. code-block:: lua
+
+    local ffi = require("ffi")
+    local ddblib = ffi.load("libdiscodb.so", true)  -- Load lib into global
+    local discodb = require("discodb")
+    local db = discodb.open("/tmp/myths.ddb")
+    local iter = db:get("Major")
+    iter = db:get("key-should-not-exist")
+    if iter then
+        for i = 1,#iter do
+            print(i, iter:next())
+        end
+    else
+        print("key does not exist")
+    end
+
+
+