Dan (#7)

* superb   [DDB-12] [DDB-13] [DDB-14]

.github/workflows/build-and-test-u20.yml

@@ -0,0 +1,51 @@
+name: Bionic Build
+
+on:
+  push:
+    branches:
+    - $default-branch
+    - dan
+jobs:
+  build:
+    runs-on:
+      - disco-runner
+    steps:
+      - uses: actions/checkout@v2
+      - name: update system
+        run:
+          sudo apt-get update -y;
+          sudo apt-get upgrade -y;
+          sudo rm -rf /usr/include/discodb.h /usr/include/ddb*.h /usr/lib/libdiscodb.so
+      - name: install c dependency
+        run:
+          sudo apt-get install -y wget libcmph-dev cmake make build-essential;
+          wget http://pkgs.bauman.space/discodb/sample.ddb -O /tmp/sample.ddb;
+          wget http://pkgs.bauman.space/discodb/animals.ddb -O /tmp/animals.ddb;
+          wget http://pkgs.bauman.space/discodb/cjk.ddb -O /tmp/cjk.ddb;
+          wget http://pkgs.bauman.space/discodb/myths.ddb -O /tmp/myths.ddb;
+      - name: build debug library
+        run:
+          rm -rf cmake-build;
+          mkdir cmake-build && cd cmake-build;
+          cmake -DCMAKE_BUILD_TYPE=Debug ..;
+          make;
+          make test;
+      - name: build release library
+        run:
+          rm -rf cmake-release;
+          mkdir cmake-release && cd cmake-release;
+          cmake -DCMAKE_BUILD_TYPE=Release ..;
+          make;
+          make test;
+          sudo install libdiscodb.so /usr/lib/libdiscodb.so ;
+          sudo install ../src/*.h /usr/include/. ;
+      - name: install python dependency
+        run:
+          sudo apt-get install -y python3-dev python3-wheel python3-pip
+      - name: build python library
+        run:
+          cd python;
+          python3 setup.py bdist_wheel ;
+          find . -name "*.so" | xargs -P1 -n1 ldd ;
+          pip3 install --force-reinstall dist/discodb*.whl ;
+          python3 util/test_merge.py ;

.gitignore

@@ -14,3 +14,5 @@ cmake_install.cmake
 CmakeCache.txt
 libdiscodb.*
 *.whl
+*.egg-info/
+

CMakeLists.txt

@@ -1,6 +1,12 @@
 cmake_minimum_required(VERSION 3.3)
 project(libdiscodb)
 
+# unclear whether SuperFastHash in ddb_hash.h is fair use within LGPL
+# a less restrictive 32bit hash function is now default hash function
+# define LGPL_IS_OK to use the original LGPL'd hashing function
+#                   undefined uses the less restrictive implementation
+# add_definitions(-DLGPL_IS_OK)
+
 INCLUDE_DIRECTORIES(/usr/include/  src/ )
 LINK_DIRECTORIES(/usr/lib64/)
 set(SOURCE_FILES    src/ddb_delta.c
@@ -23,10 +29,33 @@ TARGET_LINK_LIBRARIES(discodb cmph)
 
 #gcc -Wall -O2  -shared -o libdiscodb.so -fPIC  *.c -lcmph
 
+
+
+add_executable(ddb_create src/util/create.c)
+target_link_libraries(ddb_create discodb )
+
+add_executable(ddb_query src/util/query.c)
+target_link_libraries(ddb_query discodb )
+
+add_executable(ddb_merge src/util/merge.c)
+target_link_libraries(ddb_merge discodb )
+
+add_executable(ddb_invert src/util/invert.c)
+target_link_libraries(ddb_invert discodb )
+
 enable_testing()
-function(ddb_add_test test_name)
-    add_executable(${test_name} ${ARGN})
-    target_link_libraries(${test_name} discodb )
-endfunction()
-ddb_add_test(test-create-and-query.o src/tests/create_and_query.c)
-#ddb_add_test(testmerge src/tests/test_merge.c)
+
+add_executable(tddb_from_kv src/tests/create_and_query.c)
+target_link_libraries(tddb_from_kv discodb )
+add_test(test2_ddb_from_kv tddb_from_kv disco.out ../src/tests/input.txt)
+
+
+add_executable(tddb_merge src/tests/create_and_merge.c)
+target_link_libraries(tddb_merge discodb )
+add_test(test3_ddb_from_ddbs tddb_merge disco_merged.out disco.out)
+
+
+add_test(test10_ddbs_from_merge ddb_merge /tmp/merged.ddb /tmp/animals.ddb /tmp/myths.ddb)
+add_test(test11_ddbs_from_merge_inverted ddb_invert /tmp/inverted_myths.ddb /tmp/myths.ddb)
+
+

README.md

@@ -1,4 +1,3 @@
-[![Build Status](https://travis-ci.org/bauman/discodb.svg?branch=py3)](https://travis-ci.org/bauman/discodb)
 
 discodb
 =======
@@ -13,3 +12,16 @@ converted to cmake
     
 ```
 
+python wrapper
+==============
+
+Default python extension links against libdiscodb
+```bash
+python3 setupy.py install
+```
+
+alternate python extension packages discodb but still requires a system installed libcmph 
+
+```bash
+python3 setup_self_contained.py install 
+```

doc/capi.rst

@@ -0,0 +1,7 @@
+
+.. capi:
+
+capi -- C library to read/write DiscoDBs
+====================================================================
+
+The C library is the native control application for reading and writing DiscoDBs

doc/cnf.rst

@@ -0,0 +1,7 @@
+
+.. cnf:
+
+cnf -- Query Language
+====================================================================
+
+Both the Python API and C API will interpret CNF based queries to access values

doc/conf.py

@@ -44,7 +44,7 @@
 
 # General substitutions.
 project = 'DiscoDB'
-copyright = '2008-2012, Nokia Corporation and the Disco Project'
+copyright = '2008-2012, Nokia Corporation and the Disco Project  2014-2021, Bauman with commits from goller'
 
 if on_rtd:
     MOCK_MODULES = ['discodb._discodb']

doc/index.rst

@@ -142,8 +142,16 @@ See Also
 --------
 
 .. toctree::
+   :maxdepth: 2
+   :caption: Contents:
 
+   index
+   structure
+   pythonapi
+   cnf
    query
+   capi
+
 
 .. _conjunctive normal form: http://en.wikipedia.org/wiki/Conjunctive_normal_form
 .. _immutable: http://en.wikipedia.org/wiki/Immutable_object

doc/pythonapi.rst

@@ -0,0 +1,18 @@
+
+.. pythonapi:
+
+pythonapi -- Python library to read/write DiscoDBs
+====================================================================
+
+By default the python API links against libdiscodb at runtime and provides a thin wrapper to the functionality.
+
+
+
+Examples
+========
+
+incrementally adding key/value pairs to a constructor::
+
+    from discodb import DiscoDBConstructor
+
+

doc/query.rst

@@ -1,2 +1,7 @@
-.. automodule:: discodb.query
-   :members:
+.. query:
+
+query -- Access methods to DiscoDB
+====================================================================
+
+Overview of access methods
+

doc/structure.rst

@@ -0,0 +1,7 @@
+
+.. structure:
+
+structure -- C library to read/write DiscoDBs
+====================================================================
+
+The C library is the native control application for reading and writing DiscoDBs

python/discodb/__init__.py

@@ -1,11 +1,53 @@
+"""
+DiscoDB python wrapper to libdiscodb's handling
+
+
+This module wraps the python c extension
+at discodb._discodb
+that does the heavy lifting for this module.
+
+discodb requires a system wide (or otherwise appropriately installed)
+    libcmph (licensed LGPL)
+
+
+"""
+
 import six
-from ._discodb import _DiscoDB, DiscoDBConstructor, DiscoDBError, DiscoDBIter, DiscoDBView
+try:
+    from ._discodb import _DiscoDB, DiscoDBConstructor, DiscoDBError, DiscoDBIter, DiscoDBView
+except ImportError as imp_err:
+    from sys import stderr
+    RED = '\033[0;31m'
+    NC = '\033[0m' # No Color
+    if "libcmph" in str(imp_err):
+        stderr.write("----------------------------------------------------\n"
+                     f"{RED}DiscoDB import Error{NC}:\n"
+                     f"       {RED}libcmph{NC} not found on system, \n"
+                     f"       please obtain system-wide libcmph\n"
+                     f"       to comply with LGPL License\n"
+                     "            apt-get install libcmph0\n"
+                     "            apk add libcmph\n"
+                     "            dnf/yum install cmph (third party)\n"
+                     "            brew install libcmph (third party)\n"
+                     "----------------------------------------------------\n")
+    elif "libdisco" in str(imp_err):
+        stderr.write("-----------------------------------------------------------\n"
+                     f"{RED}DiscoDB import Error{NC}:\n"
+                     f"       {RED}libdiscodb{NC} not found on system, \n"
+                     f"       please obtain system-wide libdiscodb and libcmph\n"
+                     f"       to comply with LGPL License\n"
+                     "-----------------------------------------------------------\n")
+    raise
+
+
 from .query import Q
 from .tools import kvgroup
 
+
 def discodb_unpickle(string):
     return DiscoDB.loads(string)
 
+
 class DiscoDBInquiry(object):
     def __init__(self, iterfunc):
         self.iterfunc = iterfunc
@@ -25,6 +67,7 @@ def __nonzero__(self):
 
     def __format__(self, format_spec='%s.3'):
         format_str, precision = format_spec.rsplit('.', 1)
+
         def firstN(N):
             for n, item in enumerate(self):
                 if n == N:
@@ -36,17 +79,20 @@ def firstN(N):
     def __str__(self):
         return '%s([%s])' % (self.__class__.__name__, self.__format__())
 
+
 class DiscoDBLazyInquiry(DiscoDBInquiry):
     def __len__(self):
         return iter(self).count()
 
+
 class DiscoDBItemInquiry(DiscoDBLazyInquiry):
     def __len__(self):
         return sum(1 for i in self)
 
     def __format__(self, format_spec='(%s, %s).3'):
         return super(DiscoDBItemInquiry, self).__format__(format_spec)
 
+
 class DiscoDB(_DiscoDB):
     """DiscoDB(iter[, flags]) -> new DiscoDB from k, v[s] in iter."""
     def __len__(self):
@@ -123,6 +169,7 @@ def peek(self, key, default=None):
     def make_view(self, data):
         return DiscoDBView(self, data)
 
+
 __all__ = ['DiscoDB',
            'DiscoDBConstructor',
            'DiscoDBError',

python/discodb/query.py

@@ -112,16 +112,18 @@ def urlformat(self, safe=':()/,~'):
         return quote(str(self)
                      .replace('&', '/')
                      .replace('|', ',')
-                     .replace(' ' ,''),
+                     .replace(' ', ''),
                      safe)
 
     @classmethod
     def urlscan(cls, string):
         from urllib import unquote
-        return cls.parse(unquote(string)\
-                         .strip().strip('/')\
-                         .replace('/', '&')\
-                         .replace(',', '|'))
+        return cls.parse(unquote(string)
+                         .strip()
+                         .strip('/')
+                         .replace('/', '&')
+                         .replace(',', '|')
+                         )
 
     @classmethod
     def parse(cls, string):
@@ -194,6 +196,7 @@ def wrap(cls, proposition):
             return Q((Clause((proposition, )), ))
         return Q((Clause((Literal(proposition), )), ))
 
+
 class Clause(object):
     """
     Contains a disjunction of literals.
@@ -245,6 +248,7 @@ def expand(self, discodb):
     def resolve(self, discodb):
         return reduce(__or__, (l.resolve(discodb) for l in self.literals))
 
+
 class Literal(object):
     """
     A potential key in a discodb (or its negation).
@@ -290,6 +294,7 @@ def expand(self, discodb):
     def resolve(self, discodb):
         return Q.wrap(self)
 
+
 class MetaLiteral(Literal):
     def __str__(self):
         return '%s+(%s)' % ('~' if self.negated else '', self.term.decode())
@@ -303,3 +308,4 @@ def resolve(self, discodb):
         q = self.term.resolve(discodb)
         clause = Clause(Literal(v) for v in discodb.query(q))
         return Q.wrap(~clause if self.negated else clause)
+