[MRG] Move HLLCounter into the glorious Cython future

include/khmer/_cpy_khmer.hh

@@ -87,7 +87,6 @@ Contact: khmer-project@idyll.org
 
 #include "_cpy_graphlabels.hh" 
 #include "_cpy_hashset.hh" 
-#include "_cpy_hllcounter.hh" 
 #include "_cpy_readaligner.hh"
 
 

khmer/__init__.py

@@ -48,7 +48,6 @@
 from khmer._khmer import GraphLabels as _GraphLabels
 from khmer._khmer import Nodegraph as _Nodegraph
 from khmer._khmer import Nodetable as _Nodetable
-from khmer._khmer import HLLCounter as _HLLCounter
 from khmer._khmer import ReadAligner as _ReadAligner
 
 from khmer._khmer import HashSet
@@ -396,28 +395,6 @@ def __new__(cls, k, starting_size, n_tables):
         return nodetable
 
 
-class HLLCounter(_HLLCounter):
-    """HyperLogLog counter.
-
-    A HyperLogLog counter is a probabilistic data structure specialized on
-    cardinality estimation.
-    There is a precision/memory consumption trade-off: error rate determines
-    how much memory is consumed.
-
-    # Creating a new HLLCounter:
-
-    >>> khmer.HLLCounter(error_rate, ksize)
-
-    where the default values are:
-      - error_rate: 0.01
-      - ksize: 20
-    """
-
-    def __len__(self):
-        """Return the cardinality estimate."""
-        return _HLLCounter.estimate_cardinality(self)
-
-
 class ReadAligner(_ReadAligner):
     """Sequence to graph aligner.
 
@@ -516,3 +493,5 @@ def __init__(self, *args, **kwargs):  # pylint: disable=unused-argument
 
 from khmer._oxli.assembly import (LinearAssembler, SimpleLabeledAssembler,
                                   JunctionCountAssembler)
+
+from khmer._oxli.hllcounter import HLLCounter

khmer/_oxli/hllcounter.pxd

@@ -1,38 +1,47 @@
 from libcpp cimport bool
 from libcpp.memory cimport unique_ptr
-from libcpp.sting cimport string
+from libcpp.string cimport string
 from libcpp.vector cimport vector
 from libc.stdint cimport uint64_t
 
 from oxli_types cimport *
 from parsing cimport CpReadParser
 
+
+cdef extern from "oxli_exception_convert.hh":
+    cdef void oxli_raise_py_error()
+
+
 cdef extern from "oxli/hllcounter.hh" namespace "oxli":
     cdef cppclass CpHLLCounter "oxli::HLLCounter":
-        CpHLLCounter(double, WordLength)
-        CpHLLCounter(int, WordLength)
+        CpHLLCounter(double, WordLength) except +oxli_raise_py_error
+        CpHLLCounter(int, WordLength) except +oxli_raise_py_error
 
         void add(const string &)
         unsigned int consume_string(const string &)
         void consume_seqfile[SeqIO](const string &,
                                     bool,
                                     unsigned int &,
-                                    unsigned long long &)
+                                    uint64_t &) except +oxli_raise_py_error
 
-        void consume_seqfile[SeqIO](unique_ptr[CpReadParser[SeqIO]]&,
-                                    bool,
-                                    unsigned int &,
-                                    unsigned long long &)
+#        void consume_seqfile[SeqIO](unique_ptr[CpReadParser[SeqIO]]&,
+#                                    bool,
+#                                    unsigned int &,
+#                                    unsigned long long &)
         unsigned int check_and_process_read(string &, bool &)
         bool check_and_normalize_read(string &) const
         uint64_t estimate_cardinality()
-        void merge(CpHLLCounter &)
+        void merge(CpHLLCounter &) except +oxli_raise_py_error
         double get_alpha()
         int get_p()
         int get_m()
-        void set_ksize(WordLegth)
+        void set_ksize(WordLength) except +oxli_raise_py_error
         int get_ksize()
         vector[int] get_M()
         double get_erate()
-        void set_erate(double)
+        void set_erate(double) except +oxli_raise_py_error
+
 
+cdef class HLLCounter:
+    cdef unique_ptr[CpHLLCounter] _this
+    cpdef tuple consume_seqfile(self, filename, bool stream_records=*)

khmer/_oxli/hllcounter.pyx

@@ -0,0 +1,103 @@
+# cython: c_string_type=unicode, c_string_encoding=utf8
+from cython.operator cimport dereference as deref, address
+
+from parsing cimport CpFastxReader
+from .utils cimport _bstring, _ustring
+
+cdef class HLLCounter:
+    """HyperLogLog counter.
+
+    A HyperLogLog counter is a probabilistic data structure specialized on
+    cardinality estimation.
+    There is a precision/memory consumption trade-off: error rate determines
+    how much memory is consumed.
+
+    # Creating a new HLLCounter:
+
+    >>> khmer.HLLCounter(error_rate, ksize)
+
+    where the default values are:
+      - error_rate: 0.01
+      - ksize: 20
+    """
+
+    def __cinit__(self, double error_rate=0.01, int ksize=20):
+        self._this.reset(new CpHLLCounter(error_rate, ksize))
+
+    def __len__(self):
+        """Return the cardinality estimate."""
+        return self.estimate_cardinality()
+
+    def add(self, kmer):
+        """Add a k-mer to the counter."""
+        deref(self._this).add(_bstring(kmer))
+
+    def estimate_cardinality(self):
+        """Return the current estimative."""
+        return deref(self._this).estimate_cardinality()
+
+    def consume_string(self, seq):
+        """Break a sequence into k-mers and add each k-mer to the counter."""
+        return deref(self._this).consume_string(_bstring(seq))
+
+    cpdef tuple consume_seqfile(self, filename, bool stream_records=False):
+        "Read sequences from file, break into k-mers, "
+        "and add each k-mer to the counter. If optional keyword 'stream_out' "
+        "is True, also prints each sequence to stdout."
+        cdef unsigned long long n_consumed = 0
+        cdef unsigned int total_reads = 0
+
+        deref(self._this).consume_seqfile[CpFastxReader](
+                             _bstring(filename), stream_records,
+                             total_reads, n_consumed)
+
+        return total_reads, n_consumed
+
+    def merge(self, HLLCounter other):
+        """Merge other counter into this one."""
+        deref(self._this).merge(deref(other._this))
+
+    @property
+    def alpha(self):
+        """alpha constant for this HLL counter."""
+        return deref(self._this).get_alpha()
+
+    @property
+    def error_rate(self):
+        "Error rate for this HLL counter."
+        "Can be changed prior to first counting, but becomes read-only after "
+        "that (raising AttributeError)"
+        return deref(self._this).get_erate()
+
+    @error_rate.setter
+    def error_rate(self, erate):
+        deref(self._this).set_erate(float(erate))
+
+    @error_rate.deleter
+    def error_rate(self):
+        raise TypeError("Cannot delete attribute")
+
+    @property
+    def ksize(self):
+        "k-mer size for this HLL counter."
+        "Can be changed prior to first counting, but becomes read-only after "
+        "that (raising AttributeError)"
+        return deref(self._this).get_ksize()
+
+    @ksize.setter
+    def ksize(self, object new_k):
+        if new_k <= 0:
+            raise ValueError("Please set k-mer size to a value greater "
+                             "than zero")
+        if isinstance(new_k, float):
+            raise TypeError("Please use an integer value for k-mer size")
+        deref(self._this).set_ksize(<int>new_k)
+
+    @ksize.deleter
+    def ksize(self):
+        raise TypeError("Cannot delete attribute")
+
+    @property
+    def counters(self):
+        """Read-only internal counters."""
+        return deref(self._this).get_M()

khmer/_oxli/oxli_exception_convert.cc

@@ -0,0 +1,22 @@
+#include "Python.h"
+#include <exception>
+#include <string>
+#include "oxli/oxli_exception.hh"
+#include "oxli_exception_convert.hh"
+
+
+void oxli_raise_py_error()
+{
+  try {
+    throw;
+  }
+  catch (oxli::ReadOnlyAttribute& e) {
+    PyErr_SetString(PyExc_AttributeError, e.what());
+  }
+  catch (oxli::InvalidValue& e) {
+    PyErr_SetString(PyExc_ValueError, e.what());
+  }
+  catch (oxli::InvalidStream& e) {
+    PyErr_SetString(PyExc_OSError, e.what());
+  }
+}

khmer/_oxli/oxli_exception_convert.hh

@@ -0,0 +1 @@
+void oxli_raise_py_error();

setup.py

@@ -154,7 +154,7 @@ def build_dir():
 BUILD_DEPENDS = glob.glob(path_join("include", "khmer", "_cpy_*.hh"))
 BUILD_DEPENDS.extend(path_join("include", "oxli", bn + ".hh") for bn in [
     "khmer", "kmer_hash", "hashtable", "labelhash", "hashgraph",
-    "hllcounter", "khmer_exception", "read_aligner", "subset", "read_parsers",
+    "hllcounter", "oxli_exception", "read_aligner", "subset", "read_parsers",
     "kmer_filters", "traversal", "assembler", "alphabets", "storage"])
 
 SOURCES = glob.glob(path_join("src", "khmer", "_cpy_*.cc"))
@@ -199,7 +199,7 @@ def build_dir():
 
     CY_EXTENSION_MOD_DICT = \
         {
-            "sources": [cython_ext],
+            "sources": [cython_ext, "khmer/_oxli/oxli_exception_convert.cc"],
             "extra_compile_args": EXTRA_COMPILE_ARGS,
             "extra_link_args": EXTRA_LINK_ARGS,
             "extra_objects": [path_join(build_dir(), splitext(p)[0] + '.o')