Replace khmer.Nodegraph with rust nodegraph (#799)

* replace khmer.Nodegraph with rust nodegraph
* add update_mg
* prefer file cmp instead of raw_tables
* doesn't depend on named temporary files anymore
* fix n_occupied and tablesizes calculations
* fix occupied_bins
* use Update trait
* avoid copy
* brings it closer to khmer nodegraph API
* build all the wheels
* fix redis and py27
* fix travis config warnings
* bump core crate version
* gate travis jobs on Python 3.8
* fix multiple khmer versions tests, and add CI for them
* test expected collisions
* load compressed nodegraphs

.github/workflows/hypothesis.yml

@@ -24,4 +24,11 @@ jobs:
 
       - name: Run Hypothesis tests
         run: |
-          python -m pytest --run-hypothesis --hypothesis-show-statistics --hypothesis-profile ci
+          python -m pytest --cov=. --cov-report=xml --run-hypothesis --hypothesis-show-statistics --hypothesis-profile ci
+
+      - name: Upload coverage to Codecov  
+        uses: codecov/codecov-action@v1
+        with:
+            token: ${{ secrets.CODECOV_TOKEN }}
+            file: ./coverage.xml
+            fail_ci_if_error: true

.github/workflows/khmer.yml

@@ -0,0 +1,40 @@
+name: khmer compatibility tests
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: "3.7"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e .[test]
+
+      - name: Run tests with latest released khmer
+        run: |
+          python -m pip install khmer
+          python -m pytest -k test_nodegraph --cov=. --cov-report=xml
+
+      - name: Run tests with khmer master tests
+        run: |
+          python -m pip install -U git+https://github.com/dib-lab/khmer.git#egg=khmer
+          python -m pytest -k test_nodegraph --cov=. --cov-report=xml
+
+      - name: Upload coverage to Codecov  
+        uses: codecov/codecov-action@v1
+        with:
+            token: ${{ secrets.CODECOV_TOKEN }}
+            file: ./coverage.xml
+            fail_ci_if_error: true

.github/workflows/rust.yml

@@ -39,15 +39,19 @@ jobs:
           - build: macos
             os: macos-latest
             rust: stable
+            continue: false
           - build: windows
             os: windows-latest
             rust: stable
+            continue: true
           - build: beta
             os: ubuntu-latest
             rust: beta
+            continue: false
           - build: stable
             os: ubuntu-latest
             rust: stable
+            continue: false
     steps:
       - uses: actions/checkout@v1
 
@@ -57,13 +61,12 @@ jobs:
           override: true
 
       - name: Set up Python 3.8
-        if: matrix.os != 'windows-latest'
         uses: actions/setup-python@v1
         with:
           python-version: "3.8"
 
       - name: Install dependencies
-        if: matrix.os != 'windows-latest'
+        continue-on-error: ${{ matrix.continue }}
         run: |
           python -m pip install --upgrade pip
           python -m pip install -e .

.travis.yml

@@ -25,54 +25,53 @@ jobs:
   include:
     - &check
       stage: check # do a pre-screen to make sure this is even worth testing
-      python: 3.7
+      python: 3.8
       env:
         - PYTHONDEVMODE=1
 
     - &test
       stage: test
-      python: 3.6
+      python: 3.7
     - <<: *test
       os: osx
       osx_image: xcode10.1
-      python: 3.7
       language: generic
       env:
         - TOXENV=py37
     - <<: *test
-      python: 3.7
       name: integration (ipfs/redis)
       before_install:
         - sudo snap install ipfs
         - "/snap/bin/ipfs --version"
         - "/snap/bin/ipfs daemon --init --offline &>/dev/null &"
       services:
-        - redis-server
+        - redis
         - docker
     - <<: *test
-      python: 2.7
+      python: 3.6
     - <<: *test
       python: 3.5
+    - <<: *test
+      python: 2.7
 
     - &wheel
       stage: build wheel and send to github releases
       python: 3.7
       services:
         - docker
       env:
-        - CIBW_BUILD='cp37-*'
-        - CIBW_SKIP='*-manylinux_i686'
+        - CIBW_BUILD: "'cp37-*'"
+        - CIBW_SKIP: "'*-win32 *-manylinux_i686'"
         - CIBW_BEFORE_BUILD='source .travis/install_cargo.sh'
         - CIBW_ENVIRONMENT='PATH="$HOME/.cargo/bin:$PATH"'
         - CIBW_ENVIRONMENT_MACOS='MACOSX_DEPLOYMENT_TARGET=10.11'
-      before_script: skip
+      before_script:
+        - python3 -m pip install cibuildwheel==1.3.0
       script:
-        - python -m pip install -U pip setuptools
-        - python -m pip install cibuildwheel==1.1.0
-        - python -m cibuildwheel --output-dir wheelhouse
+        - python3 -m cibuildwheel --output-dir wheelhouse
       deploy:
         provider: releases
-        api_key:
+        token:
           secure: "FZmx00gL0m0uNVN8fpvqbUZSI20EEk4sgrEv4wpGFr8SFNwPb/VuizQRTapeF2AW9qzbfbUv4bR/+oSKu5jxvd+7+p9HlOZT285a5yfxay9OA8YUbpQz4a9J0CETOpxrA2wRRWBDqzjEWzLTGUquOhyBZn4cwujvA9syOdB7OSNKsB6ARc+zhqPeoMKc9JcjzwiEIh7tcCHoZ1epN5zyfsjqFEY8XlylHTU/WSI3SS9KIGuBqWf6/haoYD7BH+f2g56GMHvBEmGjk5cf/lrCpqiho7Z3fGgJcYf3fRWM3qSVUM9JyHUIqPS6oPZIp7zP40TztiK4Oeen4EpcS/KdoSdu5CjfgMYgF/qdlL28ntgnVYhcA/6IDSRg6V4H3b20qjsn3NlaNVdsSNCu3GzkvM/dtvp5I/41XcFCNqMOljMr3tRw2ZVo44/1vYW0FB0b0FjhjNcnMyTOQjSRR78zetdoorewTJdEmxthMieHIw1Mqzwg4e6JvsC30sh1uSaFq4FHrj2Fe6lAlgwPs1e9vx4w0Zs0kxxusNAdbBB8FH8paFuUEGjgT0Jo9KHOLtvRy+OU2OacQePuhG1kGBRvJz7O4/prAFLGWIW8tyZDnMCuv1vmk6YZd6ih3omkL178X94pjzavtlUip/ugF0NIwS5o+zucRow9txAX2jXgDCM="
         file_glob: true
         file: wheelhouse/sourmash*.whl
@@ -83,6 +82,17 @@ jobs:
       os: osx
       osx_image: xcode10.1
       language: shell
+    - <<: *wheel
+      arch: arm64
+    - <<: *wheel
+      arch: ppc64le
+    #- <<: *wheel
+    #  os: windows
+    #  language: shell
+    #  before_install:
+    #    - choco install python --version 3.8.0
+    #    - export PATH="/c/Python38:/c/Python38/Scripts:$PATH"
+    #    - ln -s /c/Python38/python.exe /c/Python38/python3.exe
 
 stages:
   - check

include/sourmash.h

@@ -182,6 +182,8 @@ bool kmerminhash_track_abundance(KmerMinHash *ptr);
 
 bool nodegraph_count(Nodegraph *ptr, uint64_t h);
 
+bool nodegraph_count_kmer(Nodegraph *ptr, const char *kmer);
+
 double nodegraph_expected_collisions(Nodegraph *ptr);
 
 void nodegraph_free(Nodegraph *ptr);
@@ -192,6 +194,8 @@ Nodegraph *nodegraph_from_path(const char *filename);
 
 uintptr_t nodegraph_get(Nodegraph *ptr, uint64_t h);
 
+uintptr_t nodegraph_get_kmer(Nodegraph *ptr, const char *kmer);
+
 uintptr_t nodegraph_ksize(Nodegraph *ptr);
 
 uintptr_t nodegraph_matches(Nodegraph *ptr, KmerMinHash *mh_ptr);
@@ -204,10 +208,16 @@ uintptr_t nodegraph_ntables(Nodegraph *ptr);
 
 void nodegraph_save(Nodegraph *ptr, const char *filename);
 
-uintptr_t nodegraph_tablesize(Nodegraph *ptr);
+uint8_t *nodegraph_to_buffer(Nodegraph *ptr, uintptr_t *size);
+
+void nodegraph_buffer_free(uint8_t *ptr, uintptr_t insize);
+
+uint64_t *nodegraph_hashsizes(Nodegraph *ptr, uintptr_t *size);
 
 void nodegraph_update(Nodegraph *ptr, Nodegraph *optr);
 
+void nodegraph_update_mh(Nodegraph *ptr, KmerMinHash *optr);
+
 Nodegraph *nodegraph_with_tables(uintptr_t ksize, uintptr_t starting_size, uintptr_t n_tables);
 
 void signature_add_sequence(Signature *ptr, const char *sequence, bool force);

setup.py

@@ -62,7 +62,7 @@ def build_native(spec):
         'sourmash = sourmash.__main__:main'
         ]
     },
-    "install_requires": ["screed>=0.9", "khmer>=2.1", "cffi>=1.14.0", 'numpy',
+    "install_requires": ["screed>=0.9", "cffi>=1.14.0", 'numpy',
                          'matplotlib', 'scipy', "deprecation>=2.0.6"],
     "setup_requires": [
         "setuptools>=38.6.0",
@@ -82,7 +82,7 @@ def build_native(spec):
         'doc' : ['sphinx', 'recommonmark', 'alabaster',
                  "sphinxcontrib-napoleon", "nbsphinx",
                  "ipython"],
-        '10x': ['bam2fasta==1.0.1'],
+        '10x': ['bam2fasta==1.0.4'],
         'storage': ["ipfshttpclient>=0.4.13", "redis"]
     },
     "include_package_data": True,

sourmash/cli/info.py

@@ -14,14 +14,13 @@ def subparser(subparsers):
 
 
 def info(verbose=False):
+    "Report sourmash version + version of installed dependencies."
     notify('sourmash version {}', sourmash.VERSION)
     notify('- loaded from path: {}', os.path.dirname(__file__))
     notify('')
 
     if verbose:
-        import khmer
-        notify('khmer version {}', khmer.__version__)
-        notify('- loaded from path: {}', os.path.dirname(khmer.__file__))
+        notify('khmer version: None (internal Nodegraph)')
         notify('')
 
         notify('screed version {}', screed.__version__)

sourmash/commands.py

@@ -24,23 +24,6 @@
 from .command_compute import compute
 
 
-def info(args):
-    "Report sourmash version + version of installed dependencies."
-    from . import VERSION
-    notify('sourmash version {}', VERSION)
-    notify('- loaded from path: {}', os.path.dirname(__file__))
-    notify('')
-
-    if args.verbose:
-        import khmer
-        notify('khmer version {}', khmer.__version__)
-        notify('- loaded from path: {}', os.path.dirname(khmer.__file__))
-        notify('')
-
-        notify('screed version {}', screed.__version__)
-        notify('- loaded from path: {}', os.path.dirname(screed.__file__))
-
-
 def compare(args):
     "Compare multiple signature files and create a distance matrix."
     import numpy