From 6425debb9fded24d7771d67278693ce0b7457724 Mon Sep 17 00:00:00 2001 From: Dmitry Yashunin Date: Sat, 28 Nov 2020 18:33:03 +0300 Subject: [PATCH 01/10] Move setup.py into root folder to fix bindings build when symlink doesn't work in windows, refactoring of bindings tests --- .gitignore | 9 +++++---- .travis.yml | 2 -- python_bindings/MANIFEST.in => MANIFEST.in | 0 python_bindings/Makefile => Makefile | 2 +- python_bindings/bindings.cpp | 2 +- python_bindings/hnswlib | 1 - python_bindings/tests/bindings_test.py | 11 +++++++---- python_bindings/tests/bindings_test_labels.py | 17 +++++++++++------ .../requirements.txt => requirements.txt | 0 python_bindings/setup.py => setup.py | 7 ++++--- 10 files changed, 29 insertions(+), 22 deletions(-) rename python_bindings/MANIFEST.in => MANIFEST.in (100%) rename python_bindings/Makefile => Makefile (56%) delete mode 120000 python_bindings/hnswlib rename python_bindings/requirements.txt => requirements.txt (100%) rename python_bindings/setup.py => setup.py (95%) diff --git a/.gitignore b/.gitignore index ff7acee9..ef2b9e50 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ -python_bindings/hnswlib.egg-info/ -python_bindings/build/ -python_bindings/dist/ -python_bindings/tmp/ +hnswlib.egg-info/ +build/ +dist/ +python_bindings/tests/__pycache__/ +*.pyd diff --git a/.travis.yml b/.travis.yml index 6b194926..587c4146 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,11 +6,9 @@ matrix: - python: 3.7 install: - | - cd python_bindings pip install -r requirements.txt python setup.py install script: - | - cd python_bindings python setup.py test diff --git a/python_bindings/MANIFEST.in b/MANIFEST.in similarity index 100% rename from python_bindings/MANIFEST.in rename to MANIFEST.in diff --git a/python_bindings/Makefile b/Makefile similarity index 56% rename from python_bindings/Makefile rename to Makefile index 02ec523b..1420b7c0 100644 --- a/python_bindings/Makefile +++ b/Makefile @@ -9,6 +9,6 @@ test: python3 setup.py test clean: - rm -rf *.egg-info build dist var first_half.bin tests/__pycache__ hnswlib.cpython-36m-darwin.so + rm -rf *.egg-info build dist var tests/__pycache__ hnswlib.cpython*.so .PHONY: dist \ No newline at end of file diff --git a/python_bindings/bindings.cpp b/python_bindings/bindings.cpp index bbfa9f8a..7875ec6e 100644 --- a/python_bindings/bindings.cpp +++ b/python_bindings/bindings.cpp @@ -2,7 +2,7 @@ #include #include #include -#include "hnswlib/hnswlib.h" +#include "hnswlib.h" #include #include #include diff --git a/python_bindings/hnswlib b/python_bindings/hnswlib deleted file mode 120000 index 236d6575..00000000 --- a/python_bindings/hnswlib +++ /dev/null @@ -1 +0,0 @@ -../hnswlib \ No newline at end of file diff --git a/python_bindings/tests/bindings_test.py b/python_bindings/tests/bindings_test.py index afc663af..009b2164 100644 --- a/python_bindings/tests/bindings_test.py +++ b/python_bindings/tests/bindings_test.py @@ -1,3 +1,4 @@ +import os import unittest @@ -43,16 +44,16 @@ def testRandomSelf(self): self.assertAlmostEqual(np.mean(labels.reshape(-1) == np.arange(len(data1))),1.0,3) # Serializing and deleting the index: - index_path='first_half.bin' + index_path = 'first_half.bin' print("Saving index to '%s'" % index_path) - p.save_index("first_half.bin") + p.save_index(index_path) del p # Reiniting, loading the index p = hnswlib.Index(space='l2', dim=dim) # you can change the sa - print("\nLoading index from 'first_half.bin'\n") - p.load_index("first_half.bin") + print("\nLoading index from '%s'\n" % index_path) + p.load_index(index_path) print("Adding the second batch of %d elements" % (len(data2))) p.add_items(data2) @@ -61,6 +62,8 @@ def testRandomSelf(self): labels, distances = p.knn_query(data, k=1) self.assertAlmostEqual(np.mean(labels.reshape(-1) == np.arange(len(data))),1.0,3) + + os.remove(index_path) if __name__ == "__main__": diff --git a/python_bindings/tests/bindings_test_labels.py b/python_bindings/tests/bindings_test_labels.py index c1887bef..e44b0988 100644 --- a/python_bindings/tests/bindings_test_labels.py +++ b/python_bindings/tests/bindings_test_labels.py @@ -1,3 +1,4 @@ +import os import unittest @@ -56,9 +57,9 @@ def testRandomSelf(self): # Serializing and deleting the index. # We need the part to check that serialization is working properly. - index_path='first_half.bin' + index_path = 'first_half.bin' print("Saving index to '%s'" % index_path) - p.save_index("first_half.bin") + p.save_index(index_path) print("Saved. Deleting...") del p print("Deleted") @@ -68,8 +69,8 @@ def testRandomSelf(self): print("Reiniting") p = hnswlib.Index(space='l2', dim=dim) - print("\nLoading index from 'first_half.bin'\n") - p.load_index("first_half.bin") + print("\nLoading index from '%s'\n" % index_path) + p.load_index(index_path) p.set_ef(100) print("Adding the second batch of %d elements" % (len(data2))) @@ -109,9 +110,10 @@ def testRandomSelf(self): print("All the data in data1 are removed") # checking saving/loading index with elements marked as deleted - p.save_index("with_deleted.bin") + del_index_path = "with_deleted.bin" + p.save_index(del_index_path) p = hnswlib.Index(space='l2', dim=dim) - p.load_index("with_deleted.bin") + p.load_index(del_index_path) p.set_ef(100) labels1_after, _ = p.knn_query(data1, k=1) @@ -119,6 +121,9 @@ def testRandomSelf(self): for lb in labels1: if la[0] == lb[0]: self.assertTrue(False) + + os.remove(index_path) + os.remove(del_index_path) diff --git a/python_bindings/requirements.txt b/requirements.txt similarity index 100% rename from python_bindings/requirements.txt rename to requirements.txt diff --git a/python_bindings/setup.py b/setup.py similarity index 95% rename from python_bindings/setup.py rename to setup.py index a6dfb81b..28a644f6 100644 --- a/python_bindings/setup.py +++ b/setup.py @@ -7,7 +7,8 @@ __version__ = '0.4.0' -source_files = ['bindings.cpp'] +source_files = ['./python_bindings/bindings.cpp'] +include_dirs = ['./hnswlib/'] libraries = [] extra_objects = [] @@ -17,7 +18,7 @@ Extension( 'hnswlib', source_files, - # include_dirs=[os.path.join(libdir, "include")], + include_dirs=include_dirs, libraries=libraries, language='c++', extra_objects=extra_objects, @@ -112,6 +113,6 @@ def build_extensions(self): ext_modules=ext_modules, install_requires=['pybind11>=2.0', 'numpy'], cmdclass={'build_ext': BuildExt}, - test_suite="tests", + test_suite="python_bindings.tests", zip_safe=False, ) From 376c8cdccff182db17a9c8a5d8254af4a5525220 Mon Sep 17 00:00:00 2001 From: Dmitry Yashunin Date: Sat, 28 Nov 2020 18:59:50 +0300 Subject: [PATCH 02/10] Update gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index ef2b9e50..c4045e98 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ hnswlib.egg-info/ build/ dist/ +tmp/ python_bindings/tests/__pycache__/ *.pyd +hnswlib.cpython*.so From 68a8a368e164fce34932dbdf9f5d192b66502e6b Mon Sep 17 00:00:00 2001 From: Dmitry Yashunin Date: Sat, 28 Nov 2020 19:09:30 +0300 Subject: [PATCH 03/10] Update Makefile to clean tmp folder --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1420b7c0..792b246e 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,6 @@ test: python3 setup.py test clean: - rm -rf *.egg-info build dist var tests/__pycache__ hnswlib.cpython*.so + rm -rf *.egg-info build dist tmp var tests/__pycache__ hnswlib.cpython*.so .PHONY: dist \ No newline at end of file From 19abf9b0d3a207d5e698bb5f6ea5ee781fe16fc8 Mon Sep 17 00:00:00 2001 From: Dmitry Yashunin Date: Sat, 28 Nov 2020 19:27:43 +0300 Subject: [PATCH 04/10] Update readme --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index d8d0eb76..aab5690d 100644 --- a/README.md +++ b/README.md @@ -214,7 +214,6 @@ You can install from sources: ```bash apt-get install -y python-setuptools python-pip pip3 install pybind11 numpy setuptools -cd python_bindings python3 setup.py install ``` From 5b2585d76221be0e41217698eaa9e3cc04a21d66 Mon Sep 17 00:00:00 2001 From: Dmitry Yashunin Date: Tue, 1 Dec 2020 16:15:02 +0300 Subject: [PATCH 05/10] Revert symlink to hnswlib and add windows to build matrix --- .travis.yml | 12 ++++++++---- python_bindings/hnswlib | 1 + 2 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 python_bindings/hnswlib diff --git a/.travis.yml b/.travis.yml index 587c4146..01ee109e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,13 @@ language: python -matrix: - include: - - python: 3.6 - - python: 3.7 +os: + - linux + - windows + +python: + - '3.6' + - '3.7' + install: - | pip install -r requirements.txt diff --git a/python_bindings/hnswlib b/python_bindings/hnswlib new file mode 100644 index 00000000..236d6575 --- /dev/null +++ b/python_bindings/hnswlib @@ -0,0 +1 @@ +../hnswlib \ No newline at end of file From dda9b31230f759dd6a674f41e85a0e3186cbacee Mon Sep 17 00:00:00 2001 From: Dmitry Yashunin Date: Tue, 1 Dec 2020 16:53:50 +0300 Subject: [PATCH 06/10] Fix symlink --- python_bindings/hnswlib | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 120000 python_bindings/hnswlib diff --git a/python_bindings/hnswlib b/python_bindings/hnswlib deleted file mode 100644 index 236d6575..00000000 --- a/python_bindings/hnswlib +++ /dev/null @@ -1 +0,0 @@ -../hnswlib \ No newline at end of file diff --git a/python_bindings/hnswlib b/python_bindings/hnswlib new file mode 120000 index 00000000..236d6575 --- /dev/null +++ b/python_bindings/hnswlib @@ -0,0 +1 @@ +../hnswlib \ No newline at end of file From b1994a5a14453c9d1fc4ee9b1a36429f28478581 Mon Sep 17 00:00:00 2001 From: Dmitry Yashunin Date: Tue, 1 Dec 2020 17:10:19 +0300 Subject: [PATCH 07/10] Update travis --- .travis.yml | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 01ee109e..f484b517 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,12 +1,30 @@ language: python -os: - - linux - - windows - -python: - - '3.6' - - '3.7' +jobs: + include: + - name: Linux Python 3.6 + os: linux + python: 3.6 + + - name: Linux Python 3.7 + os: linux + python: 3.7 + + - name: Windows Python 3.6 + os: windows + language: shell # 'language: python' is an error on Travis CI Windows + before_install: + - choco install python --version 3.6.0 + - python -m pip install --upgrade pip + env: PATH=/c/Python36:/c/Python36/Scripts:$PATH + + - name: Windows Python 3.7 + os: windows + language: shell # 'language: python' is an error on Travis CI Windows + before_install: + - choco install python --version 3.7.0 + - python -m pip install --upgrade pip + env: PATH=/c/Python37:/c/Python37/Scripts:$PATH install: - | From afd18d2edde0b3176d5eacc8875f52359be1eb12 Mon Sep 17 00:00:00 2001 From: Dmitry Yashunin Date: Tue, 1 Dec 2020 17:34:13 +0300 Subject: [PATCH 08/10] Update travis --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index f484b517..893441e9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,7 @@ jobs: before_install: - choco install python --version 3.6.0 - python -m pip install --upgrade pip + - python --version env: PATH=/c/Python36:/c/Python36/Scripts:$PATH - name: Windows Python 3.7 @@ -24,6 +25,7 @@ jobs: before_install: - choco install python --version 3.7.0 - python -m pip install --upgrade pip + - python --version env: PATH=/c/Python37:/c/Python37/Scripts:$PATH install: From 6efa48c0003bf797253eb798e53a01aeebf0854f Mon Sep 17 00:00:00 2001 From: Dmitry Yashunin Date: Tue, 8 Dec 2020 21:39:27 +0300 Subject: [PATCH 09/10] Add symlink to setup.py instead of hnswlib --- python_bindings/hnswlib | 1 - python_bindings/setup.py | 1 + setup.py | 10 ++++++++-- 3 files changed, 9 insertions(+), 3 deletions(-) delete mode 120000 python_bindings/hnswlib create mode 120000 python_bindings/setup.py diff --git a/python_bindings/hnswlib b/python_bindings/hnswlib deleted file mode 120000 index 236d6575..00000000 --- a/python_bindings/hnswlib +++ /dev/null @@ -1 +0,0 @@ -../hnswlib \ No newline at end of file diff --git a/python_bindings/setup.py b/python_bindings/setup.py new file mode 120000 index 00000000..f8f80fc2 --- /dev/null +++ b/python_bindings/setup.py @@ -0,0 +1 @@ +../setup.py \ No newline at end of file diff --git a/setup.py b/setup.py index 28a644f6..002f3893 100644 --- a/setup.py +++ b/setup.py @@ -6,9 +6,15 @@ __version__ = '0.4.0' +# compatibility when run in python_bindings +bindings_dir = 'python_bindings' +if bindings_dir in os.path.basename(os.getcwd()): + source_files = ['./bindings.cpp'] + include_dirs = ['../hnswlib/'] +else: + source_files = ['./python_bindings/bindings.cpp'] + include_dirs = ['./hnswlib/'] -source_files = ['./python_bindings/bindings.cpp'] -include_dirs = ['./hnswlib/'] libraries = [] extra_objects = [] From 6ae02a525d477694c0bd03265ed5cabfa61ea690 Mon Sep 17 00:00:00 2001 From: Dmitry Yashunin Date: Wed, 6 Jan 2021 19:09:57 +0300 Subject: [PATCH 10/10] Run sift test from separate directory --- README.md | 6 ++++-- sift_1b.cpp | 6 +++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index aab5690d..8575a681 100644 --- a/README.md +++ b/README.md @@ -242,13 +242,15 @@ Contributions are highly welcome! Please make pull requests against the `develop` branch. ### 200M SIFT test reproduction -To download and extract the bigann dataset: +To download and extract the bigann dataset (from root directory): ```bash python3 download_bigann.py ``` To compile: ```bash -cmake . +mkdir build +cd build +cmake .. make all ``` diff --git a/sift_1b.cpp b/sift_1b.cpp index 273c9828..2739490c 100644 --- a/sift_1b.cpp +++ b/sift_1b.cpp @@ -242,11 +242,11 @@ void sift_test1B() { size_t vecdim = 128; char path_index[1024]; char path_gt[1024]; - char *path_q = "bigann/bigann_query.bvecs"; - char *path_data = "bigann/bigann_base.bvecs"; + char *path_q = "../bigann/bigann_query.bvecs"; + char *path_data = "../bigann/bigann_base.bvecs"; sprintf(path_index, "sift1b_%dm_ef_%d_M_%d.bin", subset_size_milllions, efConstruction, M); - sprintf(path_gt, "bigann/gnd/idx_%dM.ivecs", subset_size_milllions); + sprintf(path_gt, "../bigann/gnd/idx_%dM.ivecs", subset_size_milllions); unsigned char *massb = new unsigned char[vecdim];