Merge pull request #45 from ashleyabraham/main

Updated build for Windows

.github/workflows/build.yml

@@ -10,7 +10,7 @@ jobs:
       matrix:
         # https://github.blog/2019-08-08-github-actions-now-supports-ci-cd/
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: [3.5, 3.6, 3.7, 3.8]
+        python-version: ['3.6', '3.7', '3.8', '3.9', '3.10']
         # https://stackoverflow.com/questions/57810623/how-to-select-the-c-c-compiler-used-for-a-github-actions-job:
         compiler: [gcc, clang, cl]
         exclude:
@@ -30,16 +30,18 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v2
       with:
         python-version: ${{ matrix.python-version }}
+        architecture: x64
     - name: Print machine info
       run: |
         uname -a
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
         pip install -r requirements.txt
+    - uses: ilammy/msvc-dev-cmd@v1
     - name: Test with CC=${{ matrix.compiler }}
       env:
         CC: ${{ matrix.compiler }}

.gitignore

@@ -100,4 +100,5 @@ ENV/
 .spyderproject
 
 # Rope project settings
-.ropeproject
+.ropeproject
+/.vs

README.md

@@ -26,7 +26,7 @@ The search can be operated for a subset of a database. | Rii remains fast even a
 
 
 ## Installing
-You can install the package via pip. This library works with Python 3.5+ on linux/mac/wsl/Windows10 (x64, using MSVC:flags - /arch:AVX2, /openmp:llvm, /fp:fast').
+You can install the package via pip. This library works with Python 3.6+ on linux/mac/wsl/Windows10
 
 ```
 pip install rii
@@ -36,17 +36,29 @@ pip install rii
 <details>
   <summary>For windows (maintained by @ashleyabraham)</summary>
 
-  ### Pre-compiled binary for Windows 10 (, may need MS Visual Studio Build tools)
+  ### Installing in Windows 10 via `pip install`
+  Requires MS Visual Studio Build tools C++ 14.0 or 14.1 toolset or above to compile and install via pip install
+
+  ### Pre-compiled binary for Windows 10 
+  Pre-compiled binaries doesn't require MS Visual Studio Build tools
+
   ```
-  pip install https://github.com/ashleyabraham/rii/releases/download/v0.2.7/rii-0.2.7-cp38-cp38-win_amd64.whl
+  #Python 3.8
+  pip install https://github.com/ashleyabraham/rii/releases/download/v0.2.8/rii-0.2.8-cp38-cp38-win_amd64.whl
   ```
+  ```
+  #Python 3.7
+  pip install https://github.com/ashleyabraham/rii/releases/download/v0.2.8/rii-0.2.8-cp37-cp37m-win_amd64.whl
+```
 
-  ### OpenMP
+  #### OpenMP
+  OpenMP requires libomp140_x86_64.dll to compile in windows, which is part of MS Visual Studio Build tools and it is not redistributable.
+
   In order to use OpenMP 3.0 /openmp:llvm flag is used which causes warnings of multiple libs loading, use at your descretion when used with other parallel processing library loadings. To supress use
 
   `SET KMP_DUPLICATE_LIB_OK=TRUE`
 
-  ### SIMD
+  #### SIMD
   The /arch:AVX2 flag is used in MSVC to set appropriate SIMD preprocessors and compiler intrinsics
 
 </details>
@@ -101,7 +113,7 @@ e = rii.Rii(fine_quantizer=nanopq.PQ(M=32).fit(vecs=Xt)).add_configure(vecs=X)
 
 ```python
 # The search can be conducted over a subset of the database
-target_ids = np.array([85, 132, 236, 551, 694, 728, 992, 1234])  # Specified by IDs
+target_ids = np.array([85, 132, 236, 551, 694, 728, 992, 1234], dtype=np.int64)  # Specified by IDs
 ids, dists = e.query(q=q, topk=3, target_ids=target_ids)
 print(ids, dists)  # e.g., [728  85 132] [14.80522156 15.92787838 16.28690338]
 ```
@@ -148,8 +160,8 @@ e.verbose = False
 # You can merge two Rii instances if they have the same fine_quantizer
 e1 = rii.Rii(fine_quantizer=codec)
 e2 = rii.Rii(fine_quantizer=codec)
-e1.add_reconfigure(vecs=X1)
-e2.add_reconfigure(vecs=X2)
+e1.add_configure(vecs=X1)
+e2.add_configure(vecs=X2)
 e1.merge(e2)  # Now e1 contains both X1 and X2
 
 ```

docs/requirements.txt

@@ -1,2 +1,2 @@
-pybind11>=2.2
+pybind11>=2.9
 nanopq

requirements.txt

@@ -1,3 +1,3 @@
-pybind11>=2.3
+pybind11>=2.9
 nanopq
 

setup.py

@@ -102,8 +102,10 @@ def build_extensions(self):
             if has_flag(self.compiler, '-fvisibility=hidden'):
                 opts.append('-fvisibility=hidden')
         elif ct == 'msvc':
-            opts.append('/DVERSION_INFO=\\"%s\\"' % self.distribution.get_version())
-            opts.append('/openmp:llvm') # support for openmp 3.0 or above
+            opts.append('/std:c++17')
+            opts.append('/permissive-')
+            opts.append('/DVERSION_INFO="%s"' % self.distribution.get_version())
+            opts.append('/openmp') 
             opts.append('/fp:fast') # -Ofast
 
         if sys.platform not in ['darwin', 'win32']:
@@ -116,10 +118,9 @@ def build_extensions(self):
             opts.append('-mtune=native')  # Do optimization (It seems this doesn't boost, but just in case)
             opts.append('-Ofast')  # This makes the program faster
 
-
         for ext in self.extensions:
             ext.extra_compile_args = opts
-            if not sys.platform == 'darwin':
+            if sys.platform not in ['darwin', 'win32']:
                 ext.extra_link_args = ['-fopenmp']  # Because of PQk-means
 
         build_ext.build_extensions(self)
@@ -136,8 +137,9 @@ def build_extensions(self):
     license='MIT',
     packages=find_packages(),
     install_requires=requirements,
-    setup_requires=['pybind11>=2.3'],
+    setup_requires=['pybind11>=2.9'],
     ext_modules=ext_modules,
     cmdclass={'build_ext': BuildExt},
-    zip_safe=False
+    zip_safe=False,
+    python_requires=">=3.6",
 )

src/main.cpp

@@ -2,6 +2,9 @@
 #include <pybind11/stl.h>  // To convert list of list <-> vec<vec<int>> for e.g. posting_lists
 #include "rii.h"
 
+#define STRINGIFY(x) #x
+#define MACRO_STRINGIFY(x) STRINGIFY(x)
+
 namespace py = pybind11;
 
 namespace rii {
@@ -51,9 +54,9 @@ PYBIND11_MODULE(main, m) {
         ));
 
 #ifdef VERSION_INFO
-    m.attr("__version__") = VERSION_INFO;
+    m.attr("__version__") = MACRO_STRINGIFY(VERSION_INFO);
 #else
     m.attr("__version__") = "dev";
 #endif
 }
-} // namespace rii
+} // namespace rii

src/pqkmeans.cpp

@@ -208,7 +208,7 @@ std::pair<std::size_t, float> PQKMeans::FindNearetCenterLinear(const std::vector
     int min_i = -1;
     for (size_t i = 0, sz = codes.size(); i < sz; ++i) {
         if (dists[i] < min_dist) {
-            min_i = i;
+            min_i = static_cast<int>(i);
             min_dist = dists[i];
         }
     }

src/rii.h

@@ -6,6 +6,11 @@
 #include "./pqkmeans.h"
 #include "./distance.h"
 
+// Handle missing ssize_t on Windows. 
+# if defined(_MSC_VER) 
+    typedef __int64 ssize_t;
+# endif
+
 // For py::array_t
 // See http://pybind11.readthedocs.io/en/master/advanced/pycpp/numpy.html#direct-access
 #include <pybind11/pybind11.h>
@@ -206,15 +211,17 @@ std::pair<std::vector<size_t>, std::vector<float> > RiiCpp::QueryLinear(const py
         size_t N = GetN();
         scores.resize(N);
 #pragma omp parallel for
-        for (size_t n = 0; n < N; ++n) {
+        for (long long n_tmp = 0LL; n_tmp < static_cast<long long>(N); ++n_tmp) {
+            size_t n = static_cast<size_t>(n_tmp);
             scores[n] = {n, ADist(dtable, flattened_codes_, n)};
         }
     } else {  // Target ids are specified
         assert((size_t) topk <= S);
         assert(S <= GetN());
         scores.resize(S);
 #pragma omp parallel for
-        for (size_t s = 0; s < S; ++s) {
+        for (long long s_tmp = 0LL; s_tmp < static_cast<long long>(S); ++s_tmp) {
+            size_t s = static_cast<size_t>(s_tmp);
             size_t tid = static_cast<size_t>(tids(s));
             scores[s] = {tid, ADist(dtable, flattened_codes_, tid)};
         }
@@ -335,18 +342,19 @@ void RiiCpp::UpdatePostingLists(size_t start, size_t num)
 
 
     // ===== (1) Construct a dummy pqkmeans class for computing Symmetric Distance =====
-    pqkmeans::PQKMeans clustering_instance(codewords_, GetNumList(), 0, true);
+    pqkmeans::PQKMeans clustering_instance(codewords_, (int)GetNumList(), 0, true);
     clustering_instance.SetClusterCenters(coarse_centers_);
 
     // ===== (2) Update posting lists =====
     std::vector<size_t> assign(num);
 #pragma omp parallel for
-    for (size_t n = 0; n < num; ++n) {
+    for (long long n_tmp = 0LL; n_tmp < static_cast<long long>(num); ++n_tmp) {
+        size_t n = static_cast<size_t>(n_tmp);
         assign[n] = clustering_instance.predict_one(NthCode(flattened_codes_, start + n));
     }
 
     for (size_t n = 0; n < num; ++n) {
-        posting_lists_[assign[n]].push_back(start + n);
+        posting_lists_[assign[n]].push_back((int)(start + n));
     }
 }