Merge pull request #442 from spotify/388-unit-test

Fix #388 issue – Windows failure with large indexes

.travis.yml

@@ -14,7 +14,7 @@ matrix:
     - os: osx
       language: generic
       env:
-        - TOXENV=py36
+        - TOXENV=py37
 
 env:
   matrix:

src/annoylib.h

@@ -31,18 +31,24 @@
 typedef unsigned char     uint8_t;
 typedef signed __int32    int32_t;
 typedef unsigned __int64  uint64_t;
+typedef signed __int64    int64_t;
 #else
 #include <stdint.h>
 #endif
 
+
 #if defined(_MSC_VER) || defined(__MINGW32__)
+ // a bit hacky, but override some definitions to support 64 bit
+ #define off_t int64_t
+ #define lseek_getsize(fd) _lseeki64(fd, 0, SEEK_END)
  #ifndef NOMINMAX
   #define NOMINMAX
  #endif
  #include "mman.h"
  #include <windows.h>
 #else
  #include <sys/mman.h>
+ #define lseek_getsize(fd) lseek(fd, 0, SEEK_END)
 #endif
 
 #include <cerrno>
@@ -983,8 +989,6 @@ template<typename S, typename T, typename Distance, typename Random>
       // Delete file if it already exists (See issue #335)
       unlink(filename);
 
-      printf("path: %s\n", filename);
-
       FILE *f = fopen(filename, "wb");
       if (f == NULL) {
         showUpdate("Unable to open: %s\n", strerror(errno));
@@ -1046,7 +1050,7 @@ template<typename S, typename T, typename Distance, typename Random>
       _fd = 0;
       return false;
     }
-    off_t size = lseek(_fd, 0, SEEK_END);
+    off_t size = lseek_getsize(_fd);
     if (size == -1) {
       showUpdate("lseek returned -1\n");
       if (error) *error = strerror(errno);

test/index_test.py

@@ -223,3 +223,25 @@ def test_build_twice(self):
         t.build(10)
         # Used to segfault:
         self.assertRaises(Exception, t.build, 10)
+
+    def test_very_large_index(self):
+        # 388
+        f = 3
+        dangerous_size = 2**31
+        size_per_vector = 4*(f+3)
+        n_vectors = int(dangerous_size / size_per_vector)
+        m = AnnoyIndex(3, 'angular')
+        m.verbose(True)
+        for i in range(100):
+            m.add_item(n_vectors+i, [random.gauss(0, 1) for z in range(f)])
+        n_trees = 10
+        m.build(n_trees)
+        path = 'test_big.annoy'
+        m.save(path)  # Raises on Windows
+
+        # Sanity check size of index
+        self.assertGreaterEqual(os.path.getsize(path), dangerous_size)
+        self.assertLess(os.path.getsize(path), dangerous_size + 100e3)
+
+        # Sanity check number of trees
+        self.assertEquals(m.get_n_trees(), n_trees)