fix the issue #152

VERSION

@@ -1 +1 @@
-2.1.4
+2.1.5

lib/NGT/Index.cpp

@@ -1229,6 +1229,12 @@ GraphAndTreeIndex::createIndex(size_t threadPoolSize, size_t sizeOfRepository)
 
       for (size_t i = 0; i < cnt; i++) {
 	CreateIndexJob &job = output[i];
+	if (GraphIndex::objectSpace->isNormalizedDistance()) {
+	  if (job.results->size() > 0) {
+	    auto *o = GraphIndex::getObjectRepository().get((*job.results)[0].id);
+	    (*job.results)[0].distance = GraphIndex::objectSpace->compareWithL1(*job.object, *o);
+	  }
+	}
 	if (((job.results->size() > 0) && ((*job.results)[0].distance != 0.0)) ||
 	    (job.results->size() == 0)) {
 #ifdef NGT_SHARED_MEMORY_ALLOCATOR

lib/NGT/Index.h

@@ -1537,12 +1537,52 @@ namespace NGT {
       ObjectDistances	seeds;
       seeds.push_back(ObjectDistance(id, 0.0));
       GraphIndex::search(so, seeds);
+      if (results.size() == 0) {
+	if (!GraphIndex::objectSpace->isNormalizedDistance()) {
 #ifdef NGT_SHARED_MEMORY_ALLOCATOR
-      GraphIndex::objectSpace->deleteObject(obj);
+	  GraphIndex::objectSpace->deleteObject(obj);
 #endif
-      if (results.size() == 0) {
-	NGTThrowException("Not found the specified id");
+	  std::stringstream msg;
+	  msg << "Not found the specified id. ID=" << id;
+	  NGTThrowException(msg);
+	}
+	so.radius = FLT_MAX;
+	so.size = 10;
+	results.clear();
+	GraphIndex::search(so, seeds);
+	for (size_t i = 0; i < results.size(); i++) {
+	  try {
+	    auto *robj = GraphIndex::objectSpace->getRepository().get(results[i].id);
+	    results[i].distance = GraphIndex::objectSpace->compareWithL1(*obj, *robj);
+	  } catch (Exception &err) {
+#ifdef NGT_SHARED_MEMORY_ALLOCATOR
+	    GraphIndex::objectSpace->deleteObject(obj);
+#endif
+	    std::stringstream msg;
+	    msg << "remove: Fatal Inner Error! Cannot get an object. ID=" << id;
+	    NGTThrowException(msg);
+	  }
+	}
+	std::sort(results.begin(), results.end());
+	results.resize(2);
+	for (auto i = results.begin(); i != results.end(); ++i) {
+	  if ((*i).distance != 0.0) {
+	    results.resize(distance(results.begin(), i));
+	    break;
+	  }
+	}
+	if (results.size() == 0) {
+#ifdef NGT_SHARED_MEMORY_ALLOCATOR
+	  GraphIndex::objectSpace->deleteObject(obj);
+#endif
+	  std::stringstream msg;
+	  msg << "Not found the specified id. ID=" << id;
+	  NGTThrowException(msg);
+	}
       }
+#ifdef NGT_SHARED_MEMORY_ALLOCATOR
+      GraphIndex::objectSpace->deleteObject(obj);
+#endif
       if (results.size() == 1) {
 	try {
 	  DVPTree::remove(id);

lib/NGT/Node.cpp

@@ -242,7 +242,8 @@ LeafNode::removeObject(size_t id, size_t replaceId) {
 #else
       if (getObjectIDs()[idx].id == replaceId) {
 #endif
-	std::cerr << " Warning. found the same ID as the replaced ID." << std::endl;
+	std::cerr << " Warning. found the same ID as the replaced ID. " << id << ":" << replaceId << std::endl;
+	std::cerr << "          ignore it, if normalized distance." << std::endl;
 	replaceId = 0;
 	break;
       }

lib/NGT/ObjectSpace.cpp

@@ -0,0 +1,67 @@
+//
+// Copyright (C) 2015 Yahoo Japan Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#include	"NGT/defines.h"
+#include	"NGT/Common.h"
+#include	"NGT/ObjectSpace.h"
+#include	"NGT/ObjectRepository.h"
+
+NGT::Distance NGT::ObjectSpace::compareWithL1(NGT::Object &o1, NGT::Object &o2) {
+  auto dim = getPaddedDimension();
+  NGT::Distance d;
+  if (getObjectType() == typeid(uint8_t)) {
+    d = PrimitiveComparator::compareL1(reinterpret_cast<uint8_t*>(o1.getPointer()), 
+				       reinterpret_cast<uint8_t*>(o2.getPointer()), dim);
+#ifdef NGT_HALF_FLOAT
+  } else if (getObjectType() == typeid(float16)) {
+    d = PrimitiveComparator::compareL1(reinterpret_cast<float16*>(o1.getPointer()), 
+				       reinterpret_cast<float16*>(o2.getPointer()), dim);
+#endif
+  } else if (getObjectType() == typeid(float)) {
+    d = PrimitiveComparator::compareL1(reinterpret_cast<float*>(o1.getPointer()), 
+				       reinterpret_cast<float*>(o2.getPointer()), dim);
+  } else {
+    std::stringstream msg;
+    msg << "ObjectSpace::compareWithL1: Fatal Inner Error! Unexpected object type.";
+    NGTThrowException(msg);
+  }
+  return d;
+}
+
+#ifdef NGT_SHARED_MEMORY_ALLOCATOR
+NGT::Distance NGT::ObjectSpace::compareWithL1(NGT::Object &o1, NGT::PersistentObject &o2) {
+  auto dim = getPaddedDimension();
+  NGT::Distance d;
+  if (getObjectType() == typeid(uint8_t)) {
+    d = PrimitiveComparator::compareL1(reinterpret_cast<uint8_t*>(o1.getPointer()), 
+				       reinterpret_cast<uint8_t*>(o2.getPointer(getRepository().getAllocator())), dim);
+#ifdef NGT_HALF_FLOAT
+  } else if (getObjectType() == typeid(float16)) {
+    d = PrimitiveComparator::compareL1(reinterpret_cast<float16*>(o1.getPointer()), 
+				       reinterpret_cast<float16*>(o2.getPointer(getRepository().getAllocator())), dim);
+#endif
+  } else if (getObjectType() == typeid(float)) {
+    d = PrimitiveComparator::compareL1(reinterpret_cast<float*>(o1.getPointer()), 
+				       reinterpret_cast<float*>(o2.getPointer(getRepository().getAllocator())), dim);
+  } else {
+    std::stringstream msg;
+    msg << "ObjectSpace::compareWithL1: Fatal Inner Error! Unexpected object type.";
+    NGTThrowException(msg);
+  }
+  return d;
+}
+
+#endif

lib/NGT/ObjectSpace.h

@@ -296,6 +296,18 @@ namespace NGT {
       }
       return prefetchSize;
     }
+
+    bool isNormalizedDistance() {
+      return (getDistanceType() == ObjectSpace::DistanceTypeNormalizedAngle) ||
+	     (getDistanceType() == ObjectSpace::DistanceTypeNormalizedCosine) ||
+	     (getDistanceType() == ObjectSpace::DistanceTypeNormalizedL2);
+    }
+
+    NGT::Distance compareWithL1(NGT::Object &o1, NGT::Object &o2);
+#ifdef NGT_SHARED_MEMORY_ALLOCATOR
+    NGT::Distance compareWithL1(NGT::Object &o1, NGT::PersistentObject &o2);
+#endif
+
   protected:
     const size_t	dimension;
     DistanceType	distanceType;
@@ -509,6 +521,9 @@ namespace NGT {
       return a[idx];
     }
 
+    void *getPointer(SharedMemoryAllocator &allocator) {
+      return getPointer(0, allocator);
+    }
     void *getPointer(size_t idx, SharedMemoryAllocator &allocator) {
       uint8_t *a = (uint8_t *)allocator.getAddr(array);
       return a + idx;

lib/NGT/Tree.cpp

@@ -67,7 +67,11 @@ DVPTree::insert(InsertContainer &iobj,  LeafNode *leafNode)
 	ObjectID loid;
         try {
 	  loid = objects[i].id;
-	  idd = comparator(iobj.object, *getObjectRepository().get(loid));
+	  if (objectSpace->isNormalizedDistance()) {
+	    idd = objectSpace->compareWithL1(iobj.object, *getObjectRepository().get(loid));
+	  } else {
+	    idd = comparator(iobj.object, *getObjectRepository().get(loid));
+	  }
         } catch (Exception &e) {
           stringstream msg;
           msg << "LeafNode::insert: Cannot find object which belongs to a leaf node. id="