finished serialization backend, optimized search to only sort the top…

… N hits instead of the full list

.gitignore

@@ -13,4 +13,22 @@ webapp/imageSearch.css
 webapp/imageSearch.css~
 webapp/imageSearch.wt
 webapp/images/
+*/*~
+*~
+doc/hackerTalk/hackerTalk.aux
+doc/hackerTalk/hackerTalk.log
+doc/hackerTalk/hackerTalk.nav
+doc/hackerTalk/hackerTalk.out
+doc/hackerTalk/hackerTalk.pdf
+doc/hackerTalk/hackerTalk.snm
+doc/hackerTalk/hackerTalk.toc
+doc/hackerTalk/hackerTalkPrintVersion.pdf
+Debug/
+doc/hackerTalk/auto/
+importer/broken.jpg
+importer/dbBasedImporter
+importer/serializationBasedImporter
+webapp/imagesearch.log
+webapp/imagesearch.valgrind
+webapp/massif.out.19596
 

backend/BinaryScoreTableSerializer.cc

@@ -0,0 +1,44 @@
+#include "BinaryScoreTableSerializer.h"
+#include "PreSelectScoreTable.h"
+
+#include <boost/archive/binary_oarchive.hpp>
+#include <boost/archive/binary_iarchive.hpp>
+#include <boost/serialization/export.hpp>
+#include <boost/serialization/vector.hpp>
+
+#include <stdexcept>
+
+using namespace ImageSearch;
+
+BOOST_CLASS_EXPORT_GUID (PreSelectScoreTable, "PreSelectScoreTable")
+
+BinaryScoreTableSerializer::BinaryScoreTableSerializer (void)
+{
+}
+
+BinaryScoreTableSerializer::~BinaryScoreTableSerializer (void)
+{
+}
+
+void
+BinaryScoreTableSerializer::doSave (const ScoreTable *scoreTable, const std::string &fileName)
+{
+  std::ofstream ofs (fileName.c_str (), std::ios::out | std::ios::binary);
+  boost::archive::binary_oarchive oa (ofs);
+  oa << scoreTable;
+}
+
+ScoreTable *
+BinaryScoreTableSerializer::doLoad (const std::string &fileName)
+{
+  ScoreTable *result;
+  std::ifstream ifs (fileName.c_str (), std::ios::in | std::ios::binary);
+  if (!ifs)
+    {
+      throw std::ios::failure ("error opening file " + fileName);
+    }
+  boost::archive::binary_iarchive ia (ifs);
+  ia >> result;
+  return result;
+}
+

backend/BinaryScoreTableSerializer.h

@@ -0,0 +1,19 @@
+#ifndef IMAGE_SEARCH_SCORE_TABLE_BINARY_SERIALIZER_H
+#define IMAGE_SEARCH_SCORE_TABLE_BINARY_SERIALIZER_H
+
+#include "ScoreTableSerializer.h"
+
+namespace ImageSearch
+{
+
+  class BinaryScoreTableSerializer : public ScoreTableSerializer {
+  public:
+    BinaryScoreTableSerializer (void);
+    virtual ~BinaryScoreTableSerializer (void);
+    virtual void doSave (const ScoreTable *scoreTable, const std::string &fileName);
+    virtual ScoreTable *doLoad (const std::string &fileName);
+  };
+
+};
+
+#endif // IMAGE_SEARCH_SCORE_TABLE_BINARY_SERIALIZER_H

backend/CMakeLists.txt

@@ -6,6 +6,8 @@ ADD_LIBRARY(Backend
 	DbBasedImageSearchBackend.cc
 	SerializationBasedImageSearchBackend.cc
 	ScoreTable.cc
+	BinaryScoreTableSerializer.cc
+	TextScoreTableSerializer.cc
 	PreSelectScoreTable.cc
 	postgresql/PostgresQl.cc
 	postgresql/PgSelect.cc

backend/DbBasedImageSearchBackend.cc

@@ -2,7 +2,7 @@
 #include "PreSelectScoreTable.h"
 #include "postgresql/PostgresQl.h"
 #include "macros.h"
-#include "config.h"
+#include "../config.h"
 
 #include <boost/thread/mutex.hpp>
 
@@ -21,7 +21,7 @@ DbBasedImageSearchBackend::DbBasedImageSearchBackend (const std::string &imageDb
 
   if (m_imageDbPrefix.size () > 0)
     {
-      initScoreTable ();
+      p_initScoreTable ();
     }
   else
     {
@@ -36,7 +36,7 @@ DbBasedImageSearchBackend::~DbBasedImageSearchBackend (void)
 }
 
 void
-DbBasedImageSearchBackend::initScoreTable (void)
+DbBasedImageSearchBackend::p_initScoreTable (void)
 {
   if (m_scoreTable == NULL)
     {
@@ -52,7 +52,6 @@ DbBasedImageSearchBackend::initScoreTable (void)
 
 	  std::cout << "loaded " << allImages.size ()
 		    << " images from the database." << std::endl;
-	  m_nDbImages = allImages.size ();
 	  m_scoreTable = new PreSelectScoreTable (getDbImageRows (),
 						  getDbImageCols (),
 						  m_nKeptCoeffs);
@@ -74,7 +73,7 @@ DbBasedImageSearchBackend::initScoreTable (void)
 std::string
 DbBasedImageSearchBackend::getImageNameById (const unsigned long id)
 {
-  std::auto_ptr<DBImage> dbImage (getDbImageById (id));
+  std::auto_ptr<DBImage> dbImage (p_getDbImageById (id));
   return dbImage->getFileName ();
 }
 
@@ -85,13 +84,13 @@ DbBasedImageSearchBackend::saveDbImage (const DBImage &image)
 }
 
 ImageFeaturesList
-DbBasedImageSearchBackend::getAllDbImages (void)
+DbBasedImageSearchBackend::p_getAllDbImages (void)
 {
   return m_database->findAll ();
 }
 
 std::auto_ptr<DBImage>
-DbBasedImageSearchBackend::getDbImageById (int id)
+DbBasedImageSearchBackend::p_getDbImageById (int id)
 {
   return m_database->getById (id);
 }

backend/DbBasedImageSearchBackend.h

@@ -25,9 +25,9 @@ namespace ImageSearch
     virtual std::string getImageNameById (const unsigned long id);
 
   private:
-    void initScoreTable (void);
-    std::auto_ptr<DBImage> getDbImageById (int id);
-    ImageFeaturesList getAllDbImages (void);
+    void p_initScoreTable (void);
+    std::auto_ptr<DBImage> p_getDbImageById (int id);
+    ImageFeaturesList p_getAllDbImages (void);
     Database *m_database;
   };
 

backend/ImageScore.h

@@ -29,6 +29,7 @@ namespace ImageSearch
 
   typedef std::vector<ImageScore> ImageScoreList;
   typedef ImageScoreList::iterator ImageScoreIterator;
+  typedef ImageScoreList::const_iterator ImageScoreConstIterator;
 
 };
 

backend/ImageSearchBackend.cc

@@ -1,7 +1,7 @@
 #include "ImageSearchBackend.h"
 #include "ImageScore.h"
 #include "macros.h"
-#include "config.h"
+#include "../config.h"
 
 #include <cxxutil/utils.h>
 
@@ -27,7 +27,6 @@
 using namespace ImageSearch;
 
 ScoreTable * ImageSearchBackend::m_scoreTable = NULL;
-unsigned long ImageSearchBackend::m_nDbImages = 0;
 
 
 static std::string guessMimeType (const std::string &fileName);
@@ -148,7 +147,7 @@ ImageSearchBackend::getScoreTableInfoText(void) const
 }
 
 BLImage
-ImageSearchBackend::makeBlImage (const int id, const std::string &fileName,
+ImageSearchBackend::p_makeBlImage (const int id, const std::string &fileName,
 				 const std::string &text)
 {
   std::string thumbNail = m_documentRoot + m_imageDbPrefix + "/thumb_"
@@ -165,21 +164,29 @@ ImageSearchBackend::performSearch (void)
   m_searchResults.clear ();
   if (isCurrentImageValid ())
     {
-      ImageScoreList result;
-      for (int i = 0; i < m_nDbImages; ++i)
+      ImageScoreList result, sorted;
+      for (int i = 0; i < m_scoreTable->nImages (); ++i)
 	{
 	  result.push_back (ImageScore (i));
 	}
-      assert (result.size () == m_nDbImages);
+      assert (result.size () == m_scoreTable->nImages ());
       std::auto_ptr<ColorImage> image (new ColorImage ());
       image->read (m_currentTempFile.c_str ());
+
       m_scoreTable->query (*image, result, false);
+
       boost::timer timer;
-      for (int i = 0; i < result.size () && i < m_maxResults; ++i)
+      p_sortScores (result, sorted);
+      int elapsed = (int)(timer.elapsed () * 1000);
+      std::cout << "sorting the query results took "
+		<< elapsed << " milliseconds." << std::endl;
+
+      timer.restart ();
+      for (int i = 0; i < sorted.size () && i < m_maxResults; ++i)
 	{
-	  m_searchResults.push_back (getBlImage (result[i]));
+	  m_searchResults.push_back (p_getBlImage (sorted[i]));
 	}
-      int elapsed = (int)(timer.elapsed () * 1000);
+      elapsed = (int)(timer.elapsed () * 1000);
       std::cout << "loading the images took "
 		<< elapsed << " milliseconds." << std::endl;
     }
@@ -188,12 +195,12 @@ ImageSearchBackend::performSearch (void)
 }
 
 BLImage
-ImageSearchBackend::getBlImage (const ImageScore &score)
+ImageSearchBackend::p_getBlImage (const ImageScore &score)
 {
   std::string fileName = getImageNameById (score.getId ());
   std::string text = "File: " + fileName
     + ", score: " + CxxUtil::dtoa (score.getScore ());
-  return makeBlImage (score.getId (), fileName, text);
+  return p_makeBlImage (score.getId (), fileName, text);
 }
 
 bool
@@ -314,6 +321,63 @@ ImageSearchBackend::createImageFeatures (const std::string &path, int rows, int
 
 }
 
+void
+ImageSearchBackend::p_sortScores (const ImageScoreList &scores, ImageScoreList &result)
+{
+  result.resize (m_maxResults);
+  ImageScoreConstIterator it = scores.begin ();
+  float worstScore = (*it).getScore ();
+  int worstPosition;
+  int lastPos = 0;
+
+  for (; it != scores.end (); ++it)
+    {
+
+      const ImageScore &score = *it;
+
+      if (lastPos < m_maxResults)
+	{
+
+	  int pos = lastPos;
+	  if (lastPos == 0 || score.getScore () > worstScore)
+	    {
+	      worstPosition = pos;
+	      worstScore = result[worstPosition].getScore ();
+	    }
+	  result[pos] = score;
+	  lastPos++;
+
+	}
+      else if (score.getScore () < worstScore)
+	{
+
+	  result[worstPosition] = score;
+	  for (int i = 0; i < m_maxResults; ++i)
+	    {
+	      if (result[i].getScore () > result[worstPosition].getScore ())
+		{
+		  worstPosition = i;
+		}
+	    }
+	  worstScore = result[worstPosition].getScore ();
+
+	  for (int i = 0; i < lastPos; ++i)
+	    {
+	      if (i != worstPosition && result[worstPosition].getScore () < result[i].getScore ())
+		{
+		  std::cerr << "Error, " << result[worstPosition].getScore ()
+			    << " is less than " << result[i].getScore () << " but it should not." << std::endl;
+		  assert (0);
+		}
+	    }
+
+	}
+
+    }
+
+  std::sort (result.begin (), result.end ());
+}
+
 static void
 fillFeatureVectors (const Image &img,
 		    Features &posFeatures, Features &negFeatures)

backend/ImageSearchBackend.h

@@ -37,20 +37,20 @@ namespace ImageSearch
     int getDbImageCols (void) const;
     std::auto_ptr<ReadOnlyImage> createImageFeatures (const std::string &path,
 						      int rows, int cols);
-    int getNImages (void) const { return m_nDbImages; }
+    int nImages (void) const { return m_scoreTable->nImages (); }
 
   protected:
     virtual std::string getImageNameById (const unsigned long id) = 0;
     static ScoreTable *m_scoreTable;
-    static unsigned long m_nDbImages;
     int m_nKeptCoeffs;
     std::string m_documentRoot;
     std::string m_imageDbPrefix;
 
   private:
-    BLImage makeBlImage (const int id, const std::string &fileName,
-			 const std::string &text);
-    BLImage getBlImage (const ImageScore &score);
+    BLImage p_makeBlImage (const int id, const std::string &fileName,
+			   const std::string &text);
+    BLImage p_getBlImage (const ImageScore &score);
+    void p_sortScores (const ImageScoreList &scores, ImageScoreList &result);
     std::string m_currentTempFile;
     int m_sizeY;
     int m_sizeX;