textindex/_xapian.dll textindex/mp3tag.py textindex/testxapian.py textindex/xapian.py @@ -4,7 +4,7 @@ /* File created by MIDL compiler version 6.00.0361 */ -/* at Sat Apr 01 15:31:23 2006 +/* at Sun Jul 16 11:58:43 2006 */ /* Compiler settings for _tstlib.idl: Oicf, W1, Zp8, env=Win32 (32b run) com/tstlib/_tstlib.h @@ -1,10 +1,10 @@ -import "E:\Program Files\Microsoft Visual Studio .NET 2003\Vc7\PlatformSDK\include\prsht.idl"; -import "E:\Program Files\Microsoft Visual Studio .NET 2003\Vc7\PlatformSDK\include\mshtml.idl"; -import "e:\program files\microsoft visual studio .net 2003\vc7\platformsdk\include\dimm.idl"; -import "E:\Program Files\Microsoft Visual Studio .NET 2003\Vc7\PlatformSDK\include\mshtmhst.idl"; -import "e:\program files\microsoft visual studio .net 2003\vc7\platformsdk\include\docobj.idl"; -import "E:\Program Files\Microsoft Visual Studio .NET 2003\Vc7\PlatformSDK\include\exdisp.idl"; -import "E:\Program Files\Microsoft Visual Studio .NET 2003\Vc7\PlatformSDK\include\objsafe.idl"; +import "C:\Program Files\Microsoft Visual Studio .NET 2003\Vc7\PlatformSDK\include\prsht.idl"; +import "C:\Program Files\Microsoft Visual Studio .NET 2003\Vc7\PlatformSDK\include\mshtml.idl"; +import "c:\program files\microsoft visual studio .net 2003\vc7\platformsdk\include\dimm.idl"; +import "C:\Program Files\Microsoft Visual Studio .NET 2003\Vc7\PlatformSDK\include\mshtmhst.idl"; +import "c:\program files\microsoft visual studio .net 2003\vc7\platformsdk\include\docobj.idl"; +import "C:\Program Files\Microsoft Visual Studio .NET 2003\Vc7\PlatformSDK\include\exdisp.idl"; +import "C:\Program Files\Microsoft Visual Studio .NET 2003\Vc7\PlatformSDK\include\objsafe.idl"; [ object, @@ -13,9 +13,9 @@ import "E:\Program Files\Microsoft Visual Studio .NET 2003\Vc7\PlatformSDK\inclu helpstring("Interface IDocumentsScore"), pointer_default(unique) ] -#line 35 "c:\\projets\\development\\tst\\com\\tstlib\\documentsscore.h" +#line 35 "d:\\projets\\tst\\com\\tstlib\\documentsscore.h" interface IDocumentsScore : IDispatch { -#line 37 "c:\\projets\\development\\tst\\com\\tstlib\\documentsscore.h" +#line 37 "d:\\projets\\tst\\com\\tstlib\\documentsscore.h" [propget,id(1),helpstring("property Size")] HRESULT Size([out,retval] LONG *pVal); [id(2),helpstring("method GetDocument")] HRESULT GetDocument([in] LONG index, [out,retval] BSTR *document ); [id(3),helpstring("method GetScore")] HRESULT GetScore([in] LONG index, [out,retval] LONG *score); @@ -28,9 +28,9 @@ interface IDocumentsScore : IDispatch { helpstring("Interface ITextIndex"), pointer_default(unique) ] -#line 18 "c:\\projets\\development\\tst\\com\\tstlib\\textindex.h" +#line 18 "d:\\projets\\tst\\com\\tstlib\\textindex.h" interface ITextIndex : IDispatch { -#line 20 "c:\\projets\\development\\tst\\com\\tstlib\\textindex.h" +#line 20 "d:\\projets\\tst\\com\\tstlib\\textindex.h" [id(1),helpstring("method AddWord")] HRESULT AddWord([in] BSTR *word , [in] BSTR *document ); [id(2),helpstring("method AddText")] HRESULT AddText([in] BSTR *text , [in] BSTR *document ); [id(3),helpstring("method Pack")] HRESULT Pack(); @@ -54,7 +54,7 @@ library tstlib uuid(98F35214-F66B-4BBF-B039-1AA4CA344DCF), helpstring("DocumentsScore Class") ] -#line 55 "c:\\projets\\development\\tst\\com\\tstlib\\documentsscore.h" +#line 55 "d:\\projets\\tst\\com\\tstlib\\documentsscore.h" coclass CDocumentsScore { interface IDocumentsScore; }; @@ -64,7 +64,7 @@ library tstlib uuid(8EC92961-CAB3-4CA6-A36E-46C39424E80B), helpstring("TextIndex Class") ] -#line 45 "c:\\projets\\development\\tst\\com\\tstlib\\textindex.h" +#line 45 "d:\\projets\\tst\\com\\tstlib\\textindex.h" coclass CTextIndex { interface ITextIndex; }; com/tstlib/_tstlib.idl com/tstlib/_tstlib.tlb @@ -6,7 +6,7 @@ /* File created by MIDL compiler version 6.00.0361 */ -/* at Sat Apr 01 15:31:23 2006 +/* at Sun Jul 16 11:58:43 2006 */ /* Compiler settings for _tstlib.idl: Oicf, W1, Zp8, env=Win32 (32b run) com/tstlib/_tstlib_i.c @@ -4,7 +4,7 @@ /* File created by MIDL compiler version 6.00.0361 */ -/* at Sat Apr 01 15:31:23 2006 +/* at Sun Jul 16 11:58:43 2006 */ /* Compiler settings for _tstlib.idl: Oicf, W1, Zp8, env=Win32 (32b run) com/tstlib/_tstlib_p.c @@ -90,7 +90,7 @@ CharacterSet="2"> <Tool Name="VCCLCompilerTool" - AdditionalIncludeDirectories="c:\projets\boost_1_33_1\;..\..\include" + AdditionalIncludeDirectories="d:\projets\boost_1_33_1\;..\..\include" PreprocessorDefinitions="WIN32;_WINDOWS;NDEBUG;_USRDLL;_ATL_ATTRIBUTES" RuntimeLibrary="2" UsePrecompiledHeader="3" @@ -105,7 +105,7 @@ AdditionalOptions="comsupp.lib" OutputFile="$(OutDir)/tstlib.dll" LinkIncremental="1" - AdditionalLibraryDirectories="&quot;c:\projets\boost_1_33_1\bin\boost\libs\regex\build\libboost_regex.lib\vc-7_1\release\threading-multi&quot;" + AdditionalLibraryDirectories="&quot;d:\projets\boost_1_33_1\bin\boost\libs\regex\build\libboost_regex.lib\vc-7_1\release\threading-multi&quot;" MergedIDLBaseFileName="_tstlib.idl" GenerateDebugInformation="TRUE" SubSystem="2" @@ -210,10 +210,10 @@ RelativePath=".\stdafx.h"> </File> <File - RelativePath="..\..\textindex\textindex.h"> + RelativePath=".\TextIndex.h"> </File> <File - RelativePath=".\TextIndex.h"> + RelativePath="..\..\textindex\textindex.h"> </File> <File RelativePath="..\..\include\tst.h"> com/tstlib/tstlib.vcproj @@ -3,7 +3,10 @@ "FILE_VERSION" = "9237" "ENLISTMENT_CHOICE" = "NEVER" "PROJECT_FILE_RELATIVE_PATH" = "" -"NUMBER_OF_EXCLUDED_FILES" = "0" +"NUMBER_OF_EXCLUDED_FILES" = "3" +"EXCLUDED_FILE0" = "_tstlib_p.c" +"EXCLUDED_FILE1" = "dlldata.c" +"EXCLUDED_FILE2" = "_tstlib_i.c" "ORIGINAL_PROJECT_FILE_PATH" = "" "NUMBER_OF_NESTED_PROJECTS" = "0" "SOURCE_CONTROL_SETTINGS_PROVIDER" = "PROJECT" com/tstlib/tstlibPS.vcproj.vspscc @@ -1436,7 +1436,7 @@ template<typename charT,typename valueT,typename storageT,typename serializerT, // We save the version number size_t version_length = strlen(TST_VERSION); file.write((char*)(&version_length),sizeof(size_t)); - file.write(TST_VERSION,version_length); + file.write(TST_VERSION,(std::streamsize)version_length); serializerT writer; writer.write(file,default_value); include/tst.h @@ -1,5 +1,5 @@ rmdir /s /q build include mkdir include & xcopy ..\include include swig -O -Iinclude -DSCANNER -c++ -python tst.i -e:\python23\python setup.py clean build --compile=mingw32 sdist bdist_wininst +c:\python23\python setup.py clean build --compile=mingw32 sdist bdist_wininst rmdir /s /q build include python/build23.bat @@ -33,7 +33,6 @@ typedef qad_string<char> string_type; - class CallableAction : public action<char,PythonReference,string_type > { public: CallableAction(PythonReference perform,PythonReference result) : _perform(perform), _result(result) { python/pythonTST.h @@ -6,7 +6,7 @@ # one containing a subdirectory called "boost/" and a sub-subdirectory # "boost/python/" full of .hpp files). Absolute paths work, too. # -path-global BOOST_ROOT : c:/projets/boost_1_33_1 ; +path-global BOOST_ROOT : d:/projets/boost_1_33_1 ; # Boost.Python configuration variables, as described in # http://www.boost.org/libs/python/doc/building.html#configuration. python/with_boost/Jamrules @@ -3,4 +3,4 @@ # Edit this path to point at the tools/build/v1 subdirectory of your # Boost installation. Absolute paths work, too. -boost-build c:/projets/boost_1_33_1/tools/build/v1 ; +boost-build d:/projets/boost_1_33_1/tools/build/v1 ; python/with_boost/boost-build.jam @@ -1,4 +1,4 @@ -bjam -sPYTHON_VERSION=2.4 -sZLIB_SOURCE=c:\projets\httpd-2.2.0\srclib\zlib -sPYTHON_ROOT=e:\python24 -sBUILD=release %* +bjam -sPYTHON_VERSION=2.4 -sZLIB_SOURCE=d:\projets\zlib-1.2.3 -sPYTHON_ROOT=c:\python24 -sBUILD=release %* copy bin\boost\libs\python\build\boost_python.dll\vc-7_1\release\threading-multi\boost_python.dll . copy bin\with_boost\tst.pyd\vc-7_1\release\threading-multi\tst.pyd . python zipit.py python/with_boost/build.bat python/with_boost/tst.pyd @@ -6,7 +6,7 @@ # one containing a subdirectory called "boost/" and a sub-subdirectory # "boost/python/" full of .hpp files). Absolute paths work, too. # -path-global BOOST_ROOT : c:/projets/boost_1_33_1 ; +path-global BOOST_ROOT : d:/projets/boost_1_33_1 ; # Boost.Python configuration variables, as described in # http://www.boost.org/libs/python/doc/building.html#configuration. textindex/Jamrules @@ -3,4 +3,4 @@ # Edit this path to point at the tools/build/v1 subdirectory of your # Boost installation. Absolute paths work, too. -boost-build c:/projets/boost_1_33_1/tools/build/v1 ; +boost-build d:/projets/boost_1_33_1/tools/build/v1 ; textindex/boost-build.jam textindex/boost_python.dll @@ -1,3 +1,3 @@ -bjam -sPYTHON_VERSION=2.4 -sPYTHON_ROOT=e:\python24 -sBUILD=release %1 %2 %3 %4 %5 %6 %7 %8 %9 +bjam -sPYTHON_VERSION=2.4 -sPYTHON_ROOT=c:\python24 -sBUILD=release %1 %2 %3 %4 %5 %6 %7 %8 %9 copy bin\boost\libs\python\build\boost_python.dll\vc-7_1\release\threading-multi\boost_python.dll . copy bin\textindex\textindex.pyd\vc-7_1\release\threading-multi\textindex.pyd . textindex/build.bat @@ -55,7 +55,7 @@ if __name__ == "__main__": self.entry = Entry(self.frame, name='input') self.entry.pack(fill=X) - self.entry.bind('<Key>',self.keyPressed) + self.entry.bind('<KeyRelease>',self.keyPressed) frame = Frame(self.frame) frame.pack(fill=BOTH, expand=1) textindex/test.py @@ -20,14 +20,19 @@ #include <string> #include <iostream> #include <fstream> -#include <boost/iostreams/filtering_stream.hpp> -#include <boost/iostreams/filter/zlib.hpp> #include <boost/python.hpp> using namespace boost::python; +#define COMPRESS_INDEX_FILE +#ifdef COMPRESS_INDEX_FILE +#include <boost/iostreams/filtering_stream.hpp> +#include <boost/iostreams/filter/zlib.hpp> +#endif + #define __PYTHON__BUILD__ #include "textindex.h" + class object_serializer { public: object_serializer(); @@ -86,22 +91,29 @@ template <typename character_type> class python_textindex : public textindex<cha void write_to_file(str file) { std::ofstream out(PyString_AsString(file.ptr()),std::ofstream::binary|std::ofstream::out|std::ofstream::trunc); - boost::iostreams::filtering_ostream fout; - fout.push(boost::iostreams::zlib_compressor()); - fout.push(out); - write(fout); - fout.strict_sync(); - // out.close(); + out.exceptions(std::ofstream::eofbit | std::ofstream::failbit | std::ofstream::badbit); + #ifdef COMPRESS_INDEX_FILE + boost::iostreams::filtering_ostream fout; + fout.push(boost::iostreams::zlib_compressor()); + fout.push(out); + write(fout); + fout.strict_sync(); + #else + write(out); + #endif } void read_from_file(object file) { std::ifstream in(PyString_AsString(file.ptr()),std::ifstream::binary|std::ifstream::in); in.exceptions(std::ifstream::eofbit | std::ifstream::failbit | std::ifstream::badbit); - boost::iostreams::filtering_istream fin; - fin.push(boost::iostreams::zlib_decompressor()); - fin.push(in); - read(fin); - // in.close(); + #ifdef COMPRESS_INDEX_FILE + boost::iostreams::filtering_istream fin; + fin.push(boost::iostreams::zlib_decompressor()); + fin.push(in); + read(fin); + #else + read(in); + #endif } protected: textindex/textindex.cpp @@ -26,21 +26,25 @@ #include <boost/regex.hpp> #include <iostream> -template <class pair> inline bool sort_by_document(const pair& lhs, const pair& rhs) +template <class pair> +inline bool sort_by_document(const pair& lhs, const pair& rhs) { return lhs.first < rhs.first; } -template <class pair> inline bool sort_by_score(const pair& lhs, const pair& rhs) +template <class pair> +inline bool sort_by_score(const pair& lhs, const pair& rhs) { return lhs.second > rhs.second; } -template <class item> inline bool sort_by_first_item_length_desc(const item& lhs, const item& rhs) { +template <class item> +inline bool sort_by_first_item_length_desc(const item& lhs, const item& rhs) { return lhs.begin()->length() > rhs.begin()->length(); } -template <typename document_id_type, typename score_type> class documents_scores { +template <typename document_id_type, typename score_type> +class documents_scores { public: typedef std::pair<document_id_type,score_type> pair_type; typedef std::map<document_id_type,score_type> storage_type; @@ -164,8 +168,11 @@ template <typename document_id_type, typename score_type> class documents_scores }; -template < typename character_type, typename document_type, typename reader_writer, typename document_id_type = unsigned int, typename score_type = unsigned int > class textindex { +template < typename character_type, typename document_type, typename reader_writer, typename document_id_type = unsigned int, typename score_type = unsigned int> +class textindex { public: + // Chaînes de caractères + typedef std::basic_string<typename character_type> string_type; // Mapping document <=> id typedef std::map< typename document_type, document_id_type > documents_ids_type; @@ -175,7 +182,7 @@ template < typename character_type, typename document_type, typename reader_writ typedef documents_scores< typename document_id_type, typename score_type > documents_scores_type; typedef typename documents_scores_type::shared_ptr documents_scores_pointer; typedef typename documents_scores_type::serializer documents_scores_serializer; - typedef string_tst < character_type, typename documents_scores_pointer, memory_storage<character_type, typename documents_scores_pointer>, documents_scores_serializer > tree_type; + typedef tst < character_type, typename documents_scores_pointer, memory_storage<character_type, typename documents_scores_pointer>, documents_scores_serializer, typename string_type > tree_type; // Résultat des requêtes typedef std::pair< document_type, score_type > pair_type; @@ -184,14 +191,14 @@ template < typename character_type, typename document_type, typename reader_writ // Regexes typedef boost::basic_regex < character_type > regex_type; - typedef boost::regex_iterator<typename std::basic_string<character_type>::const_iterator> regex_type_iterator; + typedef boost::regex_iterator<typename string_type::const_iterator> regex_type_iterator; - class collector : public action< typename character_type, typename documents_scores_pointer > { + class collector : public action< typename character_type, typename documents_scores_pointer, typename string_type> { public: collector(collector* _intersect=0) : intersect(_intersect), entries(new documents_scores_type(0)) { } - virtual void perform(const character_type* string, size_t string_length, int remaining_distance, typename documents_scores_pointer data) { + virtual void perform(const typename string_type& string, int remaining_distance, typename documents_scores_pointer data) { entries->merge_with(*data,intersect ? intersect->entries.get() : 0); } @@ -216,12 +223,12 @@ template < typename character_type, typename document_type, typename reader_writ documents_scores_pointer entries; }; - class eraser : public action< typename character_type, typename documents_scores_pointer > { + class eraser : public action< typename character_type, typename documents_scores_pointer, typename string_type > { public: eraser(document_id_type _document_id) : document_id(_document_id) { } - virtual void perform(const character_type* string, size_t string_length, int remaining_distance, typename documents_scores_pointer data) { + virtual void perform(const typename string_type& string, int remaining_distance, typename documents_scores_pointer data) { data->remove_document(document_id); } @@ -233,16 +240,16 @@ template < typename character_type, typename document_type, typename reader_writ document_id_type document_id; }; - class cleaner : public action< typename character_type, typename documents_scores_pointer > { + class cleaner : public action< typename character_type, typename documents_scores_pointer, typename string_type > { public: - typedef boost::shared_ptr< std::vector< std::basic_string< typename character_type > > > result_type; + typedef boost::shared_ptr< std::vector< typename string_type > > result_type; - cleaner(typename tree_type* _tree) : tree(_tree), empty_words(new std::vector< std::basic_string< character_type > >() ) { + cleaner(typename tree_type* _tree) : tree(_tree), empty_words(new std::vector< typename string_type >() ) { } - virtual void perform(const character_type* string, size_t string_length, int remaining_distance, typename documents_scores_pointer data) { + virtual void perform(const typename string_type& string, int remaining_distance, typename documents_scores_pointer data) { if(data->size()==0) { - empty_words->push_back(std::basic_string< typename character_type >(string,string_length)); + empty_words->push_back(string); } } @@ -258,9 +265,9 @@ template < typename character_type, typename document_type, typename reader_writ typename result_type empty_words; }; - class documents_scores_type_factory : public filter< character_type, documents_scores_pointer > { + class documents_scores_type_factory : public filter< character_type, documents_scores_pointer, typename string_type > { public: - virtual documents_scores_pointer perform(const character_type* string, size_t string_length, int remaining_distance, typename documents_scores_pointer data) { + virtual documents_scores_pointer perform(const typename string_type& string, int remaining_distance, typename documents_scores_pointer data) { return typename documents_scores_pointer(new documents_scores_type(0)); } }; @@ -271,7 +278,7 @@ template < typename character_type, typename document_type, typename reader_writ next_document_id(0) { } - int put_word(const std::basic_string< character_type >& word,const document_type& document) { + int put_word(const typename string_type& word,const document_type& document) { if(word.size()>2) { documents_scores_pointer scores(tree.get_or_build(word,&factory)); return scores->add_document(get_document_id(document),1); @@ -281,7 +288,7 @@ template < typename character_type, typename document_type, typename reader_writ } } - int put_text(const std::basic_string< character_type >& text,const document_type& document) { + int put_text(const typename string_type& text,const document_type& document) { typename regex_type_iterator token(text.begin(),text.end(),tokenizer); typename regex_type_iterator end; int count = 0; @@ -302,17 +309,17 @@ template < typename character_type, typename document_type, typename reader_writ reversed_ids.erase(document_id); eraser e(document_id); - tree.walk1(0,&e); + tree.walk(0,&e); } } - result_pointer find_word(const std::basic_string< character_type >& word) { + result_pointer find_word(const typename string_type& word) { collector c; tree.match(word,0,&c); return convert_to_result(c); } - result_pointer find_text(const std::basic_string< character_type >& text,bool intersection) { + result_pointer find_text(const typename string_type& text,bool intersection) { typename regex_type_iterator token(text.begin(),text.end(),tokenizer); typename regex_type_iterator end; if(token!=end) { @@ -343,7 +350,7 @@ template < typename character_type, typename document_type, typename reader_writ collector c; while(token != end) { if(token->begin()->length()>2) { - std::basic_string<character_type> word = token->begin()->str(); + typename string_type word = token->begin()->str(); tree.match(word,0,&c); } ++token; @@ -358,7 +365,7 @@ template < typename character_type, typename document_type, typename reader_writ void pack() { cleaner c(&tree); - tree.walk1(NULL,&c); + tree.walk(NULL,&c); tree.pack(); } textindex/textindex.h textindex/textindex.pyd 6e0528a99f6a946abdbe21e27585933671385a83 Nicolas Lehuen nicolas@lehuen.com http://github.com/nlehuen/pytst/commit/1a1493e1269799076d540d29c78395e84a95d656 1a1493e1269799076d540d29c78395e84a95d656 2006-07-17T03:22:31-07:00 2006-07-17T03:22:31-07:00 Mise à jour de textindex suite aux changements d'API de TST (passage à string partout). 071ca8928e709afeff787fbd55e5ee1955f802c1 Nicolas Lehuen nicolas@lehuen.com