Skip to content

Commit

Permalink
Issue #32
Browse files Browse the repository at this point in the history
  • Loading branch information
@grosjo
grosjo committed Jan 18, 2020
1 parent bf95117 commit 8397809
Show file tree
Hide file tree
Showing 2 changed files with 77 additions and 44 deletions.
84 changes: 59 additions & 25 deletions src/fts-backend-xapian-functions.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -297,22 +297,22 @@ class XQuerySet

qp->set_database(*db);

Xapian::Query * q = new Xapian::Query(qp->parse_query(s));
Xapian::Query * q = new Xapian::Query(qp->parse_query(s,Xapian::QueryParser::FLAG_DEFAULT | Xapian::QueryParser::FLAG_PARTIAL));

i_free(s);
delete(qp);
return q;
}
};

class XHeaderTerm
class XNGram
{
public:
long size,partial,full,maxlength;
char ** data;
bool onlyone;

XHeaderTerm(long p, long f, bool o)
XNGram(long p, long f, bool o)
{
partial=p; full=f;
size=0;
Expand All @@ -321,7 +321,7 @@ class XHeaderTerm
onlyone=o;
}

~XHeaderTerm()
~XNGram()
{
if (size>0)
{
Expand Down Expand Up @@ -728,27 +728,27 @@ bool fts_backend_xapian_index_hdr(Xapian::WritableDatabase * dbx, uint uid, cons

if(i>=HDRS_NB) return true;
const char * h=hdrs_xapian[i];
{
XHeaderTerm * xhs = new XHeaderTerm(p,f,strcmp(h,"XMID")==0);
xhs->add(data);

char *t = (char*)i_malloc(sizeof(char)*(xhs->maxlength+6));

for(i=0;i<xhs->size;i++)
XNGram * ngram = new XNGram(p,f,strcmp(h,"XMID")==0);
ngram->add(data);

char *t = (char*)i_malloc(sizeof(char)*(ngram->maxlength+6));

for(i=0;i<ngram->size;i++)
{
snprintf(t,ngram->maxlength+6,"%s%s",h,ngram->data[i]);
try
{
snprintf(t,xhs->maxlength+6,"%s%s",h,xhs->data[i]);
try
{
doc.add_term(t);
}
catch(Xapian::Error e)
{
i_error("FTS Xapian: %s",e.get_msg().c_str());
}
doc.add_term(t);
}
catch(Xapian::Error e)
{
i_error("FTS Xapian: %s",e.get_msg().c_str());
}
i_free(t);
delete(xhs);
}
i_free(t);
delete(ngram);

dbx->replace_document(docid,doc);
return true;
}
Expand All @@ -760,7 +760,7 @@ bool fts_backend_xapian_index_hdr(Xapian::WritableDatabase * dbx, uint uid, cons
return false;
}

bool fts_backend_xapian_index_text(Xapian::WritableDatabase * dbx,uint uid, const char * field, const char * data)
bool fts_backend_xapian_index_text(Xapian::WritableDatabase * dbx,uint uid, const char * field, const char * data,long p, long f)
{
try
{
Expand Down Expand Up @@ -788,9 +788,11 @@ bool fts_backend_xapian_index_text(Xapian::WritableDatabase * dbx,uint uid, cons
delete(result);
delete(xq);

Xapian::Document doc2;
Xapian::TermGenerator termgenerator;
termgenerator.set_stemmer(Xapian::Stem("en"));
termgenerator.set_document(doc);
Xapian::Stem stem("en");
termgenerator.set_stemmer(stem);
termgenerator.set_document(doc2);

const char * h;
if(strcmp(field,"subject")==0)
Expand All @@ -802,8 +804,40 @@ bool fts_backend_xapian_index_text(Xapian::WritableDatabase * dbx,uint uid, cons
h="XBDY";
}
std::string d(data);
termgenerator.set_stemming_strategy(Xapian::TermGenerator::STEM_ALL);
termgenerator.index_text(d, 1, h);


long n= doc2.termlist_count();
Xapian::TermIterator ti = doc2.termlist_begin();
XNGram * ngram = new XNGram(p,f,false);
while(n>0)
{
const std::string s = *ti;
if(s.compare(0,strlen(h),h)==0)
{
// if(verbose>1) i_info("Adding STEM %s",s.c_str());
ngram->add(s.c_str()+strlen(h));
}
ti++;
n--;
}
if(verbose>1) i_info("NGRAM(%s,%s) %d max=%d",field,h,ngram->size,ngram->maxlength);
char *t = (char*)i_malloc(sizeof(char)*(ngram->maxlength+6));
for(n=0;n<ngram->size;n++)
{
snprintf(t,ngram->maxlength+6,"%s%s",h,ngram->data[n]);
try
{
doc.add_term(t);
}
catch(Xapian::Error e)
{
i_error("FTS Xapian: %s",e.get_msg().c_str());
}
}
i_free(t);
delete(ngram);

dbx->replace_document(docid,doc);
}
catch(Xapian::Error e)
Expand Down
37 changes: 18 additions & 19 deletions src/fts-backend-xapian.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -378,46 +378,45 @@ static int fts_backend_xapian_update_build_more(struct fts_backend_update_contex
if(data == NULL) return 0;
if(size<1) return 0;

char * s = (char*)i_malloc(sizeof(char)*(size+1));
strncpy(s,(char *)data,size);
s[size]=0;

if((backend->oldbox == NULL) || (strcmp(backend->oldbox,backend->box->name)!=0))
{
fts_backend_xapian_oldbox(backend);
backend->oldbox = i_strdup(backend->box->name);
if(verbose>1) i_info("Start indexing '%s' (%s)",backend->box->name,backend->db);
}

if(!fts_backend_xapian_check_write(backend))
{
i_error("FTS Xapian: Buildmore: Can not open db");
return -1;
}

char * s = (char*)i_malloc(sizeof(char)*(size+1));
strncpy(s,(char *)data,size);
s[size]=0;

bool ok=true;

if(ctx->tbi_isfield)
{
if(!fts_backend_xapian_index_hdr(backend->dbw,ctx->tbi_uid,ctx->tbi_field, s, backend->partial,backend->full))
{
return -1;
}
ok=fts_backend_xapian_index_hdr(backend->dbw,ctx->tbi_uid,ctx->tbi_field, s, backend->partial,backend->full);
}
else
{
if(!fts_backend_xapian_index_text(backend->dbw,ctx->tbi_uid,ctx->tbi_field, s))
{
return -1;
}
ok=fts_backend_xapian_index_text(backend->dbw,ctx->tbi_uid,ctx->tbi_field, s, backend->partial,backend->full);
}

if((backend->oldbox == NULL) || (strcmp(backend->oldbox,backend->box->name)!=0))
{
fts_backend_xapian_oldbox(backend);
backend->oldbox = i_strdup(backend->box->name);
if(verbose>1) i_info("Start indexing '%s' (%s)",backend->box->name,backend->db);
}
i_free(s);

backend->nb_updates++;
if(backend->nb_updates>XAPIAN_COMMIT_LIMIT)
{
if(verbose>1) i_info("refreshing...");
fts_backend_xapian_refresh( ctx->ctx.backend);
}
return 0;

if(!ok) return -1;
return 0;
}


Expand Down

0 comments on commit 8397809

Please sign in to comment.