Be more efficient when resetting stemmers

src/forward_index.c

@@ -68,10 +68,15 @@ static void ForwardIndex_InitCommon(ForwardIndex *idx, Document *doc, uint32_t i
   idx->idxFlags = idxFlags;
   idx->maxFreq = 0;
   idx->totalFreq = 0;
-  if (idx->stemmer) {
+
+  if (idx->stemmer && !ResetStemmer(idx->stemmer, SnowballStemmer, doc->language)) {
     idx->stemmer->Free(idx->stemmer);
+    idx->stemmer = NULL;
+  }
+
+  if (!idx->stemmer) {
+    idx->stemmer = NewStemmer(SnowballStemmer, doc->language);
   }
-  idx->stemmer = NewStemmer(SnowballStemmer, doc->language);
 }
 
 ForwardIndex *NewForwardIndex(Document *doc, uint32_t idxFlags) {
@@ -81,7 +86,9 @@ ForwardIndex *NewForwardIndex(Document *doc, uint32_t idxFlags) {
   BlkAlloc_Init(&idx->entries);
 
   static const KHTableProcs procs = {
-      .Alloc = allocBucketEntry, .Compare = khtCompare, .Hash = khtHash,
+      .Alloc = allocBucketEntry,
+      .Compare = khtCompare,
+      .Hash = khtHash,
   };
 
   size_t termCount = estimtateTermCount(doc);

src/stemmer.c

@@ -2,6 +2,7 @@
 #include <string.h>
 #include <stdio.h>
 #include <sys/param.h>
+#include <assert.h>
 #include "dep/snowball/include/libstemmer.h"
 
 const char *__supportedLanguages[] = {"arabic",     "danish",   "dutch",     "english", "finnish",
@@ -61,6 +62,13 @@ void __sbstemmer_Free(Stemmer *s) {
   free(s);
 }
 
+static int sbstemmer_Reset(Stemmer *stemmer, StemmerType type, const char *language) {
+  if (type != stemmer->type || strcmp(stemmer->language, language)) {
+    return 0;
+  }
+  return 1;
+}
+
 Stemmer *__newSnowballStemmer(const char *language) {
   struct sb_stemmer *sb = sb_stemmer_new(language, NULL);
   // No stemmer available for this language
@@ -78,16 +86,34 @@ Stemmer *__newSnowballStemmer(const char *language) {
   ret->ctx = ctx;
   ret->Stem = __sbstemmer_Stem;
   ret->Free = __sbstemmer_Free;
+  ret->Reset = sbstemmer_Reset;
   return ret;
 }
 
 Stemmer *NewStemmer(StemmerType type, const char *language) {
-  switch (type) {
-    case SnowballStemmer:
+  Stemmer *ret = NULL;
+  if (type == SnowballStemmer) {
+    ret = __newSnowballStemmer(language);
+    if (!ret) {
+      return NULL;
+    }
+  } else {
+    fprintf(stderr, "Invalid stemmer type");
+    return NULL;
+  }
 
-      return __newSnowballStemmer(language);
+  for (const char **s = __supportedLanguages; *s; s++) {
+    if (!strcmp(language, *s)) {
+      ret->language = *s;
+      break;
+    }
   }
 
-  fprintf(stderr, "Invalid stemmer type");
-  return NULL;
+  assert(ret->language);
+  ret->type = type;
+  return ret;
 }
+
+int ResetStemmer(Stemmer *stemmer, StemmerType type, const char *language) {
+  return stemmer->Reset && stemmer->Reset(stemmer, type, language);
+}

src/stemmer.h

@@ -14,10 +14,19 @@ typedef struct stemmer {
   void *ctx;
   const char *(*Stem)(void *ctx, const char *word, size_t len, size_t *outlen);
   void (*Free)(struct stemmer *);
+
+  // Attempts to reset the stemmer using the given language and type. Returns 0
+  // if this stemmer cannot be reused.
+  int (*Reset)(struct stemmer *, StemmerType type, const char *language);
+
+  const char *language;
+  StemmerType type;  // Type of stemmer
 } Stemmer;
 
 Stemmer *NewStemmer(StemmerType type, const char *language);
 
+int ResetStemmer(Stemmer *stemmer, StemmerType type, const char *language);
+
 /* check if a language is supported by our stemmers */
 int IsSupportedLanguage(const char *language, size_t len);