update ranker

main.go

@@ -167,6 +167,12 @@ func main() {
 		"text",
 		"set output format [text, json]",
 	)
+	flags.StringVar(
+		&processor.Ranker,
+		"ranker",
+		"tfidfl",
+		"set ranking algorithm [tfidf, tfidfl]",
+	)
 
 	// the below flags we want but are not enabled as yet
 

processor/arguments.go

@@ -41,6 +41,9 @@ var IncludeBinaryFiles = false
 // Format sets the output format of the formatter
 var Format = ""
 
+// Ranker sets which ranking algorithm to use
+var Ranker = ""
+
 // FileOutput sets the file that output should be written to
 var FileOutput = ""
 

processor/result_ranker.go

@@ -157,7 +157,12 @@ func rankResultsTFIDF(corpusCount int, results []*fileJob, documentFrequencies m
 			tf := float64(len(wordCount)) / words
 			idf := math.Log10(float64(corpusCount) / float64(documentFrequencies[word]))
 
-			weight += tf * idf
+			// TODO adding math.Sqrt around tf can improve results https://opensourceconnections.com/blog/2015/10/16/bm25-the-next-generation-of-lucene-relevation/
+			if Ranker == "tfidfl" {
+				weight += math.Sqrt(tf) * idf * (1/math.Sqrt(words))
+			} else {
+				weight += tf * idf
+			}
 		}
 
 		// Override the score here because we don't want whatever we got originally