Merge pull request #4473 from rspamd/vstakhov-fasttext-langdet

[Feature] Add fasttext language detector to Rspamd

CMakeLists.txt

@@ -247,6 +247,7 @@ if(ENABLE_FASTTEXT MATCHES "ON")
 	ProcessPackage(FASTTEXT LIBRARY fasttext INCLUDE fasttext.h
 			INCLUDE_SUFFIXES include/fasttext
 			ROOT ${FASTTEXT_ROOT_DIR} MODULES fasttext)
+	SET(WITH_FASTTEXT "1")
 endif()
 
 include (CompilerWarnings)

conf/lang_detection.inc

@@ -0,0 +1,10 @@
+# Language detection configuration
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/lang_detection.inc' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/lang_detection.inc' to strictly override all
+# parameters defined inside this section
+

conf/rspamd.conf

@@ -24,6 +24,12 @@ options {
     .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/options.inc"
 }
 
+lang_detection {
+    .include "$CONFDIR/lang_detection.inc"
+    .include(try=true; priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/lang_detection.inc"
+    .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/lang_detection.inc"
+}
+
 .include(try=true; duplicate=merge) "$CONFDIR/cgp.inc"
 .include(try=true; priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/cgp.inc"
 

config.h.in

@@ -116,6 +116,7 @@
 #cmakedefine WITH_LIBUNWIND      1
 #cmakedefine WITH_LUA_TRACE      1
 #cmakedefine WITH_LUA_REPL       1
+#cmakedefine WITH_FASTTEXT       1
 
 #cmakedefine DISABLE_PTHREAD_MUTEX 1
 

src/libmime/CMakeLists.txt

@@ -12,6 +12,7 @@ SET(LIBRSPAMDMIMESRC
 				${CMAKE_CURRENT_SOURCE_DIR}/mime_parser.c
 				${CMAKE_CURRENT_SOURCE_DIR}/mime_encoding.c
 				${CMAKE_CURRENT_SOURCE_DIR}/lang_detection.c
+		${CMAKE_CURRENT_SOURCE_DIR}/lang_detection_fasttext.cxx
 		${CMAKE_CURRENT_SOURCE_DIR}/mime_string.cxx
 		)