High performance gains. Still experimental.

- New "batch" mode which improves performance.
- New "batch-size" configuration option.
- Move many mutex_locks/unlocks to __atomic_add_fetch/__atomic_sub_fetch
- where i could moved from strlcpy to memcpy
- change "level", "tag", "faclility",  "syslog_facility", "syslog_level".
- bug in the "syslog_priority" didn't exsist.

doc/source/high-performance.rst

@@ -0,0 +1,2 @@
+High Performance Considerations
+===============================

doc/source/index.rst

@@ -15,7 +15,7 @@ Sagan User Guide
 
    liblognorm
 
-
+   high-performance
    blogs/index
    getting-help
 

doc/source/rule-keywords.rst

@@ -281,12 +281,12 @@ port, facility, syslog priority, liblognorm JSON and the syslog message.
 
 **external: /usr/local/bin/myprogram.py**
 
-facility
---------
+syslog_facility
+---------------
 
-.. option:: facility: {sylog facility}
+.. option:: syslog_facility: {sylog facility}
 
-Searches only messages from a specified facility.
+Searches only messages from a specified facility.  This can be multiple facilities when seperated with an '|' (or) symbol.
 
 **facility: daemon;**
 
@@ -341,12 +341,12 @@ This informs Sagan that if the rule is successfully trigged, the source or desti
 
 This would firewall the offending source for 1 day. For more information about Snortsam, see: http://www.snortsam.net
 
-level
------
+syslog_level
+------------
 
-.. option:: level: {syslog level};
+.. option:: syslog_level: {syslog level};
 
-Seaches only message from a specified syslog level.
+Seaches only message from a specified syslog level.  This can be multiple levels when seperated by a '|' (or) symbol.
 
 **level: notice;**
 
@@ -620,12 +620,12 @@ sid
 
 Sagan signatures start at 5000000. To view the "last used" signature, see https://github.com/beave/sagan-rules/blob/master/.last_used_sid
 
-tag
----
+syslog_tag
+----------
 
-.. option:: tag: {syslog tag};
+.. option:: syslog_tag: {syslog tag};
 
-Informs Sagan to only search syslog message with the specified tag.
+Informs Sagan to only search syslog message with the specified tag.  This can be multiple tags when seperated with an '|' (or) symbol.
 
 **tag: 2d;**
 

src/classifications.c

@@ -48,7 +48,7 @@ struct _Class_Struct *classstruct;
 struct _SaganDebug *debug;
 struct _SaganConfig *config;
 
-pthread_mutex_t CounterClassMutex=PTHREAD_MUTEX_INITIALIZER;
+//pthread_mutex_t CounterClassMutex=PTHREAD_MUTEX_INITIALIZER;
 
 void Load_Classifications( const char *ruleset )
 {
@@ -62,9 +62,11 @@ void Load_Classifications( const char *ruleset )
     char tmpbuf2[5];
     int  linecount=0;
 
-    pthread_mutex_lock(&CounterClassMutex);
-    counters->classcount = 0;
-    pthread_mutex_unlock(&CounterClassMutex);
+//    pthread_mutex_lock(&CounterClassMutex);
+//    counters->classcount = 0;
+//    pthread_mutex_unlock(&CounterClassMutex);
+
+    __atomic_store_n (&counters->classcount, 0, __ATOMIC_SEQ_CST);
 
     Sagan_Log(NORMAL, "Loading classifications.conf file. [%s]", ruleset);
 
@@ -138,9 +140,10 @@ void Load_Classifications( const char *ruleset )
                     Sagan_Log(DEBUG, "[D-%d] Classification: %s|%s|%d", counters->classcount, classstruct[counters->classcount].s_shortname, classstruct[counters->classcount].s_desc, classstruct[counters->classcount].s_priority);
                 }
 
-            pthread_mutex_lock(&CounterClassMutex);
-            counters->classcount++;
-            pthread_mutex_unlock(&CounterClassMutex);
+//            pthread_mutex_lock(&CounterClassMutex);
+//            counters->classcount++;
+//            pthread_mutex_unlock(&CounterClassMutex);
+	      __atomic_add_fetch(&counters->classcount, 1, __ATOMIC_SEQ_CST);
 
         }
     fclose(classfile);

src/config-yaml.c

@@ -100,8 +100,8 @@ struct _Rule_Struct *rulestruct;
 
 bool reload_rules;
 
-pthread_mutex_t SaganRulesLoadedMutex;
-pthread_mutex_t CounterLoadConfigGenericMutex=PTHREAD_MUTEX_INITIALIZER;
+//pthread_mutex_t SaganRulesLoadedMutex;
+//pthread_mutex_t CounterLoadConfigGenericMutex=PTHREAD_MUTEX_INITIALIZER;
 
 
 #ifdef HAVE_LIBYAML
@@ -210,6 +210,8 @@ void Load_YAML_Config( char *yaml_file )
             config->max_threshold2 = DEFAULT_IPC_THRESHOLD2_IPC;
             config->max_track_clients = DEFAULT_IPC_CLIENT_TRACK_IPC;
 
+	    config->max_batch = MAX_SYSLOG_BATCH;
+
             config->pp_sagan_track_clients = TRACK_TIME;
 
             config->sagan_proto = 17;           /* Default to UDP */
@@ -512,9 +514,11 @@ void Load_YAML_Config( char *yaml_file )
                                                             Sagan_Log(DEBUG, "[%s, line %d] Variable: \"%s == %s\"", __FILE__, __LINE__, var[counters->var_count].var_name, var[counters->var_count].var_value);
                                                         }
 
-                                                    pthread_mutex_lock(&CounterLoadConfigGenericMutex);
-                                                    counters->var_count++;
-                                                    pthread_mutex_unlock(&CounterLoadConfigGenericMutex);
+//                                                    pthread_mutex_lock(&CounterLoadConfigGenericMutex);
+//                                                    counters->var_count++;
+//                                                    pthread_mutex_unlock(&CounterLoadConfigGenericMutex);
+
+						      __atomic_add_fetch(&counters->var_count, 1, __ATOMIC_SEQ_CST);
 
                                                     toggle = 1;
 
@@ -847,6 +851,25 @@ void Load_YAML_Config( char *yaml_file )
 
                                         }
 
+				    else if (!strcmp(last_pass, "batch-size")) 
+					{
+					    Var_To_Value(value, tmp, sizeof(tmp));
+
+                                        config->max_batch = atoi(tmp);
+
+                                            if ( config->max_batch  == 0 )
+                                                {
+                                                    Sagan_Log(ERROR, "[%s, line %d] sagan:core 'max_batch' is zero/invalid. Abort!", __FILE__, __LINE__);
+                                                }
+
+					    if ( config->max_batch > MAX_SYSLOG_BATCH )
+					       {
+						    Sagan_Log(ERROR, "[%s, line %d] sagan:core 'max_batch' is greater than %d (the max default). Abort!", __FILE__, __LINE__, MAX_SYSLOG_BATCH);
+					       }
+
+
+					}
+
                                     else if (!strcmp(last_pass, "xbit-storage"))
                                         {
 
@@ -1138,9 +1161,11 @@ void Load_YAML_Config( char *yaml_file )
                                                     memcpy(GeoIP_Skip[counters->geoip_skip_count].range.ipbits, geoip_ipbits, sizeof(geoip_ipbits));
                                                     memcpy(GeoIP_Skip[counters->geoip_skip_count].range.maskbits, geoip_maskbits, sizeof(geoip_maskbits));
 
-                                                    pthread_mutex_lock(&CounterLoadConfigGenericMutex);
-                                                    counters->geoip_skip_count++;
-                                                    pthread_mutex_unlock(&CounterLoadConfigGenericMutex);
+//                                                    pthread_mutex_lock(&CounterLoadConfigGenericMutex);
+//                                                    counters->geoip_skip_count++;
+//                                                    pthread_mutex_unlock(&CounterLoadConfigGenericMutex)
+
+					            __atomic_add_fetch(&ounters->geoip_skip_count, 1, __ATOMIC_SEQ_CST);
 
                                                     maxmind_ptr = strtok_r(NULL, ",", &tok);
 
@@ -1684,9 +1709,11 @@ void Load_YAML_Config( char *yaml_file )
                                                     memcpy(Bluedot_Skip[counters->bluedot_skip_count].range.ipbits, bluedot_ipbits, sizeof(bluedot_ipbits));
                                                     memcpy(Bluedot_Skip[counters->bluedot_skip_count].range.maskbits, bluedot_maskbits, sizeof(bluedot_maskbits));
 
-                                                    pthread_mutex_lock(&CounterLoadConfigGenericMutex);
-                                                    counters->bluedot_skip_count++;
-                                                    pthread_mutex_unlock(&CounterLoadConfigGenericMutex);
+//                                                    pthread_mutex_lock(&CounterLoadConfigGenericMutex);
+//                                                    counters->bluedot_skip_count++;
+//                                                    pthread_mutex_unlock(&CounterLoadConfigGenericMutex);
+
+						    __atomic_add_fetch(&counters->bluedot_skip_count, 1, __ATOMIC_SEQ_CST);
 
                                                     bluedot_ptr = strtok_r(NULL, ",", &tok);
 
@@ -2497,9 +2524,11 @@ void Load_YAML_Config( char *yaml_file )
                             Var_To_Value(value, tmp, sizeof(tmp));
                             strlcpy(rules_loaded[counters->rules_loaded_count].ruleset, tmp, sizeof(rules_loaded[counters->rules_loaded_count].ruleset));
 
-                            pthread_mutex_lock(&CounterLoadConfigGenericMutex);
-                            counters->rules_loaded_count++;
-                            pthread_mutex_unlock(&CounterLoadConfigGenericMutex);
+//                            pthread_mutex_lock(&CounterLoadConfigGenericMutex);
+//                            counters->rules_loaded_count++;
+//                            pthread_mutex_unlock(&CounterLoadConfigGenericMutex);
+
+			      __atomic_add_fetch(&counters->rules_loaded_count, 1, __ATOMIC_SEQ_CST);
 
                         }
 

src/gen-msg.c

@@ -42,7 +42,7 @@ struct _Sagan_Processor_Generator *generator;
 struct _SaganConfig *config;
 struct _SaganDebug *debug;
 
-pthread_mutex_t CounterGenMapMutex=PTHREAD_MUTEX_INITIALIZER;
+//pthread_mutex_t CounterGenMapMutex=PTHREAD_MUTEX_INITIALIZER;
 
 
 void Load_Gen_Map( const char *genmap )
@@ -59,9 +59,11 @@ void Load_Gen_Map( const char *genmap )
 
     Sagan_Log(NORMAL, "Loading gen-msg.map file. [%s]", genmap);
 
-    pthread_mutex_lock(&CounterGenMapMutex);
-    counters->genmapcount=0;
-    pthread_mutex_unlock(&CounterGenMapMutex);
+//    pthread_mutex_lock(&CounterGenMapMutex);
+//    counters->genmapcount=0;
+//    pthread_mutex_unlock(&CounterGenMapMutex);
+
+    __atomic_store_n (&counters->genmapcount, 0, __ATOMIC_SEQ_CST);
 
     if (( genmapfile = fopen(genmap, "r" )) == NULL )
         {
@@ -119,9 +121,12 @@ void Load_Gen_Map( const char *genmap )
             generator[counters->genmapcount].alertid=atoi(gen2);
             strlcpy(generator[counters->genmapcount].generator_msg, gen3, sizeof(generator[counters->genmapcount].generator_msg));
 
-            pthread_mutex_lock(&CounterGenMapMutex);
-            counters->genmapcount++;
-            pthread_mutex_unlock(&CounterGenMapMutex);
+//            pthread_mutex_lock(&CounterGenMapMutex);
+//            counters->genmapcount++;
+//            pthread_mutex_unlock(&CounterGenMapMutex);
+
+	      __atomic_add_fetch(&counters->genmapcount, 1, __ATOMIC_SEQ_CST);
+
         }
 
     fclose(genmapfile);

src/geoip.c

@@ -56,7 +56,7 @@ struct _SaganCounters *counters;
 struct _Sagan_GeoIP_Skip *GeoIP_Skip;
 
 
-pthread_mutex_t CountGeoIP2MissMutex=PTHREAD_MUTEX_INITIALIZER;
+//pthread_mutex_t CountGeoIP2MissMutex=PTHREAD_MUTEX_INITIALIZER;
 
 void Open_GeoIP2_Database( void )
 {
@@ -141,9 +141,11 @@ int GeoIP2_Lookup_Country( char *ipaddr, unsigned char *ip_bits, int rule_positi
 
     if (res != MMDB_SUCCESS)
         {
-            pthread_mutex_lock(&CountGeoIP2MissMutex);
-            counters->geoip2_miss++;
-            pthread_mutex_unlock(&CountGeoIP2MissMutex);
+
+	      __atomic_add_fetch(&counters->geoip2_miss, 1, __ATOMIC_SEQ_CST);
+//            pthread_mutex_lock(&CountGeoIP2MissMutex);
+//            counters->geoip2_miss++;
+//            pthread_mutex_unlock(&CountGeoIP2MissMutex);
 
             Sagan_Log(WARN, "Country code MMDB_get_value failure (%s) for %s.", MMDB_strerror(res), ipaddr);
             return(GEOIP_SKIP);
@@ -153,9 +155,10 @@ int GeoIP2_Lookup_Country( char *ipaddr, unsigned char *ip_bits, int rule_positi
     if (!entry_data.has_data || entry_data.type != MMDB_DATA_TYPE_UTF8_STRING)
         {
 
-            pthread_mutex_lock(&CountGeoIP2MissMutex);
-            counters->geoip2_miss++;
-            pthread_mutex_unlock(&CountGeoIP2MissMutex);
+//            pthread_mutex_lock(&CountGeoIP2MissMutex);
+//            counters->geoip2_miss++;
+//            pthread_mutex_unlock(&CountGeoIP2MissMutex);
+	      __atomic_add_fetch(&counters->geoip2_miss, 1, __ATOMIC_SEQ_CST);
 
             if ( debug->debuggeoip2 )
                 {

src/ignore-list.c

@@ -41,7 +41,7 @@ struct _Sagan_Ignorelist *SaganIgnorelist;
 struct _SaganCounters *counters;
 struct _SaganConfig *config;
 
-pthread_mutex_t CountDropListMutex=PTHREAD_MUTEX_INITIALIZER;
+//pthread_mutex_t CountDropListMutex=PTHREAD_MUTEX_INITIALIZER;
 
 /****************************************************************************
  * "ignore" list.
@@ -89,9 +89,12 @@ void Load_Ignore_List ( void )
 
                             strlcpy(SaganIgnorelist[counters->droplist_count].ignore_string, droplistbuf, sizeof(SaganIgnorelist[counters->droplist_count].ignore_string));
 
-                            pthread_mutex_lock(&CountDropListMutex);
-                            counters->droplist_count++;
-                            pthread_mutex_unlock(&CountDropListMutex);
+//                            pthread_mutex_lock(&CountDropListMutex);
+//                            counters->droplist_count++;
+//                            pthread_mutex_unlock(&CountDropListMutex);
+
+			      __atomic_add_fetch(&counters->droplist_count, 1, __ATOMIC_SEQ_CST);
+
 
                         }
                 }

src/input-json.c

@@ -68,7 +68,7 @@ void SyslogInput_JSON( char *syslog_string, struct _SyslogInput *SyslogInput )
     SyslogInput->syslog_date = "UNDEFINED";
     SyslogInput->syslog_time = "UNDEFINED";
     SyslogInput->syslog_program = "UNDEFINED";
-    SyslogInput->syslog_msg = "UNDEFINED";
+    SyslogInput->syslog_message = "UNDEFINED";
 
     json_obj = json_tokener_parse(syslog_string);
 
@@ -123,7 +123,7 @@ void SyslogInput_JSON( char *syslog_string, struct _SyslogInput *SyslogInput )
     if ( json_object_object_get_ex(json_obj, Syslog_JSON_Map->syslog_map_message, &tmp))
         {
             strlcpy(syslog_message,  json_object_get_string(tmp), sizeof(syslog_message));
-            SyslogInput->syslog_msg = syslog_message;
+            SyslogInput->syslog_message = syslog_message;
         }
 
 

src/input-pipe.c

@@ -39,7 +39,6 @@ struct _SaganConfig *config;
 struct _SaganDNSCache *dnscache;
 
 void SyslogInput_Pipe( char *syslog_string, struct _SyslogInput *SyslogInput )
-
 {
 
     bool dns_flag;
@@ -243,12 +242,12 @@ void SyslogInput_Pipe( char *syslog_string, struct _SyslogInput *SyslogInput )
                 }
         }
 
-    SyslogInput->syslog_msg = syslog_string != NULL ? strsep(&syslog_string, "") : NULL; /* In case the message has | in it,  we delimit on "" */
+    SyslogInput->syslog_message = syslog_string != NULL ? strsep(&syslog_string, "") : NULL; /* In case the message has | in it,  we delimit on "" */
 
-    if (  SyslogInput->syslog_msg == NULL )
+    if (  SyslogInput->syslog_message == NULL )
         {
 
-            SyslogInput->syslog_msg = "SAGAN: MESSAGE ERROR";
+            SyslogInput->syslog_message = "SAGAN: MESSAGE ERROR";
 
             counters->malformed_message++;
 
@@ -264,11 +263,11 @@ void SyslogInput_Pipe( char *syslog_string, struct _SyslogInput *SyslogInput )
 
         }
 
-    /* Strip any \n or \r from the syslog_msg */
+    /* Strip any \n or \r from the syslog_message */
 
-    if ( strcspn (  SyslogInput->syslog_msg, "\n" ) < strlen( SyslogInput->syslog_msg) )
+    if ( strcspn ( SyslogInput->syslog_message, "\n" ) < strlen( SyslogInput->syslog_message ) )
         {
-            SyslogInput->syslog_msg[strcspn (  SyslogInput->syslog_msg, "\n" )] = '\0';
+            SyslogInput->syslog_message[strcspn (  SyslogInput->syslog_message, "\n" )] = '\0';
         }
 
 

src/ipc.c

@@ -58,7 +58,7 @@ struct _Sagan_IPC_Xbit *xbit_ipc;
 
 struct _SaganConfig *config;
 
-pthread_mutex_t CounterMutex;
+//pthread_mutex_t CounterMutex;
 pthread_mutex_t After2_Mutex;
 pthread_mutex_t Thresh2_Mutex;
 pthread_mutex_t Xbit_Mutex;

src/liblognormalize.c

@@ -48,7 +48,7 @@ struct _SaganDebug *debug;
 
 struct _SaganNormalizeLiblognorm *SaganNormalizeLiblognorm = NULL;
 
-pthread_mutex_t Lognorm_Mutex = PTHREAD_MUTEX_INITIALIZER;
+//pthread_mutex_t Lognorm_Mutex = PTHREAD_MUTEX_INITIALIZER;
 
 /************************************************************************
  * liblognorm GLOBALS