日志支持syslog

CONTRIBUTING.md

@@ -44,6 +44,7 @@ Switched to a new branch 'devel'
 $ git branch --set-upstream-to=origin/devel devel
 Branch devel set up to track remote branch devel from origin.
 $ git pull 
+$ git pull upstream devel
 ```
 
 ## 创建自己的分支
@@ -89,5 +90,6 @@ $ git push origin feature-contribute-doc
 
 PR被关闭后，删除分支
 ```bash
-$ git branch -d feature-contribute-doc
+$ git branch -D feature-contribute-doc
+$ git push origin --delete feature-contribute-doc
 ```

README.md

@@ -17,7 +17,7 @@ DDOS检测功能包括:
 ## IDPS
 IDPS模块基于[Suricata](https://github.com/OISF/suricata)，并新增了如下特性，
 * 支持lib化编译安装，基于Suricata 4.1.0版本
-* 支持事件以Kafka方式输出，提升了事件吞吐量，便于进一步数据分析
+* 支持事件以Kafka方式输出，提升事件吞吐量，便于进一步数据分析
 
 ## 部署描述
 ![deploy.JPG](./resources/pic/deploy.JPG)
@@ -71,7 +71,6 @@ Centos：
 ```bash
 $ yum install -y libpcap-devel pcre-devel file-devel libyaml-devel jansson-devel libcap-ng-devel librdkafka-devel nss-devel nspr-devel make gcc
 $ yum install -y libxml2-devel
-$ ln -s /usr/include/libxml2/libxml /usr/include/libxml
 $ yum install -y  python-pip
 $ pip install configparser
 ```
@@ -90,7 +89,7 @@ $ tar vxf dpdk-16.11.2.tar.xz
 ### DPDK 编译安装
 
 ```bash
-$ cd dpdk-16.11.2
+$ cd dpdk-stable-16.11.2
 $ export RTE_SDK=`pwd`
 $ export RTE_TARGET=x86_64-native-linuxapp-gcc
 $ make install T=${RTE_TARGET} DESTDIR=install
@@ -132,6 +131,7 @@ HugePages_Rsvd:        0
 HugePages_Surp:        0
 Hugepagesize:      '2048 kB'
 ```
+ens7f0（0000:06:00.0）和ens7f1（0000:06:00.0）网卡驱动已经绑定DPDK驱动。
 
 ### 编译安装IDPS lib
 
@@ -141,9 +141,9 @@ Hugepagesize:      '2048 kB'
 ```bash
 $ cd scripts
 $ sh build_idps.sh
-$ ls /usr/local/lib
+$ ls /usr/local/lib | egrep 'suri|htp'
 libhtp.a  libhtp.la  libhtp.so  libhtp.so.2  libhtp.so.2.0.0  libsuri.a  libsuri.la  libsuri.so  libsuri.so.0  libsuri.so.0.0.0 
-$ ls /usr/local/include/
+$ ls /usr/local/include/ | egrep 'suri|htp'
 htp  suricata
 ```
 
@@ -155,7 +155,16 @@ $ sh build_qnsm_lib.sh
 $ ll $RTE_SDK/$RTE_TARGET/lib/libqnsm_service.a
 ```
 
-编译qnsm主程序。支持编译成debug或者release版本，默认release版本。debug版本提供一些调试命令用于展示运行时数据。
+支持编译成debug或者release版本，默认release版本。
+
+debug版本提供一些调试命令用于展示运行时数据。如果需要编译debug版本，执行以下命令。
+```
+$ cat ../config
+CONFIG_QNSM_LIBQNSM_IDPS=y
+CONFIG_DEBUG_QNSM=n
+$ sed -i '/CONFIG_DEBUG_QNSM/s/=n/=y/g' ../config
+```
+编译qnsm主程序。
 ```
 $ cd ..
 $ make
@@ -170,14 +179,42 @@ ddos、idps、ddos-idps是qnsm支持的三种部署形态，默认以ddos-idps
 启动QNSM之前，需要依据[`配置手册`](./doc/configure-tutorial.md)修改`/var/qnsm`安装目录下的配置文件。 
 
 ## 启动 QNSM
+QNSM日志支持syslog输出，相关配置如下，可以参考[`配置手册`](./doc/configure-tutorial.md)。
+```
+$ cat /var/qnsm/qnsm_edge.xml 
+<CONFIG>
+...
+    <log>
+            <syslog>
+                    <facility>local5</facility>
+                    <log-level>Critical</log-level>
+            </syslog>
+    </log>
+</CONFIG>
+```
+
+修改syslog配置，日志存储在/var/log/qnsm目录下。
+```
+$ mkdir -p /var/log/qnsm
+$ echo "local5.*                                                /var/log/qnsm/qnsm.log" >> /etc/rsyslog.d/qnsm.conf
+$ systemctl restart rsyslog.service
+$ cp -f conf/qnsm.logrotate /etc/logrotate.d
+$ logrotate /etc/logrotate.conf
+```
+
+创建suricata.yaml配置文件中的目录列表，包括规则文件目录，日志目录等。
+```
+$ mkdir -p /var/log/suricata
+```
 
+启动QNSM。
 ```bash
 $ cd /var/qnsm
-$ ./qnsm-inspect -f qnsm_inspect.cfg -c . -p 1
+$ ./qnsm-inspect -f qnsm_inspect.cfg -c . -p 3
 ```
 * -f 参数指定组件配置文件
 * -c 参数指定配置文件目录
-* -p 参数指定使用网卡ID的16进制掩码，如果有两张网卡，该值为3（0011），依此类推
+* -p 参数指定使用网卡ID的16进制掩码，如果有两张网卡，该值为3（0b0011），依此类推
 
 另外，可以编写多个部署配置文件（qnsm_inspect_x.cfg），这样的话，可以启动多个QNSM进程。
 
@@ -215,11 +252,11 @@ $ cat qnsm_log_qnsm | grep 'master cmd'
 QNSM: 1557921514 master cmd msg {"id":0,"op":"ip_dump_pkt_enable","content":[{"idc":"idc_aaa","proto":"any","vip":"11.22.33.44","vport":"any"}]}
 ```
 
-检查pcap文件，可以存储在本地或者云端。
+检查pcap文件，默认存储在运行目录下面的dump目录，支持配置修改存储目录。
 ```bash
-$ cd /data
-$ tail -n 20 dump_oss.log 
-2019-05-15 Wednesday 19:59:00 INFO dump_oss.py 168 26239 close write file: /data/ads-monitor-mirror/log/11.22.33.44-core4-20190515-1958.pcap 
+$ ls dump
+xx.xx.xx.xx-core5-20191018-1522.pcap
+$ 
 ```
 
 ### IDPS事件
@@ -284,7 +321,7 @@ $ tail -n 20 dump_oss.log
 
 数据包吞吐可以线性增长，但是瓶颈存在于压力最大的那个组件。
 
-在我们的测试环境中, 开启超线程， DDOS检测和IDPS混合部署，[performance](doc/performance.md)说明了测试方法和数据。
+在我们的测试环境中, 开启超线程， DDOS检测和IDPS混合部署，[performance](doc/performance.md)包含测试方法和数据。
 
 # 版权说明
 

conf/ddos-idps/suricata.yaml

@@ -119,7 +119,7 @@ threshold-file: /usr/local/etc/suricata/threshold.config
 # The default logging directory.  Any log or output file will be
 # placed here if its not specified with a full path name. This can be
 # overridden with the -l command line parameter.
-default-log-dir: /usr/local/var/log/suricata/
+default-log-dir: /var/log/suricata/
 
 # global stats configuration
 stats:
@@ -143,7 +143,7 @@ outputs:
       kafka:
         brokers: > 
          xxxx:port1,
-         yyyy:port1,
+         yyyy:port1
         topic: nsm_event
         partitions: 5
       payload-hex: yes
@@ -536,7 +536,7 @@ logging:
   - file:
       enabled: yes
       level: info
-      filename: /usr/local/var/log/suricata/suricata.log
+      filename: /var/log/suricata/suricata.log
       # type: json
   - syslog:
       enabled: no
@@ -965,7 +965,7 @@ sensor-name: dc_name:ip-1
 # Default location of the pid file. The pid file is only used in
 # daemon mode (start Suricata with -D). If not running in daemon mode
 # the --pidfile command line option must be used to create a pid file.
-#pid-file: /usr/local/var/run/suricata.pid
+#pid-file: /var/run/suricata.pid
 
 # Daemon working directory
 # Suricata will change directory to this one if provided

conf/idps/suricata.yaml

@@ -119,7 +119,7 @@ threshold-file: /usr/local/etc/suricata/threshold.config
 # The default logging directory.  Any log or output file will be
 # placed here if its not specified with a full path name. This can be
 # overridden with the -l command line parameter.
-default-log-dir: /usr/local/var/log/suricata/
+default-log-dir: /var/log/suricata/
 
 # global stats configuration
 stats:
@@ -536,7 +536,7 @@ logging:
   - file:
       enabled: yes
       level: info
-      filename: /usr/local/var/log/suricata/suricata.log
+      filename: /var/log/suricata/suricata.log
       # type: json
   - syslog:
       enabled: no
@@ -965,7 +965,7 @@ sensor-name: dc_name:ip-1
 # Default location of the pid file. The pid file is only used in
 # daemon mode (start Suricata with -D). If not running in daemon mode
 # the --pidfile command line option must be used to create a pid file.
-#pid-file: /usr/local/var/run/suricata.pid
+#pid-file: /var/run/suricata.pid
 
 # Daemon working directory
 # Suricata will change directory to this one if provided

conf/qnsm.logrotate

@@ -0,0 +1,4 @@
+/var/log/qnsm/qnsm.log
+{
+	missingok
+}

conf/qnsm_edge.xml

@@ -75,5 +75,11 @@
             <broker>yyy:9092</broker>
         </brokers>
     </kafka>
+	<log>
+		<syslog>
+			<facility>local5</facility>
+			<log-level>info</log-level>
+		</syslog>
+	</log>
 </CONFIG>
 

doc/configure-tutorial.md

@@ -69,6 +69,13 @@
     		<broker>yyyy:9092</broker>
     	</brokers>
     </kafka>
+    <log>											<!-- syslog 配置 -->
+        <syslog>
+                <facility>local5</facility>
+                <log-level>Critical</log-level>				<!-- Emergency/Alert/Critical/Error/Warning/Notice/Info/Debug，级别递增 -->
+        </syslog>
+    </log>
+	<dump-dir>/data/qnsm</dump-dir>					<!-- pcap文件存储目录 -->
 </CONFIG>
 ```
 

include/qnsm_cfg.h

@@ -216,6 +216,10 @@ typedef struct qnsm_edge_cfg {
 } QNSM_EDGE_CFG;
 #endif
 
+typedef struct qnsm_dump_cfg {
+    char *dump_dir;
+} QNSM_DUMP_CFG;
+
 #if QNSM_PART("cmd")
 typedef struct {
     cmdline_parse_ctx_t ctx[16];
@@ -230,6 +234,7 @@ QNSM_PROTO_CFG* qnsm_get_proto_conf(const char *name);
 int32_t qnsm_cmd_init(void **tbl_handle);
 int qnsm_conf_parse(void);
 inline QNSM_EDGE_CFG* qnsm_get_edge_conf(void);
+inline QNSM_DUMP_CFG* qnsm_get_dump_conf(void);
 inline QNSM_SVR_IP_GROUP* qnsm_get_group(uint32_t group_id);
 inline uint16_t qnsm_group_num(void);
 inline uint32_t qnsm_group_is_valid(uint32_t group_id);

include/util.h

@@ -18,10 +18,14 @@
 #ifndef __UTIL__
 #define __UTIL__
 
+#include <rte_common.h>
+#include <rte_log.h>
+
 #include <stdio.h>
 #include <stdlib.h>
 #include <inttypes.h>
 #include <sys/types.h>
+#include <sys/syslog.h>
 
 
 #define QNSM_DDOS_MEM_ALIGN (RTE_CACHE_LINE_SIZE)
@@ -92,22 +96,65 @@ uint64_t get_diff_time(uint64_t time_now, uint64_t prev_time);
 
 #define PREFETCH_OFFSET 4
 
-/* ctrl conditions */
-#define QnsmCtrlCondT pthread_cond_t
-#define QnsmCtrlCondInit pthread_cond_init
-#define QnsmCtrlCondSignal pthread_cond_signal
-#define QnsmCtrlCondTimedwait pthread_cond_timedwait
-#define QnsmCtrlCondWait pthread_cond_wait
-#define QnsmCtrlCondDestroy pthread_cond_destroy
-
-/* ctrl mutex */
-#define QnsmCtrlMutex pthread_mutex_t
-#define QnsmCtrlMutexAttr pthread_mutexattr_t
-#define QnsmCtrlMutexInit(mut, mutattr ) pthread_mutex_init(mut, mutattr)
-#define QnsmCtrlMutexLock(mut) pthread_mutex_lock(mut)
-#define QnsmCtrlMutexTrylock(mut) pthread_mutex_trylock(mut)
-#define QnsmCtrlMutexUnlock(mut) pthread_mutex_unlock(mut)
-#define QnsmCtrlMutexDestroy pthread_mutex_destroy
+enum en_qnsm_log_type {
+    EN_QNSM_LOG_RTE = 0,
+    EN_QNSM_LOG_SYSLOG,
+    EN_QNSM_LOG_MAX,
+};
+
+enum {
+    QNSM_LOG_NOTSET = -1,
+    QNSM_LOG_NONE = 0,
+    QNSM_LOG_EMERG,
+    QNSM_LOG_ALERT,
+    QNSM_LOG_CRIT,
+    QNSM_LOG_ERR,
+    QNSM_LOG_WARNING,
+    QNSM_LOG_NOTICE,
+    QNSM_LOG_INFO,
+    QNSM_LOG_DEBUG,
+    QNSM_LOG_LEVEL_MAX,
+};
 
+typedef struct qnsm_log_cfg {
+    enum en_qnsm_log_type type;
+    int log_level;
+
+    /*file log conf*/
+    struct {
+        char *log_dir;
+        char *log_level;
+    } file_log_conf;
+
+    /*syslog conf*/
+    struct {
+        uint8_t enabled;
+        char *facility;
+        char *log_level;
+    } sys_log_conf;
+} QNSM_LOG_CFG;
+
+inline QNSM_LOG_CFG* qnsm_get_log_conf(void);
+
+#define QNSM_LOG(level, format, ...)\
+{\
+    switch (qnsm_get_log_conf()->type) {\
+        case EN_QNSM_LOG_RTE: {\
+            RTE_LOG(level, QNSM, "%" PRIu64 " - (%s:%d) <%s> "format, \
+                jiffies(), __FILE__, __LINE__, #level, ##__VA_ARGS__);\
+            break;\
+        }\
+        case EN_QNSM_LOG_SYSLOG: {\
+            if (qnsm_get_log_conf()->log_level >= QNSM_LOG_##level) {\
+                syslog(LOG_##level, "%" PRIu64 " - (%s:%d) <%s> "format, \
+                    jiffies(), __FILE__, __LINE__, #level, ##__VA_ARGS__);\
+            }\
+            break;\
+        }\
+        case EN_QNSM_LOG_MAX: {\
+            break;\
+        }\
+    }\
+}
 
 #endif

libqnsm_service/qnsm_acl.c

@@ -92,7 +92,7 @@ int qnsm_acl_tbl_add_bulk(
 #ifdef  DEBUG_QNSM
     if (0 == ret) {
         for (index = 0; index < n_keys; index++) {
-            RTE_LOG(CRIT, QNSM, "add acl rule %u action %d\n",
+            QNSM_LOG(INFO, "add acl rule %u action %d\n",
                     index,
                     ((QNSM_ACL_ENTRY *)(entries_ptr_arr[index]))->act);
         }
@@ -217,7 +217,7 @@ int qnsm_acl_init(void **tbl_handle)
 
     tbl_hdl = rte_zmalloc_socket("ACL", sizeof(QNSM_ACL_HANDLE), QNSM_DDOS_MEM_ALIGN, rte_socket_id());
     if (NULL == tbl_hdl) {
-        RTE_LOG(CRIT, QNSM, "acl handle init failed\n");
+        QNSM_LOG(ERR, "acl handle init failed\n");
         return -1;
     }
     memset(tbl_hdl, 0, sizeof(QNSM_ACL_HANDLE));

libqnsm_service/qnsm_msg.c

@@ -285,7 +285,7 @@ int32_t qnsm_msg_publish(void)
     msg->act_head.pub_lcore = lcore_id;
     msg->value_len = 0;
 
-    RTE_LOG(CRIT, QNSM, "[INFO] lcore %d send pub msg to crm\n", lcore_id);
+    QNSM_LOG(INFO, "lcore %d send pub msg to crm\n", lcore_id);
     qnsm_crm_agent_msg_send(msg);
     return 0;
 }
@@ -314,7 +314,7 @@ int32_t qnsm_msg_subscribe(uint32_t target_lcore_id)
     msg->act_head.pub_lcore = target_lcore_id;
     msg->value_len = 0;
 
-    RTE_LOG(CRIT, QNSM, "[INFO] lcore %d send sub lcore %d msg to crm\n", lcore_id, target_lcore_id);
+    QNSM_LOG(INFO, "lcore %d send sub lcore %d msg to crm\n", lcore_id, target_lcore_id);
     qnsm_crm_agent_msg_send(msg);
 
     return 0;