Restructured scan options flags from a single bitflag field to a stru…

…cture containing multiple bitflag fields. This also required adding a new function to the bytecode API to get scan options a la carte, and modifying the existing function to hand back scan options in the old/deprecated uint32_t bitflag format. Re-generated bytecode iface header files.

Updated libclamav documentation detailing new scan options structure.
Renamed references to 'algorithmic' detection to 'heuristic' detection. Renaming references to 'properties' to 'collect metadata'.
Renamed references to 'scan all' to 'scan all match'.
Renamed a couple of 'Hueristic.*' signature names as 'Heuristics.*' signatures (plural) to match majority of other heuristics.

NEWS.md

@@ -86,7 +86,7 @@ ClamAV 0.100, including but not limited to:
 - Raw scanning of PostScript files.
 - Fix clamsubmit to use the new virus and false positive submission web
   interface.
-- Optionally, flag files with the virus "Heuristic.Limits.Exceeded" when
+- Optionally, flag files with the virus "Heuristics.Limits.Exceeded" when
   size limitations are exceeded.
 - Improved decoders for PDF files.
 - Reduced number of compile time warnings.
@@ -1147,7 +1147,7 @@ Detailed list of changes:
   - Support for Sensory Networks' NodalCore hardware acceleration technology
   - Advanced phishing detection module (experimental)
   - Signatures are stored in separate trees depending on their target type
-  - Algorithmic detection can be controlled with CL_SCAN_ALGORITHMIC
+  - Algorithmic detection can be controlled with CL_SCAN_GENERAL_HEURISTICS
   - Support for new obfuscators: SUE, Y0da Cryptor, CryptFF
   - Support for new packers: NsPack, wwpack32, MEW, Upack
   - Support for SIS files (SymbianOS packages)
@@ -1598,11 +1598,11 @@ the new version of ClamAV have detected and blocked 100% of Mydoom attacks!
 New features in this release include:
 
 - libclamav
-  - Portable Executable analyser (CL_SCAN_PE) featuring:
+  - Portable Executable analyser (CL_SCAN_PARSE_PE) featuring:
   - UPX decompression (all versions)
   - Petite decompression (2.x)
   - FSG decompression (1.3, 1.31, 1.33)
-  - detection of broken executables (CL_SCAN_BLOCKBROKEN)
+  - detection of broken executables (CL_SCAN_HEURISTIC_BROKEN)
   - new, memory efficient, pattern matching algorithm (multipattern variant
     of Boyer-Moore) - it's now primary matcher and Aho-Corasick is only used
     for regular expression extended signatures
@@ -1618,7 +1618,7 @@ New features in this release include:
   - new method of mail files detection
   - all e-mail attachments are now scanned (previously only the first ten
     attachments were scanned)
-  - added support for scanning URLs in e-mails (CL_SCAN_MAILURL)
+  - added support for scanning URLs in e-mails (CL_SCAN_PARSE_MAILURL)
   - detection of Worm.Mydoom.M.log
   - updated API (still backward compatible but please consult clamdoc.pdf
     (Section 6) and adapt your software)

clamconf/clamconf.c

@@ -39,6 +39,7 @@
 #include "shared/optparser.h"
 #include "shared/misc.h"
 
+#include "clamav-config.h"
 #include "libclamav/str.h"
 #include "libclamav/clamav.h"
 #include "libclamav/others.h"

clamd/onaccess_ddd.c

@@ -68,7 +68,7 @@ static void onas_ddd_handle_in_moved_to(struct ddd_thrarg *tharg, const char *pa
 static void onas_ddd_handle_in_create(struct ddd_thrarg *tharg, const char *path, const char *child_path, const struct inotify_event *event, int wd, uint64_t in_mask);
 static void onas_ddd_handle_in_moved_from(struct ddd_thrarg *tharg, const char *path, const char *child_path, const struct inotify_event *event, int wd);
 static void onas_ddd_handle_in_delete(struct ddd_thrarg *tharg, const char *path, const char *child_path, const struct inotify_event *event, int wd);
-static void onas_ddd_handle_extra_scanning(struct ddd_thrarg *tharg, const char *pathname, int options);
+static void onas_ddd_handle_extra_scanning(struct ddd_thrarg *tharg, const char *pathname, int extra_options);
 
 static void onas_ddd_exit(int sig);
 
@@ -527,8 +527,9 @@ static void onas_ddd_handle_in_moved_to(struct ddd_thrarg *tharg,
 	return;
 }
 
-static void onas_ddd_handle_extra_scanning(struct ddd_thrarg *tharg, const char *pathname, int options) {
+static void onas_ddd_handle_extra_scanning(struct ddd_thrarg *tharg, const char *pathname, int extra_options) {
 
+	int thread_started = 1;
 	struct scth_thrarg *scth_tharg = NULL;
 	pthread_attr_t scth_attr;
 	pthread_t scth_pid = 0;
@@ -537,20 +538,37 @@ static void onas_ddd_handle_extra_scanning(struct ddd_thrarg *tharg, const char
 		if (pthread_attr_init(&scth_attr)) break;
 		pthread_attr_setdetachstate(&scth_attr, PTHREAD_CREATE_JOINABLE);
 
-		if (!(scth_tharg = (struct scth_thrarg *) malloc(sizeof(struct scth_thrarg)))) break;
+		/* Allocate memory for arguments. Thread is responsible for freeing it. */
+		if (!(scth_tharg = (struct scth_thrarg *) calloc(sizeof(struct scth_thrarg), 1))) break;
+		if (!(scth_tharg->options = (struct cl_scan_options *) calloc(sizeof(struct cl_scan_options), 1))) break;
 
-		scth_tharg->options = options;
+		(void) memcpy(scth_tharg->options, tharg->options, sizeof(struct cl_scan_options));
+
+		scth_tharg->extra_options = extra_options;
 		scth_tharg->opts = tharg->opts;
 		scth_tharg->pathname = strdup(pathname);
 		scth_tharg->engine = tharg->engine;
 
-		if (!pthread_create(&scth_pid, &scth_attr, onas_scan_th, scth_tharg)) break;
-
-		free(scth_tharg);
-		scth_tharg = NULL;
+		thread_started = pthread_create(&scth_pid, &scth_attr, onas_scan_th, scth_tharg);
 	} while(0);
-	if (!scth_tharg) logg("!ScanOnAccess: Unable to kick off extra scanning.\n");
 
+	if (0 != thread_started) {
+		/* Failed to create thread. Free anything we may have allocated. */
+		logg("!ScanOnAccess: Unable to kick off extra scanning.\n");
+		if (NULL != scth_tharg) {
+			if (NULL != scth_tharg->pathname){
+				free(scth_tharg->pathname);
+				scth_tharg->pathname = NULL;
+			}
+			if (NULL != scth_tharg->options) {
+				free(scth_tharg->options);
+				scth_tharg->options = NULL;
+			}
+			free(scth_tharg);
+			scth_tharg = NULL;
+		}
+	}
+
 	return;
 }
 

clamd/onaccess_ddd.h

@@ -22,19 +22,24 @@
 #ifndef __ONAS_IN_H
 #define __ONAS_IN_H
 
-#define ONAS_IN 0x01
-#define ONAS_FAN 0x02
+#include "shared/optparser.h"
+#include "libclamav/clamav.h"
+
+/*
+ * Extra options for onas_scan_th(). 
+ */
+#define ONAS_IN 	0x01
+#define ONAS_FAN 	0x02
 
 #define MAX_WATCH_LEN 7
 
 struct ddd_thrarg {
 	int sid;
-	int options;
+	struct cl_scan_options *options;
 	int fan_fd;
 	uint64_t fan_mask;
 	const struct optstruct *opts;
 	const struct cl_engine *engine;
-	const struct cl_limits *limits;
 };
 
 

clamd/onaccess_fan.c

@@ -171,24 +171,36 @@ void *onas_fan_th(void *arg)
 	    }
 
     } else if (!optget(tharg->opts, "OnAccessDisableDDD")->enabled) {
+		int thread_started = 1;
 	    do {
 		    if(pthread_attr_init(&ddd_attr)) break;
 		    pthread_attr_setdetachstate(&ddd_attr, PTHREAD_CREATE_JOINABLE);
 
-		    if(!(ddd_tharg = (struct ddd_thrarg *) malloc(sizeof(struct ddd_thrarg)))) break;
+			/* Allocate memory for arguments. Thread is responsible for freeing it. */
+		    if (!(ddd_tharg = (struct ddd_thrarg *) calloc(sizeof(struct ddd_thrarg), 1))) break;
+			if (!(ddd_tharg->options = (struct cl_scan_options *) calloc(sizeof(struct cl_scan_options), 1))) break;
 
+			(void) memcpy(ddd_tharg->options, tharg->options, sizeof(struct cl_scan_options));
 		    ddd_tharg->fan_fd = onas_fan_fd;
 		    ddd_tharg->fan_mask = fan_mask;
 		    ddd_tharg->opts = tharg->opts;
 		    ddd_tharg->engine = tharg->engine;
-		    ddd_tharg->options = tharg->options;
 
-		    if(!pthread_create(&ddd_pid, &ddd_attr, onas_ddd_th, ddd_tharg)) break;
-
-		    free(ddd_tharg);
-		    ddd_tharg=NULL;
+		    thread_started = pthread_create(&ddd_pid, &ddd_attr, onas_ddd_th, ddd_tharg);
 	    } while(0);
-	    if (!ddd_tharg) logg("!Unable to start dynamic directory determination.\n");
+
+		if (0 != thread_started) {
+			/* Failed to create thread. Free anything we may have allocated. */
+			logg("!Unable to start dynamic directory determination.\n");
+			if (NULL != ddd_tharg) {
+				if (NULL != ddd_tharg->options) {
+					free(ddd_tharg->options);
+					ddd_tharg->options = NULL;
+				}
+				free(ddd_tharg);
+				ddd_tharg = NULL;
+			}
+		}
 
     } else {
 	    if((pt = optget(tharg->opts, "OnAccessIncludePath"))->enabled) {

clamd/onaccess_others.c

@@ -94,7 +94,7 @@ int onas_fan_checkowner(int pid, const struct optstruct *opts)
     return CHK_CLEAN;
 }
 
-int onas_scan(const char *fname, int fd, const char **virname, const struct cl_engine *engine, int options, int extinfo)
+int onas_scan(const char *fname, int fd, const char **virname, const struct cl_engine *engine, struct cl_scan_options *options, int extinfo)
 {
     int ret = 0;
     struct cb_context context;

clamd/onaccess_others.h

@@ -22,6 +22,7 @@
 #define __CLAMD_ONAS_OTHERS_H
 
 #include "shared/optparser.h"
+#include "libclamav/clamav.h"
 
 typedef enum {
     CHK_CLEAN,
@@ -30,6 +31,6 @@ typedef enum {
 } cli_check_t;
 
 int onas_fan_checkowner(int pid, const struct optstruct *opts);
-int onas_scan(const char *fname, int fd, const char **virname, const struct cl_engine *engine, int options, int extinfo);
+int onas_scan(const char *fname, int fd, const char **virname, const struct cl_engine *engine, struct cl_scan_options *options, int extinfo);
 
 #endif

clamd/onaccess_scth.c

@@ -38,6 +38,7 @@
 #include "priv_fts.h"
 
 #include "onaccess_scth.h"
+#include "onaccess_others.h"
 
 #include "libclamav/clamav.h"
 
@@ -132,16 +133,31 @@ void *onas_scan_th(void *arg) {
 	sigaction(SIGUSR1, &act, NULL);
 	sigaction(SIGSEGV, &act, NULL);
 
-	if (tharg->options & ONAS_SCTH_ISDIR) {
+	if (NULL == tharg || NULL == tharg->pathname || NULL == tharg->opts || NULL == tharg->engine) {
+		logg("ScanOnAccess: Invalid thread arguments for extra scanning\n");
+		goto done;
+	}
+
+	if (tharg->extra_options & ONAS_SCTH_ISDIR) {
 		logg("*ScanOnAccess: Performing additional scanning on directory '%s'\n", tharg->pathname);
 		onas_scth_handle_dir(tharg->pathname, tharg);
-	} else if (tharg->options & ONAS_SCTH_ISFILE) {
+	} else if (tharg->extra_options & ONAS_SCTH_ISFILE) {
 		logg("*ScanOnAccess: Performing additional scanning on file '%s'\n", tharg->pathname);
 		onas_scth_handle_file(tharg->pathname, tharg);
 	}
 
-	free(tharg->pathname);
-	tharg->pathname = NULL;
+done:
+	if (NULL != tharg->pathname){
+		free(tharg->pathname);
+		tharg->pathname = NULL;
+	}
+	if (NULL != tharg->options) {
+		free(tharg->options);
+		tharg->options = NULL;
+	}
+	if (NULL != tharg) {
+		free(tharg);
+	}
 
 	return NULL;
 }

clamd/onaccess_scth.h

@@ -21,13 +21,17 @@
 #ifndef __ONAS_SCTH_H
 #define __ONAS_SCTH_H
 
+#include "shared/optparser.h"
+#include "libclamav/clamav.h"
+
 #define ONAS_SCTH_ISDIR  0x01
 #define ONAS_SCTH_ISFILE 0x02
 
 struct scth_thrarg {
-	int options;
+	uint32_t extra_options;
+	struct cl_scan_options *options;
 	const struct optstruct *opts;
-        const struct cl_engine *engine;
+	const struct cl_engine *engine;
 	char *pathname;
 };
 

clamd/scanner.c

@@ -117,7 +117,7 @@ void clamd_virus_found_cb(int fd, const char *virname, void *ctx)
 
     if (d == NULL)
         return;
-    if (!(d->options & CL_SCAN_ALLMATCHES) && !(d->options & CL_SCAN_HEURISTIC_PRECEDENCE))
+    if (!(d->options->general & CL_SCAN_GENERAL_ALLMATCHES) && !(d->options->general & CL_SCAN_GENERAL_HEURISTIC_PRECEDENCE))
         return;
     if (virname == NULL)
         return;
@@ -277,7 +277,7 @@ int scan_callback(STATBUF *sb, char *filename, const char *msg, enum cli_ftw_rea
 
     if (ret == CL_VIRUS) {
 
-         if (scandata->options & CL_SCAN_ALLMATCHES || (scandata->infected && scandata->options & CL_SCAN_HEURISTIC_PRECEDENCE)) {
+         if (scandata->options->general & CL_SCAN_GENERAL_ALLMATCHES || (scandata->infected && scandata->options->general & CL_SCAN_GENERAL_HEURISTIC_PRECEDENCE)) {
             if(optget(scandata->opts, "PreludeEnable")->enabled){
                 prelude_logging(filename, virname, context.virhash, context.virsize);
             }
@@ -353,9 +353,14 @@ int scan_pathchk(const char *path, struct cli_ftw_cbdata *data)
     return 0;
 }
 
-int scanfd(const client_conn_t *conn, unsigned long int *scanned,
-	   const struct cl_engine *engine,
-	   unsigned int options, const struct optstruct *opts, int odesc, int stream)
+int scanfd(
+	const client_conn_t *conn,
+	unsigned long int *scanned,
+	const struct cl_engine *engine,
+	struct cl_scan_options *options,
+	const struct optstruct *opts,
+	int odesc,
+	int stream)
 {
     int ret, fd = conn->scanfd;
 	const char *virname = NULL;
@@ -418,7 +423,13 @@ int scanfd(const client_conn_t *conn, unsigned long int *scanned,
 	return ret;
 }
 
-int scanstream(int odesc, unsigned long int *scanned, const struct cl_engine *engine, unsigned int options, const struct optstruct *opts, char term)
+int scanstream(
+	int odesc,
+	unsigned long int *scanned,
+	const struct cl_engine *engine,
+	struct cl_scan_options *options,
+	const struct optstruct *opts,
+	char term)
 {
 	int ret, sockfd, acceptd;
 	int tmpd, bread, retval, firsttimeout, timeout, btread;

clamd/scanner.h

@@ -24,6 +24,7 @@
 
 #include <sys/types.h>
 
+#include "libclamav/others.h"
 #include "libclamav/clamav.h"
 #include "shared/optparser.h"
 #include "thrmgr.h"
@@ -42,7 +43,7 @@ struct scan_cb_data {
     const client_conn_t *conn;
     const char *toplevel_path;
     unsigned long scanned;
-    unsigned int options;
+    struct cl_scan_options *options;
     struct cl_engine *engine;
     const struct optstruct *opts;
     threadpool_t *thr_pool;
@@ -57,8 +58,8 @@ struct cb_context {
     struct scan_cb_data *scandata;
 };
 
-int scanfd(const client_conn_t *conn, unsigned long int *scanned, const struct cl_engine *engine, unsigned int options, const struct optstruct *opts, int odesc, int stream);
-int scanstream(int odesc, unsigned long int *scanned, const struct cl_engine *engine, unsigned int options, const struct optstruct *opts, char term);
+int scanfd(const client_conn_t *conn, unsigned long int *scanned, const struct cl_engine *engine, struct cl_scan_options *options, const struct optstruct *opts, int odesc, int stream);
+int scanstream(int odesc, unsigned long int *scanned, const struct cl_engine *engine, struct cl_scan_options *options, const struct optstruct *opts, char term);
 int scan_callback(STATBUF *sb, char *filename, const char *msg, enum cli_ftw_reason reason, struct cli_ftw_cbdata *data);
 int scan_pathchk(const char *path, struct cli_ftw_cbdata *data);
 void hash_callback(int fd, unsigned long long size, const unsigned char *md5, const char *virname, void *ctx);