Permalink
Browse files

Restructured scan options flags from a single bitflag field to a stru…

…cture containing multiple bitflag fields. This also required adding a new function to the bytecode API to get scan options a la carte, and modifying the existing function to hand back scan options in the old/deprecated uint32_t bitflag format. Re-generated bytecode iface header files.

Updated libclamav documentation detailing new scan options structure.
Renamed references to 'algorithmic' detection to 'heuristic' detection. Renaming references to 'properties' to 'collect metadata'.
Renamed references to 'scan all' to 'scan all match'.
Renamed a couple of 'Hueristic.*' signature names as 'Heuristics.*' signatures (plural) to match majority of other heuristics.
  • Loading branch information...
micah-at-talos authored and Micah committed Jul 21, 2018
1 parent 1158b28 commit 048a88e61558726bd9ba66ec3195b63d61d8a430
Showing with 1,379 additions and 805 deletions.
  1. +5 −5 NEWS.md
  2. +1 −0 clamconf/clamconf.c
  3. +27 −9 clamd/onaccess_ddd.c
  4. +9 −4 clamd/onaccess_ddd.h
  5. +19 −7 clamd/onaccess_fan.c
  6. +1 −1 clamd/onaccess_others.c
  7. +2 −1 clamd/onaccess_others.h
  8. +20 −4 clamd/onaccess_scth.c
  9. +6 −2 clamd/onaccess_scth.h
  10. +17 −6 clamd/scanner.c
  11. +4 −3 clamd/scanner.h
  12. +57 −38 clamd/server-th.c
  13. +1 −11 clamd/server.h
  14. +2 −2 clamd/session.c
  15. +1 −1 clamd/session.h
  16. +43 −39 clamscan/manager.c
  17. +4 −1 contrib/old-clamav-milter/clamav-milter.c
  18. +88 −43 docs/UserManual/libclamav.md
  19. +1 −1 etc/clamd.conf.sample
  20. +6 −1 examples/ex1.c
  21. +6 −6 libclamav/7z_iface.c
  22. +4 −4 libclamav/apm.c
  23. +2 −2 libclamav/autoit.c
  24. +3 −2 libclamav/blob.c
  25. +4 −4 libclamav/bytecode.c
  26. +192 −1 libclamav/bytecode_api.c
  27. +85 −2 libclamav/bytecode_api.h
  28. +126 −123 libclamav/bytecode_api_decl.c
  29. +2 −1 libclamav/bytecode_api_impl.h
  30. +1 −1 libclamav/bytecode_hooks.h
  31. +1 −1 libclamav/cache.c
  32. +74 −61 libclamav/clamav.h
  33. +1 −1 libclamav/clambc.h
  34. +6 −6 libclamav/cpio.c
  35. +1 −1 libclamav/dmg.c
  36. +12 −12 libclamav/elf.c
  37. +10 −10 libclamav/gpt.c
  38. +4 −4 libclamav/hfsplus.c
  39. +22 −22 libclamav/hwp.c
  40. +1 −1 libclamav/ishield.c
  41. +2 −2 libclamav/iso9660.c
  42. +1 −1 libclamav/json_api.c
  43. +4 −4 libclamav/libmspack.c
  44. +1 −1 libclamav/macho.c
  45. +4 −4 libclamav/matcher-ac.c
  46. +2 −2 libclamav/matcher-bm.c
  47. +1 −1 libclamav/matcher-pcre.c
  48. +18 −18 libclamav/matcher.c
  49. +3 −4 libclamav/mbox.c
  50. +9 −9 libclamav/mbr.c
  51. +9 −9 libclamav/msxml_parser.c
  52. +7 −7 libclamav/ole2_extract.c
  53. +8 −8 libclamav/others.c
  54. +32 −19 libclamav/others.h
  55. +49 −25 libclamav/pdf.c
  56. +2 −1 libclamav/pdfdecode.c
  57. +21 −21 libclamav/pe.c
  58. +3 −3 libclamav/phishcheck.c
  59. +150 −145 libclamav/scanners.c
  60. +1 −1 libclamav/tiff.c
  61. +2 −2 libclamav/untar.c
  62. +5 −5 libclamav/unzip.c
  63. +4 −4 libclamav/xar.c
  64. +15 −3 sigtool/sigtool.c
  65. +5 −1 unit_tests/check_bytecode.c
  66. +67 −13 unit_tests/check_clamav.c
  67. +15 −3 unit_tests/check_matchers.c
  68. +7 −1 unit_tests/check_regex.c
  69. +60 −43 win32/clamav-for-windows/clamav-for-windows/interface.c
  70. +1 −1 win32/conf_examples/clamd.conf.sample
View
10 NEWS.md
@@ -86,7 +86,7 @@ ClamAV 0.100, including but not limited to:
- Raw scanning of PostScript files.
- Fix clamsubmit to use the new virus and false positive submission web
interface.
- Optionally, flag files with the virus "Heuristic.Limits.Exceeded" when
- Optionally, flag files with the virus "Heuristics.Limits.Exceeded" when
size limitations are exceeded.
- Improved decoders for PDF files.
- Reduced number of compile time warnings.
@@ -1147,7 +1147,7 @@ Detailed list of changes:
- Support for Sensory Networks' NodalCore hardware acceleration technology
- Advanced phishing detection module (experimental)
- Signatures are stored in separate trees depending on their target type
- Algorithmic detection can be controlled with CL_SCAN_ALGORITHMIC
- Algorithmic detection can be controlled with CL_SCAN_GENERAL_HEURISTICS
- Support for new obfuscators: SUE, Y0da Cryptor, CryptFF
- Support for new packers: NsPack, wwpack32, MEW, Upack
- Support for SIS files (SymbianOS packages)
@@ -1598,11 +1598,11 @@ the new version of ClamAV have detected and blocked 100% of Mydoom attacks!
New features in this release include:
- libclamav
- Portable Executable analyser (CL_SCAN_PE) featuring:
- Portable Executable analyser (CL_SCAN_PARSE_PE) featuring:
- UPX decompression (all versions)
- Petite decompression (2.x)
- FSG decompression (1.3, 1.31, 1.33)
- detection of broken executables (CL_SCAN_BLOCKBROKEN)
- detection of broken executables (CL_SCAN_HEURISTIC_BROKEN)
- new, memory efficient, pattern matching algorithm (multipattern variant
of Boyer-Moore) - it's now primary matcher and Aho-Corasick is only used
for regular expression extended signatures
@@ -1618,7 +1618,7 @@ New features in this release include:
- new method of mail files detection
- all e-mail attachments are now scanned (previously only the first ten
attachments were scanned)
- added support for scanning URLs in e-mails (CL_SCAN_MAILURL)
- added support for scanning URLs in e-mails (CL_SCAN_PARSE_MAILURL)
- detection of Worm.Mydoom.M.log
- updated API (still backward compatible but please consult clamdoc.pdf
(Section 6) and adapt your software)
View
@@ -39,6 +39,7 @@
#include "shared/optparser.h"
#include "shared/misc.h"
#include "clamav-config.h"
#include "libclamav/str.h"
#include "libclamav/clamav.h"
#include "libclamav/others.h"
View
@@ -68,7 +68,7 @@ static void onas_ddd_handle_in_moved_to(struct ddd_thrarg *tharg, const char *pa
static void onas_ddd_handle_in_create(struct ddd_thrarg *tharg, const char *path, const char *child_path, const struct inotify_event *event, int wd, uint64_t in_mask);
static void onas_ddd_handle_in_moved_from(struct ddd_thrarg *tharg, const char *path, const char *child_path, const struct inotify_event *event, int wd);
static void onas_ddd_handle_in_delete(struct ddd_thrarg *tharg, const char *path, const char *child_path, const struct inotify_event *event, int wd);
static void onas_ddd_handle_extra_scanning(struct ddd_thrarg *tharg, const char *pathname, int options);
static void onas_ddd_handle_extra_scanning(struct ddd_thrarg *tharg, const char *pathname, int extra_options);
static void onas_ddd_exit(int sig);
@@ -527,8 +527,9 @@ static void onas_ddd_handle_in_moved_to(struct ddd_thrarg *tharg,
return;
}
static void onas_ddd_handle_extra_scanning(struct ddd_thrarg *tharg, const char *pathname, int options) {
static void onas_ddd_handle_extra_scanning(struct ddd_thrarg *tharg, const char *pathname, int extra_options) {
int thread_started = 1;
struct scth_thrarg *scth_tharg = NULL;
pthread_attr_t scth_attr;
pthread_t scth_pid = 0;
@@ -537,20 +538,37 @@ static void onas_ddd_handle_extra_scanning(struct ddd_thrarg *tharg, const char
if (pthread_attr_init(&scth_attr)) break;
pthread_attr_setdetachstate(&scth_attr, PTHREAD_CREATE_JOINABLE);
if (!(scth_tharg = (struct scth_thrarg *) malloc(sizeof(struct scth_thrarg)))) break;
/* Allocate memory for arguments. Thread is responsible for freeing it. */
if (!(scth_tharg = (struct scth_thrarg *) calloc(sizeof(struct scth_thrarg), 1))) break;
if (!(scth_tharg->options = (struct cl_scan_options *) calloc(sizeof(struct cl_scan_options), 1))) break;
scth_tharg->options = options;
(void) memcpy(scth_tharg->options, tharg->options, sizeof(struct cl_scan_options));
scth_tharg->extra_options = extra_options;
scth_tharg->opts = tharg->opts;
scth_tharg->pathname = strdup(pathname);
scth_tharg->engine = tharg->engine;
if (!pthread_create(&scth_pid, &scth_attr, onas_scan_th, scth_tharg)) break;
free(scth_tharg);
scth_tharg = NULL;
thread_started = pthread_create(&scth_pid, &scth_attr, onas_scan_th, scth_tharg);
} while(0);
if (!scth_tharg) logg("!ScanOnAccess: Unable to kick off extra scanning.\n");
if (0 != thread_started) {
/* Failed to create thread. Free anything we may have allocated. */
logg("!ScanOnAccess: Unable to kick off extra scanning.\n");
if (NULL != scth_tharg) {
if (NULL != scth_tharg->pathname){
free(scth_tharg->pathname);
scth_tharg->pathname = NULL;
}
if (NULL != scth_tharg->options) {
free(scth_tharg->options);
scth_tharg->options = NULL;
}
free(scth_tharg);
scth_tharg = NULL;
}
}
return;
}
View
@@ -22,19 +22,24 @@
#ifndef __ONAS_IN_H
#define __ONAS_IN_H
#define ONAS_IN 0x01
#define ONAS_FAN 0x02
#include "shared/optparser.h"
#include "libclamav/clamav.h"
/*
* Extra options for onas_scan_th().
*/
#define ONAS_IN 0x01
#define ONAS_FAN 0x02
#define MAX_WATCH_LEN 7
struct ddd_thrarg {
int sid;
int options;
struct cl_scan_options *options;
int fan_fd;
uint64_t fan_mask;
const struct optstruct *opts;
const struct cl_engine *engine;
const struct cl_limits *limits;
};
View
@@ -171,24 +171,36 @@ void *onas_fan_th(void *arg)
}
} else if (!optget(tharg->opts, "OnAccessDisableDDD")->enabled) {
int thread_started = 1;
do {
if(pthread_attr_init(&ddd_attr)) break;
pthread_attr_setdetachstate(&ddd_attr, PTHREAD_CREATE_JOINABLE);
if(!(ddd_tharg = (struct ddd_thrarg *) malloc(sizeof(struct ddd_thrarg)))) break;
/* Allocate memory for arguments. Thread is responsible for freeing it. */
if (!(ddd_tharg = (struct ddd_thrarg *) calloc(sizeof(struct ddd_thrarg), 1))) break;
if (!(ddd_tharg->options = (struct cl_scan_options *) calloc(sizeof(struct cl_scan_options), 1))) break;
(void) memcpy(ddd_tharg->options, tharg->options, sizeof(struct cl_scan_options));
ddd_tharg->fan_fd = onas_fan_fd;
ddd_tharg->fan_mask = fan_mask;
ddd_tharg->opts = tharg->opts;
ddd_tharg->engine = tharg->engine;
ddd_tharg->options = tharg->options;
if(!pthread_create(&ddd_pid, &ddd_attr, onas_ddd_th, ddd_tharg)) break;
free(ddd_tharg);
ddd_tharg=NULL;
thread_started = pthread_create(&ddd_pid, &ddd_attr, onas_ddd_th, ddd_tharg);
} while(0);
if (!ddd_tharg) logg("!Unable to start dynamic directory determination.\n");
if (0 != thread_started) {
/* Failed to create thread. Free anything we may have allocated. */
logg("!Unable to start dynamic directory determination.\n");
if (NULL != ddd_tharg) {
if (NULL != ddd_tharg->options) {
free(ddd_tharg->options);
ddd_tharg->options = NULL;
}
free(ddd_tharg);
ddd_tharg = NULL;
}
}
} else {
if((pt = optget(tharg->opts, "OnAccessIncludePath"))->enabled) {
View
@@ -94,7 +94,7 @@ int onas_fan_checkowner(int pid, const struct optstruct *opts)
return CHK_CLEAN;
}
int onas_scan(const char *fname, int fd, const char **virname, const struct cl_engine *engine, int options, int extinfo)
int onas_scan(const char *fname, int fd, const char **virname, const struct cl_engine *engine, struct cl_scan_options *options, int extinfo)
{
int ret = 0;
struct cb_context context;
View
@@ -22,6 +22,7 @@
#define __CLAMD_ONAS_OTHERS_H
#include "shared/optparser.h"
#include "libclamav/clamav.h"
typedef enum {
CHK_CLEAN,
@@ -30,6 +31,6 @@ typedef enum {
} cli_check_t;
int onas_fan_checkowner(int pid, const struct optstruct *opts);
int onas_scan(const char *fname, int fd, const char **virname, const struct cl_engine *engine, int options, int extinfo);
int onas_scan(const char *fname, int fd, const char **virname, const struct cl_engine *engine, struct cl_scan_options *options, int extinfo);
#endif
View
@@ -38,6 +38,7 @@
#include "priv_fts.h"
#include "onaccess_scth.h"
#include "onaccess_others.h"
#include "libclamav/clamav.h"
@@ -132,16 +133,31 @@ void *onas_scan_th(void *arg) {
sigaction(SIGUSR1, &act, NULL);
sigaction(SIGSEGV, &act, NULL);
if (tharg->options & ONAS_SCTH_ISDIR) {
if (NULL == tharg || NULL == tharg->pathname || NULL == tharg->opts || NULL == tharg->engine) {
logg("ScanOnAccess: Invalid thread arguments for extra scanning\n");
goto done;
}
if (tharg->extra_options & ONAS_SCTH_ISDIR) {
logg("*ScanOnAccess: Performing additional scanning on directory '%s'\n", tharg->pathname);
onas_scth_handle_dir(tharg->pathname, tharg);
} else if (tharg->options & ONAS_SCTH_ISFILE) {
} else if (tharg->extra_options & ONAS_SCTH_ISFILE) {
logg("*ScanOnAccess: Performing additional scanning on file '%s'\n", tharg->pathname);
onas_scth_handle_file(tharg->pathname, tharg);
}
free(tharg->pathname);
tharg->pathname = NULL;
done:
if (NULL != tharg->pathname){
free(tharg->pathname);
tharg->pathname = NULL;
}
if (NULL != tharg->options) {
free(tharg->options);
tharg->options = NULL;
}
if (NULL != tharg) {
free(tharg);
}
return NULL;
}
View
@@ -21,13 +21,17 @@
#ifndef __ONAS_SCTH_H
#define __ONAS_SCTH_H
#include "shared/optparser.h"
#include "libclamav/clamav.h"
#define ONAS_SCTH_ISDIR 0x01
#define ONAS_SCTH_ISFILE 0x02
struct scth_thrarg {
int options;
uint32_t extra_options;
struct cl_scan_options *options;
const struct optstruct *opts;
const struct cl_engine *engine;
const struct cl_engine *engine;
char *pathname;
};
View
@@ -117,7 +117,7 @@ void clamd_virus_found_cb(int fd, const char *virname, void *ctx)
if (d == NULL)
return;
if (!(d->options & CL_SCAN_ALLMATCHES) && !(d->options & CL_SCAN_HEURISTIC_PRECEDENCE))
if (!(d->options->general & CL_SCAN_GENERAL_ALLMATCHES) && !(d->options->general & CL_SCAN_GENERAL_HEURISTIC_PRECEDENCE))
return;
if (virname == NULL)
return;
@@ -277,7 +277,7 @@ int scan_callback(STATBUF *sb, char *filename, const char *msg, enum cli_ftw_rea
if (ret == CL_VIRUS) {
if (scandata->options & CL_SCAN_ALLMATCHES || (scandata->infected && scandata->options & CL_SCAN_HEURISTIC_PRECEDENCE)) {
if (scandata->options->general & CL_SCAN_GENERAL_ALLMATCHES || (scandata->infected && scandata->options->general & CL_SCAN_GENERAL_HEURISTIC_PRECEDENCE)) {
if(optget(scandata->opts, "PreludeEnable")->enabled){
prelude_logging(filename, virname, context.virhash, context.virsize);
}
@@ -353,9 +353,14 @@ int scan_pathchk(const char *path, struct cli_ftw_cbdata *data)
return 0;
}
int scanfd(const client_conn_t *conn, unsigned long int *scanned,
const struct cl_engine *engine,
unsigned int options, const struct optstruct *opts, int odesc, int stream)
int scanfd(
const client_conn_t *conn,
unsigned long int *scanned,
const struct cl_engine *engine,
struct cl_scan_options *options,
const struct optstruct *opts,
int odesc,
int stream)
{
int ret, fd = conn->scanfd;
const char *virname = NULL;
@@ -418,7 +423,13 @@ int scanfd(const client_conn_t *conn, unsigned long int *scanned,
return ret;
}
int scanstream(int odesc, unsigned long int *scanned, const struct cl_engine *engine, unsigned int options, const struct optstruct *opts, char term)
int scanstream(
int odesc,
unsigned long int *scanned,
const struct cl_engine *engine,
struct cl_scan_options *options,
const struct optstruct *opts,
char term)
{
int ret, sockfd, acceptd;
int tmpd, bread, retval, firsttimeout, timeout, btread;
View
@@ -24,6 +24,7 @@
#include <sys/types.h>
#include "libclamav/others.h"
#include "libclamav/clamav.h"
#include "shared/optparser.h"
#include "thrmgr.h"
@@ -42,7 +43,7 @@ struct scan_cb_data {
const client_conn_t *conn;
const char *toplevel_path;
unsigned long scanned;
unsigned int options;
struct cl_scan_options *options;
struct cl_engine *engine;
const struct optstruct *opts;
threadpool_t *thr_pool;
@@ -57,8 +58,8 @@ struct cb_context {
struct scan_cb_data *scandata;
};
int scanfd(const client_conn_t *conn, unsigned long int *scanned, const struct cl_engine *engine, unsigned int options, const struct optstruct *opts, int odesc, int stream);
int scanstream(int odesc, unsigned long int *scanned, const struct cl_engine *engine, unsigned int options, const struct optstruct *opts, char term);
int scanfd(const client_conn_t *conn, unsigned long int *scanned, const struct cl_engine *engine, struct cl_scan_options *options, const struct optstruct *opts, int odesc, int stream);
int scanstream(int odesc, unsigned long int *scanned, const struct cl_engine *engine, struct cl_scan_options *options, const struct optstruct *opts, char term);
int scan_callback(STATBUF *sb, char *filename, const char *msg, enum cli_ftw_reason reason, struct cli_ftw_cbdata *data);
int scan_pathchk(const char *path, struct cli_ftw_cbdata *data);
void hash_callback(int fd, unsigned long long size, const unsigned char *md5, const char *virname, void *ctx);
Oops, something went wrong.

0 comments on commit 048a88e

Please sign in to comment.