Merge remote-tracking branch 'upstream/master' into unary_operation_d…

…efined

# Conflicts:
#	docs/writingrules.rst
#	libyara/grammar.c
#	libyara/grammar.h
#	libyara/include/yara/exec.h
#	libyara/lexer.c
#	tests/test-rules.c

.github/workflows/build.yml

@@ -0,0 +1,80 @@
+name: Build and test
+
+on: [push, pull_request]
+
+jobs:
+  standard-build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        config:
+        - "--enable-cuckoo --enable-magic --enable-pb-tests --enable-address-sanitizer"
+        - "--enable-profiling --enable-cuckoo --enable-magic --enable-pb-tests --enable-address-sanitizer"
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y \
+            autoconf \
+            automake \
+            gcc-multilib \
+            libtool \
+            libjansson-dev \
+            libmagic-dev \
+            libssl-dev  \
+            protobuf-compiler \
+            protobuf-c-compiler \
+            libprotobuf-c-dev
+      - name: Prepare for building
+        run: |
+          ./bootstrap.sh
+          ./configure ${{ matrix.config }}
+      - name: Build
+        run: make clean && make
+      - name: Run tests
+        run: make check
+
+  mingw-build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        config:
+        - "--host=x86_64-w64-mingw32"
+        - "--host=i686-w64-mingw32"
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y \
+            autoconf \
+            automake \
+            libtool \
+            libssl-dev  \
+            gcc-mingw-w64
+      - name: Prepare for building
+        run: |
+          ./bootstrap.sh
+          ./configure ${{ matrix.config }}
+      - name: Build
+        run: make clean && make
+
+  bazel-build:
+    # The ubuntu-20.04 virtual environment has Bazel installed, see:
+    # https://github.com/actions/virtual-environments
+    runs-on: ubuntu-20.04
+    steps:
+      # Caches and restores the bazelisk download directory.
+      - name: Cache bazelisk download
+        uses: actions/cache@v2
+        env:
+          cache-name: bazel-cache
+        with:
+          path: ~/.cache/bazelisk
+          key: ${{ runner.os }}-${{ env.cache-name }}-${{ github.ref }}
+          restore-keys: |
+            ${{ runner.os }}-${{ env.cache-name }}-development
+      - uses: actions/checkout@v2
+      - name: Build and run tests
+        run: bazel test @jansson//... //tests/...

.gitignore

@@ -61,26 +61,7 @@ libyara/yara.pc
 .DS_Store
 
 # Files generated by tests
-test-alignment
-test-api
-test-arena
-test-arena-stream
-test-async
-test-atoms
-test-bitmask
-test-elf
-test-exception
-test-rules-pass-1
-test-rules-pass-2
-test-rules-pass-3
-test-rules.yarc
-test-pb
-test-pe
-test-re-split
-test-stack
-test-macho
-test-math
-test-version
+test-*
 
 # Bazel
 bazel-*

BUILD.bazel

@@ -104,12 +104,14 @@ cc_library(
     name = "cli_shared",
     srcs = [
         "cli/args.c",
+        "cli/common.c",
         "cli/threading.c",
     ],
     hdrs = [
         "cli/args.h",
         "cli/common.h",
         "cli/threading.h",
+        "cli/unicode.h",
     ],
     deps = [":libyara"],
 )

Makefile.am

@@ -56,6 +56,9 @@ yarac_SOURCES = \
 yarac_LDADD = -Llibyara/.libs -lyara
 yarac_DEPENDENCIES = libyara/.libs/libyara.la
 
+tests_mapper_SOURCES = tests/mapper.c
+tests_mapper_CFLAGS = -O0
+
 test_alignment_SOURCES = tests/test-alignment.c tests/util.c
 test_alignment_LDADD = libyara/.libs/libyara.a
 test_arena_SOURCES = tests/test-arena.c tests/util.c
@@ -64,6 +67,9 @@ test_atoms_SOURCES = tests/test-atoms.c tests/util.c
 test_atoms_LDADD = libyara/.libs/libyara.a
 test_rules_SOURCES = tests/test-rules.c tests/util.c
 test_rules_LDADD = libyara/.libs/libyara.a
+if POSIX
+EXTRA_test_rules_DEPENDENCIES = tests/mapper$(EXEEXT)
+endif
 test_pe_SOURCES = tests/test-pe.c tests/util.c
 test_pe_LDADD = libyara/.libs/libyara.a
 test_elf_SOURCES = tests/test-elf.c tests/util.c
@@ -84,7 +90,7 @@ test_async_SOURCES = tests/test-async.c tests/util.c
 test_async_LDADD = libyara/.libs/libyara.a
 
 TESTS = $(check_PROGRAMS)
-TESTS_ENVIRONMENT = TOP_SRCDIR=$(top_srcdir)
+TESTS_ENVIRONMENT = TOP_SRCDIR=$(top_srcdir) TOP_BUILDDIR=$(top_builddir)
 
 check_PROGRAMS = \
   test-arena \
@@ -101,6 +107,9 @@ check_PROGRAMS = \
   test-re-split \
   test-async
 
+EXTRA_PROGRAMS = tests/mapper
+CLEANFILES = tests/mapper$(EXEEXT)
+
 if POSIX
 # The -fsanitize=address option makes test-exception fail. Include the test
 # only if the option is not enabled.

README.md

@@ -1,5 +1,4 @@
 [![Join the chat at https://gitter.im/VirusTotal/yara](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/VirusTotal/yara?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
-[![Travis build status](https://travis-ci.com/VirusTotal/yara.svg)](https://travis-ci.com/VirusTotal/yara)
 [![AppVeyor build status](https://ci.appveyor.com/api/projects/status/7glqg19w4oolm7pr?svg=true)](https://ci.appveyor.com/project/plusvic/yara)
 [![Coverity status](https://scan.coverity.com/projects/9057/badge.svg?flat=1)](https://scan.coverity.com/projects/plusvic-yara)
 
@@ -68,6 +67,7 @@ awesome list of [YARA-related stuff](https://github.com/InQuest/awesome-yara).
 * [Blueliv](http://www.blueliv.com)
 * [Cisco Talos Intelligence Group](https://talosintelligence.com/)
 * [Claroty](https://claroty.com/continuous-threat-detection)
+* [Cloudina Security](https://cloudina.co.uk)
 * [Cofense](https://cofense.com)
 * [Conix](http://www.conix.fr)
 * [CounterCraft](https://www.countercraft.eu)

cli/yara.c

@@ -1338,7 +1338,7 @@ int _tmain(int argc, const char_t** argv)
     return EXIT_FAILURE;
   }
 
-#if defined(_WIN32)
+#if defined(_WIN32) && defined(_UNICODE)
   // In Windows set stdout to UTF-8 mode.
   if (_setmode(_fileno(stdout), _O_U8TEXT) == -1)
   {
@@ -1551,12 +1551,19 @@ int _tmain(int argc, const char_t** argv)
     yr_scanner_set_flags(scanner, flags);
     yr_scanner_set_timeout(scanner, timeout);
 
-    long pid = _tcstol(argv[argc - 1], NULL, 10);
+    // Assume the last argument is a file first. This assures we try to process
+    // files that start with numbers first.
+    result = scan_file(scanner, argv[argc - 1]);
 
-    if (pid != 0)
-      result = yr_scanner_scan_proc(scanner, (int) pid);
-    else
-      result = scan_file(scanner, argv[argc - 1]);
+    if (result == ERROR_COULD_NOT_OPEN_FILE)
+    {
+      // Is it a PID? To be a PID it must be made up entirely of digits.
+      char* endptr = NULL;
+      long pid = _tcstol(argv[argc - 1], &endptr, 10);
+
+      if (pid > 0 && argv[argc - 1] != NULL && *endptr == '\x00')
+        result = yr_scanner_scan_proc(scanner, (int) pid);
+    }
 
     if (result != ERROR_SUCCESS)
     {

cli/yarac.c

@@ -204,15 +204,13 @@ int _tmain(int argc, const char_t** argv)
     exit_with_code(EXIT_FAILURE);
   }
 
-  result = yr_initialize();
-
-  if (result != ERROR_SUCCESS)
+  if (yr_initialize() != ERROR_SUCCESS)
     exit_with_code(EXIT_FAILURE);
 
   if (yr_compiler_create(&compiler) != ERROR_SUCCESS)
     exit_with_code(EXIT_FAILURE);
 
-  if (!define_external_variables(ext_vars, NULL, compiler))
+  if (define_external_variables(ext_vars, NULL, compiler) != ERROR_SUCCESS)
     exit_with_code(EXIT_FAILURE);
 
   if (atom_quality_table != NULL)

configure.ac

@@ -1,4 +1,4 @@
-AC_INIT([yara], [4.1.0], [vmalvarez@virustotal.com])
+AC_INIT([yara], [4.1.1], [vmalvarez@virustotal.com])
 
 AM_SILENT_RULES([yes])
 AC_CONFIG_SRCDIR([cli/yara.c])
@@ -121,6 +121,14 @@ AC_ARG_ENABLE([profiling],
     CFLAGS="$CFLAGS -DYR_PROFILING_ENABLED"
   fi])
 
+AC_ARG_WITH([debug-verbose],
+  [AS_HELP_STRING([--with-debug-verbose=[[NUM]]], [Turn on runtime debugging information])],
+  [if test $withval -gt 0; then
+    AC_DEFINE_UNQUOTED([YR_DEBUG_VERBOSITY], [$withval])
+   else
+    AC_MSG_ERROR([debug verbosity must be greater than 0])
+  fi])
+
 AC_ARG_ENABLE([cuckoo],
   [AS_HELP_STRING([--enable-cuckoo], [enable cuckoo module])],
   [if test x$enableval = xyes; then
@@ -183,8 +191,8 @@ AC_ARG_ENABLE([pb-tests],
     # AC_CHECK_PROG(PROTOC_GEN_YARA, protoc-gen-yara, protoc-gen-yara)
     # AS_IF([test "x${PROTOC_GEN_YARA}" == "x"],
     #    [AC_MSG_ERROR([please install https://github.com/VirusTotal/protoc-gen-yara])])
-    PKG_CHECK_MODULES(PROTOBUF_C, libprotobuf-c >= 1.0.0)
-    AC_CHECK_LIB(protobuf-c, protobuf_c_message_unpack,,
+    PKG_CHECK_MODULES([PROTOBUF_C], [libprotobuf-c >= 1.0.0])
+    AC_CHECK_LIB([protobuf-c], protobuf_c_message_unpack,,
       AC_MSG_ERROR([please install libprotobuf-c library]))
     CFLAGS="$CFLAGS -DPB_TESTS_MODULE"
     PC_REQUIRES_PRIVATE="$PC_REQUIRES_PRIVATE libprotobuf-c"