Merge branch 'github_develop' into github_master

Author: fatchanghao

CHANGELOG.md

@@ -2,6 +2,14 @@
 
 This is a list of notable changes to Hyperscan, in reverse chronological order.
 
+## [5.4.2] 2023-04-19
+- Roll back bugfix for github issue #350: Besides using scratch for
+  corresponding database, Hyperscan also allows user to use larger scratch
+  allocated for another database. Users can leverage this property to achieve
+  safe scratch usage in multi-database scenarios. Behaviors beyond these are
+  discouraged and results are undefined.
+- Fix hsdump issue due to invalid nfa type.
+
 ## [5.4.1] 2023-02-20
 - The Intel Hyperscan team is pleased to provide a bug fix release to our open source library.
   Intel also maintains an upgraded version available through your Intel sales representative.

CMakeLists.txt

@@ -3,7 +3,7 @@ project (hyperscan C CXX)
 
 set (HS_MAJOR_VERSION 5)
 set (HS_MINOR_VERSION 4)
-set (HS_PATCH_VERSION 1)
+set (HS_PATCH_VERSION 2)
 set (HS_VERSION ${HS_MAJOR_VERSION}.${HS_MINOR_VERSION}.${HS_PATCH_VERSION})
 
 set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake)

src/hs.h

@@ -43,7 +43,7 @@
 
 #define HS_MAJOR      5
 #define HS_MINOR      4
-#define HS_PATCH      1
+#define HS_PATCH      2
 
 #include "hs_compile.h"
 #include "hs_runtime.h"

src/nfa/nfa_dump_dispatch.cpp

@@ -75,7 +75,6 @@ namespace ue2 {
         DISPATCH_CASE(LBR_NFA_VERM, LbrVerm, dbnt_func);                       \
         DISPATCH_CASE(LBR_NFA_NVERM, LbrNVerm, dbnt_func);                     \
         DISPATCH_CASE(LBR_NFA_SHUF, LbrShuf, dbnt_func);                       \
-        DISPATCH_CASE(LBR_NFA_VSHUF, LbrVShuf, dbnt_func);                     \
         DISPATCH_CASE(LBR_NFA_TRUF, LbrTruf, dbnt_func);                       \
         DISPATCH_CASE(CASTLE_NFA, Castle, dbnt_func);                          \
         DISPATCH_CASE(SHENG_NFA, Sheng, dbnt_func);                            \

src/runtime.c

@@ -90,7 +90,7 @@ u8 *getHistory(char *state, const struct RoseEngine *t, u64a offset) {
  * callers.
  */
 static really_inline
-char validScratch(const struct hs_scratch *s, u32 crc) {
+char validScratch(const struct RoseEngine *t, const struct hs_scratch *s) {
     if (!ISALIGNED_CL(s)) {
         DEBUG_PRINTF("bad alignment %p\n", s);
         return 0;
@@ -101,12 +101,18 @@ char validScratch(const struct hs_scratch *s, u32 crc) {
         return 0;
     }
 
-    /* add quick rose sanity checks by db crc*/
-    if (s->db_crc != crc) {
-        DEBUG_PRINTF("Improper scratch for current db\n");
+    if (t->mode == HS_MODE_BLOCK && t->stateOffsets.end > s->bStateSize) {
+        DEBUG_PRINTF("bad state size\n");
         return 0;
     }
 
+    if (t->queueCount > s->queueCount) {
+        DEBUG_PRINTF("bad queue count\n");
+        return 0;
+    }
+
+    /* TODO: add quick rose sanity checks */
+
     return 1;
 }
 
@@ -329,7 +335,7 @@ hs_error_t HS_CDECL hs_scan(const hs_database_t *db, const char *data,
         return HS_DB_MODE_ERROR;
     }
 
-    if (unlikely(!validScratch(scratch, db->crc32))) {
+    if (unlikely(!validScratch(rose, scratch))) {
         return HS_INVALID;
     }
 
@@ -503,7 +509,7 @@ void maintainHistoryBuffer(const struct RoseEngine *rose, char *state,
 
 static really_inline
 void init_stream(struct hs_stream *s, const struct RoseEngine *rose,
-                 char init_history, u32 crc) {
+                 char init_history) {
     char *state = getMultiState(s);
 
     if (init_history) {
@@ -518,7 +524,6 @@ void init_stream(struct hs_stream *s, const struct RoseEngine *rose,
 
     s->rose = rose;
     s->offset = 0;
-    s->crc32 = crc;
 
     setStreamStatus(state, 0);
     roseInitState(rose, state);
@@ -563,7 +568,7 @@ hs_error_t HS_CDECL hs_open_stream(const hs_database_t *db,
         return HS_NOMEM;
     }
 
-    init_stream(s, rose, 1, db->crc32);
+    init_stream(s, rose, 1);
 
     *stream = s;
     return HS_SUCCESS;
@@ -751,7 +756,7 @@ hs_error_t HS_CDECL hs_reset_and_copy_stream(hs_stream_t *to_id,
     }
 
     if (onEvent) {
-        if (!scratch || !validScratch(scratch, to_id->crc32)) {
+        if (!scratch || !validScratch(to_id->rose, scratch)) {
             return HS_INVALID;
         }
         if (unlikely(markScratchInUse(scratch))) {
@@ -977,7 +982,7 @@ hs_error_t HS_CDECL hs_scan_stream(hs_stream_t *id, const char *data,
                                    hs_scratch_t *scratch,
                                    match_event_handler onEvent, void *context) {
     if (unlikely(!id || !scratch || !data ||
-                 !validScratch(scratch, id->crc32))) {
+                 !validScratch(id->rose, scratch))) {
         return HS_INVALID;
     }
 
@@ -999,7 +1004,7 @@ hs_error_t HS_CDECL hs_close_stream(hs_stream_t *id, hs_scratch_t *scratch,
     }
 
     if (onEvent) {
-        if (!scratch || !validScratch(scratch, id->crc32)) {
+        if (!scratch || !validScratch(id->rose, scratch)) {
             return HS_INVALID;
         }
         if (unlikely(markScratchInUse(scratch))) {
@@ -1029,7 +1034,7 @@ hs_error_t HS_CDECL hs_reset_stream(hs_stream_t *id, UNUSED unsigned int flags,
     }
 
     if (onEvent) {
-        if (!scratch || !validScratch(scratch, id->crc32)) {
+        if (!scratch || !validScratch(id->rose, scratch)) {
             return HS_INVALID;
         }
         if (unlikely(markScratchInUse(scratch))) {
@@ -1044,7 +1049,7 @@ hs_error_t HS_CDECL hs_reset_stream(hs_stream_t *id, UNUSED unsigned int flags,
     }
 
     // history already initialised
-    init_stream(id, id->rose, 0, id->crc32);
+    init_stream(id, id->rose, 0);
 
     return HS_SUCCESS;
 }
@@ -1123,7 +1128,7 @@ hs_error_t HS_CDECL hs_scan_vector(const hs_database_t *db,
         return HS_DB_MODE_ERROR;
     }
 
-    if (unlikely(!validScratch(scratch, db->crc32))) {
+    if (unlikely(!validScratch(rose, scratch))) {
         return HS_INVALID;
     }
 
@@ -1133,7 +1138,7 @@ hs_error_t HS_CDECL hs_scan_vector(const hs_database_t *db,
 
     hs_stream_t *id = (hs_stream_t *)(scratch->bstate);
 
-    init_stream(id, rose, 1, db->crc32); /* open stream */
+    init_stream(id, rose, 1); /* open stream */
 
     for (u32 i = 0; i < count; i++) {
         DEBUG_PRINTF("block %u/%u offset=%llu len=%u\n", i, count, id->offset,
@@ -1248,7 +1253,7 @@ hs_error_t HS_CDECL hs_reset_and_expand_stream(hs_stream_t *to_stream,
     const struct RoseEngine *rose = to_stream->rose;
 
     if (onEvent) {
-        if (!scratch || !validScratch(scratch, to_stream->crc32)) {
+        if (!scratch || !validScratch(to_stream->rose, scratch)) {
             return HS_INVALID;
         }
         if (unlikely(markScratchInUse(scratch))) {

src/scratch.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015-2022, Intel Corporation
+ * Copyright (c) 2015-2023, Intel Corporation
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -373,15 +373,13 @@ hs_error_t HS_CDECL hs_alloc_scratch(const hs_database_t *db,
             hs_scratch_free((*scratch)->scratch_alloc);
         }
 
-        proto->db_crc = db->crc32;
         hs_error_t alloc_ret = alloc_scratch(proto, scratch);
         hs_scratch_free(proto_tmp); /* kill off temp used for sizing */
         if (alloc_ret != HS_SUCCESS) {
             *scratch = NULL;
             return alloc_ret;
         }
     } else {
-        (*scratch)->db_crc = db->crc32;
         hs_scratch_free(proto_tmp); /* kill off temp used for sizing */
         unmarkScratchInUse(*scratch);
     }

src/scratch.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015-2022, Intel Corporation
+ * Copyright (c) 2015-2023, Intel Corporation
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -171,7 +171,6 @@ struct match_deduper {
  */
 struct ALIGN_CL_DIRECTIVE hs_scratch {
     u32 magic;
-    u32 db_crc; /**< identity of a scratch space, for validity check */
     u8 in_use; /**< non-zero when being used by an API call. */
     u32 queueCount;
     u32 activeQueueArraySize; /**< size of active queue array fatbit in bytes */

src/state.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015-2022, Intel Corporation
+ * Copyright (c) 2015-2023, Intel Corporation
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -57,9 +57,6 @@ struct hs_stream {
 
     /** \brief The current stream offset. */
     u64a offset;
-
-    /** \brief Identity of hs_stream, for scratch validity check. */
-    u32 crc32;
 };
 
 #define getMultiState(hs_s)      ((char *)(hs_s) + sizeof(*(hs_s)))

src/stream_compress_impl.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-2022, Intel Corporation
+ * Copyright (c) 2017-2023, Intel Corporation
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -116,7 +116,6 @@ size_t JOIN(sc_, FN_SUFFIX)(const struct RoseEngine *rose,
         = ((STREAM_QUAL char *)stream) + sizeof(struct hs_stream);
 
     COPY_FIELD(stream->offset);
-    COPY_FIELD(stream->crc32);
     ASSIGN(stream->rose, rose);
 
     COPY(stream_body + ROSE_STATE_OFFSET_STATUS_FLAGS, 1);