Skip to content

Commit

Permalink
add AVX512 support for vermicelli model
Browse files Browse the repository at this point in the history
  • Loading branch information
@hongyang7
hongyang7 committed May 25, 2020
1 parent ec68fac commit 88a18dc
Show file tree
Hide file tree
Showing 2 changed files with 634 additions and 43 deletions.
179 changes: 137 additions & 42 deletions src/nfa/vermicelli.h
View file
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2015-2016, Intel Corporation
* Copyright (c) 2015-2020, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
Expand Down Expand Up @@ -46,7 +46,20 @@ const u8 *vermicelliExec(char c, char nocase, const u8 *buf,
nocase ? "nocase " : "", c, (size_t)(buf_end - buf));
assert(buf < buf_end);

VERM_TYPE chars = VERM_SET_FN(c); /* nocase already uppercase */

// Handle small scans.
#ifdef HAVE_AVX512
if (buf_end - buf <= VERM_BOUNDARY) {
const u8 *ptr = nocase
? vermMiniNocase(chars, buf, buf_end, 0)
: vermMini(chars, buf, buf_end, 0);
if (ptr) {
return ptr;
}
return buf_end;
}
#else
if (buf_end - buf < VERM_BOUNDARY) {
for (; buf < buf_end; buf++) {
char cur = (char)*buf;
Expand All @@ -59,8 +72,8 @@ const u8 *vermicelliExec(char c, char nocase, const u8 *buf,
}
return buf;
}
#endif

VERM_TYPE chars = VERM_SET_FN(c); /* nocase already uppercase */
uintptr_t min = (uintptr_t)buf % VERM_BOUNDARY;
if (min) {
// Input isn't aligned, so we need to run one iteration with an
Expand Down Expand Up @@ -99,7 +112,20 @@ const u8 *nvermicelliExec(char c, char nocase, const u8 *buf,
nocase ? "nocase " : "", c, (size_t)(buf_end - buf));
assert(buf < buf_end);

VERM_TYPE chars = VERM_SET_FN(c); /* nocase already uppercase */

// Handle small scans.
#ifdef HAVE_AVX512
if (buf_end - buf <= VERM_BOUNDARY) {
const u8 *ptr = nocase
? vermMiniNocase(chars, buf, buf_end, 1)
: vermMini(chars, buf, buf_end, 1);
if (ptr) {
return ptr;
}
return buf_end;
}
#else
if (buf_end - buf < VERM_BOUNDARY) {
for (; buf < buf_end; buf++) {
char cur = (char)*buf;
Expand All @@ -112,8 +138,8 @@ const u8 *nvermicelliExec(char c, char nocase, const u8 *buf,
}
return buf;
}
#endif

VERM_TYPE chars = VERM_SET_FN(c); /* nocase already uppercase */
size_t min = (size_t)buf % VERM_BOUNDARY;
if (min) {
// Input isn't aligned, so we need to run one iteration with an
Expand Down Expand Up @@ -149,12 +175,32 @@ const u8 *vermicelliDoubleExec(char c1, char c2, char nocase, const u8 *buf,
DEBUG_PRINTF("double verm scan %s\\x%02hhx%02hhx over %zu bytes\n",
nocase ? "nocase " : "", c1, c2, (size_t)(buf_end - buf));
assert(buf < buf_end);
assert((buf_end - buf) >= VERM_BOUNDARY);

uintptr_t min = (uintptr_t)buf % VERM_BOUNDARY;
VERM_TYPE chars1 = VERM_SET_FN(c1); /* nocase already uppercase */
VERM_TYPE chars2 = VERM_SET_FN(c2); /* nocase already uppercase */

#ifdef HAVE_AVX512
if (buf_end - buf <= VERM_BOUNDARY) {
const u8 *ptr = nocase
? dvermMiniNocase(chars1, chars2, buf, buf_end)
: dvermMini(chars1, chars2, buf, buf_end);
if (ptr) {
return ptr;
}

/* check for partial match at end */
u8 mask = nocase ? CASE_CLEAR : 0xff;
if ((buf_end[-1] & mask) == (u8)c1) {
DEBUG_PRINTF("partial!!!\n");
return buf_end - 1;
}

return buf_end;
}
#endif

assert((buf_end - buf) >= VERM_BOUNDARY);
uintptr_t min = (uintptr_t)buf % VERM_BOUNDARY;
if (min) {
// Input isn't aligned, so we need to run one iteration with an
// unaligned load, then skip buf forward to the next aligned address.
Expand Down Expand Up @@ -205,14 +251,32 @@ const u8 *vermicelliDoubleMaskedExec(char c1, char c2, char m1, char m2,
DEBUG_PRINTF("double verm scan (\\x%02hhx&\\x%02hhx)(\\x%02hhx&\\x%02hhx) "
"over %zu bytes\n", c1, m1, c2, m2, (size_t)(buf_end - buf));
assert(buf < buf_end);
assert((buf_end - buf) >= VERM_BOUNDARY);

uintptr_t min = (uintptr_t)buf % VERM_BOUNDARY;
VERM_TYPE chars1 = VERM_SET_FN(c1);
VERM_TYPE chars2 = VERM_SET_FN(c2);
VERM_TYPE mask1 = VERM_SET_FN(m1);
VERM_TYPE mask2 = VERM_SET_FN(m2);

#ifdef HAVE_AVX512
if (buf_end - buf <= VERM_BOUNDARY) {
const u8 *ptr = dvermMiniMasked(chars1, chars2, mask1, mask2, buf,
buf_end);
if (ptr) {
return ptr;
}

/* check for partial match at end */
if ((buf_end[-1] & m1) == (u8)c1) {
DEBUG_PRINTF("partial!!!\n");
return buf_end - 1;
}

return buf_end;
}
#endif

assert((buf_end - buf) >= VERM_BOUNDARY);
uintptr_t min = (uintptr_t)buf % VERM_BOUNDARY;
if (min) {
// Input isn't aligned, so we need to run one iteration with an
// unaligned load, then skip buf forward to the next aligned address.
Expand Down Expand Up @@ -244,6 +308,7 @@ const u8 *vermicelliDoubleMaskedExec(char c1, char c2, char m1, char m2,

/* check for partial match at end */
if ((buf_end[-1] & m1) == (u8)c1) {
DEBUG_PRINTF("partial!!!\n");
return buf_end - 1;
}

Expand All @@ -259,7 +324,20 @@ const u8 *rvermicelliExec(char c, char nocase, const u8 *buf,
nocase ? "nocase " : "", c, (size_t)(buf_end - buf));
assert(buf < buf_end);

VERM_TYPE chars = VERM_SET_FN(c); /* nocase already uppercase */

// Handle small scans.
#ifdef HAVE_AVX512
if (buf_end - buf <= VERM_BOUNDARY) {
const u8 *ptr = nocase
? rvermMiniNocase(chars, buf, buf_end, 0)
: rvermMini(chars, buf, buf_end, 0);
if (ptr) {
return ptr;
}
return buf - 1;
}
#else
if (buf_end - buf < VERM_BOUNDARY) {
for (buf_end--; buf_end >= buf; buf_end--) {
char cur = (char)*buf_end;
Expand All @@ -272,26 +350,22 @@ const u8 *rvermicelliExec(char c, char nocase, const u8 *buf,
}
return buf_end;
}
#endif

VERM_TYPE chars = VERM_SET_FN(c); /* nocase already uppercase */
size_t min = (size_t)buf_end % VERM_BOUNDARY;

if (min) {
// Input isn't aligned, so we need to run one iteration with an
// unaligned load, then skip buf backward to the next aligned address.
// There's some small overlap here, but we don't mind scanning it twice
// if we can do it quickly, do we?
if (nocase) {
const u8 *ptr =
rvermUnalignNocase(chars, buf_end - VERM_BOUNDARY, 0);
if (ptr) {
return ptr;
}
} else {
const u8 *ptr = rvermUnalign(chars, buf_end - VERM_BOUNDARY, 0);
if (ptr) {
return ptr;
}
const u8 *ptr = nocase ? rvermUnalignNocase(chars,
buf_end - VERM_BOUNDARY,
0)
: rvermUnalign(chars, buf_end - VERM_BOUNDARY,
0);

if (ptr) {
return ptr;
}

buf_end -= min;
Expand Down Expand Up @@ -322,7 +396,20 @@ const u8 *rnvermicelliExec(char c, char nocase, const u8 *buf,
nocase ? "nocase " : "", c, (size_t)(buf_end - buf));
assert(buf < buf_end);

VERM_TYPE chars = VERM_SET_FN(c); /* nocase already uppercase */

// Handle small scans.
#ifdef HAVE_AVX512
if (buf_end - buf <= VERM_BOUNDARY) {
const u8 *ptr = nocase
? rvermMiniNocase(chars, buf, buf_end, 1)
: rvermMini(chars, buf, buf_end, 1);
if (ptr) {
return ptr;
}
return buf - 1;
}
#else
if (buf_end - buf < VERM_BOUNDARY) {
for (buf_end--; buf_end >= buf; buf_end--) {
char cur = (char)*buf_end;
Expand All @@ -335,26 +422,22 @@ const u8 *rnvermicelliExec(char c, char nocase, const u8 *buf,
}
return buf_end;
}
#endif

VERM_TYPE chars = VERM_SET_FN(c); /* nocase already uppercase */
size_t min = (size_t)buf_end % VERM_BOUNDARY;

if (min) {
// Input isn't aligned, so we need to run one iteration with an
// unaligned load, then skip buf backward to the next aligned address.
// There's some small overlap here, but we don't mind scanning it twice
// if we can do it quickly, do we?
if (nocase) {
const u8 *ptr =
rvermUnalignNocase(chars, buf_end - VERM_BOUNDARY, 1);
if (ptr) {
return ptr;
}
} else {
const u8 *ptr = rvermUnalign(chars, buf_end - VERM_BOUNDARY, 1);
if (ptr) {
return ptr;
}
const u8 *ptr = nocase ? rvermUnalignNocase(chars,
buf_end - VERM_BOUNDARY,
1)
: rvermUnalign(chars, buf_end - VERM_BOUNDARY,
1);

if (ptr) {
return ptr;
}

buf_end -= min;
Expand Down Expand Up @@ -383,24 +466,36 @@ const u8 *rvermicelliDoubleExec(char c1, char c2, char nocase, const u8 *buf,
DEBUG_PRINTF("rev double verm scan %s\\x%02hhx%02hhx over %zu bytes\n",
nocase ? "nocase " : "", c1, c2, (size_t)(buf_end - buf));
assert(buf < buf_end);
assert((buf_end - buf) >= VERM_BOUNDARY);

size_t min = (size_t)buf_end % VERM_BOUNDARY;
VERM_TYPE chars1 = VERM_SET_FN(c1); /* nocase already uppercase */
VERM_TYPE chars2 = VERM_SET_FN(c2); /* nocase already uppercase */

#ifdef HAVE_AVX512
if (buf_end - buf <= VERM_BOUNDARY) {
const u8 *ptr = nocase
? rdvermMiniNocase(chars1, chars2, buf, buf_end)
: rdvermMini(chars1, chars2, buf, buf_end);

if (ptr) {
return ptr;
}

// check for partial match at end ???
return buf - 1;
}
#endif

assert((buf_end - buf) >= VERM_BOUNDARY);
size_t min = (size_t)buf_end % VERM_BOUNDARY;
if (min) {
// input not aligned, so we need to run one iteration with an unaligned
// load, then skip buf forward to the next aligned address. There's
// some small overlap here, but we don't mind scanning it twice if we
// can do it quickly, do we?
const u8 *ptr;
if (nocase) {
ptr = rdvermPreconditionNocase(chars1, chars2,
buf_end - VERM_BOUNDARY);
} else {
ptr = rdvermPrecondition(chars1, chars2, buf_end - VERM_BOUNDARY);
}
const u8 *ptr = nocase ? rdvermPreconditionNocase(chars1, chars2,
buf_end - VERM_BOUNDARY)
: rdvermPrecondition(chars1, chars2,
buf_end - VERM_BOUNDARY);

if (ptr) {
return ptr;
Expand Down

0 comments on commit 88a18dc

Please sign in to comment.