From 6dafd9a01d1b230fbdbd6748d74f868506371282 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 30 Nov 2024 09:06:55 +0900 Subject: [PATCH 1/4] fix assertion error on AIX --- Objects/unicodeobject.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index ab4f07ed054385..85cfc202ca002f 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -5022,13 +5022,13 @@ ctz(size_t v) static size_t load_unaligned(const unsigned char *p, size_t size) { - assert(size <= SIZEOF_SIZE_T); union { size_t s; unsigned char b[SIZEOF_SIZE_T]; } u; u.s = 0; switch (size) { + default: case 8: u.b[7] = p[7]; _Py_FALLTHROUGH; @@ -5055,8 +5055,6 @@ load_unaligned(const unsigned char *p, size_t size) break; case 0: break; - default: - Py_UNREACHABLE(); } return u.s; } From 2faf883b3ca2da93525a9ee06c56da3ea36de525 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 30 Nov 2024 12:18:20 +0900 Subject: [PATCH 2/4] fix 32bit build --- Objects/unicodeobject.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 85cfc202ca002f..184c61d94d462b 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -5029,6 +5029,7 @@ load_unaligned(const unsigned char *p, size_t size) u.s = 0; switch (size) { default: +#if SIZEOF_SIZE_T == 8 case 8: u.b[7] = p[7]; _Py_FALLTHROUGH; @@ -5041,6 +5042,7 @@ load_unaligned(const unsigned char *p, size_t size) case 5: u.b[4] = p[4]; _Py_FALLTHROUGH; +#endif case 4: u.b[3] = p[3]; _Py_FALLTHROUGH; From ba3e008d0ddabcdfe5606cb0478fc23073729e44 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 30 Nov 2024 13:37:24 +0900 Subject: [PATCH 3/4] fix for big endian --- Objects/unicodeobject.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 184c61d94d462b..3de1d1bf5e874a 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -5016,9 +5016,8 @@ ctz(size_t v) } #endif -#if HAVE_CTZ -// load p[0]..p[size-1] as a little-endian size_t -// without unaligned access nor read ahead. +#if HAVE_CTZ && PY_LITTLE_ENDIAN +// load p[0]..p[size-1] as a size_t without unaligned access nor read ahead. static size_t load_unaligned(const unsigned char *p, size_t size) { @@ -5027,6 +5026,9 @@ load_unaligned(const unsigned char *p, size_t size) unsigned char b[SIZEOF_SIZE_T]; } u; u.s = 0; + // This switch statement assumes little endian because: + // * union is faster than bitwise or and shift. + // * big endian machine is rare and hard to maintain. switch (size) { default: #if SIZEOF_SIZE_T == 8 @@ -5078,7 +5080,7 @@ find_first_nonascii(const unsigned char *start, const unsigned char *end) if (end - start >= SIZEOF_SIZE_T) { const unsigned char *p2 = _Py_ALIGN_UP(p, SIZEOF_SIZE_T); if (p < p2) { -#if HAVE_CTZ +#if PY_LITTLE_ENDIAN && HAVE_CTZ #if defined(_M_AMD64) || defined(_M_IX86) || defined(__x86_64__) || defined(__i386__) // x86 and amd64 are little endian and can load unaligned memory. size_t u = *(const size_t*)p & ASCII_CHAR_MASK; @@ -5113,7 +5115,7 @@ find_first_nonascii(const unsigned char *start, const unsigned char *end) p += SIZEOF_SIZE_T; } } -#if HAVE_CTZ +#if PY_LITTLE_ENDIAN && HAVE_CTZ // we can not use *(const size_t*)p to avoid buffer overrun. size_t u = load_unaligned(p, end - p) & ASCII_CHAR_MASK; if (u) { From aa4a8397e84296e8d1ab88ac3213a44212b58fef Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 30 Nov 2024 15:48:03 +0900 Subject: [PATCH 4/4] fixup --- Objects/unicodeobject.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 3de1d1bf5e874a..33fa21d4c7d1bf 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -5014,6 +5014,8 @@ ctz(size_t v) #endif /* SIZEOF_SIZE_T */ return pos; } +#else +#define HAVE_CTZ 0 #endif #if HAVE_CTZ && PY_LITTLE_ENDIAN @@ -5079,8 +5081,8 @@ find_first_nonascii(const unsigned char *start, const unsigned char *end) if (end - start >= SIZEOF_SIZE_T) { const unsigned char *p2 = _Py_ALIGN_UP(p, SIZEOF_SIZE_T); - if (p < p2) { #if PY_LITTLE_ENDIAN && HAVE_CTZ + if (p < p2) { #if defined(_M_AMD64) || defined(_M_IX86) || defined(__x86_64__) || defined(__i386__) // x86 and amd64 are little endian and can load unaligned memory. size_t u = *(const size_t*)p & ASCII_CHAR_MASK; @@ -5088,11 +5090,11 @@ find_first_nonascii(const unsigned char *start, const unsigned char *end) size_t u = load_unaligned(p, p2 - p) & ASCII_CHAR_MASK; #endif if (u) { - return p - start + (ctz(u) - 7) / 8; + return (ctz(u) - 7) / 8; } p = p2; } -#else +#else /* PY_LITTLE_ENDIAN && HAVE_CTZ */ while (p < p2) { if (*p & 0x80) { return p - start;