Fails to build with x32 abi

[kraj]

Here is regression seen on x32 builds

make[1]: Entering directory '/mnt/a/yoe/build/tmp/work/core2-64-x32-yoe-linux-gnux32/libxcrypt/4.4.1-r0/build'
IFY_SOURCE=2 -Wformat -Wformat-security -Werror=format-security --sysroot=/mnt/a/yoe/build/tmp/work/core2-64-x32-yoe-linux-gnux32/libxcrypt/4.4.1-r0/recipe-sysroot -DHAVE_CONFWextra -Walloc-zero -Walloca -Wbad-function-cast -Wcast-align -Wcast-qual -Wconversion -Wformat=2 -Wformat-overflow=2 -Wformat-signedness -Wformat-truncation=1 -Winline -Wlogiprototypes -Wundef -Wvla -Wwrite-strings -Wpedantic -Werror -I/mnt/a/yoe/build/tmp/work/core2-64-x32-yoe-linux-gnux32/libxcrypt/4.4.1-r0/recipe-sysroot/usr/include -Wno-error=xcrypt/4.4.1-r0 -fdebug-prefix-map=/mnt/a/yoe/build/tmp/work/core2-64-x32-yoe-linux-gnux32/libxcrypt/4.4.1-r0/recipe-sysroot= -fdebug-prefix-map=/mnt/a/yoe/build/tmp/work/corex86_64-yoe-linux-gnux32-libtool: compile:  x86_64-yoe-linux-gnux32-gcc -mx32 -march=core2 -mtune=core2 -msse3 -mfpmath=sse -fstack-protector-strong -D_FORTIFY_SOURCE=2 -WformaIN_LIBCRYPT -I/mnt/a/yoe/build/tmp/work/core2-64-x32-yoe-linux-gnux32/libxcrypt/4.4.1-r0/recipe-sysroot/usr/include -Wno-error=missing-attributes -Wall -Wextra -Walloc-zero -Wrations -Wmissing-prototypes -Wnested-externs -Wnull-dereference -Wold-style-definition -Wpointer-arith -Wrestrict -Wshadow -Wstrict-overflow=2 -Wstrict-prototypes -Wundef -Wv -pipe -g -feliminate-unused-debug-types -fdebug-prefix-map=/mnt/a/yoe/build/tmp/work/core2-64-x32-yoe-linux-gnux32/libxcrypt/4.4.1-r0=/usr/src/debug/libxcrypt/4.4.1-r0 -fdebux32/libxcrypt/4.4.1-r0/recipe-sysroot-native= -c ../git/yescrypt-opt.c  -fPIC -DPIC -o .libs/libcrypt_la-yescrypt-opt.o
../git/yescrypt-opt.c:665: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:665: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:665: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:665: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:665: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:665: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:665: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:665: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:665: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:665: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:665: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:665: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:665: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:665: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:665: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:665: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:665: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:665: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:665: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:665: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:666: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:715: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:716: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:715: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:715: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:715: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:715: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:715: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:715: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:715: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:715: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:715: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:715: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:715: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:715: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:715: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:715: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:715: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:716: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:720: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:720: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:720: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:720: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:720: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:720: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:720: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:720: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:720: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:720: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:720: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:720: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:720: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:720: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:720: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:720: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:720: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:721: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:720: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:721: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:720: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:721: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:720: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:721: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:767: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:768: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:767: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:768: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:767: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:768: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:767: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:768: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:767: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:768: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:767: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:768: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:767: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:768: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:767: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:768: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:767: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:768: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:767: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:768: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:767: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:768: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:767: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:768: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:767: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:768: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:767: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:768: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:767: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:768: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:767: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:768: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:767: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:768: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:767: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:768: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:767: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:768: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:767: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:768: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:767: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:768: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:767: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:768: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:767: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:768: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:767: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:768: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:771: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:772: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:771: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:772: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:771: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:772: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:771: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:772: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:771: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:772: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:771: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:772: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:771: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:772: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:771: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:772: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:771: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:772: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:771: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:772: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:771: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:772: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:771: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:772: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:771: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:772: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:771: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:772: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:771: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:772: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:771: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:772: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:771: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:772: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:771: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:772: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:771: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:772: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:771: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:772: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:771: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:772: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:771: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:772: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:771: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:772: Error: `(%edi,%rax)' is not a valid base/index expression
../git/yescrypt-opt.c:771: Error: `(%esi,%rcx)' is not a valid base/index expression
../git/yescrypt-opt.c:772: Error: `(%edi,%rax)' is not a valid base/index expression
make[1]: *** [Makefile:1314: libcrypt_la-yescrypt-opt.lo] Error 1

[kraj]

This is the commit where it is introduced

a32bb84

[solardiz]

It looks like the x86_64 inline asm is enabled on x32. I don't have any x32 system to test on, but I guess a fix would be to change the line:

#if defined(__x86_64__) && (defined(__AVX__) || !defined(__GNUC__))

to:

#if defined(__x86_64__) && (defined(__AVX__) || defined(__ILP32__) || !defined(__GNUC__))

That way, a suitable implementation with intrinsics should be used and the inline asm one not reached by the conditional compiles.

Slightly better performance could be possible through addition of x32 specific inline asm. I think it can be something like (untested, and diff is against upstream yescrypt):

--- yescrypt-opt.c	2018-07-10 17:51:52 +0000
+++ yescrypt-opt-x32.c	2018-12-17 19:29:36 +0000
@@ -515,6 +515,11 @@
 #undef MAYBE_MEMORY_BARRIER
 #define MAYBE_MEMORY_BARRIER \
 	__asm__("" : : : "memory");
+#ifdef __ILP32__ /* x32 */
+#define REGISTER_PREFIX "e"
+#else
+#define REGISTER_PREFIX "r"
+#endif
 #define PWXFORM_SIMD(X) { \
 	__m128i H; \
 	__asm__( \
@@ -524,8 +529,8 @@
 	    "pmuludq %1, %0\n\t" \
 	    "movl %%eax, %%ecx\n\t" \
 	    "shrq $0x20, %%rax\n\t" \
-	    "paddq (%3,%%rcx), %0\n\t" \
-	    "pxor (%4,%%rax), %0\n\t" \
+	    "paddq (%3,%%" REGISTER_PREFIX "cx), %0\n\t" \
+	    "pxor (%4,%%" REGISTER_PREFIX "ax), %0\n\t" \
 	    : "+x" (X), "=x" (H) \
 	    : "d" (Smask2), "S" (S0), "D" (S1) \
 	    : "cc", "ax", "cx"); \

Please test.

Yet another kind of fix would be to define two limited scope variables of type size_t and pass them to the asm block as "=a" and "=c" (much like we already do for the H variable), and then refer to them by their numbers. Then gcc itself would probably substitute properly sized registers. Maybe that would be cleaner. A minor drawback is we'll have to renumber %2 and on to higher numbers, so the diff would be bigger.

[kraj]

let me try second solution out

[besser82]

@kraj @solardiz That's odd… i686 builds [1] on Fedora don't show this issue.

[1] https://kojipkgs.fedoraproject.org//packages/libxcrypt/4.4.1/1.fc30/data/logs/i686/build.log

[ldv-alt]

x32 is not something you can try on Fedora.

[kraj]

@solardiz the inline asm change is building successfully, waiting for runtime results. hopefully tomorrow

[solardiz]

@kraj So what are the runtime results? And what project is this for? Do you actually use yescrypt there or is it merely getting compiled along with the rest of libxcrypt, then left unused? I'd like to know whether to merge this fix upstream. Thanks!

[kraj]

This patch here works as expected.
http://git.openembedded.org/openembedded-core/tree/meta/recipes-core/libxcrypt/files/0001-Add-x32-specific-inline-asm.patch?id=59be9254df9ffeda594b21af8be433564c898474

[besser82]

@kraj It would be nice, if you would open a PR with this patch here as well.