Fix GHASH-ASM implementation on s390x

s390x GHASH assembler implementation assumed it was called from a
gcm128_context structure where the Xi paramter to the ghash function was
embedded in that structure.  Since the structure layout resembles the paramter
block required for kimd-GHASH, the assembler code simply assumed the 128 bytes
after Xi are the hash subkey.

This assumption was broken with the introduction of AES-GCM-SIV which uses the
GHASH implementation without a gcm128_context structure.  Furthermore, the
bytes following the Xi input parameter to the GHASH function do not contain
the hash subkey.  To fix this, we remove the assumption about the calling
context and build the parameter block on the stack.  This requires some
copying of data to and from the stack.  While this introduces a performance
degradation, new systems anyway use kma for GHASH/AES-GCM.

Finally fixes #18693 for s390x.

Signed-off-by: Juergen Christ <jchrist@linux.ibm.com>

Reviewed-by: Todd Short <todd.short@me.com>
Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from #18939)

(cherry picked from commit cd854f2)

crypto/modes/asm/ghash-s390x.pl

@@ -90,25 +90,6 @@
 .align	32
 gcm_gmult_4bit:
 ___
-$code.=<<___ if(!$softonly && 0);	# hardware is slow for single block...
-	larl	%r1,OPENSSL_s390xcap_P
-	lghi	%r0,0
-	lg	%r1,S390X_KIMD+8(%r1)	# load second word of kimd capabilities
-					#  vector
-	tmhh	%r1,0x4000	# check for function 65
-	jz	.Lsoft_gmult
-	stg	%r0,16($sp)	# arrange 16 bytes of zero input
-	stg	%r0,24($sp)
-	lghi	%r0,S390X_GHASH	# function 65
-	la	%r1,0($Xi)	# H lies right after Xi in gcm128_context
-	la	$inp,16($sp)
-	lghi	$len,16
-	.long	0xb93e0004	# kimd %r0,$inp
-	brc	1,.-4		# pay attention to "partial completion"
-	br	%r14
-.align	32
-.Lsoft_gmult:
-___
 $code.=<<___;
 	stm${g}	%r6,%r14,6*$SIZE_T($sp)
 
@@ -132,10 +113,21 @@
 					#  vector
 	tmhh	%r0,0x4000	# check for function 65
 	jz	.Lsoft_ghash
+	# Do not assume this function is called from a gcm128_context.
+	# This is not true, e.g., for AES-GCM-SIV.
+	# Parameter Block:
+	# Chaining Value (XI) 128byte
+	# Key (Htable[8]) 128byte
+	lmg	%r0,%r1,0($Xi)
+	stmg	%r0,%r1,8($sp)
+	lmg	%r0,%r1,8*16($Htbl)
+	stmg	%r0,%r1,24($sp)
+	la	%r1,8($sp)
 	lghi	%r0,S390X_GHASH	# function 65
-	la	%r1,0($Xi)	# H lies right after Xi in gcm128_context
 	.long	0xb93e0004	# kimd %r0,$inp
 	brc	1,.-4		# pay attention to "partial completion"
+	lmg	%r0,%r1,8($sp)
+	stmg	%r0,%r1,0($Xi)
 	br	%r14
 .align	32
 .Lsoft_ghash: