From 1585f6acd92508aef2988c362db598c2e35f56dd Mon Sep 17 00:00:00 2001 From: Maamoun TK Date: Sun, 21 Mar 2021 20:48:26 +0200 Subject: [PATCH] [AArch64] Support fat build for GCM optimization --- Makefile.in | 2 +- arm64/fat/gcm-hash.asm | 38 ++++++++++ configure.ac | 12 +++- fat-arm64.c | 156 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 204 insertions(+), 4 deletions(-) create mode 100644 arm64/fat/gcm-hash.asm create mode 100644 fat-arm64.c diff --git a/Makefile.in b/Makefile.in index 2274d8be..0ace35f7 100644 --- a/Makefile.in +++ b/Makefile.in @@ -616,7 +616,7 @@ distdir: $(DISTFILES) set -e; for d in sparc32 sparc64 x86 \ x86_64 x86_64/aesni x86_64/sha_ni x86_64/fat \ arm arm/neon arm/v6 arm/fat \ - arm64 arm64/crypto \ + arm64 arm64/crypto arm64/fat \ powerpc64 powerpc64/p7 powerpc64/p8 powerpc64/fat ; do \ mkdir "$(distdir)/$$d" ; \ find "$(srcdir)/$$d" -maxdepth 1 '(' -name '*.asm' -o -name '*.m4' -o -name README ')' \ diff --git a/arm64/fat/gcm-hash.asm b/arm64/fat/gcm-hash.asm new file mode 100644 index 00000000..5ef171b5 --- /dev/null +++ b/arm64/fat/gcm-hash.asm @@ -0,0 +1,38 @@ +C arm64/fat/gcm-hash.asm + +ifelse(` + Copyright (C) 2021 Mamone Tarsha + + This file is part of GNU Nettle. + + GNU Nettle is free software: you can redistribute it and/or + modify it under the terms of either: + + * the GNU Lesser General Public License as published by the Free + Software Foundation; either version 3 of the License, or (at your + option) any later version. + + or + + * the GNU General Public License as published by the Free + Software Foundation; either version 2 of the License, or (at your + option) any later version. + + or both in parallel, as here. + + GNU Nettle is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received copies of the GNU General Public License and + the GNU Lesser General Public License along with this program. If + not, see http://www.gnu.org/licenses/. +') + +dnl picked up by configure +dnl PROLOGUE(_nettle_fat_gcm_init_key) +dnl PROLOGUE(_nettle_fat_gcm_hash) + +define(`fat_transform', `$1_arm64') +include_src(`arm64/crypto/gcm-hash.asm') diff --git a/configure.ac b/configure.ac index 6080a06a..026ae99d 100644 --- a/configure.ac +++ b/configure.ac @@ -476,9 +476,15 @@ if test "x$enable_assembler" = xyes ; then ;; aarch64*) if test "$ABI" = 64 ; then - asm_path=arm64 - if test "$enable_arm64_crypto" = yes ; then - asm_path="arm64/crypto $asm_path" + asm_path=arm64 + if test "x$enable_fat" = xyes ; then + asm_path="arm64/fat $asm_path" + OPT_NETTLE_SOURCES="fat-arm64.c $OPT_NETTLE_SOURCES" + FAT_TEST_LIST="none pmull" + else + if test "$enable_arm64_crypto" = yes ; then + asm_path="arm64/crypto $asm_path" + fi fi else # As far as I understand, Neon instructions are unlikely to be diff --git a/fat-arm64.c b/fat-arm64.c new file mode 100644 index 00000000..9f81951f --- /dev/null +++ b/fat-arm64.c @@ -0,0 +1,156 @@ +/* fat-arm64.c + + Copyright (C) 2021 Mamone Tarsha + + This file is part of GNU Nettle. + + GNU Nettle is free software: you can redistribute it and/or + modify it under the terms of either: + + * the GNU Lesser General Public License as published by the Free + Software Foundation; either version 3 of the License, or (at your + option) any later version. + + or + + * the GNU General Public License as published by the Free + Software Foundation; either version 2 of the License, or (at your + option) any later version. + + or both in parallel, as here. + + GNU Nettle is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received copies of the GNU General Public License and + the GNU Lesser General Public License along with this program. If + not, see http://www.gnu.org/licenses/. +*/ + +#define _GNU_SOURCE + +#if HAVE_CONFIG_H +# include "config.h" +#endif + +#include +#include +#include +#include + +#if defined(__linux__) && defined(__GLIBC__) && defined(__GLIBC_PREREQ) +# if __GLIBC_PREREQ(2, 16) +# define USE_GETAUXVAL 1 +# include +# include +# endif +#endif + +#include "nettle-types.h" + +#include "gcm.h" +#include "gcm-internal.h" +#include "fat-setup.h" + +/* Defines from arch/arm64/include/uapi/asm/hwcap.h in Linux kernel */ +#ifndef HWCAP_ASIMD +#define HWCAP_ASIMD (1 << 1) +#endif +#ifndef HWCAP_PMULL +#define HWCAP_PMULL (1 << 4) +#endif + +struct arm64_features +{ + int have_pmull; +}; + +#define MATCH(s, slen, literal, llen) \ + ((slen) == (llen) && memcmp ((s), (literal), llen) == 0) + +static void +get_arm64_features (struct arm64_features *features) +{ + const char *s; + features->have_pmull = 0; + + s = secure_getenv (ENV_OVERRIDE); + if (s) + for (;;) + { + const char *sep = strchr (s, ','); + size_t length = sep ? (size_t) (sep - s) : strlen(s); + + if (MATCH (s, length, "pmull", 5)) + features->have_pmull = 1; + if (!sep) + break; + s = sep + 1; + } + else + { +#if USE_GETAUXVAL + unsigned long hwcap = getauxval(AT_HWCAP); + features->have_pmull + = ((hwcap & (HWCAP_ASIMD | HWCAP_PMULL)) == (HWCAP_ASIMD | HWCAP_PMULL)); +#endif + } +} + +#if GCM_TABLE_BITS == 8 +DECLARE_FAT_FUNC(_nettle_gcm_init_key, gcm_init_key_func) +DECLARE_FAT_FUNC_VAR(gcm_init_key, gcm_init_key_func, c) +DECLARE_FAT_FUNC_VAR(gcm_init_key, gcm_init_key_func, arm64) + +DECLARE_FAT_FUNC(_nettle_gcm_hash, gcm_hash_func) +DECLARE_FAT_FUNC_VAR(gcm_hash, gcm_hash_func, c) +DECLARE_FAT_FUNC_VAR(gcm_hash, gcm_hash_func, arm64) +#endif /* GCM_TABLE_BITS == 8 */ + +static void CONSTRUCTOR +fat_init (void) +{ + struct arm64_features features; + int verbose; + + get_arm64_features (&features); + + verbose = getenv (ENV_VERBOSE) != NULL; + if (verbose) + fprintf (stderr, "libnettle: cpu features: %s\n", + features.have_pmull ? "polynomial multiply long instructions (PMULL/PMULL2)" : ""); + + if (features.have_pmull) + { + if (verbose) + fprintf (stderr, "libnettle: enabling hardware-accelerated polynomial multiply code.\n"); +#if GCM_TABLE_BITS == 8 + /* Make sure _nettle_gcm_init_key_vec function is compatible + with _nettle_gcm_hash_vec function e.g. _nettle_gcm_init_key_c() + fills gcm_key table with values that are incompatible with + _nettle_gcm_hash_arm64() */ + _nettle_gcm_init_key_vec = _nettle_gcm_init_key_arm64; + _nettle_gcm_hash_vec = _nettle_gcm_hash_arm64; +#endif /* GCM_TABLE_BITS == 8 */ + } + else + { +#if GCM_TABLE_BITS == 8 + _nettle_gcm_init_key_vec = _nettle_gcm_init_key_c; + _nettle_gcm_hash_vec = _nettle_gcm_hash_c; +#endif /* GCM_TABLE_BITS == 8 */ + } +} + +#if GCM_TABLE_BITS == 8 +DEFINE_FAT_FUNC(_nettle_gcm_init_key, void, + (union nettle_block16 *table), + (table)) + +DEFINE_FAT_FUNC(_nettle_gcm_hash, void, + (const struct gcm_key *key, union nettle_block16 *x, + size_t length, const uint8_t *data), + (key, x, length, data)) +#endif /* GCM_TABLE_BITS == 8 */