From c2945509df63a85ecf4291a36e6fc1a3d302e66e Mon Sep 17 00:00:00 2001 From: Andrew Dinn Date: Mon, 10 Nov 2025 17:44:47 +0000 Subject: [PATCH 01/30] complete stub save and restore for aarch64 --- .../cpu/aarch64/macroAssembler_aarch64.cpp | 2 +- src/hotspot/cpu/aarch64/runtime_aarch64.cpp | 2 +- .../cpu/aarch64/stubDeclarations_aarch64.hpp | 49 +- .../cpu/aarch64/stubGenerator_aarch64.cpp | 1719 ++++++++++++++--- .../cpu/aarch64/stubRoutines_aarch64.cpp | 16 + .../cpu/aarch64/stubRoutines_aarch64.hpp | 2 + src/hotspot/share/c1/c1_Runtime1.cpp | 4 - src/hotspot/share/code/aotCodeCache.cpp | 1083 +++++++++-- src/hotspot/share/code/aotCodeCache.hpp | 231 ++- src/hotspot/share/prims/downcallLinker.hpp | 2 +- src/hotspot/share/runtime/init.cpp | 11 +- .../share/runtime/stubCodeGenerator.cpp | 70 +- .../share/runtime/stubCodeGenerator.hpp | 20 +- .../share/runtime/stubDeclarations.hpp | 6 + src/hotspot/share/runtime/stubInfo.cpp | 9 + src/hotspot/share/runtime/stubInfo.hpp | 5 + src/hotspot/share/runtime/stubRoutines.cpp | 85 +- src/hotspot/share/runtime/stubRoutines.hpp | 4 + .../cds/appcds/aotCode/AOTCodeFlags.java | 2 +- 19 files changed, 2800 insertions(+), 522 deletions(-) diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp index ceedb4f10632a..ac2acfba92353 100644 --- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp @@ -3306,7 +3306,7 @@ void MacroAssembler::subw(Register Rd, Register Rn, RegisterOrConstant decrement void MacroAssembler::reinit_heapbase() { if (UseCompressedOops) { - if (Universe::is_fully_initialized()) { + if (Universe::is_fully_initialized() && !AOTCodeCache::is_on_for_dump()) { mov(rheapbase, CompressedOops::base()); } else { lea(rheapbase, ExternalAddress(CompressedOops::base_addr())); diff --git a/src/hotspot/cpu/aarch64/runtime_aarch64.cpp b/src/hotspot/cpu/aarch64/runtime_aarch64.cpp index d45f9865bd26a..2e570bee7091c 100644 --- a/src/hotspot/cpu/aarch64/runtime_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/runtime_aarch64.cpp @@ -292,7 +292,7 @@ ExceptionBlob* OptoRuntime::generate_exception_blob() { assert(SimpleRuntimeFrame::framesize % 4 == 0, "sp not 16-byte aligned"); const char* name = OptoRuntime::stub_name(StubId::c2_exception_id); - CodeBlob* blob = AOTCodeCache::load_code_blob(AOTCodeEntry::C2Blob, (uint)BlobId::c2_exception_id, name); + CodeBlob* blob = AOTCodeCache::load_code_blob(AOTCodeEntry::C2Blob, BlobId::c2_exception_id); if (blob != nullptr) { return blob->as_exception_blob(); } diff --git a/src/hotspot/cpu/aarch64/stubDeclarations_aarch64.hpp b/src/hotspot/cpu/aarch64/stubDeclarations_aarch64.hpp index 695534604b8c0..95b59bd8590f5 100644 --- a/src/hotspot/cpu/aarch64/stubDeclarations_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/stubDeclarations_aarch64.hpp @@ -84,8 +84,7 @@ do_stub(compiler, count_positives) \ do_arch_entry(aarch64, compiler, count_positives, count_positives, \ count_positives) \ - do_stub(compiler, count_positives_long) \ - do_arch_entry(aarch64, compiler, count_positives_long, \ + do_arch_entry(aarch64, compiler, count_positives, \ count_positives_long, count_positives_long) \ do_stub(compiler, compare_long_string_LL) \ do_arch_entry(aarch64, compiler, compare_long_string_LL, \ @@ -108,8 +107,9 @@ do_stub(compiler, string_indexof_linear_ul) \ do_arch_entry(aarch64, compiler, string_indexof_linear_ul, \ string_indexof_linear_ul, string_indexof_linear_ul) \ - /* this uses the entry for ghash_processBlocks */ \ - do_stub(compiler, ghash_processBlocks_wide) \ + do_stub(compiler, ghash_processBlocks_small) \ + do_arch_entry(aarch64, compiler, ghash_processBlocks_small, \ + ghash_processBlocks_small, ghash_processBlocks_small) \ #define STUBGEN_FINAL_BLOBS_ARCH_DO(do_stub, \ @@ -142,6 +142,47 @@ /* stub only -- entries are not stored in StubRoutines::aarch64 */ \ /* n.b. these are not the same as the generic atomic stubs */ \ do_stub(final, atomic_entry_points) \ + do_arch_entry(aarch64, final, atomic_entry_points, \ + atomic_fetch_add_4_impl, atomic_fetch_add_4_impl) \ + do_arch_entry(aarch64, final, atomic_entry_points, \ + atomic_fetch_add_8_impl, atomic_fetch_add_8_impl) \ + do_arch_entry(aarch64, final, atomic_entry_points, \ + atomic_fetch_add_4_relaxed_impl, \ + atomic_fetch_add_4_relaxed_impl) \ + do_arch_entry(aarch64, final, atomic_entry_points, \ + atomic_fetch_add_8_relaxed_impl, \ + atomic_fetch_add_8_relaxed_impl) \ + do_arch_entry(aarch64, final, atomic_entry_points, \ + atomic_xchg_4_impl, atomic_xchg_4_impl) \ + do_arch_entry(aarch64, final, atomic_entry_points, \ + atomic_xchg_8_impl, atomic_xchg_8_impl) \ + do_arch_entry(aarch64, final, atomic_entry_points, \ + atomic_cmpxchg_1_impl, atomic_cmpxchg_1_impl) \ + do_arch_entry(aarch64, final, atomic_entry_points, \ + atomic_cmpxchg_4_impl, atomic_cmpxchg_4_impl) \ + do_arch_entry(aarch64, final, atomic_entry_points, \ + atomic_cmpxchg_8_impl, atomic_cmpxchg_8_impl) \ + do_arch_entry(aarch64, final, atomic_entry_points, \ + atomic_cmpxchg_1_relaxed_impl, \ + atomic_cmpxchg_1_relaxed_impl) \ + do_arch_entry(aarch64, final, atomic_entry_points, \ + atomic_cmpxchg_4_relaxed_impl, \ + atomic_cmpxchg_4_relaxed_impl) \ + do_arch_entry(aarch64, final, atomic_entry_points, \ + atomic_cmpxchg_8_relaxed_impl, \ + atomic_cmpxchg_8_relaxed_impl) \ + do_arch_entry(aarch64, final, atomic_entry_points, \ + atomic_cmpxchg_4_release_impl, \ + atomic_cmpxchg_4_release_impl) \ + do_arch_entry(aarch64, final, atomic_entry_points, \ + atomic_cmpxchg_8_release_impl, \ + atomic_cmpxchg_8_release_impl) \ + do_arch_entry(aarch64, final, atomic_entry_points, \ + atomic_cmpxchg_4_seq_cst_impl, \ + atomic_cmpxchg_4_seq_cst_impl) \ + do_arch_entry(aarch64, final, atomic_entry_points, \ + atomic_cmpxchg_8_seq_cst_impl, \ + atomic_cmpxchg_8_seq_cst_impl) \ #endif // CPU_AARCH64_STUBDECLARATIONS_HPP diff --git a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp index 8dbc5dbac0343..78ba98811849d 100644 --- a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp @@ -79,6 +79,166 @@ #define BIND(label) bind(label); BLOCK_COMMENT(#label ":") +// Constant data definitions + +static const uint32_t _sha256_round_consts[64] = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, + 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, + 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, + 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, + 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, + 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, + 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2, +}; + +static const uint64_t _sha512_round_consts[80] = { + 0x428A2F98D728AE22L, 0x7137449123EF65CDL, 0xB5C0FBCFEC4D3B2FL, + 0xE9B5DBA58189DBBCL, 0x3956C25BF348B538L, 0x59F111F1B605D019L, + 0x923F82A4AF194F9BL, 0xAB1C5ED5DA6D8118L, 0xD807AA98A3030242L, + 0x12835B0145706FBEL, 0x243185BE4EE4B28CL, 0x550C7DC3D5FFB4E2L, + 0x72BE5D74F27B896FL, 0x80DEB1FE3B1696B1L, 0x9BDC06A725C71235L, + 0xC19BF174CF692694L, 0xE49B69C19EF14AD2L, 0xEFBE4786384F25E3L, + 0x0FC19DC68B8CD5B5L, 0x240CA1CC77AC9C65L, 0x2DE92C6F592B0275L, + 0x4A7484AA6EA6E483L, 0x5CB0A9DCBD41FBD4L, 0x76F988DA831153B5L, + 0x983E5152EE66DFABL, 0xA831C66D2DB43210L, 0xB00327C898FB213FL, + 0xBF597FC7BEEF0EE4L, 0xC6E00BF33DA88FC2L, 0xD5A79147930AA725L, + 0x06CA6351E003826FL, 0x142929670A0E6E70L, 0x27B70A8546D22FFCL, + 0x2E1B21385C26C926L, 0x4D2C6DFC5AC42AEDL, 0x53380D139D95B3DFL, + 0x650A73548BAF63DEL, 0x766A0ABB3C77B2A8L, 0x81C2C92E47EDAEE6L, + 0x92722C851482353BL, 0xA2BFE8A14CF10364L, 0xA81A664BBC423001L, + 0xC24B8B70D0F89791L, 0xC76C51A30654BE30L, 0xD192E819D6EF5218L, + 0xD69906245565A910L, 0xF40E35855771202AL, 0x106AA07032BBD1B8L, + 0x19A4C116B8D2D0C8L, 0x1E376C085141AB53L, 0x2748774CDF8EEB99L, + 0x34B0BCB5E19B48A8L, 0x391C0CB3C5C95A63L, 0x4ED8AA4AE3418ACBL, + 0x5B9CCA4F7763E373L, 0x682E6FF3D6B2B8A3L, 0x748F82EE5DEFB2FCL, + 0x78A5636F43172F60L, 0x84C87814A1F0AB72L, 0x8CC702081A6439ECL, + 0x90BEFFFA23631E28L, 0xA4506CEBDE82BDE9L, 0xBEF9A3F7B2C67915L, + 0xC67178F2E372532BL, 0xCA273ECEEA26619CL, 0xD186B8C721C0C207L, + 0xEADA7DD6CDE0EB1EL, 0xF57D4F7FEE6ED178L, 0x06F067AA72176FBAL, + 0x0A637DC5A2C898A6L, 0x113F9804BEF90DAEL, 0x1B710B35131C471BL, + 0x28DB77F523047D84L, 0x32CAAB7B40C72493L, 0x3C9EBE0A15C9BEBCL, + 0x431D67C49C100D4CL, 0x4CC5D4BECB3E42B6L, 0x597F299CFC657E2AL, + 0x5FCB6FAB3AD6FAECL, 0x6C44198C4A475817L +}; + +static const uint64_t _sha3_round_consts[24] = { + 0x0000000000000001L, 0x0000000000008082L, 0x800000000000808AL, + 0x8000000080008000L, 0x000000000000808BL, 0x0000000080000001L, + 0x8000000080008081L, 0x8000000000008009L, 0x000000000000008AL, + 0x0000000000000088L, 0x0000000080008009L, 0x000000008000000AL, + 0x000000008000808BL, 0x800000000000008BL, 0x8000000000008089L, + 0x8000000000008003L, 0x8000000000008002L, 0x8000000000000080L, + 0x000000000000800AL, 0x800000008000000AL, 0x8000000080008081L, + 0x8000000000008080L, 0x0000000080000001L, 0x8000000080008008L +}; + +static const uint64_t _double_keccak_round_consts[24] = { + 0x0000000000000001L, 0x0000000000008082L, 0x800000000000808AL, + 0x8000000080008000L, 0x000000000000808BL, 0x0000000080000001L, + 0x8000000080008081L, 0x8000000000008009L, 0x000000000000008AL, + 0x0000000000000088L, 0x0000000080008009L, 0x000000008000000AL, + 0x000000008000808BL, 0x800000000000008BL, 0x8000000000008089L, + 0x8000000000008003L, 0x8000000000008002L, 0x8000000000000080L, + 0x000000000000800AL, 0x800000008000000AL, 0x8000000080008081L, + 0x8000000000008080L, 0x0000000080000001L, 0x8000000080008008L +}; + +static const char _encodeBlock_toBase64[64] = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', + 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', + 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/' +}; + +static const char _encodeBlock_toBase64URL[64] = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', + 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', + 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_' +}; + +// Non-SIMD lookup tables are mostly dumped from fromBase64 array used in java.util.Base64, +// except the trailing character '=' is also treated illegal value in this intrinsic. That +// is java.util.Base64.fromBase64['='] = -2, while fromBase(URL)64ForNoSIMD['='] = 255 here. +static const uint8_t _decodeBlock_fromBase64ForNoSIMD[256] = { + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u, 255u, 63u, + 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u, 14u, + 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u, 255u, + 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u, 40u, + 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, +}; + +static const uint8_t _decodeBlock_fromBase64URLForNoSIMD[256] = { + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u, + 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u, 14u, + 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u, 63u, + 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u, 40u, + 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, +}; + +// A legal value of base64 code is in range [0, 127]. We need two lookups +// with tbl/tbx and combine them to get the decode data. The 1st table vector +// lookup use tbl, out of range indices are set to 0 in destination. The 2nd +// table vector lookup use tbx, out of range indices are unchanged in +// destination. Input [64..126] is mapped to index [65, 127] in second lookup. +// The value of index 64 is set to 0, so that we know that we already get the +// decoded data with the 1st lookup. +static const uint8_t _decodeBlock_fromBase64ForSIMD[128] = { + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u, 255u, 63u, + 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u, + 0u, 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u, + 14u, 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u, + 255u, 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u, + 40u, 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u, +}; + +static const uint8_t _decodeBlock_fromBase64URLForSIMD[128] = { + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u, + 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u, + 0u, 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u, + 14u, 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u, + 63u, 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u, + 40u, 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u, +}; + + // Stub Code definitions class StubGenerator: public StubCodeGenerator { @@ -197,12 +357,52 @@ class StubGenerator: public StubCodeGenerator { retaddr_off = 1, }; + // Helper used to restore saved ranges and handler addresses. + // Expects entries to contain 3 * count addresses beginning at + // offset begin which identify start of range, end of range and + // address of handler pc. end of range may be nullptr in which case + // it defaults to stub_end. hanlder pc may be nullptr in which case + // it defaults to default_handler. + + void register_unsafe_access_handlers(GrowableArray
&entries, int begin, int count, address stub_end) { + for (int i = 0; i < count; i++) { + int offset = begin + 3 * i; + address start = entries.at(offset); + address end = entries.at(offset + 1); + if (end == nullptr) { + end = stub_end; + } + address handler = entries.at(offset + 2); + if (handler == nullptr) { + handler = UnsafeMemoryAccess::common_exit_stub_pc(); + } + UnsafeMemoryAccess::add_to_table(start, end, handler); + } + } + + void retrieve_unsafe_access_handlers(address start, address end, GrowableArray
&entries) { + UnsafeMemoryAccess::collect_entries(start, end, entries); + } + address generate_call_stub(address& return_address) { assert((int)frame::entry_frame_after_call_words == -(int)sp_after_call_off + 1 && (int)frame::entry_frame_call_wrapper_offset == (int)call_wrapper_off, "adjust this code"); StubId stub_id = StubId::stubgen_call_stub_id; + GrowableArray
entries; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 2, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + // this stub has an extra return entry which we need to retrieve + load_archive_data(stub_id, start, end, &entries); + assert(entries.length() == 1, "expected 1 extra entry"); + return_address = entries.at(0); + return start; + } + StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -323,6 +523,7 @@ class StubGenerator: public StubCodeGenerator { // save current address for use by exception handling code return_address = __ pc(); + entries.append(return_address); // store result depending on type (everything that is not // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT) @@ -406,6 +607,9 @@ class StubGenerator: public StubCodeGenerator { __ strd(j_farg0, Address(j_rarg2, 0)); __ br(Assembler::AL, exit); + // record the stub entry and end plus the auxiliary entry + store_archive_data(stub_id, start, __ pc(), &entries); + return start; } @@ -423,6 +627,15 @@ class StubGenerator: public StubCodeGenerator { address generate_catch_exception() { StubId stub_id = StubId::stubgen_catch_exception_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -460,6 +673,9 @@ class StubGenerator: public StubCodeGenerator { "_call_stub_return_address must have been generated before"); __ b(StubRoutines::_call_stub_return_address); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -479,6 +695,14 @@ class StubGenerator: public StubCodeGenerator { address generate_forward_exception() { StubId stub_id = StubId::stubgen_forward_exception_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -551,6 +775,9 @@ class StubGenerator: public StubCodeGenerator { __ verify_oop(r0); __ br(r19); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -569,6 +796,15 @@ class StubGenerator: public StubCodeGenerator { // [tos + 5]: saved rscratch1 address generate_verify_oop() { StubId stub_id = StubId::stubgen_verify_oop_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -613,12 +849,24 @@ class StubGenerator: public StubCodeGenerator { __ blr(rscratch1); __ hlt(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } // Generate indices for iota vector. address generate_iota_indices(StubId stub_id) { __ align(CodeEntryAlignment); + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + StubCodeMark mark(this, stub_id); address start = __ pc(); // B @@ -639,6 +887,10 @@ class StubGenerator: public StubCodeGenerator { // D - FP __ emit_data64(0x0000000000000000, relocInfo::none); // 0.0d __ emit_data64(0x3FF0000000000000, relocInfo::none); // 1.0d + + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -656,14 +908,23 @@ class StubGenerator: public StubCodeGenerator { // r11 < MacroAssembler::zero_words_block_size. address generate_zero_blocks() { + StubId stub_id = StubId::stubgen_zero_blocks_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + StubCodeMark mark(this, stub_id); Label done; Label base_aligned; Register base = r10, cnt = r11; __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_zero_blocks_id; - StubCodeMark mark(this, stub_id); address start = __ pc(); if (UseBlockZeroing) { @@ -707,6 +968,9 @@ class StubGenerator: public StubCodeGenerator { __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -803,6 +1067,15 @@ class StubGenerator: public StubCodeGenerator { // s and d are adjusted to point to the remaining words to copy // address generate_copy_longs(StubId stub_id, DecoratorSet decorators, Register s, Register d, Register count) { + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + BasicType type; copy_direction direction; @@ -1154,6 +1427,9 @@ class StubGenerator: public StubCodeGenerator { __ ret(lr); } + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -1508,8 +1784,8 @@ class StubGenerator: public StubCodeGenerator { // stub_id - is used to name the stub and identify all details of // how to perform the copy. // - // entry - is assigned to the stub's post push entry point unless - // it is null + // nopush_entry - is assigned to the stub's post push entry point + // unless it is null // // Inputs: // c_rarg0 - source array address @@ -1525,8 +1801,6 @@ class StubGenerator: public StubCodeGenerator { // copy method // address generate_disjoint_copy(StubId stub_id, address *nopush_entry) { - Register s = c_rarg0, d = c_rarg1, count = c_rarg2; - RegSet saved_reg = RegSet::of(s, d, count); int size; bool aligned; bool is_oop; @@ -1607,17 +1881,48 @@ class StubGenerator: public StubCodeGenerator { ShouldNotReachHere(); break; } + // all stubs provide a 2nd entry which omits the frame push for + // use when bailing out from a conjoint copy. However we may also + // need some extra addressses for memory access protection. + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 2, "sanity check"); + assert(nopush_entry != nullptr, "all disjoint copy stubs export a nopush entry"); + + bool add_extras = !is_oop && (!aligned || sizeof(jlong) == size); + int extra_count = ((add_extras ? 1 : 0) * 3); + GrowableArray
entries; + GrowableArray
extras; + GrowableArray
*extras_ptr = (extra_count > 0 ? &extras : nullptr); + + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, &entries, extras_ptr); + assert(entries.length() == entry_count - 1, + "unexpected entries count %d", entries.length()); + *nopush_entry = entries.at(0); + assert(extras.length() == extra_count, + "unexpected extra count %d", extras.length()); + if (add_extras) { + // register one handler at offset 0 + register_unsafe_access_handlers(extras, 0, 1, end); + } + return start; + } + + Register s = c_rarg0, d = c_rarg1, count = c_rarg2; + RegSet saved_reg = RegSet::of(s, d, count); __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); - if (nopush_entry != nullptr) { - *nopush_entry = __ pc(); - // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) - BLOCK_COMMENT("Entry:"); - } + *nopush_entry = __ pc(); + entries.append(*nopush_entry); + + // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) + BLOCK_COMMENT("Post-Push Entry:"); DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_DISJOINT; if (dest_uninitialized) { @@ -1636,8 +1941,7 @@ class StubGenerator: public StubCodeGenerator { } { // UnsafeMemoryAccess page error: continue after unsafe access - bool add_entry = !is_oop && (!aligned || sizeof(jlong) == size); - UnsafeMemoryAccessMark umam(this, add_entry, true); + UnsafeMemoryAccessMark umam(this, add_extras, true); copy_memory(decorators, is_oop ? T_OBJECT : T_BYTE, aligned, s, d, count, size); } @@ -1652,6 +1956,20 @@ class StubGenerator: public StubCodeGenerator { __ leave(); __ mov(r0, zr); // return 0 __ ret(lr); + + address end = __ pc(); + + if (add_extras) { + // retrieve the registered handler addresses + retrieve_unsafe_access_handlers(start, end, extras); + assert(extras.length() == extra_count + , "incorrect handlers count %d", extras.length()); + } + + // record the stub entry and end plus the no_push entry and any + // extra handler addresses + store_archive_data(stub_id, start, end, &entries, extras_ptr); + return start; } @@ -1663,8 +1981,8 @@ class StubGenerator: public StubCodeGenerator { // corresponding disjoint copy routine which can be // jumped to if the ranges do not actually overlap // - // entry - is assigned to the stub's post push entry point unless - // it is null + // nopush_entry - is assigned to the stub's post push entry point + // unless it is null // // // Inputs: @@ -1681,8 +1999,6 @@ class StubGenerator: public StubCodeGenerator { // used by some other conjoint copy method // address generate_conjoint_copy(StubId stub_id, address nooverlap_target, address *nopush_entry) { - Register s = c_rarg0, d = c_rarg1, count = c_rarg2; - RegSet saved_regs = RegSet::of(s, d, count); int size; bool aligned; bool is_oop; @@ -1762,15 +2078,50 @@ class StubGenerator: public StubCodeGenerator { default: ShouldNotReachHere(); } + // only some conjoint stubs generate a 2nd entry + int entry_count = StubInfo::entry_count(stub_id); + int expected_entry_count = (nopush_entry == nullptr ? 1 : 2); + assert(entry_count == expected_entry_count, + "expected entry count %d does not match declared entry count %d for stub %s", + expected_entry_count, entry_count, StubInfo::name(stub_id)); + + // We need to protect memory accesses in certain cases + bool add_extras = !is_oop && (!aligned || sizeof(jlong) == size); + int extra_count = ((add_extras ? 1 : 0) * 3); + GrowableArray
entries; + GrowableArray
extras; + GrowableArray
*entries_ptr = (nopush_entry != nullptr ? &entries : nullptr); + GrowableArray
*extras_ptr = (extra_count > 0 ? &extras : nullptr); + + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, entries_ptr, extras_ptr); + assert(entries.length() == expected_entry_count - 1, + "unexpected entries count %d", entries.length()); + assert(extras.length() == extra_count, + "unexpected extra count %d", extras.length()); + if (nopush_entry != nullptr) { + *nopush_entry = entries.at(0); + } + if (add_extras) { + // register one handler at offset 0 + register_unsafe_access_handlers(extras, 0, 1, end); + } + return start; + } + Register s = c_rarg0, d = c_rarg1, count = c_rarg2; + RegSet saved_regs = RegSet::of(s, d, count); StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); if (nopush_entry != nullptr) { *nopush_entry = __ pc(); + entries.append(*nopush_entry); // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) - BLOCK_COMMENT("Entry:"); + BLOCK_COMMENT("Post-Push Entry:"); } // use fwd copy when (d-s) above_equal (count*size) @@ -1798,8 +2149,7 @@ class StubGenerator: public StubCodeGenerator { } { // UnsafeMemoryAccess page error: continue after unsafe access - bool add_entry = !is_oop && (!aligned || sizeof(jlong) == size); - UnsafeMemoryAccessMark umam(this, add_entry, true); + UnsafeMemoryAccessMark umam(this, add_extras, true); copy_memory(decorators, is_oop ? T_OBJECT : T_BYTE, aligned, s, d, count, -size); } if (is_oop) { @@ -1811,6 +2161,23 @@ class StubGenerator: public StubCodeGenerator { __ leave(); __ mov(r0, zr); // return 0 __ ret(lr); + + assert(entries.length() == expected_entry_count - 1, + "unexpected entries count %d", entries.length()); + + address end = __ pc(); + + if (add_extras) { + // retrieve the registered handler addresses + retrieve_unsafe_access_handlers(start, end, extras); + assert(extras.length() == extra_count, + "incorrect handlers count %d", extras.length()); + } + + // record the stub entry and end plus any no_push entry and/or + // extra handler addresses + store_archive_data(stub_id, start, end, entries_ptr, extras_ptr); + return start; } @@ -1864,6 +2231,30 @@ class StubGenerator: public StubCodeGenerator { ShouldNotReachHere(); } + // the normal stub provides a 2nd entry which omits the frame push + // for use when bailing out from a disjoint copy + // We need to protect memory accesses in certain cases + // only some conjoint stubs generate a 2nd entry + int entry_count = StubInfo::entry_count(stub_id); + int expected_entry_count = (nopush_entry == nullptr ? 1 : 2); + GrowableArray
entries; + GrowableArray
*entries_ptr = (expected_entry_count == 1 ? nullptr : &entries); + assert(entry_count == expected_entry_count, + "expected entry count %d does not match declared entry count %d for stub %s", + expected_entry_count, entry_count, StubInfo::name(stub_id)); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, entries_ptr); + assert(entries.length() + 1 == expected_entry_count, + "expected entry count %d does not match return entry count %d for stub %s", + expected_entry_count, entries.length() + 1, StubInfo::name(stub_id)); + if (nopush_entry != nullptr) { + *nopush_entry = entries.at(0); + } + return start; + } + Label L_load_element, L_store_element, L_do_card_marks, L_done, L_done_pop; // Input registers (after setup_arg_regs) @@ -1913,6 +2304,7 @@ class StubGenerator: public StubCodeGenerator { // Caller of this entry point must set up the argument registers. if (nopush_entry != nullptr) { *nopush_entry = __ pc(); + entries.append(*nopush_entry); BLOCK_COMMENT("Entry:"); } @@ -2010,6 +2402,8 @@ class StubGenerator: public StubCodeGenerator { __ leave(); __ ret(lr); + // record the stub entry and end plus any no_push entry + store_archive_data(stub_id, start, __ pc() , entries_ptr); return start; } @@ -2072,6 +2466,17 @@ class StubGenerator: public StubCodeGenerator { address int_copy_entry, address long_copy_entry) { StubId stub_id = StubId::stubgen_unsafe_arraycopy_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + // TODO - in debug mode we should save and restore the supplied + // copy entries and assert that the restored entries match the ones + // passed in + load_archive_data(stub_id, start, end); + return start; + } Label L_long_aligned, L_int_aligned, L_short_aligned; Register s = c_rarg0, d = c_rarg1, count = c_rarg2; @@ -2104,6 +2509,9 @@ class StubGenerator: public StubCodeGenerator { __ lsr(count, count, LogBytesPerLong); // size => long_count __ b(RuntimeAddress(long_copy_entry)); + // record the stub entry and end plus any no_push entry + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -2125,6 +2533,17 @@ class StubGenerator: public StubCodeGenerator { address int_copy_entry, address oop_copy_entry, address long_copy_entry, address checkcast_copy_entry) { StubId stub_id = StubId::stubgen_generic_arraycopy_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + // TODO - in debug mode we should save and restore the supplied + // copy entries and assert that the restored entries match the ones + // passed in + load_archive_data(stub_id, start, end); + return start; + } Label L_failed, L_objArray; Label L_copy_bytes, L_copy_shorts, L_copy_ints, L_copy_longs; @@ -2383,6 +2802,9 @@ class StubGenerator: public StubCodeGenerator { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(lr); + // record the stub entry and end plus any no_push entry + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -2427,6 +2849,14 @@ class StubGenerator: public StubCodeGenerator { default: ShouldNotReachHere(); }; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); @@ -2563,14 +2993,31 @@ class StubGenerator: public StubCodeGenerator { __ bind(L_exit2); __ leave(); __ ret(lr); + + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address generate_unsafecopy_common_error_exit() { + StubId stub_id = StubId::stubgen_unsafecopy_common_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } address start_pc = __ pc(); __ leave(); __ mov(r0, 0); __ ret(lr); + + // record the stub entry and end + store_archive_data(stub_id, start_pc, __ pc()); + return start_pc; } @@ -2589,13 +3036,30 @@ class StubGenerator: public StubCodeGenerator { // c_rarg2 - byte value // address generate_unsafe_setmemory() { + StubId stub_id = StubId::stubgen_unsafe_setmemory_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + // we expect one set of extra unsafememory access handler entries + GrowableArray
extras; + int extra_count = 3; + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, nullptr, &extras); + assert(extras.length() == extra_count, + "unexpected extra entry count %d", extras.length()); + register_unsafe_access_handlers(extras, 0, 1, end); + return start; + } + __ align(CodeEntryAlignment); - StubCodeMark mark(this, StubId::stubgen_unsafe_setmemory_id); + StubCodeMark mark(this, stub_id); address start = __ pc(); Register dest = c_rarg0, count = c_rarg1, value = c_rarg2; Label tail; + { UnsafeMemoryAccessMark umam(this, true, false); __ enter(); // required for proper stackwalking of RuntimeStub frame @@ -2679,6 +3143,17 @@ class StubGenerator: public StubCodeGenerator { __ bind(finished); __ leave(); __ ret(lr); + // have to exit the block and destroy the UnsafeMemoryAccessMark + // in order to retrieve the handler end address + } + + // install saved handler addresses in extras + address end = __ pc(); + retrieve_unsafe_access_handlers(start, end, extras); + assert(extras.length() == extra_count, + "incorrect handlers count %d", extras.length()); + // record the stub entry and end plus the extras + store_archive_data(stub_id, start, end, nullptr, &extras); return start; } @@ -2686,9 +3161,18 @@ class StubGenerator: public StubCodeGenerator { address generate_data_cache_writeback() { const Register line = c_rarg0; // address of line to write back + StubId stub_id = StubId::stubgen_data_cache_writeback_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_data_cache_writeback_id; StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -2697,15 +3181,27 @@ class StubGenerator: public StubCodeGenerator { __ leave(); __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address generate_data_cache_writeback_sync() { + StubId stub_id = StubId::stubgen_data_cache_writeback_sync_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + const Register is_pre = c_rarg0; // pre or post sync __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_data_cache_writeback_sync_id; StubCodeMark mark(this, stub_id); // pre wbsync is a no-op @@ -2720,6 +3216,9 @@ class StubGenerator: public StubCodeGenerator { __ leave(); __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -2882,8 +3381,18 @@ class StubGenerator: public StubCodeGenerator { // c_rarg2 - K (key) in little endian int array // address generate_aescrypt_encryptBlock() { - __ align(CodeEntryAlignment); + assert(UseAES, "need AES cryptographic extension support"); StubId stub_id = StubId::stubgen_aescrypt_encryptBlock_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); const Register from = c_rarg0; // source array address @@ -2904,6 +3413,9 @@ class StubGenerator: public StubCodeGenerator { __ leave(); __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -2916,8 +3428,17 @@ class StubGenerator: public StubCodeGenerator { // address generate_aescrypt_decryptBlock() { assert(UseAES, "need AES cryptographic extension support"); - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_aescrypt_decryptBlock_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); Label L_doLast; @@ -2938,6 +3459,9 @@ class StubGenerator: public StubCodeGenerator { __ leave(); __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -2955,8 +3479,17 @@ class StubGenerator: public StubCodeGenerator { // address generate_cipherBlockChaining_encryptAESCrypt() { assert(UseAES, "need AES cryptographic extension support"); - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_cipherBlockChaining_encryptAESCrypt_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); Label L_loadkeys_44, L_loadkeys_52, L_aes_loop, L_rounds_44, L_rounds_52; @@ -3043,6 +3576,9 @@ class StubGenerator: public StubCodeGenerator { __ leave(); __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -3060,8 +3596,17 @@ class StubGenerator: public StubCodeGenerator { // address generate_cipherBlockChaining_decryptAESCrypt() { assert(UseAES, "need AES cryptographic extension support"); - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_cipherBlockChaining_decryptAESCrypt_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); Label L_loadkeys_44, L_loadkeys_52, L_aes_loop, L_rounds_44, L_rounds_52; @@ -3152,6 +3697,9 @@ class StubGenerator: public StubCodeGenerator { __ leave(); __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -3188,6 +3736,16 @@ class StubGenerator: public StubCodeGenerator { // r0 - input length // address generate_counterMode_AESCrypt() { + StubId stub_id = StubId::stubgen_counterMode_AESCrypt_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + const Register in = c_rarg0; const Register out = c_rarg1; const Register key = c_rarg2; @@ -3248,7 +3806,6 @@ class StubGenerator: public StubCodeGenerator { // Wide bulk encryption of whole blocks. __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_counterMode_AESCrypt_id; StubCodeMark mark(this, stub_id); const address start = __ pc(); __ enter(); @@ -3435,6 +3992,9 @@ class StubGenerator: public StubCodeGenerator { __ strw(used, Address(used_ptr)); __ b(large_block_return); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -3451,9 +4011,17 @@ class StubGenerator: public StubCodeGenerator { // return - number of processed bytes address generate_galoisCounterMode_AESCrypt() { Label ghash_polynomial; // local data generated after code - - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_galoisCounterMode_AESCrypt_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); @@ -3567,6 +4135,9 @@ class StubGenerator: public StubCodeGenerator { // 128-bit vector __ emit_int64(0x87); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -3685,6 +4256,15 @@ class StubGenerator: public StubCodeGenerator { default: ShouldNotReachHere(); } + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); @@ -3815,6 +4395,9 @@ class StubGenerator: public StubCodeGenerator { __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -3838,6 +4421,14 @@ class StubGenerator: public StubCodeGenerator { default: ShouldNotReachHere(); } + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); @@ -3919,6 +4510,9 @@ class StubGenerator: public StubCodeGenerator { __ emit_int32(0x8f1bbcdc); __ emit_int32(0xca62c1d6); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -3943,25 +4537,14 @@ class StubGenerator: public StubCodeGenerator { default: ShouldNotReachHere(); } - - static const uint32_t round_consts[64] = { - 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, - 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, - 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, - 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, - 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, - 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, - 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, - 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, - 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, - 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, - 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, - 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, - 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, - 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, - 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, - 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2, - }; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); @@ -3987,7 +4570,7 @@ class StubGenerator: public StubCodeGenerator { // t1 == v7 // load 16 keys to v16..v31 - __ lea(rscratch1, ExternalAddress((address)round_consts)); + __ lea(rscratch1, ExternalAddress((address)_sha256_round_consts)); __ ld1(v16, v17, v18, v19, __ T4S, __ post(rscratch1, 64)); __ ld1(v20, v21, v22, v23, __ T4S, __ post(rscratch1, 64)); __ ld1(v24, v25, v26, v27, __ T4S, __ post(rscratch1, 64)); @@ -4048,6 +4631,9 @@ class StubGenerator: public StubCodeGenerator { __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -4099,36 +4685,14 @@ class StubGenerator: public StubCodeGenerator { default: ShouldNotReachHere(); } - - static const uint64_t round_consts[80] = { - 0x428A2F98D728AE22L, 0x7137449123EF65CDL, 0xB5C0FBCFEC4D3B2FL, - 0xE9B5DBA58189DBBCL, 0x3956C25BF348B538L, 0x59F111F1B605D019L, - 0x923F82A4AF194F9BL, 0xAB1C5ED5DA6D8118L, 0xD807AA98A3030242L, - 0x12835B0145706FBEL, 0x243185BE4EE4B28CL, 0x550C7DC3D5FFB4E2L, - 0x72BE5D74F27B896FL, 0x80DEB1FE3B1696B1L, 0x9BDC06A725C71235L, - 0xC19BF174CF692694L, 0xE49B69C19EF14AD2L, 0xEFBE4786384F25E3L, - 0x0FC19DC68B8CD5B5L, 0x240CA1CC77AC9C65L, 0x2DE92C6F592B0275L, - 0x4A7484AA6EA6E483L, 0x5CB0A9DCBD41FBD4L, 0x76F988DA831153B5L, - 0x983E5152EE66DFABL, 0xA831C66D2DB43210L, 0xB00327C898FB213FL, - 0xBF597FC7BEEF0EE4L, 0xC6E00BF33DA88FC2L, 0xD5A79147930AA725L, - 0x06CA6351E003826FL, 0x142929670A0E6E70L, 0x27B70A8546D22FFCL, - 0x2E1B21385C26C926L, 0x4D2C6DFC5AC42AEDL, 0x53380D139D95B3DFL, - 0x650A73548BAF63DEL, 0x766A0ABB3C77B2A8L, 0x81C2C92E47EDAEE6L, - 0x92722C851482353BL, 0xA2BFE8A14CF10364L, 0xA81A664BBC423001L, - 0xC24B8B70D0F89791L, 0xC76C51A30654BE30L, 0xD192E819D6EF5218L, - 0xD69906245565A910L, 0xF40E35855771202AL, 0x106AA07032BBD1B8L, - 0x19A4C116B8D2D0C8L, 0x1E376C085141AB53L, 0x2748774CDF8EEB99L, - 0x34B0BCB5E19B48A8L, 0x391C0CB3C5C95A63L, 0x4ED8AA4AE3418ACBL, - 0x5B9CCA4F7763E373L, 0x682E6FF3D6B2B8A3L, 0x748F82EE5DEFB2FCL, - 0x78A5636F43172F60L, 0x84C87814A1F0AB72L, 0x8CC702081A6439ECL, - 0x90BEFFFA23631E28L, 0xA4506CEBDE82BDE9L, 0xBEF9A3F7B2C67915L, - 0xC67178F2E372532BL, 0xCA273ECEEA26619CL, 0xD186B8C721C0C207L, - 0xEADA7DD6CDE0EB1EL, 0xF57D4F7FEE6ED178L, 0x06F067AA72176FBAL, - 0x0A637DC5A2C898A6L, 0x113F9804BEF90DAEL, 0x1B710B35131C471BL, - 0x28DB77F523047D84L, 0x32CAAB7B40C72493L, 0x3C9EBE0A15C9BEBCL, - 0x431D67C49C100D4CL, 0x4CC5D4BECB3E42B6L, 0x597F299CFC657E2AL, - 0x5FCB6FAB3AD6FAECL, 0x6C44198C4A475817L - }; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); @@ -4151,7 +4715,7 @@ class StubGenerator: public StubCodeGenerator { __ ld1(v8, v9, v10, v11, __ T2D, state); // load first 4 round constants - __ lea(rscratch1, ExternalAddress((address)round_consts)); + __ lea(rscratch1, ExternalAddress((address)_sha512_round_consts)); __ ld1(v20, v21, v22, v23, __ T2D, __ post(rscratch1, 64)); __ BIND(sha512_loop); @@ -4236,6 +4800,9 @@ class StubGenerator: public StubCodeGenerator { __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -4349,17 +4916,14 @@ class StubGenerator: public StubCodeGenerator { default: ShouldNotReachHere(); } - - static const uint64_t round_consts[24] = { - 0x0000000000000001L, 0x0000000000008082L, 0x800000000000808AL, - 0x8000000080008000L, 0x000000000000808BL, 0x0000000080000001L, - 0x8000000080008081L, 0x8000000000008009L, 0x000000000000008AL, - 0x0000000000000088L, 0x0000000080008009L, 0x000000008000000AL, - 0x000000008000808BL, 0x800000000000008BL, 0x8000000000008089L, - 0x8000000000008003L, 0x8000000000008002L, 0x8000000000000080L, - 0x000000000000800AL, 0x800000008000000AL, 0x8000000080008081L, - 0x8000000000008080L, 0x0000000080000001L, 0x8000000080008008L - }; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); @@ -4396,7 +4960,7 @@ class StubGenerator: public StubCodeGenerator { __ movw(rscratch2, 24); // load round_constants base - __ lea(rscratch1, ExternalAddress((address) round_consts)); + __ lea(rscratch1, ExternalAddress((address) _sha3_round_consts)); // load input __ ld1(v25, v26, v27, v28, __ T8B, __ post(buf, 32)); @@ -4488,6 +5052,9 @@ class StubGenerator: public StubCodeGenerator { __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -4495,21 +5062,21 @@ class StubGenerator: public StubCodeGenerator { // c_rarg0 - long[] state0 // c_rarg1 - long[] state1 address generate_double_keccak() { - static const uint64_t round_consts[24] = { - 0x0000000000000001L, 0x0000000000008082L, 0x800000000000808AL, - 0x8000000080008000L, 0x000000000000808BL, 0x0000000080000001L, - 0x8000000080008081L, 0x8000000000008009L, 0x000000000000008AL, - 0x0000000000000088L, 0x0000000080008009L, 0x000000008000000AL, - 0x000000008000808BL, 0x800000000000008BL, 0x8000000000008089L, - 0x8000000000008003L, 0x8000000000008002L, 0x8000000000000080L, - 0x000000000000800AL, 0x800000008000000AL, 0x8000000080008081L, - 0x8000000000008080L, 0x0000000080000001L, 0x8000000080008008L - }; + StubId stub_id = StubId::stubgen_double_keccak_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } // Implements the double_keccak() method of the // sun.secyrity.provider.SHA3Parallel class __ align(CodeEntryAlignment); - StubCodeMark mark(this, "StubRoutines", "double_keccak"); + + StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); @@ -4546,7 +5113,7 @@ class StubGenerator: public StubCodeGenerator { __ movw(rscratch2, 24); // load round_constants base - __ lea(rscratch1, ExternalAddress((address) round_consts)); + __ lea(rscratch1, ExternalAddress((address) _double_keccak_round_consts)); __ BIND(rounds24_loop); __ subw(rscratch2, rscratch2, 1); @@ -4578,6 +5145,9 @@ class StubGenerator: public StubCodeGenerator { __ mov(r0, zr); // return 0 __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -4611,9 +5181,18 @@ class StubGenerator: public StubCodeGenerator { // vectors write their first lane back to the keystream buffer, followed // by the second lane from all vectors and so on. address generate_chacha20Block_blockpar() { + StubId stub_id = StubId::stubgen_chacha20Block_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + Label L_twoRounds, L_cc20_const; __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_chacha20Block_id; StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); @@ -4770,6 +5349,9 @@ class StubGenerator: public StubCodeGenerator { __ emit_int64(0x0605040702010003UL); __ emit_int64(0x0E0D0C0F0A09080BUL); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -5258,9 +5840,17 @@ class StubGenerator: public StubCodeGenerator { // coeffs (short[256]) = c_rarg0 // ntt_zetas (short[256]) = c_rarg1 address generate_kyberNtt() { + StubId stub_id = StubId::stubgen_kyberNtt_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_kyberNtt_id; StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); @@ -5486,6 +6076,9 @@ class StubGenerator: public StubCodeGenerator { __ mov(r0, zr); // return 0 __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -5496,9 +6089,17 @@ class StubGenerator: public StubCodeGenerator { // coeffs (short[256]) = c_rarg0 // ntt_zetas (short[256]) = c_rarg1 address generate_kyberInverseNtt() { + StubId stub_id = StubId::stubgen_kyberInverseNtt_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_kyberInverseNtt_id; StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); @@ -5770,6 +6371,9 @@ class StubGenerator: public StubCodeGenerator { __ mov(r0, zr); // return 0 __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -5783,9 +6387,17 @@ class StubGenerator: public StubCodeGenerator { // nttb (short[256]) = c_rarg2 // zetas (short[128]) = c_rarg3 address generate_kyberNttMult() { + StubId stub_id = StubId::stubgen_kyberNttMult_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_kyberNttMult_id; StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); @@ -5889,6 +6501,9 @@ class StubGenerator: public StubCodeGenerator { __ mov(r0, zr); // return 0 __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -5900,9 +6515,17 @@ class StubGenerator: public StubCodeGenerator { // a (short[256]) = c_rarg1 // b (short[256]) = c_rarg2 address generate_kyberAddPoly_2() { + StubId stub_id = StubId::stubgen_kyberAddPoly_2_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_kyberAddPoly_2_id; StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); @@ -5973,6 +6596,9 @@ class StubGenerator: public StubCodeGenerator { __ mov(r0, zr); // return 0 __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -5985,9 +6611,17 @@ class StubGenerator: public StubCodeGenerator { // b (short[256]) = c_rarg2 // c (short[256]) = c_rarg3 address generate_kyberAddPoly_3() { + StubId stub_id = StubId::stubgen_kyberAddPoly_3_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_kyberAddPoly_3_id; StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); @@ -6072,6 +6706,9 @@ class StubGenerator: public StubCodeGenerator { __ mov(r0, zr); // return 0 __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -6088,10 +6725,19 @@ class StubGenerator: public StubCodeGenerator { // parsed (short[112 or 256]) = c_rarg2 // parsedLength (112 or 256) = c_rarg3 address generate_kyber12To16() { + StubId stub_id = StubId::stubgen_kyber12To16_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + Label L_F00, L_loop, L_end; __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_kyber12To16_id; StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); @@ -6288,6 +6934,9 @@ class StubGenerator: public StubCodeGenerator { __ emit_int64(0x0f000f000f000f00); __ emit_int64(0x0f000f000f000f00); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -6297,9 +6946,17 @@ class StubGenerator: public StubCodeGenerator { // // coeffs (short[256]) = c_rarg0 address generate_kyberBarrettReduce() { + StubId stub_id = StubId::stubgen_kyberBarrettReduce_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_kyberBarrettReduce_id; StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); @@ -6381,6 +7038,9 @@ class StubGenerator: public StubCodeGenerator { __ mov(r0, zr); // return 0 __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -6544,9 +7204,17 @@ class StubGenerator: public StubCodeGenerator { // coeffs (int[256]) = c_rarg0 // zetas (int[256]) = c_rarg1 address generate_dilithiumAlmostNtt() { + StubId stub_id = StubId::stubgen_dilithiumAlmostNtt_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_dilithiumAlmostNtt_id; StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); @@ -6659,6 +7327,9 @@ class StubGenerator: public StubCodeGenerator { __ mov(r0, zr); // return 0 __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -6751,9 +7422,17 @@ class StubGenerator: public StubCodeGenerator { // coeffs (int[256]) = c_rarg0 // zetas (int[256]) = c_rarg1 address generate_dilithiumAlmostInverseNtt() { + StubId stub_id = StubId::stubgen_dilithiumAlmostInverseNtt_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_dilithiumAlmostInverseNtt_id; StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); @@ -6851,6 +7530,9 @@ class StubGenerator: public StubCodeGenerator { __ mov(r0, zr); // return 0 __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -6864,9 +7546,17 @@ class StubGenerator: public StubCodeGenerator { // poly1 (int[256]) = c_rarg1 // poly2 (int[256]) = c_rarg2 address generate_dilithiumNttMult() { - - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_dilithiumNttMult_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); @@ -6917,6 +7607,9 @@ class StubGenerator: public StubCodeGenerator { __ mov(r0, zr); // return 0 __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -6928,9 +7621,17 @@ class StubGenerator: public StubCodeGenerator { // coeffs (int[256]) = c_rarg0 // constant (int) = c_rarg1 address generate_dilithiumMontMulByConstant() { + StubId stub_id = StubId::stubgen_dilithiumMontMulByConstant_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_dilithiumMontMulByConstant_id; StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); @@ -6978,6 +7679,9 @@ class StubGenerator: public StubCodeGenerator { __ mov(r0, zr); // return 0 __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -6992,9 +7696,17 @@ class StubGenerator: public StubCodeGenerator { // twoGamma2 (int) = c_rarg3 // multiplier (int) = c_rarg4 address generate_dilithiumDecomposePoly() { + StubId stub_id = StubId::stubgen_dilithiumDecomposePoly_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_dilithiumDecomposePoly_id; StubCodeMark mark(this, stub_id); address start = __ pc(); Label L_loop; @@ -7136,6 +7848,9 @@ class StubGenerator: public StubCodeGenerator { __ mov(r0, zr); // return 0 __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -7275,17 +7990,14 @@ class StubGenerator: public StubCodeGenerator { default: ShouldNotReachHere(); } - - static const uint64_t round_consts[24] = { - 0x0000000000000001L, 0x0000000000008082L, 0x800000000000808AL, - 0x8000000080008000L, 0x000000000000808BL, 0x0000000080000001L, - 0x8000000080008081L, 0x8000000000008009L, 0x000000000000008AL, - 0x0000000000000088L, 0x0000000080008009L, 0x000000008000000AL, - 0x000000008000808BL, 0x800000000000008BL, 0x8000000000008089L, - 0x8000000000008003L, 0x8000000000008002L, 0x8000000000000080L, - 0x000000000000800AL, 0x800000008000000AL, 0x8000000080008081L, - 0x8000000000008080L, 0x0000000080000001L, 0x8000000080008008L - }; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); @@ -7441,7 +8153,7 @@ class StubGenerator: public StubCodeGenerator { __ fmovs(v1, 1.0); // exact representation __ str(buf, Address(sp, 16)); - __ lea(tmp3, ExternalAddress((address) round_consts)); + __ lea(tmp3, ExternalAddress((address) _sha3_round_consts)); __ BIND(loop_body); keccak_round_gpr(can_use_fp, can_use_r18, tmp3, @@ -7496,6 +8208,9 @@ class StubGenerator: public StubCodeGenerator { __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -7512,9 +8227,17 @@ class StubGenerator: public StubCodeGenerator { */ address generate_updateBytesCRC32() { assert(UseCRC32Intrinsics, "what are we doing here?"); + StubId stub_id = StubId::stubgen_updateBytesCRC32_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_updateBytesCRC32_id; StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -7537,6 +8260,9 @@ class StubGenerator: public StubCodeGenerator { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -7554,9 +8280,17 @@ class StubGenerator: public StubCodeGenerator { */ address generate_updateBytesCRC32C() { assert(UseCRC32CIntrinsics, "what are we doing here?"); - - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_updateBytesCRC32C_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -7579,6 +8313,9 @@ class StubGenerator: public StubCodeGenerator { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -7594,8 +8331,17 @@ class StubGenerator: public StubCodeGenerator { * c_rarg0 - int adler result */ address generate_updateBytesAdler32() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_updateBytesAdler32_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -7765,6 +8511,9 @@ class StubGenerator: public StubCodeGenerator { __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -7816,8 +8565,17 @@ class StubGenerator: public StubCodeGenerator { * c_rarg4 - z address */ address generate_multiplyToLen() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_multiplyToLen_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -7842,6 +8600,9 @@ class StubGenerator: public StubCodeGenerator { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -7849,8 +8610,17 @@ class StubGenerator: public StubCodeGenerator { // squareToLen algorithm for sizes 1..127 described in java code works // faster than multiply_to_len on some CPUs and slower on others, but // multiply_to_len shows a bit better overall results - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_squareToLen_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -7879,12 +8649,25 @@ class StubGenerator: public StubCodeGenerator { __ pop(spilled_regs, sp); __ leave(); __ ret(lr); + + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address generate_mulAdd() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_mulAdd_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -7901,6 +8684,9 @@ class StubGenerator: public StubCodeGenerator { __ leave(); __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -7914,8 +8700,17 @@ class StubGenerator: public StubCodeGenerator { // c_rarg4 - numIter // address generate_bigIntegerRightShift() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_bigIntegerRightShiftWorker_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -8024,6 +8819,9 @@ class StubGenerator: public StubCodeGenerator { __ BIND(Exit); __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -8039,6 +8837,15 @@ class StubGenerator: public StubCodeGenerator { address generate_bigIntegerLeftShift() { __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_bigIntegerLeftShiftWorker_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -8135,10 +8942,28 @@ class StubGenerator: public StubCodeGenerator { __ BIND(Exit); __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address generate_count_positives(address &count_positives_long) { + StubId stub_id = StubId::stubgen_count_positives_id; + GrowableArray
entries; + int entry_count = StubInfo::entry_count(stub_id); + // We have an extra entry for count_positives_long. + assert(entry_count == 2, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, &entries); + assert(entries.length() == 1, + "unexpected extra entry count %d", entries.length()); + count_positives_long = entries.at(0); + return start; + } + const u1 large_loop_size = 64; const uint64_t UPPER_BIT_MASK=0x8080808080808080; int dcache_line = VM_Version::dcache_line_size(); @@ -8147,7 +8972,6 @@ class StubGenerator: public StubCodeGenerator { __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_count_positives_id; StubCodeMark mark(this, stub_id); address entry = __ pc(); @@ -8190,6 +9014,7 @@ class StubGenerator: public StubCodeGenerator { const RegSet spilled_regs = RegSet::range(tmp1, tmp5) + tmp6; count_positives_long = __ pc(); // 2nd entry point + entries.append(count_positives_long); __ enter(); @@ -8304,6 +9129,9 @@ class StubGenerator: public StubCodeGenerator { __ sub(result, result, len); __ ret(lr); + // record the stub entry and end plus the extra entry + store_archive_data(stub_id, entry, __ pc(), &entries); + return entry; } @@ -8394,6 +9222,16 @@ class StubGenerator: public StubCodeGenerator { // r3-r5 are reserved temporary registers // Clobbers: v0-v7 when UseSIMDForArrayEquals, rscratch1, rscratch2 address generate_large_array_equals() { + StubId stub_id = StubId::stubgen_large_array_equals_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + Register a1 = r1, a2 = r2, result = r0, cnt1 = r10, tmp1 = rscratch1, tmp2 = rscratch2, tmp3 = r3, tmp4 = r4, tmp5 = r5, tmp6 = r11, tmp7 = r12, tmp8 = r13; @@ -8409,7 +9247,6 @@ class StubGenerator: public StubCodeGenerator { __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_large_array_equals_id; StubCodeMark mark(this, stub_id); address entry = __ pc(); @@ -8484,6 +9321,10 @@ class StubGenerator: public StubCodeGenerator { __ bind(NOT_EQUAL_NO_POP); __ leave(); __ ret(lr); + + // record the stub entry and end + store_archive_data(stub_id, entry, __ pc()); + return entry; } @@ -8492,6 +9333,36 @@ class StubGenerator: public StubCodeGenerator { // cnt = r2 - elements count // Clobbers: v0-v13, rscratch1, rscratch2 address generate_large_arrays_hashcode(BasicType eltype) { + StubId stub_id; + switch (eltype) { + case T_BOOLEAN: + stub_id = StubId::stubgen_large_arrays_hashcode_boolean_id; + break; + case T_BYTE: + stub_id = StubId::stubgen_large_arrays_hashcode_byte_id; + break; + case T_CHAR: + stub_id = StubId::stubgen_large_arrays_hashcode_char_id; + break; + case T_SHORT: + stub_id = StubId::stubgen_large_arrays_hashcode_short_id; + break; + case T_INT: + stub_id = StubId::stubgen_large_arrays_hashcode_int_id; + break; + default: + stub_id = StubId::NO_STUBID; + ShouldNotReachHere(); + }; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + const Register result = r0, ary = r1, cnt = r2; const FloatRegister vdata0 = v3, vdata1 = v2, vdata2 = v1, vdata3 = v0; const FloatRegister vmul0 = v4, vmul1 = v5, vmul2 = v6, vmul3 = v7; @@ -8535,28 +9406,6 @@ class StubGenerator: public StubCodeGenerator { __ align(CodeEntryAlignment); - StubId stub_id; - switch (eltype) { - case T_BOOLEAN: - stub_id = StubId::stubgen_large_arrays_hashcode_boolean_id; - break; - case T_BYTE: - stub_id = StubId::stubgen_large_arrays_hashcode_byte_id; - break; - case T_CHAR: - stub_id = StubId::stubgen_large_arrays_hashcode_char_id; - break; - case T_SHORT: - stub_id = StubId::stubgen_large_arrays_hashcode_short_id; - break; - case T_INT: - stub_id = StubId::stubgen_large_arrays_hashcode_int_id; - break; - default: - stub_id = StubId::NO_STUBID; - ShouldNotReachHere(); - }; - StubCodeMark mark(this, stub_id); address entry = __ pc(); @@ -8791,12 +9640,24 @@ class StubGenerator: public StubCodeGenerator { __ leave(); __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, entry, __ pc()); + return entry; } address generate_dsin_dcos(bool isCos) { - __ align(CodeEntryAlignment); StubId stub_id = (isCos ? StubId::stubgen_dcos_id : StubId::stubgen_dsin_id); + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); __ generate_dsin_dcos(isCos, (address)StubRoutines::aarch64::_npio2_hw, @@ -8804,6 +9665,10 @@ class StubGenerator: public StubCodeGenerator { (address)StubRoutines::aarch64::_pio2, (address)StubRoutines::aarch64::_dsin_coef, (address)StubRoutines::aarch64::_dcos_coef); + + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -8847,8 +9712,17 @@ class StubGenerator: public StubCodeGenerator { // r10 = tmp1 // r11 = tmp2 address generate_compare_long_string_different_encoding(bool isLU) { - __ align(CodeEntryAlignment); StubId stub_id = (isLU ? StubId::stubgen_compare_long_string_LU_id : StubId::stubgen_compare_long_string_UL_id); + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address entry = __ pc(); Label SMALL_LOOP, TAIL, TAIL_LOAD_16, LOAD_LAST, DIFF1, DIFF2, @@ -8950,20 +9824,37 @@ class StubGenerator: public StubCodeGenerator { __ subw(result, tmp1, rscratch1); __ bind(DONE); __ ret(lr); - return entry; + + // record the stub entry and end + store_archive_data(stub_id, entry, __ pc()); + + return entry; } // r0 = input (float16) // v0 = result (float) // v1 = temporary float register address generate_float16ToFloat() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_hf2f_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address entry = __ pc(); BLOCK_COMMENT("Entry:"); __ flt16_to_flt(v0, r0, v1); __ ret(lr); + + // record the stub entry and end + store_archive_data(stub_id, entry, __ pc()); + return entry; } @@ -8971,19 +9862,39 @@ class StubGenerator: public StubCodeGenerator { // r0 = result (float16) // v1 = temporary float register address generate_floatToFloat16() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_f2hf_id; + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address entry = __ pc(); BLOCK_COMMENT("Entry:"); __ flt_to_flt16(r0, v0, v1); __ ret(lr); + + // record the stub entry and end + store_archive_data(stub_id, entry, __ pc()); + return entry; } address generate_method_entry_barrier() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_method_entry_barrier_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); Label deoptimize_label; @@ -9037,6 +9948,9 @@ class StubGenerator: public StubCodeGenerator { __ mov(sp, rscratch1); __ br(rscratch2); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -9048,8 +9962,18 @@ class StubGenerator: public StubCodeGenerator { // r10 = tmp1 // r11 = tmp2 address generate_compare_long_string_same_encoding(bool isLL) { - __ align(CodeEntryAlignment); StubId stub_id = (isLL ? StubId::stubgen_compare_long_string_LL_id : StubId::stubgen_compare_long_string_UU_id); + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); + StubCodeMark mark(this, stub_id); address entry = __ pc(); Register result = r0, str1 = r1, cnt1 = r2, str2 = r3, cnt2 = r4, @@ -9157,6 +10081,10 @@ class StubGenerator: public StubCodeGenerator { __ bind(LENGTH_DIFF); __ ret(lr); + + // record the stub entry and end + store_archive_data(stub_id, entry, __ pc()); + return entry; } @@ -9188,8 +10116,17 @@ class StubGenerator: public StubCodeGenerator { case UU: stub_id = StubId::stubgen_compare_long_string_UU_id; break; default: ShouldNotReachHere(); } + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); + StubCodeMark mark(this, stub_id); address entry = __ pc(); Register result = r0, str1 = r1, cnt1 = r2, str2 = r3, cnt2 = r4, tmp1 = r10, tmp2 = r11; @@ -9224,8 +10161,6 @@ class StubGenerator: public StubCodeGenerator { ShouldNotReachHere(); \ } - StubCodeMark mark(this, stub_id); - __ mov(idx, 0); __ sve_whilelt(pgtmp1, mode == LL ? __ B : __ H, idx, cnt); @@ -9269,6 +10204,10 @@ class StubGenerator: public StubCodeGenerator { __ bind(DONE); __ ret(lr); #undef LOAD_PAIR + + // record the stub entry and end + store_archive_data(stub_id, entry, __ pc()); + return entry; } @@ -9330,6 +10269,15 @@ class StubGenerator: public StubCodeGenerator { stub_id = StubId::stubgen_string_indexof_linear_uu_id; } } + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address entry = __ pc(); @@ -9598,6 +10546,10 @@ class StubGenerator: public StubCodeGenerator { __ BIND(DONE); __ pop(spilled_regs, sp); __ ret(lr); + + // record the stub entry and end + store_archive_data(stub_id, entry, __ pc()); + return entry; } @@ -9628,8 +10580,17 @@ class StubGenerator: public StubCodeGenerator { // v1 = loaded 8 bytes // Clobbers: r0, r1, r3, rscratch1, rflags, v0-v6 address generate_large_byte_array_inflate() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_large_byte_array_inflate_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address entry = __ pc(); Label LOOP, LOOP_START, LOOP_PRFM, LOOP_PRFM_START, DONE; @@ -9668,6 +10629,10 @@ class StubGenerator: public StubCodeGenerator { __ br(__ GE, LOOP); __ bind(DONE); __ ret(lr); + + // record the stub entry and end + store_archive_data(stub_id, entry, __ pc()); + return entry; } @@ -9683,7 +10648,7 @@ class StubGenerator: public StubCodeGenerator { * Output: * Updated state at c_rarg0 */ - address generate_ghash_processBlocks() { + address generate_ghash_processBlocks_small() { // Bafflingly, GCM uses little-endian for the byte order, but // big-endian for the bit order. For example, the polynomial 1 is // represented as the 16-byte string 80 00 00 00 | 12 bytes of 00. @@ -9695,7 +10660,16 @@ class StubGenerator: public StubCodeGenerator { // that) and keep the data in little-endian bit order through the // calculation, bit-reversing the inputs and outputs. - StubId stub_id = StubId::stubgen_ghash_processBlocks_id; + StubId stub_id = StubId::stubgen_ghash_processBlocks_small_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + StubCodeMark mark(this, stub_id); Label polynomial; // local data generated at end of stub __ align(CodeEntryAlignment); @@ -9759,16 +10733,26 @@ class StubGenerator: public StubCodeGenerator { // 128-bit vector __ emit_int64(0x87); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } - address generate_ghash_processBlocks_wide() { - address small = generate_ghash_processBlocks(); + address generate_ghash_processBlocks(address small) { + StubId stub_id = StubId::stubgen_ghash_processBlocks_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } - StubId stub_id = StubId::stubgen_ghash_processBlocks_wide_id; - StubCodeMark mark(this, stub_id); Label polynomial; // local data generated after stub __ align(CodeEntryAlignment); + StubCodeMark mark(this, stub_id); address start = __ pc(); Register state = c_rarg0; @@ -9811,8 +10795,10 @@ class StubGenerator: public StubCodeGenerator { // 128-bit vector __ emit_int64(0x87); - return start; + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } void generate_base64_encode_simdround(Register src, Register dst, @@ -9863,24 +10849,17 @@ class StubGenerator: public StubCodeGenerator { */ address generate_base64_encodeBlock() { - static const char toBase64[64] = { - 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', - 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', - 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', - 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', - '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/' - }; - - static const char toBase64URL[64] = { - 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', - 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', - 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', - 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', - '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_' - }; + StubId stub_id = StubId::stubgen_base64_encodeBlock_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_base64_encodeBlock_id; StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -9902,9 +10881,9 @@ class StubGenerator: public StubCodeGenerator { __ sub(length, send, soff); // load the codec base address - __ lea(codec, ExternalAddress((address) toBase64)); + __ lea(codec, ExternalAddress((address) _encodeBlock_toBase64)); __ cbz(isURL, ProcessData); - __ lea(codec, ExternalAddress((address) toBase64URL)); + __ lea(codec, ExternalAddress((address) _encodeBlock_toBase64URL)); __ BIND(ProcessData); @@ -9957,6 +10936,9 @@ class StubGenerator: public StubCodeGenerator { __ BIND(Exit); __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -10078,78 +11060,17 @@ class StubGenerator: public StubCodeGenerator { // on http://0x80.pl/articles/base64-simd-neon.html#encoding-quadwords, in section // titled "Base64 decoding". - // Non-SIMD lookup tables are mostly dumped from fromBase64 array used in java.util.Base64, - // except the trailing character '=' is also treated illegal value in this intrinsic. That - // is java.util.Base64.fromBase64['='] = -2, while fromBase(URL)64ForNoSIMD['='] = 255 here. - static const uint8_t fromBase64ForNoSIMD[256] = { - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u, 255u, 63u, - 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u, 14u, - 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u, 255u, - 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u, 40u, - 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - }; - - static const uint8_t fromBase64URLForNoSIMD[256] = { - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u, - 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u, 14u, - 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u, 63u, - 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u, 40u, - 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - }; - - // A legal value of base64 code is in range [0, 127]. We need two lookups - // with tbl/tbx and combine them to get the decode data. The 1st table vector - // lookup use tbl, out of range indices are set to 0 in destination. The 2nd - // table vector lookup use tbx, out of range indices are unchanged in - // destination. Input [64..126] is mapped to index [65, 127] in second lookup. - // The value of index 64 is set to 0, so that we know that we already get the - // decoded data with the 1st lookup. - static const uint8_t fromBase64ForSIMD[128] = { - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u, 255u, 63u, - 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u, - 0u, 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u, - 14u, 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u, - 255u, 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u, - 40u, 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u, - }; - - static const uint8_t fromBase64URLForSIMD[128] = { - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u, - 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u, - 0u, 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u, - 14u, 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u, - 63u, 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u, - 40u, 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u, - }; + StubId stub_id = StubId::stubgen_base64_decodeBlock_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_base64_decodeBlock_id; StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -10178,9 +11099,9 @@ class StubGenerator: public StubCodeGenerator { __ sub(length, send, soff); __ bfm(length, zr, 0, 1); - __ lea(nosimd_codec, ExternalAddress((address) fromBase64ForNoSIMD)); + __ lea(nosimd_codec, ExternalAddress((address) _decodeBlock_fromBase64ForNoSIMD)); __ cbz(isURL, ProcessData); - __ lea(nosimd_codec, ExternalAddress((address) fromBase64URLForNoSIMD)); + __ lea(nosimd_codec, ExternalAddress((address) _decodeBlock_fromBase64URLForNoSIMD)); __ BIND(ProcessData); __ mov(rscratch1, length); @@ -10225,9 +11146,9 @@ class StubGenerator: public StubCodeGenerator { __ cbzw(rscratch1, Exit); __ sub(length, length, 80); - __ lea(simd_codec, ExternalAddress((address) fromBase64ForSIMD)); + __ lea(simd_codec, ExternalAddress((address) _decodeBlock_fromBase64ForSIMD)); __ cbz(isURL, SIMDEnter); - __ lea(simd_codec, ExternalAddress((address) fromBase64URLForSIMD)); + __ lea(simd_codec, ExternalAddress((address) _decodeBlock_fromBase64URLForSIMD)); __ BIND(SIMDEnter); __ ld1(v0, v1, v2, v3, __ T16B, __ post(simd_codec, 64)); @@ -10260,24 +11181,55 @@ class StubGenerator: public StubCodeGenerator { __ leave(); __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } // Support for spin waits. address generate_spin_wait() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_spin_wait_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); __ spin_wait(); __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } void generate_lookup_secondary_supers_table_stub() { StubId stub_id = StubId::stubgen_lookup_secondary_supers_table_id; + GrowableArray
entries; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == Klass::SECONDARY_SUPERS_TABLE_SIZE, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, &entries); + assert(entries.length() == 1, + "unexpected extra entry count %d", entries.length()); + StubRoutines::_lookup_secondary_supers_table_stubs[0] = start; + for (int slot = 1; slot < Klass::SECONDARY_SUPERS_TABLE_SIZE; slot++) { + StubRoutines::_lookup_secondary_supers_table_stubs[0] = entries.at(slot - 1); + } + return; + } + StubCodeMark mark(this, stub_id); const Register @@ -10291,8 +11243,15 @@ class StubGenerator: public StubCodeGenerator { const FloatRegister vtemp = v0; + address start; for (int slot = 0; slot < Klass::SECONDARY_SUPERS_TABLE_SIZE; slot++) { - StubRoutines::_lookup_secondary_supers_table_stubs[slot] = __ pc(); + address next_entry = __ pc(); + StubRoutines::_lookup_secondary_supers_table_stubs[slot] = next_entry; + if (slot == 0) { + start = next_entry; + } else { + entries.append(next_entry); + } Label L_success; __ enter(); __ lookup_secondary_supers_table_const(r_sub_klass, r_super_klass, @@ -10302,11 +11261,22 @@ class StubGenerator: public StubCodeGenerator { __ leave(); __ ret(lr); } + // record the stub entry and end plus all the auxiliary entries + store_archive_data(stub_id, start, __ pc(), &entries); } // Slow path implementation for UseSecondarySupersTable. address generate_lookup_secondary_supers_table_slow_path_stub() { StubId stub_id = StubId::stubgen_lookup_secondary_supers_table_slow_path_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -10321,6 +11291,9 @@ class StubGenerator: public StubCodeGenerator { __ lookup_secondary_supers_table_slow_path(r_super_klass, r_array_base, r_array_index, r_bitmap, temp1, result); __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -10460,14 +11433,45 @@ class StubGenerator: public StubCodeGenerator { if (! UseLSE) { return; } - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_atomic_entry_points_id; + GrowableArray
entries; + int entry_count = StubInfo::entry_count(stub_id); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, &entries); + assert(entries.length() == entry_count - 1, + "unexpected extra entry count %d", entries.length()); + aarch64_atomic_fetch_add_4_impl = (aarch64_atomic_stub_t)start; + int idx = 0; + aarch64_atomic_fetch_add_8_impl = (aarch64_atomic_stub_t)entries.at(idx++); + aarch64_atomic_fetch_add_4_relaxed_impl = (aarch64_atomic_stub_t)entries.at(idx++); + aarch64_atomic_fetch_add_8_relaxed_impl = (aarch64_atomic_stub_t)entries.at(idx++); + aarch64_atomic_xchg_4_impl = (aarch64_atomic_stub_t)entries.at(idx++); + aarch64_atomic_xchg_8_impl = (aarch64_atomic_stub_t)entries.at(idx++); + aarch64_atomic_cmpxchg_1_impl = (aarch64_atomic_stub_t)entries.at(idx++); + aarch64_atomic_cmpxchg_4_impl = (aarch64_atomic_stub_t)entries.at(idx++); + aarch64_atomic_cmpxchg_8_impl = (aarch64_atomic_stub_t)entries.at(idx++); + aarch64_atomic_cmpxchg_1_relaxed_impl = (aarch64_atomic_stub_t)entries.at(idx++); + aarch64_atomic_cmpxchg_4_relaxed_impl = (aarch64_atomic_stub_t)entries.at(idx++); + aarch64_atomic_cmpxchg_8_relaxed_impl = (aarch64_atomic_stub_t)entries.at(idx++); + aarch64_atomic_cmpxchg_4_release_impl = (aarch64_atomic_stub_t)entries.at(idx++); + aarch64_atomic_cmpxchg_8_release_impl = (aarch64_atomic_stub_t)entries.at(idx++); + aarch64_atomic_cmpxchg_4_seq_cst_impl = (aarch64_atomic_stub_t)entries.at(idx++); + aarch64_atomic_cmpxchg_8_seq_cst_impl = (aarch64_atomic_stub_t)entries.at(idx++); + assert(idx == entries.length(), "sanity!"); + return; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address first_entry = __ pc(); - + address end; + { // ADD, memory_order_conservative AtomicStubMark mark_fetch_add_4(_masm, &aarch64_atomic_fetch_add_4_impl); gen_ldadd_entry(Assembler::word, memory_order_conservative); + AtomicStubMark mark_fetch_add_8(_masm, &aarch64_atomic_fetch_add_8_impl); gen_ldadd_entry(Assembler::xword, memory_order_conservative); @@ -10475,6 +11479,7 @@ class StubGenerator: public StubCodeGenerator { AtomicStubMark mark_fetch_add_4_relaxed (_masm, &aarch64_atomic_fetch_add_4_relaxed_impl); gen_ldadd_entry(MacroAssembler::word, memory_order_relaxed); + AtomicStubMark mark_fetch_add_8_relaxed (_masm, &aarch64_atomic_fetch_add_8_relaxed_impl); gen_ldadd_entry(MacroAssembler::xword, memory_order_relaxed); @@ -10482,14 +11487,17 @@ class StubGenerator: public StubCodeGenerator { // XCHG, memory_order_conservative AtomicStubMark mark_xchg_4(_masm, &aarch64_atomic_xchg_4_impl); gen_swpal_entry(Assembler::word); - AtomicStubMark mark_xchg_8_impl(_masm, &aarch64_atomic_xchg_8_impl); + + AtomicStubMark mark_xchg_8(_masm, &aarch64_atomic_xchg_8_impl); gen_swpal_entry(Assembler::xword); // CAS, memory_order_conservative AtomicStubMark mark_cmpxchg_1(_masm, &aarch64_atomic_cmpxchg_1_impl); gen_cas_entry(MacroAssembler::byte, memory_order_conservative); + AtomicStubMark mark_cmpxchg_4(_masm, &aarch64_atomic_cmpxchg_4_impl); gen_cas_entry(MacroAssembler::word, memory_order_conservative); + AtomicStubMark mark_cmpxchg_8(_masm, &aarch64_atomic_cmpxchg_8_impl); gen_cas_entry(MacroAssembler::xword, memory_order_conservative); @@ -10497,9 +11505,11 @@ class StubGenerator: public StubCodeGenerator { AtomicStubMark mark_cmpxchg_1_relaxed (_masm, &aarch64_atomic_cmpxchg_1_relaxed_impl); gen_cas_entry(MacroAssembler::byte, memory_order_relaxed); + AtomicStubMark mark_cmpxchg_4_relaxed (_masm, &aarch64_atomic_cmpxchg_4_relaxed_impl); gen_cas_entry(MacroAssembler::word, memory_order_relaxed); + AtomicStubMark mark_cmpxchg_8_relaxed (_masm, &aarch64_atomic_cmpxchg_8_relaxed_impl); gen_cas_entry(MacroAssembler::xword, memory_order_relaxed); @@ -10507,6 +11517,7 @@ class StubGenerator: public StubCodeGenerator { AtomicStubMark mark_cmpxchg_4_release (_masm, &aarch64_atomic_cmpxchg_4_release_impl); gen_cas_entry(MacroAssembler::word, memory_order_release); + AtomicStubMark mark_cmpxchg_8_release (_masm, &aarch64_atomic_cmpxchg_8_release_impl); gen_cas_entry(MacroAssembler::xword, memory_order_release); @@ -10514,11 +11525,41 @@ class StubGenerator: public StubCodeGenerator { AtomicStubMark mark_cmpxchg_4_seq_cst (_masm, &aarch64_atomic_cmpxchg_4_seq_cst_impl); gen_cas_entry(MacroAssembler::word, memory_order_seq_cst); + AtomicStubMark mark_cmpxchg_8_seq_cst (_masm, &aarch64_atomic_cmpxchg_8_seq_cst_impl); gen_cas_entry(MacroAssembler::xword, memory_order_seq_cst); - ICache::invalidate_range(first_entry, __ pc() - first_entry); + end = __ pc(); + + ICache::invalidate_range(first_entry, end - first_entry); + // exit block to force update of AtomicStubMark targets + } + + assert(first_entry == (address)aarch64_atomic_fetch_add_4_impl, + "atomic stub should be at start of buffer"); + // record the stub start and end plus all the entries saved by the + // AtomicStubMark destructor + entries.append((address)aarch64_atomic_fetch_add_8_impl); + entries.append((address)aarch64_atomic_fetch_add_4_relaxed_impl); + entries.append((address)aarch64_atomic_fetch_add_8_relaxed_impl); + entries.append((address)aarch64_atomic_xchg_4_impl); + entries.append((address)aarch64_atomic_xchg_8_impl); + entries.append((address)aarch64_atomic_cmpxchg_1_impl); + entries.append((address)aarch64_atomic_cmpxchg_4_impl); + entries.append((address)aarch64_atomic_cmpxchg_8_impl); + entries.append((address)aarch64_atomic_cmpxchg_1_relaxed_impl); + entries.append((address)aarch64_atomic_cmpxchg_4_relaxed_impl); + entries.append((address)aarch64_atomic_cmpxchg_8_relaxed_impl); + entries.append((address)aarch64_atomic_cmpxchg_4_release_impl); + entries.append((address)aarch64_atomic_cmpxchg_8_release_impl); + entries.append((address)aarch64_atomic_cmpxchg_4_seq_cst_impl); + entries.append((address)aarch64_atomic_cmpxchg_8_seq_cst_impl); + + assert(entries.length() == entry_count - 1, + "unexpected extra entry count %d", entries.length()); + + store_archive_data(stub_id, first_entry, end, &entries); } #endif // LINUX @@ -10622,9 +11663,22 @@ class StubGenerator: public StubCodeGenerator { if (!Continuations::enabled()) return nullptr; StubId stub_id = StubId::stubgen_cont_thaw_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + StubCodeMark mark(this, stub_id); address start = __ pc(); generate_cont_thaw(Continuation::thaw_top); + + // record the stub start and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -10633,11 +11687,23 @@ class StubGenerator: public StubCodeGenerator { // TODO: will probably need multiple return barriers depending on return type StubId stub_id = StubId::stubgen_cont_returnBarrier_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + StubCodeMark mark(this, stub_id); address start = __ pc(); generate_cont_thaw(Continuation::thaw_return_barrier); + // record the stub start and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -10645,17 +11711,38 @@ class StubGenerator: public StubCodeGenerator { if (!Continuations::enabled()) return nullptr; StubId stub_id = StubId::stubgen_cont_returnBarrierExc_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + StubCodeMark mark(this, stub_id); address start = __ pc(); generate_cont_thaw(Continuation::thaw_return_barrier_exception); + // record the stub start and end + store_archive_data(stub_id, start, __ pc()); + return start; } address generate_cont_preempt_stub() { if (!Continuations::enabled()) return nullptr; StubId stub_id = StubId::stubgen_cont_preempt_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -10682,6 +11769,9 @@ class StubGenerator: public StubCodeGenerator { __ ldr(rscratch1, Address(rscratch1)); __ br(rscratch1); + // record the stub start and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -10737,8 +11827,17 @@ class StubGenerator: public StubCodeGenerator { // computation. address generate_poly1305_processBlocks() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_poly1305_processBlocks_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); Label here; @@ -10848,12 +11947,24 @@ class StubGenerator: public StubCodeGenerator { __ leave(); __ ret(lr); + // record the stub start and end + store_archive_data(stub_id, start, __ pc()); + return start; } // exception handler for upcall stubs address generate_upcall_stub_exception_handler() { StubId stub_id = StubId::stubgen_upcall_stub_exception_handler_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -10864,6 +11975,9 @@ class StubGenerator: public StubCodeGenerator { __ blr(rscratch1); __ should_not_reach_here(); + // record the stub start and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -10872,6 +11986,15 @@ class StubGenerator: public StubCodeGenerator { // rmethod = result address generate_upcall_stub_load_target() { StubId stub_id = StubId::stubgen_upcall_stub_load_target_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -10887,6 +12010,9 @@ class StubGenerator: public StubCodeGenerator { __ ret(lr); + // record the stub start and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -11915,7 +13041,8 @@ class StubGenerator: public StubCodeGenerator { } if (UseGHASHIntrinsics) { // StubRoutines::_ghash_processBlocks = generate_ghash_processBlocks(); - StubRoutines::_ghash_processBlocks = generate_ghash_processBlocks_wide(); + StubRoutines::aarch64::_ghash_processBlocks_small = generate_ghash_processBlocks_small(); + StubRoutines::_ghash_processBlocks = generate_ghash_processBlocks(StubRoutines::aarch64::_ghash_processBlocks_small); } if (UseAESIntrinsics && UseGHASHIntrinsics) { StubRoutines::_galoisCounterMode_AESCrypt = generate_galoisCounterMode_AESCrypt(); @@ -11962,7 +13089,7 @@ class StubGenerator: public StubCodeGenerator { } public: - StubGenerator(CodeBuffer* code, BlobId blob_id) : StubCodeGenerator(code, blob_id) { + StubGenerator(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) : StubCodeGenerator(code, blob_id, stub_data) { switch(blob_id) { case BlobId::stubgen_preuniverse_id: generate_preuniverse_stubs(); @@ -11984,10 +13111,36 @@ class StubGenerator: public StubCodeGenerator { break; }; } + + static void init_AOTAddressTable(GrowableArray
& external_addresses) { + // external data defined in this file +#define ADD(addr) external_addresses.append((address)addr); + ADD(_sha256_round_consts); + ADD(_sha512_round_consts); + ADD(_sha3_round_consts); + ADD(_double_keccak_round_consts); + ADD(_encodeBlock_toBase64); + ADD(_encodeBlock_toBase64URL); + ADD(_decodeBlock_fromBase64ForNoSIMD); + ADD(_decodeBlock_fromBase64URLForNoSIMD); + ADD(_decodeBlock_fromBase64ForSIMD); + ADD(_decodeBlock_fromBase64URLForSIMD); +#undef ADD + } }; // end class declaration -void StubGenerator_generate(CodeBuffer* code, BlobId blob_id) { - StubGenerator g(code, blob_id); +void StubGenerator_AOTAddressTable_init() { + ResourceMark rm; + GrowableArray
external_addresses; + // publish static addresses referred to by aarch64 generator + StubGenerator::init_AOTAddressTable(external_addresses); + // publish external data addresses defined in nested aarch64 class + StubRoutines::aarch64::init_AOTAddressTable(external_addresses); + AOTCodeCache::publish_external_addresses(external_addresses); +} + +void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) { + StubGenerator g(code, blob_id, stub_data); } diff --git a/src/hotspot/cpu/aarch64/stubRoutines_aarch64.cpp b/src/hotspot/cpu/aarch64/stubRoutines_aarch64.cpp index 88993818b47ee..73f0797cb7aca 100644 --- a/src/hotspot/cpu/aarch64/stubRoutines_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/stubRoutines_aarch64.cpp @@ -413,3 +413,19 @@ ATTRIBUTE_ALIGNED(64) jdouble StubRoutines::aarch64::_pio2[] = { 2.73370053816464559624e-44, // 0x36E3822280000000 2.16741683877804819444e-51, // 0x3569F31D00000000 }; + +#define ADD(addr) external_addresses.append((address)addr); + +void StubRoutines::aarch64::init_AOTAddressTable(GrowableArray
& external_addresses) { + ADD(_kyberConsts); + ADD(_dilithiumConsts); + ADD(_crc_table); + ADD(_adler_table); + ADD(_npio2_hw); + ADD(_dsin_coef); + ADD(_dcos_coef); + ADD(_two_over_pi); + ADD(_pio2); +} + +#undef ADD diff --git a/src/hotspot/cpu/aarch64/stubRoutines_aarch64.hpp b/src/hotspot/cpu/aarch64/stubRoutines_aarch64.hpp index c35371e10833d..89c72a6a5ea11 100644 --- a/src/hotspot/cpu/aarch64/stubRoutines_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/stubRoutines_aarch64.hpp @@ -110,6 +110,8 @@ class aarch64 { _completed = true; } + static void init_AOTAddressTable(GrowableArray
& external_addresses); + private: static uint16_t _kyberConsts[]; static uint32_t _dilithiumConsts[]; diff --git a/src/hotspot/share/c1/c1_Runtime1.cpp b/src/hotspot/share/c1/c1_Runtime1.cpp index a4c956ff5bea1..e1867c7c4ab88 100644 --- a/src/hotspot/share/c1/c1_Runtime1.cpp +++ b/src/hotspot/share/c1/c1_Runtime1.cpp @@ -278,10 +278,6 @@ bool Runtime1::initialize(BufferBlob* blob) { if (!generate_blob_for(blob, id)) { return false; } - if (id == StubId::c1_forward_exception_id) { - // publish early c1 stubs at this point so later stubs can refer to them - AOTCodeCache::init_early_c1_table(); - } } // printing #ifndef PRODUCT diff --git a/src/hotspot/share/code/aotCodeCache.cpp b/src/hotspot/share/code/aotCodeCache.cpp index 04776f4c16c95..a72a7ba9d49a3 100644 --- a/src/hotspot/share/code/aotCodeCache.cpp +++ b/src/hotspot/share/code/aotCodeCache.cpp @@ -32,9 +32,12 @@ #include "classfile/javaAssertions.hpp" #include "code/aotCodeCache.hpp" #include "code/codeCache.hpp" +#include "gc/shared/barrierSetNMethod.hpp" #include "gc/shared/gcConfig.hpp" #include "logging/logStream.hpp" #include "memory/memoryReserver.hpp" +#include "prims/jvmtiThreadState.hpp" +#include "prims/upcallLinker.hpp" #include "runtime/deoptimization.hpp" #include "runtime/flags/flagSetting.hpp" #include "runtime/globals_extension.hpp" @@ -153,10 +156,13 @@ static uint32_t encode_id(AOTCodeEntry::Kind kind, int id) { } else if (kind == AOTCodeEntry::C1Blob) { assert(StubInfo::is_c1(static_cast(id)), "not a c1 blob id %d", id); return id; - } else { - // kind must be AOTCodeEntry::C2Blob + } else if (kind == AOTCodeEntry::C2Blob) { assert(StubInfo::is_c2(static_cast(id)), "not a c2 blob id %d", id); return id; + } else { + // kind must be AOTCodeEntry::StubGenBlob + assert(StubInfo::is_stubgen(static_cast(id)), "not a stubgen blob id %d", id); + return id; } } @@ -182,7 +188,7 @@ void AOTCodeCache::initialize() { } // Disable stubs caching until JDK-8357398 is fixed. - FLAG_SET_ERGO(AOTStubCaching, false); + // FLAG_SET_ERGO(AOTStubCaching, false); if (VerifyOops) { // Disable AOT stubs caching when VerifyOops flag is on. @@ -278,6 +284,19 @@ bool AOTCodeCache::open_cache(bool is_dumping, bool is_using) { return true; } +// Called after continuations_init() when continuation stub callouts +// have been initialized +void AOTCodeCache::init3() { + if (opened_cache == nullptr) { + return; + } + // initialize external routines for continuations so we can save + // generated continuation blob that references them + AOTCodeAddressTable* table = opened_cache->_table; + assert(table != nullptr, "should be initialized already"); + table->init_extrs2(); +} + void AOTCodeCache::close() { if (is_on()) { delete _cache; // Free memory @@ -354,24 +373,59 @@ AOTCodeCache::AOTCodeCache(bool is_dumping, bool is_using) : _table = new AOTCodeAddressTable(); } -void AOTCodeCache::init_early_stubs_table() { +void AOTCodeCache::add_stub_entries(StubId stub_id, address start, GrowableArray
*entries, int begin_idx) { + EntryId entry_id = StubInfo::entry_base(stub_id); + add_stub_entry(entry_id, start); + // skip past first entry + entry_id = StubInfo::next_in_stub(stub_id, entry_id); + // now check for any more entries + int count = StubInfo::entry_count(stub_id) - 1; + assert(start != nullptr, "invalid start address for stub %s", StubInfo::name(stub_id)); + assert(entries == nullptr || begin_idx + count <= entries->length(), "sanity"); + // write any extra entries + for (int i = 0; i < count; i++) { + assert(entry_id != EntryId::NO_ENTRYID, "not enough entries for stub %s", StubInfo::name(stub_id)); + address a = entries->at(begin_idx + i); + add_stub_entry(entry_id, a); + entry_id = StubInfo::next_in_stub(stub_id, entry_id); + } + assert(entry_id == EntryId::NO_ENTRYID, "too many entries for stub %s", StubInfo::name(stub_id)); +} + +void AOTCodeCache::add_stub_entry(EntryId entry_id, address a) { + if (a != nullptr) { + if (_table != nullptr) { + log_trace(aot, codecache, stubs)("Publishing stub entry %s at address " INTPTR_FORMAT, StubInfo::name(entry_id), p2i(a)); + return _table->add_stub_entry(entry_id, a); + } + } +} + +void AOTCodeCache::set_shared_stubs_complete() { AOTCodeAddressTable* table = addr_table(); if (table != nullptr) { - table->init_early_stubs(); + table->set_shared_stubs_complete(); } } -void AOTCodeCache::init_shared_blobs_table() { +void AOTCodeCache::set_c1_stubs_complete() { AOTCodeAddressTable* table = addr_table(); if (table != nullptr) { - table->init_shared_blobs(); + table->set_c1_stubs_complete(); } } -void AOTCodeCache::init_early_c1_table() { +void AOTCodeCache::set_c2_stubs_complete() { AOTCodeAddressTable* table = addr_table(); if (table != nullptr) { - table->init_early_c1(); + table->set_c2_stubs_complete(); + } +} + +void AOTCodeCache::set_stubgen_stubs_complete() { + AOTCodeAddressTable* table = addr_table(); + if (table != nullptr) { + table->set_stubgen_stubs_complete(); } } @@ -799,13 +853,42 @@ bool AOTCodeCache::finish_write() { //------------------Store/Load AOT code ---------------------- -bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind, uint id, const char* name) { +bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind, uint id, const char* name, AOTStubData* stub_data, CodeBuffer* code_buffer) { + assert(AOTCodeEntry::is_valid_entry_kind(entry_kind), "invalid entry_kind %d", entry_kind); + + // we only expect stub data and a code buffer for a multi stub blob + assert(AOTCodeEntry::is_multi_stub_blob(entry_kind) == (stub_data != nullptr), + "entry_kind %d does not match stub_data pointer %p", + entry_kind, stub_data); + + assert((stub_data == nullptr) == (code_buffer == nullptr), + "stub data and code buffer must both be null or both non null"); + + // If this is a stub and the cache is on for either load or dump we + // need to insert the stub entries into the AOTCacheAddressTable so + // that relocs which refer to entries defined by this blob get + // translated correctly. + // + // Entry insertion needs to be be done up front before writing the + // blob because some blobs rely on internal daisy-chain references + // from one entry to another. + // + // Entry insertion also needs to be done even if the cache is open + // for use but not for dump. This may be needed when an archived + // blob omits some entries -- either because of a config change or a + // load failure -- with the result that the entries end up being + // generated. These generated entry addresses may be needed to + // resolve references from subsequently loaded blobs (for either + // stubs or nmethods). + + if (is_on() && AOTCodeEntry::is_blob(entry_kind)) { + publish_stub_addresses(blob, (BlobId)id, stub_data); + } + AOTCodeCache* cache = open_for_dump(); if (cache == nullptr) { return false; } - assert(AOTCodeEntry::is_valid_entry_kind(entry_kind), "invalid entry_kind %d", entry_kind); - if (AOTCodeEntry::is_adapter(entry_kind) && !is_dumping_adapter()) { return false; } @@ -851,8 +934,44 @@ bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind } CodeBlob::archive_blob(&blob, archive_buffer); - uint reloc_data_size = blob.relocation_size(); - n = cache->write_bytes((address)blob.relocation_begin(), reloc_data_size); + // For a relocatable code blob its relocations are linked from the + // blob. However, for a non-relocatable (stubgen) blob we only have + // transient relocations attached to the code buffer that are added + // in order to support AOT-load time patching. in either case, we + // need to explicitly save these relocs when storing the blob to the + // archive so we can then reload them and reattach them to either + // the blob or to a code buffer when we reload the blob into a + // production JVM. + // + // Either way we are then in a position to iterate over the relocs + // and AOT patch the ones that refer to code that may move between + // assembly and production time. We also need to save and restore + // AOT address table indexes for the target addresses of affected + // relocs. That happens below. + + int reloc_count; + address reloc_data; + if (AOTCodeEntry::is_multi_stub_blob(entry_kind)) { + CodeSection* cs = code_buffer->code_section(CodeBuffer::SECT_INSTS); + reloc_count = (cs->has_locs() ? cs->locs_count() : 0); + reloc_data = (reloc_count > 0 ? (address)cs->locs_start() : nullptr); + } else { + reloc_count = blob.relocation_size() / sizeof(relocInfo); + reloc_data = (address)blob.relocation_begin(); + } + n = cache->write_bytes(&reloc_count, sizeof(int)); + if (n != sizeof(int)) { + return false; + } + if (AOTCodeEntry::is_multi_stub_blob(entry_kind)) { + // align to heap word size before writing the relocs so we can + // install them into a code buffer when they get restored + if (!cache->align_write()) { + return false; + } + } + uint reloc_data_size = (uint)(reloc_count * sizeof(relocInfo)); + n = cache->write_bytes(reloc_data, reloc_data_size); if (n != reloc_data_size) { return false; } @@ -875,7 +994,29 @@ bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind } #endif /* PRODUCT */ - if (!cache->write_relocations(blob)) { + // In the case of a multi-stub blob we need to write start, end, + // secondary entries and extras. For any other blob entry addresses + // beyond the blob start will be stored in the blob as offsets. + if (stub_data != nullptr) { + if (!cache->write_stub_data(blob, stub_data)) { + return false; + } + } + + // now we have added all the other data we can write the AOT + // relocations + + bool write_ok; + if (AOTCodeEntry::is_multi_stub_blob(entry_kind)) { + CodeSection* cs = code_buffer->code_section(CodeBuffer::SECT_INSTS); + RelocIterator iter(cs); + write_ok = cache->write_relocations(blob, iter); + } else { + RelocIterator iter(&blob); + write_ok = cache->write_relocations(blob, iter); + } + + if (!write_ok) { if (!cache->failed()) { // We may miss an address in AOT table - skip this code blob. cache->set_write_position(entry_position); @@ -884,6 +1025,7 @@ bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind } uint entry_size = cache->_write_position - entry_position; + AOTCodeEntry* entry = new(cache) AOTCodeEntry(entry_kind, encode_id(entry_kind, id), entry_position, entry_size, name_offset, name_size, blob_offset, has_oop_maps, blob.content_begin()); @@ -891,19 +1033,130 @@ bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind return true; } +bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind, uint id, const char* name) { + assert(!AOTCodeEntry::is_blob(entry_kind), + "wrong entry kind for numeric id %d", id); + return store_code_blob(blob, entry_kind, (uint)id, name, nullptr, nullptr); +} + bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind, BlobId id) { - assert(AOTCodeEntry::is_blob(entry_kind), + assert(AOTCodeEntry::is_single_stub_blob(entry_kind), "wrong entry kind for blob id %s", StubInfo::name(id)); - return store_code_blob(blob, entry_kind, (uint)id, StubInfo::name(id)); + return store_code_blob(blob, entry_kind, (uint)id, StubInfo::name(id), nullptr, nullptr); } -CodeBlob* AOTCodeCache::load_code_blob(AOTCodeEntry::Kind entry_kind, uint id, const char* name) { +bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind, BlobId id, AOTStubData* stub_data, CodeBuffer* code_buffer) { + assert(AOTCodeEntry::is_multi_stub_blob(entry_kind), + "wrong entry kind for multi stub blob id %s", StubInfo::name(id)); + return store_code_blob(blob, entry_kind, (uint)id, StubInfo::name(id), stub_data, code_buffer); +} + +bool AOTCodeCache::write_stub_data(CodeBlob &blob, AOTStubData *stub_data) { + BlobId blob_id = stub_data->blob_id(); + StubId stub_id = StubInfo::stub_base(blob_id); + address blob_base = blob.code_begin(); + int stub_cnt = StubInfo::stub_count(blob_id); + int n; + + LogStreamHandle(Trace, aot, codecache, stubs) log; + + if (log.is_enabled()) { + log.print_cr("======== Stub data starts at offset %d", _write_position); + } + + for (int i = 0; i < stub_cnt; i++, stub_id = StubInfo::next_in_blob(blob_id, stub_id)) { + // for each stub we find in the ranges list we write an int + // sequence where + // + // - start_pos is the stub start address encoded as a code section offset + // + // - end is the stub end address encoded as an offset from start + // + // - N counts the number of stub-local entries/extras + // + // - offseti is a stub-local entry/extra address encoded as len for + // a null address otherwise as an offset in range [1,len-1] + + StubAddrRange& range = stub_data->get_range(i); + GrowableArray
& addresses = stub_data->address_array(); + int base = range.start_index(); + if (base >= 0) { + n = write_bytes(&stub_id, sizeof(StubId)); + if (n != sizeof(StubId)) { + return false; + } + address start = addresses.at(base); + assert (blob_base <= start, "sanity"); + uint offset = (uint)(start - blob_base); + n = write_bytes(&offset, sizeof(uint)); + if (n != sizeof(int)) { + return false; + } + address end = addresses.at(base + 1); + assert (start < end, "sanity"); + offset = (uint)(end - start); + n = write_bytes(&offset, sizeof(uint)); + if (n != sizeof(int)) { + return false; + } + // write number of secondary and extra entries + int count = range.count() - 2; + n = write_bytes(&count, sizeof(int)); + if (n != sizeof(int)) { + return false; + } + for (int j = 0; j < count; j++) { + address next = addresses.at(base + 2 + j); + if (next != nullptr) { + // n.b. This maps next == end to the stub length which + // means we will reconstitute the address as nullptr. That + // happens when we have a handler range covers the end of + // a stub and needs to be handled specially by the client + // that restores the extras. + assert(start <= next && next <= end, "sanity"); + offset = (uint)(next - start); + } else { + // this can happen when a stub is not generated or an + // extra is the common handler target + offset = (uint)(end - start); + } + n = write_bytes(&offset, sizeof(uint)); + if (n != sizeof(int)) { + return false; + } + } + if (log.is_enabled()) { + log.print_cr("======== wrote stub %s and %d addresses up to offset %d", + StubInfo::name(stub_id), range.count(), _write_position); + } + } + } + // we should have exhausted all stub ids in the blob + assert(stub_id == StubId::NO_STUBID, "sanity"); + // write NO_STUBID as an end marker + n = write_bytes(&stub_id, sizeof(StubId)); + if (n != sizeof(StubId)) { + return false; + } + + if (log.is_enabled()) { + log.print_cr("======== Stub data ends at offset %d", _write_position); + } + + return true; +} + +CodeBlob* AOTCodeCache::load_code_blob(AOTCodeEntry::Kind entry_kind, uint id, const char* name, AOTStubData* stub_data) { AOTCodeCache* cache = open_for_use(); if (cache == nullptr) { return nullptr; } assert(AOTCodeEntry::is_valid_entry_kind(entry_kind), "invalid entry_kind %d", entry_kind); + assert(AOTCodeEntry::is_multi_stub_blob(entry_kind) == (stub_data != nullptr), + "entry_kind %d does not match stub_data pointer %p", + entry_kind, stub_data); + if (AOTCodeEntry::is_adapter(entry_kind) && !is_using_adapter()) { return nullptr; } @@ -917,20 +1170,32 @@ CodeBlob* AOTCodeCache::load_code_blob(AOTCodeEntry::Kind entry_kind, uint id, c return nullptr; } AOTCodeReader reader(cache, entry); - CodeBlob* blob = reader.compile_code_blob(name); + CodeBlob* blob = reader.compile_code_blob(name, entry_kind, id, stub_data); log_debug(aot, codecache, stubs)("%sRead blob '%s' (id=%u, kind=%s) from AOT Code Cache", (blob == nullptr? "Failed to " : ""), name, id, aot_code_entry_kind_name[entry_kind]); return blob; } +CodeBlob* AOTCodeCache::load_code_blob(AOTCodeEntry::Kind entry_kind, uint id, const char* name) { + assert(!AOTCodeEntry::is_blob(entry_kind), + "wrong entry kind for numeric id %d", id); + return load_code_blob(entry_kind, (uint)id, name, nullptr); +} + CodeBlob* AOTCodeCache::load_code_blob(AOTCodeEntry::Kind entry_kind, BlobId id) { - assert(AOTCodeEntry::is_blob(entry_kind), + assert(AOTCodeEntry::is_single_stub_blob(entry_kind), + "wrong entry kind for blob id %s", StubInfo::name(id)); + return load_code_blob(entry_kind, (uint)id, StubInfo::name(id), nullptr); +} + +CodeBlob* AOTCodeCache::load_code_blob(AOTCodeEntry::Kind entry_kind, BlobId id, AOTStubData* stub_data) { + assert(AOTCodeEntry::is_multi_stub_blob(entry_kind), "wrong entry kind for blob id %s", StubInfo::name(id)); - return load_code_blob(entry_kind, (uint)id, StubInfo::name(id)); + return load_code_blob(entry_kind, (uint)id, StubInfo::name(id), stub_data); } -CodeBlob* AOTCodeReader::compile_code_blob(const char* name) { +CodeBlob* AOTCodeReader::compile_code_blob(const char* name, AOTCodeEntry::Kind entry_kind, int id, AOTStubData* stub_data) { uint entry_position = _entry->offset(); // Read name @@ -945,13 +1210,19 @@ CodeBlob* AOTCodeReader::compile_code_blob(const char* name) { return nullptr; } - // Read archived code blob + // Read archived code blob and related info uint offset = entry_position + _entry->blob_offset(); CodeBlob* archived_blob = (CodeBlob*)addr(offset); offset += archived_blob->size(); + int reloc_count = *(int*)addr(offset); offset += sizeof(int); + if (AOTCodeEntry::is_multi_stub_blob(entry_kind)) { + // position of relocs will have been aligned to heap word size so + // we can install them into a code buffer + offset = align_up(offset, DATA_ALIGNMENT); + } address reloc_data = (address)addr(offset); - offset += archived_blob->relocation_size(); + offset += reloc_count * sizeof(relocInfo); set_read_position(offset); ImmutableOopMapSet* oop_maps = nullptr; @@ -959,11 +1230,13 @@ CodeBlob* AOTCodeReader::compile_code_blob(const char* name) { oop_maps = read_oop_map_set(); } + // Note that for a non-relocatable blob reloc_data will not be + // restored into the blob. We fix that later. + CodeBlob* code_blob = CodeBlob::create(archived_blob, stored_name, reloc_data, - oop_maps - ); + oop_maps); if (code_blob == nullptr) { // no space left in CodeCache return nullptr; } @@ -975,7 +1248,49 @@ CodeBlob* AOTCodeReader::compile_code_blob(const char* name) { read_dbg_strings(code_blob->dbg_strings()); #endif // PRODUCT - fix_relocations(code_blob); + if (AOTCodeEntry::is_blob(entry_kind)) { + BlobId blob_id = static_cast(id); + if (StubInfo::is_stubgen(blob_id)) { + assert(stub_data != nullptr, "sanity"); + read_stub_data(code_blob, stub_data); + } + // publish entries found either in stub_data or as offsets in blob + AOTCodeCache::publish_stub_addresses(*code_blob, blob_id, stub_data); + } + + // Now that all the entry points are in the address table we can + // read all the extra reloc info and fix up any addresses that need + // patching to adjust for a new location in a new JVM. We can be + // sure to correctly update all runtime references, including + // cross-linked stubs that are internally daisy-chained. If + // relocation fails and we have to re-generate any of the stubs then + // the entry points for newly generated stubs will get updated, + // ensuring that any other stubs or nmethods we need to relocate + // will use the correct address. + + // if we have a relocatable code blob then the relocs are already + // attached to the blob and we can iterate over it to find the ones + // we need to patch. With a non-relocatable code blob we need to + // wrap it with a CodeBuffer and then reattach the relocs to the + // code buffer. + + if (AOTCodeEntry::is_multi_stub_blob(entry_kind)) { + // the blob doesn't have any proper runtime relocs but we can + // reinstate the AOT-load time relocs we saved from the code + // buffer that generated this blob in a new code buffer and use + // the latter to iterate over them + CodeBuffer code_buffer(code_blob); + relocInfo* locs = (relocInfo*)reloc_data; + code_buffer.insts()->initialize_shared_locs(locs, reloc_count); + code_buffer.insts()->set_locs_end(locs + reloc_count); + CodeSection *cs = code_buffer.code_section(CodeBuffer::SECT_INSTS); + RelocIterator reloc_iter(cs); + fix_relocations(code_blob, reloc_iter); + } else { + // the AOT-load time relocs will be in the blob's restored relocs + RelocIterator reloc_iter(code_blob); + fix_relocations(code_blob, reloc_iter); + } #ifdef ASSERT LogStreamHandle(Trace, aot, codecache, stubs) log; @@ -987,15 +1302,146 @@ CodeBlob* AOTCodeReader::compile_code_blob(const char* name) { return code_blob; } -// ------------ process code and data -------------- +void AOTCodeReader::read_stub_data(CodeBlob* code_blob, AOTStubData* stub_data) { + GrowableArray
& addresses = stub_data->address_array(); + // Read the list of stub ids and associated start, end, secondary + // and extra addresses and install them in the stub data. + // + // Also insert all start and secondary addresses into the AOTCache + // address table so we correctly relocate this blob and any followng + // blobs/nmethods. + // + // n.b. if an error occurs and we need to regenerate any of these + // stubs the address table will be updated as a side-effect of + // regeneration. + + address blob_base = code_blob->code_begin(); + uint blob_size = (uint)(code_blob->code_end() - blob_base); + int offset = read_position(); + LogStreamHandle(Trace, aot, codecache, stubs) log; + if (log.is_enabled()) { + log.print_cr("======== Stub data starts at offset %d", offset); + } + // read stub and entries until we see NO_STUBID + StubId stub_id = *(StubId*)addr(offset); offset += sizeof(StubId); + // we ought to have at least one saved stub in the blob + assert(stub_id != StubId::NO_STUBID, "blob %s contains no stubs!", StubInfo::name(stub_data->blob_id())); + while (stub_id != StubId::NO_STUBID) { + assert(StubInfo::blob(stub_id) == stub_data->blob_id(), "sanity"); + int idx = StubInfo::stubgen_offset_in_blob(stub_data->blob_id(), stub_id); + StubAddrRange& range = stub_data->get_range(idx); + // we should only see a stub once + assert(range.start_index() < 0, "repeated entry for stub %s", StubInfo::name(stub_id)); + int address_base = addresses.length(); + // start is an offset from the blob base + uint start = *(uint*)addr(offset); offset += sizeof(uint); + assert(start < blob_size, "stub %s start offset %d exceeds buffer length %d", StubInfo::name(stub_id), start, blob_size); + address stub_start = blob_base + start; + addresses.append(stub_start); + // end is an offset from the stub start + uint end = *(uint*)addr(offset); offset += sizeof(uint); + assert(start + end <= blob_size, "stub %s end offset %d exceeds remaining buffer length %d", StubInfo::name(stub_id), end, blob_size - start); + addresses.append(stub_start + end); + // read count of secondary entries plus extras + int entries_count = *(int*)addr(offset); offset += sizeof(int); + assert(entries_count >= (StubInfo::entry_count(stub_id) - 1), "not enough entries for %s", StubInfo::name(stub_id)); + for (int i = 0; i < entries_count; i++) { + // entry offset is an offset from the stub start less than or + // equal to end + uint entry = *(uint*)addr(offset); offset += sizeof(uint); + assert(entry <= end, "stub %s entry offset %d lies beyond stub end %d", StubInfo::name(stub_id), entry, end); + if (entry < end) { + addresses.append(stub_start + entry); + } else { + // entry offset == end encodes a nullptr + addresses.append(nullptr); + } + } + if (log.is_enabled()) { + log.print_cr("======== read stub %s and %d addresses up to offset %d", + StubInfo::name(stub_id), 2 + entries_count, offset); + } + range.init_entry(address_base, 2 + entries_count); + // move on to next stub or NO_STUBID + stub_id = *(StubId*)addr(offset); offset += sizeof(StubId); + } + if (log.is_enabled()) { + log.print_cr("======== Stub data ends at offset %d", offset); + } + + set_read_position(offset); +} + +void AOTCodeCache::publish_external_addresses(GrowableArray
& addresses) { + DEBUG_ONLY( _passed_init2 = true; ) + if (opened_cache == nullptr) { + return; + } + + cache()->_table->add_external_addresses(addresses); +} + +void AOTCodeCache::publish_stub_addresses(CodeBlob &code_blob, BlobId blob_id, AOTStubData *stub_data) { + if (stub_data != nullptr) { + // register all entries in stub + assert(StubInfo::stub_count(blob_id) > 1, + "multiple stub data provided for single stub blob %s", + StubInfo::name(blob_id)); + assert(blob_id == stub_data->blob_id(), + "blob id %s does not match id in stub data %s", + StubInfo::name(blob_id), + StubInfo::name(stub_data->blob_id())); + // iterate over all stubs in the blob + StubId stub_id = StubInfo::stub_base(blob_id); + int stub_cnt = StubInfo::stub_count(blob_id); + GrowableArray
& addresses = stub_data->address_array(); + for (int i = 0; i < stub_cnt; i++) { + assert(stub_id != StubId::NO_STUBID, "sanity"); + StubAddrRange& range = stub_data->get_range(i); + int base = range.start_index(); + if (base >= 0) { + cache()->add_stub_entries(stub_id, addresses.at(base), &addresses, base + 2); + } + stub_id = StubInfo::next_in_blob(blob_id, stub_id); + } + // we should have exhausted all stub ids in the blob + assert(stub_id == StubId::NO_STUBID, "sanity"); + } else { + // register entry or entries for a single stub blob + StubId stub_id = StubInfo::stub_base(blob_id); + assert(StubInfo::stub_count(blob_id) == 1, + "multiple stub blob %s provided without stub data", + StubInfo::name(blob_id)); + address start = code_blob.code_begin(); + if (StubInfo::entry_count(stub_id) == 1) { + assert(!code_blob.is_deoptimization_stub(), "expecting multiple entries for stub %s", StubInfo::name(stub_id)); + // register the blob base address as the only entry + cache()->add_stub_entries(stub_id, start); + } else { + assert(code_blob.is_deoptimization_stub(), "only expecting one entry for stub %s", StubInfo::name(stub_id)); + DeoptimizationBlob *deopt_blob = code_blob.as_deoptimization_blob(); + assert(deopt_blob->unpack() == start, "unexpected offset 0x%lx for deopt stub entry", deopt_blob->unpack() - start); + GrowableArray
addresses; + addresses.append(deopt_blob->unpack_with_exception()); + addresses.append(deopt_blob->unpack_with_reexecution()); + addresses.append(deopt_blob->unpack_with_exception_in_tls()); +#if INCLUDE_JVMCI + addresses.append(deopt_blob->uncommon_trap()); + addresses.append(deopt_blob->implicit_exception_uncommon_trap()); +#endif // INCLUDE_JVMCI + cache()->add_stub_entries(stub_id, start, &addresses, 0); + } + } +} + + // ------------ process code and data -------------- // Can't use -1. It is valid value for jump to iteself destination // used by static call stub: see NativeJump::jump_destination(). #define BAD_ADDRESS_ID -2 -bool AOTCodeCache::write_relocations(CodeBlob& code_blob) { +bool AOTCodeCache::write_relocations(CodeBlob& code_blob, RelocIterator& iter) { GrowableArray reloc_data; - RelocIterator iter(&code_blob); LogStreamHandle(Trace, aot, codecache, reloc) log; while (iter.next()) { int idx = reloc_data.append(0); // default value @@ -1049,6 +1495,11 @@ bool AOTCodeCache::write_relocations(CodeBlob& code_blob) { // Write the count first int count = reloc_data.length(); write_bytes(&count, sizeof(int)); + if (log.is_enabled()) { + log.print_cr("======== extra relocations count=%d", count); + log.print( " {"); + } + bool first = true; for (GrowableArrayIterator iter = reloc_data.begin(); iter != reloc_data.end(); ++iter) { uint value = *iter; @@ -1056,23 +1507,43 @@ bool AOTCodeCache::write_relocations(CodeBlob& code_blob) { if (n != sizeof(uint)) { return false; } + if (log.is_enabled()) { + if (first) { + first = false; + log.print("%d", value); + } else { + log.print(", %d", value); + } + } } + log.print_cr("}"); return true; } -void AOTCodeReader::fix_relocations(CodeBlob* code_blob) { - LogStreamHandle(Trace, aot, reloc) log; +void AOTCodeReader::fix_relocations(CodeBlob *code_blob, RelocIterator& iter) { uint offset = read_position(); - int count = *(int*)addr(offset); + int reloc_count = *(int*)addr(offset); offset += sizeof(int); - if (log.is_enabled()) { - log.print_cr("======== extra relocations count=%d", count); - } uint* reloc_data = (uint*)addr(offset); - offset += (count * sizeof(uint)); + offset += (reloc_count * sizeof(uint)); set_read_position(offset); - RelocIterator iter(code_blob); + LogStreamHandle(Trace, aot, codecache, reloc) log; + if (log.is_enabled()) { + log.print_cr("======== extra relocations count=%d", reloc_count); + } + if (log.is_enabled()) { + log.print(" {"); + for(int i = 0; i < reloc_count; i++) { + if (i == 0) { + log.print("%d", reloc_data[i]); + } else { + log.print(", %d", reloc_data[i]); + } + } + log.print_cr("}"); + } + int j = 0; while (iter.next()) { switch (iter.type()) { @@ -1121,7 +1592,7 @@ void AOTCodeReader::fix_relocations(CodeBlob* code_blob) { } j++; } - assert(j == count, "sanity"); + assert(j == reloc_count, "sanity"); } bool AOTCodeCache::write_oop_map_set(CodeBlob& cb) { @@ -1231,27 +1702,23 @@ void AOTCodeReader::read_dbg_strings(DbgStrings& dbg_strings) { //======================= AOTCodeAddressTable =============== -// address table ids for generated routines, external addresses and C -// string addresses are partitioned into positive integer ranges -// defined by the following positive base and max values -// i.e. [_extrs_base, _extrs_base + _extrs_max -1], -// [_blobs_base, _blobs_base + _blobs_max -1], -// ... -// [_c_str_base, _c_str_base + _c_str_max -1], - -#define _extrs_max 100 -#define _stubs_max 3 +// address table ids for generated routine entry adresses, external +// addresses and C string addresses are partitioned into positive +// integer ranges defined by the following positive base and max +// values i.e. [_extrs_base, _extrs_base + _extrs_max -1], +// [_stubs_base, _stubs_base + _stubs_max -1], [_c_str_base, +// _c_str_base + _c_str_max -1], -#define _shared_blobs_max 20 -#define _C1_blobs_max 10 -#define _blobs_max (_shared_blobs_max+_C1_blobs_max) -#define _all_max (_extrs_max+_stubs_max+_blobs_max) +#define _extrs_max 200 +#define _stubs_max static_cast(EntryId::NUM_ENTRYIDS) #define _extrs_base 0 #define _stubs_base (_extrs_base + _extrs_max) -#define _shared_blobs_base (_stubs_base + _stubs_max) -#define _C1_blobs_base (_shared_blobs_base + _shared_blobs_max) -#define _blobs_end (_shared_blobs_base + _blobs_max) +#define _all_max (_stubs_base + _stubs_max) + +// setter for external addresses and string addresses inserts new +// addresses in the order they are encountered them which must remain +// th esame across an assembly run and subsequent production run #define SET_ADDRESS(type, addr) \ { \ @@ -1259,18 +1726,41 @@ void AOTCodeReader::read_dbg_strings(DbgStrings& dbg_strings) { assert(type##_length <= type##_max, "increase size"); \ } +// setter for stub entry addresses inserts them using the stub entry +// id as an index + +#define SET_ENTRY_ADDRESS(type, addr, entry_id) \ + { \ + int idx = static_cast(entry); \ + _stub_addr[idx] = (address) (addr); \ + } + static bool initializing_extrs = false; void AOTCodeAddressTable::init_extrs() { if (_extrs_complete || initializing_extrs) return; // Done already - assert(_blobs_end <= _all_max, "AOTCodeAddress table ranges need adjusting"); - initializing_extrs = true; _extrs_addr = NEW_C_HEAP_ARRAY(address, _extrs_max, mtCode); _extrs_length = 0; + { + // Required by initial stubs + SET_ADDRESS(_extrs, SharedRuntime::exception_handler_for_return_address); // used by forward_exception +#if defined(AMD64) || defined(AARCH64) || defined(RISCV64) + SET_ADDRESS(_extrs, MacroAssembler::debug64); // used by many eg forward_exception, call_stub +#endif // defined(AMD64) || defined(AARCH64) || defined(RISCV64) +#if defined(AMD64) + SET_ADDRESS(_extrs, StubRoutines::x86::addr_mxcsr_std()); // used by call_stub + SET_ADDRESS(_extrs, StubRoutines::x86::addr_mxcsr_rz()); // used by libmFmod +#endif + SET_ADDRESS(_extrs, CompressedOops::base_addr()); // used by call_stub + SET_ADDRESS(_extrs, Thread::current); // used by call_stub + SET_ADDRESS(_extrs, SharedRuntime::throw_StackOverflowError); + SET_ADDRESS(_extrs, SharedRuntime::throw_delayed_StackOverflowError); + } + // Record addresses of VM runtime methods SET_ADDRESS(_extrs, SharedRuntime::fixup_callers_callsite); SET_ADDRESS(_extrs, SharedRuntime::handle_wrong_method); @@ -1279,6 +1769,93 @@ void AOTCodeAddressTable::init_extrs() { #if defined(AARCH64) && !defined(ZERO) SET_ADDRESS(_extrs, JavaThread::aarch64_get_thread_helper); #endif + +#ifndef PRODUCT + SET_ADDRESS(_extrs, &SharedRuntime::_jbyte_array_copy_ctr); // used by arraycopy stub on arm32 and x86_64 + SET_ADDRESS(_extrs, &SharedRuntime::_jshort_array_copy_ctr); // used by arraycopy stub + SET_ADDRESS(_extrs, &SharedRuntime::_jint_array_copy_ctr); // used by arraycopy stub + SET_ADDRESS(_extrs, &SharedRuntime::_jlong_array_copy_ctr); // used by arraycopy stub + SET_ADDRESS(_extrs, &SharedRuntime::_oop_array_copy_ctr); // used by arraycopy stub + SET_ADDRESS(_extrs, &SharedRuntime::_checkcast_array_copy_ctr); // used by arraycopy stub + SET_ADDRESS(_extrs, &SharedRuntime::_unsafe_array_copy_ctr); // used by arraycopy stub + SET_ADDRESS(_extrs, &SharedRuntime::_generic_array_copy_ctr); // used by arraycopy stub + SET_ADDRESS(_extrs, &SharedRuntime::_unsafe_set_memory_ctr); // used by arraycopy stub +#endif /* PRODUCT */ + + SET_ADDRESS(_extrs, SharedRuntime::enable_stack_reserved_zone); + +#ifdef AMD64 + SET_ADDRESS(_extrs, SharedRuntime::montgomery_multiply); + SET_ADDRESS(_extrs, SharedRuntime::montgomery_square); +#endif // AMD64 + + SET_ADDRESS(_extrs, SharedRuntime::d2f); + SET_ADDRESS(_extrs, SharedRuntime::d2i); + SET_ADDRESS(_extrs, SharedRuntime::d2l); + SET_ADDRESS(_extrs, SharedRuntime::dcos); + SET_ADDRESS(_extrs, SharedRuntime::dexp); + SET_ADDRESS(_extrs, SharedRuntime::dlog); + SET_ADDRESS(_extrs, SharedRuntime::dlog10); + SET_ADDRESS(_extrs, SharedRuntime::dpow); + SET_ADDRESS(_extrs, SharedRuntime::drem); + SET_ADDRESS(_extrs, SharedRuntime::dsin); + SET_ADDRESS(_extrs, SharedRuntime::dtan); + SET_ADDRESS(_extrs, SharedRuntime::f2i); + SET_ADDRESS(_extrs, SharedRuntime::f2l); + SET_ADDRESS(_extrs, SharedRuntime::frem); + SET_ADDRESS(_extrs, SharedRuntime::l2d); + SET_ADDRESS(_extrs, SharedRuntime::l2f); + SET_ADDRESS(_extrs, SharedRuntime::ldiv); + SET_ADDRESS(_extrs, SharedRuntime::lmul); + SET_ADDRESS(_extrs, SharedRuntime::lrem); + +#if INCLUDE_JVMTI + SET_ADDRESS(_extrs, &JvmtiExport::_should_notify_object_alloc); +#endif /* INCLUDE_JVMTI */ + + SET_ADDRESS(_extrs, SafepointSynchronize::handle_polling_page_exception); + + SET_ADDRESS(_extrs, ThreadIdentifier::unsafe_offset()); + SET_ADDRESS(_extrs, Thread::current); + + SET_ADDRESS(_extrs, os::javaTimeMillis); + SET_ADDRESS(_extrs, os::javaTimeNanos); +#ifndef PRODUCT + SET_ADDRESS(_extrs, os::breakpoint); +#endif + +#if INCLUDE_JVMTI + SET_ADDRESS(_extrs, &JvmtiVTMSTransitionDisabler::_VTMS_notify_jvmti_events); +#endif /* INCLUDE_JVMTI */ + SET_ADDRESS(_extrs, StubRoutines::crc_table_addr()); +#if defined(AARCH64) + SET_ADDRESS(_extrs, JavaThread::aarch64_get_thread_helper); +#endif +#ifndef PRODUCT + SET_ADDRESS(_extrs, &SharedRuntime::_partial_subtype_ctr); + SET_ADDRESS(_extrs, JavaThread::verify_cross_modify_fence_failure); +#endif + +#if defined(AMD64) || defined(AARCH64) || defined(RISCV64) + SET_ADDRESS(_extrs, MacroAssembler::debug64); +#endif +#if defined(AMD64) + SET_ADDRESS(_extrs, StubRoutines::x86::arrays_hashcode_powers_of_31()); +#endif + +#ifdef X86 + SET_ADDRESS(_extrs, LIR_Assembler::float_signmask_pool); + SET_ADDRESS(_extrs, LIR_Assembler::double_signmask_pool); + SET_ADDRESS(_extrs, LIR_Assembler::float_signflip_pool); + SET_ADDRESS(_extrs, LIR_Assembler::double_signflip_pool); +#endif + + SET_ADDRESS(_extrs, JfrIntrinsicSupport::write_checkpoint); + SET_ADDRESS(_extrs, JfrIntrinsicSupport::return_lease); + + + SET_ADDRESS(_extrs, UpcallLinker::handle_uncaught_exception); // used by upcall_stub_exception_handler + { // Required by Shared blobs SET_ADDRESS(_extrs, Deoptimization::fetch_unroll_info); @@ -1366,6 +1943,11 @@ void AOTCodeAddressTable::init_extrs() { #if INCLUDE_G1GC SET_ADDRESS(_extrs, G1BarrierSetRuntime::write_ref_field_pre_entry); + SET_ADDRESS(_extrs, G1BarrierSetRuntime::write_ref_array_pre_narrow_oop_entry); // used by arraycopy stubs + SET_ADDRESS(_extrs, G1BarrierSetRuntime::write_ref_array_pre_oop_entry); // used by arraycopy stubs + SET_ADDRESS(_extrs, G1BarrierSetRuntime::write_ref_array_post_entry); // used by arraycopy stubs + SET_ADDRESS(_extrs, BarrierSetNMethod::nmethod_stub_entry_barrier); // used by method_entry_barrier + #endif #if INCLUDE_SHENANDOAHGC SET_ADDRESS(_extrs, ShenandoahRuntime::write_barrier_pre); @@ -1384,91 +1966,70 @@ void AOTCodeAddressTable::init_extrs() { #endif #endif // ZERO - _extrs_complete = true; - log_debug(aot, codecache, init)("External addresses recorded"); + log_debug(aot, codecache, init)("External addresses opened and recorded"); + // allocate storage for stub entries + _stubs_addr = NEW_C_HEAP_ARRAY(address, _stubs_max, mtCode); + log_debug(aot, codecache, init)("Stub addresses opened"); } -static bool initializing_early_stubs = false; - -void AOTCodeAddressTable::init_early_stubs() { - if (_complete || initializing_early_stubs) return; // Done already - initializing_early_stubs = true; - _stubs_addr = NEW_C_HEAP_ARRAY(address, _stubs_max, mtCode); - _stubs_length = 0; - SET_ADDRESS(_stubs, StubRoutines::forward_exception_entry()); +void AOTCodeAddressTable::init_extrs2() { + assert(initializing_extrs && !_extrs_complete, + "invalid sequence for init_extrs2"); { - // Required by C1 blobs -#if defined(AMD64) && !defined(ZERO) - SET_ADDRESS(_stubs, StubRoutines::x86::double_sign_flip()); - SET_ADDRESS(_stubs, StubRoutines::x86::d2l_fixup()); -#endif // AMD64 + SET_ADDRESS(_extrs, Continuation::prepare_thaw); // used by cont_thaw + SET_ADDRESS(_extrs, Continuation::thaw_entry()); // used by cont_thaw + SET_ADDRESS(_extrs, ContinuationEntry::thaw_call_pc_address()); // used by cont_preempt_stub } - - _early_stubs_complete = true; - log_info(aot, codecache, init)("Early stubs recorded"); + _extrs_complete = true; + initializing_extrs = false; + log_debug(aot, codecache, init)("External addresses recorded and closed"); } -static bool initializing_shared_blobs = false; - -void AOTCodeAddressTable::init_shared_blobs() { - if (_complete || initializing_shared_blobs) return; // Done already - initializing_shared_blobs = true; - address* blobs_addr = NEW_C_HEAP_ARRAY(address, _blobs_max, mtCode); - - // Divide _shared_blobs_addr array to chunks because they could be initialized in parrallel - _shared_blobs_addr = blobs_addr; - _C1_blobs_addr = _shared_blobs_addr + _shared_blobs_max; - - _shared_blobs_length = 0; - _C1_blobs_length = 0; +void AOTCodeAddressTable::add_external_addresses(GrowableArray
& addresses) { + assert(initializing_extrs && !_extrs_complete, + "invalid sequence for add_external_addresses"); + for (int i = 0; i < addresses.length(); i++) { + SET_ADDRESS(_extrs, addresses.at(i)); + } + log_debug(aot, codecache, init)("External addresses recorded"); +} - // clear the address table - memset(blobs_addr, 0, sizeof(address)* _blobs_max); +void AOTCodeAddressTable::add_stub_entry(EntryId entry_id, address a) { + assert(_extrs_complete || initializing_extrs, + "recording stub entry address before external addresses complete"); + assert(!(StubInfo::is_shared(StubInfo::stub(entry_id)) && _shared_stubs_complete), "too late to add shared entry"); + assert(!(StubInfo::is_stubgen(StubInfo::stub(entry_id)) && _stubgen_stubs_complete), "too late to add stubgen entry"); + assert(!(StubInfo::is_c1(StubInfo::stub(entry_id)) && _c1_stubs_complete), "too late to add c1 entry"); + assert(!(StubInfo::is_c2(StubInfo::stub(entry_id)) && _c2_stubs_complete), "too late to add c2 entry"); + log_debug(aot, stubs)("Recording address 0x%p for %s entry %s", a, StubInfo::name(StubInfo::stubgroup(entry_id)), StubInfo::name(entry_id)); + int idx = static_cast(entry_id); + _stubs_addr[idx] = a; +} - // Record addresses of generated code blobs - SET_ADDRESS(_shared_blobs, SharedRuntime::get_handle_wrong_method_stub()); - SET_ADDRESS(_shared_blobs, SharedRuntime::get_ic_miss_stub()); - SET_ADDRESS(_shared_blobs, SharedRuntime::deopt_blob()->unpack()); - SET_ADDRESS(_shared_blobs, SharedRuntime::deopt_blob()->unpack_with_exception()); - SET_ADDRESS(_shared_blobs, SharedRuntime::deopt_blob()->unpack_with_reexecution()); - SET_ADDRESS(_shared_blobs, SharedRuntime::deopt_blob()->unpack_with_exception_in_tls()); -#if INCLUDE_JVMCI - if (EnableJVMCI) { - SET_ADDRESS(_shared_blobs, SharedRuntime::deopt_blob()->uncommon_trap()); - SET_ADDRESS(_shared_blobs, SharedRuntime::deopt_blob()->implicit_exception_uncommon_trap()); - } -#endif +void AOTCodeAddressTable::set_shared_stubs_complete() { + assert(!_shared_stubs_complete, "repeated close for shared stubs!"); + _shared_stubs_complete = true; + log_debug(aot, codecache, init)("Shared stubs closed"); +} - _shared_blobs_complete = true; - log_debug(aot, codecache, init)("Early shared blobs recorded"); - _complete = true; +void AOTCodeAddressTable::set_c1_stubs_complete() { + assert(!_c1_stubs_complete, "repeated close for c1 stubs!"); + _c2_stubs_complete = true; + log_debug(aot, codecache, init)("C1 stubs closed"); } -void AOTCodeAddressTable::init_early_c1() { -#ifdef COMPILER1 - // Runtime1 Blobs - StubId id = StubInfo::stub_base(StubGroup::C1); - // include forward_exception in range we publish - StubId limit = StubInfo::next(StubId::c1_forward_exception_id); - for (; id != limit; id = StubInfo::next(id)) { - if (Runtime1::blob_for(id) == nullptr) { - log_info(aot, codecache, init)("C1 blob %s is missing", Runtime1::name_for(id)); - continue; - } - if (Runtime1::entry_for(id) == nullptr) { - log_info(aot, codecache, init)("C1 blob %s is missing entry", Runtime1::name_for(id)); - continue; - } - address entry = Runtime1::entry_for(id); - SET_ADDRESS(_C1_blobs, entry); - } -#endif // COMPILER1 - assert(_C1_blobs_length <= _C1_blobs_max, "increase _C1_blobs_max to %d", _C1_blobs_length); - _early_c1_complete = true; +void AOTCodeAddressTable::set_c2_stubs_complete() { + assert(!_c2_stubs_complete, "repeated close for c2 stubs!"); + _c2_stubs_complete = true; + log_debug(aot, codecache, init)("C2 stubs closed"); } -#undef SET_ADDRESS +void AOTCodeAddressTable::set_stubgen_stubs_complete() { + assert(!_stubgen_stubs_complete, "repeated close for stubgen stubs!"); + _stubgen_stubs_complete = true; + log_debug(aot, codecache, init)("Stubgen stubs closed"); +} AOTCodeAddressTable::~AOTCodeAddressTable() { if (_extrs_addr != nullptr) { @@ -1477,9 +2038,6 @@ AOTCodeAddressTable::~AOTCodeAddressTable() { if (_stubs_addr != nullptr) { FREE_C_HEAP_ARRAY(address, _stubs_addr); } - if (_shared_blobs_addr != nullptr) { - FREE_C_HEAP_ARRAY(address, _shared_blobs_addr); - } } #ifdef PRODUCT @@ -1564,7 +2122,7 @@ const char* AOTCodeCache::add_C_string(const char* str) { } const char* AOTCodeAddressTable::add_C_string(const char* str) { - if (_extrs_complete) { + if (_extrs_complete || initializing_extrs) { // Check previous strings address for (int i = 0; i < _C_strings_count; i++) { if (_C_strings_in[i] == str) { @@ -1627,7 +2185,7 @@ static int search_address(address addr, address* table, uint length) { } address AOTCodeAddressTable::address_for_id(int idx) { - assert(_extrs_complete, "AOT Code Cache VM runtime addresses table is not complete"); + assert(_extrs_complete || initializing_extrs, "AOT Code Cache VM runtime addresses table is not complete"); if (idx == -1) { return (address)-1; } @@ -1644,15 +2202,9 @@ address AOTCodeAddressTable::address_for_id(int idx) { if (/* id >= _extrs_base && */ id < _extrs_length) { return _extrs_addr[id - _extrs_base]; } - if (id >= _stubs_base && id < _stubs_base + _stubs_length) { + if (id >= _stubs_base && id < _c_str_base) { return _stubs_addr[id - _stubs_base]; } - if (id >= _shared_blobs_base && id < _shared_blobs_base + _shared_blobs_length) { - return _shared_blobs_addr[id - _shared_blobs_base]; - } - if (id >= _C1_blobs_base && id < _C1_blobs_base + _C1_blobs_length) { - return _C1_blobs_addr[id - _C1_blobs_base]; - } if (id >= _c_str_base && id < (_c_str_base + (uint)_C_strings_count)) { return address_for_C_string(id - _c_str_base); } @@ -1661,7 +2213,7 @@ address AOTCodeAddressTable::address_for_id(int idx) { } int AOTCodeAddressTable::id_for_address(address addr, RelocIterator reloc, CodeBlob* code_blob) { - assert(_extrs_complete, "AOT Code Cache VM runtime addresses table is not complete"); + assert(_extrs_complete || initializing_extrs, "AOT Code Cache VM runtime addresses table is not complete"); int id = -1; if (addr == (address)-1) { // Static call stub has jump to itself return id; @@ -1671,9 +2223,9 @@ int AOTCodeAddressTable::id_for_address(address addr, RelocIterator reloc, CodeB if (id >= 0) { return id + _c_str_base; } - if (StubRoutines::contains(addr)) { - // Search in stubs - id = search_address(addr, _stubs_addr, _stubs_length); + if (StubRoutines::contains(addr) || CodeCache::find_blob(addr) != nullptr) { + // Search for a matching stub entry + id = search_address(addr, _stubs_addr, _stubs_max); if (id < 0) { StubCodeDesc* desc = StubCodeDesc::desc_for(addr); if (desc == nullptr) { @@ -1685,51 +2237,39 @@ int AOTCodeAddressTable::id_for_address(address addr, RelocIterator reloc, CodeB return id + _stubs_base; } } else { - CodeBlob* cb = CodeCache::find_blob(addr); - if (cb != nullptr) { - // Search in code blobs - int id_base = _shared_blobs_base; - id = search_address(addr, _shared_blobs_addr, _blobs_max); - if (id < 0) { - assert(false, "Address " INTPTR_FORMAT " for Blob:%s is missing in AOT Code Cache addresses table", p2i(addr), cb->name()); - } else { - return id_base + id; - } - } else { - // Search in runtime functions - id = search_address(addr, _extrs_addr, _extrs_length); - if (id < 0) { - ResourceMark rm; - const int buflen = 1024; - char* func_name = NEW_RESOURCE_ARRAY(char, buflen); - int offset = 0; - if (os::dll_address_to_function_name(addr, func_name, buflen, &offset)) { - if (offset > 0) { - // Could be address of C string - uint dist = (uint)pointer_delta(addr, (address)os::init, 1); - log_debug(aot, codecache)("Address " INTPTR_FORMAT " (offset %d) for runtime target '%s' is missing in AOT Code Cache addresses table", - p2i(addr), dist, (const char*)addr); - assert(dist > (uint)(_all_max + MAX_STR_COUNT), "change encoding of distance"); - return dist; - } + // Search in runtime functions + id = search_address(addr, _extrs_addr, _extrs_length); + if (id < 0) { + ResourceMark rm; + const int buflen = 1024; + char* func_name = NEW_RESOURCE_ARRAY(char, buflen); + int offset = 0; + if (os::dll_address_to_function_name(addr, func_name, buflen, &offset)) { + if (offset > 0) { + // Could be address of C string + uint dist = (uint)pointer_delta(addr, (address)os::init, 1); + log_debug(aot, codecache)("Address " INTPTR_FORMAT " (offset %d) for runtime target '%s' is missing in AOT Code Cache addresses table", + p2i(addr), dist, (const char*)addr); + assert(dist > (uint)(_all_max + MAX_STR_COUNT), "change encoding of distance"); + return dist; + } #ifdef ASSERT - reloc.print_current_on(tty); - code_blob->print_on(tty); - code_blob->print_code_on(tty); - assert(false, "Address " INTPTR_FORMAT " for runtime target '%s+%d' is missing in AOT Code Cache addresses table", p2i(addr), func_name, offset); + reloc.print_current_on(tty); + code_blob->print_on(tty); + code_blob->print_code_on(tty); + assert(false, "Address " INTPTR_FORMAT " for runtime target '%s+%d' is missing in AOT Code Cache addresses table", p2i(addr), func_name, offset); #endif - } else { + } else { #ifdef ASSERT - reloc.print_current_on(tty); - code_blob->print_on(tty); - code_blob->print_code_on(tty); - os::find(addr, tty); - assert(false, "Address " INTPTR_FORMAT " for /('%s') is missing in AOT Code Cache addresses table", p2i(addr), (const char*)addr); + reloc.print_current_on(tty); + code_blob->print_on(tty); + code_blob->print_code_on(tty); + os::find(addr, tty); + assert(false, "Address " INTPTR_FORMAT " for /('%s') is missing in AOT Code Cache addresses table", p2i(addr), (const char*)addr); #endif - } - } else { - return _extrs_base + id; } + } else { + return _extrs_base + id; } } return id; @@ -1758,3 +2298,176 @@ void AOTCodeCache::print_on(outputStream* st) { } } } + +// methods for managing entries in multi-stub blobs + + +AOTStubData::AOTStubData(BlobId blob_id) : + _blob_id(blob_id), + _cached_blob(nullptr), + _stub_cnt(0), + _ranges(nullptr), + _current(StubId::NO_STUBID), + _current_idx(-1), + _flags(0) { + assert(StubInfo::is_stubgen(blob_id), + "AOTStubData expects a multi-stub blob not %s", + StubInfo::name(blob_id)); + + // we cannot save or restore preuniversestubs because the cache + // cannot be accessed before initialising the universe + if (blob_id == BlobId::stubgen_preuniverse_id) { + // invalidate any attempt to use this + _flags |= INVALID; + return; + } + if (AOTCodeCache::is_on()) { + // allow update of stub entry addresses + _flags |= OPEN; + if (AOTCodeCache::is_using_stub()) { + // allow stub loading + _flags |= USING; + } + if (AOTCodeCache::is_dumping_stub()) { + // allow stub saving + _flags |= DUMPING; + } + // we need to track all the blob's entries + _stub_cnt = StubInfo::stub_count(_blob_id); + _ranges = NEW_C_HEAP_ARRAY(StubAddrRange, _stub_cnt, mtCode); + for (int i = 0; i < _stub_cnt; i++) { + _ranges[i].default_init(); + } + } +} + +bool AOTStubData::load_code_blob() { + assert(is_using(), "should not call"); + assert(!is_invalid() && _cached_blob == nullptr, "repeated init"); + _cached_blob = AOTCodeCache::load_code_blob(AOTCodeEntry::StubGenBlob, + _blob_id, + this); + if (_cached_blob == nullptr) { + set_invalid(); + return false; + } else { + return true; + } +} + +bool AOTStubData::store_code_blob(CodeBlob& new_blob, CodeBuffer *code_buffer) { + assert(is_dumping(), "should not call"); + assert(_cached_blob == nullptr, "should not be loading and storing!"); + if (!AOTCodeCache::store_code_blob(new_blob, + AOTCodeEntry::StubGenBlob, + _blob_id, this, code_buffer)) { + set_invalid(); + return false; + } else { + return true; + } +} + +bool AOTStubData::find_archive_data(StubId stub_id) { + assert(StubInfo::blob(stub_id) == _blob_id, "sanity check"); + if (is_invalid()) { + return false; + } + int idx = StubInfo::stubgen_offset_in_blob(_blob_id, stub_id); + assert(idx >= 0 && idx < _stub_cnt, "invalid index %d for stub count %d", idx, _stub_cnt); + // ensure we have a valid associated range + StubAddrRange range = _ranges[idx]; + int start_index = range.start_index(); + if (start_index < 0) { + _current = StubId::NO_STUBID; +#ifdef DEBUG + // reset index so we can idenitfy which ones we failed to find + range.init_entry(-2, 0); +#endif + return false; + } + _current = stub_id; + _current_idx = idx; + return true; +} + +void AOTStubData::load_archive_data(StubId stub_id, address& start, address& end, GrowableArray
* entries, GrowableArray
* extras) { + assert(StubInfo::blob(stub_id) == _blob_id, "sanity check"); + assert(_current == stub_id && stub_id != StubId::NO_STUBID, "sanity check"); + assert(!is_invalid(), "should not load stubs when archive data is invalid"); + assert(_current_idx >= 0, "sanity"); + StubAddrRange& range = _ranges[_current_idx]; + int base = range.start_index(); + int count = range.count(); + assert(base >= 0, "sanity"); + assert(count >= 2, "sanity"); + // first two saved addresses are start and end + start = _address_array.at(base); + end = _address_array.at(base + 1); + assert(start != nullptr, "failed to load start address of stub %s", StubInfo::name(stub_id)); + assert(end != nullptr, "failed to load end address of stub %s", StubInfo::name(stub_id)); + assert(start < end, "start address %p should be less than end %p address for stub %s", start, end, StubInfo::name(stub_id)); + + int entry_count = StubInfo::entry_count(stub_id); + // the address count must at least include the stub start, end + // and secondary addresses + assert(count >= entry_count + 1, "stub %s requires %d saved addresses but only has %d", StubInfo::name(stub_id), entry_count + 1, count); + + // caller must retrieve secondary entries if and only if they exist + assert((entry_count == 1) == (entries == nullptr), "trying to retrieve wrong number of entries for stub %s", StubInfo::name(stub_id)); + int index = 2; + if (entries != nullptr) { + assert(entries->length() == 0, "non-empty array when retrieving entries for stub %s!", StubInfo::name(stub_id)); + while (index < entry_count + 1) { + address entry = _address_array.at(base + index++); + assert(entry == nullptr || (start < entry && entry < end), "entry address %p not in range (%p, %p) for stub %s", entry, start, end, StubInfo::name(stub_id)); + entries->append(entry); + } + } + // caller must retrieve extras if and only if they exist + assert((index < count) == (extras != nullptr), "trying to retrieve wrong number of extras for stub %s", StubInfo::name(stub_id)); + if (extras != nullptr) { + assert(extras->length() == 0, "non-empty array when retrieving extras for stub %s!", StubInfo::name(stub_id)); + while (index < count) { + address extra = _address_array.at(base + index++); + assert(extra == nullptr || (start <= extra && extra < end), "extra address %p not in range (%p, %p) for stub %s", extra, start, end, StubInfo::name(stub_id)); + extras->append(extra); + } + } +} + +void AOTStubData::store_archive_data(StubId stub_id, address start, address end, GrowableArray
* entries, GrowableArray
* extras) { + assert(StubInfo::blob(stub_id) == _blob_id, "sanity check"); + assert(start != nullptr, "start address cannot be null"); + assert(end != nullptr, "end address cannot be null"); + assert(start < end, "start address %p should be less than end %p address for stub %s", start, end, StubInfo::name(stub_id)); + _current = stub_id; + _current_idx = StubInfo::stubgen_offset_in_blob(_blob_id, stub_id); + StubAddrRange& range = _ranges[_current_idx]; + assert(range.start_index() == -1, "sanity"); + int base = _address_array.length(); + assert(base >= 0, "sanity"); + // first two saved addresses are start and end + _address_array.append(start); + _address_array.append(end); + // caller must save secondary entries if and only if they exist + assert((StubInfo::entry_count(stub_id) == 1) == (entries == nullptr), "trying to save wrong number of entries for stub %s", StubInfo::name(stub_id)); + if (entries != nullptr) { + assert(entries->length() == StubInfo::entry_count(stub_id) - 1, "incorrect entry count %d when saving entries for stub %s!", entries->length(), StubInfo::name(stub_id)); + for (int i = 0; i < entries->length(); i++) { + address entry = entries->at(i); + assert(entry == nullptr || (start < entry && entry < end), "entry address %p not in range (%p, %p) for stub %s", entry, start, end, StubInfo::name(stub_id)); + _address_array.append(entry); + } + } + // caller may wish to save extra addresses + if (extras != nullptr) { + for (int i = 0; i < extras->length(); i++) { + address extra = extras->at(i); + // handler range end may be end -- it gets restored as nullptr + assert(extra == nullptr || (start <= extra && extra <= end), "extra address %p not in range (%p, %p) for stub %s", extra, start, end, StubInfo::name(stub_id)); + _address_array.append(extra); + } + } + range.init_entry(base, _address_array.length() - base); +} diff --git a/src/hotspot/share/code/aotCodeCache.hpp b/src/hotspot/share/code/aotCodeCache.hpp index 778ad34e44817..2487dc7ff615a 100644 --- a/src/hotspot/share/code/aotCodeCache.hpp +++ b/src/hotspot/share/code/aotCodeCache.hpp @@ -37,6 +37,7 @@ class CodeBuffer; class RelocIterator; class AOTCodeCache; +class AOTCodeReader; class AdapterBlob; class ExceptionBlob; class ImmutableOopMapSet; @@ -52,6 +53,7 @@ enum CompLevel : signed char; Fn(SharedBlob) \ Fn(C1Blob) \ Fn(C2Blob) \ + Fn(StubGenBlob) \ // Descriptor of AOT Code Cache's entry class AOTCodeEntry { @@ -113,7 +115,9 @@ class AOTCodeEntry { address dumptime_content_start_addr() const { return _dumptime_content_start_addr; } static bool is_valid_entry_kind(Kind kind) { return kind > None && kind < Kind_count; } - static bool is_blob(Kind kind) { return kind == SharedBlob || kind == C1Blob || kind == C2Blob; } + static bool is_blob(Kind kind) { return kind == SharedBlob || kind == C1Blob || kind == C2Blob || kind == StubGenBlob; } + static bool is_single_stub_blob(Kind kind) { return kind == SharedBlob || kind == C1Blob || kind == C2Blob || kind == StubGenBlob; } + static bool is_multi_stub_blob(Kind kind) { return kind == StubGenBlob; } static bool is_adapter(Kind kind) { return kind == Adapter; } }; @@ -122,40 +126,36 @@ class AOTCodeAddressTable : public CHeapObj { private: address* _extrs_addr; address* _stubs_addr; - address* _shared_blobs_addr; - address* _C1_blobs_addr; uint _extrs_length; - uint _stubs_length; - uint _shared_blobs_length; - uint _C1_blobs_length; bool _extrs_complete; - bool _early_stubs_complete; - bool _shared_blobs_complete; - bool _early_c1_complete; + bool _shared_stubs_complete; + bool _c1_stubs_complete; + bool _c2_stubs_complete; + bool _stubgen_stubs_complete; bool _complete; public: AOTCodeAddressTable() : _extrs_addr(nullptr), _stubs_addr(nullptr), - _shared_blobs_addr(nullptr), - _C1_blobs_addr(nullptr), _extrs_length(0), - _stubs_length(0), - _shared_blobs_length(0), - _C1_blobs_length(0), _extrs_complete(false), - _early_stubs_complete(false), - _shared_blobs_complete(false), - _early_c1_complete(false), + _shared_stubs_complete(false), + _c1_stubs_complete(false), + _c2_stubs_complete(false), + _stubgen_stubs_complete(false), _complete(false) { } ~AOTCodeAddressTable(); void init_extrs(); - void init_early_stubs(); - void init_shared_blobs(); - void init_early_c1(); + void init_extrs2(); + void add_stub_entry(EntryId entry_id, address entry); + void add_external_addresses(GrowableArray
& addresses); + void set_shared_stubs_complete(); + void set_c1_stubs_complete(); + void set_c2_stubs_complete(); + void set_stubgen_stubs_complete(); const char* add_C_string(const char* str); int id_for_C_string(address str); address address_for_C_string(int idx); @@ -163,6 +163,138 @@ class AOTCodeAddressTable : public CHeapObj { address address_for_id(int id); }; +// Auxiliary class used by AOTStubData to locate addresses owned by a +// stub in the _address_array. + +class StubAddrRange { +private: + // Index of the first address owned by a stub or -1 if none present + int _start_index; + // Total number of addresses owned by a stub, including in order: + // start address for stub code and first entry, (exclusive) end + // address for stub code, all secondary entry addresses, any + // auxiliary addresses + uint _naddr; + public: + StubAddrRange() : _start_index(-1), _naddr(0) {} + int start_index() { return _start_index; } + int count() { return _naddr; } + + void default_init() { + _start_index = -1; + _naddr = 0; + } + + void init_entry(int start_index, int naddr) { + _start_index = start_index; + _naddr = naddr; + } +}; + +// class used to save and restore details of stubs embedded in a +// multi-stub (StubGen) blob + +class AOTStubData : public StackObj { + friend class AOTCodeCache; + friend class AOTCodeReader; +private: + BlobId _blob_id; // must be a stubgen blob id + // whatever buffer blob was successfully loaded from the AOT cache + // following a call to load_code_blob or nullptr + CodeBlob *_cached_blob; + // Array of addresses owned by stubs. Each stub appends addresses to + // this array as a block, whether at the end of generation or at the + // end of restoration from the cache. The first two addresses in + // each block are the "start" and "end2 address of the stub. Any + // other visible addresses located within the range [start,end) + // follow, either extra entries, data addresses or SEGV-protected + // subrange start, end and handler addresses. In the special case + // that the SEGV handler address is the (external) common address + // handler the array will hold value nullptr. + GrowableArray
_address_array; + // count of how many stubs exist in the current blob (not all of + // which may actually be generated) + int _stub_cnt; + // array identifying range of entries in _address_array for each stub + // indexed by offset of stub in blob + StubAddrRange* _ranges; + // id of last looked up stub or NO_STUBID if lookup not attempted or failed + StubId _current; + // offset of _current in blob or -1 if lookup not attempted or failed + int _current_idx; + + // flags indicating whether the AOT code cache is open and, if so, + // whether we are loading or storing stubs.the first of those + // cases whether we have encountered any invalid stubs or failed to + // find a stub that was being generated + enum Flags { + OPEN = 1 << 0, // cache is open for use + USING = 1 << 1, // open and loading stubs + DUMPING = 1 << 2, // open and storing stubs + INVALID = 1 << 3, // found invalid stub when loading + }; + + uint32_t _flags; + + void set_invalid() { _flags |= INVALID; } + + StubAddrRange& get_range(int idx) const { return _ranges[idx]; } + GrowableArray
& address_array() { return _address_array; } + // accessor for entry/auxiliary addresses defaults to start entry +public: + AOTStubData(BlobId blob_id); + + ~AOTStubData() { + FREE_C_HEAP_ARRAY(StubAddrRange, _ranges); + } + + bool is_open() { return (_flags & OPEN) != 0; } + bool is_using() { return (_flags & USING) != 0; } + bool is_dumping() { return (_flags & DUMPING) != 0; } + bool is_aot() { return is_using() || is_dumping(); } + bool is_invalid() { return (_flags & INVALID) != 0; } + + BlobId blob_id() { return _blob_id; } + StubId current_stub_id() { return _current; } + // + bool load_code_blob(); + bool store_code_blob(CodeBlob& new_blob, CodeBuffer *code_buffe); + // determine whether a stub is available in the AOT cache + bool find_archive_data(StubId stub_id); + // retrieve stub entry data if it we are using archived stubs and + // the stub has been found in an AOT-restored blob or store stub + // entry data if we are saving archived stubs and the stub has just + // been successfully generated into the current blob. + // + // start and end identify the inclusive start and exclusive end + // address for stub code and must lie in the current blob's code + // range. Stubs presented via this interface must declare at least + // one entry and start is always taken to be the first entry. + // + // Optional arrays entries and extras present other addresses of + // interest all of which must either lie in the interval (start, + // end) or be nullptr (verified by load and store methods). + // + // entries lists secondary entries for the stub each of which must + // match a corresponding entry declaration for the stub (entry count + // verified by load and store methods). Null entry addresses are + // allowed when an architecture does not require a specific entry + // but may not vary from one run to the next. If the cache is in use + // at a store (for loading or saving code) then non-null entry + // addresses are entered into the AOT cache stub address table + // allowing references to them from other stubs or nmethods to be + // relocated. + // + // extras lists other non-entry stub addresses of interest such as + // memory protection ranges and associated handler addresses. These + // do do not need to be declared as entries and their number and + // meaning may vary according to the architecture. + void load_archive_data(StubId stub_id, address& start, address& end, GrowableArray
* entries = nullptr, GrowableArray
* extras = nullptr); + void store_archive_data(StubId stub_id, address start, address end, GrowableArray
* entries = nullptr, GrowableArray
* extras = nullptr); + + const AOTStubData* as_const() { return (const AOTStubData*)this; } +}; + class AOTCodeCache : public CHeapObj { // Classes used to describe AOT code cache. @@ -284,6 +416,7 @@ class AOTCodeCache : public CHeapObj { void clear_lookup_failed() { _lookup_failed = false; } bool lookup_failed() const { return _lookup_failed; } + void add_stub_entry(EntryId entry_id, address entry) NOT_CDS_RETURN; public: AOTCodeCache(bool is_dumping, bool is_using); ~AOTCodeCache(); @@ -300,9 +433,12 @@ class AOTCodeCache : public CHeapObj { void load_strings(); int store_strings(); - static void init_early_stubs_table() NOT_CDS_RETURN; - static void init_shared_blobs_table() NOT_CDS_RETURN; - static void init_early_c1_table() NOT_CDS_RETURN; + static void set_shared_stubs_complete(); + static void set_c1_stubs_complete(); + static void set_c2_stubs_complete(); + static void set_stubgen_stubs_complete(); + + void add_stub_entries(StubId stub_id, address start, GrowableArray
*entries = nullptr, int offset = -1) NOT_CDS_RETURN; address address_for_C_string(int idx) const { return _table->address_for_C_string(idx); } address address_for_id(int id) const { return _table->address_for_id(id); } @@ -322,22 +458,41 @@ class AOTCodeCache : public CHeapObj { bool finish_write(); - bool write_relocations(CodeBlob& code_blob); + bool write_relocations(CodeBlob& code_blob, RelocIterator& iter); bool write_oop_map_set(CodeBlob& cb); + bool write_stub_data(CodeBlob& blob, AOTStubData *stub_data); #ifndef PRODUCT bool write_asm_remarks(CodeBlob& cb); bool write_dbg_strings(CodeBlob& cb); #endif // PRODUCT +private: + // internal private API to save and restore blobs + static bool store_code_blob(CodeBlob& blob, + AOTCodeEntry::Kind entry_kind, + uint id, + const char* name, + AOTStubData* stub_data, + CodeBuffer* code_buffer) NOT_CDS_RETURN_(false); + + static CodeBlob* load_code_blob(AOTCodeEntry::Kind kind, + uint id, + const char* name, + AOTStubData* stub_data) NOT_CDS_RETURN_(nullptr); + +public: // save and restore API for non-enumerable code blobs static bool store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind, - uint id, const char* name) NOT_CDS_RETURN_(false); + uint id, + const char* name) NOT_CDS_RETURN_(false); static CodeBlob* load_code_blob(AOTCodeEntry::Kind kind, uint id, const char* name) NOT_CDS_RETURN_(nullptr); // save and restore API for enumerable code blobs + + // API for single-stub blobs static bool store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind, BlobId id) NOT_CDS_RETURN_(false); @@ -345,6 +500,22 @@ class AOTCodeCache : public CHeapObj { static CodeBlob* load_code_blob(AOTCodeEntry::Kind kind, BlobId id) NOT_CDS_RETURN_(nullptr); + // API for multi-stub blobs -- for use by class StubGenerator. + + static bool store_code_blob(CodeBlob& blob, + AOTCodeEntry::Kind kind, + BlobId id, + AOTStubData* stub_data, + CodeBuffer *code_buffer) NOT_CDS_RETURN_(false); + + static CodeBlob* load_code_blob(AOTCodeEntry::Kind kind, + BlobId id, + AOTStubData* stub_data) NOT_CDS_RETURN_(nullptr); + + static void publish_external_addresses(GrowableArray
& addresses); + // publish all entries for a code blob in code cache address table + static void publish_stub_addresses(CodeBlob &code_blob, BlobId id, AOTStubData *stub_data); + static uint store_entries_cnt() { if (is_on_for_dump()) { return cache()->_store_entries_cnt; @@ -369,6 +540,7 @@ class AOTCodeCache : public CHeapObj { static AOTCodeCache* cache() { assert(_passed_init2, "Too early to ask"); return _cache; } static void initialize() NOT_CDS_RETURN; static void init2() NOT_CDS_RETURN; + static void init3() NOT_CDS_RETURN; static void close() NOT_CDS_RETURN; static bool is_on() CDS_ONLY({ return cache() != nullptr && !_cache->closing(); }) NOT_CDS_RETURN_(false); static bool is_on_for_use() CDS_ONLY({ return is_on() && _cache->for_use(); }) NOT_CDS_RETURN_(false); @@ -389,7 +561,7 @@ class AOTCodeCache : public CHeapObj { // Concurent AOT code reader class AOTCodeReader { private: - const AOTCodeCache* _cache; + AOTCodeCache* _cache; const AOTCodeEntry* _entry; const char* _load_buffer; // Loaded cached code buffer uint _read_position; // Position in _load_buffer @@ -406,11 +578,14 @@ class AOTCodeReader { public: AOTCodeReader(AOTCodeCache* cache, AOTCodeEntry* entry); - CodeBlob* compile_code_blob(const char* name); + CodeBlob* compile_code_blob(const char* name, AOTCodeEntry::Kind entry_kind, int id, AOTStubData* stub_data = nullptr); ImmutableOopMapSet* read_oop_map_set(); + void read_stub_data(CodeBlob* code_blob, AOTStubData *stub_data); + void publish_stub_addresses(CodeBlob &code_blob, BlobId id, AOTStubData *stub_data); + - void fix_relocations(CodeBlob* code_blob); + void fix_relocations(CodeBlob* code_blob, RelocIterator& iter); #ifndef PRODUCT void read_asm_remarks(AsmRemarks& asm_remarks); void read_dbg_strings(DbgStrings& dbg_strings); diff --git a/src/hotspot/share/prims/downcallLinker.hpp b/src/hotspot/share/prims/downcallLinker.hpp index 01ee5c5677666..b27b18aff2588 100644 --- a/src/hotspot/share/prims/downcallLinker.hpp +++ b/src/hotspot/share/prims/downcallLinker.hpp @@ -71,7 +71,7 @@ class DowncallLinker: AllStatic { bool needs_return_buffer, int captured_state_mask, bool needs_transition) - : StubCodeGenerator(buffer, PrintMethodHandleStubs), + : StubCodeGenerator(buffer, PrintMethodHandleStubs), _signature(signature), _num_args(num_args), _ret_bt(ret_bt), diff --git a/src/hotspot/share/runtime/init.cpp b/src/hotspot/share/runtime/init.cpp index adc49f84358b3..463460fe3d57a 100644 --- a/src/hotspot/share/runtime/init.cpp +++ b/src/hotspot/share/runtime/init.cpp @@ -70,6 +70,8 @@ void VM_Version_init(); void icache_init2(); void initialize_stub_info(); // must precede all blob/stub generation void preuniverse_stubs_init(); + +void stubs_AOTAddressTable_init(); void initial_stubs_init(); jint universe_init(); // depends on codeCache_init and preuniverse_stubs_init @@ -149,13 +151,17 @@ jint init_globals() { AOTCodeCache::init2(); // depends on universe_init, must be before initial_stubs_init AsyncLogWriter::initialize(); + stubs_AOTAddressTable_init(); // publish external addresses used by stubs + // depends on AOTCodeCache::init2 initial_stubs_init(); // stubgen initial stub routines // stack overflow exception blob is referenced by the interpreter - AOTCodeCache::init_early_stubs_table(); // need this after stubgen initial stubs and before shared runtime initial stubs SharedRuntime::generate_initial_stubs(); gc_barrier_stubs_init(); // depends on universe_init, must be before interpreter_init continuations_init(); // must precede continuation stub generation - continuation_stubs_init(); // depends on continuations_init + AOTCodeCache::init3(); // depends on stubs_AOTAddressTable_init + // and continuations_init and must + // precede continuation stub generation + continuation_stubs_init(); // depends on continuations_init and AOTCodeCache::init3 #if INCLUDE_JFR SharedRuntime::generate_jfr_stubs(); #endif @@ -164,7 +170,6 @@ jint init_globals() { InterfaceSupport_init(); VMRegImpl::set_regName(); // need this before generate_stubs (for printing oop maps). SharedRuntime::generate_stubs(); - AOTCodeCache::init_shared_blobs_table(); // need this after generate_stubs SharedRuntime::init_adapter_library(); // do this after AOTCodeCache::init_shared_blobs_table return JNI_OK; } diff --git a/src/hotspot/share/runtime/stubCodeGenerator.cpp b/src/hotspot/share/runtime/stubCodeGenerator.cpp index 43250c004ca3d..6a9d82d6fca2c 100644 --- a/src/hotspot/share/runtime/stubCodeGenerator.cpp +++ b/src/hotspot/share/runtime/stubCodeGenerator.cpp @@ -23,6 +23,7 @@ */ #include "asm/macroAssembler.inline.hpp" +#include "code/aotCodeCache.hpp" #include "code/codeCache.hpp" #include "compiler/disassembler.hpp" #include "oops/oop.inline.hpp" @@ -69,14 +70,16 @@ void StubCodeDesc::print() const { print_on(tty); } StubCodeGenerator::StubCodeGenerator(CodeBuffer* code, bool print_code) { _masm = new MacroAssembler(code); _blob_id = BlobId::NO_BLOBID; + _stub_data = nullptr; _print_code = PrintStubCode || print_code; } -StubCodeGenerator::StubCodeGenerator(CodeBuffer* code, BlobId blob_id, bool print_code) { +StubCodeGenerator::StubCodeGenerator(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data, bool print_code) { assert(StubInfo::is_stubgen(blob_id), "not a stubgen blob %s", StubInfo::name(blob_id)); _masm = new MacroAssembler(code); _blob_id = blob_id; + _stub_data = stub_data; _print_code = PrintStubCode || print_code; } @@ -91,11 +94,29 @@ StubCodeGenerator::~StubCodeGenerator() { #endif } +void StubCodeGenerator::setup_code_desc(const char* name, address start, address end, bool loaded_from_cache) { + StubCodeDesc* cdesc = new StubCodeDesc("StubRoutines", name, start, end); + cdesc->set_disp(uint(start - _masm->code_section()->outer()->insts_begin())); + if (loaded_from_cache) { + cdesc->set_loaded_from_cache(); + } + print_stub_code_desc(cdesc); + // copied from ~StubCodeMark() + Forte::register_stub(cdesc->name(), cdesc->begin(), cdesc->end()); + if (JvmtiExport::should_post_dynamic_code_generated()) { + JvmtiExport::post_dynamic_code_generated(cdesc->name(), cdesc->begin(), cdesc->end()); + } +} + void StubCodeGenerator::stub_prolog(StubCodeDesc* cdesc) { // default implementation - do nothing } void StubCodeGenerator::stub_epilog(StubCodeDesc* cdesc) { + print_stub_code_desc(cdesc); +} + +void StubCodeGenerator::print_stub_code_desc(StubCodeDesc* cdesc) { LogTarget(Debug, stubs) lt; if (lt.is_enabled()) { LogStream ls(lt); @@ -119,6 +140,53 @@ void StubCodeGenerator::stub_epilog(StubCodeDesc* cdesc) { } } +bool StubCodeGenerator::find_archive_data(StubId stub_id) { + // punt to stub data if it exists and is not for dumping + if (_stub_data == nullptr || _stub_data->is_dumping()) { + return false; + } + return _stub_data->find_archive_data(stub_id); +} + +void StubCodeGenerator::load_archive_data(StubId stub_id, address& start, address& end, GrowableArray
*entries, GrowableArray
* extras) { + assert(_stub_data != nullptr && _stub_data->current_stub_id() == stub_id, "no current archive data for %s", StubInfo::name(stub_id)); + + // punt to stub data + _stub_data->load_archive_data(stub_id, start, end, entries, extras); + + setup_code_desc(StubInfo::name(stub_id), start, end, true); +} + +void StubCodeGenerator::store_archive_data(StubId stub_id, address start, address end, GrowableArray
* entries, GrowableArray
* extras) { + // punt to stub data if we have any + if (_stub_data != nullptr) { + _stub_data->store_archive_data(stub_id, start, end, entries, extras); + } +} + +void StubCodeGenerator::print_statistics_on(outputStream* st) { + st->print_cr("StubRoutines Stubs:"); + st->print_cr(" Initial stubs: %d", StubInfo::stub_count(BlobId::stubgen_initial_id)); + st->print_cr(" Continuation stubs: %d", StubInfo::stub_count(BlobId::stubgen_continuation_id)); + st->print_cr(" Compiler stubs: %d", StubInfo::stub_count(BlobId::stubgen_compiler_id)); + st->print_cr(" Final stubs: %d", StubInfo::stub_count(BlobId::stubgen_final_id)); + + int emitted = 0; + int loaded_from_cache = 0; + + StubCodeDesc* scd = StubCodeDesc::first(); + while (scd != nullptr) { + if (!strcmp(scd->group(), "StubRoutines")) { + emitted += 1; + if (scd->loaded_from_cache()) { + loaded_from_cache += 1; + } + } + scd = StubCodeDesc::next(scd); + } + st->print_cr("Total stubroutines stubs emitted: %d (generated=%d, loaded from cache=%d)", emitted, emitted - loaded_from_cache, loaded_from_cache); +} + #ifdef ASSERT void StubCodeGenerator::verify_stub(StubId stub_id) { assert(StubRoutines::stub_to_blob(stub_id) == blob_id(), "wrong blob %s for generation of stub %s", StubRoutines::get_blob_name(blob_id()), StubRoutines::get_stub_name(stub_id)); diff --git a/src/hotspot/share/runtime/stubCodeGenerator.hpp b/src/hotspot/share/runtime/stubCodeGenerator.hpp index 7d8944c85eada..fb3c6a292cca0 100644 --- a/src/hotspot/share/runtime/stubCodeGenerator.hpp +++ b/src/hotspot/share/runtime/stubCodeGenerator.hpp @@ -26,6 +26,7 @@ #define SHARE_RUNTIME_STUBCODEGENERATOR_HPP #include "asm/assembler.hpp" +#include "code/aotCodeCache.hpp" #include "memory/allocation.hpp" #include "runtime/stubInfo.hpp" @@ -48,6 +49,7 @@ class StubCodeDesc: public CHeapObj { address _begin; // points to the first byte of the stub code (included) address _end; // points to the first byte after the stub code (excluded) uint _disp; // Displacement relative base address in buffer. + bool _loaded_from_cache; friend class StubCodeMark; friend class StubCodeGenerator; @@ -65,6 +67,8 @@ class StubCodeDesc: public CHeapObj { void set_disp(uint disp) { _disp = disp; } + void set_loaded_from_cache() { _loaded_from_cache = true; } + public: static StubCodeDesc* first() { return _list; } static StubCodeDesc* next(StubCodeDesc* desc) { return desc->_next; } @@ -81,6 +85,7 @@ class StubCodeDesc: public CHeapObj { _end = end; _disp = 0; _list = this; + _loaded_from_cache = false; }; static void freeze(); @@ -93,12 +98,11 @@ class StubCodeDesc: public CHeapObj { uint disp() const { return _disp; } int size_in_bytes() const { return pointer_delta_as_int(_end, _begin); } bool contains(address pc) const { return _begin <= pc && pc < _end; } + bool loaded_from_cache() const { return _loaded_from_cache; } void print_on(outputStream* st) const; void print() const; }; -// forward declare blob and stub id enums - // The base class for all stub-generating code generators. // Provides utility functions. @@ -108,10 +112,13 @@ class StubCodeGenerator: public StackObj { BlobId _blob_id; protected: MacroAssembler* _masm; + AOTStubData* _stub_data; + + void setup_code_desc(const char* name, address start, address end, bool loaded_from_cache); public: StubCodeGenerator(CodeBuffer* code, bool print_code = false); - StubCodeGenerator(CodeBuffer* code, BlobId blob_id, bool print_code = false); + StubCodeGenerator(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data = nullptr, bool print_code = false); ~StubCodeGenerator(); MacroAssembler* assembler() const { return _masm; } @@ -120,9 +127,16 @@ class StubCodeGenerator: public StackObj { virtual void stub_prolog(StubCodeDesc* cdesc); // called by StubCodeMark constructor virtual void stub_epilog(StubCodeDesc* cdesc); // called by StubCodeMark destructor + void print_stub_code_desc(StubCodeDesc* cdesc); + + static void print_statistics_on(outputStream* st); + bool find_archive_data(StubId stub_id); + void load_archive_data(StubId stub_id, address& start, address& end, GrowableArray
*entries = nullptr, GrowableArray
* extras = nullptr); + void store_archive_data(StubId stub_id, address start, address end, GrowableArray
*entries = nullptr, GrowableArray
* extras = nullptr); #ifdef ASSERT void verify_stub(StubId stub_id); #endif + }; // Stack-allocated helper class used to associate a stub code with a name. diff --git a/src/hotspot/share/runtime/stubDeclarations.hpp b/src/hotspot/share/runtime/stubDeclarations.hpp index b051c6d0e184e..15735797d7c68 100644 --- a/src/hotspot/share/runtime/stubDeclarations.hpp +++ b/src/hotspot/share/runtime/stubDeclarations.hpp @@ -975,9 +975,15 @@ do_entry_init(final, arrayof_jlong_arraycopy, \ arrayof_jlong_arraycopy, arrayof_jlong_arraycopy, \ StubRoutines::arrayof_jlong_copy) \ + do_entry(final, arrayof_jlong_arraycopy, \ + arrayof_jlong_arraycopy_nopush, \ + arrayof_jlong_arraycopy_nopush) \ do_stub(final, arrayof_oop_arraycopy) \ do_entry_init(final, arrayof_oop_arraycopy, arrayof_oop_arraycopy, \ arrayof_oop_arraycopy, StubRoutines::arrayof_oop_copy) \ + do_entry(final, arrayof_oop_arraycopy, \ + arrayof_oop_arraycopy_nopush, \ + arrayof_oop_arraycopy_nopush) \ do_stub(final, arrayof_oop_arraycopy_uninit) \ do_entry_init(final, arrayof_oop_arraycopy_uninit, \ arrayof_oop_arraycopy_uninit, \ diff --git a/src/hotspot/share/runtime/stubInfo.cpp b/src/hotspot/share/runtime/stubInfo.cpp index bca6ff344ea85..6c7769dd66d8d 100644 --- a/src/hotspot/share/runtime/stubInfo.cpp +++ b/src/hotspot/share/runtime/stubInfo.cpp @@ -1096,6 +1096,15 @@ int StubInfo::stubgen_offset(StubId id) { return local_offset(StubGroup::STUBGEN, id); } +int StubInfo::stubgen_offset_in_blob(BlobId blob_id, StubId id) { + assert(blob(id) == blob_id, "sanity!"); + StubGroup group = StubGroup::STUBGEN; + assert(stubgroup(blob_id) == group, "sanity"); + StubId base_id = stub_base(blob_id); + assert(base_id != StubId::NO_STUBID, "sanity"); + return local_offset(group, id) - local_offset(group, base_id); +} + // initialization function called to populate blob. stub and entry // tables. this must be called before any stubs are generated void initialize_stub_info() { diff --git a/src/hotspot/share/runtime/stubInfo.hpp b/src/hotspot/share/runtime/stubInfo.hpp index eaea07f7e6c02..323d006e850c4 100644 --- a/src/hotspot/share/runtime/stubInfo.hpp +++ b/src/hotspot/share/runtime/stubInfo.hpp @@ -678,6 +678,11 @@ class StubInfo: AllStatic { static int c1_offset(StubId id); static int c2_offset(StubId id); static int stubgen_offset(StubId id); + + // Convert a stub id to a unique, zero-based offset in the range of + // stub ids for a given blob in the stubgen stub group. + + static int stubgen_offset_in_blob(BlobId blob_id, StubId id); }; diff --git a/src/hotspot/share/runtime/stubRoutines.cpp b/src/hotspot/share/runtime/stubRoutines.cpp index 5246613738e46..3b8c004b36827 100644 --- a/src/hotspot/share/runtime/stubRoutines.cpp +++ b/src/hotspot/share/runtime/stubRoutines.cpp @@ -102,7 +102,10 @@ BlobId StubRoutines::stub_to_blob(StubId id) { // Initialization -extern void StubGenerator_generate(CodeBuffer* code, BlobId blob_id); // only interface to generators +extern void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data); // only interface to generators +#if (defined(X86) && defined(_LP64)) || defined(AARCH64) +extern void StubGenerator_AOTAddressTable_init(); +#endif void UnsafeMemoryAccess::create_table(int max_size) { UnsafeMemoryAccess::_table = new UnsafeMemoryAccess[max_size]; @@ -169,6 +172,33 @@ static BufferBlob* initialize_stubs(BlobId blob_id, assert(StubInfo::is_stubgen(blob_id), "not a stubgen blob %s", StubInfo::name(blob_id)); ResourceMark rm; TraceTime timer(timer_msg, TRACETIME_LOG(Info, startuptime)); + // If we are loading stubs we need to check if we can retrieve a + // blob and/or an associated archived stub descriptor from the + // AOTCodeCache. If we are storing stubs we need to create a blob + // but we still need a stub data descriptor to fill in during + // generation. + AOTStubData stub_data(blob_id); + AOTStubData* stub_data_p = nullptr; + LogTarget(Info, stubs) lt; + + if (code_size > 0 && stub_data.is_using()) { + // AOTCodeEntry tracks and logs status of any cached blob + bool loaded = stub_data.load_code_blob(); + if (loaded) { + if (lt.is_enabled()) { + LogStream ls(lt); + ls.print_cr("Found blob %s in AOT cache", StubInfo::name(blob_id)); + } + stub_data_p = &stub_data; + } + } else if (stub_data.is_dumping()) { + stub_data_p = &stub_data; + } + + // Even if we managed to load a blob from the AOT cache we still + // need to allocate a code blob and associated buffer. The AOT blob + // may not include all the stubs we need for this runtime. + // Add extra space for large CodeEntryAlignment int size = code_size + CodeEntryAlignment * max_aligned_stubs; BufferBlob* stubs_code = BufferBlob::create(buffer_name, size); @@ -178,6 +208,10 @@ static BufferBlob* initialize_stubs(BlobId blob_id, // In that case we can tolerate an allocation failure because the // compiler will have been shut down and we have no need of the // blob. + // TODO: Ideally we would still like to try to use any AOT cached + // blob here but we don't have a fallback if we find that it is + // missing stubs we need so for now we exit. This should only + // happen in cases where we have a very small code cache. if (Thread::current()->is_Compiler_thread()) { assert(blob_id == BlobId::stubgen_compiler_id, "sanity"); assert(DelayCompilerStubsGeneration, "sanity"); @@ -187,7 +221,10 @@ static BufferBlob* initialize_stubs(BlobId blob_id, vm_exit_out_of_memory(code_size, OOM_MALLOC_ERROR, "CodeCache: no room for %s", buffer_name); } CodeBuffer buffer(stubs_code); - StubGenerator_generate(&buffer, blob_id); + short buffer_locs[20]; + buffer.insts()->initialize_shared_locs((relocInfo*)buffer_locs, + sizeof(buffer_locs)/sizeof(relocInfo)); + StubGenerator_generate(&buffer, blob_id, stub_data_p); if (code_size == 0) { assert(buffer.insts_size() == 0, "should not write into buffer when bob size declared as 0"); LogTarget(Info, stubs) lt; @@ -203,7 +240,25 @@ static BufferBlob* initialize_stubs(BlobId blob_id, "increase %s, code_size: %d, used: %d, free: %d", assert_msg, code_size, buffer.total_content_size(), buffer.insts_remaining()); - LogTarget(Info, stubs) lt; + if (AOTCodeCache::is_dumping_stub()) { + if (stub_data.store_code_blob(*stubs_code, &buffer)) { + if (lt.is_enabled()) { + LogStream ls(lt); + ls.print_cr("Stored blob '%s' to Startup Code Cache", buffer_name); + } + } else { + if (lt.is_enabled()) { + LogStream ls(lt); + ls.print_cr("Failed to store blob '%s' to Startup Code Cache", buffer_name); + } + } + } + + // close off recording of any further stubgen generation + if (blob_id == BlobId::stubgen_final_id) { + AOTCodeCache::set_stubgen_stubs_complete(); + } + if (lt.is_enabled()) { LogStream ls(lt); ls.print_cr("%s\t [" INTPTR_FORMAT ", " INTPTR_FORMAT "] used: %d, free: %d", @@ -214,6 +269,8 @@ static BufferBlob* initialize_stubs(BlobId blob_id, return stubs_code; } +// per blob initializer methods StubRoutines::initialize_xxx_stubs() + #define DEFINE_BLOB_INIT_METHOD(blob_name) \ void StubRoutines::initialize_ ## blob_name ## _stubs() { \ if (STUBGEN_BLOB_FIELD_NAME(blob_name) == nullptr) { \ @@ -234,6 +291,7 @@ STUBGEN_BLOBS_DO(DEFINE_BLOB_INIT_METHOD) #undef DEFINE_BLOB_INIT_METHOD +// external driver API functions for per blob init: xxx_stubs_init() #define DEFINE_BLOB_INIT_FUNCTION(blob_name) \ void blob_name ## _stubs_init() { \ @@ -244,11 +302,24 @@ STUBGEN_BLOBS_DO(DEFINE_BLOB_INIT_FUNCTION) #undef DEFINE_BLOB_INIT_FUNCTION + +// Non-generated init method + +void StubRoutines::init_AOTAddressTable() { +#if (defined(X86) && defined(_LP64)) || defined(AARCH64) + StubGenerator_AOTAddressTable_init(); +#endif +} + +// non-generated external API init driver function + +void stubs_AOTAddressTable_init() { StubRoutines::init_AOTAddressTable(); } + /* - * we generate the underlying driver method but this wrapper is needed - * to perform special handling depending on where the compiler init - * gets called from. it ought to be possible to remove this at some - * point and have a determinate ordered init. + * we generate the underlying driver function compiler_stubs_init() + * but this wrapper is needed to perform special handling depending on + * where the compiler init gets called from. it ought to be possible + * to remove this at some point and have a determinate ordered init. */ void compiler_stubs_init(bool in_compiler_thread) { diff --git a/src/hotspot/share/runtime/stubRoutines.hpp b/src/hotspot/share/runtime/stubRoutines.hpp index 97e3e46b87063..b2384c33a998d 100644 --- a/src/hotspot/share/runtime/stubRoutines.hpp +++ b/src/hotspot/share/runtime/stubRoutines.hpp @@ -130,6 +130,7 @@ class UnsafeMemoryAccess : public CHeapObj { static UnsafeMemoryAccess* add_to_table(address start_pc, address end_pc, address error_exit_pc) { guarantee(_table_length < _table_max_length, "Incorrect UnsafeMemoryAccess::_table_max_length"); UnsafeMemoryAccess* entry = &_table[_table_length]; + assert(start_pc != nullptr, "invalid start address"); entry->set_start_pc(start_pc); entry->set_end_pc(end_pc); entry->set_error_exit_pc(error_exit_pc); @@ -283,6 +284,9 @@ class StubRoutines: AllStatic { static BlobId stub_to_blob(StubId id); #endif + // Initalization + static void init_AOTAddressTable(); + // Debugging static jint verify_oop_count() { return _verify_oop_count; } static jint* verify_oop_count_addr() { return &_verify_oop_count; } diff --git a/test/hotspot/jtreg/runtime/cds/appcds/aotCode/AOTCodeFlags.java b/test/hotspot/jtreg/runtime/cds/appcds/aotCode/AOTCodeFlags.java index 761c2c5382d61..4219cbdf6be88 100644 --- a/test/hotspot/jtreg/runtime/cds/appcds/aotCode/AOTCodeFlags.java +++ b/test/hotspot/jtreg/runtime/cds/appcds/aotCode/AOTCodeFlags.java @@ -51,7 +51,7 @@ public class AOTCodeFlags { public static void main(String... args) throws Exception { Tester t = new Tester(); // Run only 2 modes (0 - no AOT code, 1 - AOT adapters) until JDK-8357398 is fixed - for (int mode = 0; mode < 2; mode++) { + for (int mode = 0; mode < 4; mode++) { t.setTestMode(mode); t.run(new String[] {"AOT", "--two-step-training"}); } From 53cf3555bda01c22886595635d56f10a24d14745 Mon Sep 17 00:00:00 2001 From: Andrew Dinn Date: Tue, 11 Nov 2025 16:00:32 +0000 Subject: [PATCH 02/30] count and log stubgen blobs --- src/hotspot/share/code/aotCodeCache.cpp | 9 +++++++-- src/hotspot/share/code/aotCodeCache.hpp | 11 +++++++---- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/hotspot/share/code/aotCodeCache.cpp b/src/hotspot/share/code/aotCodeCache.cpp index a72a7ba9d49a3..d756cc05616e6 100644 --- a/src/hotspot/share/code/aotCodeCache.cpp +++ b/src/hotspot/share/code/aotCodeCache.cpp @@ -772,6 +772,7 @@ bool AOTCodeCache::finish_write() { AOTCodeEntry* entries_address = _store_entries; // Pointer to latest entry uint adapters_count = 0; uint shared_blobs_count = 0; + uint stubgen_blobs_count = 0; uint C1_blobs_count = 0; uint C2_blobs_count = 0; uint max_size = 0; @@ -799,6 +800,8 @@ bool AOTCodeCache::finish_write() { adapters_count++; } else if (kind == AOTCodeEntry::SharedBlob) { shared_blobs_count++; + } else if (kind == AOTCodeEntry::StubGenBlob) { + stubgen_blobs_count++; } else if (kind == AOTCodeEntry::C1Blob) { C1_blobs_count++; } else if (kind == AOTCodeEntry::C2Blob) { @@ -835,6 +838,7 @@ bool AOTCodeCache::finish_write() { log_debug(aot, codecache, exit)(" Adapters: total=%u", adapters_count); log_debug(aot, codecache, exit)(" Shared Blobs: total=%d", shared_blobs_count); + log_debug(aot, codecache, exit)(" StubGen Blobs: total=%d", stubgen_blobs_count); log_debug(aot, codecache, exit)(" C1 Blobs: total=%d", C1_blobs_count); log_debug(aot, codecache, exit)(" C2 Blobs: total=%d", C2_blobs_count); log_debug(aot, codecache, exit)(" AOT code cache size: %u bytes, max entry's size: %u bytes", size, max_size); @@ -844,7 +848,8 @@ bool AOTCodeCache::finish_write() { header->init(size, (uint)strings_count, strings_offset, entries_count, new_entries_offset, adapters_count, shared_blobs_count, - C1_blobs_count, C2_blobs_count); + stubgen_blobs_count, C1_blobs_count, + C2_blobs_count); log_info(aot, codecache, exit)("Wrote %d AOT code entries to AOT Code Cache", entries_count); } @@ -2028,7 +2033,7 @@ void AOTCodeAddressTable::set_c2_stubs_complete() { void AOTCodeAddressTable::set_stubgen_stubs_complete() { assert(!_stubgen_stubs_complete, "repeated close for stubgen stubs!"); _stubgen_stubs_complete = true; - log_debug(aot, codecache, init)("Stubgen stubs closed"); + log_debug(aot, codecache, init)("StubGen stubs closed"); } AOTCodeAddressTable::~AOTCodeAddressTable() { diff --git a/src/hotspot/share/code/aotCodeCache.hpp b/src/hotspot/share/code/aotCodeCache.hpp index 2487dc7ff615a..fb970575eb2bb 100644 --- a/src/hotspot/share/code/aotCodeCache.hpp +++ b/src/hotspot/share/code/aotCodeCache.hpp @@ -336,16 +336,18 @@ class AOTCodeCache : public CHeapObj { uint _entries_offset; // offset of AOTCodeEntry array describing entries uint _adapters_count; uint _shared_blobs_count; + uint _stubgen_blobs_count; uint _C1_blobs_count; uint _C2_blobs_count; Config _config; public: void init(uint cache_size, - uint strings_count, uint strings_offset, - uint entries_count, uint entries_offset, - uint adapters_count, uint shared_blobs_count, - uint C1_blobs_count, uint C2_blobs_count) { + uint strings_count, uint strings_offset, + uint entries_count, uint entries_offset, + uint adapters_count, uint shared_blobs_count, + uint stubgen_blobs_count, uint C1_blobs_count, + uint C2_blobs_count) { _version = AOT_CODE_VERSION; _cache_size = cache_size; _strings_count = strings_count; @@ -354,6 +356,7 @@ class AOTCodeCache : public CHeapObj { _entries_offset = entries_offset; _adapters_count = adapters_count; _shared_blobs_count = shared_blobs_count; + _stubgen_blobs_count = stubgen_blobs_count; _C1_blobs_count = C1_blobs_count; _C2_blobs_count = C2_blobs_count; _config.record(); From 1397164b55b5aaaeacdc684a50d52011ee453735 Mon Sep 17 00:00:00 2001 From: Andrew Dinn Date: Thu, 13 Nov 2025 10:22:11 +0000 Subject: [PATCH 03/30] remove redundant publication of external addresses --- src/hotspot/share/code/aotCodeCache.cpp | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/src/hotspot/share/code/aotCodeCache.cpp b/src/hotspot/share/code/aotCodeCache.cpp index d756cc05616e6..40c2b95a57827 100644 --- a/src/hotspot/share/code/aotCodeCache.cpp +++ b/src/hotspot/share/code/aotCodeCache.cpp @@ -1753,9 +1753,6 @@ void AOTCodeAddressTable::init_extrs() { { // Required by initial stubs SET_ADDRESS(_extrs, SharedRuntime::exception_handler_for_return_address); // used by forward_exception -#if defined(AMD64) || defined(AARCH64) || defined(RISCV64) - SET_ADDRESS(_extrs, MacroAssembler::debug64); // used by many eg forward_exception, call_stub -#endif // defined(AMD64) || defined(AARCH64) || defined(RISCV64) #if defined(AMD64) SET_ADDRESS(_extrs, StubRoutines::x86::addr_mxcsr_std()); // used by call_stub SET_ADDRESS(_extrs, StubRoutines::x86::addr_mxcsr_rz()); // used by libmFmod @@ -1818,8 +1815,6 @@ void AOTCodeAddressTable::init_extrs() { SET_ADDRESS(_extrs, &JvmtiExport::_should_notify_object_alloc); #endif /* INCLUDE_JVMTI */ - SET_ADDRESS(_extrs, SafepointSynchronize::handle_polling_page_exception); - SET_ADDRESS(_extrs, ThreadIdentifier::unsafe_offset()); SET_ADDRESS(_extrs, Thread::current); @@ -1833,17 +1828,11 @@ void AOTCodeAddressTable::init_extrs() { SET_ADDRESS(_extrs, &JvmtiVTMSTransitionDisabler::_VTMS_notify_jvmti_events); #endif /* INCLUDE_JVMTI */ SET_ADDRESS(_extrs, StubRoutines::crc_table_addr()); -#if defined(AARCH64) - SET_ADDRESS(_extrs, JavaThread::aarch64_get_thread_helper); -#endif #ifndef PRODUCT SET_ADDRESS(_extrs, &SharedRuntime::_partial_subtype_ctr); SET_ADDRESS(_extrs, JavaThread::verify_cross_modify_fence_failure); #endif -#if defined(AMD64) || defined(AARCH64) || defined(RISCV64) - SET_ADDRESS(_extrs, MacroAssembler::debug64); -#endif #if defined(AMD64) SET_ADDRESS(_extrs, StubRoutines::x86::arrays_hashcode_powers_of_31()); #endif @@ -1869,7 +1858,6 @@ void AOTCodeAddressTable::init_extrs() { SET_ADDRESS(_extrs, SharedRuntime::resolve_opt_virtual_call_C); SET_ADDRESS(_extrs, SharedRuntime::resolve_virtual_call_C); SET_ADDRESS(_extrs, SharedRuntime::resolve_static_call_C); - SET_ADDRESS(_extrs, SharedRuntime::throw_StackOverflowError); SET_ADDRESS(_extrs, SharedRuntime::throw_delayed_StackOverflowError); SET_ADDRESS(_extrs, SharedRuntime::throw_AbstractMethodError); SET_ADDRESS(_extrs, SharedRuntime::throw_IncompatibleClassChangeError); @@ -1880,7 +1868,6 @@ void AOTCodeAddressTable::init_extrs() { { // Required by C1 blobs SET_ADDRESS(_extrs, static_cast(SharedRuntime::dtrace_object_alloc)); - SET_ADDRESS(_extrs, SharedRuntime::exception_handler_for_return_address); SET_ADDRESS(_extrs, SharedRuntime::register_finalizer); SET_ADDRESS(_extrs, Runtime1::is_instance_of); SET_ADDRESS(_extrs, Runtime1::exception_handler_for_pc); @@ -1897,7 +1884,6 @@ void AOTCodeAddressTable::init_extrs() { SET_ADDRESS(_extrs, Runtime1::throw_array_store_exception); SET_ADDRESS(_extrs, Runtime1::throw_class_cast_exception); SET_ADDRESS(_extrs, Runtime1::throw_incompatible_class_change_error); - SET_ADDRESS(_extrs, Runtime1::is_instance_of); SET_ADDRESS(_extrs, Runtime1::monitorenter); SET_ADDRESS(_extrs, Runtime1::monitorexit); SET_ADDRESS(_extrs, Runtime1::deoptimize); @@ -1909,9 +1895,6 @@ void AOTCodeAddressTable::init_extrs() { SET_ADDRESS(_extrs, Runtime1::unimplemented_entry); SET_ADDRESS(_extrs, Thread::current); SET_ADDRESS(_extrs, CompressedKlassPointers::base_addr()); -#ifndef PRODUCT - SET_ADDRESS(_extrs, os::breakpoint); -#endif } #endif @@ -1940,9 +1923,6 @@ void AOTCodeAddressTable::init_extrs() { SET_ADDRESS(_extrs, OptoRuntime::rethrow_C); SET_ADDRESS(_extrs, OptoRuntime::slow_arraycopy_C); SET_ADDRESS(_extrs, OptoRuntime::register_finalizer_C); -#if defined(AARCH64) - SET_ADDRESS(_extrs, JavaThread::verify_cross_modify_fence_failure); -#endif // AARCH64 } #endif // COMPILER2 From 3f052ce97441227bc2863ade18e43460c239f9eb Mon Sep 17 00:00:00 2001 From: Andrew Dinn Date: Mon, 17 Nov 2025 15:48:00 +0000 Subject: [PATCH 04/30] fix missing external address --- src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp index 78ba98811849d..6bc70e0d07016 100644 --- a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp @@ -663,7 +663,8 @@ class StubGenerator: public StubCodeGenerator { __ verify_oop(r0); __ str(r0, Address(rthread, Thread::pending_exception_offset())); - __ mov(rscratch1, (address)__FILE__); + __ lea(rscratch1, ExternalAddress((address)__FILE__)); + AOTCodeCache::add_C_string(__FILE__); __ str(rscratch1, Address(rthread, Thread::exception_file_offset())); __ movw(rscratch1, (int)__LINE__); __ strw(rscratch1, Address(rthread, Thread::exception_line_offset())); From 32f1320d5302865bf6406b20e3a60a5add92a5c7 Mon Sep 17 00:00:00 2001 From: Andrew Dinn Date: Mon, 17 Nov 2025 16:11:39 +0000 Subject: [PATCH 05/30] fix external address problem --- src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp index 6bc70e0d07016..8a4f9e2517321 100644 --- a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp @@ -663,8 +663,9 @@ class StubGenerator: public StubCodeGenerator { __ verify_oop(r0); __ str(r0, Address(rthread, Thread::pending_exception_offset())); - __ lea(rscratch1, ExternalAddress((address)__FILE__)); - AOTCodeCache::add_C_string(__FILE__); + // special case -- add file name string to AOT address table + address file = (address)AOTCodeCache::add_C_string(__FILE__); + __ lea(rscratch1, ExternalAddress(file)); __ str(rscratch1, Address(rthread, Thread::exception_file_offset())); __ movw(rscratch1, (int)__LINE__); __ strw(rscratch1, Address(rthread, Thread::exception_line_offset())); From b9adda3eb21922056a968f657dcb8a157cb77f9f Mon Sep 17 00:00:00 2001 From: Andrew Dinn Date: Mon, 17 Nov 2025 16:40:23 +0000 Subject: [PATCH 06/30] fix stub publish/save checks --- src/hotspot/share/runtime/stubRoutines.cpp | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/hotspot/share/runtime/stubRoutines.cpp b/src/hotspot/share/runtime/stubRoutines.cpp index 3b8c004b36827..84d469af5c513 100644 --- a/src/hotspot/share/runtime/stubRoutines.cpp +++ b/src/hotspot/share/runtime/stubRoutines.cpp @@ -227,7 +227,6 @@ static BufferBlob* initialize_stubs(BlobId blob_id, StubGenerator_generate(&buffer, blob_id, stub_data_p); if (code_size == 0) { assert(buffer.insts_size() == 0, "should not write into buffer when bob size declared as 0"); - LogTarget(Info, stubs) lt; if (lt.is_enabled()) { LogStream ls(lt); ls.print_cr("%s\t not generated", buffer_name); @@ -240,7 +239,17 @@ static BufferBlob* initialize_stubs(BlobId blob_id, "increase %s, code_size: %d, used: %d, free: %d", assert_msg, code_size, buffer.total_content_size(), buffer.insts_remaining()); - if (AOTCodeCache::is_dumping_stub()) { + if (stub_data.is_using()) { + // we generated some new entries so republish all entries TODO - + // ensure we publish collect and publish the preuniverse stubs but + // don't try to save them + AOTCodeCache::publish_stub_addresses(*stubs_code, blob_id, &stub_data); + if (lt.is_enabled()) { + LogStream ls(lt); + ls.print_cr("Republished entries for blob '%s'", buffer_name); + } + } else if (stub_data.is_dumping()) { + // save the blob and publihs the entry addresses if (stub_data.store_code_blob(*stubs_code, &buffer)) { if (lt.is_enabled()) { LogStream ls(lt); From 16f4d1fa427e6bdf97ef7931ab821c72553f6416 Mon Sep 17 00:00:00 2001 From: Andrew Dinn Date: Mon, 17 Nov 2025 16:48:18 +0000 Subject: [PATCH 07/30] move handler management routines to shared code --- .../cpu/aarch64/stubGenerator_aarch64.cpp | 29 +----------------- .../share/runtime/stubCodeGenerator.cpp | 30 +++++++++++++++++++ .../share/runtime/stubCodeGenerator.hpp | 3 ++ 3 files changed, 34 insertions(+), 28 deletions(-) diff --git a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp index 8a4f9e2517321..51741452d14d8 100644 --- a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp @@ -357,33 +357,6 @@ class StubGenerator: public StubCodeGenerator { retaddr_off = 1, }; - // Helper used to restore saved ranges and handler addresses. - // Expects entries to contain 3 * count addresses beginning at - // offset begin which identify start of range, end of range and - // address of handler pc. end of range may be nullptr in which case - // it defaults to stub_end. hanlder pc may be nullptr in which case - // it defaults to default_handler. - - void register_unsafe_access_handlers(GrowableArray
&entries, int begin, int count, address stub_end) { - for (int i = 0; i < count; i++) { - int offset = begin + 3 * i; - address start = entries.at(offset); - address end = entries.at(offset + 1); - if (end == nullptr) { - end = stub_end; - } - address handler = entries.at(offset + 2); - if (handler == nullptr) { - handler = UnsafeMemoryAccess::common_exit_stub_pc(); - } - UnsafeMemoryAccess::add_to_table(start, end, handler); - } - } - - void retrieve_unsafe_access_handlers(address start, address end, GrowableArray
&entries) { - UnsafeMemoryAccess::collect_entries(start, end, entries); - } - address generate_call_stub(address& return_address) { assert((int)frame::entry_frame_after_call_words == -(int)sp_after_call_off + 1 && (int)frame::entry_frame_call_wrapper_offset == (int)call_wrapper_off, @@ -859,7 +832,6 @@ class StubGenerator: public StubCodeGenerator { // Generate indices for iota vector. address generate_iota_indices(StubId stub_id) { - __ align(CodeEntryAlignment); int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); if (find_archive_data(stub_id)) { @@ -869,6 +841,7 @@ class StubGenerator: public StubCodeGenerator { return start; } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); // B diff --git a/src/hotspot/share/runtime/stubCodeGenerator.cpp b/src/hotspot/share/runtime/stubCodeGenerator.cpp index 6a9d82d6fca2c..4859a110ca2ce 100644 --- a/src/hotspot/share/runtime/stubCodeGenerator.cpp +++ b/src/hotspot/share/runtime/stubCodeGenerator.cpp @@ -108,6 +108,36 @@ void StubCodeGenerator::setup_code_desc(const char* name, address start, address } } +// Helper used to restore ranges and handler addresses restored from +// AOT cache. Expects entries to contain 3 * count addresses beginning +// at offset begin which identify start of range, end of range and +// address of handler pc. end of range may be nullptr in which case it +// defaults to stub_end. hanlder pc may be nullptr in which case it +// defaults to default_handler. + +void StubCodeGenerator::register_unsafe_access_handlers(GrowableArray
&entries, int begin, int count, address stub_end) { + for (int i = 0; i < count; i++) { + int offset = begin + 3 * i; + address start = entries.at(offset); + address end = entries.at(offset + 1); + if (end == nullptr) { + end = stub_end; + } + address handler = entries.at(offset + 2); + if (handler == nullptr) { + handler = UnsafeMemoryAccess::common_exit_stub_pc(); + } + UnsafeMemoryAccess::add_to_table(start, end, handler); + } +} + +// Helper used to retrieve ranges and handler addresses registered +// during generation of the stub which spans [start, end) in order to +// allow them to be saved to an AOT cache. +void StubCodeGenerator::retrieve_unsafe_access_handlers(address start, address end, GrowableArray
&entries) { + UnsafeMemoryAccess::collect_entries(start, end, entries); +} + void StubCodeGenerator::stub_prolog(StubCodeDesc* cdesc) { // default implementation - do nothing } diff --git a/src/hotspot/share/runtime/stubCodeGenerator.hpp b/src/hotspot/share/runtime/stubCodeGenerator.hpp index fb3c6a292cca0..18cf461f27e35 100644 --- a/src/hotspot/share/runtime/stubCodeGenerator.hpp +++ b/src/hotspot/share/runtime/stubCodeGenerator.hpp @@ -115,6 +115,9 @@ class StubCodeGenerator: public StackObj { AOTStubData* _stub_data; void setup_code_desc(const char* name, address start, address end, bool loaded_from_cache); + // unsafe handler management + void register_unsafe_access_handlers(GrowableArray
&entries, int begin, int count, address stub_end); + void retrieve_unsafe_access_handlers(address start, address end, GrowableArray
&entries); public: StubCodeGenerator(CodeBuffer* code, bool print_code = false); From 9f27b2bf1fa74e47bed751426ec9bd354cf4acda Mon Sep 17 00:00:00 2001 From: Andrew Dinn Date: Tue, 18 Nov 2025 17:04:53 +0000 Subject: [PATCH 08/30] first complete version of x86 stub save and restore --- .../cpu/aarch64/stubGenerator_aarch64.cpp | 2 +- src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp | 9 + src/hotspot/cpu/x86/c1_LIRAssembler_x86.hpp | 1 + src/hotspot/cpu/x86/macroAssembler_x86.cpp | 2 +- .../cpu/x86/macroAssembler_x86_sha.cpp | 10 +- src/hotspot/cpu/x86/stubDeclarations_x86.hpp | 9 + src/hotspot/cpu/x86/stubGenerator_x86_64.cpp | 973 +++++++++++++++++- src/hotspot/cpu/x86/stubGenerator_x86_64.hpp | 28 +- .../cpu/x86/stubGenerator_x86_64_adler.cpp | 8 + .../cpu/x86/stubGenerator_x86_64_aes.cpp | 22 + .../x86/stubGenerator_x86_64_arraycopy.cpp | 469 ++++++++- .../cpu/x86/stubGenerator_x86_64_cbrt.cpp | 21 + .../cpu/x86/stubGenerator_x86_64_chacha.cpp | 7 + .../x86/stubGenerator_x86_64_constants.cpp | 24 + .../x86/stubGenerator_x86_64_dilithium.cpp | 11 + .../cpu/x86/stubGenerator_x86_64_exp.cpp | 19 + .../cpu/x86/stubGenerator_x86_64_fmod.cpp | 10 + .../cpu/x86/stubGenerator_x86_64_ghash.cpp | 9 + .../cpu/x86/stubGenerator_x86_64_kyber.cpp | 16 + .../cpu/x86/stubGenerator_x86_64_log.cpp | 20 + .../cpu/x86/stubGenerator_x86_64_poly1305.cpp | 9 + .../x86/stubGenerator_x86_64_poly_mont.cpp | 13 + .../cpu/x86/stubGenerator_x86_64_pow.cpp | 28 + .../cpu/x86/stubGenerator_x86_64_sha3.cpp | 9 + .../cpu/x86/stubGenerator_x86_64_sin.cpp | 6 + .../cpu/x86/stubGenerator_x86_64_sinh.cpp | 20 + .../cpu/x86/stubGenerator_x86_64_tan.cpp | 21 + .../cpu/x86/stubGenerator_x86_64_tanh.cpp | 20 + src/hotspot/cpu/x86/stubRoutines_x86.cpp | 25 + src/hotspot/cpu/x86/stubRoutines_x86.hpp | 2 + src/hotspot/share/code/aotCodeCache.cpp | 17 +- 31 files changed, 1737 insertions(+), 103 deletions(-) diff --git a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp index 51741452d14d8..266bd7097144b 100644 --- a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp @@ -11196,7 +11196,7 @@ class StubGenerator: public StubCodeGenerator { address start = nullptr; address end = nullptr; load_archive_data(stub_id, start, end, &entries); - assert(entries.length() == 1, + assert(entries.length() == Klass::SECONDARY_SUPERS_TABLE_SIZE - 1, "unexpected extra entry count %d", entries.length()); StubRoutines::_lookup_secondary_supers_table_stubs[0] = start; for (int slot = 1; slot < Klass::SECONDARY_SUPERS_TABLE_SIZE; slot++) { diff --git a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp index edeb0baea0e95..fcdf55324e8ad 100644 --- a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp @@ -69,6 +69,15 @@ static jlong *double_signmask_pool = double_quadword(&fp_signmask_pool[2*2], static jlong *float_signflip_pool = double_quadword(&fp_signmask_pool[3*2], (jlong)UCONST64(0x8000000080000000), (jlong)UCONST64(0x8000000080000000)); static jlong *double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], (jlong)UCONST64(0x8000000000000000), (jlong)UCONST64(0x8000000000000000)); +// publish external addresses defined in this file +void LIR_Assembler::init_AOTAddressTable(GrowableArray
& external_addresses) { +#define ADD(addr) external_addresses.append((address)addr); + ADD(float_signmask_pool); + ADD(double_signmask_pool); + ADD(float_signflip_pool); + ADD(double_signflip_pool); +#undef ADD +} NEEDS_CLEANUP // remove this definitions ? const Register SYNC_header = rax; // synchronization header diff --git a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.hpp b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.hpp index 8524dc90276f0..71a137a1e901a 100644 --- a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.hpp +++ b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.hpp @@ -58,4 +58,5 @@ void store_parameter(jobject c, int offset_from_esp_in_words); void store_parameter(Metadata* c, int offset_from_esp_in_words); + void static init_AOTAddressTable(GrowableArray
& external_addresses); #endif // CPU_X86_C1_LIRASSEMBLER_X86_HPP diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.cpp b/src/hotspot/cpu/x86/macroAssembler_x86.cpp index 44f1a35d443ab..fb033732ef6b6 100644 --- a/src/hotspot/cpu/x86/macroAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/macroAssembler_x86.cpp @@ -5574,7 +5574,7 @@ void MacroAssembler::cmp_narrow_klass(Address dst, Klass* k) { void MacroAssembler::reinit_heapbase() { if (UseCompressedOops) { - if (Universe::heap() != nullptr) { + if (Universe::heap() != nullptr && !AOTCodeCache::is_on_for_dump()) { if (CompressedOops::base() == nullptr) { MacroAssembler::xorptr(r12_heapbase, r12_heapbase); } else { diff --git a/src/hotspot/cpu/x86/macroAssembler_x86_sha.cpp b/src/hotspot/cpu/x86/macroAssembler_x86_sha.cpp index 9f0232075cd6a..822400062fdd1 100644 --- a/src/hotspot/cpu/x86/macroAssembler_x86_sha.cpp +++ b/src/hotspot/cpu/x86/macroAssembler_x86_sha.cpp @@ -792,6 +792,9 @@ enum { movl(h, Address(CTX, 4*7)); pshuffle_byte_flip_mask_addr = pshuffle_byte_flip_mask; + assert(pshuffle_byte_flip_mask_addr + 32 == StubRoutines::x86::pshuffle_byte_flip_mask_00ba_addr(), "sanity"); + assert(pshuffle_byte_flip_mask_addr + 64 == StubRoutines::x86::pshuffle_byte_flip_mask_dc00_addr(), "sanity"); + vmovdqu(BYTE_FLIP_MASK, ExternalAddress(pshuffle_byte_flip_mask_addr + 0)); // [PSHUFFLE_BYTE_FLIP_MASK wrt rip] vmovdqu(SHUF_00BA, ExternalAddress(pshuffle_byte_flip_mask_addr + 32)); // [_SHUF_00BA wrt rip] vmovdqu(SHUF_DC00, ExternalAddress(pshuffle_byte_flip_mask_addr + 64)); // [_SHUF_DC00 wrt rip] @@ -955,6 +958,9 @@ bind(only_one_block); pshuffle_byte_flip_mask_addr = pshuffle_byte_flip_mask; + assert(pshuffle_byte_flip_mask_addr + 32 == StubRoutines::x86::pshuffle_byte_flip_mask_00ba_addr(), "sanity"); + assert(pshuffle_byte_flip_mask_addr + 64 == StubRoutines::x86::pshuffle_byte_flip_mask_dc00_addr(), "sanity"); + vmovdqu(BYTE_FLIP_MASK, ExternalAddress(pshuffle_byte_flip_mask_addr + 0)); // [PSHUFFLE_BYTE_FLIP_MASK wrt rip] vmovdqu(SHUF_00BA, ExternalAddress(pshuffle_byte_flip_mask_addr + 32)); // [_SHUF_00BA wrt rip] vmovdqu(SHUF_DC00, ExternalAddress(pshuffle_byte_flip_mask_addr + 64)); // [_SHUF_DC00 wrt rip] @@ -1347,7 +1353,9 @@ void MacroAssembler::sha512_AVX2(XMMRegister msg, XMMRegister state0, XMMRegiste movq(h, Address(CTX, 8 * 7)); pshuffle_byte_flip_mask_addr = pshuffle_byte_flip_mask_sha512; - vmovdqu(BYTE_FLIP_MASK, ExternalAddress(pshuffle_byte_flip_mask_addr + 0)); // PSHUFFLE_BYTE_FLIP_MASK wrt rip + assert(pshuffle_byte_flip_mask_addr + 32 == StubRoutines::x86::pshuffle_byte_flip_mask_ymm_lo_addr_sha512(), "sanity"); + + vmovdqu(BYTE_FLIP_MASK, ExternalAddress(pshuffle_byte_flip_mask_addr + 0)); // PSHUFFLE_BYTE_FLIP_MASK wrt rip vmovdqu(YMM_MASK_LO, ExternalAddress(pshuffle_byte_flip_mask_addr + 32)); movq(g, Address(CTX, 8 * 6)); diff --git a/src/hotspot/cpu/x86/stubDeclarations_x86.hpp b/src/hotspot/cpu/x86/stubDeclarations_x86.hpp index 3e0e1d5bf074a..61deb113d6fa2 100644 --- a/src/hotspot/cpu/x86/stubDeclarations_x86.hpp +++ b/src/hotspot/cpu/x86/stubDeclarations_x86.hpp @@ -161,6 +161,12 @@ do_arch_entry(x86, compiler, pshuffle_byte_flip_mask, \ pshuffle_byte_flip_mask_addr, \ pshuffle_byte_flip_mask_addr) \ + do_arch_entry(x86, compiler, pshuffle_byte_flip_mask, \ + pshuffle_byte_flip_mask_00ba_addr, \ + pshuffle_byte_flip_mask_00ba_addr) \ + do_arch_entry(x86, compiler, pshuffle_byte_flip_mask, \ + pshuffle_byte_flip_mask_dc00_addr, \ + pshuffle_byte_flip_mask_dc00_addr) \ /* x86_64 exposes these 3 stubs via a generic entry array */ \ /* other arches use arch-specific entries */ \ /* this really needs rationalising */ \ @@ -168,6 +174,9 @@ do_stub(compiler, string_indexof_linear_uu) \ do_stub(compiler, string_indexof_linear_ul) \ do_stub(compiler, pshuffle_byte_flip_mask_sha512) \ + do_arch_entry(x86, compiler, pshuffle_byte_flip_mask_sha512, \ + pshuffle_byte_flip_mask_ymm_lo_addr_sha512, \ + pshuffle_byte_flip_mask_ymm_lo_addr_sha512) \ do_arch_entry(x86, compiler, pshuffle_byte_flip_mask_sha512, \ pshuffle_byte_flip_mask_addr_sha512, \ pshuffle_byte_flip_mask_addr_sha512) \ diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp index efb0411aa39df..e04729f070722 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp @@ -39,6 +39,9 @@ #include "runtime/sharedRuntime.hpp" #include "runtime/stubRoutines.hpp" #include "stubGenerator_x86_64.hpp" +#ifdef COMPILER1 +#include "c1/c1_LIRAssembler.hpp" +#endif #ifdef COMPILER2 #include "opto/runtime.hpp" #include "opto/c2_globals.hpp" @@ -188,6 +191,19 @@ address StubGenerator::generate_call_stub(address& return_address) { (int)frame::entry_frame_call_wrapper_offset == (int)call_wrapper_off, "adjust this code"); StubId stub_id = StubId::stubgen_call_stub_id; + GrowableArray
entries; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 2, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + // this stub has an extra return entry which we need to retrieve + load_archive_data(stub_id, start, end, &entries); + assert(entries.length() == 1, "expected 1 extra entry"); + return_address = entries.at(0); + return start; + } + StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -298,6 +314,7 @@ address StubGenerator::generate_call_stub(address& return_address) { BLOCK_COMMENT("call_stub_return_address:"); return_address = __ pc(); + entries.append(return_address); // store result depending on type (everything that is not // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT) @@ -394,6 +411,9 @@ address StubGenerator::generate_call_stub(address& return_address) { __ movdbl(Address(c_rarg0, 0), xmm0); __ jmp(exit); + // record the stub entry and end plus the auxiliary entry + store_archive_data(stub_id, start, __ pc(), &entries); + return start; } @@ -411,6 +431,15 @@ address StubGenerator::generate_call_stub(address& return_address) { address StubGenerator::generate_catch_exception() { StubId stub_id = StubId::stubgen_catch_exception_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -442,7 +471,9 @@ address StubGenerator::generate_catch_exception() { __ verify_oop(rax); __ movptr(Address(r15_thread, Thread::pending_exception_offset()), rax); - __ lea(rscratch1, ExternalAddress((address)__FILE__)); + // special case -- add file name string to AOT address table + address file = (address)AOTCodeCache::add_C_string(__FILE__); + __ lea(rscratch1, ExternalAddress(file)); __ movptr(Address(r15_thread, Thread::exception_file_offset()), rscratch1); __ movl(Address(r15_thread, Thread::exception_line_offset()), (int) __LINE__); @@ -451,6 +482,9 @@ address StubGenerator::generate_catch_exception() { "_call_stub_return_address must have been generated before"); __ jump(RuntimeAddress(StubRoutines::_call_stub_return_address)); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -467,6 +501,14 @@ address StubGenerator::generate_catch_exception() { address StubGenerator::generate_forward_exception() { StubId stub_id = StubId::stubgen_forward_exception_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -521,6 +563,9 @@ address StubGenerator::generate_forward_exception() { __ verify_oop(rax); __ jmp(rbx); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -531,12 +576,23 @@ address StubGenerator::generate_forward_exception() { // Result: address StubGenerator::generate_orderaccess_fence() { StubId stub_id = StubId::stubgen_fence_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } StubCodeMark mark(this, stub_id); address start = __ pc(); __ membar(Assembler::StoreLoad); __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -550,6 +606,14 @@ address StubGenerator::generate_orderaccess_fence() { address StubGenerator::generate_verify_mxcsr() { StubId stub_id = StubId::stubgen_verify_mxcsr_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -574,11 +638,22 @@ address StubGenerator::generate_verify_mxcsr() { __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::generate_f2i_fixup() { StubId stub_id = StubId::stubgen_f2i_fixup_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } StubCodeMark mark(this, stub_id); Address inout(rsp, 5 * wordSize); // return address + 4 saves @@ -613,11 +688,22 @@ address StubGenerator::generate_f2i_fixup() { __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::generate_f2l_fixup() { StubId stub_id = StubId::stubgen_f2l_fixup_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } StubCodeMark mark(this, stub_id); Address inout(rsp, 5 * wordSize); // return address + 4 saves address start = __ pc(); @@ -651,11 +737,22 @@ address StubGenerator::generate_f2l_fixup() { __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::generate_d2i_fixup() { StubId stub_id = StubId::stubgen_d2i_fixup_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } StubCodeMark mark(this, stub_id); Address inout(rsp, 6 * wordSize); // return address + 5 saves @@ -699,11 +796,22 @@ address StubGenerator::generate_d2i_fixup() { __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::generate_d2l_fixup() { StubId stub_id = StubId::stubgen_d2l_fixup_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } StubCodeMark mark(this, stub_id); Address inout(rsp, 6 * wordSize); // return address + 5 saves @@ -747,12 +855,23 @@ address StubGenerator::generate_d2l_fixup() { __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::generate_count_leading_zeros_lut() { - __ align64(); StubId stub_id = StubId::stubgen_vector_count_leading_zeros_lut_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align64(); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -765,12 +884,23 @@ address StubGenerator::generate_count_leading_zeros_lut() { __ emit_data64(0x0101010102020304, relocInfo::none); __ emit_data64(0x0000000000000000, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::generate_popcount_avx_lut() { - __ align64(); StubId stub_id = StubId::stubgen_vector_popcount_lut_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align64(); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -783,12 +913,23 @@ address StubGenerator::generate_popcount_avx_lut() { __ emit_data64(0x0302020102010100, relocInfo::none); __ emit_data64(0x0403030203020201, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::generate_iota_indices() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_vector_iota_indices_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); // B @@ -845,12 +986,24 @@ address StubGenerator::generate_iota_indices() { __ emit_data64(0x4014000000000000, relocInfo::none); // 5.0d __ emit_data64(0x4018000000000000, relocInfo::none); // 6.0d __ emit_data64(0x401c000000000000, relocInfo::none); // 7.0d + + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::generate_vector_reverse_bit_lut() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_vector_reverse_bit_lut_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -863,12 +1016,23 @@ address StubGenerator::generate_vector_reverse_bit_lut() { __ emit_data64(0x0E060A020C040800, relocInfo::none); __ emit_data64(0x0F070B030D050901, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::generate_vector_reverse_byte_perm_mask_long() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_vector_reverse_byte_perm_mask_long_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -881,12 +1045,23 @@ address StubGenerator::generate_vector_reverse_byte_perm_mask_long() { __ emit_data64(0x0001020304050607, relocInfo::none); __ emit_data64(0x08090A0B0C0D0E0F, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::generate_vector_reverse_byte_perm_mask_int() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_vector_reverse_byte_perm_mask_int_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -899,12 +1074,23 @@ address StubGenerator::generate_vector_reverse_byte_perm_mask_int() { __ emit_data64(0x0405060700010203, relocInfo::none); __ emit_data64(0x0C0D0E0F08090A0B, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::generate_vector_reverse_byte_perm_mask_short() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_vector_reverse_byte_perm_mask_short_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -917,12 +1103,23 @@ address StubGenerator::generate_vector_reverse_byte_perm_mask_short() { __ emit_data64(0x0607040502030001, relocInfo::none); __ emit_data64(0x0E0F0C0D0A0B0809, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::generate_vector_byte_shuffle_mask() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_vector_byte_shuffle_mask_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -931,10 +1128,21 @@ address StubGenerator::generate_vector_byte_shuffle_mask() { __ emit_data64(0xF0F0F0F0F0F0F0F0, relocInfo::none); __ emit_data64(0xF0F0F0F0F0F0F0F0, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::generate_fp_mask(StubId stub_id, int64_t mask) { + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -942,6 +1150,9 @@ address StubGenerator::generate_fp_mask(StubId stub_id, int64_t mask) { __ emit_data64( mask, relocInfo::none ); __ emit_data64( mask, relocInfo::none ); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -957,6 +1168,14 @@ address StubGenerator::generate_compress_perm_table(StubId stub_id) { default: ShouldNotReachHere(); } + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -997,6 +1216,9 @@ address StubGenerator::generate_compress_perm_table(StubId stub_id) { } } } + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -1012,6 +1234,14 @@ address StubGenerator::generate_expand_perm_table(StubId stub_id) { default: ShouldNotReachHere(); } + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -1050,10 +1280,21 @@ address StubGenerator::generate_expand_perm_table(StubId stub_id) { } } } + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::generate_vector_mask(StubId stub_id, int64_t mask) { + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -1067,12 +1308,23 @@ address StubGenerator::generate_vector_mask(StubId stub_id, int64_t mask) { __ emit_data64(mask, relocInfo::none); __ emit_data64(mask, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::generate_vector_byte_perm_mask() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_vector_byte_perm_mask_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -1085,10 +1337,21 @@ address StubGenerator::generate_vector_byte_perm_mask() { __ emit_data64(0x0000000000000004, relocInfo::none); __ emit_data64(0x0000000000000006, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::generate_vector_fp_mask(StubId stub_id, int64_t mask) { + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -1102,6 +1365,9 @@ address StubGenerator::generate_vector_fp_mask(StubId stub_id, int64_t mask) { __ emit_data64(mask, relocInfo::none); __ emit_data64(mask, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -1110,6 +1376,14 @@ address StubGenerator::generate_vector_custom_i32(StubId stub_id, Assembler::Avx int32_t val4, int32_t val5, int32_t val6, int32_t val7, int32_t val8, int32_t val9, int32_t val10, int32_t val11, int32_t val12, int32_t val13, int32_t val14, int32_t val15) { + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -1135,6 +1409,9 @@ address StubGenerator::generate_vector_custom_i32(StubId stub_id, Assembler::Avx __ emit_data(val15, relocInfo::none, 0); } } + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -1156,6 +1433,14 @@ address StubGenerator::generate_vector_custom_i32(StubId stub_id, Assembler::Avx // * = popped on exit address StubGenerator::generate_verify_oop() { StubId stub_id = StubId::stubgen_verify_oop_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -1235,6 +1520,9 @@ address StubGenerator::generate_verify_oop() { __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64))); __ hlt(); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -1350,10 +1638,16 @@ void StubGenerator::restore_argument_regs(BasicType type) { address StubGenerator::generate_data_cache_writeback() { const Register src = c_rarg0; // source address - - __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_data_cache_writeback_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -1363,15 +1657,24 @@ address StubGenerator::generate_data_cache_writeback() { __ leave(); __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::generate_data_cache_writeback_sync() { const Register is_pre = c_rarg0; // pre or post sync - - __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_data_cache_writeback_sync_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); // pre wbsync is a no-op @@ -1388,6 +1691,9 @@ address StubGenerator::generate_data_cache_writeback_sync() { __ leave(); __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -1405,6 +1711,14 @@ address StubGenerator::generate_md5_implCompress(StubId stub_id) { default: ShouldNotReachHere(); } + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -1437,30 +1751,55 @@ address StubGenerator::generate_md5_implCompress(StubId stub_id) { __ leave(); __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::generate_upper_word_mask() { - __ align64(); StubId stub_id = StubId::stubgen_upper_word_mask_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align64(); StubCodeMark mark(this, stub_id); address start = __ pc(); __ emit_data64(0x0000000000000000, relocInfo::none); __ emit_data64(0xFFFFFFFF00000000, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::generate_shuffle_byte_flip_mask() { - __ align64(); StubId stub_id = StubId::stubgen_shuffle_byte_flip_mask_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align64(); StubCodeMark mark(this, stub_id); address start = __ pc(); __ emit_data64(0x08090a0b0c0d0e0f, relocInfo::none); __ emit_data64(0x0001020304050607, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -1478,6 +1817,14 @@ address StubGenerator::generate_sha1_implCompress(StubId stub_id) { default: ShouldNotReachHere(); } + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -1509,15 +1856,34 @@ address StubGenerator::generate_sha1_implCompress(StubId stub_id) { __ leave(); __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } -address StubGenerator::generate_pshuffle_byte_flip_mask() { - __ align64(); +address StubGenerator::generate_pshuffle_byte_flip_mask(address& entry_00ba, address& entry_dc00) { StubId stub_id = StubId::stubgen_pshuffle_byte_flip_mask_id; + GrowableArray
entries; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 3, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, &entries); + assert(entries.length() == entry_count - 1, + "unexpected extra entry count %d", entries.length()); + entry_00ba = entries.at(0); + entry_dc00 = entries.at(1); + assert(VM_Version::supports_avx2() == (entry_00ba != nullptr && entry_dc00 != nullptr), + "entries cannot be null when avx2 is enabled"); + return start; + } + __ align64(); StubCodeMark mark(this, stub_id); address start = __ pc(); - + address entry2 = nullptr; + address entry3 = nullptr; __ emit_data64(0x0405060700010203, relocInfo::none); __ emit_data64(0x0c0d0e0f08090a0b, relocInfo::none); @@ -1525,37 +1891,68 @@ address StubGenerator::generate_pshuffle_byte_flip_mask() { __ emit_data64(0x0405060700010203, relocInfo::none); // second copy __ emit_data64(0x0c0d0e0f08090a0b, relocInfo::none); // _SHUF_00BA + entry2 = __ pc(); __ emit_data64(0x0b0a090803020100, relocInfo::none); __ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none); __ emit_data64(0x0b0a090803020100, relocInfo::none); __ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none); // _SHUF_DC00 + entry3 = __ pc(); __ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none); __ emit_data64(0x0b0a090803020100, relocInfo::none); __ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none); __ emit_data64(0x0b0a090803020100, relocInfo::none); } + // have to track the 2nd and 3rd entries even if they are null + entry_00ba = entry2; + entries.push(entry_00ba); + entry_dc00 = entry3; + entries.push(entry_dc00); + + // record the stub entry and end plus all the auxiliary entries + store_archive_data(stub_id, start, __ pc(), &entries); return start; } //Mask for byte-swapping a couple of qwords in an XMM register using (v)pshufb. -address StubGenerator::generate_pshuffle_byte_flip_mask_sha512() { - __ align32(); +address StubGenerator::generate_pshuffle_byte_flip_mask_sha512(address& entry_ymm_lo) { StubId stub_id = StubId::stubgen_pshuffle_byte_flip_mask_sha512_id; + GrowableArray
entries; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 2, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, &entries); + assert(entries.length() == entry_count - 1, + "unexpected extra entry count %d", entries.length()); + entry_ymm_lo = entries.at(0); + assert(VM_Version::supports_avx2() == (entry_ymm_lo != nullptr), + "entry cannot be null when avx2 is enabled"); + return start; + } + __ align32(); StubCodeMark mark(this, stub_id); address start = __ pc(); - + address entry2 = nullptr; if (VM_Version::supports_avx2()) { __ emit_data64(0x0001020304050607, relocInfo::none); // PSHUFFLE_BYTE_FLIP_MASK __ emit_data64(0x08090a0b0c0d0e0f, relocInfo::none); __ emit_data64(0x1011121314151617, relocInfo::none); __ emit_data64(0x18191a1b1c1d1e1f, relocInfo::none); + // capture 2nd entry + entry2 = __ pc(); __ emit_data64(0x0000000000000000, relocInfo::none); //MASK_YMM_LO __ emit_data64(0x0000000000000000, relocInfo::none); __ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none); __ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none); } + // have to track the 2nd entry even if it is null + entry_ymm_lo = entry2; + entries.push(entry2); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc(), &entries); return start; } @@ -1575,6 +1972,14 @@ address StubGenerator::generate_sha256_implCompress(StubId stub_id) { ShouldNotReachHere(); } assert(VM_Version::supports_sha() || VM_Version::supports_avx2(), ""); + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -1612,6 +2017,9 @@ address StubGenerator::generate_sha256_implCompress(StubId stub_id) { __ leave(); __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -1629,6 +2037,14 @@ address StubGenerator::generate_sha512_implCompress(StubId stub_id) { } assert(VM_Version::supports_avx2(), ""); assert(VM_Version::supports_bmi2() || VM_Version::supports_sha512(), ""); + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -1660,12 +2076,23 @@ address StubGenerator::generate_sha512_implCompress(StubId stub_id) { __ leave(); __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::base64_shuffle_addr() { - __ align64(); StubId stub_id = StubId::stubgen_shuffle_base64_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align64(); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -1680,12 +2107,23 @@ address StubGenerator::base64_shuffle_addr() { __ emit_data64(0x2829272825262425, relocInfo::none); __ emit_data64(0x2e2f2d2e2b2c2a2b, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::base64_avx2_shuffle_addr() { - __ align32(); StubId stub_id = StubId::stubgen_avx2_shuffle_base64_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align32(); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -1694,12 +2132,23 @@ address StubGenerator::base64_avx2_shuffle_addr() { __ emit_data64(0x0405030401020001, relocInfo::none); __ emit_data64(0x0a0b090a07080607, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::base64_avx2_input_mask_addr() { - __ align32(); StubId stub_id = StubId::stubgen_avx2_input_mask_base64_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align32(); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -1708,12 +2157,23 @@ address StubGenerator::base64_avx2_input_mask_addr() { __ emit_data64(0x8000000080000000, relocInfo::none); __ emit_data64(0x8000000080000000, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::base64_avx2_lut_addr() { - __ align32(); StubId stub_id = StubId::stubgen_avx2_lut_base64_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align32(); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -1728,12 +2188,23 @@ address StubGenerator::base64_avx2_lut_addr() { __ emit_data64(0xfcfcfcfcfcfc4741, relocInfo::none); __ emit_data64(0x000020effcfcfcfc, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::base64_encoding_table_addr() { - __ align64(); StubId stub_id = StubId::stubgen_encoding_table_base64_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align64(); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -1757,6 +2228,9 @@ address StubGenerator::base64_encoding_table_addr() { __ emit_data64(0x333231307a797877, relocInfo::none); __ emit_data64(0x5f2d393837363534, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -1766,8 +2240,16 @@ address StubGenerator::base64_encoding_table_addr() { // boolean isURL) { address StubGenerator::generate_base64_encodeBlock() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_base64_encodeBlock_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -2144,13 +2626,24 @@ address StubGenerator::generate_base64_encodeBlock() __ leave(); __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } // base64 AVX512vbmi tables address StubGenerator::base64_vbmi_lookup_lo_addr() { - __ align64(); StubId stub_id = StubId::stubgen_lookup_lo_base64_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align64(); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -2165,12 +2658,23 @@ address StubGenerator::base64_vbmi_lookup_lo_addr() { __ emit_data64(0x3b3a393837363534, relocInfo::none); __ emit_data64(0x8080808080803d3c, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::base64_vbmi_lookup_hi_addr() { - __ align64(); StubId stub_id = StubId::stubgen_lookup_hi_base64_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align64(); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -2185,11 +2689,22 @@ address StubGenerator::base64_vbmi_lookup_hi_addr() { __ emit_data64(0x302f2e2d2c2b2a29, relocInfo::none); __ emit_data64(0x8080808080333231, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::base64_vbmi_lookup_lo_url_addr() { - __ align64(); StubId stub_id = StubId::stubgen_lookup_lo_base64url_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align64(); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -2204,12 +2719,23 @@ address StubGenerator::base64_vbmi_lookup_lo_url_addr() { __ emit_data64(0x3b3a393837363534, relocInfo::none); __ emit_data64(0x8080808080803d3c, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::base64_vbmi_lookup_hi_url_addr() { - __ align64(); StubId stub_id = StubId::stubgen_lookup_hi_base64url_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align64(); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -2224,12 +2750,23 @@ address StubGenerator::base64_vbmi_lookup_hi_url_addr() { __ emit_data64(0x302f2e2d2c2b2a29, relocInfo::none); __ emit_data64(0x8080808080333231, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::base64_vbmi_pack_vec_addr() { - __ align64(); StubId stub_id = StubId::stubgen_pack_vec_base64_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align64(); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -2244,12 +2781,23 @@ address StubGenerator::base64_vbmi_pack_vec_addr() { __ emit_data64(0x0000000000000000, relocInfo::none); __ emit_data64(0x0000000000000000, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::base64_vbmi_join_0_1_addr() { - __ align64(); StubId stub_id = StubId::stubgen_join_0_1_base64_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align64(); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -2264,12 +2812,23 @@ address StubGenerator::base64_vbmi_join_0_1_addr() { __ emit_data64(0x494a444546404142, relocInfo::none); __ emit_data64(0x565051524c4d4e48, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::base64_vbmi_join_1_2_addr() { - __ align64(); StubId stub_id = StubId::stubgen_join_1_2_base64_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align64(); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -2284,12 +2843,23 @@ address StubGenerator::base64_vbmi_join_1_2_addr() { __ emit_data64(0x5c5d5e58595a5455, relocInfo::none); __ emit_data64(0x696a646566606162, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::base64_vbmi_join_2_3_addr() { - __ align64(); StubId stub_id = StubId::stubgen_join_2_3_base64_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align64(); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -2304,12 +2874,23 @@ address StubGenerator::base64_vbmi_join_2_3_addr() { __ emit_data64(0x767071726c6d6e68, relocInfo::none); __ emit_data64(0x7c7d7e78797a7475, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::base64_AVX2_decode_tables_addr() { - __ align64(); StubId stub_id = StubId::stubgen_avx2_decode_tables_base64_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align64(); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -2339,12 +2920,23 @@ address StubGenerator::base64_AVX2_decode_tables_addr() { // merge multiplier __ emit_data(0x00011000, relocInfo::none, 0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::base64_AVX2_decode_LUT_tables_addr() { - __ align64(); StubId stub_id = StubId::stubgen_avx2_decode_lut_tables_base64_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align64(); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -2380,11 +2972,22 @@ address StubGenerator::base64_AVX2_decode_LUT_tables_addr() { __ emit_data64(0x0804080402011010, relocInfo::none); __ emit_data64(0x1010101010101010, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::base64_decoding_table_addr() { StubId stub_id = StubId::stubgen_decoding_table_base64_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -2455,6 +3058,9 @@ address StubGenerator::base64_decoding_table_addr() { __ emit_data64(0xffffffffffffffff, relocInfo::none); __ emit_data64(0xffffffffffffffff, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -2466,8 +3072,16 @@ address StubGenerator::base64_decoding_table_addr() { // Intrinsic function prototype in Base64.java: // private void decodeBlock(byte[] src, int sp, int sl, byte[] dst, int dp, boolean isURL, isMIME) { address StubGenerator::generate_base64_decodeBlock() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_base64_decodeBlock_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -2982,6 +3596,9 @@ address StubGenerator::generate_base64_decodeBlock() { __ leave(); __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -3000,8 +3617,16 @@ address StubGenerator::generate_base64_decodeBlock() { address StubGenerator::generate_updateBytesCRC32() { assert(UseCRC32Intrinsics, "need AVX and CLMUL instructions"); - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_updateBytesCRC32_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -3039,6 +3664,9 @@ address StubGenerator::generate_updateBytesCRC32() { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -3057,8 +3685,16 @@ address StubGenerator::generate_updateBytesCRC32() { */ address StubGenerator::generate_updateBytesCRC32C(bool is_pclmulqdq_supported) { assert(UseCRC32CIntrinsics, "need SSE4_2"); - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_updateBytesCRC32C_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -3120,6 +3756,9 @@ address StubGenerator::generate_updateBytesCRC32C(bool is_pclmulqdq_supported) { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -3138,8 +3777,16 @@ address StubGenerator::generate_updateBytesCRC32C(bool is_pclmulqdq_supported) { * rsp+40 - z address */ address StubGenerator::generate_multiplyToLen() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_multiplyToLen_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -3179,6 +3826,9 @@ address StubGenerator::generate_multiplyToLen() { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -3195,8 +3845,16 @@ address StubGenerator::generate_multiplyToLen() { * rax - int >= mismatched index, < 0 bitwise complement of tail */ address StubGenerator::generate_vectorizedMismatch() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_vectorizedMismatch_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -3232,6 +3890,9 @@ address StubGenerator::generate_vectorizedMismatch() { __ leave(); __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -3247,8 +3908,16 @@ address StubGenerator::generate_vectorizedMismatch() { */ address StubGenerator::generate_squareToLen() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_squareToLen_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -3279,12 +3948,23 @@ address StubGenerator::generate_squareToLen() { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::generate_method_entry_barrier() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_method_entry_barrier_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -3356,6 +4036,9 @@ address StubGenerator::generate_method_entry_barrier() { __ movptr(rsp, Address(rsp, 0)); // new rsp was written in the barrier __ jmp(Address(rsp, -1 * wordSize)); // jmp target should be callers verified_entry_point + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -3373,8 +4056,16 @@ address StubGenerator::generate_method_entry_barrier() { * rsp+40 - k */ address StubGenerator::generate_mulAdd() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_mulAdd_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -3411,12 +4102,23 @@ address StubGenerator::generate_mulAdd() { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::generate_bigIntegerRightShift() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_bigIntegerRightShiftWorker_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -3534,6 +4236,9 @@ address StubGenerator::generate_bigIntegerRightShift() { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -3551,8 +4256,16 @@ address StubGenerator::generate_bigIntegerRightShift() { * rsp40 - numIter */ address StubGenerator::generate_bigIntegerLeftShift() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_bigIntegerLeftShiftWorker_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -3659,6 +4372,9 @@ address StubGenerator::generate_bigIntegerLeftShift() { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -3708,6 +4424,14 @@ void StubGenerator::generate_libm_stubs() { */ address StubGenerator::generate_float16ToFloat() { StubId stub_id = StubId::stubgen_hf2f_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -3720,6 +4444,9 @@ address StubGenerator::generate_float16ToFloat() { __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -3734,6 +4461,14 @@ address StubGenerator::generate_float16ToFloat() { */ address StubGenerator::generate_floatToFloat16() { StubId stub_id = StubId::stubgen_f2hf_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -3746,6 +4481,9 @@ address StubGenerator::generate_floatToFloat16() { __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -3775,6 +4513,14 @@ address StubGenerator::generate_cont_thaw(StubId stub_id) { default: ShouldNotReachHere(); } + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -3889,6 +4635,9 @@ address StubGenerator::generate_cont_thaw(StubId stub_id) { __ ret(0); } + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -3909,6 +4658,14 @@ address StubGenerator::generate_cont_returnBarrier_exception() { address StubGenerator::generate_cont_preempt_stub() { if (!Continuations::enabled()) return nullptr; StubId stub_id = StubId::stubgen_cont_preempt_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -3934,12 +4691,23 @@ address StubGenerator::generate_cont_preempt_stub() { __ movptr(rscratch1, ExternalAddress(ContinuationEntry::thaw_call_pc_address())); __ jmp(rscratch1); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } // exception handler for upcall stubs address StubGenerator::generate_upcall_stub_exception_handler() { StubId stub_id = StubId::stubgen_upcall_stub_exception_handler_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -3953,6 +4721,9 @@ address StubGenerator::generate_upcall_stub_exception_handler() { __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, UpcallLinker::handle_uncaught_exception))); __ should_not_reach_here(); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -3961,6 +4732,14 @@ address StubGenerator::generate_upcall_stub_exception_handler() { // rbx = result address StubGenerator::generate_upcall_stub_load_target() { StubId stub_id = StubId::stubgen_upcall_stub_load_target_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -3981,6 +4760,21 @@ address StubGenerator::generate_upcall_stub_load_target() { void StubGenerator::generate_lookup_secondary_supers_table_stub() { StubId stub_id = StubId::stubgen_lookup_secondary_supers_table_id; + GrowableArray
entries; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == Klass::SECONDARY_SUPERS_TABLE_SIZE, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, &entries); + assert(entries.length() == Klass::SECONDARY_SUPERS_TABLE_SIZE - 1, + "unexpected extra entry count %d", entries.length()); + StubRoutines::_lookup_secondary_supers_table_stubs[0] = start; + for (int slot = 1; slot < Klass::SECONDARY_SUPERS_TABLE_SIZE; slot++) { + StubRoutines::_lookup_secondary_supers_table_stubs[0] = entries.at(slot - 1); + } + return; + } StubCodeMark mark(this, stub_id); const Register @@ -3988,19 +4782,38 @@ void StubGenerator::generate_lookup_secondary_supers_table_stub() { r_sub_klass = rsi, result = rdi; + address start; for (int slot = 0; slot < Klass::SECONDARY_SUPERS_TABLE_SIZE; slot++) { - StubRoutines::_lookup_secondary_supers_table_stubs[slot] = __ pc(); + address next_entry = __ pc(); + if (slot == 0) { + start = next_entry; + } else { + entries.append(next_entry); + } + StubRoutines::_lookup_secondary_supers_table_stubs[slot] = next_entry; __ lookup_secondary_supers_table_const(r_sub_klass, r_super_klass, rdx, rcx, rbx, r11, // temps result, slot); __ ret(0); } + + // record the stub entry and end plus all the auxiliary entries + store_archive_data(stub_id, start, __ pc(), &entries); } // Slow path implementation for UseSecondarySupersTable. address StubGenerator::generate_lookup_secondary_supers_table_slow_path_stub() { StubId stub_id = StubId::stubgen_lookup_secondary_supers_table_slow_path_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -4025,6 +4838,9 @@ address StubGenerator::generate_lookup_secondary_supers_table_slow_path_stub() { __ movl(result, 0); __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -4232,6 +5048,8 @@ void StubGenerator::generate_compiler_stubs() { } if (UseSHA256Intrinsics) { + address entry2 = nullptr; + address entry3 = nullptr; StubRoutines::x86::_k256_adr = (address)StubRoutines::x86::_k256; char* dst = (char*)StubRoutines::x86::_k256_W; char* src = (char*)StubRoutines::x86::_k256; @@ -4240,14 +5058,18 @@ void StubGenerator::generate_compiler_stubs() { memcpy(dst + 32 * ii + 16, src + 16 * ii, 16); } StubRoutines::x86::_k256_W_adr = (address)StubRoutines::x86::_k256_W; - StubRoutines::x86::_pshuffle_byte_flip_mask_addr = generate_pshuffle_byte_flip_mask(); + StubRoutines::x86::_pshuffle_byte_flip_mask_addr = generate_pshuffle_byte_flip_mask(entry2, entry3); + StubRoutines::x86::_pshuffle_byte_flip_mask_00ba_addr = entry2; + StubRoutines::x86::_pshuffle_byte_flip_mask_dc00_addr = entry3; StubRoutines::_sha256_implCompress = generate_sha256_implCompress(StubId::stubgen_sha256_implCompress_id); StubRoutines::_sha256_implCompressMB = generate_sha256_implCompress(StubId::stubgen_sha256_implCompressMB_id); } if (UseSHA512Intrinsics) { + address entry2 = nullptr; StubRoutines::x86::_k512_W_addr = (address)StubRoutines::x86::_k512_W; - StubRoutines::x86::_pshuffle_byte_flip_mask_addr_sha512 = generate_pshuffle_byte_flip_mask_sha512(); + StubRoutines::x86::_pshuffle_byte_flip_mask_addr_sha512 = generate_pshuffle_byte_flip_mask_sha512(entry2); + StubRoutines::x86::_pshuffle_byte_flip_mask_ymm_lo_addr_sha512 = entry2; StubRoutines::_sha512_implCompress = generate_sha512_implCompress(StubId::stubgen_sha512_implCompress_id); StubRoutines::_sha512_implCompressMB = generate_sha512_implCompress(StubId::stubgen_sha512_implCompressMB_id); } @@ -4325,7 +5147,7 @@ void StubGenerator::generate_compiler_stubs() { #endif // COMPILER2_OR_JVMCI } -StubGenerator::StubGenerator(CodeBuffer* code, BlobId blob_id) : StubCodeGenerator(code, blob_id) { +StubGenerator::StubGenerator(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) : StubCodeGenerator(code, blob_id, stub_data) { switch(blob_id) { case BlobId::stubgen_preuniverse_id: generate_preuniverse_stubs(); @@ -4348,8 +5170,47 @@ StubGenerator::StubGenerator(CodeBuffer* code, BlobId blob_id) : StubCodeGenerat }; } -void StubGenerator_generate(CodeBuffer* code, BlobId blob_id) { - StubGenerator g(code, blob_id); +// publish addresses of static data defined in this file and in other +// stubgen stub generator files +void StubGenerator::init_AOTAddressTable(GrowableArray
& external_addresses) { + init_AOTAddressTable_adler(external_addresses); + init_AOTAddressTable_aes(external_addresses); + init_AOTAddressTable_cbrt(external_addresses); + init_AOTAddressTable_chacha(external_addresses); + // constants publishes for all of address use by cos and almost all of sin + init_AOTAddressTable_constants(external_addresses); + init_AOTAddressTable_dilithium(external_addresses); + init_AOTAddressTable_exp(external_addresses); + init_AOTAddressTable_fmod(external_addresses); + init_AOTAddressTable_ghash(external_addresses); + init_AOTAddressTable_kyber(external_addresses); + init_AOTAddressTable_log(external_addresses); + init_AOTAddressTable_poly1305(external_addresses); + init_AOTAddressTable_poly_mont(external_addresses); + init_AOTAddressTable_pow(external_addresses); + init_AOTAddressTable_sha3(external_addresses); + init_AOTAddressTable_sin(external_addresses); + init_AOTAddressTable_sinh(external_addresses); + init_AOTAddressTable_tan(external_addresses); + init_AOTAddressTable_tanh(external_addresses); +} + +void StubGenerator_AOTAddressTable_init() { + ResourceMark rm; + GrowableArray
external_addresses; + // publish static addresses referred to by main x86 generator and + // auxiliary x86 generators + StubGenerator::init_AOTAddressTable(external_addresses); + // publish external data addresses defined in nested x86 class + StubRoutines::x86::init_AOTAddressTable(external_addresses); +#ifdef COMPILER1 + LIR_Assembler::init_AOTAddressTable(external_addresses); +#endif + AOTCodeCache::publish_external_addresses(external_addresses); +} + +void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) { + StubGenerator g(code, blob_id, stub_data); } #undef __ diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64.hpp b/src/hotspot/cpu/x86/stubGenerator_x86_64.hpp index 36315535d1607..b72a965ba905a 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64.hpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64.hpp @@ -303,11 +303,11 @@ class StubGenerator: public StubCodeGenerator { address generate_sha512_implCompress(StubId stub_id); // Mask for byte-swapping a couple of qwords in an XMM register using (v)pshufb. - address generate_pshuffle_byte_flip_mask_sha512(); + address generate_pshuffle_byte_flip_mask_sha512(address& entry_ymm_lo); address generate_upper_word_mask(); address generate_shuffle_byte_flip_mask(); - address generate_pshuffle_byte_flip_mask(); + address generate_pshuffle_byte_flip_mask(address& entry_00ba, address& entry_dc0); // AES intrinsic stubs @@ -637,8 +637,30 @@ class StubGenerator: public StubCodeGenerator { void generate_compiler_stubs(); void generate_final_stubs(); + static void init_AOTAddressTable_adler(GrowableArray
& external_addresses); + static void init_AOTAddressTable_aes(GrowableArray
& external_addresses); + static void init_AOTAddressTable_cbrt(GrowableArray
& external_addresses); + static void init_AOTAddressTable_(GrowableArray
& external_addresses); + static void init_AOTAddressTable_chacha(GrowableArray
& external_addresses); + static void init_AOTAddressTable_constants(GrowableArray
& external_addresses); + static void init_AOTAddressTable_dilithium(GrowableArray
& external_addresses); + static void init_AOTAddressTable_exp(GrowableArray
& external_addresses); + static void init_AOTAddressTable_fmod(GrowableArray
& external_addresses); + static void init_AOTAddressTable_ghash(GrowableArray
& external_addresses); + static void init_AOTAddressTable_kyber(GrowableArray
& external_addresses); + static void init_AOTAddressTable_log(GrowableArray
& external_addresses); + static void init_AOTAddressTable_poly1305(GrowableArray
& external_addresses); + static void init_AOTAddressTable_poly_mont(GrowableArray
& external_addresses); + static void init_AOTAddressTable_pow(GrowableArray
& external_addresses); + static void init_AOTAddressTable_sha3(GrowableArray
& external_addresses); + static void init_AOTAddressTable_sin(GrowableArray
& external_addresses); + static void init_AOTAddressTable_sinh(GrowableArray
& external_addresses); + static void init_AOTAddressTable_tan(GrowableArray
& external_addresses); + static void init_AOTAddressTable_tanh(GrowableArray
& external_addresses); + public: - StubGenerator(CodeBuffer* code, BlobId blob_id); + StubGenerator(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data); + static void init_AOTAddressTable(GrowableArray
& external_addresses); }; #endif // CPU_X86_STUBGENERATOR_X86_64_HPP diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_adler.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_adler.cpp index 2799997a761da..c5aa877b4fcf7 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_adler.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_adler.cpp @@ -338,3 +338,11 @@ address StubGenerator::generate_updateBytesAdler32() { } #undef __ + +void StubGenerator::init_AOTAddressTable_adler(GrowableArray
& external_addresses) { +#define ADD(addr) external_addresses.append((address)(addr)) + ADD(ADLER32_ASCALE_TABLE); + ADD(ADLER32_SHUF0_TABLE); + ADD(ADLER32_SHUF1_TABLE); +#undef ADD +} diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_aes.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_aes.cpp index f0726ded7e546..bb991e30b76af 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_aes.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_aes.cpp @@ -4093,3 +4093,25 @@ void StubGenerator::aesgcm_avx2(Register in, Register len, Register ct, Register } #undef __ + +void StubGenerator::init_AOTAddressTable_aes(GrowableArray
& external_addresses) { +#define ADD(addr) external_addresses.append((address)(addr)) + ADD(key_shuffle_mask_addr()); + ADD(counter_shuffle_mask_addr()); + ADD(counter_mask_linc0_addr()); + ADD(counter_mask_linc1_addr()); + ADD(counter_mask_linc1f_addr()); + ADD(counter_mask_linc2_addr()); + ADD(counter_mask_linc2f_addr()); + ADD(counter_mask_linc4_addr()); + ADD(counter_mask_linc8_addr()); + ADD(counter_mask_linc16_addr()); + ADD(counter_mask_linc32_addr()); + ADD(counter_mask_ones_addr()); + ADD(ghash_polynomial_reduction_addr()); + ADD(ghash_polynomial_two_one_addr()); + ADD(counter_mask_addbe_4444_addr()); + ADD(counter_mask_addbe_1234_addr()); + ADD(counter_mask_add_1234_addr()); +#undef ADD +} diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_arraycopy.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_arraycopy.cpp index d53fafafdb440..49acc64164082 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_arraycopy.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_arraycopy.cpp @@ -570,6 +570,32 @@ address StubGenerator::generate_disjoint_copy_avx3_masked(StubId stub_id, addres default: ShouldNotReachHere(); } + GrowableArray
entries; + GrowableArray
extras; + bool add_extras = !is_oop && !aligned; + int expected_entry_count = (entry != nullptr ? 2 : 1); + int expected_extra_count = 3 * (add_extras ? 2 : 0); // 0/2 x UMAM {start,end,handler} + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == expected_entry_count, "sanity check"); + GrowableArray
* entries_ptr = (entry_count == 1 ? nullptr : &entries); + GrowableArray
* extras_ptr = (add_extras ? &extras : nullptr); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, entries_ptr, extras_ptr); + assert(entries.length() == expected_entry_count - 1, + "unexpected extra entry count %d", entries.length()); + assert(extras.length() == expected_extra_count, + "unexpected extra addresses count %d", extras.length()); + if (entry != nullptr) { + *entry = entries.at(0); + } + if (add_extras) { + // restore 1 x UMAM {start,end,handler} addresses from extras + register_unsafe_access_handlers(extras, 0, 1, end); + } + return start; + } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); @@ -596,6 +622,7 @@ address StubGenerator::generate_disjoint_copy_avx3_masked(StubId stub_id, addres if (entry != nullptr) { *entry = __ pc(); + entries.append(*entry); // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) BLOCK_COMMENT("Entry:"); } @@ -794,6 +821,18 @@ address StubGenerator::generate_disjoint_copy_avx3_masked(StubId stub_id, addres arraycopy_avx3_large(to, from, temp1, temp2, temp3, temp4, count, xmm1, xmm2, xmm3, xmm4, shift); __ jmp(L_finish); } + // retrieve the registered handler addresses + address end = __ pc(); + if (add_extras) { + retrieve_unsafe_access_handlers(start, end, extras); + } + assert(extras.length() == expected_extra_count, + "unexpected extra addresses count %d", extras.length()); + + // record the stub entry and end plus the no_push entry and any + // extra handler addresses + store_archive_data(stub_id, start, end, entries_ptr, extras_ptr); + return start; } @@ -908,7 +947,32 @@ address StubGenerator::generate_conjoint_copy_avx3_masked(StubId stub_id, addres default: ShouldNotReachHere(); } - + GrowableArray
entries; + GrowableArray
extras; + bool add_extras = !is_oop && !aligned; + int expected_entry_count = (entry != nullptr ? 2 : 1); + int expected_extra_count = 3 * (add_extras ? 1 : 0); // 0/1 x UMAM {start,end,handler} + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == expected_entry_count, "sanity check"); + GrowableArray
* entries_ptr = (entry_count == 1 ? nullptr : &entries); + GrowableArray
* extras_ptr = (add_extras ? &extras : nullptr); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, entries_ptr, extras_ptr); + assert(entries.length() == expected_entry_count - 1, + "unexpected extra entry count %d", entries.length()); + assert(extras.length() == expected_extra_count, + "unexpected extra addresses count %d", extras.length()); + if (entry != nullptr) { + *entry = entries.at(0); + } + if (add_extras) { + // restore 1 x UMAM {start,end,handler} addresses from extras + register_unsafe_access_handlers(extras, 0, 1, end); + } + return start; + } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -933,6 +997,7 @@ address StubGenerator::generate_conjoint_copy_avx3_masked(StubId stub_id, addres if (entry != nullptr) { *entry = __ pc(); + entries.append(*entry); // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) BLOCK_COMMENT("Entry:"); } @@ -959,7 +1024,7 @@ address StubGenerator::generate_conjoint_copy_avx3_masked(StubId stub_id, addres int threshold[] = { 4096, 2048, 1024, 512}; // UnsafeMemoryAccess page error: continue after unsafe access - UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, true); + UnsafeMemoryAccessMark umam(this, add_extras, true); // 'from', 'to' and 'count' are now valid // temp1 holds remaining count. @@ -1073,6 +1138,18 @@ address StubGenerator::generate_conjoint_copy_avx3_masked(StubId stub_id, addres __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // retrieve the registered handler addresses + address end = __ pc(); + if (add_extras) { + retrieve_unsafe_access_handlers(start, end, extras); + } + assert(extras.length() == expected_extra_count, + "unexpected extra addresses count %d", extras.length()); + + // record the stub entry and end plus the no_push entry and any + // extra handler addresses + store_archive_data(stub_id, start, end, entries_ptr, extras_ptr); + return start; } @@ -1387,6 +1464,28 @@ address StubGenerator::generate_disjoint_byte_copy(address* entry) { return generate_disjoint_copy_avx3_masked(stub_id, entry); } #endif + GrowableArray
entries; + GrowableArray
extras; + int expected_entry_count = (entry != nullptr ? 2 : 1); + int expected_extra_count = (2 * 3); // 2 x UMAM {start,end,handler} + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == expected_entry_count, "sanity check"); + GrowableArray
* entries_ptr = (entry_count == 1 ? nullptr : &entries); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, entries_ptr, &extras); + assert(entries.length() == expected_entry_count - 1, + "unexpected extra entry count %d", entries.length()); + assert(extras.length() == expected_extra_count, + "unexpected extra addresses count %d", extras.length()); + if (entry != nullptr) { + *entry = entries.at(0); + } + // restore 2 UMAM {start,end,handler} addresses from extras + register_unsafe_access_handlers(extras, 0, 2, end); + return start; + } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -1409,6 +1508,7 @@ address StubGenerator::generate_disjoint_byte_copy(address* entry) { if (entry != nullptr) { *entry = __ pc(); + entries.append(*entry); // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) BLOCK_COMMENT("Entry:"); } @@ -1478,6 +1578,17 @@ __ BIND(L_exit); copy_bytes_forward(end_from, end_to, qword_count, rax, r10, L_copy_bytes, L_copy_8_bytes, decorators, T_BYTE); __ jmp(L_copy_4_bytes); } + + // retrieve the registered handler addresses + address end = __ pc(); + retrieve_unsafe_access_handlers(start, end, extras); + assert(extras.length() == expected_extra_count, + "unexpected extra addresses count %d", extras.length()); + + // record the stub entry and end plus the no_push entry and any + // extra handler addresses + store_archive_data(stub_id, start, end, entries_ptr, &extras); + return start; } @@ -1505,6 +1616,28 @@ address StubGenerator::generate_conjoint_byte_copy(address nooverlap_target, add return generate_conjoint_copy_avx3_masked(stub_id, entry, nooverlap_target); } #endif + GrowableArray
entries; + GrowableArray
extras; + int expected_entry_count = (entry != nullptr ? 2 : 1); + int expected_extra_count = (2 * 3); // 2 x UMAM {start,end,handler} + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == expected_entry_count, "sanity check"); + GrowableArray
* entries_ptr = (entry_count == 1 ? nullptr : &entries); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, entries_ptr, &extras); + assert(entries.length() == expected_entry_count - 1, + "unexpected extra entry count %d", entries.length()); + assert(extras.length() == expected_extra_count, + "unexpected extra addresses count %d", extras.length()); + if (entry != nullptr) { + *entry = entries.at(0); + } + // restore 2 UMAM {start,end,handler} addresses from extras + register_unsafe_access_handlers(extras, 0, 2, end); + return start; + } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -1522,6 +1655,7 @@ address StubGenerator::generate_conjoint_byte_copy(address nooverlap_target, add if (entry != nullptr) { *entry = __ pc(); + entries.append(*entry); // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) BLOCK_COMMENT("Entry:"); } @@ -1588,6 +1722,16 @@ address StubGenerator::generate_conjoint_byte_copy(address nooverlap_target, add __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // retrieve the registered handler addresses + address end = __ pc(); + retrieve_unsafe_access_handlers(start, end, extras); + assert(extras.length() == expected_extra_count, + "unexpected extra addresses count %d", extras.length()); + + // record the stub entry and end plus the no_push entry and any + // extra handler addresses + store_archive_data(stub_id, start, end, entries_ptr, &extras); + return start; } @@ -1618,7 +1762,28 @@ address StubGenerator::generate_disjoint_short_copy(address *entry) { return generate_disjoint_copy_avx3_masked(stub_id, entry); } #endif - + GrowableArray
entries; + GrowableArray
extras; + int expected_entry_count = (entry != nullptr ? 2 : 1); + int expected_extra_count = (2 * 3); // 2 x UMAM {start,end,handler} + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == expected_entry_count, "sanity check"); + GrowableArray
* entries_ptr = (entry_count == 1 ? nullptr : &entries); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, entries_ptr, &extras); + assert(entries.length() == expected_entry_count - 1, + "unexpected extra entry count %d", entries.length()); + assert(extras.length() == expected_extra_count, + "unexpected extra addresses count %d", extras.length()); + if (entry != nullptr) { + *entry = entries.at(0); + } + // restore 2 UMAM {start,end,handler} addresses from extras + register_unsafe_access_handlers(extras, 0, 2, end); + return start; + } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -1640,6 +1805,7 @@ address StubGenerator::generate_disjoint_short_copy(address *entry) { if (entry != nullptr) { *entry = __ pc(); + entries.append(*entry); // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) BLOCK_COMMENT("Entry:"); } @@ -1703,6 +1869,16 @@ __ BIND(L_exit); __ jmp(L_copy_4_bytes); } + // retrieve the registered handler addresses + address end = __ pc(); + retrieve_unsafe_access_handlers(start, end, extras); + assert(extras.length() == expected_extra_count, + "unexpected extra addresses count %d", extras.length()); + + // record the stub entry and end plus the no_push entry and any + // extra handler addresses + store_archive_data(stub_id, start, end, entries_ptr, &extras); + return start; } @@ -1710,7 +1886,14 @@ __ BIND(L_exit); address StubGenerator::generate_fill(StubId stub_id) { BasicType t; bool aligned; - + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } switch (stub_id) { case StubId::stubgen_jbyte_fill_id: t = T_BYTE; @@ -1763,6 +1946,9 @@ address StubGenerator::generate_fill(StubId stub_id) { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -1790,7 +1976,28 @@ address StubGenerator::generate_conjoint_short_copy(address nooverlap_target, ad return generate_conjoint_copy_avx3_masked(stub_id, entry, nooverlap_target); } #endif - + GrowableArray
entries; + GrowableArray
extras; + int expected_entry_count = (entry != nullptr ? 2 : 1); + int expected_extra_count = (2 * 3); // 2 x UMAM {start,end,handler} + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == expected_entry_count, "sanity check"); + GrowableArray
* entries_ptr = (entry_count == 1 ? nullptr : &entries); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, entries_ptr, &extras); + assert(entries.length() == expected_entry_count - 1, + "unexpected extra entry count %d", entries.length()); + assert(extras.length() == expected_extra_count, + "unexpected extra addresses count %d", extras.length()); + if (entry != nullptr) { + *entry = entries.at(0); + } + // restore 2 UMAM {start,end,handler} addresses from extras + register_unsafe_access_handlers(extras, 0, 2, end); + return start; + } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -1808,6 +2015,7 @@ address StubGenerator::generate_conjoint_short_copy(address nooverlap_target, ad if (entry != nullptr) { *entry = __ pc(); + entries.append(*entry); // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) BLOCK_COMMENT("Entry:"); } @@ -1866,6 +2074,16 @@ address StubGenerator::generate_conjoint_short_copy(address nooverlap_target, ad __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // retrieve the registered handler addresses + address end = __ pc(); + retrieve_unsafe_access_handlers(start, end, extras); + assert(extras.length() == expected_extra_count, + "unexpected extra addresses count %d", extras.length()); + + // record the stub entry and end plus the no_push entry and any + // extra handler addresses + store_archive_data(stub_id, start, end, entries_ptr, &extras); + return start; } @@ -1918,6 +2136,28 @@ address StubGenerator::generate_disjoint_int_oop_copy(StubId stub_id, address* e return generate_disjoint_copy_avx3_masked(stub_id, entry); } #endif + GrowableArray
entries; + GrowableArray
extras; + int expected_entry_count = (entry != nullptr ? 2 : 1); + int expected_extra_count = (2 * 3); // 2 x UMAM {start,end,handler} + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == expected_entry_count, "sanity check"); + GrowableArray
* entries_ptr = (entry_count == 1 ? nullptr : &entries); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, entries_ptr, &extras); + assert(entries.length() == expected_entry_count - 1, + "unexpected extra entry count %d", entries.length()); + assert(extras.length() == expected_extra_count, + "unexpected extra addresses count %d", extras.length()); + if (entry != nullptr) { + *entry = entries.at(0); + } + // restore 2 UMAM {start,end,handler} addresses from extras + register_unsafe_access_handlers(extras, 0, 2, end); + return start; + } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); @@ -1939,6 +2179,7 @@ address StubGenerator::generate_disjoint_int_oop_copy(StubId stub_id, address* e if (entry != nullptr) { *entry = __ pc(); + entries.append(*entry); // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) BLOCK_COMMENT("Entry:"); } @@ -1971,20 +2212,20 @@ address StubGenerator::generate_disjoint_int_oop_copy(StubId stub_id, address* e __ jmp(L_copy_bytes); // Copy trailing qwords - __ BIND(L_copy_8_bytes); + __ BIND(L_copy_8_bytes); __ movq(rax, Address(end_from, qword_count, Address::times_8, 8)); __ movq(Address(end_to, qword_count, Address::times_8, 8), rax); __ increment(qword_count); __ jcc(Assembler::notZero, L_copy_8_bytes); // Check for and copy trailing dword - __ BIND(L_copy_4_bytes); + __ BIND(L_copy_4_bytes); __ testl(dword_count, 1); // Only byte test since the value is 0 or 1 __ jccb(Assembler::zero, L_exit); __ movl(rax, Address(end_from, 8)); __ movl(Address(end_to, 8), rax); } -__ BIND(L_exit); + __ BIND(L_exit); address ucme_exit_pc = __ pc(); bs->arraycopy_epilogue(_masm, decorators, type, from, to, dword_count); restore_arg_regs_using_thread(); @@ -2001,6 +2242,16 @@ __ BIND(L_exit); __ jmp(L_copy_4_bytes); } + // retrieve the registered handler addresses + address end = __ pc(); + retrieve_unsafe_access_handlers(start, end, extras); + assert(extras.length() == expected_extra_count, + "unexpected extra addresses count %d", extras.length()); + + // record the stub entry and end plus the no_push entry and any + // extra handler addresses + store_archive_data(stub_id, start, end, entries_ptr, &extras); + return start; } @@ -2049,6 +2300,28 @@ address StubGenerator::generate_conjoint_int_oop_copy(StubId stub_id, address no return generate_conjoint_copy_avx3_masked(stub_id, entry, nooverlap_target); } #endif + GrowableArray
entries; + GrowableArray
extras; + int expected_entry_count = (entry != nullptr ? 2 : 1); + int expected_extra_count = (2 * 3); // 2 x UMAM {start,end,handler} + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == expected_entry_count, "sanity check"); + GrowableArray
* entries_ptr = (entry_count == 1 ? nullptr : &entries); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, entries_ptr, &extras); + assert(entries.length() == expected_entry_count - 1, + "unexpected extra entry count %d", entries.length()); + assert(extras.length() == expected_extra_count, + "unexpected extra addresses count %d", extras.length()); + if (entry != nullptr) { + *entry = entries.at(0); + } + // restore 2 UMAM {start,end,handler} addresses from extras + register_unsafe_access_handlers(extras, 0, 2, end); + return start; + } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); @@ -2066,6 +2339,7 @@ address StubGenerator::generate_conjoint_int_oop_copy(StubId stub_id, address no if (entry != nullptr) { *entry = __ pc(); + entries.append(*entry); // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) BLOCK_COMMENT("Entry:"); } @@ -2136,6 +2410,16 @@ __ BIND(L_exit); __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // retrieve the registered handler addresses + address end = __ pc(); + retrieve_unsafe_access_handlers(start, end, extras); + assert(extras.length() == expected_extra_count, + "unexpected extra addresses count %d", extras.length()); + + // record the stub entry and end plus the no_push entry and any + // extra handler addresses + store_archive_data(stub_id, start, end, entries_ptr, &extras); + return start; } @@ -2182,6 +2466,28 @@ address StubGenerator::generate_disjoint_long_oop_copy(StubId stub_id, address * return generate_disjoint_copy_avx3_masked(stub_id, entry); } #endif + GrowableArray
entries; + GrowableArray
extras; + int expected_entry_count = (entry != nullptr ? 2 : 1); + int expected_extra_count = (2 * 3); // 2 x UMAM {start,end,handler} + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == expected_entry_count, "sanity check"); + GrowableArray
* entries_ptr = (entry_count == 1 ? nullptr : &entries); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, entries_ptr, &extras); + assert(entries.length() == expected_entry_count - 1, + "unexpected extra entry count %d", entries.length()); + assert(extras.length() == expected_extra_count, + "unexpected extra addresses count %d", extras.length()); + if (entry != nullptr) { + *entry = entries.at(0); + } + // restore 2 UMAM {start,end,handler} addresses from extras + register_unsafe_access_handlers(extras, 0, 2, end); + return start; + } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); @@ -2203,6 +2509,7 @@ address StubGenerator::generate_disjoint_long_oop_copy(StubId stub_id, address * if (entry != nullptr) { *entry = __ pc(); + entries.append(*entry); // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) BLOCK_COMMENT("Entry:"); } @@ -2271,6 +2578,16 @@ address StubGenerator::generate_disjoint_long_oop_copy(StubId stub_id, address * __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // retrieve the registered handler addresses + address end = __ pc(); + retrieve_unsafe_access_handlers(start, end, extras); + assert(extras.length() == expected_extra_count, + "unexpected extra addresses count %d", extras.length()); + + // record the stub entry and end plus the no_push entry and any + // extra handler addresses + store_archive_data(stub_id, start, end, entries_ptr, &extras); + return start; } @@ -2315,6 +2632,28 @@ address StubGenerator::generate_conjoint_long_oop_copy(StubId stub_id, address n return generate_conjoint_copy_avx3_masked(stub_id, entry, nooverlap_target); } #endif + GrowableArray
entries; + GrowableArray
extras; + int expected_entry_count = (entry != nullptr ? 2 : 1); + int expected_extra_count = (2 * 3); // 2 x UMAM {start,end,handler} + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == expected_entry_count, "sanity check"); + GrowableArray
* entries_ptr = (entry_count == 1 ? nullptr : &entries); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, entries_ptr, &extras); + assert(entries.length() == expected_entry_count - 1, + "unexpected extra entry count %d", entries.length()); + assert(extras.length() == expected_extra_count, + "unexpected extra addresses count %d", extras.length()); + if (entry != nullptr) { + *entry = entries.at(0); + } + // restore 2 UMAM {start,end,handler} addresses from extras + register_unsafe_access_handlers(extras, 0, 2, end); + return start; + } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); @@ -2331,6 +2670,7 @@ address StubGenerator::generate_conjoint_long_oop_copy(StubId stub_id, address n if (entry != nullptr) { *entry = __ pc(); + entries.append(*entry); // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) BLOCK_COMMENT("Entry:"); } @@ -2395,6 +2735,17 @@ address StubGenerator::generate_conjoint_long_oop_copy(StubId stub_id, address n __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + + // retrieve the registered handler addresses + address end = __ pc(); + retrieve_unsafe_access_handlers(start, end, extras); + assert(extras.length() == expected_extra_count, + "unexpected extra addresses count %d", extras.length()); + + // record the stub entry and end plus the no_push entry and any + // extra handler addresses + store_archive_data(stub_id, start, end, entries_ptr, &extras); + return start; } @@ -2450,6 +2801,23 @@ address StubGenerator::generate_checkcast_copy(StubId stub_id, address *entry) { ShouldNotReachHere(); } + GrowableArray
entries; + int expected_entry_count = (entry != nullptr ? 2 : 1); + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == expected_entry_count, "sanity check"); + GrowableArray
* entries_ptr = (entry_count == 1 ? nullptr : &entries); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, entries_ptr); + assert(entries.length() == expected_entry_count - 1, + "unexpected extra entry count %d", entries.length()); + if (entry != nullptr) { + *entry = entries.at(0); + } + return start; + } + Label L_load_element, L_store_element, L_do_card_marks, L_done; // Input registers (after setup_arg_regs) @@ -2504,6 +2872,7 @@ address StubGenerator::generate_checkcast_copy(StubId stub_id, address *entry) { // Caller of this entry point must set up the argument registers. if (entry != nullptr) { *entry = __ pc(); + entries.append(*entry); BLOCK_COMMENT("Entry:"); } @@ -2638,6 +3007,9 @@ address StubGenerator::generate_checkcast_copy(StubId stub_id, address *entry) { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // record the stub entry and end plus the no_push entry + store_archive_data(stub_id, start, __ pc(), entries_ptr); + return start; } @@ -2657,6 +3029,19 @@ address StubGenerator::generate_checkcast_copy(StubId stub_id, address *entry) { address StubGenerator::generate_unsafe_copy(address byte_copy_entry, address short_copy_entry, address int_copy_entry, address long_copy_entry) { + StubId stub_id = StubId::stubgen_unsafe_arraycopy_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + // TODO - in debug mode we should save and restore the supplied + // copy entries and assert that the restored entries match the ones + // passed in + load_archive_data(stub_id, start, end); + return start; + } + Label L_long_aligned, L_int_aligned, L_short_aligned; // Input registers (before setup_arg_regs) @@ -2668,7 +3053,6 @@ address StubGenerator::generate_unsafe_copy(address byte_copy_entry, address sho const Register bits = rax; // test copy of low bits __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_unsafe_arraycopy_id; StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -2702,6 +3086,9 @@ address StubGenerator::generate_unsafe_copy(address byte_copy_entry, address sho __ shrptr(size, LogBytesPerLong); // size => qword_count __ jump(RuntimeAddress(long_copy_entry)); + // record the stub entry and end plus + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -2803,8 +3190,23 @@ static void do_setmemory_atomic_loop(USM_TYPE type, Register dest, // to an int, short, or byte fill loop. // address StubGenerator::generate_unsafe_setmemory(address unsafe_byte_fill) { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_unsafe_setmemory_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + // we expect one set of extra unsafememory access handler entries + GrowableArray
extras; + int expected_extra_count = 3 * 3; + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, nullptr, &extras); + assert(extras.length() == expected_extra_count, + "unexpected extra entry count %d", extras.length()); + register_unsafe_access_handlers(extras, 0, 3, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); // required for proper stackwalking of RuntimeStub frame @@ -2896,6 +3298,16 @@ address StubGenerator::generate_unsafe_setmemory(address unsafe_byte_fill) { __ jump(RuntimeAddress(unsafe_byte_fill)); } + // retrieve the registered handler addresses + address end = __ pc(); + retrieve_unsafe_access_handlers(start, end, extras); + assert(extras.length() == expected_extra_count, + "unexpected extra addresses count %d", extras.length()); + + // record the stub entry and end plus the no_push entry and any + // extra handler addresses + store_archive_data(stub_id, start, end, nullptr, &extras); + return start; } @@ -2952,7 +3364,16 @@ address StubGenerator::generate_generic_copy(address byte_copy_entry, address sh address int_copy_entry, address oop_copy_entry, address long_copy_entry, address checkcast_copy_entry) { - Label L_failed, L_failed_0, L_objArray; + StubId stub_id = StubId::stubgen_generic_arraycopy_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + Label L_failed, L_failed_0, L_skip_failed_0, L_objArray; Label L_copy_shorts, L_copy_ints, L_copy_longs; // Input registers @@ -2968,20 +3389,7 @@ address StubGenerator::generate_generic_copy(address byte_copy_entry, address sh const Register rklass_tmp = rdi; // load_klass #endif - { int modulus = CodeEntryAlignment; - int target = modulus - 5; // 5 = sizeof jmp(L_failed) - int advance = target - (__ offset() % modulus); - if (advance < 0) advance += modulus; - if (advance > 0) __ nop(advance); - } - StubId stub_id = StubId::stubgen_generic_arraycopy_id; StubCodeMark mark(this, stub_id); - - // Short-hop target to L_failed. Makes for denser prologue code. - __ BIND(L_failed_0); - __ jmp(L_failed); - assert(__ offset() % CodeEntryAlignment == 0, "no further alignment needed"); - __ align(CodeEntryAlignment); address start = __ pc(); @@ -3024,7 +3432,8 @@ address StubGenerator::generate_generic_copy(address byte_copy_entry, address sh // if (dst_pos < 0) return -1; __ testl(dst_pos, dst_pos); // dst_pos (32-bits) size_t j4off = __ offset(); - __ jccb(Assembler::negative, L_failed_0); + // skip over the failure trampoline + __ jccb(Assembler::positive, L_skip_failed_0); // The first four tests are very dense code, // but not quite dense enough to put four @@ -3034,6 +3443,13 @@ address StubGenerator::generate_generic_copy(address byte_copy_entry, address sh // Make sure of this. guarantee(((j1off ^ j4off) & ~15) != 0, "I$ line of 1st & 4th jumps"); + // Short-hop target to L_failed. Makes for denser prologue code. + __ BIND(L_failed_0); + __ jmp(L_failed); + + // continue here if first 4 checks pass + __ bind(L_skip_failed_0); + // registers used as temp const Register r11_length = r11; // elements count to copy const Register r10_src_klass = r10; // array klass @@ -3256,6 +3672,9 @@ __ BIND(L_failed); __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_cbrt.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_cbrt.cpp index 73330dedc0f7c..d386d4a27440f 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_cbrt.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_cbrt.cpp @@ -339,3 +339,24 @@ address StubGenerator::generate_libmCbrt() { } #undef __ +void StubGenerator::init_AOTAddressTable_cbrt(GrowableArray
& external_addresses) { +#define ADD(addr) external_addresses.append((address)(addr)) + ADD(_ABS_MASK); + ADD(_SIG_MASK); + ADD(_EXP_MASK); + ADD(_EXP_MSK2); + ADD(_EXP_MSK3); + ADD(_SCALE63); + ADD(_ZERON); + ADD(_INF); + ADD(_NEG_INF); + address coeff_table = (address)_coeff_table; + ADD(coeff_table); + ADD(coeff_table + 16); + ADD(coeff_table + 32); + ADD(coeff_table + 48); + ADD(_rcp_table); + ADD(_cbrt_table); + ADD(_D_table); +#undef ADD +} diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_chacha.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_chacha.cpp index 7afaf34e031cc..2366f37112d6f 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_chacha.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_chacha.cpp @@ -584,3 +584,10 @@ bVec, } #undef __ +void StubGenerator::init_AOTAddressTable_chacha(GrowableArray
& external_addresses) { +#define ADD(addr) external_addresses.append((address)(addr)) + ADD(CC20_COUNTER_ADD_AVX); + ADD(CC20_COUNTER_ADD_AVX512); + ADD(CC20_LROT_CONSTS); +#undef ADD +} diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_constants.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_constants.cpp index 93fa7e650db6a..af02736347c86 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_constants.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_constants.cpp @@ -233,3 +233,27 @@ ATTRIBUTE_ALIGNED(16) static const juint _Ctable[] = { }; address StubGenerator::Ctable = (address)_Ctable; +void StubGenerator::init_AOTAddressTable_constants(GrowableArray
& external_addresses) { +#define ADD(addr) external_addresses.append((address)(addr)) + ADD(_ONE); + ADD(_ONEHALF); + ADD(_SIGN_MASK); + ADD(_TWO_POW_55); + ADD(_TWO_POW_M55); + ADD(_SHIFTER); + ADD(_ZERO); + ADD(_SC_1); + ADD(_SC_2); + ADD(_SC_3); + ADD(_SC_4); + ADD(_PI_4); + ADD(((address)_PI_4+8)); + ADD(_PI32INV); + ADD(_NEG_ZERO); + ADD(_P_1); + ADD(_P_2); + ADD(_P_3); + ADD(_PI_INV_TABLE); + ADD(_Ctable); +#undef ADD +} diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_dilithium.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_dilithium.cpp index 9555d60c8a42c..1c88906f09aa6 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_dilithium.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_dilithium.cpp @@ -1032,3 +1032,14 @@ void StubGenerator::generate_dilithium_stubs() { generate_dilithiumDecomposePoly_avx512(this, _masm); } } + +void StubGenerator::init_AOTAddressTable_dilithium(GrowableArray
& external_addresses) { +#define ADD(addr) external_addresses.append((address)(addr)) + // use accessors to correctly identify the relevant addresses + ADD(dilithiumAvx512PermsAddr()); + ADD(dilithiumAvx512ConstsAddr(montQInvModRIdx)); + ADD(dilithiumAvx512ConstsAddr(dilithium_qIdx)); + ADD(dilithiumAvx512ConstsAddr(montRSquareModQIdx)); + ADD(dilithiumAvx512ConstsAddr(barrettAddendIdx)); +#undef ADD +} diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_exp.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_exp.cpp index 5130fd2c9d2d4..32bbee73f5430 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_exp.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_exp.cpp @@ -385,3 +385,22 @@ address StubGenerator::generate_libmExp() { } #undef __ + +void StubGenerator::init_AOTAddressTable_exp(GrowableArray
& external_addresses) { +#define ADD(addr) external_addresses.append((address)addr); + ADD(_cv); + ADD(((address)_cv+16)); + ADD(((address)_cv+32)); + ADD(((address)_cv+48)); + ADD(((address)_cv+64)); + ADD(((address)_cv+80)); + ADD(_mmask); + ADD(_bias); + ADD(_Tbl_addr); + ADD(_ALLONES); + ADD(_ebias); + ADD(_XMAX); + ADD(_XMIN); + ADD(_INF); +#undef ADD +} diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_fmod.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_fmod.cpp index b1eaa4b8031ad..cfbaf97a5aa32 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_fmod.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_fmod.cpp @@ -525,3 +525,13 @@ address StubGenerator::generate_libmFmod() { } #undef __ + +void StubGenerator::init_AOTAddressTable_fmod(GrowableArray
& external_addresses) { +#define ADD(addr) external_addresses.append((address)addr); + ADD(CONST_NaN); + ADD(CONST_1p260); + ADD(CONST_MAX); + ADD(CONST_INF); + ADD(CONST_e307); +#undef ADD +} diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_ghash.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_ghash.cpp index 6f05b1ab5e6b9..4286088b45e68 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_ghash.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_ghash.cpp @@ -538,3 +538,12 @@ void StubGenerator::generateHtbl_eight_blocks(Register htbl) { } #undef __ + +void StubGenerator::init_AOTAddressTable_ghash(GrowableArray
& external_addresses) { +#define ADD(addr) external_addresses.append((address)addr); + ADD(GHASH_SHUFFLE_MASK); + ADD(GHASH_LONG_SWAP_MASK); + ADD(GHASH_BYTE_SWAP_MASK); + ADD(GHASH_POLYNOMIAL); +#undef ADD +} diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_kyber.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_kyber.cpp index 3e5593322d5ea..c934d3c8ceb2a 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_kyber.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_kyber.cpp @@ -950,3 +950,19 @@ void StubGenerator::generate_kyber_stubs() { } } } + +void StubGenerator::init_AOTAddressTable_kyber(GrowableArray
& external_addresses) { +#define ADD(addr) external_addresses.append((address)(addr)) + // use accessors to correctly identify the relevant addresses + ADD(kyberAvx512NttPermsAddr()); + ADD(kyberAvx512InverseNttPermsAddr()); + ADD(kyberAvx512_nttMultPermsAddr()); + ADD(kyberAvx512_12To16PermsAddr()); + ADD(kyberAvx512ConstsAddr(qOffset)); + ADD(kyberAvx512ConstsAddr(qInvModROffset)); + ADD(kyberAvx512ConstsAddr(dimHalfInverseOffset)); + ADD(kyberAvx512ConstsAddr(barretMultiplierOffset)); + ADD(kyberAvx512ConstsAddr(montRSquareModqOffset)); + ADD(kyberAvx512ConstsAddr(f00Offset)); +#undef ADD +} diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_log.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_log.cpp index 6b5b4d704e3e3..13da0591a592e 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_log.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_log.cpp @@ -708,3 +708,23 @@ address StubGenerator::generate_libmLog10() { } #undef __ + +void StubGenerator::init_AOTAddressTable_log(GrowableArray
& external_addresses) { +#define ADD(addr) external_addresses.append((address)addr); + ADD(_L_tbl); + ADD(_log2); + ADD(((address)_log2+8)); + ADD(_coeff); + ADD(((address)_coeff+16)); + ADD(((address)_coeff+32)); + ADD(_HIGHSIGMASK_log10); + ADD(_LOG10_E); + ADD(((address)_LOG10_E+8)); + ADD(_L_tbl_log10); + ADD(_log2_log10); + ADD(((address)_log2_log10+8)); + ADD(_coeff_log10); + ADD(((address)_coeff_log10+16)); + ADD(((address)_coeff_log10+32)); +#undef ADD +} diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_poly1305.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_poly1305.cpp index c80b2d16181c6..2a4874a78108f 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_poly1305.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_poly1305.cpp @@ -1695,3 +1695,12 @@ void StubGenerator::poly1305_msg_mul_reduce_vec4_avx2( __ vpaddq(A1, A1, YTMP2, Assembler::AVX_256bit); //Add medium 42-bit bits from new blocks to accumulator __ vpaddq(A1, A1, YTMP5, Assembler::AVX_256bit); } +#undef __ + +void StubGenerator::init_AOTAddressTable_poly1305(GrowableArray
& external_addresses) { +#define ADD(addr) external_addresses.append((address)addr); + ADD(POLY1305_PAD_MSG); + ADD(POLY1305_MASK42); + ADD(POLY1305_MASK44); +#undef ADD +} diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_poly_mont.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_poly_mont.cpp index c439e0b370faf..ebe2414486900 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_poly_mont.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_poly_mont.cpp @@ -764,3 +764,16 @@ address StubGenerator::generate_intpoly_assign() { __ ret(0); return start; } +#undef __ + +void StubGenerator::init_AOTAddressTable_poly_mont(GrowableArray
& external_addresses) { +#define ADD(addr) external_addresses.append((address)addr); + // use accessors to retrieve all correct addresses + ADD(shift_1L()); + ADD(shift_1R()); + ADD(p256_mask52()); + ADD(mask_limb5()); + ADD(modulus_p256()); + ADD(modulus_p256(1)); +#undef ADD +} diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_pow.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_pow.cpp index 3c3df7e6ac4f4..581b0ff1ca976 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_pow.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_pow.cpp @@ -1863,3 +1863,31 @@ address StubGenerator::generate_libmPow() { } #undef __ + +void StubGenerator::init_AOTAddressTable_pow(GrowableArray
& external_addresses) { +#define ADD(addr) external_addresses.append((address)addr); + ADD(_HIGHSIGMASK); + ADD(_LOG2_E); + ADD(_HIGHMASK_Y); + ADD((address)_HIGHMASK_Y+8); + ADD(_T_exp); + ADD(_e_coeff); + ADD((address)_e_coeff+16); + ADD((address)_e_coeff+32); + ADD(_coeff_h); + ADD((address)_coeff_h+8); + ADD(_HIGHMASK_LOG_X); + ADD(_HALFMASK); + ADD(_coeff_pow); + ADD((address)_coeff_pow+16); + ADD((address)_coeff_pow+32); + ADD((address)_coeff_pow+48); + ADD((address)_coeff_pow+64); + ADD((address)_coeff_pow+80); + ADD(_L_tbl_pow); + ADD(_log2_pow); + ADD(_DOUBLE2); + ADD(_DOUBLE0); + ADD(_DOUBLE0DOT5); +#undef ADD +} diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_sha3.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_sha3.cpp index f9d876f34f377..6b830f02e6c9b 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_sha3.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_sha3.cpp @@ -508,3 +508,12 @@ void StubGenerator::generate_sha3_stubs() { generate_sha3_implCompress(StubId::stubgen_sha3_implCompressMB_id, this, _masm); } } + +#undef __ + +void StubGenerator::init_AOTAddressTable_sha3(GrowableArray
& external_addresses) { +#define ADD(addr) external_addresses.append((address)addr); + ADD(round_constsAddr()); + ADD(permsAndRotsAddr()); +#undef ADD +} diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_sin.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_sin.cpp index 5290e73758128..2b313d6c10fa6 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_sin.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_sin.cpp @@ -649,3 +649,9 @@ address StubGenerator::generate_libmSin() { } #undef __ + +void StubGenerator::init_AOTAddressTable_sin(GrowableArray
& external_addresses) { +#define ADD(addr) external_addresses.append((address)addr); + ADD(_ALL_ONES); +#undef ADD +} diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_sinh.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_sinh.cpp index 86e4ac2017605..7f190068467e6 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_sinh.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_sinh.cpp @@ -523,3 +523,23 @@ address StubGenerator::generate_libmSinh() { } #undef __ + +void StubGenerator::init_AOTAddressTable_sinh(GrowableArray
& external_addresses) { +#define ADD(addr) external_addresses.append((address)addr); + ADD(_L2E); + ADD(_L2E + 8); + ADD(_HALFMASK); + ADD(_Shifter); + ADD(_cv); + ADD(_cv + 16); + ADD(_cv + 32); + ADD(_cv + 48); + ADD(_cv + 64); + ADD(_T2f); + ADD(_T2_neg_f); + ADD(_pv); + ADD(_pv + 16); + ADD(_pv + 32); + ADD(_MASK3); +#undef ADD +} diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_tan.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_tan.cpp index 4f14414652c2d..04534ed50dadd 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_tan.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_tan.cpp @@ -1029,3 +1029,24 @@ address StubGenerator::generate_libmTan() { } #undef __ + +void StubGenerator::init_AOTAddressTable_tan(GrowableArray
& external_addresses) { +#define ADD(addr) external_addresses.append((address)addr); + ADD(_MUL16); + ADD(_sign_mask_tan); + ADD(_PI32INV_tan); + ADD(_P_1_tan); + ADD(_P_2_tan); + ADD(_P_3_tan); + ADD(_Ctable_tan); + ADD(_MASK_35_tan); + ADD(_Q_11_tan); + ADD(_Q_9_tan); + ADD(_Q_7_tan); + ADD(_Q_5_tan); + ADD(_Q_3_tan); + ADD(_PI_4_tan); + ADD(((address)_PI_4_tan+8)); + ADD(_QQ_2_tan); +#undef ADD +} diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_tanh.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_tanh.cpp index dce4fbfc4550c..e0618ced9d820 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_tanh.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_tanh.cpp @@ -499,3 +499,23 @@ address StubGenerator::generate_libmTanh() { } #undef __ + +void StubGenerator::init_AOTAddressTable_tanh(GrowableArray
& external_addresses) { +#define ADD(addr) external_addresses.append((address)addr); + ADD(_L2E); + ADD(_L2E + 8); + ADD(_HALFMASK); + ADD(_ONEMASK); + ADD(_TWOMASK); + ADD(_Shifter); + ADD(_cv); + ADD(_cv + 16); + ADD(_cv + 32); + ADD(_T2_neg_f); + ADD(_pv); + ADD(_pv + 16); + ADD(_pv + 32); + ADD(_MASK3); + ADD(_RMASK); +#undef ADD +} diff --git a/src/hotspot/cpu/x86/stubRoutines_x86.cpp b/src/hotspot/cpu/x86/stubRoutines_x86.cpp index ee9cea08e647c..0a9a1838b4cf6 100644 --- a/src/hotspot/cpu/x86/stubRoutines_x86.cpp +++ b/src/hotspot/cpu/x86/stubRoutines_x86.cpp @@ -411,3 +411,28 @@ ATTRIBUTE_ALIGNED(64) const julong StubRoutines::x86::_k512_W[] = 0x4cc5d4becb3e42b6ULL, 0x597f299cfc657e2aULL, 0x5fcb6fab3ad6faecULL, 0x6c44198c4a475817ULL, }; + +// publish addresses of external data defined in this file which may +// be referenced from stub or code +void StubRoutines::x86::init_AOTAddressTable(GrowableArray
& external_addresses) { +#define ADD(addr) external_addresses.append((address)addr); + ADD(&_mxcsr_std); + ADD(&_mxcsr_rz); + ADD(crc_by128_masks_addr()); + ADD(crc_by128_masks_addr() + 16); + ADD(crc_by128_masks_addr() + 32); + ADD(_crc_table); + ADD(crc_by128_masks_avx512_addr()); + ADD(crc_by128_masks_avx512_addr() + 16); + ADD(crc_by128_masks_avx512_addr() + 32); + ADD(_crc_table_avx512); + ADD(_crc32c_table_avx512); + ADD(_shuf_table_crc32_avx512); + // n.b. call accessor for this one to ensure the table is generated + ADD(crc32c_table_addr()); + ADD(_arrays_hashcode_powers_of_31); + ADD(_k256); + ADD(_k256_W); + ADD(_k512_W); +#undef ADD +} diff --git a/src/hotspot/cpu/x86/stubRoutines_x86.hpp b/src/hotspot/cpu/x86/stubRoutines_x86.hpp index 7d13c4f6e7ac9..f4d9cd6eda30f 100644 --- a/src/hotspot/cpu/x86/stubRoutines_x86.hpp +++ b/src/hotspot/cpu/x86/stubRoutines_x86.hpp @@ -112,6 +112,8 @@ class x86 { static address arrays_hashcode_powers_of_31() { return (address)_arrays_hashcode_powers_of_31; } static void generate_CRC32C_table(bool is_pclmulqdq_supported); + + static void init_AOTAddressTable(GrowableArray
& external_addresses); }; #endif // CPU_X86_STUBROUTINES_X86_HPP diff --git a/src/hotspot/share/code/aotCodeCache.cpp b/src/hotspot/share/code/aotCodeCache.cpp index 40c2b95a57827..4a535a14eb440 100644 --- a/src/hotspot/share/code/aotCodeCache.cpp +++ b/src/hotspot/share/code/aotCodeCache.cpp @@ -1714,7 +1714,7 @@ void AOTCodeReader::read_dbg_strings(DbgStrings& dbg_strings) { // [_stubs_base, _stubs_base + _stubs_max -1], [_c_str_base, // _c_str_base + _c_str_max -1], -#define _extrs_max 200 +#define _extrs_max 350 #define _stubs_max static_cast(EntryId::NUM_ENTRYIDS) #define _extrs_base 0 @@ -1753,10 +1753,6 @@ void AOTCodeAddressTable::init_extrs() { { // Required by initial stubs SET_ADDRESS(_extrs, SharedRuntime::exception_handler_for_return_address); // used by forward_exception -#if defined(AMD64) - SET_ADDRESS(_extrs, StubRoutines::x86::addr_mxcsr_std()); // used by call_stub - SET_ADDRESS(_extrs, StubRoutines::x86::addr_mxcsr_rz()); // used by libmFmod -#endif SET_ADDRESS(_extrs, CompressedOops::base_addr()); // used by call_stub SET_ADDRESS(_extrs, Thread::current); // used by call_stub SET_ADDRESS(_extrs, SharedRuntime::throw_StackOverflowError); @@ -1833,17 +1829,6 @@ void AOTCodeAddressTable::init_extrs() { SET_ADDRESS(_extrs, JavaThread::verify_cross_modify_fence_failure); #endif -#if defined(AMD64) - SET_ADDRESS(_extrs, StubRoutines::x86::arrays_hashcode_powers_of_31()); -#endif - -#ifdef X86 - SET_ADDRESS(_extrs, LIR_Assembler::float_signmask_pool); - SET_ADDRESS(_extrs, LIR_Assembler::double_signmask_pool); - SET_ADDRESS(_extrs, LIR_Assembler::float_signflip_pool); - SET_ADDRESS(_extrs, LIR_Assembler::double_signflip_pool); -#endif - SET_ADDRESS(_extrs, JfrIntrinsicSupport::write_checkpoint); SET_ADDRESS(_extrs, JfrIntrinsicSupport::return_lease); From 24a7c8817d378a9c1dbf4a219eb0ffe0a3207628 Mon Sep 17 00:00:00 2001 From: Andrew Dinn Date: Wed, 19 Nov 2025 11:21:41 +0000 Subject: [PATCH 09/30] fix x86 avx2 stub generation --- .../x86/stubGenerator_x86_64_arraycopy.cpp | 94 ++++++++++++------- 1 file changed, 59 insertions(+), 35 deletions(-) diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_arraycopy.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_arraycopy.cpp index 49acc64164082..a07be16747f3b 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_arraycopy.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_arraycopy.cpp @@ -2138,15 +2138,17 @@ address StubGenerator::generate_disjoint_int_oop_copy(StubId stub_id, address* e #endif GrowableArray
entries; GrowableArray
extras; + bool add_extras = !is_oop && !aligned; int expected_entry_count = (entry != nullptr ? 2 : 1); - int expected_extra_count = (2 * 3); // 2 x UMAM {start,end,handler} + int expected_extra_count = 3 * (add_extras ? 2 : 0); // 0/2 x UMAM {start,end,handler} int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == expected_entry_count, "sanity check"); GrowableArray
* entries_ptr = (entry_count == 1 ? nullptr : &entries); + GrowableArray
* extras_ptr = (add_extras ? extras_ptr : nullptr); if (find_archive_data(stub_id)) { address start = nullptr; address end = nullptr; - load_archive_data(stub_id, start, end, entries_ptr, &extras); + load_archive_data(stub_id, start, end, entries_ptr, extras_ptr); assert(entries.length() == expected_entry_count - 1, "unexpected extra entry count %d", entries.length()); assert(extras.length() == expected_extra_count, @@ -2154,8 +2156,10 @@ address StubGenerator::generate_disjoint_int_oop_copy(StubId stub_id, address* e if (entry != nullptr) { *entry = entries.at(0); } - // restore 2 UMAM {start,end,handler} addresses from extras - register_unsafe_access_handlers(extras, 0, 2, end); + if (add_extras) { + // restore 2 UMAM {start,end,handler} addresses from extras + register_unsafe_access_handlers(extras, 0, 2, end); + } return start; } @@ -2200,7 +2204,7 @@ address StubGenerator::generate_disjoint_int_oop_copy(StubId stub_id, address* e { // UnsafeMemoryAccess page error: continue after unsafe access - UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, true); + UnsafeMemoryAccessMark umam(this, add_extras, true); // 'from', 'to' and 'count' are now valid __ movptr(dword_count, count); __ shrptr(count, 1); // count => qword_count @@ -2236,7 +2240,7 @@ address StubGenerator::generate_disjoint_int_oop_copy(StubId stub_id, address* e __ ret(0); { - UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, false, ucme_exit_pc); + UnsafeMemoryAccessMark umam(this, add_extras, false, ucme_exit_pc); // Copy in multi-bytes chunks copy_bytes_forward(end_from, end_to, qword_count, rax, r10, L_copy_bytes, L_copy_8_bytes, decorators, is_oop ? T_OBJECT : T_INT); __ jmp(L_copy_4_bytes); @@ -2244,13 +2248,15 @@ address StubGenerator::generate_disjoint_int_oop_copy(StubId stub_id, address* e // retrieve the registered handler addresses address end = __ pc(); - retrieve_unsafe_access_handlers(start, end, extras); + if (add_extras) { + retrieve_unsafe_access_handlers(start, end, extras); + } assert(extras.length() == expected_extra_count, "unexpected extra addresses count %d", extras.length()); // record the stub entry and end plus the no_push entry and any // extra handler addresses - store_archive_data(stub_id, start, end, entries_ptr, &extras); + store_archive_data(stub_id, start, end, entries_ptr, extras_ptr); return start; } @@ -2300,17 +2306,19 @@ address StubGenerator::generate_conjoint_int_oop_copy(StubId stub_id, address no return generate_conjoint_copy_avx3_masked(stub_id, entry, nooverlap_target); } #endif + bool add_extras = !is_oop && !aligned; GrowableArray
entries; GrowableArray
extras; int expected_entry_count = (entry != nullptr ? 2 : 1); - int expected_extra_count = (2 * 3); // 2 x UMAM {start,end,handler} + int expected_extra_count = 3 * (add_extras ? 2 : 0); // 0/2 x UMAM {start,end,handler} int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == expected_entry_count, "sanity check"); GrowableArray
* entries_ptr = (entry_count == 1 ? nullptr : &entries); + GrowableArray
* extras_ptr = (add_extras ? &extras : nullptr); if (find_archive_data(stub_id)) { address start = nullptr; address end = nullptr; - load_archive_data(stub_id, start, end, entries_ptr, &extras); + load_archive_data(stub_id, start, end, entries_ptr, extras_ptr); assert(entries.length() == expected_entry_count - 1, "unexpected extra entry count %d", entries.length()); assert(extras.length() == expected_extra_count, @@ -2318,8 +2326,10 @@ address StubGenerator::generate_conjoint_int_oop_copy(StubId stub_id, address no if (entry != nullptr) { *entry = entries.at(0); } - // restore 2 UMAM {start,end,handler} addresses from extras - register_unsafe_access_handlers(extras, 0, 2, end); + if (add_extras) { + // restore 2 UMAM {start,end,handler} addresses from extras + register_unsafe_access_handlers(extras, 0, 2, end); + } return start; } @@ -2340,7 +2350,7 @@ address StubGenerator::generate_conjoint_int_oop_copy(StubId stub_id, address no if (entry != nullptr) { *entry = __ pc(); entries.append(*entry); - // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) + // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) BLOCK_COMMENT("Entry:"); } @@ -2363,7 +2373,7 @@ address StubGenerator::generate_conjoint_int_oop_copy(StubId stub_id, address no assert_clean_int(count, rax); // Make sure 'count' is clean int. { // UnsafeMemoryAccess page error: continue after unsafe access - UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, true); + UnsafeMemoryAccessMark umam(this, add_extras, true); // 'from', 'to' and 'count' are now valid __ movptr(dword_count, count); __ shrptr(count, 1); // count => qword_count @@ -2378,7 +2388,7 @@ address StubGenerator::generate_conjoint_int_oop_copy(StubId stub_id, address no __ jmp(L_copy_bytes); // Copy trailing qwords - __ BIND(L_copy_8_bytes); + __ BIND(L_copy_8_bytes); __ movq(rax, Address(from, qword_count, Address::times_8, -8)); __ movq(Address(to, qword_count, Address::times_8, -8), rax); __ decrement(qword_count); @@ -2396,12 +2406,12 @@ address StubGenerator::generate_conjoint_int_oop_copy(StubId stub_id, address no { // UnsafeMemoryAccess page error: continue after unsafe access - UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, true); + UnsafeMemoryAccessMark umam(this, add_extras, true); // Copy in multi-bytes chunks copy_bytes_backward(from, to, qword_count, rax, r10, L_copy_bytes, L_copy_8_bytes, decorators, is_oop ? T_OBJECT : T_INT); } -__ BIND(L_exit); + __ BIND(L_exit); bs->arraycopy_epilogue(_masm, decorators, type, from, to, dword_count); restore_arg_regs_using_thread(); INC_COUNTER_NP(SharedRuntime::_jint_array_copy_ctr, rscratch1); // Update counter after rscratch1 is free @@ -2412,13 +2422,15 @@ __ BIND(L_exit); // retrieve the registered handler addresses address end = __ pc(); - retrieve_unsafe_access_handlers(start, end, extras); + if (add_extras) { + retrieve_unsafe_access_handlers(start, end, extras); + } assert(extras.length() == expected_extra_count, "unexpected extra addresses count %d", extras.length()); // record the stub entry and end plus the no_push entry and any // extra handler addresses - store_archive_data(stub_id, start, end, entries_ptr, &extras); + store_archive_data(stub_id, start, end, entries_ptr, extras_ptr); return start; } @@ -2466,17 +2478,19 @@ address StubGenerator::generate_disjoint_long_oop_copy(StubId stub_id, address * return generate_disjoint_copy_avx3_masked(stub_id, entry); } #endif + bool add_extras = !is_oop && !aligned; GrowableArray
entries; GrowableArray
extras; int expected_entry_count = (entry != nullptr ? 2 : 1); - int expected_extra_count = (2 * 3); // 2 x UMAM {start,end,handler} + int expected_extra_count = 3 * (add_extras ? 2 : 0); // 0/2 x UMAM {start,end,handler} int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == expected_entry_count, "sanity check"); GrowableArray
* entries_ptr = (entry_count == 1 ? nullptr : &entries); + GrowableArray
* extras_ptr = (add_extras ? &extras : nullptr); if (find_archive_data(stub_id)) { address start = nullptr; address end = nullptr; - load_archive_data(stub_id, start, end, entries_ptr, &extras); + load_archive_data(stub_id, start, end, entries_ptr, extras_ptr); assert(entries.length() == expected_entry_count - 1, "unexpected extra entry count %d", entries.length()); assert(extras.length() == expected_extra_count, @@ -2484,8 +2498,10 @@ address StubGenerator::generate_disjoint_long_oop_copy(StubId stub_id, address * if (entry != nullptr) { *entry = entries.at(0); } - // restore 2 UMAM {start,end,handler} addresses from extras - register_unsafe_access_handlers(extras, 0, 2, end); + if (add_extras) { + // restore 2 UMAM {start,end,handler} addresses from extras + register_unsafe_access_handlers(extras, 0, 2, end); + } return start; } @@ -2530,7 +2546,7 @@ address StubGenerator::generate_disjoint_long_oop_copy(StubId stub_id, address * bs->arraycopy_prologue(_masm, decorators, type, from, to, qword_count); { // UnsafeMemoryAccess page error: continue after unsafe access - UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, true); + UnsafeMemoryAccessMark umam(this, add_extras, true); // Copy from low to high addresses. Use 'to' as scratch. __ lea(end_from, Address(from, qword_count, Address::times_8, -8)); @@ -2562,7 +2578,7 @@ address StubGenerator::generate_disjoint_long_oop_copy(StubId stub_id, address * { // UnsafeMemoryAccess page error: continue after unsafe access - UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, true); + UnsafeMemoryAccessMark umam(this, add_extras, true); // Copy in multi-bytes chunks copy_bytes_forward(end_from, end_to, qword_count, rax, r10, L_copy_bytes, L_copy_8_bytes, decorators, is_oop ? T_OBJECT : T_LONG); } @@ -2580,13 +2596,15 @@ address StubGenerator::generate_disjoint_long_oop_copy(StubId stub_id, address * // retrieve the registered handler addresses address end = __ pc(); - retrieve_unsafe_access_handlers(start, end, extras); + if (add_extras) { + retrieve_unsafe_access_handlers(start, end, extras); + } assert(extras.length() == expected_extra_count, "unexpected extra addresses count %d", extras.length()); // record the stub entry and end plus the no_push entry and any // extra handler addresses - store_archive_data(stub_id, start, end, entries_ptr, &extras); + store_archive_data(stub_id, start, end, entries_ptr, extras_ptr); return start; } @@ -2632,17 +2650,19 @@ address StubGenerator::generate_conjoint_long_oop_copy(StubId stub_id, address n return generate_conjoint_copy_avx3_masked(stub_id, entry, nooverlap_target); } #endif + bool add_extras = !is_oop && !aligned; GrowableArray
entries; GrowableArray
extras; int expected_entry_count = (entry != nullptr ? 2 : 1); - int expected_extra_count = (2 * 3); // 2 x UMAM {start,end,handler} + int expected_extra_count = 3 * (add_extras ? 2 : 0); // 0/2 x UMAM {start,end,handler} int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == expected_entry_count, "sanity check"); GrowableArray
* entries_ptr = (entry_count == 1 ? nullptr : &entries); + GrowableArray
* extras_ptr = (add_extras ? &extras : nullptr); if (find_archive_data(stub_id)) { address start = nullptr; address end = nullptr; - load_archive_data(stub_id, start, end, entries_ptr, &extras); + load_archive_data(stub_id, start, end, entries_ptr, extras_ptr); assert(entries.length() == expected_entry_count - 1, "unexpected extra entry count %d", entries.length()); assert(extras.length() == expected_extra_count, @@ -2650,8 +2670,10 @@ address StubGenerator::generate_conjoint_long_oop_copy(StubId stub_id, address n if (entry != nullptr) { *entry = entries.at(0); } - // restore 2 UMAM {start,end,handler} addresses from extras - register_unsafe_access_handlers(extras, 0, 2, end); + if (add_extras) { + // restore 2 UMAM {start,end,handler} addresses from extras + register_unsafe_access_handlers(extras, 0, 2, end); + } return start; } @@ -2692,7 +2714,7 @@ address StubGenerator::generate_conjoint_long_oop_copy(StubId stub_id, address n bs->arraycopy_prologue(_masm, decorators, type, from, to, qword_count); { // UnsafeMemoryAccess page error: continue after unsafe access - UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, true); + UnsafeMemoryAccessMark umam(this, add_extras, true); __ jmp(L_copy_bytes); @@ -2719,7 +2741,7 @@ address StubGenerator::generate_conjoint_long_oop_copy(StubId stub_id, address n } { // UnsafeMemoryAccess page error: continue after unsafe access - UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, true); + UnsafeMemoryAccessMark umam(this, add_extras, true); // Copy in multi-bytes chunks copy_bytes_backward(from, to, qword_count, rax, r10, L_copy_bytes, L_copy_8_bytes, decorators, is_oop ? T_OBJECT : T_LONG); @@ -2738,13 +2760,15 @@ address StubGenerator::generate_conjoint_long_oop_copy(StubId stub_id, address n // retrieve the registered handler addresses address end = __ pc(); - retrieve_unsafe_access_handlers(start, end, extras); + if (add_extras) { + retrieve_unsafe_access_handlers(start, end, extras); + } assert(extras.length() == expected_extra_count, "unexpected extra addresses count %d", extras.length()); // record the stub entry and end plus the no_push entry and any // extra handler addresses - store_archive_data(stub_id, start, end, entries_ptr, &extras); + store_archive_data(stub_id, start, end, entries_ptr, extras_ptr); return start; } From 1da6590e3becf8d72c1f62f8eaf8e4f9738b74e1 Mon Sep 17 00:00:00 2001 From: Andrew Dinn Date: Wed, 19 Nov 2025 11:22:21 +0000 Subject: [PATCH 10/30] fix zero/minimal build issues --- src/hotspot/share/code/aotCodeCache.cpp | 3 +- src/hotspot/share/code/aotCodeCache.hpp | 40 ++++++++++++------------- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/src/hotspot/share/code/aotCodeCache.cpp b/src/hotspot/share/code/aotCodeCache.cpp index 4a535a14eb440..81b2c79c4f014 100644 --- a/src/hotspot/share/code/aotCodeCache.cpp +++ b/src/hotspot/share/code/aotCodeCache.cpp @@ -1829,9 +1829,10 @@ void AOTCodeAddressTable::init_extrs() { SET_ADDRESS(_extrs, JavaThread::verify_cross_modify_fence_failure); #endif +#if INCLUDE_JFR SET_ADDRESS(_extrs, JfrIntrinsicSupport::write_checkpoint); SET_ADDRESS(_extrs, JfrIntrinsicSupport::return_lease); - +#endif SET_ADDRESS(_extrs, UpcallLinker::handle_uncaught_exception); // used by upcall_stub_exception_handler diff --git a/src/hotspot/share/code/aotCodeCache.hpp b/src/hotspot/share/code/aotCodeCache.hpp index fb970575eb2bb..8b7ecfe088d07 100644 --- a/src/hotspot/share/code/aotCodeCache.hpp +++ b/src/hotspot/share/code/aotCodeCache.hpp @@ -151,7 +151,7 @@ class AOTCodeAddressTable : public CHeapObj { void init_extrs(); void init_extrs2(); void add_stub_entry(EntryId entry_id, address entry); - void add_external_addresses(GrowableArray
& addresses); + void add_external_addresses(GrowableArray
& addresses) NOT_CDS_RETURN; void set_shared_stubs_complete(); void set_c1_stubs_complete(); void set_c2_stubs_complete(); @@ -242,25 +242,23 @@ class AOTStubData : public StackObj { GrowableArray
& address_array() { return _address_array; } // accessor for entry/auxiliary addresses defaults to start entry public: - AOTStubData(BlobId blob_id); + AOTStubData(BlobId blob_id) NOT_CDS({}); - ~AOTStubData() { - FREE_C_HEAP_ARRAY(StubAddrRange, _ranges); - } + ~AOTStubData() CDS_ONLY({FREE_C_HEAP_ARRAY(StubAddrRange, _ranges);}) NOT_CDS({}) - bool is_open() { return (_flags & OPEN) != 0; } - bool is_using() { return (_flags & USING) != 0; } - bool is_dumping() { return (_flags & DUMPING) != 0; } - bool is_aot() { return is_using() || is_dumping(); } - bool is_invalid() { return (_flags & INVALID) != 0; } + bool is_open() CDS_ONLY({ return (_flags & OPEN) != 0; }) NOT_CDS_RETURN_(false); + bool is_using() CDS_ONLY({ return (_flags & USING) != 0; }) NOT_CDS_RETURN_(false); + bool is_dumping() CDS_ONLY({ return (_flags & DUMPING) != 0; }) NOT_CDS_RETURN_(false); + bool is_aot() CDS_ONLY({ return is_using() || is_dumping(); }) NOT_CDS_RETURN_(false); + bool is_invalid() CDS_ONLY({ return (_flags & INVALID) != 0; }) NOT_CDS_RETURN_(false); BlobId blob_id() { return _blob_id; } StubId current_stub_id() { return _current; } // - bool load_code_blob(); - bool store_code_blob(CodeBlob& new_blob, CodeBuffer *code_buffe); + bool load_code_blob() NOT_CDS_RETURN_(true); + bool store_code_blob(CodeBlob& new_blob, CodeBuffer *code_buffer) NOT_CDS_RETURN_(true); // determine whether a stub is available in the AOT cache - bool find_archive_data(StubId stub_id); + bool find_archive_data(StubId stub_id) NOT_CDS_RETURN_(false); // retrieve stub entry data if it we are using archived stubs and // the stub has been found in an AOT-restored blob or store stub // entry data if we are saving archived stubs and the stub has just @@ -289,8 +287,8 @@ class AOTStubData : public StackObj { // memory protection ranges and associated handler addresses. These // do do not need to be declared as entries and their number and // meaning may vary according to the architecture. - void load_archive_data(StubId stub_id, address& start, address& end, GrowableArray
* entries = nullptr, GrowableArray
* extras = nullptr); - void store_archive_data(StubId stub_id, address start, address end, GrowableArray
* entries = nullptr, GrowableArray
* extras = nullptr); + void load_archive_data(StubId stub_id, address& start, address& end, GrowableArray
* entries = nullptr, GrowableArray
* extras = nullptr) NOT_CDS_RETURN; + void store_archive_data(StubId stub_id, address start, address end, GrowableArray
* entries = nullptr, GrowableArray
* extras = nullptr) NOT_CDS_RETURN; const AOTStubData* as_const() { return (const AOTStubData*)this; } }; @@ -436,10 +434,10 @@ class AOTCodeCache : public CHeapObj { void load_strings(); int store_strings(); - static void set_shared_stubs_complete(); - static void set_c1_stubs_complete(); - static void set_c2_stubs_complete(); - static void set_stubgen_stubs_complete(); + static void set_shared_stubs_complete() NOT_CDS_RETURN; + static void set_c1_stubs_complete() NOT_CDS_RETURN ; + static void set_c2_stubs_complete() NOT_CDS_RETURN; + static void set_stubgen_stubs_complete() NOT_CDS_RETURN; void add_stub_entries(StubId stub_id, address start, GrowableArray
*entries = nullptr, int offset = -1) NOT_CDS_RETURN; @@ -515,9 +513,9 @@ class AOTCodeCache : public CHeapObj { BlobId id, AOTStubData* stub_data) NOT_CDS_RETURN_(nullptr); - static void publish_external_addresses(GrowableArray
& addresses); + static void publish_external_addresses(GrowableArray
& addresses) NOT_CDS_RETURN; // publish all entries for a code blob in code cache address table - static void publish_stub_addresses(CodeBlob &code_blob, BlobId id, AOTStubData *stub_data); + static void publish_stub_addresses(CodeBlob &code_blob, BlobId id, AOTStubData *stub_data) NOT_CDS_RETURN; static uint store_entries_cnt() { if (is_on_for_dump()) { From 3774e5fedb6d5133ef5c9be9e27fac8cfaae7b45 Mon Sep 17 00:00:00 2001 From: Andrew Dinn Date: Wed, 19 Nov 2025 13:00:53 +0000 Subject: [PATCH 11/30] fix more zero issues --- src/hotspot/share/code/aotCodeCache.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/hotspot/share/code/aotCodeCache.cpp b/src/hotspot/share/code/aotCodeCache.cpp index 81b2c79c4f014..17c478038cb0b 100644 --- a/src/hotspot/share/code/aotCodeCache.cpp +++ b/src/hotspot/share/code/aotCodeCache.cpp @@ -1782,10 +1782,10 @@ void AOTCodeAddressTable::init_extrs() { SET_ADDRESS(_extrs, SharedRuntime::enable_stack_reserved_zone); -#ifdef AMD64 +#if defined(AMD64) && !defined(ZERO) SET_ADDRESS(_extrs, SharedRuntime::montgomery_multiply); SET_ADDRESS(_extrs, SharedRuntime::montgomery_square); -#endif // AMD64 +#endif // defined(AMD64) && !defined(ZERO) SET_ADDRESS(_extrs, SharedRuntime::d2f); SET_ADDRESS(_extrs, SharedRuntime::d2i); @@ -1795,12 +1795,16 @@ void AOTCodeAddressTable::init_extrs() { SET_ADDRESS(_extrs, SharedRuntime::dlog); SET_ADDRESS(_extrs, SharedRuntime::dlog10); SET_ADDRESS(_extrs, SharedRuntime::dpow); +#ifndef ZERO SET_ADDRESS(_extrs, SharedRuntime::drem); +#endif SET_ADDRESS(_extrs, SharedRuntime::dsin); SET_ADDRESS(_extrs, SharedRuntime::dtan); SET_ADDRESS(_extrs, SharedRuntime::f2i); SET_ADDRESS(_extrs, SharedRuntime::f2l); +#ifndef ZERO SET_ADDRESS(_extrs, SharedRuntime::frem); +#endif SET_ADDRESS(_extrs, SharedRuntime::l2d); SET_ADDRESS(_extrs, SharedRuntime::l2f); SET_ADDRESS(_extrs, SharedRuntime::ldiv); From 2043042f8e1a4e68d638034a8344e738d56d7976 Mon Sep 17 00:00:00 2001 From: Andrew Dinn Date: Wed, 19 Nov 2025 17:02:56 +0000 Subject: [PATCH 12/30] correct var init on x86 --- src/hotspot/cpu/x86/stubGenerator_x86_64_arraycopy.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_arraycopy.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_arraycopy.cpp index a07be16747f3b..c9649fbfa3933 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_arraycopy.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_arraycopy.cpp @@ -2144,7 +2144,7 @@ address StubGenerator::generate_disjoint_int_oop_copy(StubId stub_id, address* e int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == expected_entry_count, "sanity check"); GrowableArray
* entries_ptr = (entry_count == 1 ? nullptr : &entries); - GrowableArray
* extras_ptr = (add_extras ? extras_ptr : nullptr); + GrowableArray
* extras_ptr = (add_extras ? &extras : nullptr); if (find_archive_data(stub_id)) { address start = nullptr; address end = nullptr; From 351712a423f2b6dd509c8196c9e2f13896119792 Mon Sep 17 00:00:00 2001 From: Andrew Dinn Date: Wed, 19 Nov 2025 17:12:15 +0000 Subject: [PATCH 13/30] update zero port with required changes --- src/hotspot/cpu/zero/stubGenerator_zero.cpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/hotspot/cpu/zero/stubGenerator_zero.cpp b/src/hotspot/cpu/zero/stubGenerator_zero.cpp index 08cb173b50709..d46da4b3723ab 100644 --- a/src/hotspot/cpu/zero/stubGenerator_zero.cpp +++ b/src/hotspot/cpu/zero/stubGenerator_zero.cpp @@ -213,7 +213,7 @@ class StubGenerator: public StubCodeGenerator { } public: - StubGenerator(CodeBuffer* code, BlobId blob_id) : StubCodeGenerator(code, blob_id) { + StubGenerator(CodeBuffer* code, BlobId blob_id, AOTStubData *stub_data) : StubCodeGenerator(code, blob_id, stub_data) { switch(blob_id) { case BlobId::stubgen_preuniverse_id: generate_preuniverse_stubs(); @@ -237,8 +237,12 @@ class StubGenerator: public StubCodeGenerator { } }; -void StubGenerator_generate(CodeBuffer* code, BlobId blob_id) { - StubGenerator g(code, blob_id); +// nothing to do for ZERO +void StubGenerator_AOTAddressTable_init() { +} + +void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData *stub_data) { + StubGenerator g(code, blob_id, stub_data); } EntryFrame *EntryFrame::build(const intptr_t* parameters, From 63d4d1c62217d73f5127e8df426c71d390026438 Mon Sep 17 00:00:00 2001 From: Andrew Dinn Date: Wed, 19 Nov 2025 17:25:04 +0000 Subject: [PATCH 14/30] update arm/ppc/riscv/s390/ ports with required changes --- src/hotspot/cpu/arm/stubGenerator_arm.cpp | 10 +++++++--- src/hotspot/cpu/ppc/stubGenerator_ppc.cpp | 10 +++++++--- src/hotspot/cpu/riscv/stubGenerator_riscv.cpp | 10 +++++++--- src/hotspot/cpu/s390/stubGenerator_s390.cpp | 10 +++++++--- 4 files changed, 28 insertions(+), 12 deletions(-) diff --git a/src/hotspot/cpu/arm/stubGenerator_arm.cpp b/src/hotspot/cpu/arm/stubGenerator_arm.cpp index a36ad3a0c4701..c6a30fa837dc5 100644 --- a/src/hotspot/cpu/arm/stubGenerator_arm.cpp +++ b/src/hotspot/cpu/arm/stubGenerator_arm.cpp @@ -3211,7 +3211,7 @@ class StubGenerator: public StubCodeGenerator { } public: - StubGenerator(CodeBuffer* code, BlobId blob_id) : StubCodeGenerator(code, blob_id) { + StubGenerator(CodeBuffer* code, BlobId blob_id, AOTStubDat a*stub_data) : StubCodeGenerator(code, blob_id, stub_data) { switch(blob_id) { case BlobId::stubgen_preuniverse_id: generate_preuniverse_stubs(); @@ -3235,8 +3235,12 @@ class StubGenerator: public StubCodeGenerator { } }; // end class declaration -void StubGenerator_generate(CodeBuffer* code, BlobId blob_id) { - StubGenerator g(code, blob_id); +// nothing to do for arm +void StubGenerator_AOTAddressTable_init() { +} + +void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) { + StubGenerator g(code, blob_id, stub_data); } // implementation of internal development flag diff --git a/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp b/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp index 948092bbb9aad..2156f742c3a10 100644 --- a/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp +++ b/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp @@ -5095,7 +5095,7 @@ void generate_lookup_secondary_supers_table_stub() { } public: - StubGenerator(CodeBuffer* code, BlobId blob_id) : StubCodeGenerator(code, blob_id) { + StubGenerator(CodeBuffer* code, BlobId blob_id, AOTStubData *stub_data) : StubCodeGenerator(code, blob_id, stub_data) { switch(blob_id) { case BlobId::stubgen_preuniverse_id: generate_preuniverse_stubs(); @@ -5119,7 +5119,11 @@ void generate_lookup_secondary_supers_table_stub() { } }; -void StubGenerator_generate(CodeBuffer* code, BlobId blob_id) { - StubGenerator g(code, blob_id); +// nothing to do for ppc +void StubGenerator_AOTAddressTable_init() { +} + +void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData *stub_data) { + StubGenerator g(code, blob_id, stub_data); } diff --git a/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp b/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp index e5eb15cb8e4ef..e8bd832612a4d 100644 --- a/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp +++ b/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp @@ -6882,7 +6882,7 @@ static const int64_t right_3_bits = right_n_bits(3); } public: - StubGenerator(CodeBuffer* code, BlobId blob_id) : StubCodeGenerator(code, blob_id) { + StubGenerator(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) : StubCodeGenerator(code, blob_id, stub_data) { switch(blob_id) { case BlobId::stubgen_preuniverse_id: generate_preuniverse_stubs(); @@ -6906,6 +6906,10 @@ static const int64_t right_3_bits = right_n_bits(3); } }; // end class declaration -void StubGenerator_generate(CodeBuffer* code, BlobId blob_id) { - StubGenerator g(code, blob_id); +// nothing to do for riscv +void StubGenerator_AOTAddressTable_init() { +} + +void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) { + StubGenerator g(code, blob_id, stub_data); } diff --git a/src/hotspot/cpu/s390/stubGenerator_s390.cpp b/src/hotspot/cpu/s390/stubGenerator_s390.cpp index 2aa365be99906..56ba3a85cbae4 100644 --- a/src/hotspot/cpu/s390/stubGenerator_s390.cpp +++ b/src/hotspot/cpu/s390/stubGenerator_s390.cpp @@ -3422,7 +3422,7 @@ class StubGenerator: public StubCodeGenerator { } public: - StubGenerator(CodeBuffer* code, BlobId blob_id) : StubCodeGenerator(code, blob_id) { + StubGenerator(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) : StubCodeGenerator(code, blob_id, stub_data) { switch(blob_id) { case BlobId::stubgen_preuniverse_id: generate_preuniverse_stubs(); @@ -3479,6 +3479,10 @@ class StubGenerator: public StubCodeGenerator { }; -void StubGenerator_generate(CodeBuffer* code, BlobId blob_id) { - StubGenerator g(code, blob_id); +// nothing to do for s390 +void StubGenerator_AOTAddressTable_init() { +} + +void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) { + StubGenerator g(code, blob_id, stub_data); } From 7cf62b2303d9438e72144a96f23b8e484dfeaa99 Mon Sep 17 00:00:00 2001 From: Andrew Dinn Date: Wed, 19 Nov 2025 22:18:50 +0000 Subject: [PATCH 15/30] fix typo --- src/hotspot/cpu/arm/stubGenerator_arm.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/cpu/arm/stubGenerator_arm.cpp b/src/hotspot/cpu/arm/stubGenerator_arm.cpp index c6a30fa837dc5..16155203278e2 100644 --- a/src/hotspot/cpu/arm/stubGenerator_arm.cpp +++ b/src/hotspot/cpu/arm/stubGenerator_arm.cpp @@ -3211,7 +3211,7 @@ class StubGenerator: public StubCodeGenerator { } public: - StubGenerator(CodeBuffer* code, BlobId blob_id, AOTStubDat a*stub_data) : StubCodeGenerator(code, blob_id, stub_data) { + StubGenerator(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) : StubCodeGenerator(code, blob_id, stub_data) { switch(blob_id) { case BlobId::stubgen_preuniverse_id: generate_preuniverse_stubs(); From 042be3fabff3f9658d7bbfe1fb84bad82df91e46 Mon Sep 17 00:00:00 2001 From: Andrew Dinn Date: Thu, 20 Nov 2025 09:37:34 +0000 Subject: [PATCH 16/30] fix format issue --- src/hotspot/share/code/aotCodeCache.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/code/aotCodeCache.cpp b/src/hotspot/share/code/aotCodeCache.cpp index 17c478038cb0b..7fb0464e2b927 100644 --- a/src/hotspot/share/code/aotCodeCache.cpp +++ b/src/hotspot/share/code/aotCodeCache.cpp @@ -1425,7 +1425,7 @@ void AOTCodeCache::publish_stub_addresses(CodeBlob &code_blob, BlobId blob_id, A } else { assert(code_blob.is_deoptimization_stub(), "only expecting one entry for stub %s", StubInfo::name(stub_id)); DeoptimizationBlob *deopt_blob = code_blob.as_deoptimization_blob(); - assert(deopt_blob->unpack() == start, "unexpected offset 0x%lx for deopt stub entry", deopt_blob->unpack() - start); + assert(deopt_blob->unpack() == start, "unexpected offset 0x%x for deopt stub entry", (int)(deopt_blob->unpack() - start)); GrowableArray
addresses; addresses.append(deopt_blob->unpack_with_exception()); addresses.append(deopt_blob->unpack_with_reexecution()); From 9cd737673f1d410e9ca01e2af96d89bdcecbfcbd Mon Sep 17 00:00:00 2001 From: Andrew Dinn Date: Tue, 25 Nov 2025 14:59:19 +0000 Subject: [PATCH 17/30] adjust extry/extra address offset encoding to distinguish end from nullptr --- src/hotspot/share/code/aotCodeCache.cpp | 13 +++++++++---- src/hotspot/share/code/aotCodeCache.hpp | 4 ++++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/hotspot/share/code/aotCodeCache.cpp b/src/hotspot/share/code/aotCodeCache.cpp index 7fb0464e2b927..524a89386c54d 100644 --- a/src/hotspot/share/code/aotCodeCache.cpp +++ b/src/hotspot/share/code/aotCodeCache.cpp @@ -1123,7 +1123,7 @@ bool AOTCodeCache::write_stub_data(CodeBlob &blob, AOTStubData *stub_data) { } else { // this can happen when a stub is not generated or an // extra is the common handler target - offset = (uint)(end - start); + offset = NULL_ADDRESS_MARKER; } n = write_bytes(&offset, sizeof(uint)); if (n != sizeof(int)) { @@ -1355,10 +1355,15 @@ void AOTCodeReader::read_stub_data(CodeBlob* code_blob, AOTStubData* stub_data) // equal to end uint entry = *(uint*)addr(offset); offset += sizeof(uint); assert(entry <= end, "stub %s entry offset %d lies beyond stub end %d", StubInfo::name(stub_id), entry, end); - if (entry < end) { + if (entry <= end) { + // entry addresses may not address end but extras can + assert(entry < end || i >= StubInfo::entry_count(stub_id), + "entry offset 0x%x exceeds stub length 0x%x for stub %s", + entry, end, StubInfo::name(stub_id)); addresses.append(stub_start + entry); } else { - // entry offset == end encodes a nullptr + // special case: entry encodes a nullptr + assert(entry == AOTCodeCache::NULL_ADDRESS_MARKER, "sanity"); addresses.append(nullptr); } } @@ -2405,7 +2410,7 @@ void AOTStubData::load_archive_data(StubId stub_id, address& start, address& end assert(extras->length() == 0, "non-empty array when retrieving extras for stub %s!", StubInfo::name(stub_id)); while (index < count) { address extra = _address_array.at(base + index++); - assert(extra == nullptr || (start <= extra && extra < end), "extra address %p not in range (%p, %p) for stub %s", extra, start, end, StubInfo::name(stub_id)); + assert(extra == nullptr || (start <= extra && extra <= end), "extra address %p not in range (%p, %p) for stub %s", extra, start, end, StubInfo::name(stub_id)); extras->append(extra); } } diff --git a/src/hotspot/share/code/aotCodeCache.hpp b/src/hotspot/share/code/aotCodeCache.hpp index 8b7ecfe088d07..bfe085c0a4e5c 100644 --- a/src/hotspot/share/code/aotCodeCache.hpp +++ b/src/hotspot/share/code/aotCodeCache.hpp @@ -538,6 +538,10 @@ class AOTCodeCache : public CHeapObj { return true; } public: + // marker used where an address offset needs to be stored for later + // retrieval and the address turns out to be null + static const uint NULL_ADDRESS_MARKER = UINT_MAX; + static AOTCodeCache* cache() { assert(_passed_init2, "Too early to ask"); return _cache; } static void initialize() NOT_CDS_RETURN; static void init2() NOT_CDS_RETURN; From d20c218222c7ef41adec5803bc974dda65ebcd32 Mon Sep 17 00:00:00 2001 From: Andrew Dinn Date: Tue, 25 Nov 2025 15:14:13 +0000 Subject: [PATCH 18/30] tighten up unsafe handler address checks --- src/hotspot/share/runtime/stubCodeGenerator.cpp | 13 +++++++------ src/hotspot/share/runtime/stubRoutines.cpp | 3 ++- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/hotspot/share/runtime/stubCodeGenerator.cpp b/src/hotspot/share/runtime/stubCodeGenerator.cpp index 4859a110ca2ce..24f641e53e69f 100644 --- a/src/hotspot/share/runtime/stubCodeGenerator.cpp +++ b/src/hotspot/share/runtime/stubCodeGenerator.cpp @@ -111,20 +111,21 @@ void StubCodeGenerator::setup_code_desc(const char* name, address start, address // Helper used to restore ranges and handler addresses restored from // AOT cache. Expects entries to contain 3 * count addresses beginning // at offset begin which identify start of range, end of range and -// address of handler pc. end of range may be nullptr in which case it -// defaults to stub_end. hanlder pc may be nullptr in which case it -// defaults to default_handler. +// address of handler pc. start and end of range may not be null. +// handler pc may be null in which case it defaults to the +// default_handler. void StubCodeGenerator::register_unsafe_access_handlers(GrowableArray
&entries, int begin, int count, address stub_end) { for (int i = 0; i < count; i++) { int offset = begin + 3 * i; address start = entries.at(offset); address end = entries.at(offset + 1); - if (end == nullptr) { - end = stub_end; - } address handler = entries.at(offset + 2); + assert(start != nullptr, "sanity"); + assert(end != nullptr, "sanity"); if (handler == nullptr) { + assert(UnsafeMemoryAccess::common_exit_stub_pc() != nullptr, + "default unsafe handler must be set before registering unsafe rgeionwiht no handler!"); handler = UnsafeMemoryAccess::common_exit_stub_pc(); } UnsafeMemoryAccess::add_to_table(start, end, handler); diff --git a/src/hotspot/share/runtime/stubRoutines.cpp b/src/hotspot/share/runtime/stubRoutines.cpp index 84d469af5c513..82094b6909452 100644 --- a/src/hotspot/share/runtime/stubRoutines.cpp +++ b/src/hotspot/share/runtime/stubRoutines.cpp @@ -157,7 +157,8 @@ void UnsafeMemoryAccess::collect_entries(address range_start, address range_end, if (e._error_exit_pc != _common_exit_stub_pc) { entries.append(e._error_exit_pc); } else { - // an address outside the stub must be the common exit stub address + // an address outside the stub must be the common exit stub + // address which is marked with a null address entries.append(nullptr); } } From e9442c36e14636fab87126618fb6267ec25ce119 Mon Sep 17 00:00:00 2001 From: Andrew Dinn Date: Wed, 26 Nov 2025 09:42:22 +0000 Subject: [PATCH 19/30] use simpler AOT lookup in all x86 stub generators --- src/hotspot/cpu/x86/stubGenerator_x86_64.cpp | 562 +++++++----------- .../cpu/x86/stubGenerator_x86_64_adler.cpp | 10 +- .../cpu/x86/stubGenerator_x86_64_aes.cpp | 143 ++++- .../x86/stubGenerator_x86_64_arraycopy.cpp | 146 ++--- .../cpu/x86/stubGenerator_x86_64_cbrt.cpp | 11 +- .../cpu/x86/stubGenerator_x86_64_chacha.cpp | 28 +- .../cpu/x86/stubGenerator_x86_64_cos.cpp | 11 +- .../x86/stubGenerator_x86_64_dilithium.cpp | 65 +- .../cpu/x86/stubGenerator_x86_64_exp.cpp | 11 +- .../cpu/x86/stubGenerator_x86_64_fmod.cpp | 13 +- .../cpu/x86/stubGenerator_x86_64_ghash.cpp | 29 +- .../cpu/x86/stubGenerator_x86_64_kyber.cpp | 97 ++- .../cpu/x86/stubGenerator_x86_64_log.cpp | 22 +- .../cpu/x86/stubGenerator_x86_64_poly1305.cpp | 14 +- .../x86/stubGenerator_x86_64_poly_mont.cpp | 28 +- .../cpu/x86/stubGenerator_x86_64_pow.cpp | 11 +- .../cpu/x86/stubGenerator_x86_64_sha3.cpp | 25 +- .../cpu/x86/stubGenerator_x86_64_sin.cpp | 11 +- .../cpu/x86/stubGenerator_x86_64_sinh.cpp | 11 +- .../cpu/x86/stubGenerator_x86_64_tan.cpp | 11 +- .../cpu/x86/stubGenerator_x86_64_tanh.cpp | 11 +- src/hotspot/share/code/aotCodeCache.cpp | 36 +- src/hotspot/share/code/aotCodeCache.hpp | 39 +- .../share/runtime/stubCodeGenerator.cpp | 21 +- .../share/runtime/stubCodeGenerator.hpp | 49 +- 25 files changed, 812 insertions(+), 603 deletions(-) diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp index e04729f070722..af583cf860471 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp @@ -194,18 +194,15 @@ address StubGenerator::generate_call_stub(address& return_address) { GrowableArray
entries; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 2, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - // this stub has an extra return entry which we need to retrieve - load_archive_data(stub_id, start, end, &entries); + address start = load_archive_data(stub_id, &entries); + if (start != nullptr) { assert(entries.length() == 1, "expected 1 extra entry"); return_address = entries.at(0); return start; } StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); // same as in generate_catch_exception()! const Address rsp_after_call(rbp, rsp_after_call_off * wordSize); @@ -433,15 +430,13 @@ address StubGenerator::generate_catch_exception() { StubId stub_id = StubId::stubgen_catch_exception_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); // same as in generate_call_stub(): const Address rsp_after_call(rbp, rsp_after_call_off * wordSize); @@ -503,14 +498,12 @@ address StubGenerator::generate_forward_exception() { StubId stub_id = StubId::stubgen_forward_exception_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); // Upon entry, the sp points to the return address returning into // Java (interpreted or compiled) code; i.e., the return address @@ -578,14 +571,12 @@ address StubGenerator::generate_orderaccess_fence() { StubId stub_id = StubId::stubgen_fence_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ membar(Assembler::StoreLoad); __ ret(0); @@ -608,14 +599,12 @@ address StubGenerator::generate_verify_mxcsr() { StubId stub_id = StubId::stubgen_verify_mxcsr_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); const Address mxcsr_save(rsp, 0); @@ -648,16 +637,14 @@ address StubGenerator::generate_f2i_fixup() { StubId stub_id = StubId::stubgen_f2i_fixup_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } StubCodeMark mark(this, stub_id); Address inout(rsp, 5 * wordSize); // return address + 4 saves - address start = __ pc(); + start = __ pc(); Label L; @@ -698,15 +685,13 @@ address StubGenerator::generate_f2l_fixup() { StubId stub_id = StubId::stubgen_f2l_fixup_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } StubCodeMark mark(this, stub_id); Address inout(rsp, 5 * wordSize); // return address + 4 saves - address start = __ pc(); + start = __ pc(); Label L; @@ -747,16 +732,14 @@ address StubGenerator::generate_d2i_fixup() { StubId stub_id = StubId::stubgen_d2i_fixup_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } StubCodeMark mark(this, stub_id); Address inout(rsp, 6 * wordSize); // return address + 5 saves - address start = __ pc(); + start = __ pc(); Label L; @@ -806,16 +789,14 @@ address StubGenerator::generate_d2l_fixup() { StubId stub_id = StubId::stubgen_d2l_fixup_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } StubCodeMark mark(this, stub_id); Address inout(rsp, 6 * wordSize); // return address + 5 saves - address start = __ pc(); + start = __ pc(); Label L; @@ -865,15 +846,13 @@ address StubGenerator::generate_count_leading_zeros_lut() { StubId stub_id = StubId::stubgen_vector_count_leading_zeros_lut_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align64(); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ emit_data64(0x0101010102020304, relocInfo::none); __ emit_data64(0x0000000000000000, relocInfo::none); @@ -894,15 +873,13 @@ address StubGenerator::generate_popcount_avx_lut() { StubId stub_id = StubId::stubgen_vector_popcount_lut_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align64(); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ emit_data64(0x0302020102010100, relocInfo::none); __ emit_data64(0x0403030203020201, relocInfo::none); @@ -923,15 +900,13 @@ address StubGenerator::generate_iota_indices() { StubId stub_id = StubId::stubgen_vector_iota_indices_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); // B __ emit_data64(0x0706050403020100, relocInfo::none); __ emit_data64(0x0F0E0D0C0B0A0908, relocInfo::none); @@ -997,15 +972,13 @@ address StubGenerator::generate_vector_reverse_bit_lut() { StubId stub_id = StubId::stubgen_vector_reverse_bit_lut_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ emit_data64(0x0E060A020C040800, relocInfo::none); __ emit_data64(0x0F070B030D050901, relocInfo::none); @@ -1026,15 +999,13 @@ address StubGenerator::generate_vector_reverse_byte_perm_mask_long() { StubId stub_id = StubId::stubgen_vector_reverse_byte_perm_mask_long_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ emit_data64(0x0001020304050607, relocInfo::none); __ emit_data64(0x08090A0B0C0D0E0F, relocInfo::none); @@ -1055,15 +1026,13 @@ address StubGenerator::generate_vector_reverse_byte_perm_mask_int() { StubId stub_id = StubId::stubgen_vector_reverse_byte_perm_mask_int_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ emit_data64(0x0405060700010203, relocInfo::none); __ emit_data64(0x0C0D0E0F08090A0B, relocInfo::none); @@ -1084,15 +1053,13 @@ address StubGenerator::generate_vector_reverse_byte_perm_mask_short() { StubId stub_id = StubId::stubgen_vector_reverse_byte_perm_mask_short_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ emit_data64(0x0607040502030001, relocInfo::none); __ emit_data64(0x0E0F0C0D0A0B0809, relocInfo::none); @@ -1113,15 +1080,13 @@ address StubGenerator::generate_vector_byte_shuffle_mask() { StubId stub_id = StubId::stubgen_vector_byte_shuffle_mask_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ emit_data64(0x7070707070707070, relocInfo::none); __ emit_data64(0x7070707070707070, relocInfo::none); @@ -1137,15 +1102,13 @@ address StubGenerator::generate_vector_byte_shuffle_mask() { address StubGenerator::generate_fp_mask(StubId stub_id, int64_t mask) { int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ emit_data64( mask, relocInfo::none ); __ emit_data64( mask, relocInfo::none ); @@ -1170,15 +1133,13 @@ address StubGenerator::generate_compress_perm_table(StubId stub_id) { } int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); if (esize == 32) { // Loop to generate 256 x 8 int compression permute index table. A row is // accessed using 8 bit index computed using vector mask. An entry in @@ -1236,15 +1197,13 @@ address StubGenerator::generate_expand_perm_table(StubId stub_id) { } int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); if (esize == 32) { // Loop to generate 256 x 8 int expand permute index table. A row is accessed // using 8 bit index computed using vector mask. An entry in a row holds either @@ -1289,15 +1248,13 @@ address StubGenerator::generate_expand_perm_table(StubId stub_id) { address StubGenerator::generate_vector_mask(StubId stub_id, int64_t mask) { int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ emit_data64(mask, relocInfo::none); __ emit_data64(mask, relocInfo::none); @@ -1318,15 +1275,13 @@ address StubGenerator::generate_vector_byte_perm_mask() { StubId stub_id = StubId::stubgen_vector_byte_perm_mask_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ emit_data64(0x0000000000000001, relocInfo::none); __ emit_data64(0x0000000000000003, relocInfo::none); @@ -1346,15 +1301,13 @@ address StubGenerator::generate_vector_byte_perm_mask() { address StubGenerator::generate_vector_fp_mask(StubId stub_id, int64_t mask) { int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ emit_data64(mask, relocInfo::none); __ emit_data64(mask, relocInfo::none); @@ -1378,15 +1331,13 @@ address StubGenerator::generate_vector_custom_i32(StubId stub_id, Assembler::Avx int32_t val12, int32_t val13, int32_t val14, int32_t val15) { int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); assert(len != Assembler::AVX_NoVec, "vector len must be specified"); __ emit_data(val0, relocInfo::none, 0); @@ -1435,14 +1386,12 @@ address StubGenerator::generate_verify_oop() { StubId stub_id = StubId::stubgen_verify_oop_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); Label exit, error; @@ -1641,16 +1590,14 @@ address StubGenerator::generate_data_cache_writeback() { StubId stub_id = StubId::stubgen_data_cache_writeback_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ enter(); __ cache_wb(Address(src, 0)); @@ -1668,10 +1615,8 @@ address StubGenerator::generate_data_cache_writeback_sync() { StubId stub_id = StubId::stubgen_data_cache_writeback_sync_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align(CodeEntryAlignment); @@ -1681,7 +1626,7 @@ address StubGenerator::generate_data_cache_writeback_sync() { // post wbsync translates to an sfence Label skip; - address start = __ pc(); + start = __ pc(); __ enter(); __ cmpl(is_pre, 0); @@ -1713,15 +1658,13 @@ address StubGenerator::generate_md5_implCompress(StubId stub_id) { } int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); const Register buf_param = r15; const Address state_param(rsp, 0 * wordSize); @@ -1761,15 +1704,13 @@ address StubGenerator::generate_upper_word_mask() { StubId stub_id = StubId::stubgen_upper_word_mask_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align64(); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ emit_data64(0x0000000000000000, relocInfo::none); __ emit_data64(0xFFFFFFFF00000000, relocInfo::none); @@ -1784,15 +1725,13 @@ address StubGenerator::generate_shuffle_byte_flip_mask() { StubId stub_id = StubId::stubgen_shuffle_byte_flip_mask_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align64(); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ emit_data64(0x08090a0b0c0d0e0f, relocInfo::none); __ emit_data64(0x0001020304050607, relocInfo::none); @@ -1819,15 +1758,13 @@ address StubGenerator::generate_sha1_implCompress(StubId stub_id) { } int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); Register buf = c_rarg0; Register state = c_rarg1; @@ -1867,10 +1804,8 @@ address StubGenerator::generate_pshuffle_byte_flip_mask(address& entry_00ba, add GrowableArray
entries; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 3, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end, &entries); + address start = load_archive_data(stub_id, &entries); + if (start != nullptr) { assert(entries.length() == entry_count - 1, "unexpected extra entry count %d", entries.length()); entry_00ba = entries.at(0); @@ -1881,7 +1816,7 @@ address StubGenerator::generate_pshuffle_byte_flip_mask(address& entry_00ba, add } __ align64(); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); address entry2 = nullptr; address entry3 = nullptr; __ emit_data64(0x0405060700010203, relocInfo::none); @@ -1921,10 +1856,8 @@ address StubGenerator::generate_pshuffle_byte_flip_mask_sha512(address& entry_ym GrowableArray
entries; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 2, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end, &entries); + address start = load_archive_data(stub_id, &entries); + if (start != nullptr) { assert(entries.length() == entry_count - 1, "unexpected extra entry count %d", entries.length()); entry_ymm_lo = entries.at(0); @@ -1934,7 +1867,7 @@ address StubGenerator::generate_pshuffle_byte_flip_mask_sha512(address& entry_ym } __ align32(); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); address entry2 = nullptr; if (VM_Version::supports_avx2()) { __ emit_data64(0x0001020304050607, relocInfo::none); // PSHUFFLE_BYTE_FLIP_MASK @@ -1974,15 +1907,13 @@ address StubGenerator::generate_sha256_implCompress(StubId stub_id) { assert(VM_Version::supports_sha() || VM_Version::supports_avx2(), ""); int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); Register buf = c_rarg0; Register state = c_rarg1; @@ -2039,15 +1970,13 @@ address StubGenerator::generate_sha512_implCompress(StubId stub_id) { assert(VM_Version::supports_bmi2() || VM_Version::supports_sha512(), ""); int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); Register buf = c_rarg0; Register state = c_rarg1; @@ -2086,15 +2015,13 @@ address StubGenerator::base64_shuffle_addr() { StubId stub_id = StubId::stubgen_shuffle_base64_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align64(); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); assert(((unsigned long long)start & 0x3f) == 0, "Alignment problem (0x%08llx)", (unsigned long long)start); @@ -2117,15 +2044,13 @@ address StubGenerator::base64_avx2_shuffle_addr() { StubId stub_id = StubId::stubgen_avx2_shuffle_base64_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align32(); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ emit_data64(0x0809070805060405, relocInfo::none); __ emit_data64(0x0e0f0d0e0b0c0a0b, relocInfo::none); @@ -2142,15 +2067,13 @@ address StubGenerator::base64_avx2_input_mask_addr() { StubId stub_id = StubId::stubgen_avx2_input_mask_base64_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align32(); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ emit_data64(0x8000000000000000, relocInfo::none); __ emit_data64(0x8000000080000000, relocInfo::none); @@ -2167,15 +2090,13 @@ address StubGenerator::base64_avx2_lut_addr() { StubId stub_id = StubId::stubgen_avx2_lut_base64_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align32(); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ emit_data64(0xfcfcfcfcfcfc4741, relocInfo::none); __ emit_data64(0x0000f0edfcfcfcfc, relocInfo::none); @@ -2198,15 +2119,13 @@ address StubGenerator::base64_encoding_table_addr() { StubId stub_id = StubId::stubgen_encoding_table_base64_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align64(); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); assert(((unsigned long long)start & 0x3f) == 0, "Alignment problem (0x%08llx)", (unsigned long long)start); __ emit_data64(0x4847464544434241, relocInfo::none); @@ -2243,15 +2162,13 @@ address StubGenerator::generate_base64_encodeBlock() StubId stub_id = StubId::stubgen_base64_encodeBlock_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ enter(); @@ -2637,15 +2554,13 @@ address StubGenerator::base64_vbmi_lookup_lo_addr() { StubId stub_id = StubId::stubgen_lookup_lo_base64_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align64(); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); assert(((unsigned long long)start & 0x3f) == 0, "Alignment problem (0x%08llx)", (unsigned long long)start); @@ -2668,15 +2583,13 @@ address StubGenerator::base64_vbmi_lookup_hi_addr() { StubId stub_id = StubId::stubgen_lookup_hi_base64_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align64(); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); assert(((unsigned long long)start & 0x3f) == 0, "Alignment problem (0x%08llx)", (unsigned long long)start); @@ -2698,15 +2611,13 @@ address StubGenerator::base64_vbmi_lookup_lo_url_addr() { StubId stub_id = StubId::stubgen_lookup_lo_base64url_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align64(); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); assert(((unsigned long long)start & 0x3f) == 0, "Alignment problem (0x%08llx)", (unsigned long long)start); @@ -2729,15 +2640,13 @@ address StubGenerator::base64_vbmi_lookup_hi_url_addr() { StubId stub_id = StubId::stubgen_lookup_hi_base64url_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align64(); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); assert(((unsigned long long)start & 0x3f) == 0, "Alignment problem (0x%08llx)", (unsigned long long)start); @@ -2760,15 +2669,13 @@ address StubGenerator::base64_vbmi_pack_vec_addr() { StubId stub_id = StubId::stubgen_pack_vec_base64_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align64(); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); assert(((unsigned long long)start & 0x3f) == 0, "Alignment problem (0x%08llx)", (unsigned long long)start); @@ -2791,15 +2698,13 @@ address StubGenerator::base64_vbmi_join_0_1_addr() { StubId stub_id = StubId::stubgen_join_0_1_base64_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align64(); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); assert(((unsigned long long)start & 0x3f) == 0, "Alignment problem (0x%08llx)", (unsigned long long)start); @@ -2822,15 +2727,13 @@ address StubGenerator::base64_vbmi_join_1_2_addr() { StubId stub_id = StubId::stubgen_join_1_2_base64_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align64(); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); assert(((unsigned long long)start & 0x3f) == 0, "Alignment problem (0x%08llx)", (unsigned long long)start); @@ -2853,15 +2756,13 @@ address StubGenerator::base64_vbmi_join_2_3_addr() { StubId stub_id = StubId::stubgen_join_2_3_base64_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align64(); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); assert(((unsigned long long)start & 0x3f) == 0, "Alignment problem (0x%08llx)", (unsigned long long)start); @@ -2884,15 +2785,13 @@ address StubGenerator::base64_AVX2_decode_tables_addr() { StubId stub_id = StubId::stubgen_avx2_decode_tables_base64_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align64(); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); assert(((unsigned long long)start & 0x3f) == 0, "Alignment problem (0x%08llx)", (unsigned long long)start); @@ -2930,15 +2829,13 @@ address StubGenerator::base64_AVX2_decode_LUT_tables_addr() { StubId stub_id = StubId::stubgen_avx2_decode_lut_tables_base64_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align64(); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); assert(((unsigned long long)start & 0x3f) == 0, "Alignment problem (0x%08llx)", (unsigned long long)start); @@ -2982,14 +2879,12 @@ address StubGenerator::base64_decoding_table_addr() { StubId stub_id = StubId::stubgen_decoding_table_base64_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ emit_data64(0xffffffffffffffff, relocInfo::none); __ emit_data64(0xffffffffffffffff, relocInfo::none); @@ -3075,15 +2970,13 @@ address StubGenerator::generate_base64_decodeBlock() { StubId stub_id = StubId::stubgen_base64_decodeBlock_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ enter(); @@ -3620,16 +3513,14 @@ address StubGenerator::generate_updateBytesCRC32() { StubId stub_id = StubId::stubgen_updateBytesCRC32_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...) // Unix: rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...) @@ -3688,15 +3579,13 @@ address StubGenerator::generate_updateBytesCRC32C(bool is_pclmulqdq_supported) { StubId stub_id = StubId::stubgen_updateBytesCRC32C_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); //reg.arg int#0 int#1 int#2 int#3 int#4 int#5 float regs //Windows RCX RDX R8 R9 none none XMM0..XMM3 @@ -3780,15 +3669,13 @@ address StubGenerator::generate_multiplyToLen() { StubId stub_id = StubId::stubgen_multiplyToLen_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...) // Unix: rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...) @@ -3848,15 +3735,13 @@ address StubGenerator::generate_vectorizedMismatch() { StubId stub_id = StubId::stubgen_vectorizedMismatch_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); BLOCK_COMMENT("Entry:"); __ enter(); @@ -3911,15 +3796,13 @@ address StubGenerator::generate_squareToLen() { StubId stub_id = StubId::stubgen_squareToLen_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...) // Unix: rdi, rsi, rdx, rcx (c_rarg0, c_rarg1, ...) @@ -3958,15 +3841,13 @@ address StubGenerator::generate_method_entry_barrier() { StubId stub_id = StubId::stubgen_method_entry_barrier_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); Label deoptimize_label; @@ -4059,15 +3940,13 @@ address StubGenerator::generate_mulAdd() { StubId stub_id = StubId::stubgen_mulAdd_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...) // Unix: rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...) @@ -4112,15 +3991,13 @@ address StubGenerator::generate_bigIntegerRightShift() { StubId stub_id = StubId::stubgen_bigIntegerRightShiftWorker_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); Label Shift512Loop, ShiftTwo, ShiftTwoLoop, ShiftOne, Exit; // For Unix, the arguments are as follows: rdi, rsi, rdx, rcx, r8. @@ -4259,15 +4136,13 @@ address StubGenerator::generate_bigIntegerLeftShift() { StubId stub_id = StubId::stubgen_bigIntegerLeftShiftWorker_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); Label Shift512Loop, ShiftTwo, ShiftTwoLoop, ShiftOne, Exit; // For Unix, the arguments are as follows: rdi, rsi, rdx, rcx, r8. @@ -4426,15 +4301,13 @@ address StubGenerator::generate_float16ToFloat() { StubId stub_id = StubId::stubgen_hf2f_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); BLOCK_COMMENT("Entry:"); // No need for RuntimeStub frame since it is called only during JIT compilation @@ -4463,15 +4336,13 @@ address StubGenerator::generate_floatToFloat16() { StubId stub_id = StubId::stubgen_f2hf_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); BLOCK_COMMENT("Entry:"); // No need for RuntimeStub frame since it is called only during JIT compilation @@ -4515,14 +4386,12 @@ address StubGenerator::generate_cont_thaw(StubId stub_id) { } int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); // TODO: Handle Valhalla return types. May require generating different return barriers. @@ -4660,14 +4529,12 @@ address StubGenerator::generate_cont_preempt_stub() { StubId stub_id = StubId::stubgen_cont_preempt_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ reset_last_Java_frame(true); @@ -4702,14 +4569,12 @@ address StubGenerator::generate_upcall_stub_exception_handler() { StubId stub_id = StubId::stubgen_upcall_stub_exception_handler_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); // native caller has no idea how to handle exceptions // we just crash here. Up to callee to catch exceptions. @@ -4734,14 +4599,12 @@ address StubGenerator::generate_upcall_stub_load_target() { StubId stub_id = StubId::stubgen_upcall_stub_load_target_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ resolve_global_jobject(j_rarg0, rscratch1); // Load target method from receiver @@ -4763,10 +4626,8 @@ void StubGenerator::generate_lookup_secondary_supers_table_stub() { GrowableArray
entries; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == Klass::SECONDARY_SUPERS_TABLE_SIZE, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end, &entries); + address start = load_archive_data(stub_id, &entries); + if (start != nullptr) { assert(entries.length() == Klass::SECONDARY_SUPERS_TABLE_SIZE - 1, "unexpected extra entry count %d", entries.length()); StubRoutines::_lookup_secondary_supers_table_stubs[0] = start; @@ -4782,7 +4643,6 @@ void StubGenerator::generate_lookup_secondary_supers_table_stub() { r_sub_klass = rsi, result = rdi; - address start; for (int slot = 0; slot < Klass::SECONDARY_SUPERS_TABLE_SIZE; slot++) { address next_entry = __ pc(); if (slot == 0) { @@ -4807,16 +4667,12 @@ address StubGenerator::generate_lookup_secondary_supers_table_slow_path_stub() { StubId stub_id = StubId::stubgen_lookup_secondary_supers_table_slow_path_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - StubCodeMark mark(this, stub_id); - - address start = __ pc(); + start = __ pc(); const Register r_super_klass = rax, diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_adler.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_adler.cpp index c5aa877b4fcf7..66928c6e92893 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_adler.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_adler.cpp @@ -67,8 +67,14 @@ address StubGenerator::generate_updateBytesAdler32() { __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_updateBytesAdler32_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + address start = load_archive_data(stub_id); + if (start != nullptr) { + return start; + } StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); // Choose an appropriate LIMIT for inner loop based on the granularity // of intermediate results. For int, LIMIT of 5552 will ensure intermediate @@ -334,6 +340,8 @@ address StubGenerator::generate_updateBytesAdler32() { __ leave(); __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); return start; } diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_aes.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_aes.cpp index bb991e30b76af..e7572126a27f7 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_aes.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_aes.cpp @@ -248,10 +248,16 @@ void StubGenerator::generate_aes_stubs() { // Output: // rax - number of processed bytes address StubGenerator::generate_galoisCounterMode_AESCrypt() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_galoisCounterMode_AESCrypt_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + address start = load_archive_data(stub_id); + if (start != nullptr) { + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); const Register in = c_rarg0; const Register len = c_rarg1; @@ -317,6 +323,9 @@ address StubGenerator::generate_galoisCounterMode_AESCrypt() { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -335,10 +344,16 @@ address StubGenerator::generate_galoisCounterMode_AESCrypt() { // Output: // rax - number of processed bytes address StubGenerator::generate_avx2_galoisCounterMode_AESCrypt() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_galoisCounterMode_AESCrypt_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + address start = load_archive_data(stub_id); + if (start != nullptr) { + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); const Register in = c_rarg0; const Register len = c_rarg1; @@ -402,15 +417,24 @@ address StubGenerator::generate_avx2_galoisCounterMode_AESCrypt() { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } // Vector AES Counter implementation address StubGenerator::generate_counterMode_VectorAESCrypt() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_counterMode_AESCrypt_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + address start = load_archive_data(stub_id); + if (start != nullptr) { + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); const Register from = c_rarg0; // source array address const Register to = c_rarg1; // destination array address @@ -469,6 +493,9 @@ address StubGenerator::generate_counterMode_VectorAESCrypt() { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -496,10 +523,16 @@ address StubGenerator::generate_counterMode_VectorAESCrypt() { // address StubGenerator::generate_counterMode_AESCrypt_Parallel() { assert(UseAES, "need AES instructions and misaligned SSE support"); - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_counterMode_AESCrypt_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + address start = load_archive_data(stub_id); + if (start != nullptr) { + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); const Register from = c_rarg0; // source array address const Register to = c_rarg1; // destination array address @@ -779,15 +812,24 @@ address StubGenerator::generate_counterMode_AESCrypt_Parallel() { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::generate_cipherBlockChaining_decryptVectorAESCrypt() { assert(VM_Version::supports_avx512_vaes(), "need AES instructions and misaligned SSE support"); - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_cipherBlockChaining_decryptAESCrypt_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + address start = load_archive_data(stub_id); + if (start != nullptr) { + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); const Register from = c_rarg0; // source array address const Register to = c_rarg1; // destination array address @@ -1055,6 +1097,9 @@ address StubGenerator::generate_cipherBlockChaining_decryptVectorAESCrypt() { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -1067,11 +1112,17 @@ address StubGenerator::generate_cipherBlockChaining_decryptVectorAESCrypt() { // address StubGenerator::generate_aescrypt_encryptBlock() { assert(UseAES, "need AES instructions and misaligned SSE support"); - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_aescrypt_encryptBlock_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + address start = load_archive_data(stub_id); + if (start != nullptr) { + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); Label L_doLast; - address start = __ pc(); + start = __ pc(); const Register from = c_rarg0; // source array address const Register to = c_rarg1; // destination array address @@ -1150,6 +1201,9 @@ address StubGenerator::generate_aescrypt_encryptBlock() { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -1162,11 +1216,17 @@ address StubGenerator::generate_aescrypt_encryptBlock() { // address StubGenerator::generate_aescrypt_decryptBlock() { assert(UseAES, "need AES instructions and misaligned SSE support"); - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_aescrypt_decryptBlock_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + address start = load_archive_data(stub_id); + if (start != nullptr) { + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); Label L_doLast; - address start = __ pc(); + start = __ pc(); const Register from = c_rarg0; // source array address const Register to = c_rarg1; // destination array address @@ -1246,6 +1306,9 @@ address StubGenerator::generate_aescrypt_decryptBlock() { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -1264,10 +1327,16 @@ address StubGenerator::generate_aescrypt_decryptBlock() { // address StubGenerator::generate_cipherBlockChaining_encryptAESCrypt() { assert(UseAES, "need AES instructions and misaligned SSE support"); - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_cipherBlockChaining_encryptAESCrypt_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + address start = load_archive_data(stub_id); + if (start != nullptr) { + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); Label L_exit, L_key_192_256, L_key_256, L_loopTop_128, L_loopTop_192, L_loopTop_256; const Register from = c_rarg0; // source array address @@ -1396,6 +1465,9 @@ address StubGenerator::generate_cipherBlockChaining_encryptAESCrypt() { __ jcc(Assembler::notEqual, L_loopTop_256); __ jmp(L_exit); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -1416,10 +1488,16 @@ address StubGenerator::generate_cipherBlockChaining_encryptAESCrypt() { // address StubGenerator::generate_cipherBlockChaining_decryptAESCrypt_Parallel() { assert(UseAES, "need AES instructions and misaligned SSE support"); - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_cipherBlockChaining_decryptAESCrypt_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + address start = load_archive_data(stub_id); + if (start != nullptr) { + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); const Register from = c_rarg0; // source array address const Register to = c_rarg1; // destination array address @@ -1655,14 +1733,23 @@ __ opc(xmm_result3, src_reg); \ __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::generate_electronicCodeBook_encryptAESCrypt() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_electronicCodeBook_encryptAESCrypt_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + address start = load_archive_data(stub_id); + if (start != nullptr) { + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); const Register from = c_rarg0; // source array address const Register to = c_rarg1; // destination array address @@ -1676,14 +1763,23 @@ address StubGenerator::generate_electronicCodeBook_encryptAESCrypt() { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::generate_electronicCodeBook_decryptAESCrypt() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_electronicCodeBook_decryptAESCrypt_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + address start = load_archive_data(stub_id); + if (start != nullptr) { + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); const Register from = c_rarg0; // source array address const Register to = c_rarg1; // destination array address @@ -1697,6 +1793,9 @@ address StubGenerator::generate_electronicCodeBook_decryptAESCrypt() { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_arraycopy.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_arraycopy.cpp index c9649fbfa3933..44249b57b9974 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_arraycopy.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_arraycopy.cpp @@ -579,10 +579,8 @@ address StubGenerator::generate_disjoint_copy_avx3_masked(StubId stub_id, addres assert(entry_count == expected_entry_count, "sanity check"); GrowableArray
* entries_ptr = (entry_count == 1 ? nullptr : &entries); GrowableArray
* extras_ptr = (add_extras ? &extras : nullptr); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end, entries_ptr, extras_ptr); + address start = load_archive_data(stub_id, entries_ptr, extras_ptr); + if (start != nullptr) { assert(entries.length() == expected_entry_count - 1, "unexpected extra entry count %d", entries.length()); assert(extras.length() == expected_extra_count, @@ -592,14 +590,14 @@ address StubGenerator::generate_disjoint_copy_avx3_masked(StubId stub_id, addres } if (add_extras) { // restore 1 x UMAM {start,end,handler} addresses from extras - register_unsafe_access_handlers(extras, 0, 1, end); + register_unsafe_access_handlers(extras, 0, 1); } return start; } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); int avx3threshold = VM_Version::avx3_threshold(); bool use64byteVector = (MaxVectorSize > 32) && (avx3threshold == 0); @@ -956,10 +954,8 @@ address StubGenerator::generate_conjoint_copy_avx3_masked(StubId stub_id, addres assert(entry_count == expected_entry_count, "sanity check"); GrowableArray
* entries_ptr = (entry_count == 1 ? nullptr : &entries); GrowableArray
* extras_ptr = (add_extras ? &extras : nullptr); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end, entries_ptr, extras_ptr); + address start = load_archive_data(stub_id, entries_ptr, extras_ptr); + if (start != nullptr) { assert(entries.length() == expected_entry_count - 1, "unexpected extra entry count %d", entries.length()); assert(extras.length() == expected_extra_count, @@ -969,13 +965,13 @@ address StubGenerator::generate_conjoint_copy_avx3_masked(StubId stub_id, addres } if (add_extras) { // restore 1 x UMAM {start,end,handler} addresses from extras - register_unsafe_access_handlers(extras, 0, 1, end); + register_unsafe_access_handlers(extras, 0, 1); } return start; } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); int avx3threshold = VM_Version::avx3_threshold(); bool use64byteVector = (MaxVectorSize > 32) && (avx3threshold == 0); @@ -1471,10 +1467,8 @@ address StubGenerator::generate_disjoint_byte_copy(address* entry) { int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == expected_entry_count, "sanity check"); GrowableArray
* entries_ptr = (entry_count == 1 ? nullptr : &entries); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end, entries_ptr, &extras); + address start = load_archive_data(stub_id, entries_ptr, &extras); + if (start != nullptr) { assert(entries.length() == expected_entry_count - 1, "unexpected extra entry count %d", entries.length()); assert(extras.length() == expected_extra_count, @@ -1483,12 +1477,12 @@ address StubGenerator::generate_disjoint_byte_copy(address* entry) { *entry = entries.at(0); } // restore 2 UMAM {start,end,handler} addresses from extras - register_unsafe_access_handlers(extras, 0, 2, end); + register_unsafe_access_handlers(extras, 0, 2); return start; } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_DISJOINT; Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes, L_copy_2_bytes; @@ -1623,10 +1617,8 @@ address StubGenerator::generate_conjoint_byte_copy(address nooverlap_target, add int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == expected_entry_count, "sanity check"); GrowableArray
* entries_ptr = (entry_count == 1 ? nullptr : &entries); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end, entries_ptr, &extras); + address start = load_archive_data(stub_id, entries_ptr, &extras); + if (start != nullptr) { assert(entries.length() == expected_entry_count - 1, "unexpected extra entry count %d", entries.length()); assert(extras.length() == expected_extra_count, @@ -1635,12 +1627,12 @@ address StubGenerator::generate_conjoint_byte_copy(address nooverlap_target, add *entry = entries.at(0); } // restore 2 UMAM {start,end,handler} addresses from extras - register_unsafe_access_handlers(extras, 0, 2, end); + register_unsafe_access_handlers(extras, 0, 2); return start; } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); DecoratorSet decorators = IN_HEAP | IS_ARRAY; Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes, L_copy_2_bytes; @@ -1769,10 +1761,8 @@ address StubGenerator::generate_disjoint_short_copy(address *entry) { int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == expected_entry_count, "sanity check"); GrowableArray
* entries_ptr = (entry_count == 1 ? nullptr : &entries); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end, entries_ptr, &extras); + address start = load_archive_data(stub_id, entries_ptr, &extras); + if (start != nullptr) { assert(entries.length() == expected_entry_count - 1, "unexpected extra entry count %d", entries.length()); assert(extras.length() == expected_extra_count, @@ -1781,12 +1771,12 @@ address StubGenerator::generate_disjoint_short_copy(address *entry) { *entry = entries.at(0); } // restore 2 UMAM {start,end,handler} addresses from extras - register_unsafe_access_handlers(extras, 0, 2, end); + register_unsafe_access_handlers(extras, 0, 2); return start; } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_DISJOINT; Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes,L_copy_2_bytes,L_exit; @@ -1888,10 +1878,8 @@ address StubGenerator::generate_fill(StubId stub_id) { bool aligned; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } switch (stub_id) { @@ -1925,7 +1913,7 @@ address StubGenerator::generate_fill(StubId stub_id) { __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); BLOCK_COMMENT("Entry:"); @@ -1983,10 +1971,8 @@ address StubGenerator::generate_conjoint_short_copy(address nooverlap_target, ad int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == expected_entry_count, "sanity check"); GrowableArray
* entries_ptr = (entry_count == 1 ? nullptr : &entries); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end, entries_ptr, &extras); + address start = load_archive_data(stub_id, entries_ptr, &extras); + if (start != nullptr) { assert(entries.length() == expected_entry_count - 1, "unexpected extra entry count %d", entries.length()); assert(extras.length() == expected_extra_count, @@ -1995,12 +1981,12 @@ address StubGenerator::generate_conjoint_short_copy(address nooverlap_target, ad *entry = entries.at(0); } // restore 2 UMAM {start,end,handler} addresses from extras - register_unsafe_access_handlers(extras, 0, 2, end); + register_unsafe_access_handlers(extras, 0, 2); return start; } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); DecoratorSet decorators = IN_HEAP | IS_ARRAY; Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes; @@ -2145,10 +2131,8 @@ address StubGenerator::generate_disjoint_int_oop_copy(StubId stub_id, address* e assert(entry_count == expected_entry_count, "sanity check"); GrowableArray
* entries_ptr = (entry_count == 1 ? nullptr : &entries); GrowableArray
* extras_ptr = (add_extras ? &extras : nullptr); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end, entries_ptr, extras_ptr); + address start = load_archive_data(stub_id, entries_ptr, extras_ptr); + if (start != nullptr) { assert(entries.length() == expected_entry_count - 1, "unexpected extra entry count %d", entries.length()); assert(extras.length() == expected_extra_count, @@ -2158,14 +2142,14 @@ address StubGenerator::generate_disjoint_int_oop_copy(StubId stub_id, address* e } if (add_extras) { // restore 2 UMAM {start,end,handler} addresses from extras - register_unsafe_access_handlers(extras, 0, 2, end); + register_unsafe_access_handlers(extras, 0, 2); } return start; } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes, L_exit; const Register from = rdi; // source array address @@ -2315,10 +2299,8 @@ address StubGenerator::generate_conjoint_int_oop_copy(StubId stub_id, address no assert(entry_count == expected_entry_count, "sanity check"); GrowableArray
* entries_ptr = (entry_count == 1 ? nullptr : &entries); GrowableArray
* extras_ptr = (add_extras ? &extras : nullptr); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end, entries_ptr, extras_ptr); + address start = load_archive_data(stub_id, entries_ptr, extras_ptr); + if (start != nullptr) { assert(entries.length() == expected_entry_count - 1, "unexpected extra entry count %d", entries.length()); assert(extras.length() == expected_extra_count, @@ -2328,14 +2310,14 @@ address StubGenerator::generate_conjoint_int_oop_copy(StubId stub_id, address no } if (add_extras) { // restore 2 UMAM {start,end,handler} addresses from extras - register_unsafe_access_handlers(extras, 0, 2, end); + register_unsafe_access_handlers(extras, 0, 2); } return start; } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); Label L_copy_bytes, L_copy_8_bytes, L_exit; const Register from = rdi; // source array address @@ -2487,10 +2469,8 @@ address StubGenerator::generate_disjoint_long_oop_copy(StubId stub_id, address * assert(entry_count == expected_entry_count, "sanity check"); GrowableArray
* entries_ptr = (entry_count == 1 ? nullptr : &entries); GrowableArray
* extras_ptr = (add_extras ? &extras : nullptr); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end, entries_ptr, extras_ptr); + address start = load_archive_data(stub_id, entries_ptr, extras_ptr); + if (start != nullptr) { assert(entries.length() == expected_entry_count - 1, "unexpected extra entry count %d", entries.length()); assert(extras.length() == expected_extra_count, @@ -2500,14 +2480,14 @@ address StubGenerator::generate_disjoint_long_oop_copy(StubId stub_id, address * } if (add_extras) { // restore 2 UMAM {start,end,handler} addresses from extras - register_unsafe_access_handlers(extras, 0, 2, end); + register_unsafe_access_handlers(extras, 0, 2); } return start; } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); Label L_copy_bytes, L_copy_8_bytes, L_exit; const Register from = rdi; // source array address @@ -2659,10 +2639,8 @@ address StubGenerator::generate_conjoint_long_oop_copy(StubId stub_id, address n assert(entry_count == expected_entry_count, "sanity check"); GrowableArray
* entries_ptr = (entry_count == 1 ? nullptr : &entries); GrowableArray
* extras_ptr = (add_extras ? &extras : nullptr); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end, entries_ptr, extras_ptr); + address start = load_archive_data(stub_id, entries_ptr, extras_ptr); + if (start != nullptr) { assert(entries.length() == expected_entry_count - 1, "unexpected extra entry count %d", entries.length()); assert(extras.length() == expected_extra_count, @@ -2672,14 +2650,14 @@ address StubGenerator::generate_conjoint_long_oop_copy(StubId stub_id, address n } if (add_extras) { // restore 2 UMAM {start,end,handler} addresses from extras - register_unsafe_access_handlers(extras, 0, 2, end); + register_unsafe_access_handlers(extras, 0, 2); } return start; } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); Label L_copy_bytes, L_copy_8_bytes, L_exit; const Register from = rdi; // source array address @@ -2830,10 +2808,8 @@ address StubGenerator::generate_checkcast_copy(StubId stub_id, address *entry) { int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == expected_entry_count, "sanity check"); GrowableArray
* entries_ptr = (entry_count == 1 ? nullptr : &entries); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end, entries_ptr); + address start = load_archive_data(stub_id, entries_ptr); + if (start != nullptr) { assert(entries.length() == expected_entry_count - 1, "unexpected extra entry count %d", entries.length()); if (entry != nullptr) { @@ -2871,7 +2847,7 @@ address StubGenerator::generate_checkcast_copy(StubId stub_id, address *entry) { __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ enter(); // required for proper stackwalking of RuntimeStub frame @@ -3056,13 +3032,8 @@ address StubGenerator::generate_unsafe_copy(address byte_copy_entry, address sho StubId stub_id = StubId::stubgen_unsafe_arraycopy_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - // TODO - in debug mode we should save and restore the supplied - // copy entries and assert that the restored entries match the ones - // passed in - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } @@ -3078,7 +3049,7 @@ address StubGenerator::generate_unsafe_copy(address byte_copy_entry, address sho __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ enter(); // required for proper stackwalking of RuntimeStub frame @@ -3220,19 +3191,17 @@ address StubGenerator::generate_unsafe_setmemory(address unsafe_byte_fill) { // we expect one set of extra unsafememory access handler entries GrowableArray
extras; int expected_extra_count = 3 * 3; - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end, nullptr, &extras); + address start = load_archive_data(stub_id, nullptr, &extras); + if (start != nullptr) { assert(extras.length() == expected_extra_count, "unexpected extra entry count %d", extras.length()); - register_unsafe_access_handlers(extras, 0, 3, end); + register_unsafe_access_handlers(extras, 0, 3); return start; } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ enter(); // required for proper stackwalking of RuntimeStub frame assert(unsafe_byte_fill != nullptr, "Invalid call"); @@ -3391,12 +3360,11 @@ address StubGenerator::generate_generic_copy(address byte_copy_entry, address sh StubId stub_id = StubId::stubgen_generic_arraycopy_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } + Label L_failed, L_failed_0, L_skip_failed_0, L_objArray; Label L_copy_shorts, L_copy_ints, L_copy_longs; @@ -3415,7 +3383,7 @@ address StubGenerator::generate_generic_copy(address byte_copy_entry, address sh StubCodeMark mark(this, stub_id); __ align(CodeEntryAlignment); - address start = __ pc(); + start = __ pc(); __ enter(); // required for proper stackwalking of RuntimeStub frame diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_cbrt.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_cbrt.cpp index d386d4a27440f..444e9ad191f2d 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_cbrt.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_cbrt.cpp @@ -191,8 +191,14 @@ ATTRIBUTE_ALIGNED(4) static const juint _D_table[] = address StubGenerator::generate_libmCbrt() { StubId stub_id = StubId::stubgen_dcbrt_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + address start = load_archive_data(stub_id); + if (start != nullptr) { + return start; + } StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); Label L_2TAG_PACKET_0_0_1, L_2TAG_PACKET_1_0_1, L_2TAG_PACKET_2_0_1; Label B1_1, B1_2, B1_4; @@ -335,6 +341,9 @@ address StubGenerator::generate_libmCbrt() { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_chacha.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_chacha.cpp index 2366f37112d6f..8bce50fd638dc 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_chacha.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_chacha.cpp @@ -111,10 +111,16 @@ void StubGenerator::generate_chacha_stubs() { /* The 2-block AVX/AVX2-enabled ChaCha20 block function implementation */ address StubGenerator::generate_chacha20Block_avx() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_chacha20Block_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + address start = load_archive_data(stub_id); + if (start != nullptr) { + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); Label L_twoRounds; const Register state = c_rarg0; @@ -295,15 +301,25 @@ address StubGenerator::generate_chacha20Block_avx() { } __ leave(); __ ret(0); + + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } /* The 4-block AVX512-enabled ChaCha20 block function implementation */ address StubGenerator::generate_chacha20Block_avx512() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_chacha20Block_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + address start = load_archive_data(stub_id); + if (start != nullptr) { + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); Label L_twoRounds; const Register state = c_rarg0; @@ -466,6 +482,10 @@ address StubGenerator::generate_chacha20Block_avx512() { __ vzeroupper(); __ leave(); __ ret(0); + + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_cos.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_cos.cpp index 8cb6ead21fd1b..8dedd50cd9790 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_cos.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_cos.cpp @@ -174,8 +174,14 @@ address StubGenerator::generate_libmCos() { StubId stub_id = StubId::stubgen_dcos_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + address start = load_archive_data(stub_id); + if (start != nullptr) { + return start; + } StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); Label L_2TAG_PACKET_0_0_1, L_2TAG_PACKET_1_0_1, L_2TAG_PACKET_2_0_1, L_2TAG_PACKET_3_0_1; Label L_2TAG_PACKET_4_0_1, L_2TAG_PACKET_5_0_1, L_2TAG_PACKET_6_0_1, L_2TAG_PACKET_7_0_1; @@ -619,6 +625,9 @@ address StubGenerator::generate_libmCos() { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_dilithium.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_dilithium.cpp index 1c88906f09aa6..e92c17b1313b9 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_dilithium.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_dilithium.cpp @@ -273,10 +273,16 @@ static int xmm29_29[] = {29, 29, 29, 29}; static address generate_dilithiumAlmostNtt_avx512(StubGenerator *stubgen, MacroAssembler *_masm) { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_dilithiumAlmostNtt_id; + __ align(CodeEntryAlignment); + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + address start = stubgen->load_archive_data(stub_id); + if (start != nullptr) { + return start; + } StubCodeMark mark(stubgen, stub_id); - address start = __ pc(); + start = __ pc(); __ enter(); Label L_loop, L_end; @@ -451,6 +457,9 @@ static address generate_dilithiumAlmostNtt_avx512(StubGenerator *stubgen, __ mov64(rax, 0); // return 0 __ ret(0); + // record the stub entry and end + stubgen->store_archive_data(stub_id, start, __ pc()); + return start; } @@ -463,10 +472,16 @@ static address generate_dilithiumAlmostNtt_avx512(StubGenerator *stubgen, static address generate_dilithiumAlmostInverseNtt_avx512(StubGenerator *stubgen, MacroAssembler *_masm) { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_dilithiumAlmostInverseNtt_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + address start = stubgen->load_archive_data(stub_id); + if (start != nullptr) { + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(stubgen, stub_id); - address start = __ pc(); + start = __ pc(); __ enter(); Label L_loop, L_end; @@ -630,6 +645,9 @@ static address generate_dilithiumAlmostInverseNtt_avx512(StubGenerator *stubgen, __ mov64(rax, 0); // return 0 __ ret(0); + // record the stub entry and end + stubgen->store_archive_data(stub_id, start, __ pc()); + return start; } @@ -644,10 +662,16 @@ static address generate_dilithiumAlmostInverseNtt_avx512(StubGenerator *stubgen, static address generate_dilithiumNttMult_avx512(StubGenerator *stubgen, MacroAssembler *_masm) { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_dilithiumNttMult_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + address start = stubgen->load_archive_data(stub_id); + if (start != nullptr) { + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(stubgen, stub_id); - address start = __ pc(); + start = __ pc(); __ enter(); Label L_loop; @@ -696,6 +720,9 @@ static address generate_dilithiumNttMult_avx512(StubGenerator *stubgen, __ mov64(rax, 0); // return 0 __ ret(0); + // record the stub entry and end + stubgen->store_archive_data(stub_id, start, __ pc()); + return start; } @@ -708,10 +735,16 @@ static address generate_dilithiumNttMult_avx512(StubGenerator *stubgen, static address generate_dilithiumMontMulByConstant_avx512(StubGenerator *stubgen, MacroAssembler *_masm) { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_dilithiumMontMulByConstant_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + address start = stubgen->load_archive_data(stub_id); + if (start != nullptr) { + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(stubgen, stub_id); - address start = __ pc(); + start = __ pc(); __ enter(); Label L_loop; @@ -757,6 +790,9 @@ static address generate_dilithiumMontMulByConstant_avx512(StubGenerator *stubgen __ mov64(rax, 0); // return 0 __ ret(0); + // record the stub entry and end + stubgen->store_archive_data(stub_id, start, __ pc()); + return start; } @@ -772,10 +808,16 @@ static address generate_dilithiumMontMulByConstant_avx512(StubGenerator *stubgen static address generate_dilithiumDecomposePoly_avx512(StubGenerator *stubgen, MacroAssembler *_masm) { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_dilithiumDecomposePoly_id; + __ align(CodeEntryAlignment); + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + address start = stubgen->load_archive_data(stub_id); + if (start != nullptr) { + return start; + } StubCodeMark mark(stubgen, stub_id); - address start = __ pc(); + start = __ pc(); __ enter(); Label L_loop; @@ -1014,6 +1056,9 @@ static address generate_dilithiumDecomposePoly_avx512(StubGenerator *stubgen, __ mov64(rax, 0); // return 0 __ ret(0); + // record the stub entry and end + stubgen->store_archive_data(stub_id, start, __ pc()); + return start; } diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_exp.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_exp.cpp index 32bbee73f5430..2d36cd919fc41 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_exp.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_exp.cpp @@ -166,8 +166,14 @@ ATTRIBUTE_ALIGNED(4) static const juint _INF[] = address StubGenerator::generate_libmExp() { StubId stub_id = StubId::stubgen_dexp_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + address start = load_archive_data(stub_id); + if (start != nullptr) { + return start; + } StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); Label L_2TAG_PACKET_0_0_2, L_2TAG_PACKET_1_0_2, L_2TAG_PACKET_2_0_2, L_2TAG_PACKET_3_0_2; Label L_2TAG_PACKET_4_0_2, L_2TAG_PACKET_5_0_2, L_2TAG_PACKET_6_0_2, L_2TAG_PACKET_7_0_2; @@ -381,6 +387,9 @@ address StubGenerator::generate_libmExp() { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_fmod.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_fmod.cpp index cfbaf97a5aa32..3b03497ae3176 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_fmod.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_fmod.cpp @@ -72,10 +72,16 @@ ATTRIBUTE_ALIGNED(32) static const uint64_t CONST_e307[] = { }; address StubGenerator::generate_libmFmod() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_fmod_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + address start = load_archive_data(stub_id); + if (start != nullptr) { + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ enter(); // required for proper stackwalking of RuntimeStub frame if (VM_Version::supports_avx512vlbwdq()) { // AVX512 version @@ -521,6 +527,9 @@ address StubGenerator::generate_libmFmod() { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_ghash.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_ghash.cpp index 4286088b45e68..e9eb9b64838cd 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_ghash.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_ghash.cpp @@ -80,11 +80,17 @@ void StubGenerator::generate_ghash_stubs() { // Single and multi-block ghash operations. address StubGenerator::generate_ghash_processBlocks() { - __ align(CodeEntryAlignment); - Label L_ghash_loop, L_exit; StubId stub_id = StubId::stubgen_ghash_processBlocks_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + address start = load_archive_data(stub_id); + if (start != nullptr) { + return start; + } + Label L_ghash_loop, L_exit; + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); const Register state = c_rarg0; const Register subkeyH = c_rarg1; @@ -211,17 +217,25 @@ address StubGenerator::generate_ghash_processBlocks() { __ leave(); __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } // Ghash single and multi block operations using AVX instructions address StubGenerator::generate_avx_ghash_processBlocks() { - __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_ghash_processBlocks_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + address start = load_archive_data(stub_id); + if (start != nullptr) { + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); // arguments const Register state = c_rarg0; @@ -237,6 +251,9 @@ address StubGenerator::generate_avx_ghash_processBlocks() { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_kyber.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_kyber.cpp index c934d3c8ceb2a..6b5081401a8da 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_kyber.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_kyber.cpp @@ -367,10 +367,16 @@ static int xmm29_29[] = {29, 29, 29, 29}; // ntt_zetas (short[256]) = c_rarg1 address generate_kyberNtt_avx512(StubGenerator *stubgen, MacroAssembler *_masm) { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_kyberNtt_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + address start = stubgen->load_archive_data(stub_id); + if (start != nullptr) { + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(stubgen, stub_id); - address start = __ pc(); + start = __ pc(); __ enter(); const Register coeffs = c_rarg0; @@ -454,6 +460,9 @@ address generate_kyberNtt_avx512(StubGenerator *stubgen, __ mov64(rax, 0); // return 0 __ ret(0); + // record the stub entry and end + stubgen->store_archive_data(stub_id, start, __ pc()); + return start; } @@ -463,11 +472,16 @@ address generate_kyberNtt_avx512(StubGenerator *stubgen, // ntt_zetas (short[256]) = c_rarg1 address generate_kyberInverseNtt_avx512(StubGenerator *stubgen, MacroAssembler *_masm) { - - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_kyberInverseNtt_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + address start = stubgen->load_archive_data(stub_id); + if (start != nullptr) { + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(stubgen, stub_id); - address start = __ pc(); + start = __ pc(); __ enter(); const Register coeffs = c_rarg0; @@ -577,6 +591,9 @@ address generate_kyberInverseNtt_avx512(StubGenerator *stubgen, __ mov64(rax, 0); // return 0 __ ret(0); + // record the stub entry and end + stubgen->store_archive_data(stub_id, start, __ pc()); + return start; } @@ -588,11 +605,16 @@ address generate_kyberInverseNtt_avx512(StubGenerator *stubgen, // zetas (short[128]) = c_rarg3 address generate_kyberNttMult_avx512(StubGenerator *stubgen, MacroAssembler *_masm) { - - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_kyberNttMult_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + address start = stubgen->load_archive_data(stub_id); + if (start != nullptr) { + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(stubgen, stub_id); - address start = __ pc(); + start = __ pc(); __ enter(); const Register result = c_rarg0; @@ -698,6 +720,9 @@ address generate_kyberNttMult_avx512(StubGenerator *stubgen, __ mov64(rax, 0); // return 0 __ ret(0); + // record the stub entry and end + stubgen->store_archive_data(stub_id, start, __ pc()); + return start; } @@ -708,11 +733,16 @@ address generate_kyberNttMult_avx512(StubGenerator *stubgen, // b (short[256]) = c_rarg2 address generate_kyberAddPoly_2_avx512(StubGenerator *stubgen, MacroAssembler *_masm) { - - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_kyberAddPoly_2_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + address start = stubgen->load_archive_data(stub_id); + if (start != nullptr) { + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(stubgen, stub_id); - address start = __ pc(); + start = __ pc(); __ enter(); const Register result = c_rarg0; @@ -743,6 +773,9 @@ address generate_kyberAddPoly_2_avx512(StubGenerator *stubgen, __ mov64(rax, 0); // return 0 __ ret(0); + // record the stub entry and end + stubgen->store_archive_data(stub_id, start, __ pc()); + return start; } @@ -754,11 +787,16 @@ address generate_kyberAddPoly_2_avx512(StubGenerator *stubgen, // c (short[256]) = c_rarg3 address generate_kyberAddPoly_3_avx512(StubGenerator *stubgen, MacroAssembler *_masm) { - - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_kyberAddPoly_3_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + address start = stubgen->load_archive_data(stub_id); + if (start != nullptr) { + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(stubgen, stub_id); - address start = __ pc(); + start = __ pc(); __ enter(); const Register result = c_rarg0; @@ -797,6 +835,9 @@ address generate_kyberAddPoly_3_avx512(StubGenerator *stubgen, __ mov64(rax, 0); // return 0 __ ret(0); + // record the stub entry and end + stubgen->store_archive_data(stub_id, start, __ pc()); + return start; } @@ -808,11 +849,16 @@ address generate_kyberAddPoly_3_avx512(StubGenerator *stubgen, // parsedLength (int) = c_rarg3 address generate_kyber12To16_avx512(StubGenerator *stubgen, MacroAssembler *_masm) { - - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_kyber12To16_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + address start = stubgen->load_archive_data(stub_id); + if (start != nullptr) { + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(stubgen, stub_id); - address start = __ pc(); + start = __ pc(); __ enter(); const Register condensed = c_rarg0; @@ -896,6 +942,9 @@ address generate_kyber12To16_avx512(StubGenerator *stubgen, __ mov64(rax, 0); // return 0 __ ret(0); + // record the stub entry and end + stubgen->store_archive_data(stub_id, start, __ pc()); + return start; } @@ -905,11 +954,16 @@ address generate_kyber12To16_avx512(StubGenerator *stubgen, // coeffs (short[256]) = c_rarg0 address generate_kyberBarrettReduce_avx512(StubGenerator *stubgen, MacroAssembler *_masm) { - - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_kyberBarrettReduce_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + address start = stubgen->load_archive_data(stub_id); + if (start != nullptr) { + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(stubgen, stub_id); - address start = __ pc(); + start = __ pc(); __ enter(); const Register coeffs = c_rarg0; @@ -933,6 +987,9 @@ address generate_kyberBarrettReduce_avx512(StubGenerator *stubgen, __ mov64(rax, 0); // return 0 __ ret(0); + // record the stub entry and end + stubgen->store_archive_data(stub_id, start, __ pc()); + return start; } diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_log.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_log.cpp index 13da0591a592e..70ca9d3cf75d1 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_log.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_log.cpp @@ -177,8 +177,14 @@ ATTRIBUTE_ALIGNED(16) static const juint _coeff[] = address StubGenerator::generate_libmLog() { StubId stub_id = StubId::stubgen_dlog_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + address start = load_archive_data(stub_id); + if (start != nullptr) { + return start; + } StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); Label L_2TAG_PACKET_0_0_2, L_2TAG_PACKET_1_0_2, L_2TAG_PACKET_2_0_2, L_2TAG_PACKET_3_0_2; Label L_2TAG_PACKET_4_0_2, L_2TAG_PACKET_5_0_2, L_2TAG_PACKET_6_0_2, L_2TAG_PACKET_7_0_2; @@ -359,6 +365,9 @@ address StubGenerator::generate_libmLog() { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -516,8 +525,14 @@ ATTRIBUTE_ALIGNED(16) static const juint _coeff_log10[] = address StubGenerator::generate_libmLog10() { StubId stub_id = StubId::stubgen_dlog10_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + address start = load_archive_data(stub_id); + if (start != nullptr) { + return start; + } StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); Label L_2TAG_PACKET_0_0_2, L_2TAG_PACKET_1_0_2, L_2TAG_PACKET_2_0_2, L_2TAG_PACKET_3_0_2; Label L_2TAG_PACKET_4_0_2, L_2TAG_PACKET_5_0_2, L_2TAG_PACKET_6_0_2, L_2TAG_PACKET_7_0_2; @@ -704,6 +719,9 @@ address StubGenerator::generate_libmLog10() { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_poly1305.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_poly1305.cpp index 2a4874a78108f..2c794e201cac2 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_poly1305.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_poly1305.cpp @@ -909,10 +909,16 @@ void StubGenerator::poly1305_process_blocks_avx512( // After execution, input and length will point at remaining (unprocessed) data // and accumulator will point to the current accumulator value address StubGenerator::generate_poly1305_processBlocks() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_poly1305_processBlocks_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + address start = load_archive_data(stub_id); + if (start != nullptr) { + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ enter(); // Save all 'SOE' registers @@ -1028,6 +1034,10 @@ address StubGenerator::generate_poly1305_processBlocks() { __ leave(); __ ret(0); + + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_poly_mont.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_poly_mont.cpp index ebe2414486900..0fc4a62a24c31 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_poly_mont.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_poly_mont.cpp @@ -558,10 +558,16 @@ void montgomeryMultiplyAVX2(const Register aLimbs, const Register bLimbs, const } address StubGenerator::generate_intpoly_montgomeryMult_P256() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_intpoly_montgomeryMult_P256_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + address start = load_archive_data(stub_id); + if (start != nullptr) { + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ enter(); if (VM_Version::supports_avx512ifma() && VM_Version::supports_avx512vlbw()) { @@ -620,6 +626,10 @@ address StubGenerator::generate_intpoly_montgomeryMult_P256() { __ leave(); __ ret(0); + + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -680,10 +690,16 @@ address StubGenerator::generate_intpoly_assign() { // P521OrderField: 19 = 8 + 8 + 2 + 1 // Special Cases 5, 10, 14, 16, 19 - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_intpoly_assign_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + address start = load_archive_data(stub_id); + if (start != nullptr) { + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ enter(); // Inputs @@ -762,6 +778,10 @@ address StubGenerator::generate_intpoly_assign() { __ bind(L_Done); __ leave(); __ ret(0); + + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } #undef __ diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_pow.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_pow.cpp index 581b0ff1ca976..a81064868038a 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_pow.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_pow.cpp @@ -760,8 +760,14 @@ ATTRIBUTE_ALIGNED(8) static const juint _DOUBLE0DOT5[] = { address StubGenerator::generate_libmPow() { StubId stub_id = StubId::stubgen_dpow_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + address start = load_archive_data(stub_id); + if (start != nullptr) { + return start; + } StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); Label L_2TAG_PACKET_0_0_2, L_2TAG_PACKET_1_0_2, L_2TAG_PACKET_2_0_2, L_2TAG_PACKET_3_0_2; Label L_2TAG_PACKET_4_0_2, L_2TAG_PACKET_5_0_2, L_2TAG_PACKET_6_0_2, L_2TAG_PACKET_7_0_2; @@ -1859,6 +1865,9 @@ address StubGenerator::generate_libmPow() { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_sha3.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_sha3.cpp index 6b830f02e6c9b..7b5ee166c6dc0 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_sha3.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_sha3.cpp @@ -104,10 +104,15 @@ static address generate_sha3_implCompress(StubId stub_id, default: ShouldNotReachHere(); } - + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + address start = stubgen->load_archive_data(stub_id); + if (start != nullptr) { + return start; + } __ align(CodeEntryAlignment); StubCodeMark mark(stubgen, stub_id); - address start = __ pc(); + start = __ pc(); const Register buf = c_rarg0; const Register state = c_rarg1; @@ -316,6 +321,9 @@ static address generate_sha3_implCompress(StubId stub_id, __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // record the stub entry and end + stubgen->store_archive_data(stub_id, start, __ pc()); + return start; } @@ -326,10 +334,16 @@ static address generate_sha3_implCompress(StubId stub_id, // Performs two keccak() computations in parallel. The steps of the // two computations are executed interleaved. static address generate_double_keccak(StubGenerator *stubgen, MacroAssembler *_masm) { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_double_keccak_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + address start = stubgen->load_archive_data(stub_id); + if (start != nullptr) { + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(stubgen, stub_id); - address start = __ pc(); + start = __ pc(); const Register state0 = c_rarg0; const Register state1 = c_rarg1; @@ -495,6 +509,9 @@ static address generate_double_keccak(StubGenerator *stubgen, MacroAssembler *_m __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // record the stub entry and end + stubgen->store_archive_data(stub_id, start, __ pc()); + return start; } diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_sin.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_sin.cpp index 2b313d6c10fa6..7f68cdf9bfd01 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_sin.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_sin.cpp @@ -181,8 +181,14 @@ ATTRIBUTE_ALIGNED(8) static const juint _ALL_ONES[] = address StubGenerator::generate_libmSin() { StubId stub_id = StubId::stubgen_dsin_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + address start = load_archive_data(stub_id); + if (start != nullptr) { + return start; + } StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); Label L_2TAG_PACKET_0_0_1, L_2TAG_PACKET_1_0_1, L_2TAG_PACKET_2_0_1, L_2TAG_PACKET_3_0_1; Label L_2TAG_PACKET_4_0_1, L_2TAG_PACKET_5_0_1, L_2TAG_PACKET_6_0_1, L_2TAG_PACKET_7_0_1; @@ -645,6 +651,9 @@ address StubGenerator::generate_libmSin() { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_sinh.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_sinh.cpp index 7f190068467e6..aa1d80348d9c4 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_sinh.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_sinh.cpp @@ -290,8 +290,14 @@ ATTRIBUTE_ALIGNED(16) static const juint _T2_neg_f[] = address StubGenerator::generate_libmSinh() { StubId stub_id = StubId::stubgen_dsinh_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + address start = load_archive_data(stub_id); + if (start != nullptr) { + return start; + } StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); Label L_2TAG_PACKET_0_0_2, L_2TAG_PACKET_1_0_2, L_2TAG_PACKET_3_0_2, L_2TAG_PACKET_4_0_2; Label L_2TAG_PACKET_5_0_2, L_2TAG_PACKET_6_0_2; @@ -519,6 +525,9 @@ address StubGenerator::generate_libmSinh() { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_tan.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_tan.cpp index 04534ed50dadd..f03a4df2c34f5 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_tan.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_tan.cpp @@ -456,8 +456,14 @@ ATTRIBUTE_ALIGNED(8) static const juint _QQ_2_tan[] = address StubGenerator::generate_libmTan() { StubId stub_id = StubId::stubgen_dtan_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + address start = load_archive_data(stub_id); + if (start != nullptr) { + return start; + } StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); Label L_2TAG_PACKET_0_0_1, L_2TAG_PACKET_1_0_1, L_2TAG_PACKET_2_0_1, L_2TAG_PACKET_3_0_1; Label L_2TAG_PACKET_4_0_1, L_2TAG_PACKET_5_0_1, L_2TAG_PACKET_6_0_1, L_2TAG_PACKET_7_0_1; @@ -1025,6 +1031,9 @@ address StubGenerator::generate_libmTan() { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_tanh.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_tanh.cpp index e0618ced9d820..a1bc8143e863e 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_tanh.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_tanh.cpp @@ -303,8 +303,14 @@ ATTRIBUTE_ALIGNED(16) static const juint _T2_neg_f[] = address StubGenerator::generate_libmTanh() { StubId stub_id = StubId::stubgen_dtanh_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + address start = load_archive_data(stub_id); + if (start != nullptr) { + return start; + } StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); Label L_2TAG_PACKET_0_0_1, L_2TAG_PACKET_1_0_1, L_2TAG_PACKET_2_0_1, L_2TAG_PACKET_3_0_1; Label L_2TAG_PACKET_4_0_1, L_2TAG_PACKET_5_0_1; @@ -495,6 +501,9 @@ address StubGenerator::generate_libmTanh() { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } diff --git a/src/hotspot/share/code/aotCodeCache.cpp b/src/hotspot/share/code/aotCodeCache.cpp index 524a89386c54d..138c1b3f1b751 100644 --- a/src/hotspot/share/code/aotCodeCache.cpp +++ b/src/hotspot/share/code/aotCodeCache.cpp @@ -2287,8 +2287,6 @@ AOTStubData::AOTStubData(BlobId blob_id) : _cached_blob(nullptr), _stub_cnt(0), _ranges(nullptr), - _current(StubId::NO_STUBID), - _current_idx(-1), _flags(0) { assert(StubInfo::is_stubgen(blob_id), "AOTStubData expects a multi-stub blob not %s", @@ -2348,41 +2346,28 @@ bool AOTStubData::store_code_blob(CodeBlob& new_blob, CodeBuffer *code_buffer) { } } -bool AOTStubData::find_archive_data(StubId stub_id) { +address AOTStubData::load_archive_data(StubId stub_id, address& end, GrowableArray
* entries, GrowableArray
* extras) { assert(StubInfo::blob(stub_id) == _blob_id, "sanity check"); if (is_invalid()) { - return false; + return nullptr; } int idx = StubInfo::stubgen_offset_in_blob(_blob_id, stub_id); assert(idx >= 0 && idx < _stub_cnt, "invalid index %d for stub count %d", idx, _stub_cnt); // ensure we have a valid associated range - StubAddrRange range = _ranges[idx]; - int start_index = range.start_index(); - if (start_index < 0) { - _current = StubId::NO_STUBID; + StubAddrRange &range = _ranges[idx]; + int base = range.start_index(); + if (base < 0) { #ifdef DEBUG // reset index so we can idenitfy which ones we failed to find range.init_entry(-2, 0); #endif - return false; + return nullptr; } - _current = stub_id; - _current_idx = idx; - return true; -} - -void AOTStubData::load_archive_data(StubId stub_id, address& start, address& end, GrowableArray
* entries, GrowableArray
* extras) { - assert(StubInfo::blob(stub_id) == _blob_id, "sanity check"); - assert(_current == stub_id && stub_id != StubId::NO_STUBID, "sanity check"); - assert(!is_invalid(), "should not load stubs when archive data is invalid"); - assert(_current_idx >= 0, "sanity"); - StubAddrRange& range = _ranges[_current_idx]; - int base = range.start_index(); int count = range.count(); assert(base >= 0, "sanity"); assert(count >= 2, "sanity"); // first two saved addresses are start and end - start = _address_array.at(base); + address start = _address_array.at(base); end = _address_array.at(base + 1); assert(start != nullptr, "failed to load start address of stub %s", StubInfo::name(stub_id)); assert(end != nullptr, "failed to load end address of stub %s", StubInfo::name(stub_id)); @@ -2414,6 +2399,8 @@ void AOTStubData::load_archive_data(StubId stub_id, address& start, address& end extras->append(extra); } } + + return start; } void AOTStubData::store_archive_data(StubId stub_id, address start, address end, GrowableArray
* entries, GrowableArray
* extras) { @@ -2421,9 +2408,8 @@ void AOTStubData::store_archive_data(StubId stub_id, address start, address end, assert(start != nullptr, "start address cannot be null"); assert(end != nullptr, "end address cannot be null"); assert(start < end, "start address %p should be less than end %p address for stub %s", start, end, StubInfo::name(stub_id)); - _current = stub_id; - _current_idx = StubInfo::stubgen_offset_in_blob(_blob_id, stub_id); - StubAddrRange& range = _ranges[_current_idx]; + int idx = StubInfo::stubgen_offset_in_blob(_blob_id, stub_id); + StubAddrRange& range = _ranges[idx]; assert(range.start_index() == -1, "sanity"); int base = _address_array.length(); assert(base >= 0, "sanity"); diff --git a/src/hotspot/share/code/aotCodeCache.hpp b/src/hotspot/share/code/aotCodeCache.hpp index bfe085c0a4e5c..6dab240a21954 100644 --- a/src/hotspot/share/code/aotCodeCache.hpp +++ b/src/hotspot/share/code/aotCodeCache.hpp @@ -218,10 +218,6 @@ class AOTStubData : public StackObj { // array identifying range of entries in _address_array for each stub // indexed by offset of stub in blob StubAddrRange* _ranges; - // id of last looked up stub or NO_STUBID if lookup not attempted or failed - StubId _current; - // offset of _current in blob or -1 if lookup not attempted or failed - int _current_idx; // flags indicating whether the AOT code cache is open and, if so, // whether we are loading or storing stubs.the first of those @@ -253,41 +249,10 @@ class AOTStubData : public StackObj { bool is_invalid() CDS_ONLY({ return (_flags & INVALID) != 0; }) NOT_CDS_RETURN_(false); BlobId blob_id() { return _blob_id; } - StubId current_stub_id() { return _current; } - // bool load_code_blob() NOT_CDS_RETURN_(true); bool store_code_blob(CodeBlob& new_blob, CodeBuffer *code_buffer) NOT_CDS_RETURN_(true); - // determine whether a stub is available in the AOT cache - bool find_archive_data(StubId stub_id) NOT_CDS_RETURN_(false); - // retrieve stub entry data if it we are using archived stubs and - // the stub has been found in an AOT-restored blob or store stub - // entry data if we are saving archived stubs and the stub has just - // been successfully generated into the current blob. - // - // start and end identify the inclusive start and exclusive end - // address for stub code and must lie in the current blob's code - // range. Stubs presented via this interface must declare at least - // one entry and start is always taken to be the first entry. - // - // Optional arrays entries and extras present other addresses of - // interest all of which must either lie in the interval (start, - // end) or be nullptr (verified by load and store methods). - // - // entries lists secondary entries for the stub each of which must - // match a corresponding entry declaration for the stub (entry count - // verified by load and store methods). Null entry addresses are - // allowed when an architecture does not require a specific entry - // but may not vary from one run to the next. If the cache is in use - // at a store (for loading or saving code) then non-null entry - // addresses are entered into the AOT cache stub address table - // allowing references to them from other stubs or nmethods to be - // relocated. - // - // extras lists other non-entry stub addresses of interest such as - // memory protection ranges and associated handler addresses. These - // do do not need to be declared as entries and their number and - // meaning may vary according to the architecture. - void load_archive_data(StubId stub_id, address& start, address& end, GrowableArray
* entries = nullptr, GrowableArray
* extras = nullptr) NOT_CDS_RETURN; + + address load_archive_data(StubId stub_id, address &end, GrowableArray
* entries = nullptr, GrowableArray
* extras = nullptr) NOT_CDS_RETURN_(nullptr); void store_archive_data(StubId stub_id, address start, address end, GrowableArray
* entries = nullptr, GrowableArray
* extras = nullptr) NOT_CDS_RETURN; const AOTStubData* as_const() { return (const AOTStubData*)this; } diff --git a/src/hotspot/share/runtime/stubCodeGenerator.cpp b/src/hotspot/share/runtime/stubCodeGenerator.cpp index 24f641e53e69f..48ee6757a6e21 100644 --- a/src/hotspot/share/runtime/stubCodeGenerator.cpp +++ b/src/hotspot/share/runtime/stubCodeGenerator.cpp @@ -115,7 +115,7 @@ void StubCodeGenerator::setup_code_desc(const char* name, address start, address // handler pc may be null in which case it defaults to the // default_handler. -void StubCodeGenerator::register_unsafe_access_handlers(GrowableArray
&entries, int begin, int count, address stub_end) { +void StubCodeGenerator::register_unsafe_access_handlers(GrowableArray
&entries, int begin, int count) { for (int i = 0; i < count; i++) { int offset = begin + 3 * i; address start = entries.at(offset); @@ -171,21 +171,20 @@ void StubCodeGenerator::print_stub_code_desc(StubCodeDesc* cdesc) { } } -bool StubCodeGenerator::find_archive_data(StubId stub_id) { +address StubCodeGenerator::load_archive_data(StubId stub_id, GrowableArray
*entries, GrowableArray
* extras) { // punt to stub data if it exists and is not for dumping if (_stub_data == nullptr || _stub_data->is_dumping()) { - return false; + return nullptr; } - return _stub_data->find_archive_data(stub_id); -} - -void StubCodeGenerator::load_archive_data(StubId stub_id, address& start, address& end, GrowableArray
*entries, GrowableArray
* extras) { - assert(_stub_data != nullptr && _stub_data->current_stub_id() == stub_id, "no current archive data for %s", StubInfo::name(stub_id)); - // punt to stub data - _stub_data->load_archive_data(stub_id, start, end, entries, extras); + address start, end; + start = _stub_data->load_archive_data(stub_id, end, entries, extras); + + if (start != nullptr) { + setup_code_desc(StubInfo::name(stub_id), start, end, true); + } - setup_code_desc(StubInfo::name(stub_id), start, end, true); + return start; } void StubCodeGenerator::store_archive_data(StubId stub_id, address start, address end, GrowableArray
* entries, GrowableArray
* extras) { diff --git a/src/hotspot/share/runtime/stubCodeGenerator.hpp b/src/hotspot/share/runtime/stubCodeGenerator.hpp index 18cf461f27e35..b8ec8f5520022 100644 --- a/src/hotspot/share/runtime/stubCodeGenerator.hpp +++ b/src/hotspot/share/runtime/stubCodeGenerator.hpp @@ -116,7 +116,7 @@ class StubCodeGenerator: public StackObj { void setup_code_desc(const char* name, address start, address end, bool loaded_from_cache); // unsafe handler management - void register_unsafe_access_handlers(GrowableArray
&entries, int begin, int count, address stub_end); + void register_unsafe_access_handlers(GrowableArray
&entries, int begin, int count); void retrieve_unsafe_access_handlers(address start, address end, GrowableArray
&entries); public: @@ -133,8 +133,51 @@ class StubCodeGenerator: public StackObj { void print_stub_code_desc(StubCodeDesc* cdesc); static void print_statistics_on(outputStream* st); - bool find_archive_data(StubId stub_id); - void load_archive_data(StubId stub_id, address& start, address& end, GrowableArray
*entries = nullptr, GrowableArray
* extras = nullptr); + + // load_archive_data should be called before generating the stub + // identified by stub_id. If AOT caching of stubs is enabled and the + // stubis found then the address of the stub's first and, possibly, + // only entry is returned and the caller should use it instead of + // generating thestub. Otherwise a null address is returned and the + // caller should proceed to generate the stub. + // + // store_archive_data should be called when a stub has been + // successfully generated into the current blob irrespctive of + // whether the current JVM is generating or consuming an AOT archive + // (the caller should not check for either case). When generating an + // archive the stub entry and end addresses are recorded for storage + // along with the current blob and also to allow rences to the stub + // from other stubs or from compiled Java methods can be detected + // and marked as requiring relocation. When consuming an archive the + // stub entry address is still inorer to identify it as a relocation + // target. When no archive is in use the call has no side effects. + // + // start and end identify the inclusive start and exclusive end + // address for stub code and must lie in the current blob's code + // range. Stubs presented via this interface must declare at least + // one entry and start is always taken to be the first entry. + // + // Optional arrays entries and extras store other addresses of + // interest all of which must either lie in the interval (start, + // end) or be nullptr (verified by load and store methods). + // + // entries lists secondary entries for the stub each of which must + // match a corresponding entry declaration for the stub (entry count + // verified by load and store methods). Null entry addresses are + // allowed when an architecture does not require a specific entry + // but may not vary from one run to the next. If the cache is in use + // at a store (for loading or saving code) then non-null entry + // addresses are entered into the AOT cache stub address table + // allowing references to them from other stubs or nmethods to be + // relocated. + // + // extras lists other non-entry stub addresses of interest such as + // memory protection ranges and associated handler addresses + // (potentially including a null address). These do do not need to + // be declared as entries and their number and meaning may vary + // according to the architecture. + + address load_archive_data(StubId stub_id, GrowableArray
*entries = nullptr, GrowableArray
* extras = nullptr); void store_archive_data(StubId stub_id, address start, address end, GrowableArray
*entries = nullptr, GrowableArray
* extras = nullptr); #ifdef ASSERT void verify_stub(StubId stub_id); From 1aab99c56ed64db50a169d18c1c5c7668ac52401 Mon Sep 17 00:00:00 2001 From: Andrew Dinn Date: Wed, 26 Nov 2025 09:54:07 +0000 Subject: [PATCH 20/30] add missing aot+codecache+init statistic --- src/hotspot/share/code/aotCodeCache.cpp | 1 + src/hotspot/share/code/aotCodeCache.hpp | 1 + 2 files changed, 2 insertions(+) diff --git a/src/hotspot/share/code/aotCodeCache.cpp b/src/hotspot/share/code/aotCodeCache.cpp index 138c1b3f1b751..1d115c590fd0b 100644 --- a/src/hotspot/share/code/aotCodeCache.cpp +++ b/src/hotspot/share/code/aotCodeCache.cpp @@ -356,6 +356,7 @@ AOTCodeCache::AOTCodeCache(bool is_dumping, bool is_using) : log_info (aot, codecache, init)("Loaded %u AOT code entries from AOT Code Cache", _load_header->entries_count()); log_debug(aot, codecache, init)(" Adapters: total=%u", _load_header->adapters_count()); log_debug(aot, codecache, init)(" Shared Blobs: total=%u", _load_header->shared_blobs_count()); + log_debug(aot, codecache, init)(" StubGen Blobs: total=%d", _load_header->stubgen_blobs_count()); log_debug(aot, codecache, init)(" C1 Blobs: total=%u", _load_header->C1_blobs_count()); log_debug(aot, codecache, init)(" C2 Blobs: total=%u", _load_header->C2_blobs_count()); log_debug(aot, codecache, init)(" AOT code cache size: %u bytes", _load_header->cache_size()); diff --git a/src/hotspot/share/code/aotCodeCache.hpp b/src/hotspot/share/code/aotCodeCache.hpp index 6dab240a21954..32ca318459135 100644 --- a/src/hotspot/share/code/aotCodeCache.hpp +++ b/src/hotspot/share/code/aotCodeCache.hpp @@ -332,6 +332,7 @@ class AOTCodeCache : public CHeapObj { uint entries_count() const { return _entries_count; } uint entries_offset() const { return _entries_offset; } uint adapters_count() const { return _adapters_count; } + uint stubgen_blobs_count() const { return _stubgen_blobs_count; } uint shared_blobs_count() const { return _shared_blobs_count; } uint C1_blobs_count() const { return _C1_blobs_count; } uint C2_blobs_count() const { return _C2_blobs_count; } From a6ea6376a036db796a68f41887dfecb314e1aaf8 Mon Sep 17 00:00:00 2001 From: Andrew Dinn Date: Wed, 26 Nov 2025 10:12:10 +0000 Subject: [PATCH 21/30] use simpler AOT lookup in all aarch64 stub generators --- .../cpu/aarch64/stubGenerator_aarch64.cpp | 710 ++++++------------ 1 file changed, 232 insertions(+), 478 deletions(-) diff --git a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp index 266bd7097144b..4651095d33d3e 100644 --- a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp @@ -366,18 +366,14 @@ class StubGenerator: public StubCodeGenerator { GrowableArray
entries; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 2, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - // this stub has an extra return entry which we need to retrieve - load_archive_data(stub_id, start, end, &entries); + address start = load_archive_data(stub_id, &entries); + if (start != nullptr) { assert(entries.length() == 1, "expected 1 extra entry"); return_address = entries.at(0); return start; } - StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); const Address sp_after_call (rfp, sp_after_call_off * wordSize); @@ -602,15 +598,12 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_catch_exception_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); // same as in generate_call_stub(): const Address sp_after_call(rfp, sp_after_call_off * wordSize); @@ -672,14 +665,12 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_forward_exception_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); // Upon entry, LR points to the return address returning into // Java (interpreted or compiled) code; i.e., the return address @@ -773,15 +764,12 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_verify_oop_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); Label exit, error; @@ -834,16 +822,13 @@ class StubGenerator: public StubCodeGenerator { address generate_iota_indices(StubId stub_id) { int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); // B __ emit_data64(0x0706050403020100, relocInfo::none); __ emit_data64(0x0F0E0D0C0B0A0908, relocInfo::none); @@ -886,13 +871,10 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_zero_blocks_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - StubCodeMark mark(this, stub_id); Label done; Label base_aligned; @@ -900,7 +882,7 @@ class StubGenerator: public StubCodeGenerator { Register base = r10, cnt = r11; __ align(CodeEntryAlignment); - address start = __ pc(); + start = __ pc(); if (UseBlockZeroing) { int zva_length = VM_Version::zva_length(); @@ -1044,13 +1026,10 @@ class StubGenerator: public StubCodeGenerator { address generate_copy_longs(StubId stub_id, DecoratorSet decorators, Register s, Register d, Register count) { int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - BasicType type; copy_direction direction; @@ -1102,7 +1081,7 @@ class StubGenerator: public StubCodeGenerator { StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); Label unaligned_copy_long; if (AvoidUnalignedAccesses) { @@ -1868,11 +1847,8 @@ class StubGenerator: public StubCodeGenerator { GrowableArray
entries; GrowableArray
extras; GrowableArray
*extras_ptr = (extra_count > 0 ? &extras : nullptr); - - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end, &entries, extras_ptr); + address start = load_archive_data(stub_id, &entries, extras_ptr); + if (start != nullptr) { assert(entries.length() == entry_count - 1, "unexpected entries count %d", entries.length()); *nopush_entry = entries.at(0); @@ -1880,7 +1856,7 @@ class StubGenerator: public StubCodeGenerator { "unexpected extra count %d", extras.length()); if (add_extras) { // register one handler at offset 0 - register_unsafe_access_handlers(extras, 0, 1, end); + register_unsafe_access_handlers(extras, 0, 1); } return start; } @@ -1890,7 +1866,7 @@ class StubGenerator: public StubCodeGenerator { __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ enter(); *nopush_entry = __ pc(); @@ -2067,11 +2043,8 @@ class StubGenerator: public StubCodeGenerator { GrowableArray
extras; GrowableArray
*entries_ptr = (nopush_entry != nullptr ? &entries : nullptr); GrowableArray
*extras_ptr = (extra_count > 0 ? &extras : nullptr); - - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end, entries_ptr, extras_ptr); + address start = load_archive_data(stub_id, entries_ptr, extras_ptr); + if (start != nullptr) { assert(entries.length() == expected_entry_count - 1, "unexpected entries count %d", entries.length()); assert(extras.length() == extra_count, @@ -2081,7 +2054,7 @@ class StubGenerator: public StubCodeGenerator { } if (add_extras) { // register one handler at offset 0 - register_unsafe_access_handlers(extras, 0, 1, end); + register_unsafe_access_handlers(extras, 0, 1); } return start; } @@ -2089,7 +2062,7 @@ class StubGenerator: public StubCodeGenerator { Register s = c_rarg0, d = c_rarg1, count = c_rarg2; RegSet saved_regs = RegSet::of(s, d, count); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ enter(); if (nopush_entry != nullptr) { @@ -2217,10 +2190,8 @@ class StubGenerator: public StubCodeGenerator { assert(entry_count == expected_entry_count, "expected entry count %d does not match declared entry count %d for stub %s", expected_entry_count, entry_count, StubInfo::name(stub_id)); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end, entries_ptr); + address start = load_archive_data(stub_id, entries_ptr); + if (start != nullptr) { assert(entries.length() + 1 == expected_entry_count, "expected entry count %d does not match return entry count %d for stub %s", expected_entry_count, entries.length() + 1, StubInfo::name(stub_id)); @@ -2262,7 +2233,7 @@ class StubGenerator: public StubCodeGenerator { __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ enter(); // required for proper stackwalking of RuntimeStub frame @@ -2443,22 +2414,16 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_unsafe_arraycopy_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - // TODO - in debug mode we should save and restore the supplied - // copy entries and assert that the restored entries match the ones - // passed in - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - Label L_long_aligned, L_int_aligned, L_short_aligned; Register s = c_rarg0, d = c_rarg1, count = c_rarg2; __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ enter(); // required for proper stackwalking of RuntimeStub frame // bump this on entry, not on exit: @@ -2510,16 +2475,10 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_generic_arraycopy_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - // TODO - in debug mode we should save and restore the supplied - // copy entries and assert that the restored entries match the ones - // passed in - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - Label L_failed, L_objArray; Label L_copy_bytes, L_copy_shorts, L_copy_ints, L_copy_longs; @@ -2538,7 +2497,7 @@ class StubGenerator: public StubCodeGenerator { StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ enter(); // required for proper stackwalking of RuntimeStub frame @@ -2826,16 +2785,13 @@ class StubGenerator: public StubCodeGenerator { }; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); BLOCK_COMMENT("Entry:"); @@ -2979,21 +2935,19 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_unsafecopy_common_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - address start_pc = __ pc(); + start = __ pc(); __ leave(); __ mov(r0, 0); __ ret(lr); // record the stub entry and end - store_archive_data(stub_id, start_pc, __ pc()); + store_archive_data(stub_id, start, __ pc()); - return start_pc; + return start; } // @@ -3017,19 +2971,17 @@ class StubGenerator: public StubCodeGenerator { // we expect one set of extra unsafememory access handler entries GrowableArray
extras; int extra_count = 3; - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end, nullptr, &extras); + address start = load_archive_data(stub_id, nullptr, &extras); + if (start != nullptr) { assert(extras.length() == extra_count, "unexpected extra entry count %d", extras.length()); - register_unsafe_access_handlers(extras, 0, 1, end); + register_unsafe_access_handlers(extras, 0, 1); return start; } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); Register dest = c_rarg0, count = c_rarg1, value = c_rarg2; Label tail; @@ -3139,18 +3091,14 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_data_cache_writeback_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - __ align(CodeEntryAlignment); - StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ enter(); __ cache_wb(Address(line, 0)); __ leave(); @@ -3166,24 +3114,19 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_data_cache_writeback_sync_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - const Register is_pre = c_rarg0; // pre or post sync - __ align(CodeEntryAlignment); - StubCodeMark mark(this, stub_id); // pre wbsync is a no-op // post wbsync translates to an sfence Label skip; - address start = __ pc(); + start = __ pc(); __ enter(); __ cbnz(is_pre, skip); __ cache_wbsync(false); @@ -3360,13 +3303,10 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_aescrypt_encryptBlock_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); @@ -3375,7 +3315,7 @@ class StubGenerator: public StubCodeGenerator { const Register key = c_rarg2; // key array address const Register keylen = rscratch1; - address start = __ pc(); + start = __ pc(); __ enter(); __ ldrw(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT))); @@ -3406,13 +3346,10 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_aescrypt_decryptBlock_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); Label L_doLast; @@ -3422,7 +3359,7 @@ class StubGenerator: public StubCodeGenerator { const Register key = c_rarg2; // key array address const Register keylen = rscratch1; - address start = __ pc(); + start = __ pc(); __ enter(); // required for proper stackwalking of RuntimeStub frame __ ldrw(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT))); @@ -3457,13 +3394,10 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_cipherBlockChaining_encryptAESCrypt_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); @@ -3477,7 +3411,7 @@ class StubGenerator: public StubCodeGenerator { const Register len_reg = c_rarg4; // src len (must be multiple of blocksize 16) const Register keylen = rscratch1; - address start = __ pc(); + start = __ pc(); __ enter(); @@ -3574,13 +3508,10 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_cipherBlockChaining_decryptAESCrypt_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); @@ -3594,7 +3525,7 @@ class StubGenerator: public StubCodeGenerator { const Register len_reg = c_rarg4; // src len (must be multiple of blocksize 16) const Register keylen = rscratch1; - address start = __ pc(); + start = __ pc(); __ enter(); @@ -3714,13 +3645,10 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_counterMode_AESCrypt_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - const Register in = c_rarg0; const Register out = c_rarg1; const Register key = c_rarg2; @@ -3782,7 +3710,7 @@ class StubGenerator: public StubCodeGenerator { __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - const address start = __ pc(); + const start = __ pc(); __ enter(); Label DONE, CTR_large_block, large_block_return; @@ -3989,16 +3917,13 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_galoisCounterMode_AESCrypt_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ enter(); const Register in = c_rarg0; @@ -4233,17 +4158,14 @@ class StubGenerator: public StubCodeGenerator { } int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); Register buf = c_rarg0; Register state = c_rarg1; @@ -4398,17 +4320,14 @@ class StubGenerator: public StubCodeGenerator { } int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); Register buf = c_rarg0; Register state = c_rarg1; @@ -4514,17 +4433,13 @@ class StubGenerator: public StubCodeGenerator { } int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - __ align(CodeEntryAlignment); - StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); Register buf = c_rarg0; Register state = c_rarg1; @@ -4662,17 +4577,13 @@ class StubGenerator: public StubCodeGenerator { } int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - __ align(CodeEntryAlignment); - StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); Register buf = c_rarg0; Register state = c_rarg1; @@ -4893,17 +4804,13 @@ class StubGenerator: public StubCodeGenerator { } int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - __ align(CodeEntryAlignment); - StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); Register buf = c_rarg0; Register state = c_rarg1; @@ -5040,19 +4947,15 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_double_keccak_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - // Implements the double_keccak() method of the // sun.secyrity.provider.SHA3Parallel class __ align(CodeEntryAlignment); - StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ enter(); Register state0 = c_rarg0; @@ -5159,17 +5062,14 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_chacha20Block_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - Label L_twoRounds, L_cc20_const; __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ enter(); int i, j; @@ -5818,16 +5718,13 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_kyberNtt_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ enter(); const Register coeffs = c_rarg0; @@ -6067,16 +5964,13 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_kyberInverseNtt_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ enter(); const Register coeffs = c_rarg0; @@ -6365,16 +6259,13 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_kyberNttMult_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ enter(); const Register result = c_rarg0; @@ -6493,16 +6384,13 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_kyberAddPoly_2_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ enter(); const Register result = c_rarg0; @@ -6589,16 +6477,13 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_kyberAddPoly_3_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ enter(); const Register result = c_rarg0; @@ -6703,18 +6588,14 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_kyber12To16_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - Label L_F00, L_loop, L_end; - __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ enter(); const Register condensed = c_rarg0; @@ -6924,16 +6805,13 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_kyberBarrettReduce_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ enter(); const Register coeffs = c_rarg0; @@ -7182,16 +7060,13 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_dilithiumAlmostNtt_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ enter(); const Register coeffs = c_rarg0; @@ -7400,16 +7275,13 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_dilithiumAlmostInverseNtt_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ enter(); const Register coeffs = c_rarg0; @@ -7524,16 +7396,13 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_dilithiumNttMult_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ enter(); Label L_loop; @@ -7599,16 +7468,13 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_dilithiumMontMulByConstant_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ enter(); Label L_loop; @@ -7674,16 +7540,13 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_dilithiumDecomposePoly_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); Label L_loop; const Register input = c_rarg0; @@ -7967,16 +7830,13 @@ class StubGenerator: public StubCodeGenerator { } int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); Register buf = c_rarg0; Register state = c_rarg1; @@ -8205,17 +8065,14 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_updateBytesCRC32_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); const Register crc = c_rarg0; // crc const Register buf = c_rarg1; // source java byte array address @@ -8258,17 +8115,14 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_updateBytesCRC32C_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); const Register crc = c_rarg0; // crc const Register buf = c_rarg1; // source java byte array address @@ -8309,16 +8163,13 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_updateBytesAdler32_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); Label L_simple_by1_loop, L_nmax, L_nmax_loop, L_by16, L_by16_loop, L_by1_loop, L_do_mod, L_combine, L_by1; @@ -8543,17 +8394,14 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_multiplyToLen_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); const Register x = r0; const Register xlen = r1; const Register y = r2; @@ -8588,16 +8436,13 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_squareToLen_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); const Register x = r0; const Register xlen = r1; @@ -8635,17 +8480,14 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_mulAdd_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); const Register out = r0; const Register in = r1; @@ -8678,16 +8520,13 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_bigIntegerRightShiftWorker_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); Label ShiftSIMDLoop, ShiftTwoLoop, ShiftThree, ShiftTwo, ShiftOne, Exit; @@ -8814,15 +8653,12 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_bigIntegerLeftShiftWorker_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); Label ShiftSIMDLoop, ShiftTwoLoop, ShiftThree, ShiftTwo, ShiftOne, Exit; @@ -8929,16 +8765,13 @@ class StubGenerator: public StubCodeGenerator { int entry_count = StubInfo::entry_count(stub_id); // We have an extra entry for count_positives_long. assert(entry_count == 2, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end, &entries); + address start = load_archive_data(stub_id, &entries); + if (start != nullptr) { assert(entries.length() == 1, "unexpected extra entry count %d", entries.length()); count_positives_long = entries.at(0); return start; } - const u1 large_loop_size = 64; const uint64_t UPPER_BIT_MASK=0x8080808080808080; int dcache_line = VM_Version::dcache_line_size(); @@ -8946,7 +8779,6 @@ class StubGenerator: public StubCodeGenerator { Register ary1 = r1, len = r2, result = r0; __ align(CodeEntryAlignment); - StubCodeMark mark(this, stub_id); address entry = __ pc(); @@ -9200,13 +9032,10 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_large_array_equals_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - Register a1 = r1, a2 = r2, result = r0, cnt1 = r10, tmp1 = rscratch1, tmp2 = rscratch2, tmp3 = r3, tmp4 = r4, tmp5 = r5, tmp6 = r11, tmp7 = r12, tmp8 = r13; @@ -9331,13 +9160,10 @@ class StubGenerator: public StubCodeGenerator { }; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - const Register result = r0, ary = r1, cnt = r2; const FloatRegister vdata0 = v3, vdata1 = v2, vdata2 = v1, vdata3 = v0; const FloatRegister vmul0 = v4, vmul1 = v5, vmul2 = v6, vmul3 = v7; @@ -9625,16 +9451,13 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = (isCos ? StubId::stubgen_dcos_id : StubId::stubgen_dsin_id); int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ generate_dsin_dcos(isCos, (address)StubRoutines::aarch64::_npio2_hw, (address)StubRoutines::aarch64::_two_over_pi, (address)StubRoutines::aarch64::_pio2, @@ -9690,13 +9513,10 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = (isLU ? StubId::stubgen_compare_long_string_LU_id : StubId::stubgen_compare_long_string_UL_id); int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address entry = __ pc(); @@ -9813,13 +9633,10 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_hf2f_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address entry = __ pc(); @@ -9838,13 +9655,10 @@ class StubGenerator: public StubCodeGenerator { // v1 = temporary float register address generate_floatToFloat16() { StubId stub_id = StubId::stubgen_f2hf_id; - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address entry = __ pc(); @@ -9862,19 +9676,16 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_method_entry_barrier_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); Label deoptimize_label; - address start = __ pc(); + start = __ pc(); BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler(); @@ -9940,15 +9751,11 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = (isLL ? StubId::stubgen_compare_long_string_LL_id : StubId::stubgen_compare_long_string_UU_id); int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - __ align(CodeEntryAlignment); - StubCodeMark mark(this, stub_id); address entry = __ pc(); Register result = r0, str1 = r1, cnt1 = r2, str2 = r3, cnt2 = r4, @@ -10093,13 +9900,10 @@ class StubGenerator: public StubCodeGenerator { } int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address entry = __ pc(); @@ -10246,13 +10050,10 @@ class StubGenerator: public StubCodeGenerator { } int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address entry = __ pc(); @@ -10558,13 +10359,10 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_large_byte_array_inflate_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address entry = __ pc(); @@ -10638,17 +10436,14 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_ghash_processBlocks_small_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - StubCodeMark mark(this, stub_id); Label polynomial; // local data generated at end of stub __ align(CodeEntryAlignment); - address start = __ pc(); + start = __ pc(); Register state = c_rarg0; Register subkeyH = c_rarg1; @@ -10718,17 +10513,14 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_ghash_processBlocks_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - Label polynomial; // local data generated after stub __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); Register state = c_rarg0; Register subkeyH = c_rarg1; @@ -10827,16 +10619,13 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_base64_encodeBlock_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); Register src = c_rarg0; // source array Register soff = c_rarg1; // source start offset @@ -11038,16 +10827,13 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_base64_decodeBlock_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); Register src = c_rarg0; // source array Register soff = c_rarg1; // source start offset @@ -11167,16 +10953,13 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_spin_wait_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ spin_wait(); __ ret(lr); @@ -11192,10 +10975,8 @@ class StubGenerator: public StubCodeGenerator { GrowableArray
entries; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == Klass::SECONDARY_SUPERS_TABLE_SIZE, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end, &entries); + address start = load_archive_data(stub_id, &entries); + if (start != nullptr) { assert(entries.length() == Klass::SECONDARY_SUPERS_TABLE_SIZE - 1, "unexpected extra entry count %d", entries.length()); StubRoutines::_lookup_secondary_supers_table_stubs[0] = start; @@ -11245,16 +11026,12 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_lookup_secondary_supers_table_slow_path_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - StubCodeMark mark(this, stub_id); - - address start = __ pc(); + start = __ pc(); const Register r_super_klass = r0, // argument r_array_base = r1, // argument @@ -11411,10 +11188,8 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_atomic_entry_points_id; GrowableArray
entries; int entry_count = StubInfo::entry_count(stub_id); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end, &entries); + address start = load_archive_data(stub_id, &entries); + if (start != nullptr) { assert(entries.length() == entry_count - 1, "unexpected extra entry count %d", entries.length()); aarch64_atomic_fetch_add_4_impl = (aarch64_atomic_stub_t)start; @@ -11440,7 +11215,7 @@ class StubGenerator: public StubCodeGenerator { __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address first_entry = __ pc(); + start = __ pc(); address end; { // ADD, memory_order_conservative @@ -11507,11 +11282,11 @@ class StubGenerator: public StubCodeGenerator { end = __ pc(); - ICache::invalidate_range(first_entry, end - first_entry); + ICache::invalidate_range(start, end - start); // exit block to force update of AtomicStubMark targets } - assert(first_entry == (address)aarch64_atomic_fetch_add_4_impl, + assert(start == (address)aarch64_atomic_fetch_add_4_impl, "atomic stub should be at start of buffer"); // record the stub start and end plus all the entries saved by the // AtomicStubMark destructor @@ -11534,7 +11309,7 @@ class StubGenerator: public StubCodeGenerator { assert(entries.length() == entry_count - 1, "unexpected extra entry count %d", entries.length()); - store_archive_data(stub_id, first_entry, end, &entries); + store_archive_data(stub_id, start, end, &entries); } #endif // LINUX @@ -11542,7 +11317,7 @@ class StubGenerator: public StubCodeGenerator { bool return_barrier = Continuation::is_thaw_return_barrier(kind); bool return_barrier_exception = Continuation::is_thaw_return_barrier_exception(kind); - address start = __ pc(); + start = __ pc(); if (return_barrier) { __ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset())); @@ -11640,15 +11415,12 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_cont_thaw_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); generate_cont_thaw(Continuation::thaw_top); // record the stub start and end @@ -11664,15 +11436,12 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_cont_returnBarrier_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); generate_cont_thaw(Continuation::thaw_return_barrier); @@ -11688,15 +11457,12 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_cont_returnBarrierExc_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); generate_cont_thaw(Continuation::thaw_return_barrier_exception); @@ -11711,15 +11477,12 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_cont_preempt_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ reset_last_Java_frame(true); @@ -11805,16 +11568,13 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_poly1305_processBlocks_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); Label here; __ enter(); RegSet callee_saved = RegSet::range(r19, r28); @@ -11933,15 +11693,12 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_upcall_stub_exception_handler_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); // Native caller has no idea how to handle exceptions, // so we just crash here. Up to callee to catch exceptions. @@ -11963,15 +11720,12 @@ class StubGenerator: public StubCodeGenerator { StubId stub_id = StubId::stubgen_upcall_stub_load_target_id; int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == 1, "sanity check"); - if (find_archive_data(stub_id)) { - address start = nullptr; - address end = nullptr; - load_archive_data(stub_id, start, end); + address start = load_archive_data(stub_id); + if (start != nullptr) { return start; } - StubCodeMark mark(this, stub_id); - address start = __ pc(); + start = __ pc(); __ resolve_global_jobject(j_rarg0, rscratch1, rscratch2); // Load target method from receiver From 022cb3c8349029794136db698fb98adb8c3d1615 Mon Sep 17 00:00:00 2001 From: Andrew Dinn Date: Wed, 26 Nov 2025 10:31:23 +0000 Subject: [PATCH 22/30] fix typos --- src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp index 4651095d33d3e..9d0f3e5ec3cce 100644 --- a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp @@ -3710,7 +3710,7 @@ class StubGenerator: public StubCodeGenerator { __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); - const start = __ pc(); + start = __ pc(); __ enter(); Label DONE, CTR_large_block, large_block_return; @@ -10999,7 +10999,6 @@ class StubGenerator: public StubCodeGenerator { const FloatRegister vtemp = v0; - address start; for (int slot = 0; slot < Klass::SECONDARY_SUPERS_TABLE_SIZE; slot++) { address next_entry = __ pc(); StubRoutines::_lookup_secondary_supers_table_stubs[slot] = next_entry; @@ -11317,7 +11316,7 @@ class StubGenerator: public StubCodeGenerator { bool return_barrier = Continuation::is_thaw_return_barrier(kind); bool return_barrier_exception = Continuation::is_thaw_return_barrier_exception(kind); - start = __ pc(); + address start = __ pc(); if (return_barrier) { __ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset())); From 29b478c221f1b6ec95736b88bb2a671d8e91db84 Mon Sep 17 00:00:00 2001 From: Andrew Dinn Date: Wed, 26 Nov 2025 10:38:03 +0000 Subject: [PATCH 23/30] correct assert --- src/hotspot/share/code/aotCodeCache.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/hotspot/share/code/aotCodeCache.cpp b/src/hotspot/share/code/aotCodeCache.cpp index 1d115c590fd0b..d7cbd4c456794 100644 --- a/src/hotspot/share/code/aotCodeCache.cpp +++ b/src/hotspot/share/code/aotCodeCache.cpp @@ -1355,7 +1355,6 @@ void AOTCodeReader::read_stub_data(CodeBlob* code_blob, AOTStubData* stub_data) // entry offset is an offset from the stub start less than or // equal to end uint entry = *(uint*)addr(offset); offset += sizeof(uint); - assert(entry <= end, "stub %s entry offset %d lies beyond stub end %d", StubInfo::name(stub_id), entry, end); if (entry <= end) { // entry addresses may not address end but extras can assert(entry < end || i >= StubInfo::entry_count(stub_id), @@ -1364,7 +1363,7 @@ void AOTCodeReader::read_stub_data(CodeBlob* code_blob, AOTStubData* stub_data) addresses.append(stub_start + entry); } else { // special case: entry encodes a nullptr - assert(entry == AOTCodeCache::NULL_ADDRESS_MARKER, "sanity"); + assert(entry == AOTCodeCache::NULL_ADDRESS_MARKER, "stub %s entry offset %d lies beyond stub end %d and does not equal NULL_ADDRESS_MARKER", StubInfo::name(stub_id), entry, end); addresses.append(nullptr); } } From 1a996eb94cb7d58bb1e1d5be2169b745fc92cf0a Mon Sep 17 00:00:00 2001 From: Andrew Dinn Date: Wed, 26 Nov 2025 10:40:31 +0000 Subject: [PATCH 24/30] remove redundant comment --- src/hotspot/cpu/aarch64/stubDeclarations_aarch64.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/hotspot/cpu/aarch64/stubDeclarations_aarch64.hpp b/src/hotspot/cpu/aarch64/stubDeclarations_aarch64.hpp index 95b59bd8590f5..9dac6a39b82bc 100644 --- a/src/hotspot/cpu/aarch64/stubDeclarations_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/stubDeclarations_aarch64.hpp @@ -139,7 +139,6 @@ do_stub(final, spin_wait) \ do_arch_entry_init(aarch64, final, spin_wait, spin_wait, \ spin_wait, empty_spin_wait) \ - /* stub only -- entries are not stored in StubRoutines::aarch64 */ \ /* n.b. these are not the same as the generic atomic stubs */ \ do_stub(final, atomic_entry_points) \ do_arch_entry(aarch64, final, atomic_entry_points, \ From af40c885a5bc4f4fa9396b61ac9818b87223ae6f Mon Sep 17 00:00:00 2001 From: Andrew Dinn Date: Wed, 26 Nov 2025 10:44:22 +0000 Subject: [PATCH 25/30] rmeove whitespace --- src/hotspot/share/runtime/stubRoutines.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/runtime/stubRoutines.cpp b/src/hotspot/share/runtime/stubRoutines.cpp index 82094b6909452..912a184b37d37 100644 --- a/src/hotspot/share/runtime/stubRoutines.cpp +++ b/src/hotspot/share/runtime/stubRoutines.cpp @@ -260,7 +260,7 @@ static BufferBlob* initialize_stubs(BlobId blob_id, if (lt.is_enabled()) { LogStream ls(lt); ls.print_cr("Failed to store blob '%s' to Startup Code Cache", buffer_name); - } + } } } From 59cc4b298f87414483a6c312fc1d687ab3caf095 Mon Sep 17 00:00:00 2001 From: Andrew Dinn Date: Wed, 26 Nov 2025 10:48:08 +0000 Subject: [PATCH 26/30] more whitespace --- src/hotspot/share/runtime/stubCodeGenerator.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/runtime/stubCodeGenerator.cpp b/src/hotspot/share/runtime/stubCodeGenerator.cpp index 48ee6757a6e21..bac26e5c66bae 100644 --- a/src/hotspot/share/runtime/stubCodeGenerator.cpp +++ b/src/hotspot/share/runtime/stubCodeGenerator.cpp @@ -209,7 +209,7 @@ void StubCodeGenerator::print_statistics_on(outputStream* st) { if (!strcmp(scd->group(), "StubRoutines")) { emitted += 1; if (scd->loaded_from_cache()) { - loaded_from_cache += 1; + loaded_from_cache += 1; } } scd = StubCodeDesc::next(scd); From 765d9d6e5ac1d3a83d2cdc67ba7772fb75461855 Mon Sep 17 00:00:00 2001 From: Andrew Dinn Date: Wed, 26 Nov 2025 13:56:14 +0000 Subject: [PATCH 27/30] fix extras count to match number of unsafe handler regions --- src/hotspot/cpu/x86/stubGenerator_x86_64_arraycopy.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_arraycopy.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_arraycopy.cpp index 44249b57b9974..c8bef71e530f4 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_arraycopy.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_arraycopy.cpp @@ -573,8 +573,11 @@ address StubGenerator::generate_disjoint_copy_avx3_masked(StubId stub_id, addres GrowableArray
entries; GrowableArray
extras; bool add_extras = !is_oop && !aligned; + // The stub employs one unsafe handler region by default but has two + // when MaxVectorSize == 64 So we may expect 0, 3 or 6 extras. + int handlers_count = (MaxVectorSize == 64 ? 2 : 1); int expected_entry_count = (entry != nullptr ? 2 : 1); - int expected_extra_count = 3 * (add_extras ? 2 : 0); // 0/2 x UMAM {start,end,handler} + int expected_extra_count = 3 * (add_extras ? handlers_count : 0); // 0/1/2 x UMAM {start,end,handler} int entry_count = StubInfo::entry_count(stub_id); assert(entry_count == expected_entry_count, "sanity check"); GrowableArray
* entries_ptr = (entry_count == 1 ? nullptr : &entries); From f3d7bfaeca1d8b35032ec17933d4129cf935312b Mon Sep 17 00:00:00 2001 From: Andrew Dinn Date: Wed, 26 Nov 2025 17:12:00 +0000 Subject: [PATCH 28/30] put AOT address table init code under INCLUDE_CDS --- src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp | 4 ++++ src/hotspot/cpu/aarch64/stubRoutines_aarch64.cpp | 2 ++ src/hotspot/cpu/aarch64/stubRoutines_aarch64.hpp | 3 +++ src/hotspot/cpu/arm/stubGenerator_arm.cpp | 2 ++ src/hotspot/cpu/ppc/stubGenerator_ppc.cpp | 2 ++ src/hotspot/cpu/riscv/stubGenerator_riscv.cpp | 2 ++ src/hotspot/cpu/s390/stubGenerator_s390.cpp | 2 ++ src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp | 2 ++ src/hotspot/cpu/x86/c1_LIRAssembler_x86.hpp | 2 ++ src/hotspot/cpu/x86/stubGenerator_x86_64.cpp | 2 ++ src/hotspot/cpu/x86/stubGenerator_x86_64.hpp | 4 ++++ src/hotspot/cpu/x86/stubGenerator_x86_64_adler.cpp | 2 ++ src/hotspot/cpu/x86/stubGenerator_x86_64_aes.cpp | 2 ++ src/hotspot/cpu/x86/stubGenerator_x86_64_cbrt.cpp | 3 +++ src/hotspot/cpu/x86/stubGenerator_x86_64_chacha.cpp | 3 +++ src/hotspot/cpu/x86/stubGenerator_x86_64_constants.cpp | 2 ++ src/hotspot/cpu/x86/stubGenerator_x86_64_dilithium.cpp | 2 ++ src/hotspot/cpu/x86/stubGenerator_x86_64_exp.cpp | 2 ++ src/hotspot/cpu/x86/stubGenerator_x86_64_fmod.cpp | 2 ++ src/hotspot/cpu/x86/stubGenerator_x86_64_ghash.cpp | 2 ++ src/hotspot/cpu/x86/stubGenerator_x86_64_kyber.cpp | 2 ++ src/hotspot/cpu/x86/stubGenerator_x86_64_log.cpp | 2 ++ src/hotspot/cpu/x86/stubGenerator_x86_64_poly1305.cpp | 2 ++ src/hotspot/cpu/x86/stubGenerator_x86_64_poly_mont.cpp | 2 ++ src/hotspot/cpu/x86/stubGenerator_x86_64_pow.cpp | 2 ++ src/hotspot/cpu/x86/stubGenerator_x86_64_sha3.cpp | 2 ++ src/hotspot/cpu/x86/stubGenerator_x86_64_sin.cpp | 2 ++ src/hotspot/cpu/x86/stubGenerator_x86_64_sinh.cpp | 2 ++ src/hotspot/cpu/x86/stubGenerator_x86_64_tan.cpp | 2 ++ src/hotspot/cpu/x86/stubGenerator_x86_64_tanh.cpp | 2 ++ src/hotspot/cpu/x86/stubRoutines_x86.cpp | 2 ++ src/hotspot/cpu/zero/stubGenerator_zero.cpp | 2 ++ src/hotspot/share/runtime/init.cpp | 4 ++++ src/hotspot/share/runtime/stubRoutines.cpp | 4 ++++ src/hotspot/share/runtime/stubRoutines.hpp | 2 ++ 35 files changed, 81 insertions(+) diff --git a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp index 9d0f3e5ec3cce..1a18f5f7b85ea 100644 --- a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp @@ -12840,6 +12840,7 @@ class StubGenerator: public StubCodeGenerator { }; } +#if INCLUDE_CDS static void init_AOTAddressTable(GrowableArray
& external_addresses) { // external data defined in this file #define ADD(addr) external_addresses.append((address)addr); @@ -12855,8 +12856,10 @@ class StubGenerator: public StubCodeGenerator { ADD(_decodeBlock_fromBase64URLForSIMD); #undef ADD } +#endif // INCLUDE_CDS }; // end class declaration +#if INCLUDE_CDS void StubGenerator_AOTAddressTable_init() { ResourceMark rm; GrowableArray
external_addresses; @@ -12866,6 +12869,7 @@ void StubGenerator_AOTAddressTable_init() { StubRoutines::aarch64::init_AOTAddressTable(external_addresses); AOTCodeCache::publish_external_addresses(external_addresses); } +#endif // INCLUDE_CDS void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) { StubGenerator g(code, blob_id, stub_data); diff --git a/src/hotspot/cpu/aarch64/stubRoutines_aarch64.cpp b/src/hotspot/cpu/aarch64/stubRoutines_aarch64.cpp index 73f0797cb7aca..601dd2c67b681 100644 --- a/src/hotspot/cpu/aarch64/stubRoutines_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/stubRoutines_aarch64.cpp @@ -414,6 +414,7 @@ ATTRIBUTE_ALIGNED(64) jdouble StubRoutines::aarch64::_pio2[] = { 2.16741683877804819444e-51, // 0x3569F31D00000000 }; +#if INCLUDE_CDS #define ADD(addr) external_addresses.append((address)addr); void StubRoutines::aarch64::init_AOTAddressTable(GrowableArray
& external_addresses) { @@ -429,3 +430,4 @@ void StubRoutines::aarch64::init_AOTAddressTable(GrowableArray
& externa } #undef ADD +#endif // INCLUDE_CDS diff --git a/src/hotspot/cpu/aarch64/stubRoutines_aarch64.hpp b/src/hotspot/cpu/aarch64/stubRoutines_aarch64.hpp index 89c72a6a5ea11..f77192a374185 100644 --- a/src/hotspot/cpu/aarch64/stubRoutines_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/stubRoutines_aarch64.hpp @@ -110,7 +110,10 @@ class aarch64 { _completed = true; } +#if INCLUDE_CDS static void init_AOTAddressTable(GrowableArray
& external_addresses); +#endif // INCLUDE_CDS + private: static uint16_t _kyberConsts[]; diff --git a/src/hotspot/cpu/arm/stubGenerator_arm.cpp b/src/hotspot/cpu/arm/stubGenerator_arm.cpp index 16155203278e2..d267e4f373c5a 100644 --- a/src/hotspot/cpu/arm/stubGenerator_arm.cpp +++ b/src/hotspot/cpu/arm/stubGenerator_arm.cpp @@ -3235,8 +3235,10 @@ class StubGenerator: public StubCodeGenerator { } }; // end class declaration +#if INCLUDE_CDS // nothing to do for arm void StubGenerator_AOTAddressTable_init() { +#endif // INCLUDE_CDS } void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) { diff --git a/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp b/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp index 2156f742c3a10..fd1e3a2adeaa3 100644 --- a/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp +++ b/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp @@ -5119,8 +5119,10 @@ void generate_lookup_secondary_supers_table_stub() { } }; +#if INCLUDE_CDS // nothing to do for ppc void StubGenerator_AOTAddressTable_init() { +#endif // INCLUDE_CDS } void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData *stub_data) { diff --git a/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp b/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp index e8bd832612a4d..4d1293286db30 100644 --- a/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp +++ b/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp @@ -6906,8 +6906,10 @@ static const int64_t right_3_bits = right_n_bits(3); } }; // end class declaration +#if INCLUDE_CDS // nothing to do for riscv void StubGenerator_AOTAddressTable_init() { +#endif // INCLUDE_CDS } void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) { diff --git a/src/hotspot/cpu/s390/stubGenerator_s390.cpp b/src/hotspot/cpu/s390/stubGenerator_s390.cpp index 56ba3a85cbae4..18da3ece6fbdd 100644 --- a/src/hotspot/cpu/s390/stubGenerator_s390.cpp +++ b/src/hotspot/cpu/s390/stubGenerator_s390.cpp @@ -3479,8 +3479,10 @@ class StubGenerator: public StubCodeGenerator { }; +#if INCLUDE_CDS // nothing to do for s390 void StubGenerator_AOTAddressTable_init() { +#endif // INCLUDE_CDS } void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) { diff --git a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp index 14a2ea6f308b8..f01eca899c58c 100644 --- a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp @@ -69,6 +69,7 @@ static jlong *double_signmask_pool = double_quadword(&fp_signmask_pool[2*2], static jlong *float_signflip_pool = double_quadword(&fp_signmask_pool[3*2], (jlong)UCONST64(0x8000000080000000), (jlong)UCONST64(0x8000000080000000)); static jlong *double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], (jlong)UCONST64(0x8000000000000000), (jlong)UCONST64(0x8000000000000000)); +#if INCLUDE_CDS // publish external addresses defined in this file void LIR_Assembler::init_AOTAddressTable(GrowableArray
& external_addresses) { #define ADD(addr) external_addresses.append((address)addr); @@ -78,6 +79,7 @@ void LIR_Assembler::init_AOTAddressTable(GrowableArray
& external_addres ADD(double_signflip_pool); #undef ADD } +#endif // INCLUDE_CDS NEEDS_CLEANUP // remove this definitions ? const Register SYNC_header = rax; // synchronization header diff --git a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.hpp b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.hpp index c8050d5242be2..88fee305d67af 100644 --- a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.hpp +++ b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.hpp @@ -58,5 +58,7 @@ void store_parameter(jobject c, int offset_from_esp_in_words); void store_parameter(Metadata* c, int offset_from_esp_in_words); +#if INCLUDE_CDS void static init_AOTAddressTable(GrowableArray
& external_addresses); +#endif // INCLUDE_CDS #endif // CPU_X86_C1_LIRASSEMBLER_X86_HPP diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp index af583cf860471..950197ead6b7c 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp @@ -5026,6 +5026,7 @@ StubGenerator::StubGenerator(CodeBuffer* code, BlobId blob_id, AOTStubData* stub }; } +#if INCLUDE_CDS // publish addresses of static data defined in this file and in other // stubgen stub generator files void StubGenerator::init_AOTAddressTable(GrowableArray
& external_addresses) { @@ -5064,6 +5065,7 @@ void StubGenerator_AOTAddressTable_init() { #endif AOTCodeCache::publish_external_addresses(external_addresses); } +#endif // INCLUDE_CDS void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) { StubGenerator g(code, blob_id, stub_data); diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64.hpp b/src/hotspot/cpu/x86/stubGenerator_x86_64.hpp index b72a965ba905a..264e4dc502853 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64.hpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64.hpp @@ -637,6 +637,7 @@ class StubGenerator: public StubCodeGenerator { void generate_compiler_stubs(); void generate_final_stubs(); +#if INCLUDE_CDS static void init_AOTAddressTable_adler(GrowableArray
& external_addresses); static void init_AOTAddressTable_aes(GrowableArray
& external_addresses); static void init_AOTAddressTable_cbrt(GrowableArray
& external_addresses); @@ -657,10 +658,13 @@ class StubGenerator: public StubCodeGenerator { static void init_AOTAddressTable_sinh(GrowableArray
& external_addresses); static void init_AOTAddressTable_tan(GrowableArray
& external_addresses); static void init_AOTAddressTable_tanh(GrowableArray
& external_addresses); +#endif // INCLUDE_CDS public: StubGenerator(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data); +#if INCLUDE_CDS static void init_AOTAddressTable(GrowableArray
& external_addresses); +#endif // INCLUDE_CDS }; #endif // CPU_X86_STUBGENERATOR_X86_64_HPP diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_adler.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_adler.cpp index 66928c6e92893..2ea6ac244e2e6 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_adler.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_adler.cpp @@ -347,6 +347,7 @@ address StubGenerator::generate_updateBytesAdler32() { #undef __ +#if INCLUDE_CDS void StubGenerator::init_AOTAddressTable_adler(GrowableArray
& external_addresses) { #define ADD(addr) external_addresses.append((address)(addr)) ADD(ADLER32_ASCALE_TABLE); @@ -354,3 +355,4 @@ void StubGenerator::init_AOTAddressTable_adler(GrowableArray
& external_ ADD(ADLER32_SHUF1_TABLE); #undef ADD } +#endif // INCLUDE_CDS diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_aes.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_aes.cpp index e7572126a27f7..fbb01bb830c54 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_aes.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_aes.cpp @@ -4193,6 +4193,7 @@ void StubGenerator::aesgcm_avx2(Register in, Register len, Register ct, Register #undef __ +#if INCLUDE_CDS void StubGenerator::init_AOTAddressTable_aes(GrowableArray
& external_addresses) { #define ADD(addr) external_addresses.append((address)(addr)) ADD(key_shuffle_mask_addr()); @@ -4214,3 +4215,4 @@ void StubGenerator::init_AOTAddressTable_aes(GrowableArray
& external_ad ADD(counter_mask_add_1234_addr()); #undef ADD } +#endif // INCLUDE_CDS diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_cbrt.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_cbrt.cpp index 444e9ad191f2d..4c647b7d9dcef 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_cbrt.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_cbrt.cpp @@ -348,6 +348,8 @@ address StubGenerator::generate_libmCbrt() { } #undef __ + +#if INCLUDE_CDS void StubGenerator::init_AOTAddressTable_cbrt(GrowableArray
& external_addresses) { #define ADD(addr) external_addresses.append((address)(addr)) ADD(_ABS_MASK); @@ -369,3 +371,4 @@ void StubGenerator::init_AOTAddressTable_cbrt(GrowableArray
& external_a ADD(_D_table); #undef ADD } +#endif // INCLUDE_CDS diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_chacha.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_chacha.cpp index 8bce50fd638dc..1fa51cd2f1869 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_chacha.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_chacha.cpp @@ -604,6 +604,8 @@ bVec, } #undef __ + +#if INCLUDE_CDS void StubGenerator::init_AOTAddressTable_chacha(GrowableArray
& external_addresses) { #define ADD(addr) external_addresses.append((address)(addr)) ADD(CC20_COUNTER_ADD_AVX); @@ -611,3 +613,4 @@ void StubGenerator::init_AOTAddressTable_chacha(GrowableArray
& external ADD(CC20_LROT_CONSTS); #undef ADD } +#endif // INCLUDE_CDS diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_constants.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_constants.cpp index af02736347c86..45c13b7b39728 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_constants.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_constants.cpp @@ -233,6 +233,7 @@ ATTRIBUTE_ALIGNED(16) static const juint _Ctable[] = { }; address StubGenerator::Ctable = (address)_Ctable; +#if INCLUDE_CDS void StubGenerator::init_AOTAddressTable_constants(GrowableArray
& external_addresses) { #define ADD(addr) external_addresses.append((address)(addr)) ADD(_ONE); @@ -257,3 +258,4 @@ void StubGenerator::init_AOTAddressTable_constants(GrowableArray
& exter ADD(_Ctable); #undef ADD } +#endif // INCLUDE_CDS diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_dilithium.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_dilithium.cpp index 1f52b4867f385..de8f52a3c0569 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_dilithium.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_dilithium.cpp @@ -1386,6 +1386,7 @@ void StubGenerator::generate_dilithium_stubs() { } } +#if INCLUDE_CDS void StubGenerator::init_AOTAddressTable_dilithium(GrowableArray
& external_addresses) { #define ADD(addr) external_addresses.append((address)(addr)) // use accessors to correctly identify the relevant addresses @@ -1401,3 +1402,4 @@ void StubGenerator::init_AOTAddressTable_dilithium(GrowableArray
& exter ADD(dilithiumAvx512ConstsAddr(barrettAddendIdx)); #undef ADD } +#endif // INCLUDE_CDS diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_exp.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_exp.cpp index 2d36cd919fc41..2ed9858bf0c7d 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_exp.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_exp.cpp @@ -395,6 +395,7 @@ address StubGenerator::generate_libmExp() { #undef __ +#if INCLUDE_CDS void StubGenerator::init_AOTAddressTable_exp(GrowableArray
& external_addresses) { #define ADD(addr) external_addresses.append((address)addr); ADD(_cv); @@ -413,3 +414,4 @@ void StubGenerator::init_AOTAddressTable_exp(GrowableArray
& external_ad ADD(_INF); #undef ADD } +#endif // INCLUDE_CDS diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_fmod.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_fmod.cpp index 3b03497ae3176..86d5efaf9f91d 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_fmod.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_fmod.cpp @@ -535,6 +535,7 @@ address StubGenerator::generate_libmFmod() { #undef __ +#if INCLUDE_CDS void StubGenerator::init_AOTAddressTable_fmod(GrowableArray
& external_addresses) { #define ADD(addr) external_addresses.append((address)addr); ADD(CONST_NaN); @@ -544,3 +545,4 @@ void StubGenerator::init_AOTAddressTable_fmod(GrowableArray
& external_a ADD(CONST_e307); #undef ADD } +#endif // INCLUDE_CDS diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_ghash.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_ghash.cpp index e9eb9b64838cd..557fe62335103 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_ghash.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_ghash.cpp @@ -556,6 +556,7 @@ void StubGenerator::generateHtbl_eight_blocks(Register htbl) { #undef __ +#if INCLUDE_CDS void StubGenerator::init_AOTAddressTable_ghash(GrowableArray
& external_addresses) { #define ADD(addr) external_addresses.append((address)addr); ADD(GHASH_SHUFFLE_MASK); @@ -564,3 +565,4 @@ void StubGenerator::init_AOTAddressTable_ghash(GrowableArray
& external_ ADD(GHASH_POLYNOMIAL); #undef ADD } +#endif // INCLUDE_CDS diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_kyber.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_kyber.cpp index 6b5081401a8da..f5647b9b5c0cd 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_kyber.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_kyber.cpp @@ -1008,6 +1008,7 @@ void StubGenerator::generate_kyber_stubs() { } } +#if INCLUDE_CDS void StubGenerator::init_AOTAddressTable_kyber(GrowableArray
& external_addresses) { #define ADD(addr) external_addresses.append((address)(addr)) // use accessors to correctly identify the relevant addresses @@ -1023,3 +1024,4 @@ void StubGenerator::init_AOTAddressTable_kyber(GrowableArray
& external_ ADD(kyberAvx512ConstsAddr(f00Offset)); #undef ADD } +#endif // INCLUDE_CDS diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_log.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_log.cpp index 70ca9d3cf75d1..8849597c94b15 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_log.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_log.cpp @@ -727,6 +727,7 @@ address StubGenerator::generate_libmLog10() { #undef __ +#if INCLUDE_CDS void StubGenerator::init_AOTAddressTable_log(GrowableArray
& external_addresses) { #define ADD(addr) external_addresses.append((address)addr); ADD(_L_tbl); @@ -746,3 +747,4 @@ void StubGenerator::init_AOTAddressTable_log(GrowableArray
& external_ad ADD(((address)_coeff_log10+32)); #undef ADD } +#endif // INCLUDE_CDS diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_poly1305.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_poly1305.cpp index 2c794e201cac2..1d0e961c82d92 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_poly1305.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_poly1305.cpp @@ -1707,6 +1707,7 @@ void StubGenerator::poly1305_msg_mul_reduce_vec4_avx2( } #undef __ +#if INCLUDE_CDS void StubGenerator::init_AOTAddressTable_poly1305(GrowableArray
& external_addresses) { #define ADD(addr) external_addresses.append((address)addr); ADD(POLY1305_PAD_MSG); @@ -1714,3 +1715,4 @@ void StubGenerator::init_AOTAddressTable_poly1305(GrowableArray
& extern ADD(POLY1305_MASK44); #undef ADD } +#endif // INCLUDE_CDS diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_poly_mont.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_poly_mont.cpp index 0fc4a62a24c31..4648fe03aa049 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_poly_mont.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_poly_mont.cpp @@ -786,6 +786,7 @@ address StubGenerator::generate_intpoly_assign() { } #undef __ +#if INCLUDE_CDS void StubGenerator::init_AOTAddressTable_poly_mont(GrowableArray
& external_addresses) { #define ADD(addr) external_addresses.append((address)addr); // use accessors to retrieve all correct addresses @@ -797,3 +798,4 @@ void StubGenerator::init_AOTAddressTable_poly_mont(GrowableArray
& exter ADD(modulus_p256(1)); #undef ADD } +#endif // INCLUDE_CDS diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_pow.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_pow.cpp index a81064868038a..5ff09e2b37753 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_pow.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_pow.cpp @@ -1873,6 +1873,7 @@ address StubGenerator::generate_libmPow() { #undef __ +#if INCLUDE_CDS void StubGenerator::init_AOTAddressTable_pow(GrowableArray
& external_addresses) { #define ADD(addr) external_addresses.append((address)addr); ADD(_HIGHSIGMASK); @@ -1900,3 +1901,4 @@ void StubGenerator::init_AOTAddressTable_pow(GrowableArray
& external_ad ADD(_DOUBLE0DOT5); #undef ADD } +#endif // INCLUDE_CDS diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_sha3.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_sha3.cpp index 7b5ee166c6dc0..075d25dcac89d 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_sha3.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_sha3.cpp @@ -528,9 +528,11 @@ void StubGenerator::generate_sha3_stubs() { #undef __ +#if INCLUDE_CDS void StubGenerator::init_AOTAddressTable_sha3(GrowableArray
& external_addresses) { #define ADD(addr) external_addresses.append((address)addr); ADD(round_constsAddr()); ADD(permsAndRotsAddr()); #undef ADD } +#endif // INCLUDE_CDS diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_sin.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_sin.cpp index 7f68cdf9bfd01..eaeaea2c5662e 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_sin.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_sin.cpp @@ -659,8 +659,10 @@ address StubGenerator::generate_libmSin() { #undef __ +#if INCLUDE_CDS void StubGenerator::init_AOTAddressTable_sin(GrowableArray
& external_addresses) { #define ADD(addr) external_addresses.append((address)addr); ADD(_ALL_ONES); #undef ADD } +#endif // INCLUDE_CDS diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_sinh.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_sinh.cpp index aa1d80348d9c4..f6e1d241948f5 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_sinh.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_sinh.cpp @@ -533,6 +533,7 @@ address StubGenerator::generate_libmSinh() { #undef __ +#if INCLUDE_CDS void StubGenerator::init_AOTAddressTable_sinh(GrowableArray
& external_addresses) { #define ADD(addr) external_addresses.append((address)addr); ADD(_L2E); @@ -552,3 +553,4 @@ void StubGenerator::init_AOTAddressTable_sinh(GrowableArray
& external_a ADD(_MASK3); #undef ADD } +#endif // INCLUDE_CDS diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_tan.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_tan.cpp index f03a4df2c34f5..3bfa5a7277fce 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_tan.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_tan.cpp @@ -1039,6 +1039,7 @@ address StubGenerator::generate_libmTan() { #undef __ +#if INCLUDE_CDS void StubGenerator::init_AOTAddressTable_tan(GrowableArray
& external_addresses) { #define ADD(addr) external_addresses.append((address)addr); ADD(_MUL16); @@ -1059,3 +1060,4 @@ void StubGenerator::init_AOTAddressTable_tan(GrowableArray
& external_ad ADD(_QQ_2_tan); #undef ADD } +#endif // INCLUDE_CDS diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_tanh.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_tanh.cpp index a1bc8143e863e..dcf5f3eb824d2 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_tanh.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_tanh.cpp @@ -509,6 +509,7 @@ address StubGenerator::generate_libmTanh() { #undef __ +#if INCLUDE_CDS void StubGenerator::init_AOTAddressTable_tanh(GrowableArray
& external_addresses) { #define ADD(addr) external_addresses.append((address)addr); ADD(_L2E); @@ -528,3 +529,4 @@ void StubGenerator::init_AOTAddressTable_tanh(GrowableArray
& external_a ADD(_RMASK); #undef ADD } +#endif // INCLUDE_CDS diff --git a/src/hotspot/cpu/x86/stubRoutines_x86.cpp b/src/hotspot/cpu/x86/stubRoutines_x86.cpp index 0a9a1838b4cf6..7ad8c21847585 100644 --- a/src/hotspot/cpu/x86/stubRoutines_x86.cpp +++ b/src/hotspot/cpu/x86/stubRoutines_x86.cpp @@ -412,6 +412,7 @@ ATTRIBUTE_ALIGNED(64) const julong StubRoutines::x86::_k512_W[] = 0x5fcb6fab3ad6faecULL, 0x6c44198c4a475817ULL, }; +#if INCLUDE_CDS // publish addresses of external data defined in this file which may // be referenced from stub or code void StubRoutines::x86::init_AOTAddressTable(GrowableArray
& external_addresses) { @@ -436,3 +437,4 @@ void StubRoutines::x86::init_AOTAddressTable(GrowableArray
& external_ad ADD(_k512_W); #undef ADD } +#endif // INCLUDE_CDS diff --git a/src/hotspot/cpu/zero/stubGenerator_zero.cpp b/src/hotspot/cpu/zero/stubGenerator_zero.cpp index d46da4b3723ab..6b82004f124fb 100644 --- a/src/hotspot/cpu/zero/stubGenerator_zero.cpp +++ b/src/hotspot/cpu/zero/stubGenerator_zero.cpp @@ -237,9 +237,11 @@ class StubGenerator: public StubCodeGenerator { } }; +#if INCLUDE_CDS // nothing to do for ZERO void StubGenerator_AOTAddressTable_init() { } +#endif // INCLUDE_CDS void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData *stub_data) { StubGenerator g(code, blob_id, stub_data); diff --git a/src/hotspot/share/runtime/init.cpp b/src/hotspot/share/runtime/init.cpp index 463460fe3d57a..d820968495e96 100644 --- a/src/hotspot/share/runtime/init.cpp +++ b/src/hotspot/share/runtime/init.cpp @@ -71,7 +71,9 @@ void icache_init2(); void initialize_stub_info(); // must precede all blob/stub generation void preuniverse_stubs_init(); +#if INCLUDE_CDS void stubs_AOTAddressTable_init(); +#endif // INCLUDE_CDS void initial_stubs_init(); jint universe_init(); // depends on codeCache_init and preuniverse_stubs_init @@ -151,8 +153,10 @@ jint init_globals() { AOTCodeCache::init2(); // depends on universe_init, must be before initial_stubs_init AsyncLogWriter::initialize(); +#if INCLUDE_CDS stubs_AOTAddressTable_init(); // publish external addresses used by stubs // depends on AOTCodeCache::init2 +#endif // INCLUDE_CDS initial_stubs_init(); // stubgen initial stub routines // stack overflow exception blob is referenced by the interpreter SharedRuntime::generate_initial_stubs(); diff --git a/src/hotspot/share/runtime/stubRoutines.cpp b/src/hotspot/share/runtime/stubRoutines.cpp index 912a184b37d37..5dd85ba821da2 100644 --- a/src/hotspot/share/runtime/stubRoutines.cpp +++ b/src/hotspot/share/runtime/stubRoutines.cpp @@ -103,9 +103,11 @@ BlobId StubRoutines::stub_to_blob(StubId id) { // Initialization extern void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data); // only interface to generators +#if INCLUDE_CDS #if (defined(X86) && defined(_LP64)) || defined(AARCH64) extern void StubGenerator_AOTAddressTable_init(); #endif +#endif // INCLUDE_CDS void UnsafeMemoryAccess::create_table(int max_size) { UnsafeMemoryAccess::_table = new UnsafeMemoryAccess[max_size]; @@ -313,6 +315,7 @@ STUBGEN_BLOBS_DO(DEFINE_BLOB_INIT_FUNCTION) #undef DEFINE_BLOB_INIT_FUNCTION +#if INCLUDE_CDS // Non-generated init method void StubRoutines::init_AOTAddressTable() { @@ -324,6 +327,7 @@ void StubRoutines::init_AOTAddressTable() { // non-generated external API init driver function void stubs_AOTAddressTable_init() { StubRoutines::init_AOTAddressTable(); } +#endif // INCLUDE_CDS /* * we generate the underlying driver function compiler_stubs_init() diff --git a/src/hotspot/share/runtime/stubRoutines.hpp b/src/hotspot/share/runtime/stubRoutines.hpp index b2384c33a998d..e4d5c14e6f0b1 100644 --- a/src/hotspot/share/runtime/stubRoutines.hpp +++ b/src/hotspot/share/runtime/stubRoutines.hpp @@ -284,8 +284,10 @@ class StubRoutines: AllStatic { static BlobId stub_to_blob(StubId id); #endif +#if INCLUDE_CDS // Initalization static void init_AOTAddressTable(); +#endif // INCLUDE_CDS // Debugging static jint verify_oop_count() { return _verify_oop_count; } From 094e1738ce0e4b385a729f2d868a80da65b9da79 Mon Sep 17 00:00:00 2001 From: Andrew Dinn Date: Thu, 27 Nov 2025 14:14:20 +0000 Subject: [PATCH 29/30] move AOT address init impl into arch tree below StubRoutines --- .../cpu/aarch64/stubGenerator_aarch64.cpp | 17 +++++------------ .../cpu/aarch64/stubRoutines_aarch64.cpp | 17 +++++++++++++++++ src/hotspot/cpu/arm/stubGenerator_arm.cpp | 6 ------ src/hotspot/cpu/arm/stubRoutines_arm.cpp | 6 ++++++ src/hotspot/cpu/ppc/stubGenerator_ppc.cpp | 6 ------ src/hotspot/cpu/ppc/stubRoutines_ppc_64.cpp | 6 ++++++ src/hotspot/cpu/riscv/stubGenerator_riscv.cpp | 6 ------ src/hotspot/cpu/riscv/stubRoutines_riscv.cpp | 6 ++++++ src/hotspot/cpu/s390/stubGenerator_s390.cpp | 6 ------ src/hotspot/cpu/s390/stubRoutines_s390.cpp | 6 ++++++ src/hotspot/cpu/x86/stubGenerator_x86_64.cpp | 14 -------------- src/hotspot/cpu/x86/stubRoutines_x86.cpp | 15 +++++++++++++++ src/hotspot/cpu/zero/stubGenerator_zero.cpp | 6 ------ src/hotspot/cpu/zero/stubRoutines_zero.cpp | 6 ++++++ src/hotspot/share/runtime/stubRoutines.cpp | 14 -------------- src/hotspot/share/runtime/stubRoutines.hpp | 2 +- 16 files changed, 68 insertions(+), 71 deletions(-) diff --git a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp index 1a18f5f7b85ea..874c8cf5c0bb8 100644 --- a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp @@ -12859,22 +12859,15 @@ class StubGenerator: public StubCodeGenerator { #endif // INCLUDE_CDS }; // end class declaration -#if INCLUDE_CDS -void StubGenerator_AOTAddressTable_init() { - ResourceMark rm; - GrowableArray
external_addresses; - // publish static addresses referred to by aarch64 generator - StubGenerator::init_AOTAddressTable(external_addresses); - // publish external data addresses defined in nested aarch64 class - StubRoutines::aarch64::init_AOTAddressTable(external_addresses); - AOTCodeCache::publish_external_addresses(external_addresses); -} -#endif // INCLUDE_CDS - void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) { StubGenerator g(code, blob_id, stub_data); } +#if INCLUDE_CDS +void StubGenerator_init_AOTAddressTable(GrowableArray
& addresses) { + StubGenerator::init_AOTAddressTable(addresses); +} +#endif // INCLUDE_CDS #if defined (LINUX) diff --git a/src/hotspot/cpu/aarch64/stubRoutines_aarch64.cpp b/src/hotspot/cpu/aarch64/stubRoutines_aarch64.cpp index 601dd2c67b681..971f16724ded9 100644 --- a/src/hotspot/cpu/aarch64/stubRoutines_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/stubRoutines_aarch64.cpp @@ -415,6 +415,23 @@ ATTRIBUTE_ALIGNED(64) jdouble StubRoutines::aarch64::_pio2[] = { }; #if INCLUDE_CDS +extern void StubGenerator_init_AOTAddressTable(GrowableArray
& addresses); + +void StubRoutines::init_AOTAddressTable() { + ResourceMark rm; + GrowableArray
external_addresses; + // publish static addresses referred to by aarch64 generator + // n.b. we have to use use an extern call here because class + // StubGenerator, which provides the static method that knows how to + // add the relevant addresses, is declared in a source file rather + // than in a separately includeable header. + StubGenerator_init_AOTAddressTable(external_addresses); + // publish external data addresses defined in nested aarch64 class + StubRoutines::aarch64::init_AOTAddressTable(external_addresses); + AOTCodeCache::publish_external_addresses(external_addresses); +} + + #define ADD(addr) external_addresses.append((address)addr); void StubRoutines::aarch64::init_AOTAddressTable(GrowableArray
& external_addresses) { diff --git a/src/hotspot/cpu/arm/stubGenerator_arm.cpp b/src/hotspot/cpu/arm/stubGenerator_arm.cpp index d267e4f373c5a..a705b15eff523 100644 --- a/src/hotspot/cpu/arm/stubGenerator_arm.cpp +++ b/src/hotspot/cpu/arm/stubGenerator_arm.cpp @@ -3235,12 +3235,6 @@ class StubGenerator: public StubCodeGenerator { } }; // end class declaration -#if INCLUDE_CDS -// nothing to do for arm -void StubGenerator_AOTAddressTable_init() { -#endif // INCLUDE_CDS -} - void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) { StubGenerator g(code, blob_id, stub_data); } diff --git a/src/hotspot/cpu/arm/stubRoutines_arm.cpp b/src/hotspot/cpu/arm/stubRoutines_arm.cpp index a4f2b5e1bd915..3ed747ea11a26 100644 --- a/src/hotspot/cpu/arm/stubRoutines_arm.cpp +++ b/src/hotspot/cpu/arm/stubRoutines_arm.cpp @@ -39,3 +39,9 @@ STUBGEN_ARCH_ENTRIES_DO(DEFINE_ARCH_ENTRY, DEFINE_ARCH_ENTRY_INIT) address StubRoutines::crc_table_addr() { ShouldNotCallThis(); return nullptr; } address StubRoutines::crc32c_table_addr() { ShouldNotCallThis(); return nullptr; } + +#if INCLUDE_CDS +// nothing to do for arm +void StubRoutines::init_AOTAddressTable() { +} +#endif // INCLUDE_CDS diff --git a/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp b/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp index fd1e3a2adeaa3..2760d7e475e4a 100644 --- a/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp +++ b/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp @@ -5119,12 +5119,6 @@ void generate_lookup_secondary_supers_table_stub() { } }; -#if INCLUDE_CDS -// nothing to do for ppc -void StubGenerator_AOTAddressTable_init() { -#endif // INCLUDE_CDS -} - void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData *stub_data) { StubGenerator g(code, blob_id, stub_data); } diff --git a/src/hotspot/cpu/ppc/stubRoutines_ppc_64.cpp b/src/hotspot/cpu/ppc/stubRoutines_ppc_64.cpp index 914c5a17a19b7..3b7ee66348a05 100644 --- a/src/hotspot/cpu/ppc/stubRoutines_ppc_64.cpp +++ b/src/hotspot/cpu/ppc/stubRoutines_ppc_64.cpp @@ -183,3 +183,9 @@ address StubRoutines::ppc::generate_crc_constants(juint reverse_poly) { return consts; } + +#if INCLUDE_CDS +// nothing to do for ppc +void StubRoutines::init_AOTAddressTable() { +} +#endif // INCLUDE_CDS diff --git a/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp b/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp index 4d1293286db30..bd12270eed628 100644 --- a/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp +++ b/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp @@ -6906,12 +6906,6 @@ static const int64_t right_3_bits = right_n_bits(3); } }; // end class declaration -#if INCLUDE_CDS -// nothing to do for riscv -void StubGenerator_AOTAddressTable_init() { -#endif // INCLUDE_CDS -} - void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) { StubGenerator g(code, blob_id, stub_data); } diff --git a/src/hotspot/cpu/riscv/stubRoutines_riscv.cpp b/src/hotspot/cpu/riscv/stubRoutines_riscv.cpp index 2aac95d71fa94..51e31aa3672d3 100644 --- a/src/hotspot/cpu/riscv/stubRoutines_riscv.cpp +++ b/src/hotspot/cpu/riscv/stubRoutines_riscv.cpp @@ -501,3 +501,9 @@ ATTRIBUTE_ALIGNED(4096) juint StubRoutines::riscv::_crc_table[] = 0x751997d0UL, 0x00000001UL, 0xccaa009eUL, 0x00000000UL, }; + +#if INCLUDE_CDS +// nothing to do for riscv +void StubRoutines::init_AOTAddressTable() { +} +#endif // INCLUDE_CDS diff --git a/src/hotspot/cpu/s390/stubGenerator_s390.cpp b/src/hotspot/cpu/s390/stubGenerator_s390.cpp index 18da3ece6fbdd..3f16312eb48f4 100644 --- a/src/hotspot/cpu/s390/stubGenerator_s390.cpp +++ b/src/hotspot/cpu/s390/stubGenerator_s390.cpp @@ -3479,12 +3479,6 @@ class StubGenerator: public StubCodeGenerator { }; -#if INCLUDE_CDS -// nothing to do for s390 -void StubGenerator_AOTAddressTable_init() { -#endif // INCLUDE_CDS -} - void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) { StubGenerator g(code, blob_id, stub_data); } diff --git a/src/hotspot/cpu/s390/stubRoutines_s390.cpp b/src/hotspot/cpu/s390/stubRoutines_s390.cpp index 6feb20f960473..3db4995338de7 100644 --- a/src/hotspot/cpu/s390/stubRoutines_s390.cpp +++ b/src/hotspot/cpu/s390/stubRoutines_s390.cpp @@ -736,3 +736,9 @@ juint StubRoutines::zarch::_crc32c_table[CRC32_TABLES][CRC32_COLUMN_SIZE] = { } #endif }; + +#if INCLUDE_CDS +// nothing to do for s390 +void StubRoutines::init_AOTAddressTable() { +} +#endif // INCLUDE_CDS diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp index 950197ead6b7c..94327b4d1b612 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp @@ -5051,20 +5051,6 @@ void StubGenerator::init_AOTAddressTable(GrowableArray
& external_addres init_AOTAddressTable_tan(external_addresses); init_AOTAddressTable_tanh(external_addresses); } - -void StubGenerator_AOTAddressTable_init() { - ResourceMark rm; - GrowableArray
external_addresses; - // publish static addresses referred to by main x86 generator and - // auxiliary x86 generators - StubGenerator::init_AOTAddressTable(external_addresses); - // publish external data addresses defined in nested x86 class - StubRoutines::x86::init_AOTAddressTable(external_addresses); -#ifdef COMPILER1 - LIR_Assembler::init_AOTAddressTable(external_addresses); -#endif - AOTCodeCache::publish_external_addresses(external_addresses); -} #endif // INCLUDE_CDS void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) { diff --git a/src/hotspot/cpu/x86/stubRoutines_x86.cpp b/src/hotspot/cpu/x86/stubRoutines_x86.cpp index 7ad8c21847585..7f60d007e1f13 100644 --- a/src/hotspot/cpu/x86/stubRoutines_x86.cpp +++ b/src/hotspot/cpu/x86/stubRoutines_x86.cpp @@ -413,6 +413,21 @@ ATTRIBUTE_ALIGNED(64) const julong StubRoutines::x86::_k512_W[] = }; #if INCLUDE_CDS + +void StubRoutines::init_AOTAddressTable() { + ResourceMark rm; + GrowableArray
external_addresses; + // publish static addresses referred to by main x86 generator and + // auxiliary x86 generators + StubGenerator::init_AOTAddressTable(external_addresses); + // publish external data addresses defined in nested x86 class + StubRoutines::x86::init_AOTAddressTable(external_addresses); +#ifdef COMPILER1 + LIR_Assembler::init_AOTAddressTable(external_addresses); +#endif + AOTCodeCache::publish_external_addresses(external_addresses); +} + // publish addresses of external data defined in this file which may // be referenced from stub or code void StubRoutines::x86::init_AOTAddressTable(GrowableArray
& external_addresses) { diff --git a/src/hotspot/cpu/zero/stubGenerator_zero.cpp b/src/hotspot/cpu/zero/stubGenerator_zero.cpp index 6b82004f124fb..569a2fa8ca9af 100644 --- a/src/hotspot/cpu/zero/stubGenerator_zero.cpp +++ b/src/hotspot/cpu/zero/stubGenerator_zero.cpp @@ -237,12 +237,6 @@ class StubGenerator: public StubCodeGenerator { } }; -#if INCLUDE_CDS -// nothing to do for ZERO -void StubGenerator_AOTAddressTable_init() { -} -#endif // INCLUDE_CDS - void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData *stub_data) { StubGenerator g(code, blob_id, stub_data); } diff --git a/src/hotspot/cpu/zero/stubRoutines_zero.cpp b/src/hotspot/cpu/zero/stubRoutines_zero.cpp index 9b53f09be5dc7..723ca44740853 100644 --- a/src/hotspot/cpu/zero/stubRoutines_zero.cpp +++ b/src/hotspot/cpu/zero/stubRoutines_zero.cpp @@ -30,3 +30,9 @@ address StubRoutines::crc_table_addr() { ShouldNotCallThis(); return nullptr; } address StubRoutines::crc32c_table_addr() { ShouldNotCallThis(); return nullptr; } + +#if INCLUDE_CDS +// nothing to do for xero +void StubRoutines::init_AOTAddressTable() { +} +#endif // INCLUDE_CDS diff --git a/src/hotspot/share/runtime/stubRoutines.cpp b/src/hotspot/share/runtime/stubRoutines.cpp index 5dd85ba821da2..dad4d1f8fb6f3 100644 --- a/src/hotspot/share/runtime/stubRoutines.cpp +++ b/src/hotspot/share/runtime/stubRoutines.cpp @@ -103,12 +103,6 @@ BlobId StubRoutines::stub_to_blob(StubId id) { // Initialization extern void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data); // only interface to generators -#if INCLUDE_CDS -#if (defined(X86) && defined(_LP64)) || defined(AARCH64) -extern void StubGenerator_AOTAddressTable_init(); -#endif -#endif // INCLUDE_CDS - void UnsafeMemoryAccess::create_table(int max_size) { UnsafeMemoryAccess::_table = new UnsafeMemoryAccess[max_size]; UnsafeMemoryAccess::_table_max_length = max_size; @@ -316,14 +310,6 @@ STUBGEN_BLOBS_DO(DEFINE_BLOB_INIT_FUNCTION) #if INCLUDE_CDS -// Non-generated init method - -void StubRoutines::init_AOTAddressTable() { -#if (defined(X86) && defined(_LP64)) || defined(AARCH64) - StubGenerator_AOTAddressTable_init(); -#endif -} - // non-generated external API init driver function void stubs_AOTAddressTable_init() { StubRoutines::init_AOTAddressTable(); } diff --git a/src/hotspot/share/runtime/stubRoutines.hpp b/src/hotspot/share/runtime/stubRoutines.hpp index e4d5c14e6f0b1..bfd230a4f80b5 100644 --- a/src/hotspot/share/runtime/stubRoutines.hpp +++ b/src/hotspot/share/runtime/stubRoutines.hpp @@ -285,7 +285,7 @@ class StubRoutines: AllStatic { #endif #if INCLUDE_CDS - // Initalization + // AOT Initalization -- implementation is arch-specific static void init_AOTAddressTable(); #endif // INCLUDE_CDS From 0597bc6b260145224b027a261f8cff96f373f911 Mon Sep 17 00:00:00 2001 From: Andrew Dinn Date: Thu, 27 Nov 2025 14:34:51 +0000 Subject: [PATCH 30/30] fix header declarations --- src/hotspot/cpu/x86/stubGenerator_x86_64.cpp | 3 --- src/hotspot/cpu/x86/stubRoutines_x86.cpp | 4 ++++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp index 94327b4d1b612..ecca0a4dd62fc 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp @@ -39,9 +39,6 @@ #include "runtime/sharedRuntime.hpp" #include "runtime/stubRoutines.hpp" #include "stubGenerator_x86_64.hpp" -#ifdef COMPILER1 -#include "c1/c1_LIRAssembler.hpp" -#endif #ifdef COMPILER2 #include "opto/runtime.hpp" #include "opto/c2_globals.hpp" diff --git a/src/hotspot/cpu/x86/stubRoutines_x86.cpp b/src/hotspot/cpu/x86/stubRoutines_x86.cpp index 7f60d007e1f13..eec5541830da5 100644 --- a/src/hotspot/cpu/x86/stubRoutines_x86.cpp +++ b/src/hotspot/cpu/x86/stubRoutines_x86.cpp @@ -28,6 +28,10 @@ #include "runtime/stubRoutines.hpp" #include "utilities/globalDefinitions.hpp" #include "crc32c.h" +#include "stubGenerator_x86_64.hpp" +#ifdef COMPILER1 +#include "c1/c1_LIRAssembler.hpp" +#endif // Implementation of the platform-specific part of StubRoutines - for // a description of how to extend it, see the stubRoutines.hpp file.