Skip to content

Commit

Permalink
8231349: Move intrinsic stubs generation to compiler runtime initiali…
Browse files Browse the repository at this point in the history
…zation code

Reviewed-by: redestad, vlivanov
  • Loading branch information
Vladimir Kozlov committed Mar 23, 2023
1 parent f37674a commit 3859faf
Show file tree
Hide file tree
Showing 38 changed files with 595 additions and 352 deletions.
4 changes: 3 additions & 1 deletion src/hotspot/cpu/aarch64/globals_aarch64.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,11 @@
// (see globals.hpp)

define_pd_global(bool, ImplicitNullChecks, true); // Generate code for implicit null checks
define_pd_global(bool, TrapBasedNullChecks, false);
define_pd_global(bool, TrapBasedNullChecks, false);
define_pd_global(bool, UncommonNullCast, true); // Uncommon-trap NULLs past to check cast

define_pd_global(bool, DelayCompilerStubsGeneration, COMPILER2_OR_JVMCI);

define_pd_global(uintx, CodeCacheSegmentSize, 64 COMPILER1_AND_COMPILER2_PRESENT(+64)); // Tiered compilation has large code-entry alignment.
define_pd_global(intx, CodeEntryAlignment, 64);
define_pd_global(intx, OptoLoopAlignment, 16);
Expand Down
17 changes: 11 additions & 6 deletions src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -570,26 +570,31 @@ void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm,
// caller, but with an uncorrected stack, causing delayed havoc.

if (VerifyAdapterCalls &&
(Interpreter::code() != NULL || StubRoutines::code1() != NULL)) {
(Interpreter::code() != NULL || StubRoutines::final_stubs_code() != NULL)) {
#if 0
// So, let's test for cascading c2i/i2c adapters right now.
// assert(Interpreter::contains($return_addr) ||
// StubRoutines::contains($return_addr),
// "i2c adapter must return to an interpreter frame");
__ block_comment("verify_i2c { ");
Label L_ok;
if (Interpreter::code() != NULL)
if (Interpreter::code() != NULL) {
range_check(masm, rax, r11,
Interpreter::code()->code_start(), Interpreter::code()->code_end(),
L_ok);
if (StubRoutines::code1() != NULL)
}
if (StubRoutines::initial_stubs_code() != NULL) {
range_check(masm, rax, r11,
StubRoutines::code1()->code_begin(), StubRoutines::code1()->code_end(),
StubRoutines::initial_stubs_code()->code_begin(),
StubRoutines::initial_stubs_code()->code_end(),
L_ok);
if (StubRoutines::code2() != NULL)
}
if (StubRoutines::final_stubs_code() != NULL) {
range_check(masm, rax, r11,
StubRoutines::code2()->code_begin(), StubRoutines::code2()->code_end(),
StubRoutines::final_stubs_code()->code_begin(),
StubRoutines::final_stubs_code()->code_end(),
L_ok);
}
const char* msg = "i2c adapter must return to an interpreter frame";
__ block_comment(msg);
__ stop(msg);
Expand Down
98 changes: 58 additions & 40 deletions src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -7997,7 +7997,7 @@ class StubGenerator: public StubCodeGenerator {


// Initialization
void generate_initial() {
void generate_initial_stubs() {
// Generate initial stubs and initializes the entry points

// entry points that exist in all platforms Note: This is code
Expand All @@ -8023,6 +8023,12 @@ class StubGenerator: public StubCodeGenerator {
generate_throw_exception("delayed StackOverflowError throw_exception",
CAST_FROM_FN_PTR(address,
SharedRuntime::throw_delayed_StackOverflowError));

// Initialize table for copy memory (arraycopy) check.
if (UnsafeCopyMemory::_table == nullptr) {
UnsafeCopyMemory::create_table(8);
}

if (UseCRC32Intrinsics) {
// set table address before stub generation which use it
StubRoutines::_crc_table_adr = (address)StubRoutines::aarch64::_crc_table;
Expand All @@ -8047,7 +8053,7 @@ class StubGenerator: public StubCodeGenerator {
}
}

void generate_phase1() {
void generate_continuation_stubs() {
// Continuation stubs:
StubRoutines::_cont_thaw = generate_cont_thaw();
StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier();
Expand All @@ -8057,7 +8063,7 @@ class StubGenerator: public StubCodeGenerator {
JFR_ONLY(StubRoutines::_jfr_write_checkpoint = StubRoutines::_jfr_write_checkpoint_stub->entry_point();)
}

void generate_all() {
void generate_final_stubs() {
// support for verify_oop (must happen after universe_init)
if (VerifyOops) {
StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop();
Expand All @@ -8080,32 +8086,47 @@ class StubGenerator: public StubCodeGenerator {
SharedRuntime::
throw_NullPointerException_at_call));

if (UseSVE == 0) {
StubRoutines::aarch64::_vector_iota_indices = generate_iota_indices("iota_indices");
}

// arraycopy stubs used by compilers
generate_arraycopy_stubs();

// countPositives stub for large arrays.
StubRoutines::aarch64::_count_positives = generate_count_positives(StubRoutines::aarch64::_count_positives_long);
BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
if (bs_nm != NULL) {
StubRoutines::aarch64::_method_entry_barrier = generate_method_entry_barrier();
}

StubRoutines::aarch64::_spin_wait = generate_spin_wait();

#if defined (LINUX) && !defined (__ARM_FEATURE_ATOMICS)

generate_atomic_entry_points();

#endif // LINUX

StubRoutines::aarch64::set_completed(); // Inidicate that arraycopy and zero_blocks stubs are generated
}

void generate_compiler_stubs() {
#if COMPILER2_OR_JVMCI

if (UseSVE == 0) {
StubRoutines::aarch64::_vector_iota_indices = generate_iota_indices("iota_indices");
}

// array equals stub for large arrays.
if (!UseSimpleArrayEquals) {
StubRoutines::aarch64::_large_array_equals = generate_large_array_equals();
}

// byte_array_inflate stub for large arrays.
StubRoutines::aarch64::_large_byte_array_inflate = generate_large_byte_array_inflate();

// countPositives stub for large arrays.
StubRoutines::aarch64::_count_positives = generate_count_positives(StubRoutines::aarch64::_count_positives_long);

generate_compare_long_strings();

generate_string_indexof_stubs();

// byte_array_inflate stub for large arrays.
StubRoutines::aarch64::_large_byte_array_inflate = generate_large_byte_array_inflate();

BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
if (bs_nm != NULL) {
StubRoutines::aarch64::_method_entry_barrier = generate_method_entry_barrier();
}
#ifdef COMPILER2
if (UseMultiplyToLenIntrinsic) {
StubRoutines::_multiplyToLen = generate_multiplyToLen();
Expand Down Expand Up @@ -8192,36 +8213,33 @@ class StubGenerator: public StubCodeGenerator {
if (UseAdler32Intrinsics) {
StubRoutines::_updateBytesAdler32 = generate_updateBytesAdler32();
}

StubRoutines::aarch64::_spin_wait = generate_spin_wait();

#if defined (LINUX) && !defined (__ARM_FEATURE_ATOMICS)

generate_atomic_entry_points();

#endif // LINUX

StubRoutines::aarch64::set_completed();
#endif // COMPILER2_OR_JVMCI
}

public:
StubGenerator(CodeBuffer* code, int phase) : StubCodeGenerator(code) {
if (phase == 0) {
generate_initial();
} else if (phase == 1) {
generate_phase1(); // stubs that must be available for the interpreter
} else {
generate_all();
}
StubGenerator(CodeBuffer* code, StubsKind kind) : StubCodeGenerator(code) {
switch(kind) {
case Initial_stubs:
generate_initial_stubs();
break;
case Continuation_stubs:
generate_continuation_stubs();
break;
case Compiler_stubs:
generate_compiler_stubs();
break;
case Final_stubs:
generate_final_stubs();
break;
default:
fatal("unexpected stubs kind: %d", kind);
break;
};
}
}; // end class declaration

#define UCM_TABLE_MAX_ENTRIES 8
void StubGenerator_generate(CodeBuffer* code, int phase) {
if (UnsafeCopyMemory::_table == NULL) {
UnsafeCopyMemory::create_table(UCM_TABLE_MAX_ENTRIES);
}
StubGenerator g(code, phase);
void StubGenerator_generate(CodeBuffer* code, StubCodeGenerator::StubsKind kind) {
StubGenerator g(code, kind);
}


Expand Down
9 changes: 6 additions & 3 deletions src/hotspot/cpu/aarch64/stubRoutines_aarch64.hpp
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2003, 2020, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
Expand Down Expand Up @@ -35,8 +35,11 @@ static bool returns_to_call_stub(address return_pc) {
}

enum platform_dependent_constants {
code_size1 = 19000, // simply increase if too small (assembler will crash if too small)
code_size2 = 45000 // simply increase if too small (assembler will crash if too small)
// simply increase sizes if too small (assembler will crash if too small)
_initial_stubs_code_size = 10000,
_continuation_stubs_code_size = 2000,
_compiler_stubs_code_size = 30000,
_final_stubs_code_size = 20000
};

class aarch64 {
Expand Down
2 changes: 2 additions & 0 deletions src/hotspot/cpu/arm/globals_arm.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ define_pd_global(bool, ImplicitNullChecks, true); // Generate code for i
define_pd_global(bool, UncommonNullCast, true); // Uncommon-trap nulls past to check cast
define_pd_global(bool, TrapBasedNullChecks, false); // Not needed

define_pd_global(bool, DelayCompilerStubsGeneration, false); // No need - only few compiler's stubs

define_pd_global(uintx, CodeCacheSegmentSize, 64 COMPILER1_AND_COMPILER2_PRESENT(+64)); // Tiered compilation has large code-entry alignment.
define_pd_global(intx, CodeEntryAlignment, 16);
define_pd_global(intx, OptoLoopAlignment, 16);
Expand Down
59 changes: 36 additions & 23 deletions src/hotspot/cpu/arm/stubGenerator_arm.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3077,7 +3077,7 @@ class StubGenerator: public StubCodeGenerator {
//---------------------------------------------------------------------------
// Initialization

void generate_initial() {
void generate_initial_stubs() {
// Generates all stubs and initializes the entry points

//------------------------------------------------------------------------------------------------------------------------
Expand All @@ -3094,6 +3094,10 @@ class StubGenerator: public StubCodeGenerator {
// stub for throwing stack overflow error used both by interpreter and compiler
StubRoutines::_throw_StackOverflowError_entry = generate_throw_exception("StackOverflowError throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_StackOverflowError));

if (UnsafeCopyMemory::_table == nullptr) {
UnsafeCopyMemory::create_table(32);
}

// integer division used both by interpreter and compiler
StubRoutines::Arm::_idiv_irem_entry = generate_idiv_irem();

Expand All @@ -3106,7 +3110,7 @@ class StubGenerator: public StubCodeGenerator {

}

void generate_phase1() {
void generate_continuation_stubs() {
// Continuation stubs:
StubRoutines::_cont_thaw = generate_cont_thaw();
StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier();
Expand All @@ -3116,14 +3120,9 @@ class StubGenerator: public StubCodeGenerator {
JFR_ONLY(StubRoutines::_jfr_write_checkpoint = StubRoutines::_jfr_write_checkpoint_stub->entry_point();)
}

void generate_all() {
void generate_final_stubs() {
// Generates all stubs and initializes the entry points

#ifdef COMPILER2
// Generate partial_subtype_check first here since its code depends on
// UseZeroBaseCompressedOops which is defined after heap initialization.
StubRoutines::Arm::_partial_subtype_check = generate_partial_subtype_check();
#endif
// These entry points require SharedInfo::stack0 to be set up in non-core builds
// and need to be relocatable, so they each fabricate a RuntimeStub internally.
StubRoutines::_throw_AbstractMethodError_entry = generate_throw_exception("AbstractMethodError throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_AbstractMethodError));
Expand All @@ -3144,6 +3143,14 @@ class StubGenerator: public StubCodeGenerator {
StubRoutines::Arm::_method_entry_barrier = generate_method_entry_barrier();
}

}

void generate_compiler_stubs() {
#ifdef COMPILER2
// Generate partial_subtype_check first here since its code depends on
// UseZeroBaseCompressedOops which is defined after heap initialization.
StubRoutines::Arm::_partial_subtype_check = generate_partial_subtype_check();

#ifdef COMPILE_CRYPTO
// generate AES intrinsics code
if (UseAESIntrinsics) {
Expand All @@ -3154,25 +3161,31 @@ class StubGenerator: public StubCodeGenerator {
StubRoutines::_cipherBlockChaining_decryptAESCrypt = generate_cipherBlockChaining_decryptAESCrypt();
}
#endif // COMPILE_CRYPTO
#endif // COMPILER2
}


public:
StubGenerator(CodeBuffer* code, int phase) : StubCodeGenerator(code) {
if (phase == 0) {
generate_initial();
} else if (phase == 1) {
generate_phase1();
} else {
generate_all();
}
StubGenerator(CodeBuffer* code, StubsKind kind) : StubCodeGenerator(code) {
switch(kind) {
case Initial_stubs:
generate_initial_stubs();
break;
case Continuation_stubs:
generate_continuation_stubs();
break;
case Compiler_stubs:
generate_compiler_stubs();
break;
case Final_stubs:
generate_final_stubs();
break;
default:
fatal("unexpected stubs kind: %d", kind);
break;
};
}
}; // end class declaration

#define UCM_TABLE_MAX_ENTRIES 32
void StubGenerator_generate(CodeBuffer* code, int phase) {
if (UnsafeCopyMemory::_table == nullptr) {
UnsafeCopyMemory::create_table(UCM_TABLE_MAX_ENTRIES);
}
StubGenerator g(code, phase);
void StubGenerator_generate(CodeBuffer* code, StubCodeGenerator::StubsKind kind) {
StubGenerator g(code, kind);
}
9 changes: 6 additions & 3 deletions src/hotspot/cpu/arm/stubRoutines_arm.hpp
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2008, 2019, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2008, 2023, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
Expand Down Expand Up @@ -30,8 +30,11 @@
// extend it.

enum platform_dependent_constants {
code_size1 = 9000, // simply increase if too small (assembler will crash if too small)
code_size2 = 22000 // simply increase if too small (assembler will crash if too small)
// simply increase sizes if too small (assembler will crash if too small)
_initial_stubs_code_size = 9000,
_continuation_stubs_code_size = 2000,
_compiler_stubs_code_size = 22000,
_final_stubs_code_size = 22000
};

class Arm {
Expand Down
4 changes: 3 additions & 1 deletion src/hotspot/cpu/ppc/globals_ppc.hpp
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2002, 2020, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2002, 2023, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012, 2020 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
Expand Down Expand Up @@ -36,6 +36,8 @@ define_pd_global(bool, ImplicitNullChecks, true); // Generate code for impli
define_pd_global(bool, TrapBasedNullChecks, true);
define_pd_global(bool, UncommonNullCast, true); // Uncommon-trap NULLs passed to check cast.

define_pd_global(bool, DelayCompilerStubsGeneration, COMPILER2_OR_JVMCI);

#define DEFAULT_STACK_YELLOW_PAGES (2)
#define DEFAULT_STACK_RED_PAGES (1)
// Java_java_net_SocketOutputStream_socketWrite0() uses a 64k buffer on the
Expand Down
Loading

1 comment on commit 3859faf

@openjdk-notifier
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please sign in to comment.