Skip to content

Commit

Permalink
Merge
Browse files Browse the repository at this point in the history
  • Loading branch information
@JesperIRL
JesperIRL committed Jan 14, 2021
2 parents c7e2174 + 5926d75 commit 51e14f2
Show file tree
Hide file tree
Showing 48 changed files with 868 additions and 227 deletions.
18 changes: 8 additions & 10 deletions src/hotspot/cpu/aarch64/aarch64.ad
View file
@@ -1,6 +1,6 @@
//
// Copyright (c) 2003, 2021, Oracle and/or its affiliates. All rights reserved.
// Copyright (c) 2014, 2020, Red Hat, Inc. All rights reserved.
// Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
//
// This code is free software; you can redistribute it and/or modify it
Expand Down Expand Up @@ -1916,7 +1916,7 @@ void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
__ bind(L_skip_barrier);
}

if (UseSVE > 0 && C->max_vector_size() >= 16) {
if (C->max_vector_size() >= 16) {
__ reinitialize_ptrue();
}

Expand Down Expand Up @@ -3798,11 +3798,9 @@ encode %{
}
}

if (UseSVE > 0 && Compile::current()->max_vector_size() >= 16) {
// Only non uncommon_trap calls need to reinitialize ptrue.
if (uncommon_trap_request() == 0) {
__ reinitialize_ptrue();
}
// Only non uncommon_trap calls need to reinitialize ptrue.
if (Compile::current()->max_vector_size() >= 16 && uncommon_trap_request() == 0) {
__ reinitialize_ptrue();
}
%}

Expand All @@ -3813,7 +3811,7 @@ encode %{
if (call == NULL) {
ciEnv::current()->record_failure("CodeCache is full");
return;
} else if (UseSVE > 0 && Compile::current()->max_vector_size() >= 16) {
} else if (Compile::current()->max_vector_size() >= 16) {
__ reinitialize_ptrue();
}
%}
Expand Down Expand Up @@ -3851,7 +3849,7 @@ encode %{
__ bind(retaddr);
__ add(sp, sp, 2 * wordSize);
}
if (UseSVE > 0 && Compile::current()->max_vector_size() >= 16) {
if (Compile::current()->max_vector_size() >= 16) {
__ reinitialize_ptrue();
}
%}
Expand All @@ -3864,7 +3862,7 @@ encode %{
enc_class aarch64_enc_ret() %{
C2_MacroAssembler _masm(&cbuf);
#ifdef ASSERT
if (UseSVE > 0 && Compile::current()->max_vector_size() >= 16) {
if (Compile::current()->max_vector_size() >= 16) {
__ verify_ptrue();
}
#endif
Expand Down
23 changes: 12 additions & 11 deletions src/hotspot/cpu/aarch64/aarch64_neon.ad
View file
@@ -1,5 +1,5 @@
// Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
// Copyright (c) 2020, Arm Limited. All rights reserved.
// Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved.
// Copyright (c) 2020, 2021, Arm Limited. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
//
// This code is free software; you can redistribute it and/or modify it
Expand Down Expand Up @@ -84,9 +84,11 @@ instruct reinterpretD2X(vecX dst, vecD src)
n->in(1)->bottom_type()->is_vect()->length_in_bytes() == 8);
match(Set dst (VectorReinterpret src));
ins_cost(INSN_COST);
format %{ " # reinterpret $dst,$src" %}
format %{ " # reinterpret $dst,$src\t# D2X" %}
ins_encode %{
// If register is the same, then move is not needed.
// If registers are the same, no register move is required - the
// upper 64 bits of 'src' are expected to have been initialized
// to zero.
if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
__ orr(as_FloatRegister($dst$$reg), __ T8B,
as_FloatRegister($src$$reg),
Expand All @@ -102,14 +104,13 @@ instruct reinterpretX2D(vecD dst, vecX src)
n->in(1)->bottom_type()->is_vect()->length_in_bytes() == 16);
match(Set dst (VectorReinterpret src));
ins_cost(INSN_COST);
format %{ " # reinterpret $dst,$src" %}
format %{ " # reinterpret $dst,$src\t# X2D" %}
ins_encode %{
// If register is the same, then move is not needed.
if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
__ orr(as_FloatRegister($dst$$reg), __ T8B,
as_FloatRegister($src$$reg),
as_FloatRegister($src$$reg));
}
// Resize the vector from 128-bits to 64-bits. The higher 64-bits of
// the "dst" register must be cleared to zero.
__ orr(as_FloatRegister($dst$$reg), __ T8B,
as_FloatRegister($src$$reg),
as_FloatRegister($src$$reg));
%}
ins_pipe(vlogical64);
%}
Expand Down
41 changes: 28 additions & 13 deletions src/hotspot/cpu/aarch64/aarch64_neon_ad.m4
View file
@@ -1,5 +1,5 @@
// Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
// Copyright (c) 2020, Arm Limited. All rights reserved.
// Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved.
// Copyright (c) 2020, 2021, Arm Limited. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
//
// This code is free software; you can redistribute it and/or modify it
Expand Down Expand Up @@ -97,28 +97,43 @@ dnl $1 $2
REINTERPRET(D, 8)
REINTERPRET(X, 16)
dnl
define(`REINTERPRET_X', `
instruct reinterpret$1`'2$2`'(vec$2 dst, vec$1 src)

instruct reinterpretD2X(vecX dst, vecD src)
%{
predicate(n->bottom_type()->is_vect()->length_in_bytes() == $3 &&
n->in(1)->bottom_type()->is_vect()->length_in_bytes() == $4);
predicate(n->bottom_type()->is_vect()->length_in_bytes() == 16 &&
n->in(1)->bottom_type()->is_vect()->length_in_bytes() == 8);
match(Set dst (VectorReinterpret src));
ins_cost(INSN_COST);
format %{ " # reinterpret $dst,$src" %}
format %{ " # reinterpret $dst,$src\t# D2X" %}
ins_encode %{
// If register is the same, then move is not needed.
// If registers are the same, no register move is required - the
// upper 64 bits of 'src' are expected to have been initialized
// to zero.
if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
__ orr(as_FloatRegister($dst$$reg), __ T8B,
as_FloatRegister($src$$reg),
as_FloatRegister($src$$reg));
}
%}
ins_pipe(vlogical64);
%}')dnl
dnl $1 $2 $3 $4
REINTERPRET_X(D, X, 16, 8)
REINTERPRET_X(X, D, 8, 16)
dnl
%}

instruct reinterpretX2D(vecD dst, vecX src)
%{
predicate(n->bottom_type()->is_vect()->length_in_bytes() == 8 &&
n->in(1)->bottom_type()->is_vect()->length_in_bytes() == 16);
match(Set dst (VectorReinterpret src));
ins_cost(INSN_COST);
format %{ " # reinterpret $dst,$src\t# X2D" %}
ins_encode %{
// Resize the vector from 128-bits to 64-bits. The higher 64-bits of
// the "dst" register must be cleared to zero.
__ orr(as_FloatRegister($dst$$reg), __ T8B,
as_FloatRegister($src$$reg),
as_FloatRegister($src$$reg));
%}
ins_pipe(vlogical64);
%}

// ------------------------------ Vector cast -------------------------------
dnl
Expand Down
11 changes: 5 additions & 6 deletions src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp
View file
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
Expand Down Expand Up @@ -354,6 +354,10 @@ class ZSaveLiveRegisters {
~ZSaveLiveRegisters() {
// Restore registers
__ pop_fp(_fp_regs, sp);

// External runtime call may clobber ptrue reg
__ reinitialize_ptrue();

__ pop(_gp_regs, sp);
}
};
Expand Down Expand Up @@ -428,11 +432,6 @@ void ZBarrierSetAssembler::generate_c2_load_barrier_stub(MacroAssembler* masm, Z
ZSetupArguments setup_arguments(masm, stub);
__ mov(rscratch1, stub->slow_path());
__ blr(rscratch1);
if (UseSVE > 0) {
// Reinitialize the ptrue predicate register, in case the external runtime
// call clobbers ptrue reg, as we may return to SVE compiled code.
__ reinitialize_ptrue();
}
}
// Stub exit
__ b(*stub->continuation());
Expand Down
13 changes: 11 additions & 2 deletions src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
View file
@@ -1,6 +1,6 @@
/*
* Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
* Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
Expand Down Expand Up @@ -2663,6 +2663,8 @@ void MacroAssembler::pop_call_clobbered_registers_except(RegSet exclude) {
as_FloatRegister(i+3), T1D, Address(post(sp, 4 * wordSize)));
}

reinitialize_ptrue();

pop(call_clobbered_registers() - exclude, sp);
}

Expand Down Expand Up @@ -2699,6 +2701,11 @@ void MacroAssembler::pop_CPU_state(bool restore_vectors, bool use_sve,
ld1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2),
as_FloatRegister(i+3), restore_vectors ? T2D : T1D, Address(post(sp, step)));
}

if (restore_vectors) {
reinitialize_ptrue();
}

pop(0x3fffffff, sp); // integer registers except lr & sp
}

Expand Down Expand Up @@ -5309,7 +5316,9 @@ void MacroAssembler::verify_sve_vector_length() {

void MacroAssembler::verify_ptrue() {
Label verify_ok;
assert(UseSVE > 0, "should only be used for SVE");
if (!UseSVE) {
return;
}
sve_cntp(rscratch1, B, ptrue, ptrue); // get true elements count.
sve_dec(rscratch1, B);
cbz(rscratch1, verify_ok);
Expand Down
8 changes: 5 additions & 3 deletions src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp
View file
@@ -1,6 +1,6 @@
/*
* Copyright (c) 1997, 2020, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
* Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
Expand Down Expand Up @@ -967,7 +967,9 @@ class MacroAssembler: public Assembler {

void verify_sve_vector_length();
void reinitialize_ptrue() {
sve_ptrue(ptrue, B);
if (UseSVE > 0) {
sve_ptrue(ptrue, B);
}
}
void verify_ptrue();

Expand Down
11 changes: 4 additions & 7 deletions src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp
View file
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2003, 2021, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
* Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
Expand Down Expand Up @@ -2781,12 +2781,6 @@ SafepointBlob* SharedRuntime::generate_handler_blob(address call_ptr, int poll_t

__ membar(Assembler::LoadLoad | Assembler::LoadStore);

if (UseSVE > 0 && save_vectors) {
// Reinitialize the ptrue predicate register, in case the external runtime
// call clobbers ptrue reg, as we may return to SVE compiled code.
__ reinitialize_ptrue();
}

__ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset()));
__ cbz(rscratch1, noException);

Expand Down Expand Up @@ -3009,6 +3003,9 @@ void OptoRuntime::generate_exception_blob() {
// handle_exception_C is a special VM call which does not require an explicit
// instruction sync afterwards.

// May jump to SVE compiled code
__ reinitialize_ptrue();

// Set an oopmap for the call site. This oopmap will only be used if we
// are unwinding the stack. Hence, all locations will be dead.
// Callee-saved registers will be the same as the frame above (i.e.,
Expand Down
19 changes: 8 additions & 11 deletions src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp
View file
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2003, 2021, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
* Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
Expand Down Expand Up @@ -490,11 +490,10 @@ class StubGenerator: public StubCodeGenerator {
__ call_VM_leaf(CAST_FROM_FN_PTR(address,
SharedRuntime::exception_handler_for_return_address),
rthread, c_rarg1);
if (UseSVE > 0 ) {
// Reinitialize the ptrue predicate register, in case the external runtime
// call clobbers ptrue reg, as we may return to SVE compiled code.
__ reinitialize_ptrue();
}
// Reinitialize the ptrue predicate register, in case the external runtime
// call clobbers ptrue reg, as we may return to SVE compiled code.
__ reinitialize_ptrue();

// we should not really care that lr is no longer the callee
// address. we saved the value the handler needs in r19 so we can
// just copy it to r3. however, the C2 handler will push its own
Expand Down Expand Up @@ -5655,11 +5654,9 @@ class StubGenerator: public StubCodeGenerator {

__ reset_last_Java_frame(true);

if (UseSVE > 0) {
// Reinitialize the ptrue predicate register, in case the external runtime
// call clobbers ptrue reg, as we may return to SVE compiled code.
__ reinitialize_ptrue();
}
// Reinitialize the ptrue predicate register, in case the external runtime
// call clobbers ptrue reg, as we may return to SVE compiled code.
__ reinitialize_ptrue();

__ leave();

Expand Down
5 changes: 5 additions & 0 deletions src/hotspot/share/ci/ciMethodData.cpp
View file
Expand Up @@ -203,7 +203,12 @@ void ciMethodData::load_data() {
// _extra_data_size = extra_data_limit - extra_data_base
// total_size = _data_size + _extra_data_size
// args_data_limit = data_base + total_size - parameter_data_size

#ifndef ZERO
// Some Zero platforms do not have expected alignment, and do not use
// this code. static_assert would still fire and fail for them.
static_assert(sizeof(_orig) % HeapWordSize == 0, "align");
#endif
Copy::disjoint_words_atomic((HeapWord*) &mdo->_compiler_counters,
(HeapWord*) &_orig,
sizeof(_orig) / HeapWordSize);
Expand Down

1 comment on commit 51e14f2

@openjdk-notifier
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Review

Please sign in to comment.