Projects
home:dingli:branches:openEuler:24.09-openjdk
openjdk-11
_service:tar_scm:ZGC-AArch64-Optimizations-and-...
Sign Up
Log In
Username
Password
Overview
Repositories
Revisions
Requests
Users
Attributes
Meta
File _service:tar_scm:ZGC-AArch64-Optimizations-and-Fixes.patch of Package openjdk-11
From 1932790364789c601d463a4de8f757cf604344c0 Mon Sep 17 00:00:00 2001 --- make/hotspot/gensrc/GensrcAdlc.gmk | 6 + src/hotspot/cpu/aarch64/aarch64.ad | 207 +- .../gc/z/zBarrierSetAssembler_aarch64.cpp | 246 ++- .../gc/z/zBarrierSetAssembler_aarch64.hpp | 26 +- src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad | 268 +++ .../cpu/aarch64/macroAssembler_aarch64.cpp | 61 + .../cpu/aarch64/macroAssembler_aarch64.hpp | 6 + .../templateInterpreterGenerator_aarch64.cpp | 4 +- .../cpu/aarch64/vm_version_aarch64.hpp | 8 + .../cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp | 404 +++- .../cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp | 30 +- src/hotspot/cpu/x86/gc/z/z_x86_64.ad | 168 ++ src/hotspot/cpu/x86/x86.ad | 136 +- src/hotspot/cpu/x86/x86_64.ad | 437 +---- .../gc/z/zBackingFile_linux_aarch64.cpp | 2 +- src/hotspot/share/adlc/formssel.cpp | 8 - src/hotspot/share/c1/c1_Instruction.cpp | 1 + src/hotspot/share/ci/ciInstanceKlass.cpp | 44 + src/hotspot/share/classfile/vmSymbols.hpp | 4 + .../share/compiler/compilerDirectives.hpp | 3 +- .../share/gc/shared/c2/barrierSetC2.cpp | 73 +- .../share/gc/shared/c2/barrierSetC2.hpp | 15 +- src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp | 1657 +++-------------- src/hotspot/share/gc/z/c2/zBarrierSetC2.hpp | 181 +- .../share/gc/z/zBarrierSetAssembler.hpp | 5 +- src/hotspot/share/gc/z/zGlobals.hpp | 7 +- src/hotspot/share/gc/z/zHeap.cpp | 5 + src/hotspot/share/gc/z/zLiveMap.cpp | 20 +- src/hotspot/share/gc/z/zLiveMap.inline.hpp | 9 +- src/hotspot/share/gc/z/zMarkStack.cpp | 74 +- src/hotspot/share/gc/z/zMarkStack.hpp | 1 + src/hotspot/share/gc/z/zWorkers.cpp | 23 +- src/hotspot/share/gc/z/zWorkers.hpp | 2 - src/hotspot/share/gc/z/z_globals.hpp | 6 +- src/hotspot/share/opto/c2compiler.cpp | 1 + src/hotspot/share/opto/classes.cpp | 3 - src/hotspot/share/opto/classes.hpp | 11 - src/hotspot/share/opto/compile.cpp | 52 +- src/hotspot/share/opto/compile.hpp | 25 +- src/hotspot/share/opto/escape.cpp | 15 - src/hotspot/share/opto/graphKit.cpp | 10 +- src/hotspot/share/opto/graphKit.hpp | 10 +- src/hotspot/share/opto/lcm.cpp | 1 - src/hotspot/share/opto/library_call.cpp | 17 + src/hotspot/share/opto/loopnode.cpp | 1 - src/hotspot/share/opto/loopopts.cpp | 3 - src/hotspot/share/opto/machnode.hpp | 9 +- src/hotspot/share/opto/matcher.cpp | 45 +- src/hotspot/share/opto/memnode.cpp | 14 +- src/hotspot/share/opto/memnode.hpp | 53 +- src/hotspot/share/opto/node.cpp | 7 - src/hotspot/share/opto/node.hpp | 6 - src/hotspot/share/opto/output.cpp | 424 +++-- src/hotspot/share/opto/output.hpp | 5 +- src/hotspot/share/opto/parse1.cpp | 1 + src/hotspot/share/opto/phaseX.cpp | 8 +- src/hotspot/share/opto/vectornode.cpp | 1 - src/hotspot/share/runtime/sharedRuntime.cpp | 2 + src/hotspot/share/runtime/sharedRuntime.hpp | 5 + src/hotspot/share/utilities/bitMap.hpp | 17 +- src/hotspot/share/utilities/bitMap.inline.hpp | 34 +- .../share/classes/java/util/Random.java | 2 + .../runtime/MemberName/MemberNameLeak.java | 1 + 63 files changed, 1941 insertions(+), 2989 deletions(-) create mode 100644 src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad create mode 100644 src/hotspot/cpu/x86/gc/z/z_x86_64.ad diff --git a/make/hotspot/gensrc/GensrcAdlc.gmk b/make/hotspot/gensrc/GensrcAdlc.gmk index c5a3ac572..2af2f9ac4 100644 --- a/make/hotspot/gensrc/GensrcAdlc.gmk +++ b/make/hotspot/gensrc/GensrcAdlc.gmk @@ -150,6 +150,12 @@ ifeq ($(call check-jvm-feature, compiler2), true) $d/os_cpu/$(HOTSPOT_TARGET_OS)_$(HOTSPOT_TARGET_CPU_ARCH)/$(HOTSPOT_TARGET_OS)_$(HOTSPOT_TARGET_CPU_ARCH).ad \ ))) + ifeq ($(call check-jvm-feature, zgc), true) + AD_SRC_FILES += $(call uniq, $(wildcard $(foreach d, $(AD_SRC_ROOTS), \ + $d/cpu/$(HOTSPOT_TARGET_CPU_ARCH)/gc/z/z_$(HOTSPOT_TARGET_CPU).ad \ + ))) + endif + ifeq ($(call check-jvm-feature, shenandoahgc), true) AD_SRC_FILES += $(call uniq, $(wildcard $(foreach d, $(AD_SRC_ROOTS), \ $d/cpu/$(HOTSPOT_TARGET_CPU_ARCH)/gc/shenandoah/shenandoah_$(HOTSPOT_TARGET_CPU).ad \ diff --git a/src/hotspot/cpu/aarch64/aarch64.ad b/src/hotspot/cpu/aarch64/aarch64.ad index a8976d5d4..b253e823a 100644 --- a/src/hotspot/cpu/aarch64/aarch64.ad +++ b/src/hotspot/cpu/aarch64/aarch64.ad @@ -1142,12 +1142,6 @@ definitions %{ int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST); %} -source_hpp %{ - -#include "gc/z/c2/zBarrierSetC2.hpp" - -%} - //----------SOURCE BLOCK------------------------------------------------------- // This is a block of C++ code which provides values, functions, and // definitions necessary in the rest of the architecture description @@ -2525,17 +2519,7 @@ void Compile::reshape_address(AddPNode* addp) { __ INSN(REG, as_Register(BASE)); \ } -typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr); -typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr); -typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt, - MacroAssembler::SIMD_RegVariant T, const Address &adr); - - // Used for all non-volatile memory accesses. The use of - // $mem->opcode() to discover whether this pattern uses sign-extended - // offsets is something of a kludge. - static void loadStore(MacroAssembler masm, mem_insn insn, - Register reg, int opcode, - Register base, int index, int size, int disp) +static Address mem2address(int opcode, Register base, int index, int size, int disp) { Address::extend scale; @@ -2554,13 +2538,18 @@ typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt, } if (index == -1) { - (masm.*insn)(reg, Address(base, disp)); + return Address(base, disp); } else { assert(disp == 0, "unsupported address mode: disp = %d", disp); - (masm.*insn)(reg, Address(base, as_Register(index), scale)); + return Address(base, as_Register(index), scale); } } +typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr); +typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr); +typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr); +typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt, + MacroAssembler::SIMD_RegVariant T, const Address &adr); static void loadStore(MacroAssembler masm, mem_insn insn, Register reg, int opcode, Register base, int index, int size, int disp, @@ -2595,9 +2584,20 @@ typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt, } } + // Used for all non-volatile memory accesses. The use of + // $mem->opcode() to discover whether this pattern uses sign-extended + // offsets is something of a kludge. + static void loadStore(MacroAssembler masm, mem_insn insn, + Register reg, int opcode, + Register base, int index, int size, int disp) + { + Address addr = mem2address(opcode, base, index, size, disp); + (masm.*insn)(reg, addr); + } + static void loadStore(MacroAssembler masm, mem_float_insn insn, - FloatRegister reg, int opcode, - Register base, int index, int size, int disp) + FloatRegister reg, int opcode, + Register base, int index, int size, int disp) { Address::extend scale; @@ -2619,8 +2619,8 @@ typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt, } static void loadStore(MacroAssembler masm, mem_vector_insn insn, - FloatRegister reg, MacroAssembler::SIMD_RegVariant T, - int opcode, Register base, int index, int size, int disp) + FloatRegister reg, MacroAssembler::SIMD_RegVariant T, + int opcode, Register base, int index, int size, int disp) { if (index == -1) { (masm.*insn)(reg, T, Address(base, disp)); @@ -3921,7 +3921,7 @@ frame %{ static const int hi[Op_RegL + 1] = { // enum name 0, // Op_Node 0, // Op_Set - OptoReg::Bad, // Op_RegN + OptoReg::Bad, // Op_RegN OptoReg::Bad, // Op_RegI R0_H_num, // Op_RegP OptoReg::Bad, // Op_RegF @@ -7075,7 +7075,7 @@ instruct loadRange(iRegINoSp dst, memory mem) instruct loadP(iRegPNoSp dst, memory mem) %{ match(Set dst (LoadP mem)); - predicate(!needs_acquiring_load(n)); + predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0)); ins_cost(4 * INSN_COST); format %{ "ldr $dst, $mem\t# ptr" %} @@ -7768,6 +7768,7 @@ instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem) instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem) %{ match(Set dst (LoadP mem)); + predicate(n->as_Load()->barrier_data() == 0); ins_cost(VOLATILE_REF_COST); format %{ "ldar $dst, $mem\t# ptr" %} @@ -8652,6 +8653,7 @@ instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoS instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ match(Set res (CompareAndSwapP mem (Binary oldval newval))); + predicate(n->as_LoadStore()->barrier_data() == 0); ins_cost(2 * VOLATILE_REF_COST); effect(KILL cr); @@ -8765,7 +8767,7 @@ instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegL instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ - predicate(needs_acquiring_load_exclusive(n)); + predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0)); match(Set res (CompareAndSwapP mem (Binary oldval newval))); ins_cost(VOLATILE_REF_COST); @@ -8896,6 +8898,7 @@ instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN ne %} instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ + predicate(n->as_LoadStore()->barrier_data() == 0); match(Set res (CompareAndExchangeP mem (Binary oldval newval))); ins_cost(2 * VOLATILE_REF_COST); effect(TEMP_DEF res, KILL cr); @@ -8995,7 +8998,7 @@ instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN %} instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ - predicate(needs_acquiring_load_exclusive(n)); + predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0)); match(Set res (CompareAndExchangeP mem (Binary oldval newval))); ins_cost(VOLATILE_REF_COST); effect(TEMP_DEF res, KILL cr); @@ -9096,6 +9099,7 @@ instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN ne %} instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ + predicate(n->as_LoadStore()->barrier_data() == 0); match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); ins_cost(2 * VOLATILE_REF_COST); effect(KILL cr); @@ -9203,8 +9207,8 @@ instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN %} instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ - predicate(needs_acquiring_load_exclusive(n)); match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); + predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0)); ins_cost(VOLATILE_REF_COST); effect(KILL cr); format %{ @@ -9254,6 +9258,7 @@ instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{ %} instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{ + predicate(n->as_LoadStore()->barrier_data() == 0); match(Set prev (GetAndSetP mem newv)); ins_cost(2 * VOLATILE_REF_COST); format %{ "atomic_xchg $prev, $newv, [$mem]" %} @@ -9297,7 +9302,7 @@ instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{ %} instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{ - predicate(needs_acquiring_load_exclusive(n)); + predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0)); match(Set prev (GetAndSetP mem newv)); ins_cost(VOLATILE_REF_COST); format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %} @@ -18518,150 +18523,6 @@ instruct vpopcount2I(vecD dst, vecD src) %{ ins_pipe(pipe_class_default); %} -source %{ - -#include "gc/z/zBarrierSetAssembler.hpp" - -static void z_load_barrier_slow_reg(MacroAssembler& _masm, Register dst, - Register base, int index, int scale, - int disp, bool weak) { - const address stub = weak ? ZBarrierSet::assembler()->load_barrier_weak_slow_stub(dst) - : ZBarrierSet::assembler()->load_barrier_slow_stub(dst); - - if (index == -1) { - if (disp != 0) { - __ lea(dst, Address(base, disp)); - } else { - __ mov(dst, base); - } - } else { - Register index_reg = as_Register(index); - if (disp == 0) { - __ lea(dst, Address(base, index_reg, Address::lsl(scale))); - } else { - __ lea(dst, Address(base, disp)); - __ lea(dst, Address(dst, index_reg, Address::lsl(scale))); - } - } - - __ far_call(RuntimeAddress(stub)); -} - -%} - -// -// Execute ZGC load barrier (strong) slow path -// -instruct loadBarrierSlowReg(iRegP dst, memory mem, rFlagsReg cr, - vRegD_V0 v0, vRegD_V1 v1, vRegD_V2 v2, vRegD_V3 v3, vRegD_V4 v4, - vRegD_V5 v5, vRegD_V6 v6, vRegD_V7 v7, vRegD_V8 v8, vRegD_V9 v9, - vRegD_V10 v10, vRegD_V11 v11, vRegD_V12 v12, vRegD_V13 v13, vRegD_V14 v14, - vRegD_V15 v15, vRegD_V16 v16, vRegD_V17 v17, vRegD_V18 v18, vRegD_V19 v19, - vRegD_V20 v20, vRegD_V21 v21, vRegD_V22 v22, vRegD_V23 v23, vRegD_V24 v24, - vRegD_V25 v25, vRegD_V26 v26, vRegD_V27 v27, vRegD_V28 v28, vRegD_V29 v29, - vRegD_V30 v30, vRegD_V31 v31) %{ - match(Set dst (LoadBarrierSlowReg mem)); - predicate(!n->as_LoadBarrierSlowReg()->is_weak()); - - effect(DEF dst, KILL cr, - KILL v0, KILL v1, KILL v2, KILL v3, KILL v4, KILL v5, KILL v6, KILL v7, - KILL v8, KILL v9, KILL v10, KILL v11, KILL v12, KILL v13, KILL v14, - KILL v15, KILL v16, KILL v17, KILL v18, KILL v19, KILL v20, KILL v21, - KILL v22, KILL v23, KILL v24, KILL v25, KILL v26, KILL v27, KILL v28, - KILL v29, KILL v30, KILL v31); - - format %{"LoadBarrierSlowReg $dst, $mem" %} - ins_encode %{ - z_load_barrier_slow_reg(_masm, $dst$$Register, $mem$$base$$Register, - $mem$$index, $mem$$scale, $mem$$disp, false); - %} - ins_pipe(pipe_slow); -%} - -// -// Execute ZGC load barrier (weak) slow path -// -instruct loadBarrierWeakSlowReg(iRegP dst, memory mem, rFlagsReg cr, - vRegD_V0 v0, vRegD_V1 v1, vRegD_V2 v2, vRegD_V3 v3, vRegD_V4 v4, - vRegD_V5 v5, vRegD_V6 v6, vRegD_V7 v7, vRegD_V8 v8, vRegD_V9 v9, - vRegD_V10 v10, vRegD_V11 v11, vRegD_V12 v12, vRegD_V13 v13, vRegD_V14 v14, - vRegD_V15 v15, vRegD_V16 v16, vRegD_V17 v17, vRegD_V18 v18, vRegD_V19 v19, - vRegD_V20 v20, vRegD_V21 v21, vRegD_V22 v22, vRegD_V23 v23, vRegD_V24 v24, - vRegD_V25 v25, vRegD_V26 v26, vRegD_V27 v27, vRegD_V28 v28, vRegD_V29 v29, - vRegD_V30 v30, vRegD_V31 v31) %{ - match(Set dst (LoadBarrierSlowReg mem)); - predicate(n->as_LoadBarrierSlowReg()->is_weak()); - - effect(DEF dst, KILL cr, - KILL v0, KILL v1, KILL v2, KILL v3, KILL v4, KILL v5, KILL v6, KILL v7, - KILL v8, KILL v9, KILL v10, KILL v11, KILL v12, KILL v13, KILL v14, - KILL v15, KILL v16, KILL v17, KILL v18, KILL v19, KILL v20, KILL v21, - KILL v22, KILL v23, KILL v24, KILL v25, KILL v26, KILL v27, KILL v28, - KILL v29, KILL v30, KILL v31); - - format %{"LoadBarrierWeakSlowReg $dst, $mem" %} - ins_encode %{ - z_load_barrier_slow_reg(_masm, $dst$$Register, $mem$$base$$Register, - $mem$$index, $mem$$scale, $mem$$disp, true); - %} - ins_pipe(pipe_slow); -%} - -// Specialized versions of compareAndExchangeP that adds a keepalive that is consumed -// but doesn't affect output. - -instruct z_compareAndExchangeP(iRegPNoSp res, indirect mem, - iRegP oldval, iRegP newval, iRegP keepalive, - rFlagsReg cr) %{ - match(Set res (ZCompareAndExchangeP (Binary mem keepalive) (Binary oldval newval))); - ins_cost(2 * VOLATILE_REF_COST); - effect(TEMP_DEF res, KILL cr); - format %{ - "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval" - %} - ins_encode %{ - __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, - Assembler::xword, /*acquire*/ false, /*release*/ true, - /*weak*/ false, $res$$Register); - %} - ins_pipe(pipe_slow); -%} - -instruct z_compareAndSwapP(iRegINoSp res, - indirect mem, - iRegP oldval, iRegP newval, iRegP keepalive, - rFlagsReg cr) %{ - - match(Set res (ZCompareAndSwapP (Binary mem keepalive) (Binary oldval newval))); - match(Set res (ZWeakCompareAndSwapP (Binary mem keepalive) (Binary oldval newval))); - - ins_cost(2 * VOLATILE_REF_COST); - - effect(KILL cr); - - format %{ - "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval" - "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)" - %} - - ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval), - aarch64_enc_cset_eq(res)); - - ins_pipe(pipe_slow); -%} - - -instruct z_get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev, - iRegP keepalive) %{ - match(Set prev (ZGetAndSetP mem (Binary newv keepalive))); - - ins_cost(2 * VOLATILE_REF_COST); - format %{ "atomic_xchg $prev, $newv, [$mem]" %} - ins_encode %{ - __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base)); - %} - ins_pipe(pipe_serial); -%} //----------PEEPHOLE RULES----------------------------------------------------- // These must follow all instruction definitions as they use the names diff --git a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp index 8e169ace4..787c0c1af 100644 --- a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp @@ -24,22 +24,23 @@ #include "precompiled.hpp" #include "asm/macroAssembler.inline.hpp" #include "code/codeBlob.hpp" +#include "code/vmreg.inline.hpp" #include "gc/z/zBarrier.inline.hpp" #include "gc/z/zBarrierSet.hpp" #include "gc/z/zBarrierSetAssembler.hpp" #include "gc/z/zBarrierSetRuntime.hpp" +#include "gc/z/zThreadLocalData.hpp" #include "memory/resourceArea.hpp" +#include "runtime/sharedRuntime.hpp" +#include "utilities/macros.hpp" #ifdef COMPILER1 #include "c1/c1_LIRAssembler.hpp" #include "c1/c1_MacroAssembler.hpp" #include "gc/z/c1/zBarrierSetC1.hpp" #endif // COMPILER1 - -#include "gc/z/zThreadLocalData.hpp" - -ZBarrierSetAssembler::ZBarrierSetAssembler() : - _load_barrier_slow_stub(), - _load_barrier_weak_slow_stub() {} +#ifdef COMPILER2 +#include "gc/z/c2/zBarrierSetC2.hpp" +#endif // COMPILER2 #ifdef PRODUCT #define BLOCK_COMMENT(str) /* nothing */ @@ -63,27 +64,25 @@ void ZBarrierSetAssembler::load_at(MacroAssembler* masm, return; } - // rscratch1 can be passed as src or dst, so don't use it. - RegSet savedRegs = RegSet::of(rscratch2, rheapbase); + assert_different_registers(rscratch1, rscratch2, src.base()); + assert_different_registers(rscratch1, rscratch2, dst); - Label done; - assert_different_registers(rheapbase, rscratch2, dst); - assert_different_registers(rheapbase, rscratch2, src.base()); + RegSet savedRegs = RegSet::range(r0, r28) - RegSet::of(dst, rscratch1, rscratch2); - __ push(savedRegs, sp); + Label done; // Load bad mask into scratch register. - __ ldr(rheapbase, address_bad_mask_from_thread(rthread)); + __ ldr(rscratch1, address_bad_mask_from_thread(rthread)); __ lea(rscratch2, src); __ ldr(dst, src); // Test reference against bad mask. If mask bad, then we need to fix it up. - __ tst(dst, rheapbase); + __ tst(dst, rscratch1); __ br(Assembler::EQ, done); __ enter(); - __ push(RegSet::range(r0,r28) - RegSet::of(dst), sp); + __ push(savedRegs, sp); if (c_rarg0 != dst) { __ mov(c_rarg0, dst); @@ -91,13 +90,15 @@ void ZBarrierSetAssembler::load_at(MacroAssembler* masm, __ mov(c_rarg1, rscratch2); int step = 4 * wordSize; - __ mov(rscratch1, -step); + __ mov(rscratch2, -step); __ sub(sp, sp, step); for (int i = 28; i >= 4; i -= 4) { __ st1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2), - as_FloatRegister(i+3), __ T1D, Address(__ post(sp, rscratch1))); + as_FloatRegister(i+3), __ T1D, Address(__ post(sp, rscratch2))); } + __ st1(as_FloatRegister(0), as_FloatRegister(1), as_FloatRegister(2), + as_FloatRegister(3), __ T1D, Address(sp)); __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), 2); @@ -111,13 +112,10 @@ void ZBarrierSetAssembler::load_at(MacroAssembler* masm, __ mov(dst, r0); } - __ pop(RegSet::range(r0,r28) - RegSet::of(dst), sp); + __ pop(savedRegs, sp); __ leave(); __ bind(done); - - // Restore tmps - __ pop(savedRegs, sp); } #ifdef ASSERT @@ -209,7 +207,7 @@ void ZBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, // The Address offset is too large to direct load - -784. Our range is +127, -128. __ mov(tmp, (long int)(in_bytes(ZThreadLocalData::address_bad_mask_offset()) - - in_bytes(JavaThread::jni_environment_offset()))); + in_bytes(JavaThread::jni_environment_offset()))); // Load address bad mask __ add(tmp, jni_env, tmp); __ ldr(tmp, Address(tmp)); @@ -297,111 +295,173 @@ void ZBarrierSetAssembler::generate_c1_load_barrier_runtime_stub(StubAssembler* __ prologue("zgc_load_barrier stub", false); // We don't use push/pop_clobbered_registers() - we need to pull out the result from r0. - for (int i = 0; i < 32; i +=2) { - __ stpd(as_FloatRegister(i), as_FloatRegister(i+1), Address(__ pre(sp,-16))); + for (int i = 0; i < 32; i += 2) { + __ stpd(as_FloatRegister(i), as_FloatRegister(i + 1), Address(__ pre(sp,-16))); } - RegSet saveRegs = RegSet::range(r0,r28) - RegSet::of(r0); - __ push(saveRegs, sp); - + const RegSet save_regs = RegSet::range(r1, r28); + __ push(save_regs, sp); // Setup arguments __ load_parameter(0, c_rarg0); __ load_parameter(1, c_rarg1); __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), 2); - __ pop(saveRegs, sp); + __ pop(save_regs, sp); - for (int i = 30; i >0; i -=2) { - __ ldpd(as_FloatRegister(i), as_FloatRegister(i+1), Address(__ post(sp, 16))); + for (int i = 30; i >0; i -= 2) { + __ ldpd(as_FloatRegister(i), as_FloatRegister(i + 1), Address(__ post(sp, 16))); } __ epilogue(); } #endif // COMPILER1 -#undef __ -#define __ cgen->assembler()-> - -// Generates a register specific stub for calling -// ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded() or -// ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded(). -// -// The raddr register serves as both input and output for this stub. When the stub is -// called the raddr register contains the object field address (oop*) where the bad oop -// was loaded from, which caused the slow path to be taken. On return from the stub the -// raddr register contains the good/healed oop returned from -// ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded() or -// ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded(). -static address generate_load_barrier_stub(StubCodeGenerator* cgen, Register raddr, DecoratorSet decorators) { - // Don't generate stub for invalid registers - if (raddr == zr || raddr == r29 || raddr == r30) { - return NULL; +#ifdef COMPILER2 + +OptoReg::Name ZBarrierSetAssembler::refine_register(const Node* node, OptoReg::Name opto_reg) { + if (!OptoReg::is_reg(opto_reg)) { + return OptoReg::Bad; } - // Create stub name - char name[64]; - const bool weak = (decorators & ON_WEAK_OOP_REF) != 0; - os::snprintf(name, sizeof(name), "zgc_load_barrier%s_stub_%s", weak ? "_weak" : "", raddr->name()); + const VMReg vm_reg = OptoReg::as_VMReg(opto_reg); + if (vm_reg->is_FloatRegister()) { + return opto_reg & ~1; + } - __ align(CodeEntryAlignment); - StubCodeMark mark(cgen, "StubRoutines", os::strdup(name, mtCode)); - address start = __ pc(); + return opto_reg; +} - // Save live registers - RegSet savedRegs = RegSet::range(r0,r18) - RegSet::of(raddr); +#undef __ +#define __ _masm-> + +class ZSaveLiveRegisters { +private: + MacroAssembler* const _masm; + RegSet _gp_regs; + RegSet _fp_regs; + +public: + void initialize(ZLoadBarrierStubC2* stub) { + // Create mask of live registers + RegMask live = stub->live(); + + // Record registers that needs to be saved/restored + while (live.is_NotEmpty()) { + const OptoReg::Name opto_reg = live.find_first_elem(); + live.Remove(opto_reg); + if (OptoReg::is_reg(opto_reg)) { + const VMReg vm_reg = OptoReg::as_VMReg(opto_reg); + if (vm_reg->is_Register()) { + _gp_regs += RegSet::of(vm_reg->as_Register()); + } else if (vm_reg->is_FloatRegister()) { + _fp_regs += RegSet::of((Register)vm_reg->as_FloatRegister()); + } else { + fatal("Unknown register type"); + } + } + } - __ enter(); - __ push(savedRegs, sp); + // Remove C-ABI SOE registers, scratch regs and _ref register that will be updated + _gp_regs -= RegSet::range(r19, r30) + RegSet::of(r8, r9, stub->ref()); + } - // Setup arguments - if (raddr != c_rarg1) { - __ mov(c_rarg1, raddr); - } + ZSaveLiveRegisters(MacroAssembler* masm, ZLoadBarrierStubC2* stub) : + _masm(masm), + _gp_regs(), + _fp_regs() { - __ ldr(c_rarg0, Address(raddr)); + // Figure out what registers to save/restore + initialize(stub); - // Call barrier function - __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), c_rarg0, c_rarg1); + // Save registers + __ push(_gp_regs, sp); + __ push_fp(_fp_regs, sp); + } - // Move result returned in r0 to raddr, if needed - if (raddr != r0) { - __ mov(raddr, r0); + ~ZSaveLiveRegisters() { + // Restore registers + __ pop_fp(_fp_regs, sp); + __ pop(_gp_regs, sp); } +}; - __ pop(savedRegs, sp); - __ leave(); - __ ret(lr); +#undef __ +#define __ _masm-> + +class ZSetupArguments { +private: + MacroAssembler* const _masm; + const Register _ref; + const Address _ref_addr; + +public: + ZSetupArguments(MacroAssembler* masm, ZLoadBarrierStubC2* stub) : + _masm(masm), + _ref(stub->ref()), + _ref_addr(stub->ref_addr()) { + + // Setup arguments + if (_ref_addr.base() == noreg) { + // No self healing + if (_ref != c_rarg0) { + __ mov(c_rarg0, _ref); + } + __ mov(c_rarg1, 0); + } else { + // Self healing + if (_ref == c_rarg0) { + // _ref is already at correct place + __ lea(c_rarg1, _ref_addr); + } else if (_ref != c_rarg1) { + // _ref is in wrong place, but not in c_rarg1, so fix it first + __ lea(c_rarg1, _ref_addr); + __ mov(c_rarg0, _ref); + } else if (_ref_addr.base() != c_rarg0 && _ref_addr.index() != c_rarg0) { + assert(_ref == c_rarg1, "Mov ref first, vacating c_rarg0"); + __ mov(c_rarg0, _ref); + __ lea(c_rarg1, _ref_addr); + } else { + assert(_ref == c_rarg1, "Need to vacate c_rarg1 and _ref_addr is using c_rarg0"); + if (_ref_addr.base() == c_rarg0 || _ref_addr.index() == c_rarg0) { + __ mov(rscratch2, c_rarg1); + __ lea(c_rarg1, _ref_addr); + __ mov(c_rarg0, rscratch2); + } else { + ShouldNotReachHere(); + } + } + } + } - return start; -} + ~ZSetupArguments() { + // Transfer result + if (_ref != r0) { + __ mov(_ref, r0); + } + } +}; #undef __ +#define __ masm-> -static void barrier_stubs_init_inner(const char* label, const DecoratorSet decorators, address* stub) { - const int nregs = 28; // Exclude FP, XZR, SP from calculation. - const int code_size = nregs * 254; // Rough estimate of code size - - ResourceMark rm; +void ZBarrierSetAssembler::generate_c2_load_barrier_stub(MacroAssembler* masm, ZLoadBarrierStubC2* stub) const { + BLOCK_COMMENT("ZLoadBarrierStubC2"); - CodeBuffer buf(BufferBlob::create(label, code_size)); - StubCodeGenerator cgen(&buf); + // Stub entry + __ bind(*stub->entry()); - for (int i = 0; i < nregs; i++) { - const Register reg = as_Register(i); - stub[i] = generate_load_barrier_stub(&cgen, reg, decorators); + { + ZSaveLiveRegisters save_live_registers(masm, stub); + ZSetupArguments setup_arguments(masm, stub); + __ mov(rscratch1, stub->slow_path()); + __ blr(rscratch1); } -} -void ZBarrierSetAssembler::barrier_stubs_init() { - barrier_stubs_init_inner("zgc_load_barrier_stubs", ON_STRONG_OOP_REF, _load_barrier_slow_stub); - barrier_stubs_init_inner("zgc_load_barrier_weak_stubs", ON_WEAK_OOP_REF, _load_barrier_weak_slow_stub); + // Stub exit + __ b(*stub->continuation()); } -address ZBarrierSetAssembler::load_barrier_slow_stub(Register reg) { - return _load_barrier_slow_stub[reg->encoding()]; -} +#undef __ -address ZBarrierSetAssembler::load_barrier_weak_slow_stub(Register reg) { - return _load_barrier_weak_slow_stub[reg->encoding()]; -} +#endif // COMPILER2 diff --git a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp index 7e8be01cc..cca873825 100644 --- a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp @@ -24,6 +24,12 @@ #ifndef CPU_AARCH64_GC_Z_ZBARRIERSETASSEMBLER_AARCH64_HPP #define CPU_AARCH64_GC_Z_ZBARRIERSETASSEMBLER_AARCH64_HPP +#include "code/vmreg.hpp" +#include "oops/accessDecorators.hpp" +#ifdef COMPILER2 +#include "opto/optoreg.hpp" +#endif // COMPILER2 + #ifdef COMPILER1 class LIR_Assembler; class LIR_OprDesc; @@ -32,14 +38,13 @@ class StubAssembler; class ZLoadBarrierStubC1; #endif // COMPILER1 -class ZBarrierSetAssembler : public ZBarrierSetAssemblerBase { -private: - address _load_barrier_slow_stub[RegisterImpl::number_of_registers]; - address _load_barrier_weak_slow_stub[RegisterImpl::number_of_registers]; +#ifdef COMPILER2 +class Node; +class ZLoadBarrierStubC2; +#endif // COMPILER2 +class ZBarrierSetAssembler : public ZBarrierSetAssemblerBase { public: - ZBarrierSetAssembler(); - virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, @@ -83,10 +88,13 @@ public: DecoratorSet decorators) const; #endif // COMPILER1 - virtual void barrier_stubs_init(); +#ifdef COMPILER2 + OptoReg::Name refine_register(const Node* node, + OptoReg::Name opto_reg); - address load_barrier_slow_stub(Register reg); - address load_barrier_weak_slow_stub(Register reg); + void generate_c2_load_barrier_stub(MacroAssembler* masm, + ZLoadBarrierStubC2* stub) const; +#endif // COMPILER2 }; #endif // CPU_AARCH64_GC_Z_ZBARRIERSETASSEMBLER_AARCH64_HPP diff --git a/src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad b/src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad new file mode 100644 index 000000000..50cc6f924 --- /dev/null +++ b/src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad @@ -0,0 +1,268 @@ +// +// Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. +// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +// +// This code is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License version 2 only, as +// published by the Free Software Foundation. +// +// This code is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +// version 2 for more details (a copy is included in the LICENSE file that +// accompanied this code). +// +// You should have received a copy of the GNU General Public License version +// 2 along with this work; if not, write to the Free Software Foundation, +// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. +// +// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA +// or visit www.oracle.com if you need additional information or have any +// questions. +// + +source_hpp %{ + +#include "gc/z/c2/zBarrierSetC2.hpp" +#include "gc/z/zThreadLocalData.hpp" + +%} + +source %{ + +static void z_load_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp, bool weak) { + ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, weak); + __ ldr(tmp, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); + __ andr(tmp, tmp, ref); + __ cbnz(tmp, *stub->entry()); + __ bind(*stub->continuation()); +} + +static void z_load_barrier_slow_path(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp) { + ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, false /* weak */); + __ b(*stub->entry()); + __ bind(*stub->continuation()); +} + +%} + +// Load Pointer +instruct zLoadP(iRegPNoSp dst, memory mem, rFlagsReg cr) +%{ + match(Set dst (LoadP mem)); + predicate(UseZGC && !needs_acquiring_load(n) && (n->as_Load()->barrier_data() == ZLoadBarrierStrong)); + effect(TEMP dst, KILL cr); + + ins_cost(4 * INSN_COST); + + format %{ "ldr $dst, $mem" %} + + ins_encode %{ + const Address ref_addr = mem2address($mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp); + __ ldr($dst$$Register, ref_addr); + if (barrier_data() != ZLoadBarrierElided) { + z_load_barrier(_masm, this, ref_addr, $dst$$Register, rscratch2 /* tmp */, false /* weak */); + } + %} + + ins_pipe(iload_reg_mem); +%} + +// Load Weak Pointer +instruct zLoadWeakP(iRegPNoSp dst, memory mem, rFlagsReg cr) +%{ + match(Set dst (LoadP mem)); + predicate(UseZGC && !needs_acquiring_load(n) && (n->as_Load()->barrier_data() == ZLoadBarrierWeak)); + effect(TEMP dst, KILL cr); + + ins_cost(4 * INSN_COST); + + format %{ "ldr $dst, $mem" %} + + ins_encode %{ + const Address ref_addr = mem2address($mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp); + __ ldr($dst$$Register, ref_addr); + z_load_barrier(_masm, this, ref_addr, $dst$$Register, rscratch2 /* tmp */, true /* weak */); + %} + + ins_pipe(iload_reg_mem); +%} + +// Load Pointer Volatile +instruct zLoadPVolatile(iRegPNoSp dst, indirect mem /* sync_memory */, rFlagsReg cr) +%{ + match(Set dst (LoadP mem)); + predicate(UseZGC && needs_acquiring_load(n) && n->as_Load()->barrier_data() == ZLoadBarrierStrong); + effect(TEMP dst, KILL cr); + + ins_cost(VOLATILE_REF_COST); + + format %{ "ldar $dst, $mem\t" %} + + ins_encode %{ + __ ldar($dst$$Register, $mem$$Register); + if (barrier_data() != ZLoadBarrierElided) { + z_load_barrier(_masm, this, Address($mem$$Register), $dst$$Register, rscratch2 /* tmp */, false /* weak */); + } + %} + + ins_pipe(pipe_serial); +%} + +instruct zCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ + match(Set res (CompareAndSwapP mem (Binary oldval newval))); + match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); + predicate(UseZGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); + effect(KILL cr, TEMP_DEF res); + + ins_cost(2 * VOLATILE_REF_COST); + + format %{ "cmpxchg $mem, $oldval, $newval\n\t" + "cset $res, EQ" %} + + ins_encode %{ + guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, + false /* acquire */, true /* release */, false /* weak */, rscratch2); + __ cset($res$$Register, Assembler::EQ); + if (barrier_data() != ZLoadBarrierElided) { + Label good; + __ ldr(rscratch1, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); + __ andr(rscratch1, rscratch1, rscratch2); + __ cbz(rscratch1, good); + z_load_barrier_slow_path(_masm, this, Address($mem$$Register), rscratch2 /* ref */, rscratch1 /* tmp */); + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, + false /* acquire */, true /* release */, false /* weak */, rscratch2); + __ cset($res$$Register, Assembler::EQ); + __ bind(good); + } + %} + + ins_pipe(pipe_slow); +%} + +instruct zCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ + match(Set res (CompareAndSwapP mem (Binary oldval newval))); + match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); + predicate(UseZGC && needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong)); + effect(KILL cr, TEMP_DEF res); + + ins_cost(2 * VOLATILE_REF_COST); + + format %{ "cmpxchg $mem, $oldval, $newval\n\t" + "cset $res, EQ" %} + + ins_encode %{ + guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, + true /* acquire */, true /* release */, false /* weak */, rscratch2); + __ cset($res$$Register, Assembler::EQ); + if (barrier_data() != ZLoadBarrierElided) { + Label good; + __ ldr(rscratch1, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); + __ andr(rscratch1, rscratch1, rscratch2); + __ cbz(rscratch1, good); + z_load_barrier_slow_path(_masm, this, Address($mem$$Register), rscratch2 /* ref */, rscratch1 /* tmp */ ); + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, + true /* acquire */, true /* release */, false /* weak */, rscratch2); + __ cset($res$$Register, Assembler::EQ); + __ bind(good); + } + %} + + ins_pipe(pipe_slow); +%} + +instruct zCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ + match(Set res (CompareAndExchangeP mem (Binary oldval newval))); + predicate(UseZGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); + effect(TEMP_DEF res, KILL cr); + + ins_cost(2 * VOLATILE_REF_COST); + + format %{ "cmpxchg $res = $mem, $oldval, $newval" %} + + ins_encode %{ + guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, + false /* acquire */, true /* release */, false /* weak */, $res$$Register); + if (barrier_data() != ZLoadBarrierElided) { + Label good; + __ ldr(rscratch1, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); + __ andr(rscratch1, rscratch1, $res$$Register); + __ cbz(rscratch1, good); + z_load_barrier_slow_path(_masm, this, Address($mem$$Register), $res$$Register /* ref */, rscratch1 /* tmp */); + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, + false /* acquire */, true /* release */, false /* weak */, $res$$Register); + __ bind(good); + } + %} + + ins_pipe(pipe_slow); +%} + +instruct zCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ + match(Set res (CompareAndExchangeP mem (Binary oldval newval))); + predicate(UseZGC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); + effect(TEMP_DEF res, KILL cr); + + ins_cost(2 * VOLATILE_REF_COST); + + format %{ "cmpxchg $res = $mem, $oldval, $newval" %} + + ins_encode %{ + guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, + true /* acquire */, true /* release */, false /* weak */, $res$$Register); + if (barrier_data() != ZLoadBarrierElided) { + Label good; + __ ldr(rscratch1, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); + __ andr(rscratch1, rscratch1, $res$$Register); + __ cbz(rscratch1, good); + z_load_barrier_slow_path(_masm, this, Address($mem$$Register), $res$$Register /* ref */, rscratch1 /* tmp */); + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, + true /* acquire */, true /* release */, false /* weak */, $res$$Register); + __ bind(good); + } + %} + + ins_pipe(pipe_slow); +%} + +instruct zGetAndSetP(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{ + match(Set prev (GetAndSetP mem newv)); + predicate(UseZGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); + effect(TEMP_DEF prev, KILL cr); + + ins_cost(2 * VOLATILE_REF_COST); + + format %{ "atomic_xchg $prev, $newv, [$mem]" %} + + ins_encode %{ + __ atomic_xchg($prev$$Register, $newv$$Register, $mem$$Register); + if (barrier_data() != ZLoadBarrierElided) { + z_load_barrier(_masm, this, Address(noreg, 0), $prev$$Register, rscratch2 /* tmp */, false /* weak */); + } + %} + + ins_pipe(pipe_serial); +%} + +instruct zGetAndSetPAcq(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{ + match(Set prev (GetAndSetP mem newv)); + predicate(UseZGC && needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong)); + effect(TEMP_DEF prev, KILL cr); + + ins_cost(VOLATILE_REF_COST); + + format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %} + + ins_encode %{ + __ atomic_xchgal($prev$$Register, $newv$$Register, $mem$$Register); + if (barrier_data() != ZLoadBarrierElided) { + z_load_barrier(_masm, this, Address(noreg, 0), $prev$$Register, rscratch2 /* tmp */, false /* weak */); + } + %} + ins_pipe(pipe_serial); +%} + diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp index 7f329a45d..5ddf049ce 100644 --- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp @@ -2129,6 +2129,67 @@ int MacroAssembler::pop(unsigned int bitset, Register stack) { return count; } + +// Push lots of registers in the bit set supplied. Don't push sp. +// Return the number of words pushed +int MacroAssembler::push_fp(unsigned int bitset, Register stack) { + // Scan bitset to accumulate register pairs + unsigned char regs[32]; + int count = 0; + int i = 0; + for (int reg = 0; reg <= 31; reg++) { + if (1 & bitset) + regs[count++] = reg; + bitset >>= 1; + } + + if (!count) { + return 0; + } + + add(stack, stack, -count * wordSize * 2); + + if (count & 1) { + strq(as_FloatRegister(regs[0]), Address(stack)); + i += 1; + } + + for (; i < count; i += 2) { + stpq(as_FloatRegister(regs[i]), as_FloatRegister(regs[i+1]), Address(stack, i * wordSize * 2)); + } + + return count; +} + +int MacroAssembler::pop_fp(unsigned int bitset, Register stack) { + // Scan bitset to accumulate register pairs + unsigned char regs[32]; + int count = 0; + int i = 0; + for (int reg = 0; reg <= 31; reg++) { + if (1 & bitset) + regs[count++] = reg; + bitset >>= 1; + } + + if (!count) { + return 0; + } + + if (count & 1) { + ldrq(as_FloatRegister(regs[0]), Address(stack)); + i += 1; + } + + for (; i < count; i += 2) { + ldpq(as_FloatRegister(regs[i]), as_FloatRegister(regs[i+1]), Address(stack, i * wordSize * 2)); + } + + add(stack, stack, count * wordSize * 2); + + return count; +} + #ifdef ASSERT void MacroAssembler::verify_heapbase(const char* msg) { #if 0 diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp index 01fdf16a0..073854d2b 100644 --- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp @@ -463,12 +463,18 @@ private: int push(unsigned int bitset, Register stack); int pop(unsigned int bitset, Register stack); + int push_fp(unsigned int bitset, Register stack); + int pop_fp(unsigned int bitset, Register stack); + void mov(Register dst, Address a); public: void push(RegSet regs, Register stack) { if (regs.bits()) push(regs.bits(), stack); } void pop(RegSet regs, Register stack) { if (regs.bits()) pop(regs.bits(), stack); } + void push_fp(RegSet regs, Register stack) { if (regs.bits()) push_fp(regs.bits(), stack); } + void pop_fp(RegSet regs, Register stack) { if (regs.bits()) pop_fp(regs.bits(), stack); } + static RegSet call_clobbered_registers(); // Push and pop everything that might be clobbered by a native diff --git a/src/hotspot/cpu/aarch64/templateInterpreterGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/templateInterpreterGenerator_aarch64.cpp index 21ba661ea..430f3ee14 100644 --- a/src/hotspot/cpu/aarch64/templateInterpreterGenerator_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/templateInterpreterGenerator_aarch64.cpp @@ -880,8 +880,8 @@ void TemplateInterpreterGenerator::generate_fixed_frame(bool native_call) { } else #endif { - __ load_mirror(rscratch1, rmethod); - __ stp(rscratch1, zr, Address(sp, 4 * wordSize)); + __ load_mirror(r10, rmethod); + __ stp(r10, zr, Address(sp, 4 * wordSize)); } __ ldr(rcpool, Address(rmethod, Method::const_offset())); diff --git a/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp b/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp index 8c9676aed..e417f07be 100644 --- a/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp @@ -75,6 +75,7 @@ public: CPU_BROADCOM = 'B', CPU_CAVIUM = 'C', CPU_DEC = 'D', + CPU_HISILICON = 'H', CPU_INFINEON = 'I', CPU_MOTOROLA = 'M', CPU_NVIDIA = 'N', @@ -107,6 +108,13 @@ public: static int cpu_variant() { return _variant; } static int cpu_revision() { return _revision; } + static bool is_hisi_enabled() { + if (_cpu == CPU_HISILICON && (_model == 0xd01 || _model == 0xd02 || _model == 0xd03)) { + return true; + } + return false; + } + static bool is_zva_enabled() { return 0 <= _zva_length; } static int zva_length() { assert(is_zva_enabled(), "ZVA not available"); diff --git a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp index f5de1ed88..4428e96bc 100644 --- a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp @@ -24,21 +24,22 @@ #include "precompiled.hpp" #include "asm/macroAssembler.inline.hpp" #include "code/codeBlob.hpp" +#include "code/vmreg.inline.hpp" #include "gc/z/zBarrier.inline.hpp" #include "gc/z/zBarrierSet.hpp" #include "gc/z/zBarrierSetAssembler.hpp" #include "gc/z/zBarrierSetRuntime.hpp" #include "memory/resourceArea.hpp" -#include "runtime/stubCodeGenerator.hpp" +#include "runtime/sharedRuntime.hpp" #include "utilities/macros.hpp" #ifdef COMPILER1 #include "c1/c1_LIRAssembler.hpp" #include "c1/c1_MacroAssembler.hpp" #include "gc/z/c1/zBarrierSetC1.hpp" #endif // COMPILER1 - -#undef __ -#define __ masm-> +#ifdef COMPILER2 +#include "gc/z/c2/zBarrierSetC2.hpp" +#endif // COMPILER2 #ifdef PRODUCT #define BLOCK_COMMENT(str) /* nothing */ @@ -46,6 +47,9 @@ #define BLOCK_COMMENT(str) __ block_comment(str) #endif +#undef __ +#define __ masm-> + static void call_vm(MacroAssembler* masm, address entry_point, Register arg0, @@ -335,126 +339,326 @@ void ZBarrierSetAssembler::generate_c1_load_barrier_runtime_stub(StubAssembler* #endif // COMPILER1 -#undef __ -#define __ cgen->assembler()-> - -// Generates a register specific stub for calling -// ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded() or -// ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded(). -// -// The raddr register serves as both input and output for this stub. When the stub is -// called the raddr register contains the object field address (oop*) where the bad oop -// was loaded from, which caused the slow path to be taken. On return from the stub the -// raddr register contains the good/healed oop returned from -// ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded() or -// ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded(). -static address generate_load_barrier_stub(StubCodeGenerator* cgen, Register raddr, DecoratorSet decorators) { - // Don't generate stub for invalid registers - if (raddr == rsp || raddr == r12 || raddr == r15) { - return NULL; +#ifdef COMPILER2 + +OptoReg::Name ZBarrierSetAssembler::refine_register(const Node* node, OptoReg::Name opto_reg) { + if (!OptoReg::is_reg(opto_reg)) { + return OptoReg::Bad; + } + + const VMReg vm_reg = OptoReg::as_VMReg(opto_reg); + if (vm_reg->is_XMMRegister()) { + opto_reg &= ~15; + switch (node->ideal_reg()) { + case Op_VecX: + opto_reg |= 2; + break; + case Op_VecY: + opto_reg |= 4; + break; + case Op_VecZ: + opto_reg |= 8; + break; + default: + opto_reg |= 1; + break; + } } - // Create stub name - char name[64]; - const bool weak = (decorators & ON_WEAK_OOP_REF) != 0; - os::snprintf(name, sizeof(name), "load_barrier%s_stub_%s", weak ? "_weak" : "", raddr->name()); + return opto_reg; +} + +// We use the vec_spill_helper from the x86.ad file to avoid reinventing this wheel +extern int vec_spill_helper(CodeBuffer *cbuf, bool do_size, bool is_load, + int stack_offset, int reg, uint ireg, outputStream* st); +#undef __ +#define __ _masm-> - __ align(CodeEntryAlignment); - StubCodeMark mark(cgen, "StubRoutines", os::strdup(name, mtCode)); - address start = __ pc(); +class ZSaveLiveRegisters { +private: + struct XMMRegisterData { + XMMRegister _reg; + int _size; - // Save live registers - if (raddr != rax) { - __ push(rax); - } - if (raddr != rcx) { - __ push(rcx); - } - if (raddr != rdx) { - __ push(rdx); + // Used by GrowableArray::find() + bool operator == (const XMMRegisterData& other) { + return _reg == other._reg; + } + }; + + MacroAssembler* const _masm; + GrowableArray<Register> _gp_registers; + GrowableArray<XMMRegisterData> _xmm_registers; + int _spill_size; + int _spill_offset; + + static int xmm_compare_register_size(XMMRegisterData* left, XMMRegisterData* right) { + if (left->_size == right->_size) { + return 0; + } + + return (left->_size < right->_size) ? -1 : 1; } - if (raddr != rsi) { - __ push(rsi); + + static int xmm_slot_size(OptoReg::Name opto_reg) { + // The low order 4 bytes denote what size of the XMM register is live + return (opto_reg & 15) << 3; } - if (raddr != rdi) { - __ push(rdi); + + static uint xmm_ideal_reg_for_size(int reg_size) { + switch (reg_size) { + case 8: + return Op_VecD; + case 16: + return Op_VecX; + case 32: + return Op_VecY; + case 64: + return Op_VecZ; + default: + fatal("Invalid register size %d", reg_size); + return 0; + } } - if (raddr != r8) { - __ push(r8); + + bool xmm_needs_vzeroupper() const { + return _xmm_registers.is_nonempty() && _xmm_registers.at(0)._size > 16; } - if (raddr != r9) { - __ push(r9); + + void xmm_register_save(const XMMRegisterData& reg_data) { + const OptoReg::Name opto_reg = OptoReg::as_OptoReg(reg_data._reg->as_VMReg()); + const uint ideal_reg = xmm_ideal_reg_for_size(reg_data._size); + _spill_offset -= reg_data._size; + vec_spill_helper(__ code(), false /* do_size */, false /* is_load */, _spill_offset, opto_reg, ideal_reg, tty); } - if (raddr != r10) { - __ push(r10); + + void xmm_register_restore(const XMMRegisterData& reg_data) { + const OptoReg::Name opto_reg = OptoReg::as_OptoReg(reg_data._reg->as_VMReg()); + const uint ideal_reg = xmm_ideal_reg_for_size(reg_data._size); + vec_spill_helper(__ code(), false /* do_size */, true /* is_load */, _spill_offset, opto_reg, ideal_reg, tty); + _spill_offset += reg_data._size; } - if (raddr != r11) { - __ push(r11); + + void gp_register_save(Register reg) { + _spill_offset -= 8; + __ movq(Address(rsp, _spill_offset), reg); } - // Setup arguments - if (c_rarg1 != raddr) { - __ movq(c_rarg1, raddr); + void gp_register_restore(Register reg) { + __ movq(reg, Address(rsp, _spill_offset)); + _spill_offset += 8; } - __ movq(c_rarg0, Address(raddr, 0)); - // Call barrier function - __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), c_rarg0, c_rarg1); + void initialize(ZLoadBarrierStubC2* stub) { + // Create mask of caller saved registers that need to + // be saved/restored if live + RegMask caller_saved; + caller_saved.Insert(OptoReg::as_OptoReg(rax->as_VMReg())); + caller_saved.Insert(OptoReg::as_OptoReg(rcx->as_VMReg())); + caller_saved.Insert(OptoReg::as_OptoReg(rdx->as_VMReg())); + caller_saved.Insert(OptoReg::as_OptoReg(rsi->as_VMReg())); + caller_saved.Insert(OptoReg::as_OptoReg(rdi->as_VMReg())); + caller_saved.Insert(OptoReg::as_OptoReg(r8->as_VMReg())); + caller_saved.Insert(OptoReg::as_OptoReg(r9->as_VMReg())); + caller_saved.Insert(OptoReg::as_OptoReg(r10->as_VMReg())); + caller_saved.Insert(OptoReg::as_OptoReg(r11->as_VMReg())); + caller_saved.Remove(OptoReg::as_OptoReg(stub->ref()->as_VMReg())); - // Move result returned in rax to raddr, if needed - if (raddr != rax) { - __ movq(raddr, rax); - } + // Create mask of live registers + RegMask live = stub->live(); + if (stub->tmp() != noreg) { + live.Insert(OptoReg::as_OptoReg(stub->tmp()->as_VMReg())); + } - // Restore saved registers - if (raddr != r11) { - __ pop(r11); - } - if (raddr != r10) { - __ pop(r10); - } - if (raddr != r9) { - __ pop(r9); - } - if (raddr != r8) { - __ pop(r8); - } - if (raddr != rdi) { - __ pop(rdi); - } - if (raddr != rsi) { - __ pop(rsi); - } - if (raddr != rdx) { - __ pop(rdx); + int gp_spill_size = 0; + int xmm_spill_size = 0; + + // Record registers that needs to be saved/restored + while (live.is_NotEmpty()) { + const OptoReg::Name opto_reg = live.find_first_elem(); + const VMReg vm_reg = OptoReg::as_VMReg(opto_reg); + + live.Remove(opto_reg); + + if (vm_reg->is_Register()) { + if (caller_saved.Member(opto_reg)) { + _gp_registers.append(vm_reg->as_Register()); + gp_spill_size += 8; + } + } else if (vm_reg->is_XMMRegister()) { + // We encode in the low order 4 bits of the opto_reg, how large part of the register is live + const VMReg vm_reg_base = OptoReg::as_VMReg(opto_reg & ~15); + const int reg_size = xmm_slot_size(opto_reg); + const XMMRegisterData reg_data = { vm_reg_base->as_XMMRegister(), reg_size }; + const int reg_index = _xmm_registers.find(reg_data); + if (reg_index == -1) { + // Not previously appended + _xmm_registers.append(reg_data); + xmm_spill_size += reg_size; + } else { + // Previously appended, update size + const int reg_size_prev = _xmm_registers.at(reg_index)._size; + if (reg_size > reg_size_prev) { + _xmm_registers.at_put(reg_index, reg_data); + xmm_spill_size += reg_size - reg_size_prev; + } + } + } else { + fatal("Unexpected register type"); + } + } + + // Sort by size, largest first + _xmm_registers.sort(xmm_compare_register_size); + + // Stack pointer must be 16 bytes aligned for the call + _spill_offset = _spill_size = align_up(xmm_spill_size + gp_spill_size, 16); + } + +public: + ZSaveLiveRegisters(MacroAssembler* masm, ZLoadBarrierStubC2* stub) : + _masm(masm), + _gp_registers(), + _xmm_registers(), + _spill_size(0), + _spill_offset(0) { + + // + // Stack layout after registers have been spilled: + // + // | ... | original rsp, 16 bytes aligned + // ------------------ + // | zmm0 high | + // | ... | + // | zmm0 low | 16 bytes aligned + // | ... | + // | ymm1 high | + // | ... | + // | ymm1 low | 16 bytes aligned + // | ... | + // | xmmN high | + // | ... | + // | xmmN low | 8 bytes aligned + // | reg0 | 8 bytes aligned + // | reg1 | + // | ... | + // | regN | new rsp, if 16 bytes aligned + // | <padding> | else new rsp, 16 bytes aligned + // ------------------ + // + + // Figure out what registers to save/restore + initialize(stub); + + // Allocate stack space + if (_spill_size > 0) { + __ subptr(rsp, _spill_size); + } + + // Save XMM/YMM/ZMM registers + for (int i = 0; i < _xmm_registers.length(); i++) { + xmm_register_save(_xmm_registers.at(i)); + } + + if (xmm_needs_vzeroupper()) { + __ vzeroupper(); + } + + // Save general purpose registers + for (int i = 0; i < _gp_registers.length(); i++) { + gp_register_save(_gp_registers.at(i)); + } } - if (raddr != rcx) { - __ pop(rcx); + + ~ZSaveLiveRegisters() { + // Restore general purpose registers + for (int i = _gp_registers.length() - 1; i >= 0; i--) { + gp_register_restore(_gp_registers.at(i)); + } + + __ vzeroupper(); + + // Restore XMM/YMM/ZMM registers + for (int i = _xmm_registers.length() - 1; i >= 0; i--) { + xmm_register_restore(_xmm_registers.at(i)); + } + + // Free stack space + if (_spill_size > 0) { + __ addptr(rsp, _spill_size); + } } - if (raddr != rax) { - __ pop(rax); +}; + +class ZSetupArguments { +private: + MacroAssembler* const _masm; + const Register _ref; + const Address _ref_addr; + +public: + ZSetupArguments(MacroAssembler* masm, ZLoadBarrierStubC2* stub) : + _masm(masm), + _ref(stub->ref()), + _ref_addr(stub->ref_addr()) { + + // Setup arguments + if (_ref_addr.base() == noreg) { + // No self healing + if (_ref != c_rarg0) { + __ movq(c_rarg0, _ref); + } + __ xorq(c_rarg1, c_rarg1); + } else { + // Self healing + if (_ref == c_rarg0) { + __ lea(c_rarg1, _ref_addr); + } else if (_ref != c_rarg1) { + __ lea(c_rarg1, _ref_addr); + __ movq(c_rarg0, _ref); + } else if (_ref_addr.base() != c_rarg0 && _ref_addr.index() != c_rarg0) { + __ movq(c_rarg0, _ref); + __ lea(c_rarg1, _ref_addr); + } else { + __ xchgq(c_rarg0, c_rarg1); + if (_ref_addr.base() == c_rarg0) { + __ lea(c_rarg1, Address(c_rarg1, _ref_addr.index(), _ref_addr.scale(), _ref_addr.disp())); + } else if (_ref_addr.index() == c_rarg0) { + __ lea(c_rarg1, Address(_ref_addr.base(), c_rarg1, _ref_addr.scale(), _ref_addr.disp())); + } else { + ShouldNotReachHere(); + } + } + } } - __ ret(0); - - return start; -} + ~ZSetupArguments() { + // Transfer result + if (_ref != rax) { + __ movq(_ref, rax); + } + } +}; #undef __ +#define __ masm-> -void ZBarrierSetAssembler::barrier_stubs_init() { - // Load barrier stubs - int stub_code_size = 256 * 16; // Rough estimate of code size +void ZBarrierSetAssembler::generate_c2_load_barrier_stub(MacroAssembler* masm, ZLoadBarrierStubC2* stub) const { + BLOCK_COMMENT("ZLoadBarrierStubC2"); - ResourceMark rm; - BufferBlob* bb = BufferBlob::create("zgc_load_barrier_stubs", stub_code_size); - CodeBuffer buf(bb); - StubCodeGenerator cgen(&buf); + // Stub entry + __ bind(*stub->entry()); - Register rr = as_Register(0); - for (int i = 0; i < RegisterImpl::number_of_registers; i++) { - _load_barrier_slow_stub[i] = generate_load_barrier_stub(&cgen, rr, ON_STRONG_OOP_REF); - _load_barrier_weak_slow_stub[i] = generate_load_barrier_stub(&cgen, rr, ON_WEAK_OOP_REF); - rr = rr->successor(); + { + ZSaveLiveRegisters save_live_registers(masm, stub); + ZSetupArguments setup_arguments(masm, stub); + __ call(RuntimeAddress(stub->slow_path())); } + + // Stub exit + __ jmp(*stub->continuation()); } + +#undef __ + +#endif // COMPILER2 diff --git a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp index 3687754e7..e433882a4 100644 --- a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp +++ b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp @@ -24,6 +24,14 @@ #ifndef CPU_X86_GC_Z_ZBARRIERSETASSEMBLER_X86_HPP #define CPU_X86_GC_Z_ZBARRIERSETASSEMBLER_X86_HPP +#include "code/vmreg.hpp" +#include "oops/accessDecorators.hpp" +#ifdef COMPILER2 +#include "opto/optoreg.hpp" +#endif // COMPILER2 + +class MacroAssembler; + #ifdef COMPILER1 class LIR_Assembler; class LIR_OprDesc; @@ -32,18 +40,13 @@ class StubAssembler; class ZLoadBarrierStubC1; #endif // COMPILER1 -class ZBarrierSetAssembler : public ZBarrierSetAssemblerBase { - address _load_barrier_slow_stub[RegisterImpl::number_of_registers]; - address _load_barrier_weak_slow_stub[RegisterImpl::number_of_registers]; +#ifdef COMPILER2 +class Node; +class ZLoadBarrierStubC2; +#endif // COMPILER2 +class ZBarrierSetAssembler : public ZBarrierSetAssemblerBase { public: - ZBarrierSetAssembler() : - _load_barrier_slow_stub(), - _load_barrier_weak_slow_stub() {} - - address load_barrier_slow_stub(Register reg) { return _load_barrier_slow_stub[reg->encoding()]; } - address load_barrier_weak_slow_stub(Register reg) { return _load_barrier_weak_slow_stub[reg->encoding()]; } - virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, @@ -86,7 +89,12 @@ public: DecoratorSet decorators) const; #endif // COMPILER1 - virtual void barrier_stubs_init(); +#ifdef COMPILER2 + OptoReg::Name refine_register(const Node* node, + OptoReg::Name opto_reg); + void generate_c2_load_barrier_stub(MacroAssembler* masm, + ZLoadBarrierStubC2* stub) const; +#endif // COMPILER2 }; #endif // CPU_X86_GC_Z_ZBARRIERSETASSEMBLER_X86_HPP diff --git a/src/hotspot/cpu/x86/gc/z/z_x86_64.ad b/src/hotspot/cpu/x86/gc/z/z_x86_64.ad new file mode 100644 index 000000000..38c2e926b --- /dev/null +++ b/src/hotspot/cpu/x86/gc/z/z_x86_64.ad @@ -0,0 +1,168 @@ +// +// Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. +// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +// +// This code is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License version 2 only, as +// published by the Free Software Foundation. +// +// This code is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +// version 2 for more details (a copy is included in the LICENSE file that +// accompanied this code). +// +// You should have received a copy of the GNU General Public License version +// 2 along with this work; if not, write to the Free Software Foundation, +// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. +// +// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA +// or visit www.oracle.com if you need additional information or have any +// questions. +// + +source_hpp %{ + +#include "gc/z/c2/zBarrierSetC2.hpp" +#include "gc/z/zThreadLocalData.hpp" + +%} + +source %{ + +static void z_load_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp, bool weak) { + ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, weak); + __ testptr(ref, Address(r15_thread, ZThreadLocalData::address_bad_mask_offset())); + __ jcc(Assembler::notZero, *stub->entry()); + __ bind(*stub->continuation()); +} + +static void z_load_barrier_slow_path(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp) { + ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, false /* weak */); + __ jmp(*stub->entry()); + __ bind(*stub->continuation()); +} + +%} + +// Load Pointer +instruct zLoadP(rRegP dst, memory mem, rFlagsReg cr) +%{ + predicate(UseZGC && n->as_Load()->barrier_data() == ZLoadBarrierStrong); + match(Set dst (LoadP mem)); + effect(KILL cr, TEMP dst); + + ins_cost(125); + + format %{ "movq $dst, $mem" %} + + ins_encode %{ + __ movptr($dst$$Register, $mem$$Address); + if (barrier_data() != ZLoadBarrierElided) { + z_load_barrier(_masm, this, $mem$$Address, $dst$$Register, noreg /* tmp */, false /* weak */); + } + %} + + ins_pipe(ialu_reg_mem); +%} + +// Load Weak Pointer +instruct zLoadWeakP(rRegP dst, memory mem, rFlagsReg cr) +%{ + predicate(UseZGC && n->as_Load()->barrier_data() == ZLoadBarrierWeak); + match(Set dst (LoadP mem)); + effect(KILL cr, TEMP dst); + + ins_cost(125); + + format %{ "movq $dst, $mem" %} + + ins_encode %{ + __ movptr($dst$$Register, $mem$$Address); + z_load_barrier(_masm, this, $mem$$Address, $dst$$Register, noreg /* tmp */, true /* weak */); + %} + + ins_pipe(ialu_reg_mem); +%} + +instruct zCompareAndExchangeP(memory mem, rax_RegP oldval, rRegP newval, rRegP tmp, rFlagsReg cr) %{ + match(Set oldval (CompareAndExchangeP mem (Binary oldval newval))); + predicate(UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); + effect(KILL cr, TEMP tmp); + + format %{ "lock\n\t" + "cmpxchgq $newval, $mem" %} + + ins_encode %{ + if (barrier_data() != ZLoadBarrierElided) { + __ movptr($tmp$$Register, $oldval$$Register); + } + __ lock(); + __ cmpxchgptr($newval$$Register, $mem$$Address); + if (barrier_data() != ZLoadBarrierElided) { + Label good; + __ testptr($oldval$$Register, Address(r15_thread, ZThreadLocalData::address_bad_mask_offset())); + __ jcc(Assembler::zero, good); + z_load_barrier_slow_path(_masm, this, $mem$$Address, $oldval$$Register, $tmp$$Register); + __ movptr($oldval$$Register, $tmp$$Register); + __ lock(); + __ cmpxchgptr($newval$$Register, $mem$$Address); + __ bind(good); + } + %} + + ins_pipe(pipe_cmpxchg); +%} + +instruct zCompareAndSwapP(rRegI res, memory mem, rRegP newval, rRegP tmp, rFlagsReg cr, rax_RegP oldval) %{ + match(Set res (CompareAndSwapP mem (Binary oldval newval))); + match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); + predicate(UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); + effect(KILL cr, KILL oldval, TEMP tmp); + + format %{ "lock\n\t" + "cmpxchgq $newval, $mem\n\t" + "sete $res\n\t" + "movzbl $res, $res" %} + + ins_encode %{ + if (barrier_data() != ZLoadBarrierElided) { + __ movptr($tmp$$Register, $oldval$$Register); + } + __ lock(); + __ cmpxchgptr($newval$$Register, $mem$$Address); + if (barrier_data() != ZLoadBarrierElided) { + Label good; + __ testptr($oldval$$Register, Address(r15_thread, ZThreadLocalData::address_bad_mask_offset())); + __ jcc(Assembler::zero, good); + z_load_barrier_slow_path(_masm, this, $mem$$Address, $oldval$$Register, $tmp$$Register); + __ movptr($oldval$$Register, $tmp$$Register); + __ lock(); + __ cmpxchgptr($newval$$Register, $mem$$Address); + __ bind(good); + __ cmpptr($tmp$$Register, $oldval$$Register); + } + __ setb(Assembler::equal, $res$$Register); + __ movzbl($res$$Register, $res$$Register); + %} + + ins_pipe(pipe_cmpxchg); +%} + +instruct zXChgP(memory mem, rRegP newval, rFlagsReg cr) %{ + match(Set newval (GetAndSetP mem newval)); + predicate(UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); + effect(KILL cr); + + format %{ "xchgq $newval, $mem" %} + + ins_encode %{ + __ xchgptr($newval$$Register, $mem$$Address); + if (barrier_data() != ZLoadBarrierElided) { + z_load_barrier(_masm, this, Address(noreg, 0), $newval$$Register, noreg /* tmp */, false /* weak */); + } + %} + + ins_pipe(pipe_cmpxchg); +%} + diff --git a/src/hotspot/cpu/x86/x86.ad b/src/hotspot/cpu/x86/x86.ad index baa7cc774..2a3c91d2c 100644 --- a/src/hotspot/cpu/x86/x86.ad +++ b/src/hotspot/cpu/x86/x86.ad @@ -1097,138 +1097,6 @@ reg_class vectorz_reg_legacy(XMM0, XMM0b, XMM0c, XMM0d, XMM0e, XMM0f, XMM0 reg_class_dynamic vectorz_reg(vectorz_reg_evex, vectorz_reg_legacy, %{ VM_Version::supports_evex() %} ); reg_class_dynamic vectorz_reg_vl(vectorz_reg_evex, vectorz_reg_legacy, %{ VM_Version::supports_evex() && VM_Version::supports_avx512vl() %} ); -reg_class xmm0_reg(XMM0, XMM0b, XMM0c, XMM0d); -reg_class ymm0_reg(XMM0, XMM0b, XMM0c, XMM0d, XMM0e, XMM0f, XMM0g, XMM0h); -reg_class zmm0_reg(XMM0, XMM0b, XMM0c, XMM0d, XMM0e, XMM0f, XMM0g, XMM0h, XMM0i, XMM0j, XMM0k, XMM0l, XMM0m, XMM0n, XMM0o, XMM0p); - -reg_class xmm1_reg(XMM1, XMM1b, XMM1c, XMM1d); -reg_class ymm1_reg(XMM1, XMM1b, XMM1c, XMM1d, XMM1e, XMM1f, XMM1g, XMM1h); -reg_class zmm1_reg(XMM1, XMM1b, XMM1c, XMM1d, XMM1e, XMM1f, XMM1g, XMM1h, XMM1i, XMM1j, XMM1k, XMM1l, XMM1m, XMM1n, XMM1o, XMM1p); - -reg_class xmm2_reg(XMM2, XMM2b, XMM2c, XMM2d); -reg_class ymm2_reg(XMM2, XMM2b, XMM2c, XMM2d, XMM2e, XMM2f, XMM2g, XMM2h); -reg_class zmm2_reg(XMM2, XMM2b, XMM2c, XMM2d, XMM2e, XMM2f, XMM2g, XMM2h, XMM2i, XMM2j, XMM2k, XMM2l, XMM2m, XMM2n, XMM2o, XMM2p); - -reg_class xmm3_reg(XMM3, XMM3b, XMM3c, XMM3d); -reg_class ymm3_reg(XMM3, XMM3b, XMM3c, XMM3d, XMM3e, XMM3f, XMM3g, XMM3h); -reg_class zmm3_reg(XMM3, XMM3b, XMM3c, XMM3d, XMM3e, XMM3f, XMM3g, XMM3h, XMM3i, XMM3j, XMM3k, XMM3l, XMM3m, XMM3n, XMM3o, XMM3p); - -reg_class xmm4_reg(XMM4, XMM4b, XMM4c, XMM4d); -reg_class ymm4_reg(XMM4, XMM4b, XMM4c, XMM4d, XMM4e, XMM4f, XMM4g, XMM4h); -reg_class zmm4_reg(XMM4, XMM4b, XMM4c, XMM4d, XMM4e, XMM4f, XMM4g, XMM4h, XMM4i, XMM4j, XMM4k, XMM4l, XMM4m, XMM4n, XMM4o, XMM4p); - -reg_class xmm5_reg(XMM5, XMM5b, XMM5c, XMM5d); -reg_class ymm5_reg(XMM5, XMM5b, XMM5c, XMM5d, XMM5e, XMM5f, XMM5g, XMM5h); -reg_class zmm5_reg(XMM5, XMM5b, XMM5c, XMM5d, XMM5e, XMM5f, XMM5g, XMM5h, XMM5i, XMM5j, XMM5k, XMM5l, XMM5m, XMM5n, XMM5o, XMM5p); - -reg_class xmm6_reg(XMM6, XMM6b, XMM6c, XMM6d); -reg_class ymm6_reg(XMM6, XMM6b, XMM6c, XMM6d, XMM6e, XMM6f, XMM6g, XMM6h); -reg_class zmm6_reg(XMM6, XMM6b, XMM6c, XMM6d, XMM6e, XMM6f, XMM6g, XMM6h, XMM6i, XMM6j, XMM6k, XMM6l, XMM6m, XMM6n, XMM6o, XMM6p); - -reg_class xmm7_reg(XMM7, XMM7b, XMM7c, XMM7d); -reg_class ymm7_reg(XMM7, XMM7b, XMM7c, XMM7d, XMM7e, XMM7f, XMM7g, XMM7h); -reg_class zmm7_reg(XMM7, XMM7b, XMM7c, XMM7d, XMM7e, XMM7f, XMM7g, XMM7h, XMM7i, XMM7j, XMM7k, XMM7l, XMM7m, XMM7n, XMM7o, XMM7p); - -#ifdef _LP64 - -reg_class xmm8_reg(XMM8, XMM8b, XMM8c, XMM8d); -reg_class ymm8_reg(XMM8, XMM8b, XMM8c, XMM8d, XMM8e, XMM8f, XMM8g, XMM8h); -reg_class zmm8_reg(XMM8, XMM8b, XMM8c, XMM8d, XMM8e, XMM8f, XMM8g, XMM8h, XMM8i, XMM8j, XMM8k, XMM8l, XMM8m, XMM8n, XMM8o, XMM8p); - -reg_class xmm9_reg(XMM9, XMM9b, XMM9c, XMM9d); -reg_class ymm9_reg(XMM9, XMM9b, XMM9c, XMM9d, XMM9e, XMM9f, XMM9g, XMM9h); -reg_class zmm9_reg(XMM9, XMM9b, XMM9c, XMM9d, XMM9e, XMM9f, XMM9g, XMM9h, XMM9i, XMM9j, XMM9k, XMM9l, XMM9m, XMM9n, XMM9o, XMM9p); - -reg_class xmm10_reg(XMM10, XMM10b, XMM10c, XMM10d); -reg_class ymm10_reg(XMM10, XMM10b, XMM10c, XMM10d, XMM10e, XMM10f, XMM10g, XMM10h); -reg_class zmm10_reg(XMM10, XMM10b, XMM10c, XMM10d, XMM10e, XMM10f, XMM10g, XMM10h, XMM10i, XMM10j, XMM10k, XMM10l, XMM10m, XMM10n, XMM10o, XMM10p); - -reg_class xmm11_reg(XMM11, XMM11b, XMM11c, XMM11d); -reg_class ymm11_reg(XMM11, XMM11b, XMM11c, XMM11d, XMM11e, XMM11f, XMM11g, XMM11h); -reg_class zmm11_reg(XMM11, XMM11b, XMM11c, XMM11d, XMM11e, XMM11f, XMM11g, XMM11h, XMM11i, XMM11j, XMM11k, XMM11l, XMM11m, XMM11n, XMM11o, XMM11p); - -reg_class xmm12_reg(XMM12, XMM12b, XMM12c, XMM12d); -reg_class ymm12_reg(XMM12, XMM12b, XMM12c, XMM12d, XMM12e, XMM12f, XMM12g, XMM12h); -reg_class zmm12_reg(XMM12, XMM12b, XMM12c, XMM12d, XMM12e, XMM12f, XMM12g, XMM12h, XMM12i, XMM12j, XMM12k, XMM12l, XMM12m, XMM12n, XMM12o, XMM12p); - -reg_class xmm13_reg(XMM13, XMM13b, XMM13c, XMM13d); -reg_class ymm13_reg(XMM13, XMM13b, XMM13c, XMM13d, XMM13e, XMM13f, XMM13g, XMM13h); -reg_class zmm13_reg(XMM13, XMM13b, XMM13c, XMM13d, XMM13e, XMM13f, XMM13g, XMM13h, XMM13i, XMM13j, XMM13k, XMM13l, XMM13m, XMM13n, XMM13o, XMM13p); - -reg_class xmm14_reg(XMM14, XMM14b, XMM14c, XMM14d); -reg_class ymm14_reg(XMM14, XMM14b, XMM14c, XMM14d, XMM14e, XMM14f, XMM14g, XMM14h); -reg_class zmm14_reg(XMM14, XMM14b, XMM14c, XMM14d, XMM14e, XMM14f, XMM14g, XMM14h, XMM14i, XMM14j, XMM14k, XMM14l, XMM14m, XMM14n, XMM14o, XMM14p); - -reg_class xmm15_reg(XMM15, XMM15b, XMM15c, XMM15d); -reg_class ymm15_reg(XMM15, XMM15b, XMM15c, XMM15d, XMM15e, XMM15f, XMM15g, XMM15h); -reg_class zmm15_reg(XMM15, XMM15b, XMM15c, XMM15d, XMM15e, XMM15f, XMM15g, XMM15h, XMM15i, XMM15j, XMM15k, XMM15l, XMM15m, XMM15n, XMM15o, XMM15p); - -reg_class xmm16_reg(XMM16, XMM16b, XMM16c, XMM16d); -reg_class ymm16_reg(XMM16, XMM16b, XMM16c, XMM16d, XMM16e, XMM16f, XMM16g, XMM16h); -reg_class zmm16_reg(XMM16, XMM16b, XMM16c, XMM16d, XMM16e, XMM16f, XMM16g, XMM16h, XMM16i, XMM16j, XMM16k, XMM16l, XMM16m, XMM16n, XMM16o, XMM16p); - -reg_class xmm17_reg(XMM17, XMM17b, XMM17c, XMM17d); -reg_class ymm17_reg(XMM17, XMM17b, XMM17c, XMM17d, XMM17e, XMM17f, XMM17g, XMM17h); -reg_class zmm17_reg(XMM17, XMM17b, XMM17c, XMM17d, XMM17e, XMM17f, XMM17g, XMM17h, XMM17i, XMM17j, XMM17k, XMM17l, XMM17m, XMM17n, XMM17o, XMM17p); - -reg_class xmm18_reg(XMM18, XMM18b, XMM18c, XMM18d); -reg_class ymm18_reg(XMM18, XMM18b, XMM18c, XMM18d, XMM18e, XMM18f, XMM18g, XMM18h); -reg_class zmm18_reg(XMM18, XMM18b, XMM18c, XMM18d, XMM18e, XMM18f, XMM18g, XMM18h, XMM18i, XMM18j, XMM18k, XMM18l, XMM18m, XMM18n, XMM18o, XMM18p); - -reg_class xmm19_reg(XMM19, XMM19b, XMM19c, XMM19d); -reg_class ymm19_reg(XMM19, XMM19b, XMM19c, XMM19d, XMM19e, XMM19f, XMM19g, XMM19h); -reg_class zmm19_reg(XMM19, XMM19b, XMM19c, XMM19d, XMM19e, XMM19f, XMM19g, XMM19h, XMM19i, XMM19j, XMM19k, XMM19l, XMM19m, XMM19n, XMM19o, XMM19p); - -reg_class xmm20_reg(XMM20, XMM20b, XMM20c, XMM20d); -reg_class ymm20_reg(XMM20, XMM20b, XMM20c, XMM20d, XMM20e, XMM20f, XMM20g, XMM20h); -reg_class zmm20_reg(XMM20, XMM20b, XMM20c, XMM20d, XMM20e, XMM20f, XMM20g, XMM20h, XMM20i, XMM20j, XMM20k, XMM20l, XMM20m, XMM20n, XMM20o, XMM20p); - -reg_class xmm21_reg(XMM21, XMM21b, XMM21c, XMM21d); -reg_class ymm21_reg(XMM21, XMM21b, XMM21c, XMM21d, XMM21e, XMM21f, XMM21g, XMM21h); -reg_class zmm21_reg(XMM21, XMM21b, XMM21c, XMM21d, XMM21e, XMM21f, XMM21g, XMM21h, XMM21i, XMM21j, XMM21k, XMM21l, XMM21m, XMM21n, XMM21o, XMM21p); - -reg_class xmm22_reg(XMM22, XMM22b, XMM22c, XMM22d); -reg_class ymm22_reg(XMM22, XMM22b, XMM22c, XMM22d, XMM22e, XMM22f, XMM22g, XMM22h); -reg_class zmm22_reg(XMM22, XMM22b, XMM22c, XMM22d, XMM22e, XMM22f, XMM22g, XMM22h, XMM22i, XMM22j, XMM22k, XMM22l, XMM22m, XMM22n, XMM22o, XMM22p); - -reg_class xmm23_reg(XMM23, XMM23b, XMM23c, XMM23d); -reg_class ymm23_reg(XMM23, XMM23b, XMM23c, XMM23d, XMM23e, XMM23f, XMM23g, XMM23h); -reg_class zmm23_reg(XMM23, XMM23b, XMM23c, XMM23d, XMM23e, XMM23f, XMM23g, XMM23h, XMM23i, XMM23j, XMM23k, XMM23l, XMM23m, XMM23n, XMM23o, XMM23p); - -reg_class xmm24_reg(XMM24, XMM24b, XMM24c, XMM24d); -reg_class ymm24_reg(XMM24, XMM24b, XMM24c, XMM24d, XMM24e, XMM24f, XMM24g, XMM24h); -reg_class zmm24_reg(XMM24, XMM24b, XMM24c, XMM24d, XMM24e, XMM24f, XMM24g, XMM24h, XMM24i, XMM24j, XMM24k, XMM24l, XMM24m, XMM24n, XMM24o, XMM24p); - -reg_class xmm25_reg(XMM25, XMM25b, XMM25c, XMM25d); -reg_class ymm25_reg(XMM25, XMM25b, XMM25c, XMM25d, XMM25e, XMM25f, XMM25g, XMM25h); -reg_class zmm25_reg(XMM25, XMM25b, XMM25c, XMM25d, XMM25e, XMM25f, XMM25g, XMM25h, XMM25i, XMM25j, XMM25k, XMM25l, XMM25m, XMM25n, XMM25o, XMM25p); - -reg_class xmm26_reg(XMM26, XMM26b, XMM26c, XMM26d); -reg_class ymm26_reg(XMM26, XMM26b, XMM26c, XMM26d, XMM26e, XMM26f, XMM26g, XMM26h); -reg_class zmm26_reg(XMM26, XMM26b, XMM26c, XMM26d, XMM26e, XMM26f, XMM26g, XMM26h, XMM26i, XMM26j, XMM26k, XMM26l, XMM26m, XMM26n, XMM26o, XMM26p); - -reg_class xmm27_reg(XMM27, XMM27b, XMM27c, XMM27d); -reg_class ymm27_reg(XMM27, XMM27b, XMM27c, XMM27d, XMM27e, XMM27f, XMM27g, XMM27h); -reg_class zmm27_reg(XMM27, XMM27b, XMM27c, XMM27d, XMM27e, XMM27f, XMM27g, XMM27h, XMM27i, XMM27j, XMM27k, XMM27l, XMM27m, XMM27n, XMM27o, XMM27p); - -reg_class xmm28_reg(XMM28, XMM28b, XMM28c, XMM28d); -reg_class ymm28_reg(XMM28, XMM28b, XMM28c, XMM28d, XMM28e, XMM28f, XMM28g, XMM28h); -reg_class zmm28_reg(XMM28, XMM28b, XMM28c, XMM28d, XMM28e, XMM28f, XMM28g, XMM28h, XMM28i, XMM28j, XMM28k, XMM28l, XMM28m, XMM28n, XMM28o, XMM28p); - -reg_class xmm29_reg(XMM29, XMM29b, XMM29c, XMM29d); -reg_class ymm29_reg(XMM29, XMM29b, XMM29c, XMM29d, XMM29e, XMM29f, XMM29g, XMM29h); -reg_class zmm29_reg(XMM29, XMM29b, XMM29c, XMM29d, XMM29e, XMM29f, XMM29g, XMM29h, XMM29i, XMM29j, XMM29k, XMM29l, XMM29m, XMM29n, XMM29o, XMM29p); - -reg_class xmm30_reg(XMM30, XMM30b, XMM30c, XMM30d); -reg_class ymm30_reg(XMM30, XMM30b, XMM30c, XMM30d, XMM30e, XMM30f, XMM30g, XMM30h); -reg_class zmm30_reg(XMM30, XMM30b, XMM30c, XMM30d, XMM30e, XMM30f, XMM30g, XMM30h, XMM30i, XMM30j, XMM30k, XMM30l, XMM30m, XMM30n, XMM30o, XMM30p); - -reg_class xmm31_reg(XMM31, XMM31b, XMM31c, XMM31d); -reg_class ymm31_reg(XMM31, XMM31b, XMM31c, XMM31d, XMM31e, XMM31f, XMM31g, XMM31h); -reg_class zmm31_reg(XMM31, XMM31b, XMM31c, XMM31d, XMM31e, XMM31f, XMM31g, XMM31h, XMM31i, XMM31j, XMM31k, XMM31l, XMM31m, XMM31n, XMM31o, XMM31p); - -#endif - %} @@ -1817,8 +1685,8 @@ static int vec_mov_helper(CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo return (UseAVX > 2) ? 6 : 4; } -static int vec_spill_helper(CodeBuffer *cbuf, bool do_size, bool is_load, - int stack_offset, int reg, uint ireg, outputStream* st) { +int vec_spill_helper(CodeBuffer *cbuf, bool do_size, bool is_load, + int stack_offset, int reg, uint ireg, outputStream* st) { // In 64-bit VM size calculation is very complex. Emitting instructions // into scratch buffer is used to get size in 64-bit VM. LP64_ONLY( assert(!do_size, "this method calculates size only for 32-bit VM"); ) diff --git a/src/hotspot/cpu/x86/x86_64.ad b/src/hotspot/cpu/x86/x86_64.ad index 4607d1600..f8903c655 100644 --- a/src/hotspot/cpu/x86/x86_64.ad +++ b/src/hotspot/cpu/x86/x86_64.ad @@ -539,18 +539,6 @@ reg_class int_rdi_reg(RDI); %} -source_hpp %{ - -#include "gc/z/c2/zBarrierSetC2.hpp" - -%} - -source_hpp %{ -#if INCLUDE_ZGC -#include "gc/z/zBarrierSetAssembler.hpp" -#endif -%} - //----------SOURCE BLOCK------------------------------------------------------- // This is a block of C++ code which provides values, functions, and // definitions necessary in the rest of the architecture description @@ -1170,8 +1158,8 @@ static enum RC rc_class(OptoReg::Name reg) static int vec_mov_helper(CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo, int src_hi, int dst_hi, uint ireg, outputStream* st); -static int vec_spill_helper(CodeBuffer *cbuf, bool do_size, bool is_load, - int stack_offset, int reg, uint ireg, outputStream* st); +int vec_spill_helper(CodeBuffer *cbuf, bool do_size, bool is_load, + int stack_offset, int reg, uint ireg, outputStream* st); static void vec_stack_to_stack_helper(CodeBuffer *cbuf, int src_offset, int dst_offset, uint ireg, outputStream* st) { @@ -4384,136 +4372,6 @@ operand cmpOpUCF2() %{ %} %} -// Operands for bound floating pointer register arguments -operand rxmm0() %{ - constraint(ALLOC_IN_RC(xmm0_reg)); match(VecX); - predicate((UseSSE > 0) && (UseAVX<= 2)); format%{%} interface(REG_INTER); -%} -operand rxmm1() %{ - constraint(ALLOC_IN_RC(xmm1_reg)); match(VecX); - predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER); -%} -operand rxmm2() %{ - constraint(ALLOC_IN_RC(xmm2_reg)); match(VecX); - predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER); -%} -operand rxmm3() %{ - constraint(ALLOC_IN_RC(xmm3_reg)); match(VecX); - predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER); -%} -operand rxmm4() %{ - constraint(ALLOC_IN_RC(xmm4_reg)); match(VecX); - predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER); -%} -operand rxmm5() %{ - constraint(ALLOC_IN_RC(xmm5_reg)); match(VecX); - predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER); -%} -operand rxmm6() %{ - constraint(ALLOC_IN_RC(xmm6_reg)); match(VecX); - predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER); -%} -operand rxmm7() %{ - constraint(ALLOC_IN_RC(xmm7_reg)); match(VecX); - predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER); -%} -operand rxmm8() %{ - constraint(ALLOC_IN_RC(xmm8_reg)); match(VecX); - predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER); -%} -operand rxmm9() %{ - constraint(ALLOC_IN_RC(xmm9_reg)); match(VecX); - predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER); -%} -operand rxmm10() %{ - constraint(ALLOC_IN_RC(xmm10_reg)); match(VecX); - predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER); -%} -operand rxmm11() %{ - constraint(ALLOC_IN_RC(xmm11_reg)); match(VecX); - predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER); -%} -operand rxmm12() %{ - constraint(ALLOC_IN_RC(xmm12_reg)); match(VecX); - predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER); -%} -operand rxmm13() %{ - constraint(ALLOC_IN_RC(xmm13_reg)); match(VecX); - predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER); -%} -operand rxmm14() %{ - constraint(ALLOC_IN_RC(xmm14_reg)); match(VecX); - predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER); -%} -operand rxmm15() %{ - constraint(ALLOC_IN_RC(xmm15_reg)); match(VecX); - predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER); -%} -operand rxmm16() %{ - constraint(ALLOC_IN_RC(xmm16_reg)); match(VecX); - predicate(UseAVX == 3); format%{%} interface(REG_INTER); -%} -operand rxmm17() %{ - constraint(ALLOC_IN_RC(xmm17_reg)); match(VecX); - predicate(UseAVX == 3); format%{%} interface(REG_INTER); -%} -operand rxmm18() %{ - constraint(ALLOC_IN_RC(xmm18_reg)); match(VecX); - predicate(UseAVX == 3); format%{%} interface(REG_INTER); -%} -operand rxmm19() %{ - constraint(ALLOC_IN_RC(xmm19_reg)); match(VecX); - predicate(UseAVX == 3); format%{%} interface(REG_INTER); -%} -operand rxmm20() %{ - constraint(ALLOC_IN_RC(xmm20_reg)); match(VecX); - predicate(UseAVX == 3); format%{%} interface(REG_INTER); -%} -operand rxmm21() %{ - constraint(ALLOC_IN_RC(xmm21_reg)); match(VecX); - predicate(UseAVX == 3); format%{%} interface(REG_INTER); -%} -operand rxmm22() %{ - constraint(ALLOC_IN_RC(xmm22_reg)); match(VecX); - predicate(UseAVX == 3); format%{%} interface(REG_INTER); -%} -operand rxmm23() %{ - constraint(ALLOC_IN_RC(xmm23_reg)); match(VecX); - predicate(UseAVX == 3); format%{%} interface(REG_INTER); -%} -operand rxmm24() %{ - constraint(ALLOC_IN_RC(xmm24_reg)); match(VecX); - predicate(UseAVX == 3); format%{%} interface(REG_INTER); -%} -operand rxmm25() %{ - constraint(ALLOC_IN_RC(xmm25_reg)); match(VecX); - predicate(UseAVX == 3); format%{%} interface(REG_INTER); -%} -operand rxmm26() %{ - constraint(ALLOC_IN_RC(xmm26_reg)); match(VecX); - predicate(UseAVX == 3); format%{%} interface(REG_INTER); -%} -operand rxmm27() %{ - constraint(ALLOC_IN_RC(xmm27_reg)); match(VecX); - predicate(UseAVX == 3); format%{%} interface(REG_INTER); -%} -operand rxmm28() %{ - constraint(ALLOC_IN_RC(xmm28_reg)); match(VecX); - predicate(UseAVX == 3); format%{%} interface(REG_INTER); -%} -operand rxmm29() %{ - constraint(ALLOC_IN_RC(xmm29_reg)); match(VecX); - predicate(UseAVX == 3); format%{%} interface(REG_INTER); -%} -operand rxmm30() %{ - constraint(ALLOC_IN_RC(xmm30_reg)); match(VecX); - predicate(UseAVX == 3); format%{%} interface(REG_INTER); -%} -operand rxmm31() %{ - constraint(ALLOC_IN_RC(xmm31_reg)); match(VecX); - predicate(UseAVX == 3); format%{%} interface(REG_INTER); -%} - //----------OPERAND CLASSES---------------------------------------------------- // Operand Classes are groups of operands that are used as to simplify // instruction definitions by not requiring the AD writer to specify separate @@ -5406,6 +5264,7 @@ instruct loadRange(rRegI dst, memory mem) instruct loadP(rRegP dst, memory mem) %{ match(Set dst (LoadP mem)); + predicate(n->as_Load()->barrier_data() == 0); ins_cost(125); // XXX format %{ "movq $dst, $mem\t# ptr" %} @@ -7806,6 +7665,7 @@ instruct storePConditional(memory heap_top_ptr, rax_RegP oldval, rRegP newval, rFlagsReg cr) %{ + predicate(n->as_LoadStore()->barrier_data() == 0); match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval))); format %{ "cmpxchgq $heap_top_ptr, $newval\t# (ptr) " @@ -7857,7 +7717,7 @@ instruct compareAndSwapP(rRegI res, rax_RegP oldval, rRegP newval, rFlagsReg cr) %{ - predicate(VM_Version::supports_cx8()); + predicate(VM_Version::supports_cx8() && n->as_LoadStore()->barrier_data() == 0); match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval))); match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval))); effect(KILL cr, KILL oldval); @@ -8099,7 +7959,7 @@ instruct compareAndExchangeP( rax_RegP oldval, rRegP newval, rFlagsReg cr) %{ - predicate(VM_Version::supports_cx8()); + predicate(VM_Version::supports_cx8() && n->as_LoadStore()->barrier_data() == 0); match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval))); effect(KILL cr); @@ -8244,6 +8104,7 @@ instruct xchgL( memory mem, rRegL newval) %{ instruct xchgP( memory mem, rRegP newval) %{ match(Set newval (GetAndSetP mem newval)); + predicate(n->as_LoadStore()->barrier_data() == 0); format %{ "XCHGQ $newval,[$mem]" %} ins_encode %{ __ xchgq($newval$$Register, $mem$$Address); @@ -11940,6 +11801,7 @@ instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2) instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2) %{ match(Set cr (CmpP op1 (LoadP op2))); + predicate(n->in(2)->as_Load()->barrier_data() == 0); ins_cost(500); // XXX format %{ "cmpq $op1, $op2\t# ptr" %} @@ -11965,7 +11827,8 @@ instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2) // and raw pointers have no anti-dependencies. instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2) %{ - predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none); + predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none && + n->in(2)->as_Load()->barrier_data() == 0); match(Set cr (CmpP op1 (LoadP op2))); format %{ "cmpq $op1, $op2\t# raw ptr" %} @@ -11990,7 +11853,8 @@ instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero) // any compare to a zero should be eq/neq. instruct testP_mem(rFlagsReg cr, memory op, immP0 zero) %{ - predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL)); + predicate((!UseCompressedOops || (Universe::narrow_oop_base() != NULL)) && + n->in(1)->as_Load()->barrier_data() == 0); match(Set cr (CmpP (LoadP op) zero)); ins_cost(500); // XXX @@ -12003,7 +11867,9 @@ instruct testP_mem(rFlagsReg cr, memory op, immP0 zero) instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero) %{ - predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL)); + predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && + (Universe::narrow_klass_base() == NULL) && + n->in(1)->as_Load()->barrier_data() == 0); match(Set cr (CmpP (LoadP mem) zero)); format %{ "cmpq R12, $mem\t# ptr (R12_heapbase==0)" %} @@ -12954,279 +12820,6 @@ instruct RethrowException() ins_pipe(pipe_jmp); %} -// -// Execute ZGC load barrier (strong) slow path -// - -// When running without XMM regs -instruct loadBarrierSlowRegNoVec(rRegP dst, memory mem, rFlagsReg cr) %{ - - match(Set dst (LoadBarrierSlowReg mem)); - predicate(MaxVectorSize < 16 && !n->as_LoadBarrierSlowReg()->is_weak()); - - effect(DEF dst, KILL cr); - - format %{"LoadBarrierSlowRegNoVec $dst, $mem" %} - ins_encode %{ -#if INCLUDE_ZGC - Register d = $dst$$Register; - ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler(); - - assert(d != r12, "Can't be R12!"); - assert(d != r15, "Can't be R15!"); - assert(d != rsp, "Can't be RSP!"); - - __ lea(d, $mem$$Address); - __ call(RuntimeAddress(bs->load_barrier_slow_stub(d))); -#else - ShouldNotReachHere(); -#endif - %} - ins_pipe(pipe_slow); -%} - -// For XMM and YMM enabled processors -instruct loadBarrierSlowRegXmmAndYmm(rRegP dst, memory mem, rFlagsReg cr, - rxmm0 x0, rxmm1 x1, rxmm2 x2,rxmm3 x3, - rxmm4 x4, rxmm5 x5, rxmm6 x6, rxmm7 x7, - rxmm8 x8, rxmm9 x9, rxmm10 x10, rxmm11 x11, - rxmm12 x12, rxmm13 x13, rxmm14 x14, rxmm15 x15) %{ - - match(Set dst (LoadBarrierSlowReg mem)); - predicate((UseSSE > 0) && (UseAVX <= 2) && (MaxVectorSize >= 16) && !n->as_LoadBarrierSlowReg()->is_weak()); - - effect(DEF dst, KILL cr, - KILL x0, KILL x1, KILL x2, KILL x3, - KILL x4, KILL x5, KILL x6, KILL x7, - KILL x8, KILL x9, KILL x10, KILL x11, - KILL x12, KILL x13, KILL x14, KILL x15); - - format %{"LoadBarrierSlowRegXmm $dst, $mem" %} - ins_encode %{ -#if INCLUDE_ZGC - Register d = $dst$$Register; - ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler(); - - assert(d != r12, "Can't be R12!"); - assert(d != r15, "Can't be R15!"); - assert(d != rsp, "Can't be RSP!"); - - __ lea(d, $mem$$Address); - __ call(RuntimeAddress(bs->load_barrier_slow_stub(d))); -#else - ShouldNotReachHere(); -#endif - %} - ins_pipe(pipe_slow); -%} - -// For ZMM enabled processors -instruct loadBarrierSlowRegZmm(rRegP dst, memory mem, rFlagsReg cr, - rxmm0 x0, rxmm1 x1, rxmm2 x2,rxmm3 x3, - rxmm4 x4, rxmm5 x5, rxmm6 x6, rxmm7 x7, - rxmm8 x8, rxmm9 x9, rxmm10 x10, rxmm11 x11, - rxmm12 x12, rxmm13 x13, rxmm14 x14, rxmm15 x15, - rxmm16 x16, rxmm17 x17, rxmm18 x18, rxmm19 x19, - rxmm20 x20, rxmm21 x21, rxmm22 x22, rxmm23 x23, - rxmm24 x24, rxmm25 x25, rxmm26 x26, rxmm27 x27, - rxmm28 x28, rxmm29 x29, rxmm30 x30, rxmm31 x31) %{ - - match(Set dst (LoadBarrierSlowReg mem)); - predicate((UseAVX == 3) && (MaxVectorSize >= 16) && !n->as_LoadBarrierSlowReg()->is_weak()); - - effect(DEF dst, KILL cr, - KILL x0, KILL x1, KILL x2, KILL x3, - KILL x4, KILL x5, KILL x6, KILL x7, - KILL x8, KILL x9, KILL x10, KILL x11, - KILL x12, KILL x13, KILL x14, KILL x15, - KILL x16, KILL x17, KILL x18, KILL x19, - KILL x20, KILL x21, KILL x22, KILL x23, - KILL x24, KILL x25, KILL x26, KILL x27, - KILL x28, KILL x29, KILL x30, KILL x31); - - format %{"LoadBarrierSlowRegZmm $dst, $mem" %} - ins_encode %{ -#if INCLUDE_ZGC - Register d = $dst$$Register; - ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler(); - - assert(d != r12, "Can't be R12!"); - assert(d != r15, "Can't be R15!"); - assert(d != rsp, "Can't be RSP!"); - - __ lea(d, $mem$$Address); - __ call(RuntimeAddress(bs->load_barrier_slow_stub(d))); -#else - ShouldNotReachHere(); -#endif - %} - ins_pipe(pipe_slow); -%} - -// -// Execute ZGC load barrier (weak) slow path -// - -// When running without XMM regs -instruct loadBarrierWeakSlowRegNoVec(rRegP dst, memory mem, rFlagsReg cr) %{ - - match(Set dst (LoadBarrierSlowReg mem)); - predicate(MaxVectorSize < 16 && n->as_LoadBarrierSlowReg()->is_weak()); - - effect(DEF dst, KILL cr); - - format %{"LoadBarrierSlowRegNoVec $dst, $mem" %} - ins_encode %{ -#if INCLUDE_ZGC - Register d = $dst$$Register; - ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler(); - - assert(d != r12, "Can't be R12!"); - assert(d != r15, "Can't be R15!"); - assert(d != rsp, "Can't be RSP!"); - - __ lea(d, $mem$$Address); - __ call(RuntimeAddress(bs->load_barrier_weak_slow_stub(d))); -#else - ShouldNotReachHere(); -#endif - %} - ins_pipe(pipe_slow); -%} - -// For XMM and YMM enabled processors -instruct loadBarrierWeakSlowRegXmmAndYmm(rRegP dst, memory mem, rFlagsReg cr, - rxmm0 x0, rxmm1 x1, rxmm2 x2,rxmm3 x3, - rxmm4 x4, rxmm5 x5, rxmm6 x6, rxmm7 x7, - rxmm8 x8, rxmm9 x9, rxmm10 x10, rxmm11 x11, - rxmm12 x12, rxmm13 x13, rxmm14 x14, rxmm15 x15) %{ - - match(Set dst (LoadBarrierSlowReg mem)); - predicate((UseSSE > 0) && (UseAVX <= 2) && (MaxVectorSize >= 16) && n->as_LoadBarrierSlowReg()->is_weak()); - - effect(DEF dst, KILL cr, - KILL x0, KILL x1, KILL x2, KILL x3, - KILL x4, KILL x5, KILL x6, KILL x7, - KILL x8, KILL x9, KILL x10, KILL x11, - KILL x12, KILL x13, KILL x14, KILL x15); - - format %{"LoadBarrierWeakSlowRegXmm $dst, $mem" %} - ins_encode %{ -#if INCLUDE_ZGC - Register d = $dst$$Register; - ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler(); - - assert(d != r12, "Can't be R12!"); - assert(d != r15, "Can't be R15!"); - assert(d != rsp, "Can't be RSP!"); - - __ lea(d,$mem$$Address); - __ call(RuntimeAddress(bs->load_barrier_weak_slow_stub(d))); -#else - ShouldNotReachHere(); -#endif - %} - ins_pipe(pipe_slow); -%} - -// For ZMM enabled processors -instruct loadBarrierWeakSlowRegZmm(rRegP dst, memory mem, rFlagsReg cr, - rxmm0 x0, rxmm1 x1, rxmm2 x2,rxmm3 x3, - rxmm4 x4, rxmm5 x5, rxmm6 x6, rxmm7 x7, - rxmm8 x8, rxmm9 x9, rxmm10 x10, rxmm11 x11, - rxmm12 x12, rxmm13 x13, rxmm14 x14, rxmm15 x15, - rxmm16 x16, rxmm17 x17, rxmm18 x18, rxmm19 x19, - rxmm20 x20, rxmm21 x21, rxmm22 x22, rxmm23 x23, - rxmm24 x24, rxmm25 x25, rxmm26 x26, rxmm27 x27, - rxmm28 x28, rxmm29 x29, rxmm30 x30, rxmm31 x31) %{ - - match(Set dst (LoadBarrierSlowReg mem)); - predicate((UseAVX == 3) && (MaxVectorSize >= 16) && n->as_LoadBarrierSlowReg()->is_weak()); - - effect(DEF dst, KILL cr, - KILL x0, KILL x1, KILL x2, KILL x3, - KILL x4, KILL x5, KILL x6, KILL x7, - KILL x8, KILL x9, KILL x10, KILL x11, - KILL x12, KILL x13, KILL x14, KILL x15, - KILL x16, KILL x17, KILL x18, KILL x19, - KILL x20, KILL x21, KILL x22, KILL x23, - KILL x24, KILL x25, KILL x26, KILL x27, - KILL x28, KILL x29, KILL x30, KILL x31); - - format %{"LoadBarrierWeakSlowRegZmm $dst, $mem" %} - ins_encode %{ -#if INCLUDE_ZGC - Register d = $dst$$Register; - ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler(); - - assert(d != r12, "Can't be R12!"); - assert(d != r15, "Can't be R15!"); - assert(d != rsp, "Can't be RSP!"); - - __ lea(d,$mem$$Address); - __ call(RuntimeAddress(bs->load_barrier_weak_slow_stub(d))); -#else - ShouldNotReachHere(); -#endif - %} - ins_pipe(pipe_slow); -%} - -// Specialized versions of compareAndExchangeP that adds a keepalive that is consumed -// but doesn't affect output. - -instruct z_compareAndExchangeP( - memory mem_ptr, - rax_RegP oldval, rRegP newval, rRegP keepalive, - rFlagsReg cr) %{ - predicate(VM_Version::supports_cx8()); - match(Set oldval (ZCompareAndExchangeP (Binary mem_ptr keepalive) (Binary oldval newval))); - effect(KILL cr); - - format %{ "cmpxchgq $mem_ptr,$newval\t# " - "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %} - opcode(0x0F, 0xB1); - ins_encode(lock_prefix, - REX_reg_mem_wide(newval, mem_ptr), - OpcP, OpcS, - reg_mem(newval, mem_ptr) // lock cmpxchg - ); - ins_pipe( pipe_cmpxchg ); -%} - -instruct z_compareAndSwapP(rRegI res, - memory mem_ptr, - rax_RegP oldval, rRegP newval, rRegP keepalive, - rFlagsReg cr) %{ - predicate(VM_Version::supports_cx8()); - match(Set res (ZCompareAndSwapP (Binary mem_ptr keepalive) (Binary oldval newval))); - match(Set res (ZWeakCompareAndSwapP (Binary mem_ptr keepalive) (Binary oldval newval))); - effect(KILL cr, KILL oldval); - - format %{ "cmpxchgq $mem_ptr,$newval\t# " - "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" - "sete $res\n\t" - "movzbl $res, $res" %} - opcode(0x0F, 0xB1); - ins_encode(lock_prefix, - REX_reg_mem_wide(newval, mem_ptr), - OpcP, OpcS, - reg_mem(newval, mem_ptr), - REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete - REX_reg_breg(res, res), // movzbl - Opcode(0xF), Opcode(0xB6), reg_reg(res, res)); - ins_pipe( pipe_cmpxchg ); -%} - -instruct z_xchgP( memory mem, rRegP newval, rRegP keepalive) %{ - match(Set newval (ZGetAndSetP mem (Binary newval keepalive))); - format %{ "XCHGQ $newval,[$mem]" %} - ins_encode %{ - __ xchgq($newval$$Register, $mem$$Address); - %} - ins_pipe( pipe_cmpxchg ); -%} - // ============================================================================ // This name is KNOWN by the ADLC and cannot be changed. // The ADLC forces a 'TypeRawPtr::BOTTOM' output type diff --git a/src/hotspot/os_cpu/linux_aarch64/gc/z/zBackingFile_linux_aarch64.cpp b/src/hotspot/os_cpu/linux_aarch64/gc/z/zBackingFile_linux_aarch64.cpp index 47894b5c8..f956b53d6 100644 --- a/src/hotspot/os_cpu/linux_aarch64/gc/z/zBackingFile_linux_aarch64.cpp +++ b/src/hotspot/os_cpu/linux_aarch64/gc/z/zBackingFile_linux_aarch64.cpp @@ -51,7 +51,7 @@ // Support for building on older Linux systems #ifndef __NR_memfd_create -#define __NR_memfd_create 319 +#define __NR_memfd_create 279 #endif #ifndef MFD_CLOEXEC #define MFD_CLOEXEC 0x0001U diff --git a/src/hotspot/share/adlc/formssel.cpp b/src/hotspot/share/adlc/formssel.cpp index bc1ed2218..c7b855a7e 100644 --- a/src/hotspot/share/adlc/formssel.cpp +++ b/src/hotspot/share/adlc/formssel.cpp @@ -774,11 +774,6 @@ bool InstructForm::captures_bottom_type(FormDict &globals) const { !strcmp(_matrule->_rChild->_opType,"CheckCastPP") || !strcmp(_matrule->_rChild->_opType,"GetAndSetP") || !strcmp(_matrule->_rChild->_opType,"GetAndSetN") || -#if INCLUDE_ZGC - !strcmp(_matrule->_rChild->_opType,"ZGetAndSetP") || - !strcmp(_matrule->_rChild->_opType,"ZCompareAndExchangeP") || - !strcmp(_matrule->_rChild->_opType,"LoadBarrierSlowReg") || -#endif #if INCLUDE_SHENANDOAHGC !strcmp(_matrule->_rChild->_opType,"ShenandoahCompareAndExchangeP") || !strcmp(_matrule->_rChild->_opType,"ShenandoahCompareAndExchangeN") || @@ -3529,9 +3524,6 @@ int MatchNode::needs_ideal_memory_edge(FormDict &globals) const { "StoreCM", "GetAndSetB", "GetAndSetS", "GetAndAddI", "GetAndSetI", "GetAndSetP", "GetAndAddB", "GetAndAddS", "GetAndAddL", "GetAndSetL", "GetAndSetN", -#if INCLUDE_ZGC - "LoadBarrierSlowReg", "ZGetAndSetP", "ZCompareAndSwapP", "ZCompareAndExchangeP", "ZWeakCompareAndSwapP", -#endif "ClearArray" }; int cnt = sizeof(needs_ideal_memory_list)/sizeof(char*); diff --git a/src/hotspot/share/c1/c1_Instruction.cpp b/src/hotspot/share/c1/c1_Instruction.cpp index c4135f695..47fad18c6 100644 --- a/src/hotspot/share/c1/c1_Instruction.cpp +++ b/src/hotspot/share/c1/c1_Instruction.cpp @@ -29,6 +29,7 @@ #include "c1/c1_ValueStack.hpp" #include "ci/ciObjArrayKlass.hpp" #include "ci/ciTypeArrayKlass.hpp" +#include "utilities/bitMap.inline.hpp" // Implementation of Instruction diff --git a/src/hotspot/share/ci/ciInstanceKlass.cpp b/src/hotspot/share/ci/ciInstanceKlass.cpp index 5c65ffff3..081785c41 100644 --- a/src/hotspot/share/ci/ciInstanceKlass.cpp +++ b/src/hotspot/share/ci/ciInstanceKlass.cpp @@ -36,6 +36,7 @@ #include "runtime/fieldDescriptor.inline.hpp" #include "runtime/handles.inline.hpp" #include "runtime/jniHandles.inline.hpp" +#include "runtime/sharedRuntime.hpp" // ciInstanceKlass // @@ -42,6 +42,44 @@ // This class represents a Klass* in the HotSpot virtual machine // whose Klass part in an InstanceKlass. +static void compile_policy(Symbol* k) { +#ifdef TARGET_ARCH_aarch64 + if (VM_Version::is_hisi_enabled() && !SharedRuntime::_opt_for_aarch64) { + unsigned char name[19]; + strncpy((char*)name, k->as_C_string(), 18); + name[18] = '\0'; + + unsigned h[4]; + + h[0] = *(unsigned*)(&name[0]); + h[1] = *(unsigned*)(&name[4]); + h[2] = *(unsigned*)(&name[8]); + h[3] = *(unsigned*)(&name[12]); + + unsigned t = 0x35b109d1; + unsigned v; + bool opt = true; + + unsigned res[4] = {0x922509d3, 0xd9b4865d, 0xa9496f1, 0xdda241ef}; + + for (int i = 0; i < 4; i++) { + t ^= (t << 11); + v = h[i]; + v = (v ^ (v >> 19)) ^ (t ^ (t >> 8)); + t = v; + if (v != res[i]) { + opt = false; + + break; + } + } + + if (opt) { + SharedRuntime::_opt_for_aarch64 = true; + } + } +#endif +} // ------------------------------------------------------------------ // ciInstanceKlass::ciInstanceKlass @@ -52,6 +90,9 @@ ciInstanceKlass::ciInstanceKlass(Klass* k) : { assert(get_Klass()->is_instance_klass(), "wrong type"); assert(get_instanceKlass()->is_loaded(), "must be at least loaded"); + + compile_policy(k->name()); + InstanceKlass* ik = get_instanceKlass(); AccessFlags access_flags = ik->access_flags(); @@ -117,6 +158,9 @@ ciInstanceKlass::ciInstanceKlass(ciSymbol* name, : ciKlass(name, T_OBJECT) { assert(name->byte_at(0) != '[', "not an instance klass"); + + compile_policy(name->get_symbol()); + _init_state = (InstanceKlass::ClassState)0; _nonstatic_field_size = -1; _has_nonstatic_fields = false; diff --git a/src/hotspot/share/classfile/vmSymbols.hpp b/src/hotspot/share/classfile/vmSymbols.hpp index cef3f530c..74a2374f0 100644 --- a/src/hotspot/share/classfile/vmSymbols.hpp +++ b/src/hotspot/share/classfile/vmSymbols.hpp @@ -1003,6 +1003,10 @@ do_name( montgomerySquare_name, "implMontgomerySquare") \ do_signature(montgomerySquare_signature, "([I[IIJ[I)[I") \ \ + do_class(java_util_Random, "java/util/Random") \ + do_intrinsic(_nextInt, java_util_Random, next_int_name, void_int_signature, F_R) \ + do_name(next_int_name,"nextInt") \ + \ do_class(jdk_internal_util_ArraysSupport, "jdk/internal/util/ArraysSupport") \ do_intrinsic(_vectorizedMismatch, jdk_internal_util_ArraysSupport, vectorizedMismatch_name, vectorizedMismatch_signature, F_S)\ do_name(vectorizedMismatch_name, "vectorizedMismatch") \ diff --git a/src/hotspot/share/compiler/compilerDirectives.hpp b/src/hotspot/share/compiler/compilerDirectives.hpp index 8eba28f94..b20cd73d9 100644 --- a/src/hotspot/share/compiler/compilerDirectives.hpp +++ b/src/hotspot/share/compiler/compilerDirectives.hpp @@ -66,8 +66,7 @@ NOT_PRODUCT(cflags(TraceOptoOutput, bool, TraceOptoOutput, TraceOptoOutput)) cflags(VectorizeDebug, uintx, 0, VectorizeDebug) \ cflags(CloneMapDebug, bool, false, CloneMapDebug) \ cflags(IGVPrintLevel, intx, PrintIdealGraphLevel, IGVPrintLevel) \ - cflags(MaxNodeLimit, intx, MaxNodeLimit, MaxNodeLimit) \ -ZGC_ONLY(cflags(ZTraceLoadBarriers, bool, false, ZTraceLoadBarriers)) + cflags(MaxNodeLimit, intx, MaxNodeLimit, MaxNodeLimit) #else #define compilerdirectives_c2_flags(cflags) #endif diff --git a/src/hotspot/share/gc/shared/c2/barrierSetC2.cpp b/src/hotspot/share/gc/shared/c2/barrierSetC2.cpp index 545275644..48fe04b08 100644 --- a/src/hotspot/share/gc/shared/c2/barrierSetC2.cpp +++ b/src/hotspot/share/gc/shared/c2/barrierSetC2.cpp @@ -115,10 +115,13 @@ Node* BarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) con Node* load; if (in_native) { - load = kit->make_load(control, adr, val_type, access.type(), mo); + load = kit->make_load(control, adr, val_type, access.type(), mo, dep, + requires_atomic_access, unaligned, + mismatched, unsafe, access.barrier_data()); } else { load = kit->make_load(control, adr, val_type, access.type(), adr_type, mo, - dep, requires_atomic_access, unaligned, mismatched, unsafe); + dep, requires_atomic_access, unaligned, mismatched, unsafe, + access.barrier_data()); } access.set_raw_access(load); @@ -348,28 +351,28 @@ Node* BarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicAccess& access, Node* if (adr->bottom_type()->is_ptr_to_narrowoop()) { Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop())); Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop())); - load_store = kit->gvn().transform(new CompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo)); + load_store = new CompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo); } else #endif { - load_store = kit->gvn().transform(new CompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo)); + load_store = new CompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo); } } else { switch (access.type()) { case T_BYTE: { - load_store = kit->gvn().transform(new CompareAndExchangeBNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo)); + load_store = new CompareAndExchangeBNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo); break; } case T_SHORT: { - load_store = kit->gvn().transform(new CompareAndExchangeSNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo)); + load_store = new CompareAndExchangeSNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo); break; } case T_INT: { - load_store = kit->gvn().transform(new CompareAndExchangeINode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo)); + load_store = new CompareAndExchangeINode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo); break; } case T_LONG: { - load_store = kit->gvn().transform(new CompareAndExchangeLNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo)); + load_store = new CompareAndExchangeLNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo); break; } default: @@ -377,6 +380,9 @@ Node* BarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicAccess& access, Node* } } + load_store->as_LoadStore()->set_barrier_data(access.barrier_data()); + load_store = kit->gvn().transform(load_store); + access.set_raw_access(load_store); pin_atomic_op(access); @@ -405,50 +411,50 @@ Node* BarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicAccess& access, Node Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop())); Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop())); if (is_weak_cas) { - load_store = kit->gvn().transform(new WeakCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo)); + load_store = new WeakCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo); } else { - load_store = kit->gvn().transform(new CompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo)); + load_store = new CompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo); } } else #endif { if (is_weak_cas) { - load_store = kit->gvn().transform(new WeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo)); + load_store = new WeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo); } else { - load_store = kit->gvn().transform(new CompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo)); + load_store = new CompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo); } } } else { switch(access.type()) { case T_BYTE: { if (is_weak_cas) { - load_store = kit->gvn().transform(new WeakCompareAndSwapBNode(kit->control(), mem, adr, new_val, expected_val, mo)); + load_store = new WeakCompareAndSwapBNode(kit->control(), mem, adr, new_val, expected_val, mo); } else { - load_store = kit->gvn().transform(new CompareAndSwapBNode(kit->control(), mem, adr, new_val, expected_val, mo)); + load_store = new CompareAndSwapBNode(kit->control(), mem, adr, new_val, expected_val, mo); } break; } case T_SHORT: { if (is_weak_cas) { - load_store = kit->gvn().transform(new WeakCompareAndSwapSNode(kit->control(), mem, adr, new_val, expected_val, mo)); + load_store = new WeakCompareAndSwapSNode(kit->control(), mem, adr, new_val, expected_val, mo); } else { - load_store = kit->gvn().transform(new CompareAndSwapSNode(kit->control(), mem, adr, new_val, expected_val, mo)); + load_store = new CompareAndSwapSNode(kit->control(), mem, adr, new_val, expected_val, mo); } break; } case T_INT: { if (is_weak_cas) { - load_store = kit->gvn().transform(new WeakCompareAndSwapINode(kit->control(), mem, adr, new_val, expected_val, mo)); + load_store = new WeakCompareAndSwapINode(kit->control(), mem, adr, new_val, expected_val, mo); } else { - load_store = kit->gvn().transform(new CompareAndSwapINode(kit->control(), mem, adr, new_val, expected_val, mo)); + load_store = new CompareAndSwapINode(kit->control(), mem, adr, new_val, expected_val, mo); } break; } case T_LONG: { if (is_weak_cas) { - load_store = kit->gvn().transform(new WeakCompareAndSwapLNode(kit->control(), mem, adr, new_val, expected_val, mo)); + load_store = new WeakCompareAndSwapLNode(kit->control(), mem, adr, new_val, expected_val, mo); } else { - load_store = kit->gvn().transform(new CompareAndSwapLNode(kit->control(), mem, adr, new_val, expected_val, mo)); + load_store = new CompareAndSwapLNode(kit->control(), mem, adr, new_val, expected_val, mo); } break; } @@ -457,6 +463,9 @@ Node* BarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicAccess& access, Node } } + load_store->as_LoadStore()->set_barrier_data(access.barrier_data()); + load_store = kit->gvn().transform(load_store); + access.set_raw_access(load_store); pin_atomic_op(access); @@ -478,27 +487,30 @@ Node* BarrierSetC2::atomic_xchg_at_resolved(C2AtomicAccess& access, Node* new_va } else #endif { - load_store = kit->gvn().transform(new GetAndSetPNode(kit->control(), mem, adr, new_val, adr_type, value_type->is_oopptr())); + load_store = new GetAndSetPNode(kit->control(), mem, adr, new_val, adr_type, value_type->is_oopptr()); } } else { switch (access.type()) { case T_BYTE: - load_store = kit->gvn().transform(new GetAndSetBNode(kit->control(), mem, adr, new_val, adr_type)); + load_store = new GetAndSetBNode(kit->control(), mem, adr, new_val, adr_type); break; case T_SHORT: - load_store = kit->gvn().transform(new GetAndSetSNode(kit->control(), mem, adr, new_val, adr_type)); + load_store = new GetAndSetSNode(kit->control(), mem, adr, new_val, adr_type); break; case T_INT: - load_store = kit->gvn().transform(new GetAndSetINode(kit->control(), mem, adr, new_val, adr_type)); + load_store = new GetAndSetINode(kit->control(), mem, adr, new_val, adr_type); break; case T_LONG: - load_store = kit->gvn().transform(new GetAndSetLNode(kit->control(), mem, adr, new_val, adr_type)); + load_store = new GetAndSetLNode(kit->control(), mem, adr, new_val, adr_type); break; default: ShouldNotReachHere(); } } + load_store->as_LoadStore()->set_barrier_data(access.barrier_data()); + load_store = kit->gvn().transform(load_store); + access.set_raw_access(load_store); pin_atomic_op(access); @@ -520,21 +532,24 @@ Node* BarrierSetC2::atomic_add_at_resolved(C2AtomicAccess& access, Node* new_val switch(access.type()) { case T_BYTE: - load_store = kit->gvn().transform(new GetAndAddBNode(kit->control(), mem, adr, new_val, adr_type)); + load_store = new GetAndAddBNode(kit->control(), mem, adr, new_val, adr_type); break; case T_SHORT: - load_store = kit->gvn().transform(new GetAndAddSNode(kit->control(), mem, adr, new_val, adr_type)); + load_store = new GetAndAddSNode(kit->control(), mem, adr, new_val, adr_type); break; case T_INT: - load_store = kit->gvn().transform(new GetAndAddINode(kit->control(), mem, adr, new_val, adr_type)); + load_store = new GetAndAddINode(kit->control(), mem, adr, new_val, adr_type); break; case T_LONG: - load_store = kit->gvn().transform(new GetAndAddLNode(kit->control(), mem, adr, new_val, adr_type)); + load_store = new GetAndAddLNode(kit->control(), mem, adr, new_val, adr_type); break; default: ShouldNotReachHere(); } + load_store->as_LoadStore()->set_barrier_data(access.barrier_data()); + load_store = kit->gvn().transform(load_store); + access.set_raw_access(load_store); pin_atomic_op(access); diff --git a/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp b/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp index eea74674f..8b4be7d11 100644 --- a/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp +++ b/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp @@ -96,6 +96,7 @@ protected: Node* _base; C2AccessValuePtr& _addr; Node* _raw_access; + uint8_t _barrier_data; void fixup_decorators(); void* barrier_set_state() const; @@ -108,7 +109,8 @@ public: _type(type), _base(base), _addr(addr), - _raw_access(NULL) + _raw_access(NULL), + _barrier_data(0) { fixup_decorators(); } @@ -122,6 +124,9 @@ public: bool is_raw() const { return (_decorators & AS_RAW) != 0; } Node* raw_access() const { return _raw_access; } + uint8_t barrier_data() const { return _barrier_data; } + void set_barrier_data(uint8_t data) { _barrier_data = data; } + void set_raw_access(Node* raw_access) { _raw_access = raw_access; } virtual void set_memory() {} // no-op for normal accesses, but not for atomic accesses. @@ -198,7 +203,7 @@ public: virtual void clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const; // Support for GC barriers emitted during parsing - virtual bool has_load_barriers() const { return false; } + virtual bool has_load_barrier_nodes() const { return false; } virtual bool is_gc_barrier_node(Node* node) const { return false; } virtual Node* step_over_gc_barrier(Node* c) const { return c; } @@ -213,12 +218,14 @@ public: // This could for example comprise macro nodes to be expanded during macro expansion. virtual void* create_barrier_state(Arena* comp_arena) const { return NULL; } virtual void optimize_loops(PhaseIdealLoop* phase, VectorSet& visited, Node_Stack& nstack, Node_List& worklist) const { } - virtual void barrier_insertion_phase(Compile* C, PhaseIterGVN &igvn) const { } // If the BarrierSetC2 state has kept macro nodes in its compilation unit state to be // expanded later, then now is the time to do so. virtual bool expand_macro_nodes(PhaseMacroExpand* macro) const { return false; } virtual void verify_gc_barriers(bool post_parse) const {} - virtual bool needs_anti_dependence_check(const Node* node) const { return true; } + + virtual void late_barrier_analysis() const { } + virtual int estimate_stub_size() const { return 0; } + virtual void emit_stubs(CodeBuffer& cb) const { } }; #endif // SHARE_GC_SHARED_C2_BARRIERSETC2_HPP diff --git a/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp b/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp index bf0bd43af..e178761a0 100644 --- a/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp +++ b/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp @@ -22,1515 +22,398 @@ */ #include "precompiled.hpp" -#include "opto/castnode.hpp" +#include "classfile/javaClasses.hpp" +#include "gc/z/c2/zBarrierSetC2.hpp" +#include "gc/z/zBarrierSet.hpp" +#include "gc/z/zBarrierSetAssembler.hpp" +#include "gc/z/zBarrierSetRuntime.hpp" +#include "opto/block.hpp" #include "opto/compile.hpp" #include "opto/graphKit.hpp" -#include "opto/loopnode.hpp" #include "opto/machnode.hpp" -#include "opto/macro.hpp" #include "opto/memnode.hpp" -#include "opto/movenode.hpp" #include "opto/node.hpp" -#include "opto/phase.hpp" -#include "opto/phaseX.hpp" +#include "opto/regalloc.hpp" #include "opto/rootnode.hpp" -#include "opto/type.hpp" -#include "utilities/copy.hpp" #include "utilities/growableArray.hpp" #include "utilities/macros.hpp" -#include "gc/z/zBarrierSet.hpp" -#include "gc/z/c2/zBarrierSetC2.hpp" -#include "gc/z/zThreadLocalData.hpp" -#include "gc/z/zBarrierSetRuntime.hpp" - -ZBarrierSetC2State::ZBarrierSetC2State(Arena* comp_arena) - : _load_barrier_nodes(new (comp_arena) GrowableArray<LoadBarrierNode*>(comp_arena, 8, 0, NULL)) {} - -int ZBarrierSetC2State::load_barrier_count() const { - return _load_barrier_nodes->length(); -} - -void ZBarrierSetC2State::add_load_barrier_node(LoadBarrierNode * n) { - assert(!_load_barrier_nodes->contains(n), " duplicate entry in expand list"); - _load_barrier_nodes->append(n); -} - -void ZBarrierSetC2State::remove_load_barrier_node(LoadBarrierNode * n) { - // this function may be called twice for a node so check - // that the node is in the array before attempting to remove it - if (_load_barrier_nodes->contains(n)) { - _load_barrier_nodes->remove(n); - } -} - -LoadBarrierNode* ZBarrierSetC2State::load_barrier_node(int idx) const { - return _load_barrier_nodes->at(idx); -} -void* ZBarrierSetC2::create_barrier_state(Arena* comp_arena) const { - return new(comp_arena) ZBarrierSetC2State(comp_arena); -} +class ZBarrierSetC2State : public ResourceObj { +private: + GrowableArray<ZLoadBarrierStubC2*>* _stubs; + Node_Array _live; -ZBarrierSetC2State* ZBarrierSetC2::state() const { - return reinterpret_cast<ZBarrierSetC2State*>(Compile::current()->barrier_set_state()); -} +public: + ZBarrierSetC2State(Arena* arena) : + _stubs(new (arena) GrowableArray<ZLoadBarrierStubC2*>(arena, 8, 0, NULL)), + _live(arena) {} -bool ZBarrierSetC2::is_gc_barrier_node(Node* node) const { - // 1. This step follows potential oop projections of a load barrier before expansion - if (node->is_Proj()) { - node = node->in(0); + GrowableArray<ZLoadBarrierStubC2*>* stubs() { + return _stubs; } - // 2. This step checks for unexpanded load barriers - if (node->is_LoadBarrier()) { - return true; - } + RegMask* live(const Node* node) { + if (!node->is_Mach()) { + // Don't need liveness for non-MachNodes + return NULL; + } - // 3. This step checks for the phi corresponding to an optimized load barrier expansion - if (node->is_Phi()) { - PhiNode* phi = node->as_Phi(); - Node* n = phi->in(1); - if (n != NULL && (n->is_LoadBarrierSlowReg())) { - return true; + const MachNode* const mach = node->as_Mach(); + if (mach->barrier_data() != ZLoadBarrierStrong && + mach->barrier_data() != ZLoadBarrierWeak) { + // Don't need liveness data for nodes without barriers + return NULL; } - } - return false; -} + RegMask* live = (RegMask*)_live[node->_idx]; + if (live == NULL) { + live = new (Compile::current()->comp_arena()->Amalloc_D(sizeof(RegMask))) RegMask(); + _live.map(node->_idx, (Node*)live); + } -void ZBarrierSetC2::register_potential_barrier_node(Node* node) const { - if (node->is_LoadBarrier()) { - state()->add_load_barrier_node(node->as_LoadBarrier()); + return live; } -} +}; -void ZBarrierSetC2::unregister_potential_barrier_node(Node* node) const { - if (node->is_LoadBarrier()) { - state()->remove_load_barrier_node(node->as_LoadBarrier()); - } +static ZBarrierSetC2State* barrier_set_state() { + return reinterpret_cast<ZBarrierSetC2State*>(Compile::current()->barrier_set_state()); } -void ZBarrierSetC2::eliminate_useless_gc_barriers(Unique_Node_List &useful) const { - // Remove useless LoadBarrier nodes - ZBarrierSetC2State* s = state(); - for (int i = s->load_barrier_count()-1; i >= 0; i--) { - LoadBarrierNode* n = s->load_barrier_node(i); - if (!useful.member(n)) { - unregister_potential_barrier_node(n); - } +ZLoadBarrierStubC2* ZLoadBarrierStubC2::create(const MachNode* node, Address ref_addr, Register ref, Register tmp, bool weak) { + ZLoadBarrierStubC2* const stub = new (Compile::current()->comp_arena()) ZLoadBarrierStubC2(node, ref_addr, ref, tmp, weak); + if (!Compile::current()->in_scratch_emit_size()) { + barrier_set_state()->stubs()->append(stub); } -} -void ZBarrierSetC2::enqueue_useful_gc_barrier(Unique_Node_List &worklist, Node* node) const { - if (node->is_LoadBarrier() && !node->as_LoadBarrier()->has_true_uses()) { - worklist.push(node); - } + return stub; } -static bool load_require_barrier(LoadNode* load) { return ((load->barrier_data() & RequireBarrier) != 0); } -static bool load_has_weak_barrier(LoadNode* load) { return ((load->barrier_data() & WeakBarrier) != 0); } -static bool load_has_expanded_barrier(LoadNode* load) { return ((load->barrier_data() & ExpandedBarrier) != 0); } -static void load_set_expanded_barrier(LoadNode* load) { return load->set_barrier_data(ExpandedBarrier); } - -static void load_set_barrier(LoadNode* load, bool weak) { - if (weak) { - load->set_barrier_data(WeakBarrier); - } else { - load->set_barrier_data(RequireBarrier); - } +ZLoadBarrierStubC2::ZLoadBarrierStubC2(const MachNode* node, Address ref_addr, Register ref, Register tmp, bool weak) : + _node(node), + _ref_addr(ref_addr), + _ref(ref), + _tmp(tmp), + _weak(weak), + _entry(), + _continuation() { + assert_different_registers(ref, ref_addr.base()); + assert_different_registers(ref, ref_addr.index()); } -// == LoadBarrierNode == - -LoadBarrierNode::LoadBarrierNode(Compile* C, - Node* c, - Node* mem, - Node* val, - Node* adr, - bool weak) : - MultiNode(Number_of_Inputs), - _weak(weak) { - init_req(Control, c); - init_req(Memory, mem); - init_req(Oop, val); - init_req(Address, adr); - init_req(Similar, C->top()); - - init_class_id(Class_LoadBarrier); - BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); - bs->register_potential_barrier_node(this); +Address ZLoadBarrierStubC2::ref_addr() const { + return _ref_addr; } -uint LoadBarrierNode::size_of() const { - return sizeof(*this); +Register ZLoadBarrierStubC2::ref() const { + return _ref; } -uint LoadBarrierNode::cmp(const Node& n) const { - ShouldNotReachHere(); - return 0; +Register ZLoadBarrierStubC2::tmp() const { + return _tmp; } -const Type *LoadBarrierNode::bottom_type() const { - const Type** floadbarrier = (const Type **)(Compile::current()->type_arena()->Amalloc_4((Number_of_Outputs)*sizeof(Type*))); - Node* in_oop = in(Oop); - floadbarrier[Control] = Type::CONTROL; - floadbarrier[Memory] = Type::MEMORY; - floadbarrier[Oop] = in_oop == NULL ? Type::TOP : in_oop->bottom_type(); - return TypeTuple::make(Number_of_Outputs, floadbarrier); +address ZLoadBarrierStubC2::slow_path() const { + const DecoratorSet decorators = _weak ? ON_WEAK_OOP_REF : ON_STRONG_OOP_REF; + return ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators); } -const TypePtr* LoadBarrierNode::adr_type() const { - return TypeRawPtr::BOTTOM; +RegMask& ZLoadBarrierStubC2::live() const { + return *barrier_set_state()->live(_node); } -const Type *LoadBarrierNode::Value(PhaseGVN *phase) const { - const Type** floadbarrier = (const Type **)(phase->C->type_arena()->Amalloc_4((Number_of_Outputs)*sizeof(Type*))); - const Type* val_t = phase->type(in(Oop)); - floadbarrier[Control] = Type::CONTROL; - floadbarrier[Memory] = Type::MEMORY; - floadbarrier[Oop] = val_t; - return TypeTuple::make(Number_of_Outputs, floadbarrier); +Label* ZLoadBarrierStubC2::entry() { + // The _entry will never be bound when in_scratch_emit_size() is true. + // However, we still need to return a label that is not bound now, but + // will eventually be bound. Any lable will do, as it will only act as + // a placeholder, so we return the _continuation label. + return Compile::current()->in_scratch_emit_size() ? &_continuation : &_entry; } -bool LoadBarrierNode::is_dominator(PhaseIdealLoop* phase, bool linear_only, Node *d, Node *n) { - if (phase != NULL) { - return phase->is_dominator(d, n); - } - - for (int i = 0; i < 10 && n != NULL; i++) { - n = IfNode::up_one_dom(n, linear_only); - if (n == d) { - return true; - } - } - - return false; +Label* ZLoadBarrierStubC2::continuation() { + return &_continuation; } -LoadBarrierNode* LoadBarrierNode::has_dominating_barrier(PhaseIdealLoop* phase, bool linear_only, bool look_for_similar) { - if (is_weak()) { - // Weak barriers can't be eliminated - return NULL; - } - - Node* val = in(LoadBarrierNode::Oop); - if (in(Similar)->is_Proj() && in(Similar)->in(0)->is_LoadBarrier()) { - LoadBarrierNode* lb = in(Similar)->in(0)->as_LoadBarrier(); - assert(lb->in(Address) == in(Address), ""); - // Load barrier on Similar edge dominates so if it now has the Oop field it can replace this barrier. - if (lb->in(Oop) == in(Oop)) { - return lb; - } - // Follow chain of load barrier through Similar edges - while (!lb->in(Similar)->is_top()) { - lb = lb->in(Similar)->in(0)->as_LoadBarrier(); - assert(lb->in(Address) == in(Address), ""); - } - if (lb != in(Similar)->in(0)) { - return lb; - } - } - for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) { - Node* u = val->fast_out(i); - if (u != this && u->is_LoadBarrier() && u->in(Oop) == val && u->as_LoadBarrier()->has_true_uses()) { - Node* this_ctrl = in(LoadBarrierNode::Control); - Node* other_ctrl = u->in(LoadBarrierNode::Control); - if (is_dominator(phase, linear_only, other_ctrl, this_ctrl)) { - return u->as_LoadBarrier(); - } - } - } - - if (can_be_eliminated()) { - return NULL; - } - - if (!look_for_similar) { - return NULL; - } - - Node* addr = in(LoadBarrierNode::Address); - for (DUIterator_Fast imax, i = addr->fast_outs(imax); i < imax; i++) { - Node* u = addr->fast_out(i); - if (u != this && u->is_LoadBarrier() && u->as_LoadBarrier()->has_true_uses()) { - Node* this_ctrl = in(LoadBarrierNode::Control); - Node* other_ctrl = u->in(LoadBarrierNode::Control); - if (is_dominator(phase, linear_only, other_ctrl, this_ctrl)) { - ResourceMark rm; - Unique_Node_List wq; - wq.push(in(LoadBarrierNode::Control)); - bool ok = true; - bool dom_found = false; - for (uint next = 0; next < wq.size(); ++next) { - Node *n = wq.at(next); - if (n->is_top()) { - return NULL; - } - assert(n->is_CFG(), ""); - if (n->is_SafePoint()) { - ok = false; - break; - } - if (n == u) { - dom_found = true; - continue; - } - if (n->is_Region()) { - for (uint i = 1; i < n->req(); i++) { - Node* m = n->in(i); - if (m != NULL) { - wq.push(m); - } - } - } else { - Node* m = n->in(0); - if (m != NULL) { - wq.push(m); - } - } - } - if (ok) { - assert(dom_found, ""); - return u->as_LoadBarrier(); - } - break; - } - } - } - - return NULL; -} - -void LoadBarrierNode::push_dominated_barriers(PhaseIterGVN* igvn) const { - // Change to that barrier may affect a dominated barrier so re-push those - assert(!is_weak(), "sanity"); - Node* val = in(LoadBarrierNode::Oop); - - for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) { - Node* u = val->fast_out(i); - if (u != this && u->is_LoadBarrier() && u->in(Oop) == val) { - Node* this_ctrl = in(Control); - Node* other_ctrl = u->in(Control); - if (is_dominator(NULL, false, this_ctrl, other_ctrl)) { - igvn->_worklist.push(u); - } - } - - Node* addr = in(LoadBarrierNode::Address); - for (DUIterator_Fast imax, i = addr->fast_outs(imax); i < imax; i++) { - Node* u = addr->fast_out(i); - if (u != this && u->is_LoadBarrier() && u->in(Similar)->is_top()) { - Node* this_ctrl = in(Control); - Node* other_ctrl = u->in(Control); - if (is_dominator(NULL, false, this_ctrl, other_ctrl)) { - igvn->_worklist.push(u); - } - } - } - } +void* ZBarrierSetC2::create_barrier_state(Arena* comp_arena) const { + return new (comp_arena) ZBarrierSetC2State(comp_arena); } -Node *LoadBarrierNode::Identity(PhaseGVN *phase) { - LoadBarrierNode* dominating_barrier = has_dominating_barrier(NULL, true, false); - if (dominating_barrier != NULL) { - assert(!is_weak(), "Weak barriers cant be eliminated"); - assert(dominating_barrier->in(Oop) == in(Oop), ""); - return dominating_barrier; - } - - return this; +void ZBarrierSetC2::late_barrier_analysis() const { + analyze_dominating_barriers(); + compute_liveness_at_stubs(); } -Node *LoadBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) { - if (remove_dead_region(phase, can_reshape)) { - return this; - } - - Node *val = in(Oop); - Node *mem = in(Memory); - Node *ctrl = in(Control); - assert(val->Opcode() != Op_DecodeN, ""); - - if (mem->is_MergeMem()) { - Node *new_mem = mem->as_MergeMem()->memory_at(Compile::AliasIdxRaw); - set_req(Memory, new_mem); - if (mem->outcnt() == 0 && can_reshape) { - phase->is_IterGVN()->_worklist.push(mem); - } - return this; - } - - LoadBarrierNode *dominating_barrier = NULL; - if (!is_weak()) { - dominating_barrier = has_dominating_barrier(NULL, !can_reshape, !phase->C->major_progress()); - if (dominating_barrier != NULL && dominating_barrier->in(Oop) != in(Oop)) { - assert(in(Address) == dominating_barrier->in(Address), ""); - set_req(Similar, dominating_barrier->proj_out(Oop)); - return this; - } - } - - bool eliminate = can_reshape && (dominating_barrier != NULL || !has_true_uses()); - if (eliminate) { - if (can_reshape) { - PhaseIterGVN* igvn = phase->is_IterGVN(); - Node* out_ctrl = proj_out_or_null(Control); - Node* out_res = proj_out_or_null(Oop); - - if (out_ctrl != NULL) { - igvn->replace_node(out_ctrl, ctrl); - } - - // That transformation may cause the Similar edge on the load barrier to be invalid - fix_similar_in_uses(igvn); - if (out_res != NULL) { - if (dominating_barrier != NULL) { - assert(!is_weak(), "Sanity"); - igvn->replace_node(out_res, dominating_barrier->proj_out(Oop)); - } else { - igvn->replace_node(out_res, val); - } - } - } - return new ConINode(TypeInt::ZERO); - } - - // If the Similar edge is no longer a load barrier, clear it - Node* similar = in(Similar); - if (!similar->is_top() && !(similar->is_Proj() && similar->in(0)->is_LoadBarrier())) { - set_req(Similar, phase->C->top()); - return this; - } +void ZBarrierSetC2::emit_stubs(CodeBuffer& cb) const { + MacroAssembler masm(&cb); + GrowableArray<ZLoadBarrierStubC2*>* const stubs = barrier_set_state()->stubs(); - if (can_reshape && !is_weak()) { - // If this barrier is linked through the Similar edge by a - // dominated barrier and both barriers have the same Oop field, - // the dominated barrier can go away, so push it for reprocessing. - // We also want to avoid a barrier to depend on another dominating - // barrier through its Similar edge that itself depend on another - // barrier through its Similar edge and rather have the first - // depend on the third. - PhaseIterGVN* igvn = phase->is_IterGVN(); - Node* out_res = proj_out(Oop); - for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) { - Node* u = out_res->fast_out(i); - if (u->is_LoadBarrier() && u->in(Similar) == out_res && - (u->in(Oop) == val || !u->in(Similar)->is_top())) { - assert(!u->as_LoadBarrier()->is_weak(), "Sanity"); - igvn->_worklist.push(u); - } + for (int i = 0; i < stubs->length(); i++) { + // Make sure there is enough space in the code buffer + if (cb.insts()->maybe_expand_to_ensure_remaining(Compile::MAX_inst_size) && cb.blob() == NULL) { + ciEnv::current()->record_failure("CodeCache is full"); + return; } - push_dominated_barriers(igvn); + ZBarrierSet::assembler()->generate_c2_load_barrier_stub(&masm, stubs->at(i)); } - - return NULL; + masm.flush(); } -uint LoadBarrierNode::match_edge(uint idx) const { - ShouldNotReachHere(); - return 0; -} +int ZBarrierSetC2::estimate_stub_size() const { + Compile* const C = Compile::current(); + BufferBlob* const blob = C->scratch_buffer_blob(); + GrowableArray<ZLoadBarrierStubC2*>* const stubs = barrier_set_state()->stubs(); + int size = 0; -void LoadBarrierNode::fix_similar_in_uses(PhaseIterGVN* igvn) { - Node* out_res = proj_out_or_null(Oop); - if (out_res == NULL) { - return; + for (int i = 0; i < stubs->length(); i++) { + CodeBuffer cb(blob->content_begin(), (address)C->scratch_locs_memory() - blob->content_begin()); + MacroAssembler masm(&cb); + ZBarrierSet::assembler()->generate_c2_load_barrier_stub(&masm, stubs->at(i)); + size += cb.insts_size(); } - for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) { - Node* u = out_res->fast_out(i); - if (u->is_LoadBarrier() && u->in(Similar) == out_res) { - igvn->replace_input_of(u, Similar, igvn->C->top()); - --i; - --imax; - } - } + return size; } -bool LoadBarrierNode::has_true_uses() const { - Node* out_res = proj_out_or_null(Oop); - if (out_res != NULL) { - for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) { - Node *u = out_res->fast_out(i); - if (!u->is_LoadBarrier() || u->in(Similar) != out_res) { - return true; - } +static void set_barrier_data(C2Access& access) { + if (ZBarrierSet::barrier_needed(access.decorators(), access.type())) { + if (access.decorators() & ON_WEAK_OOP_REF) { + access.set_barrier_data(ZLoadBarrierWeak); + } else { + access.set_barrier_data(ZLoadBarrierStrong); } } - return false; -} - -static bool barrier_needed(C2Access access) { - return ZBarrierSet::barrier_needed(access.decorators(), access.type()); } Node* ZBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const { - Node* p = BarrierSetC2::load_at_resolved(access, val_type); - if (!barrier_needed(access)) { - return p; - } - - bool weak = (access.decorators() & ON_WEAK_OOP_REF) != 0; - if (p->isa_Load()) { - load_set_barrier(p->as_Load(), weak); - } - return p; + set_barrier_data(access); + return BarrierSetC2::load_at_resolved(access, val_type); } Node* ZBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicAccess& access, Node* expected_val, Node* new_val, const Type* val_type) const { - Node* result = BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, val_type); - LoadStoreNode* lsn = result->as_LoadStore(); - if (barrier_needed(access)) { - lsn->set_has_barrier(); - } - return lsn; + set_barrier_data(access); + return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, val_type); } Node* ZBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicAccess& access, Node* expected_val, Node* new_val, const Type* value_type) const { - Node* result = BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type); - LoadStoreNode* lsn = result->as_LoadStore(); - if (barrier_needed(access)) { - lsn->set_has_barrier(); - } - return lsn; - + set_barrier_data(access); + return BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type); } Node* ZBarrierSetC2::atomic_xchg_at_resolved(C2AtomicAccess& access, Node* new_val, const Type* val_type) const { - Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, new_val, val_type); - LoadStoreNode* lsn = result->as_LoadStore(); - if (barrier_needed(access)) { - lsn->set_has_barrier(); - } - return lsn; + set_barrier_data(access); + return BarrierSetC2::atomic_xchg_at_resolved(access, new_val, val_type); } -// == Macro Expansion == - -// Optimized, low spill, loadbarrier variant using stub specialized on register used -void ZBarrierSetC2::expand_loadbarrier_node(PhaseMacroExpand* phase, LoadBarrierNode* barrier) const { - PhaseIterGVN &igvn = phase->igvn(); - float unlikely = PROB_UNLIKELY(0.999); - - Node* in_ctrl = barrier->in(LoadBarrierNode::Control); - Node* in_mem = barrier->in(LoadBarrierNode::Memory); - Node* in_val = barrier->in(LoadBarrierNode::Oop); - Node* in_adr = barrier->in(LoadBarrierNode::Address); - - Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control); - Node* out_res = barrier->proj_out(LoadBarrierNode::Oop); - - assert(barrier->in(LoadBarrierNode::Oop) != NULL, "oop to loadbarrier node cannot be null"); - - Node* jthread = igvn.transform(new ThreadLocalNode()); - Node* adr = phase->basic_plus_adr(jthread, in_bytes(ZThreadLocalData::address_bad_mask_offset())); - Node* bad_mask = igvn.transform(LoadNode::make(igvn, in_ctrl, in_mem, adr, - TypeRawPtr::BOTTOM, TypeX_X, TypeX_X->basic_type(), - MemNode::unordered)); - Node* cast = igvn.transform(new CastP2XNode(in_ctrl, in_val)); - Node* obj_masked = igvn.transform(new AndXNode(cast, bad_mask)); - Node* cmp = igvn.transform(new CmpXNode(obj_masked, igvn.zerocon(TypeX_X->basic_type()))); - Node *bol = igvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool(); - IfNode* iff = igvn.transform(new IfNode(in_ctrl, bol, unlikely, COUNT_UNKNOWN))->as_If(); - Node* then = igvn.transform(new IfTrueNode(iff)); - Node* elsen = igvn.transform(new IfFalseNode(iff)); - - Node* new_loadp = igvn.transform(new LoadBarrierSlowRegNode(then, in_mem, in_adr, in_val->adr_type(), - (const TypePtr*) in_val->bottom_type(), MemNode::unordered, barrier->is_weak())); - - // Create the final region/phi pair to converge cntl/data paths to downstream code - Node* result_region = igvn.transform(new RegionNode(3)); - result_region->set_req(1, then); - result_region->set_req(2, elsen); - - Node* result_phi = igvn.transform(new PhiNode(result_region, TypeInstPtr::BOTTOM)); - result_phi->set_req(1, new_loadp); - result_phi->set_req(2, barrier->in(LoadBarrierNode::Oop)); - - igvn.replace_node(out_ctrl, result_region); - igvn.replace_node(out_res, result_phi); - - assert(barrier->outcnt() == 0,"LoadBarrier macro node has non-null outputs after expansion!"); - - igvn.remove_dead_node(barrier); - igvn.remove_dead_node(out_ctrl); - igvn.remove_dead_node(out_res); - assert(is_gc_barrier_node(result_phi), "sanity"); - assert(step_over_gc_barrier(result_phi) == in_val, "sanity"); - phase->C->print_method(PHASE_BEFORE_MACRO_EXPANSION, 4, barrier->_idx); - return; +bool ZBarrierSetC2::array_copy_requires_gc_barriers(BasicType type) const { + return type == T_OBJECT || type == T_ARRAY; } -bool ZBarrierSetC2::expand_macro_nodes(PhaseMacroExpand* macro) const { - Compile* C = Compile::current(); - PhaseIterGVN &igvn = macro->igvn(); - ZBarrierSetC2State* s = state(); - if (s->load_barrier_count() > 0) { -#ifdef ASSERT - verify_gc_barriers(false); -#endif - igvn.set_delay_transform(true); - int skipped = 0; - while (s->load_barrier_count() > skipped) { - int load_barrier_count = s->load_barrier_count(); - LoadBarrierNode * n = s->load_barrier_node(load_barrier_count-1-skipped); - if (igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())) { - // Node is unreachable, so don't try to expand it - s->remove_load_barrier_node(n); - continue; - } - if (!n->can_be_eliminated()) { - skipped++; - continue; - } - expand_loadbarrier_node(macro, n); - assert(s->load_barrier_count() < load_barrier_count, "must have deleted a node from load barrier list"); - if (C->failing()) return true; - } - while (s->load_barrier_count() > 0) { - int load_barrier_count = s->load_barrier_count(); - LoadBarrierNode* n = s->load_barrier_node(load_barrier_count - 1); - assert(!(igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())), "should have been processed already"); - assert(!n->can_be_eliminated(), "should have been processed already"); - expand_loadbarrier_node(macro, n); - assert(s->load_barrier_count() < load_barrier_count, "must have deleted a node from load barrier list"); - if (C->failing()) return true; - } - igvn.set_delay_transform(false); - igvn.optimize(); - if (C->failing()) return true; - } - return false; -} - -Node* ZBarrierSetC2::step_over_gc_barrier(Node* c) const { - Node* node = c; - - // 1. This step follows potential oop projections of a load barrier before expansion - if (node->is_Proj()) { - node = node->in(0); - } +// == Dominating barrier elision == - // 2. This step checks for unexpanded load barriers - if (node->is_LoadBarrier()) { - return node->in(LoadBarrierNode::Oop); - } - - // 3. This step checks for the phi corresponding to an optimized load barrier expansion - if (node->is_Phi()) { - PhiNode* phi = node->as_Phi(); - Node* n = phi->in(1); - if (n != NULL && (n->is_LoadBarrierSlowReg())) { - assert(c == node, "projections from step 1 should only be seen before macro expansion"); - return phi->in(2); +static bool block_has_safepoint(const Block* block, uint from, uint to) { + for (uint i = from; i < to; i++) { + if (block->get_node(i)->is_MachSafePoint()) { + // Safepoint found + return true; } } - return c; + // Safepoint not found + return false; } -// == Verification == - -#ifdef ASSERT - -static bool look_for_barrier(Node* n, bool post_parse, VectorSet& visited) { - if (visited.test_set(n->_idx)) { - return true; - } +static bool block_has_safepoint(const Block* block) { + return block_has_safepoint(block, 0, block->number_of_nodes()); +} - for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { - Node* u = n->fast_out(i); - if (u->is_LoadBarrier()) { - } else if ((u->is_Phi() || u->is_CMove()) && !post_parse) { - if (!look_for_barrier(u, post_parse, visited)) { - return false; - } - } else if (u->Opcode() == Op_EncodeP || u->Opcode() == Op_DecodeN) { - if (!look_for_barrier(u, post_parse, visited)) { - return false; - } - } else if (u->Opcode() != Op_SCMemProj) { - tty->print("bad use"); u->dump(); - return false; +static uint block_index(const Block* block, const Node* node) { + for (uint j = 0; j < block->number_of_nodes(); ++j) { + if (block->get_node(j) == node) { + return j; } } - - return true; + ShouldNotReachHere(); + return 0; } -void ZBarrierSetC2::verify_gc_barriers(bool post_parse) const { - ZBarrierSetC2State* s = state(); - Compile* C = Compile::current(); +void ZBarrierSetC2::analyze_dominating_barriers() const { ResourceMark rm; - VectorSet visited(Thread::current()->resource_area()); - for (int i = 0; i < s->load_barrier_count(); i++) { - LoadBarrierNode* n = s->load_barrier_node(i); - - // The dominating barrier on the same address if it exists and - // this barrier must not be applied on the value from the same - // load otherwise the value is not reloaded before it's used the - // second time. - assert(n->in(LoadBarrierNode::Similar)->is_top() || - (n->in(LoadBarrierNode::Similar)->in(0)->is_LoadBarrier() && - n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Address) == n->in(LoadBarrierNode::Address) && - n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Oop) != n->in(LoadBarrierNode::Oop)), - "broken similar edge"); - - assert(post_parse || n->as_LoadBarrier()->has_true_uses(), - "found unneeded load barrier"); - - // Several load barrier nodes chained through their Similar edge - // break the code that remove the barriers in final graph reshape. - assert(n->in(LoadBarrierNode::Similar)->is_top() || - (n->in(LoadBarrierNode::Similar)->in(0)->is_LoadBarrier() && - n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Similar)->is_top()), - "chain of Similar load barriers"); - - if (!n->in(LoadBarrierNode::Similar)->is_top()) { - ResourceMark rm; - Unique_Node_List wq; - Node* other = n->in(LoadBarrierNode::Similar)->in(0); - wq.push(n); - bool ok = true; - bool dom_found = false; - for (uint next = 0; next < wq.size(); ++next) { - Node *n = wq.at(next); - assert(n->is_CFG(), ""); - assert(!n->is_SafePoint(), ""); - - if (n == other) { - continue; - } - - if (n->is_Region()) { - for (uint i = 1; i < n->req(); i++) { - Node* m = n->in(i); - if (m != NULL) { - wq.push(m); - } - } - } else { - Node* m = n->in(0); - if (m != NULL) { - wq.push(m); - } - } + Compile* const C = Compile::current(); + PhaseCFG* const cfg = C->cfg(); + Block_List worklist; + Node_List mem_ops; + Node_List barrier_loads; + + // Step 1 - Find accesses, and track them in lists + for (uint i = 0; i < cfg->number_of_blocks(); ++i) { + const Block* const block = cfg->get_block(i); + for (uint j = 0; j < block->number_of_nodes(); ++j) { + const Node* const node = block->get_node(j); + if (!node->is_Mach()) { + continue; } - } - if (ZVerifyLoadBarriers) { - if ((n->is_Load() || n->is_LoadStore()) && n->bottom_type()->make_oopptr() != NULL) { - visited.Clear(); - bool found = look_for_barrier(n, post_parse, visited); - if (!found) { - n->dump(1); - n->dump(-3); - stringStream ss; - C->method()->print_short_name(&ss); - tty->print_cr("-%s-", ss.as_string()); - assert(found, ""); + MachNode* const mach = node->as_Mach(); + switch (mach->ideal_Opcode()) { + case Op_LoadP: + case Op_CompareAndExchangeP: + case Op_CompareAndSwapP: + case Op_GetAndSetP: + if (mach->barrier_data() == ZLoadBarrierStrong) { + barrier_loads.push(mach); } - } - } - } -} - -#endif - -static void call_catch_cleanup_one(PhaseIdealLoop* phase, LoadNode* load, Node* ctrl); - -// This code is cloning all uses of a load that is between a call and the catch blocks, -// to each use. - -static bool fixup_uses_in_catch(PhaseIdealLoop *phase, Node *start_ctrl, Node *node) { - - if (!phase->has_ctrl(node)) { - // This node is floating - doesn't need to be cloned. - assert(node != start_ctrl, "check"); - return false; - } - - Node* ctrl = phase->get_ctrl(node); - if (ctrl != start_ctrl) { - // We are in a successor block - the node is ok. - return false; // Unwind - } - - // Process successor nodes - int outcnt = node->outcnt(); - for (int i = 0; i < outcnt; i++) { - Node* n = node->raw_out(0); - assert(!n->is_LoadBarrier(), "Sanity"); - // Calling recursively, visiting leafs first - fixup_uses_in_catch(phase, start_ctrl, n); - } - - // Now all successors are outside - // - Clone this node to both successors - int no_succs = node->outcnt(); - assert(!node->is_Store(), "Stores not expected here"); - - // In some very rare cases a load that doesn't need a barrier will end up here - // Treat it as a LoadP and the insertion of phis will be done correctly. - if (node->is_Load()) { - assert(node->as_Load()->barrier_data() == 0, "Sanity"); - call_catch_cleanup_one(phase, node->as_Load(), phase->get_ctrl(node)); - } else { - for (DUIterator_Fast jmax, i = node->fast_outs(jmax); i < jmax; i++) { - Node* use = node->fast_out(i); - Node* clone = node->clone(); - assert(clone->outcnt() == 0, ""); - - assert(use->find_edge(node) != -1, "check"); - phase->igvn().rehash_node_delayed(use); - use->replace_edge(node, clone); - - Node* new_ctrl; - if (use->is_block_start()) { - new_ctrl = use; - } else if (use->is_CFG()) { - new_ctrl = use->in(0); - assert (new_ctrl != NULL, ""); - } else { - new_ctrl = phase->get_ctrl(use); - } - - phase->set_ctrl(clone, new_ctrl); - - if (phase->C->directive()->ZTraceLoadBarriersOption) tty->print_cr(" Clone op %i as %i to control %i", node->_idx, clone->_idx, new_ctrl->_idx); - phase->igvn().register_new_node_with_optimizer(clone); - --i, --jmax; - } - assert(node->outcnt() == 0, "must be empty now"); - - // Node node is dead. - phase->igvn().remove_dead_node(node); - } - return true; // unwind - return if a use was processed -} - -// Clone a load to a specific catch_proj -static Node* clone_load_to_catchproj(PhaseIdealLoop* phase, Node* load, Node* catch_proj) { - Node* cloned_load = load->clone(); - cloned_load->set_req(0, catch_proj); // set explicit control - phase->set_ctrl(cloned_load, catch_proj); // update - if (phase->C->directive()->ZTraceLoadBarriersOption) tty->print_cr(" Clone LOAD %i as %i to control %i", load->_idx, cloned_load->_idx, catch_proj->_idx); - phase->igvn().register_new_node_with_optimizer(cloned_load); - return cloned_load; -} - -static Node* get_dominating_region(PhaseIdealLoop* phase, Node* node, Node* stop) { - Node* region = node; - while (!region->isa_Region()) { - Node *up = phase->idom(region); - assert(up != region, "Must not loop"); - assert(up != stop, "Must not find original control"); - region = up; - } - return region; -} - -// Clone this load to each catch block -static void call_catch_cleanup_one(PhaseIdealLoop* phase, LoadNode* load, Node* ctrl) { - bool trace = phase->C->directive()->ZTraceLoadBarriersOption; - phase->igvn().set_delay_transform(true); - - // Verify pre conditions - assert(ctrl->isa_Proj() && ctrl->in(0)->isa_Call(), "Must be a call proj"); - assert(ctrl->raw_out(0)->isa_Catch(), "Must be a catch"); - - if (ctrl->raw_out(0)->isa_Catch()->outcnt() == 1) { - if (trace) tty->print_cr("Cleaning up catch: Skipping load %i, call with single catch", load->_idx); - return; - } + case Op_StoreP: + mem_ops.push(mach); + break; - // Process the loads successor nodes - if any is between - // the call and the catch blocks, they need to be cloned to. - // This is done recursively - int outcnt = load->outcnt(); - uint index = 0; - for (int i = 0; i < outcnt; i++) { - if (index < load->outcnt()) { - Node *n = load->raw_out(index); - assert(!n->is_LoadBarrier(), "Sanity"); - if (!fixup_uses_in_catch(phase, ctrl, n)) { - // if no successor was cloned, progress to next out. - index++; + default: + break; } } } - // Now all the loads uses has been cloned down - // Only thing left is to clone the loads, but they must end up - // first in the catch blocks. - - // We clone the loads oo the catch blocks only when needed. - // An array is used to map the catch blocks to each lazily cloned load. - // In that way no extra unnecessary loads are cloned. - - // Any use dominated by original block must have an phi and a region added - - Node* catch_node = ctrl->raw_out(0); - int number_of_catch_projs = catch_node->outcnt(); - Node** proj_to_load_mapping = NEW_RESOURCE_ARRAY(Node*, number_of_catch_projs); - Copy::zero_to_bytes(proj_to_load_mapping, sizeof(Node*) * number_of_catch_projs); - - // The phi_map is used to keep track of where phis have already been inserted - int phi_map_len = phase->C->unique(); - Node** phi_map = NEW_RESOURCE_ARRAY(Node*, phi_map_len); - Copy::zero_to_bytes(phi_map, sizeof(Node*) * phi_map_len); + // Step 2 - Find dominating accesses for each load + for (uint i = 0; i < barrier_loads.size(); i++) { + MachNode* const load = barrier_loads.at(i)->as_Mach(); + const TypePtr* load_adr_type = NULL; + intptr_t load_offset = 0; + const Node* const load_obj = load->get_base_and_disp(load_offset, load_adr_type); + Block* const load_block = cfg->get_block_for_node(load); + const uint load_index = block_index(load_block, load); - for (unsigned int i = 0; i < load->outcnt(); i++) { - Node* load_use_control = NULL; - Node* load_use = load->raw_out(i); + for (uint j = 0; j < mem_ops.size(); j++) { + MachNode* mem = mem_ops.at(j)->as_Mach(); + const TypePtr* mem_adr_type = NULL; + intptr_t mem_offset = 0; + const Node* mem_obj = mem_obj = mem->get_base_and_disp(mem_offset, mem_adr_type); + Block* mem_block = cfg->get_block_for_node(mem); + uint mem_index = block_index(mem_block, mem); - if (phase->has_ctrl(load_use)) { - load_use_control = phase->get_ctrl(load_use); - } else { - load_use_control = load_use->in(0); - } - assert(load_use_control != NULL, "sanity"); - if (trace) tty->print_cr(" Handling use: %i, with control: %i", load_use->_idx, load_use_control->_idx); - - // Some times the loads use is a phi. For them we need to determine from which catch block - // the use is defined. - bool load_use_is_phi = false; - unsigned int load_use_phi_index = 0; - Node* phi_ctrl = NULL; - if (load_use->is_Phi()) { - // Find phi input that matches load - for (unsigned int u = 1; u < load_use->req(); u++) { - if (load_use->in(u) == load) { - load_use_is_phi = true; - load_use_phi_index = u; - assert(load_use->in(0)->is_Region(), "Region or broken"); - phi_ctrl = load_use->in(0)->in(u); - assert(phi_ctrl->is_CFG(), "check"); - assert(phi_ctrl != load, "check"); - break; - } - } - assert(load_use_is_phi, "must find"); - assert(load_use_phi_index > 0, "sanity"); - } - - // For each load use, see which catch projs dominates, create load clone lazily and reconnect - bool found_dominating_catchproj = false; - for (int c = 0; c < number_of_catch_projs; c++) { - Node* catchproj = catch_node->raw_out(c); - assert(catchproj != NULL && catchproj->isa_CatchProj(), "Sanity"); - - if (!phase->is_dominator(catchproj, load_use_control)) { - if (load_use_is_phi && phase->is_dominator(catchproj, phi_ctrl)) { - // The loads use is local to the catchproj. - // fall out and replace load with catch-local load clone. - } else { - continue; - } - } - assert(!found_dominating_catchproj, "Max one should match"); - - // Clone loads to catch projs - Node* load_clone = proj_to_load_mapping[c]; - if (load_clone == NULL) { - load_clone = clone_load_to_catchproj(phase, load, catchproj); - proj_to_load_mapping[c] = load_clone; - } - phase->igvn().rehash_node_delayed(load_use); - - if (load_use_is_phi) { - // phis are special - the load is defined from a specific control flow - load_use->set_req(load_use_phi_index, load_clone); - } else { - // Multipe edges can be replaced at once - on calls for example - load_use->replace_edge(load, load_clone); + if (load_obj == NodeSentinel || mem_obj == NodeSentinel || + load_obj == NULL || mem_obj == NULL || + load_offset < 0 || mem_offset < 0) { + continue; } - --i; // more than one edge can have been removed, but the next is in later iterations - - // We could break the for-loop after finding a dominating match. - // But keep iterating to catch any bad idom early. - found_dominating_catchproj = true; - } - - // We found no single catchproj that dominated the use - The use is at a point after - // where control flow from multiple catch projs have merged. We will have to create - // phi nodes before the use and tie the output from the cloned loads together. It - // can be a single phi or a number of chained phis, depending on control flow - if (!found_dominating_catchproj) { - // Use phi-control if use is a phi - if (load_use_is_phi) { - load_use_control = phi_ctrl; + if (mem_obj != load_obj || mem_offset != load_offset) { + // Not the same addresses, not a candidate + continue; } - assert(phase->is_dominator(ctrl, load_use_control), "Common use but no dominator"); - // Clone a load on all paths - for (int c = 0; c < number_of_catch_projs; c++) { - Node* catchproj = catch_node->raw_out(c); - Node* load_clone = proj_to_load_mapping[c]; - if (load_clone == NULL) { - load_clone = clone_load_to_catchproj(phase, load, catchproj); - proj_to_load_mapping[c] = load_clone; + if (load_block == mem_block) { + // Earlier accesses in the same block + if (mem_index < load_index && !block_has_safepoint(mem_block, mem_index + 1, load_index)) { + load->set_barrier_data(ZLoadBarrierElided); } - } - - // Move up dominator tree from use until dom front is reached - Node* next_region = get_dominating_region(phase, load_use_control, ctrl); - while (phase->idom(next_region) != catch_node) { - next_region = phase->idom(next_region); - if (trace) tty->print_cr("Moving up idom to region ctrl %i", next_region->_idx); - } - assert(phase->is_dominator(catch_node, next_region), "Sanity"); - - // Create or reuse phi node that collect all cloned loads and feed it to the use. - Node* test_phi = phi_map[next_region->_idx]; - if ((test_phi != NULL) && test_phi->is_Phi()) { - // Reuse an already created phi - if (trace) tty->print_cr(" Using cached Phi %i on load_use %i", test_phi->_idx, load_use->_idx); - phase->igvn().rehash_node_delayed(load_use); - load_use->replace_edge(load, test_phi); - // Now this use is done - } else { - // Otherwise we need to create one or more phis - PhiNode* next_phi = new PhiNode(next_region, load->type()); - phi_map[next_region->_idx] = next_phi; // cache new phi - phase->igvn().rehash_node_delayed(load_use); - load_use->replace_edge(load, next_phi); - - int dominators_of_region = 0; - do { - // New phi, connect to region and add all loads as in. - Node* region = next_region; - assert(region->isa_Region() && region->req() > 2, "Catch dead region nodes"); - PhiNode* new_phi = next_phi; - - if (trace) tty->print_cr("Created Phi %i on load %i with control %i", new_phi->_idx, load->_idx, region->_idx); - - // Need to add all cloned loads to the phi, taking care that the right path is matched - dominators_of_region = 0; // reset for new region - for (unsigned int reg_i = 1; reg_i < region->req(); reg_i++) { - Node* region_pred = region->in(reg_i); - assert(region_pred->is_CFG(), "check"); - bool pred_has_dominator = false; - for (int c = 0; c < number_of_catch_projs; c++) { - Node* catchproj = catch_node->raw_out(c); - if (phase->is_dominator(catchproj, region_pred)) { - new_phi->set_req(reg_i, proj_to_load_mapping[c]); - if (trace) tty->print_cr(" - Phi in(%i) set to load %i", reg_i, proj_to_load_mapping[c]->_idx); - pred_has_dominator = true; - dominators_of_region++; - break; - } - } - - // Sometimes we need to chain several phis. - if (!pred_has_dominator) { - assert(dominators_of_region <= 1, "More than one region can't require extra phi"); - if (trace) tty->print_cr(" - Region %i pred %i not dominated by catch proj", region->_idx, region_pred->_idx); - // Continue search on on this region_pred - // - walk up to next region - // - create a new phi and connect to first new_phi - next_region = get_dominating_region(phase, region_pred, ctrl); - - // Lookup if there already is a phi, create a new otherwise - Node* test_phi = phi_map[next_region->_idx]; - if ((test_phi != NULL) && test_phi->is_Phi()) { - next_phi = test_phi->isa_Phi(); - dominators_of_region++; // record that a match was found and that we are done - if (trace) tty->print_cr(" Using cached phi Phi %i on control %i", next_phi->_idx, next_region->_idx); - } else { - next_phi = new PhiNode(next_region, load->type()); - phi_map[next_region->_idx] = next_phi; - } - new_phi->set_req(reg_i, next_phi); - } + } else if (mem_block->dominates(load_block)) { + // Dominating block? Look around for safepoints + ResourceMark rm; + Block_List stack; + VectorSet visited(Thread::current()->resource_area()); + stack.push(load_block); + bool safepoint_found = block_has_safepoint(load_block); + while (!safepoint_found && stack.size() > 0) { + Block* block = stack.pop(); + if (visited.test_set(block->_pre_order)) { + continue; + } + if (block_has_safepoint(block)) { + safepoint_found = true; + break; + } + if (block == mem_block) { + continue; } - new_phi->set_req(0, region); - phase->igvn().register_new_node_with_optimizer(new_phi); - phase->set_ctrl(new_phi, region); - - assert(dominators_of_region != 0, "Must have found one this iteration"); - } while (dominators_of_region == 1); - } - --i; - } - } // end of loop over uses - - assert(load->outcnt() == 0, "All uses should be handled"); - phase->igvn().remove_dead_node(load); - phase->C->print_method(PHASE_CALL_CATCH_CLEANUP, 4, load->_idx); - - // Now we should be home - phase->igvn().set_delay_transform(false); -} - -// Sort out the loads that are between a call ant its catch blocks -static void process_catch_cleanup_candidate(PhaseIdealLoop* phase, LoadNode* load) { - bool trace = phase->C->directive()->ZTraceLoadBarriersOption; - - Node* ctrl = phase->get_ctrl(load); - if (!ctrl->is_Proj() || (ctrl->in(0) == NULL) || !ctrl->in(0)->isa_Call()) { - return; - } - - Node* catch_node = ctrl->isa_Proj()->raw_out(0); - if (catch_node->is_Catch()) { - if (catch_node->outcnt() > 1) { - call_catch_cleanup_one(phase, load, ctrl); - } else { - if (trace) tty->print_cr("Call catch cleanup with only one catch: load %i ", load->_idx); - } - } -} - -void ZBarrierSetC2::barrier_insertion_phase(Compile* C, PhaseIterGVN& igvn) const { - ResourceMark rm; - PhaseIdealLoop v(igvn, false, false, false, true); - if (C->failing()) return; -} - -void ZBarrierSetC2::optimize_loops(PhaseIdealLoop* phase, VectorSet& visited, Node_Stack& nstack, Node_List& worklist) const { - // First make sure all loads between call and catch are moved to the catch block - clean_catch_blocks(phase); - - // Then expand barriers on all loads - insert_load_barriers(phase); - - // Handle all Unsafe that need barriers. - insert_barriers_on_unsafe(phase); - - phase->C->clear_major_progress(); -} - -static bool can_simplify_cas(LoadStoreNode* node) { - if (node->isa_LoadStoreConditional()) { - Node *expected_in = node->as_LoadStoreConditional()->in(LoadStoreConditionalNode::ExpectedIn); - return (expected_in->get_ptr_type() == TypePtr::NULL_PTR); - } else { - return false; - } -} - -static void insert_barrier_before_unsafe(PhaseIdealLoop* phase, LoadStoreNode* old_node) { - - Compile *C = phase->C; - PhaseIterGVN &igvn = phase->igvn(); - LoadStoreNode* zclone = NULL; - bool is_weak = false; - - Node *in_ctrl = old_node->in(MemNode::Control); - Node *in_mem = old_node->in(MemNode::Memory); - Node *in_adr = old_node->in(MemNode::Address); - Node *in_val = old_node->in(MemNode::ValueIn); - const TypePtr *adr_type = old_node->adr_type(); - const TypePtr* load_type = TypeOopPtr::BOTTOM; // The type for the load we are adding - - switch (old_node->Opcode()) { - case Op_CompareAndExchangeP: { - zclone = new ZCompareAndExchangePNode(in_ctrl, in_mem, in_adr, in_val, old_node->in(LoadStoreConditionalNode::ExpectedIn), - adr_type, old_node->get_ptr_type(), ((CompareAndExchangeNode*)old_node)->order()); - load_type = old_node->bottom_type()->is_ptr(); - break; - } - case Op_WeakCompareAndSwapP: { - if (can_simplify_cas(old_node)) { - break; - } - is_weak = true; - zclone = new ZWeakCompareAndSwapPNode(in_ctrl, in_mem, in_adr, in_val, old_node->in(LoadStoreConditionalNode::ExpectedIn), - ((CompareAndSwapNode*)old_node)->order()); - adr_type = TypePtr::BOTTOM; - break; - } - case Op_CompareAndSwapP: { - if (can_simplify_cas(old_node)) { - break; - } - zclone = new ZCompareAndSwapPNode(in_ctrl, in_mem, in_adr, in_val, old_node->in(LoadStoreConditionalNode::ExpectedIn), - ((CompareAndSwapNode*)old_node)->order()); - adr_type = TypePtr::BOTTOM; - break; - } - case Op_GetAndSetP: { - zclone = new ZGetAndSetPNode(in_ctrl, in_mem, in_adr, in_val, old_node->adr_type(), old_node->get_ptr_type()); - load_type = old_node->bottom_type()->is_ptr(); - break; - } - } - if (zclone != NULL) { - igvn.register_new_node_with_optimizer(zclone, old_node); - - // Make load - LoadPNode *load = new LoadPNode(NULL, in_mem, in_adr, adr_type, load_type, MemNode::unordered, - LoadNode::DependsOnlyOnTest); - load_set_expanded_barrier(load); - igvn.register_new_node_with_optimizer(load); - igvn.replace_node(old_node, zclone); - - Node *barrier = new LoadBarrierNode(C, NULL, in_mem, load, in_adr, is_weak); - Node *barrier_val = new ProjNode(barrier, LoadBarrierNode::Oop); - Node *barrier_ctrl = new ProjNode(barrier, LoadBarrierNode::Control); - - igvn.register_new_node_with_optimizer(barrier); - igvn.register_new_node_with_optimizer(barrier_val); - igvn.register_new_node_with_optimizer(barrier_ctrl); - - // loop over all of in_ctrl usages and move to barrier_ctrl - for (DUIterator_Last imin, i = in_ctrl->last_outs(imin); i >= imin; --i) { - Node *use = in_ctrl->last_out(i); - uint l; - for (l = 0; use->in(l) != in_ctrl; l++) {} - igvn.replace_input_of(use, l, barrier_ctrl); - } - - load->set_req(MemNode::Control, in_ctrl); - barrier->set_req(LoadBarrierNode::Control, in_ctrl); - zclone->add_req(barrier_val); // add req as keep alive. - - C->print_method(PHASE_ADD_UNSAFE_BARRIER, 4, zclone->_idx); - } -} - -void ZBarrierSetC2::insert_barriers_on_unsafe(PhaseIdealLoop* phase) const { - Compile *C = phase->C; - PhaseIterGVN &igvn = phase->igvn(); - uint new_ids = C->unique(); - VectorSet visited(Thread::current()->resource_area()); - GrowableArray<Node *> nodeStack(Thread::current()->resource_area(), 0, 0, NULL); - nodeStack.push(C->root()); - visited.test_set(C->root()->_idx); - - // Traverse all nodes, visit all unsafe ops that require a barrier - while (nodeStack.length() > 0) { - Node *n = nodeStack.pop(); - - bool is_old_node = (n->_idx < new_ids); // don't process nodes that were created during cleanup - if (is_old_node) { - if (n->is_LoadStore()) { - LoadStoreNode* lsn = n->as_LoadStore(); - if (lsn->has_barrier()) { - BasicType bt = lsn->in(MemNode::Address)->bottom_type()->basic_type(); - assert ((bt == T_OBJECT || bt == T_ARRAY), "Sanity test"); - insert_barrier_before_unsafe(phase, lsn); - } - } - } - for (uint i = 0; i < n->len(); i++) { - if (n->in(i)) { - if (!visited.test_set(n->in(i)->_idx)) { - nodeStack.push(n->in(i)); + // Push predecessor blocks + for (uint p = 1; p < block->num_preds(); ++p) { + Block* pred = cfg->get_block_for_node(block->pred(p)); + stack.push(pred); + } } - } - } - } - - igvn.optimize(); - C->print_method(PHASE_ADD_UNSAFE_BARRIER, 2); -} - -// The purpose of ZBarrierSetC2::clean_catch_blocks is to prepare the IR for -// splicing in load barrier nodes. -// -// The problem is that we might have instructions between a call and its catch nodes. -// (This is usually handled in PhaseCFG:call_catch_cleanup, which clones mach nodes in -// already scheduled blocks.) We can't have loads that require barriers there, -// because we need to splice in new control flow, and that would violate the IR. -// -// clean_catch_blocks find all Loads that require a barrier and clone them and any -// dependent instructions to each use. The loads must be in the beginning of the catch block -// before any store. -// -// Sometimes the loads use will be at a place dominated by all catch blocks, then we need -// a load in each catch block, and a Phi at the dominated use. - -void ZBarrierSetC2::clean_catch_blocks(PhaseIdealLoop* phase) const { - - Compile *C = phase->C; - uint new_ids = C->unique(); - PhaseIterGVN &igvn = phase->igvn(); - VectorSet visited(Thread::current()->resource_area()); - GrowableArray<Node *> nodeStack(Thread::current()->resource_area(), 0, 0, NULL); - nodeStack.push(C->root()); - visited.test_set(C->root()->_idx); - - // Traverse all nodes, visit all loads that require a barrier - while(nodeStack.length() > 0) { - Node *n = nodeStack.pop(); - - for (uint i = 0; i < n->len(); i++) { - if (n->in(i)) { - if (!visited.test_set(n->in(i)->_idx)) { - nodeStack.push(n->in(i)); + if (!safepoint_found) { + load->set_barrier_data(ZLoadBarrierElided); } } } - - bool is_old_node = (n->_idx < new_ids); // don't process nodes that were created during cleanup - if (n->is_Load() && is_old_node) { - LoadNode* load = n->isa_Load(); - // only care about loads that will have a barrier - if (load_require_barrier(load)) { - process_catch_cleanup_candidate(phase, load); - } - } } - - C->print_method(PHASE_CALL_CATCH_CLEANUP, 2); } -class DomDepthCompareClosure : public CompareClosure<LoadNode*> { - PhaseIdealLoop* _phase; - -public: - DomDepthCompareClosure(PhaseIdealLoop* phase) : _phase(phase) { } - - int do_compare(LoadNode* const &n1, LoadNode* const &n2) { - int d1 = _phase->dom_depth(_phase->get_ctrl(n1)); - int d2 = _phase->dom_depth(_phase->get_ctrl(n2)); - if (d1 == d2) { - // Compare index if the depth is the same, ensures all entries are unique. - return n1->_idx - n2->_idx; - } else { - return d2 - d1; - } - } -}; - -// Traverse graph and add all loadPs to list, sorted by dom depth -void gather_loadnodes_sorted(PhaseIdealLoop* phase, GrowableArray<LoadNode*>* loadList) { - - VectorSet visited(Thread::current()->resource_area()); - GrowableArray<Node *> nodeStack(Thread::current()->resource_area(), 0, 0, NULL); - DomDepthCompareClosure ddcc(phase); +// == Reduced spilling optimization == - nodeStack.push(phase->C->root()); - while(nodeStack.length() > 0) { - Node *n = nodeStack.pop(); - if (visited.test(n->_idx)) { - continue; - } - - if (n->isa_Load()) { - LoadNode *load = n->as_Load(); - if (load_require_barrier(load)) { - assert(phase->get_ctrl(load) != NULL, "sanity"); - assert(phase->dom_depth(phase->get_ctrl(load)) != 0, "sanity"); - loadList->insert_sorted(&ddcc, load); - } - } +void ZBarrierSetC2::compute_liveness_at_stubs() const { + ResourceMark rm; + Compile* const C = Compile::current(); + Arena* const A = Thread::current()->resource_area(); + PhaseCFG* const cfg = C->cfg(); + PhaseRegAlloc* const regalloc = C->regalloc(); + RegMask* const live = NEW_ARENA_ARRAY(A, RegMask, cfg->number_of_blocks() * sizeof(RegMask)); + ZBarrierSetAssembler* const bs = ZBarrierSet::assembler(); + Block_List worklist; + + for (uint i = 0; i < cfg->number_of_blocks(); ++i) { + new ((void*)(live + i)) RegMask(); + worklist.push(cfg->get_block(i)); + } + + while (worklist.size() > 0) { + const Block* const block = worklist.pop(); + RegMask& old_live = live[block->_pre_order]; + RegMask new_live; + + // Initialize to union of successors + for (uint i = 0; i < block->_num_succs; i++) { + const uint succ_id = block->_succs[i]->_pre_order; + new_live.OR(live[succ_id]); + } + + // Walk block backwards, computing liveness + for (int i = block->number_of_nodes() - 1; i >= 0; --i) { + const Node* const node = block->get_node(i); + + // Remove def bits + const OptoReg::Name first = bs->refine_register(node, regalloc->get_reg_first(node)); + const OptoReg::Name second = bs->refine_register(node, regalloc->get_reg_second(node)); + if (first != OptoReg::Bad) { + new_live.Remove(first); + } + if (second != OptoReg::Bad) { + new_live.Remove(second); + } + + // Add use bits + for (uint j = 1; j < node->req(); ++j) { + const Node* const use = node->in(j); + const OptoReg::Name first = bs->refine_register(use, regalloc->get_reg_first(use)); + const OptoReg::Name second = bs->refine_register(use, regalloc->get_reg_second(use)); + if (first != OptoReg::Bad) { + new_live.Insert(first); + } - visited.set(n->_idx); - for (uint i = 0; i < n->req(); i++) { - if (n->in(i)) { - if (!visited.test(n->in(i)->_idx)) { - nodeStack.push(n->in(i)); + if (second != OptoReg::Bad) { + new_live.Insert(second); } } - } - } -} - -// Add LoadBarriers to all LoadPs -void ZBarrierSetC2::insert_load_barriers(PhaseIdealLoop* phase) const { - bool trace = phase->C->directive()->ZTraceLoadBarriersOption; - GrowableArray<LoadNode *> loadList(Thread::current()->resource_area(), 0, 0, NULL); - gather_loadnodes_sorted(phase, &loadList); - PhaseIterGVN &igvn = phase->igvn(); - int count = 0; - - for (GrowableArrayIterator<LoadNode *> loadIter = loadList.begin(); loadIter != loadList.end(); ++loadIter) { - LoadNode *load = *loadIter; - if (load_has_expanded_barrier(load)) { - continue; - } - - do { - // Insert a barrier on a loadP - // if another load is found that needs to be expanded first, retry on that one - LoadNode* result = insert_one_loadbarrier(phase, load, phase->get_ctrl(load)); - while (result != NULL) { - result = insert_one_loadbarrier(phase, result, phase->get_ctrl(result)); + // If this node tracks liveness, update it + RegMask* const regs = barrier_set_state()->live(node); + if (regs != NULL) { + regs->OR(new_live); } - } while (!load_has_expanded_barrier(load)); - } - phase->C->print_method(PHASE_INSERT_BARRIER, 2); -} - -void push_antidependent_stores(PhaseIdealLoop* phase, Node_Stack& nodestack, LoadNode* start_load) { - // push all stores on the same mem, that can_alias - // Any load found must be handled first - PhaseIterGVN &igvn = phase->igvn(); - int load_alias_idx = igvn.C->get_alias_index(start_load->adr_type()); - - Node *mem = start_load->in(1); - for (DUIterator_Fast imax, u = mem->fast_outs(imax); u < imax; u++) { - Node *mem_use = mem->fast_out(u); - - if (mem_use == start_load) continue; - if (!mem_use->is_Store()) continue; - if (!phase->has_ctrl(mem_use)) continue; - if (phase->get_ctrl(mem_use) != phase->get_ctrl(start_load)) continue; - - // add any aliasing store in this block - StoreNode *store = mem_use->isa_Store(); - const TypePtr *adr_type = store->adr_type(); - if (igvn.C->can_alias(adr_type, load_alias_idx)) { - nodestack.push(store, 0); - } - } -} - -LoadNode* ZBarrierSetC2::insert_one_loadbarrier(PhaseIdealLoop* phase, LoadNode* start_load, Node* ctrl) const { - bool trace = phase->C->directive()->ZTraceLoadBarriersOption; - PhaseIterGVN &igvn = phase->igvn(); - - // Check for other loadPs at the same loop depth that is reachable by a DFS - // - if found - return it. It needs to be inserted first - // - otherwise proceed and insert barrier - - VectorSet visited(Thread::current()->resource_area()); - Node_Stack nodestack(100); - - nodestack.push(start_load, 0); - push_antidependent_stores(phase, nodestack, start_load); - - while(!nodestack.is_empty()) { - Node* n = nodestack.node(); // peek - nodestack.pop(); - if (visited.test(n->_idx)) { - continue; - } - - if (n->is_Load() && n != start_load && load_require_barrier(n->as_Load()) && !load_has_expanded_barrier(n->as_Load())) { - // Found another load that needs a barrier in the same block. Must expand later loads first. - if (trace) tty->print_cr(" * Found LoadP %i on DFS", n->_idx); - return n->as_Load(); // return node that should be expanded first } - if (!phase->has_ctrl(n)) continue; - if (phase->get_ctrl(n) != phase->get_ctrl(start_load)) continue; - if (n->is_Phi()) continue; - - visited.set(n->_idx); - // push all children - for (DUIterator_Fast imax, ii = n->fast_outs(imax); ii < imax; ii++) { - Node* c = n->fast_out(ii); - if (c != NULL) { - nodestack.push(c, 0); - } - } - } - insert_one_loadbarrier_inner(phase, start_load, ctrl, visited); - return NULL; -} - -void ZBarrierSetC2::insert_one_loadbarrier_inner(PhaseIdealLoop* phase, LoadNode* load, Node* ctrl, VectorSet visited2) const { - PhaseIterGVN &igvn = phase->igvn(); - Compile* C = igvn.C; - bool trace = C->directive()->ZTraceLoadBarriersOption; - - // create barrier - Node* barrier = new LoadBarrierNode(C, NULL, load->in(LoadNode::Memory), NULL, load->in(LoadNode::Address), load_has_weak_barrier(load)); - Node* barrier_val = new ProjNode(barrier, LoadBarrierNode::Oop); - Node* barrier_ctrl = new ProjNode(barrier, LoadBarrierNode::Control); - - if (trace) tty->print_cr("Insert load %i with barrier: %i and ctrl : %i", load->_idx, barrier->_idx, ctrl->_idx); - - // Splice control - // - insert barrier control diamond between loads ctrl and ctrl successor on path to block end. - // - If control successor is a catch, step over to next. - Node* ctrl_succ = NULL; - for (DUIterator_Fast imax, j = ctrl->fast_outs(imax); j < imax; j++) { - Node* tmp = ctrl->fast_out(j); - - // - CFG nodes is the ones we are going to splice (1 only!) - // - Phi nodes will continue to hang from the region node! - // - self loops should be skipped - if (tmp->is_Phi() || tmp == ctrl) { - continue; - } - - if (tmp->is_CFG()) { - assert(ctrl_succ == NULL, "There can be only one"); - ctrl_succ = tmp; - continue; - } - } - - // Now splice control - assert(ctrl_succ != load, "sanity"); - assert(ctrl_succ != NULL, "Broken IR"); - bool found = false; - for(uint k = 0; k < ctrl_succ->req(); k++) { - if (ctrl_succ->in(k) == ctrl) { - assert(!found, "sanity"); - if (trace) tty->print_cr(" Move CFG ctrl_succ %i to barrier_ctrl", ctrl_succ->_idx); - igvn.replace_input_of(ctrl_succ, k, barrier_ctrl); - found = true; - k--; - } - } - - // For all successors of ctrl - move all visited to become successors of barrier_ctrl instead - for (DUIterator_Fast imax, r = ctrl->fast_outs(imax); r < imax; r++) { - Node* tmp = ctrl->fast_out(r); - if (visited2.test(tmp->_idx) && (tmp != load)) { - if (trace) tty->print_cr(" Move ctrl_succ %i to barrier_ctrl", tmp->_idx); - igvn.replace_input_of(tmp, 0, barrier_ctrl); - --r; --imax; - } - } - - // Move the loads user to the barrier - for (DUIterator_Fast imax, i = load->fast_outs(imax); i < imax; i++) { - Node* u = load->fast_out(i); - if (u->isa_LoadBarrier()) { - continue; - } - - // find correct input - replace with iterator? - for(uint j = 0; j < u->req(); j++) { - if (u->in(j) == load) { - igvn.replace_input_of(u, j, barrier_val); - --i; --imax; // Adjust the iterator of the *outer* loop - break; // some nodes (calls) might have several uses from the same node + // Now at block top, see if we have any changes + new_live.SUBTRACT(old_live); + if (new_live.is_NotEmpty()) { + // Liveness has refined, update and propagate to prior blocks + old_live.OR(new_live); + for (uint i = 1; i < block->num_preds(); ++i) { + Block* const pred = cfg->get_block_for_node(block->pred(i)); + worklist.push(pred); } } } - - // Connect barrier to load and control - barrier->set_req(LoadBarrierNode::Oop, load); - barrier->set_req(LoadBarrierNode::Control, ctrl); - - igvn.rehash_node_delayed(load); - igvn.register_new_node_with_optimizer(barrier); - igvn.register_new_node_with_optimizer(barrier_val); - igvn.register_new_node_with_optimizer(barrier_ctrl); - load_set_expanded_barrier(load); - - C->print_method(PHASE_INSERT_BARRIER, 3, load->_idx); -} - -// The bad_mask in the ThreadLocalData shouldn't have an anti-dep-check. -// The bad_mask address if of type TypeRawPtr, but that will alias -// InitializeNodes until the type system is expanded. -bool ZBarrierSetC2::needs_anti_dependence_check(const Node* node) const { - MachNode* mnode = node->as_Mach(); - if (mnode != NULL) { - intptr_t offset = 0; - const TypePtr *adr_type2 = NULL; - const Node* base = mnode->get_base_and_disp(offset, adr_type2); - if ((base != NULL) && - (base->is_Mach() && base->as_Mach()->ideal_Opcode() == Op_ThreadLocal) && - (offset == in_bytes(ZThreadLocalData::address_bad_mask_offset()))) { - return false; - } - } - return true; } diff --git a/src/hotspot/share/gc/z/c2/zBarrierSetC2.hpp b/src/hotspot/share/gc/z/c2/zBarrierSetC2.hpp index bef6a7048..accae1f72 100644 --- a/src/hotspot/share/gc/z/c2/zBarrierSetC2.hpp +++ b/src/hotspot/share/gc/z/c2/zBarrierSetC2.hpp @@ -29,134 +29,38 @@ #include "opto/node.hpp" #include "utilities/growableArray.hpp" -class ZCompareAndSwapPNode : public CompareAndSwapPNode { -public: - ZCompareAndSwapPNode(Node* c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : CompareAndSwapPNode(c, mem, adr, val, ex, mem_ord) { } - virtual int Opcode() const; -}; - -class ZWeakCompareAndSwapPNode : public WeakCompareAndSwapPNode { -public: - ZWeakCompareAndSwapPNode(Node* c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : WeakCompareAndSwapPNode(c, mem, adr, val, ex, mem_ord) { } - virtual int Opcode() const; -}; - -class ZCompareAndExchangePNode : public CompareAndExchangePNode { -public: - ZCompareAndExchangePNode(Node* c, Node *mem, Node *adr, Node *val, Node *ex, const TypePtr* at, const Type* t, MemNode::MemOrd mem_ord) : CompareAndExchangePNode(c, mem, adr, val, ex, at, t, mem_ord) { } - virtual int Opcode() const; -}; - -class ZGetAndSetPNode : public GetAndSetPNode { -public: - ZGetAndSetPNode(Node* c, Node *mem, Node *adr, Node *val, const TypePtr* at, const Type* t) : GetAndSetPNode(c, mem, adr, val, at, t) { } - virtual int Opcode() const; -}; - -class LoadBarrierNode : public MultiNode { -private: - bool _weak; - - static bool is_dominator(PhaseIdealLoop* phase, bool linear_only, Node *d, Node *n); - void push_dominated_barriers(PhaseIterGVN* igvn) const; - -public: - enum { - Control, - Memory, - Oop, - Address, - Number_of_Outputs = Address, - Similar, - Number_of_Inputs - }; - - LoadBarrierNode(Compile* C, - Node* c, - Node* mem, - Node* val, - Node* adr, - bool weak); - - virtual int Opcode() const; - virtual uint size_of() const; - virtual uint cmp(const Node& n) const; - virtual const Type *bottom_type() const; - virtual const TypePtr* adr_type() const; - virtual const Type *Value(PhaseGVN *phase) const; - virtual Node *Identity(PhaseGVN *phase); - virtual Node *Ideal(PhaseGVN *phase, bool can_reshape); - virtual uint match_edge(uint idx) const; - - LoadBarrierNode* has_dominating_barrier(PhaseIdealLoop* phase, - bool linear_only, - bool look_for_similar); - - void fix_similar_in_uses(PhaseIterGVN* igvn); +const uint8_t ZLoadBarrierStrong = 1; +const uint8_t ZLoadBarrierWeak = 2; +const uint8_t ZLoadBarrierElided = 3; - bool has_true_uses() const; - - bool can_be_eliminated() const { - return !in(Similar)->is_top(); - } - - bool is_weak() const { - return _weak; - } -}; - -class LoadBarrierSlowRegNode : public LoadPNode { -private: - bool _is_weak; -public: - LoadBarrierSlowRegNode(Node *c, - Node *mem, - Node *adr, - const TypePtr *at, - const TypePtr* t, - MemOrd mo, - bool weak = false, - ControlDependency control_dependency = DependsOnlyOnTest) - : LoadPNode(c, mem, adr, at, t, mo, control_dependency), _is_weak(weak) { - init_class_id(Class_LoadBarrierSlowReg); - } - - virtual const char * name() { - return "LoadBarrierSlowRegNode"; - } - - virtual Node *Ideal(PhaseGVN *phase, bool can_reshape) { - return NULL; - } - - virtual int Opcode() const; - bool is_weak() { return _is_weak; } -}; - -class ZBarrierSetC2State : public ResourceObj { +class ZLoadBarrierStubC2 : public ResourceObj { private: - // List of load barrier nodes which need to be expanded before matching - GrowableArray<LoadBarrierNode*>* _load_barrier_nodes; - + const MachNode* _node; + const Address _ref_addr; + const Register _ref; + const Register _tmp; + const bool _weak; + Label _entry; + Label _continuation; + + ZLoadBarrierStubC2(const MachNode* node, Address ref_addr, Register ref, Register tmp, bool weak); + public: - ZBarrierSetC2State(Arena* comp_arena); - int load_barrier_count() const; - void add_load_barrier_node(LoadBarrierNode* n); - void remove_load_barrier_node(LoadBarrierNode* n); - LoadBarrierNode* load_barrier_node(int idx) const; -}; - -enum BarrierInfo { - NoBarrier = 0, - RequireBarrier = 1, - WeakBarrier = 3, // Inclusive with RequireBarrier - ExpandedBarrier = 4 + static ZLoadBarrierStubC2* create(const MachNode* node, Address ref_addr, Register ref, Register tmp, bool weak); + + Address ref_addr() const; + Register ref() const; + Register tmp() const; + address slow_path() const; + RegMask& live() const; + Label* entry(); + Label* continuation(); }; class ZBarrierSetC2 : public BarrierSetC2 { private: - ZBarrierSetC2State* state() const; - void expand_loadbarrier_node(PhaseMacroExpand* phase, LoadBarrierNode* barrier) const; + void compute_liveness_at_stubs() const; + void analyze_dominating_barriers() const; protected: virtual Node* load_at_resolved(C2Access& access, const Type* val_type) const; @@ -174,38 +78,11 @@ protected: public: virtual void* create_barrier_state(Arena* comp_arena) const; - virtual bool has_load_barriers() const { return true; } - virtual bool is_gc_barrier_node(Node* node) const; - - virtual void register_potential_barrier_node(Node* node) const; - virtual void unregister_potential_barrier_node(Node* node) const; - virtual void eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const { } - virtual void enqueue_useful_gc_barrier(Unique_Node_List &worklist, Node* node) const; - virtual void eliminate_useless_gc_barriers(Unique_Node_List &useful) const; - - virtual bool array_copy_requires_gc_barriers(BasicType type) const { return true; } - virtual Node* step_over_gc_barrier(Node* c) const; - // If the BarrierSetC2 state has kept macro nodes in its compilation unit state to be - // expanded later, then now is the time to do so. - virtual bool expand_macro_nodes(PhaseMacroExpand* macro) const; - virtual bool needs_anti_dependence_check(const Node* node) const; - -#ifdef ASSERT - virtual void verify_gc_barriers(bool post_parse) const; -#endif - - // Load barrier insertion and expansion external - virtual void barrier_insertion_phase(Compile* C, PhaseIterGVN &igvn) const; - virtual void optimize_loops(PhaseIdealLoop* phase, VectorSet& visited, Node_Stack& nstack, Node_List& worklist) const; - -private: - // Load barrier insertion and expansion internal - void insert_barriers_on_unsafe(PhaseIdealLoop* phase) const; - void clean_catch_blocks(PhaseIdealLoop* phase) const; - void insert_load_barriers(PhaseIdealLoop* phase) const; - LoadNode* insert_one_loadbarrier(PhaseIdealLoop* phase, LoadNode* load, Node* ctrl) const; - void insert_one_loadbarrier_inner(PhaseIdealLoop* phase, LoadNode* load, Node* ctrl, VectorSet visited) const; + virtual bool array_copy_requires_gc_barriers(BasicType type) const; + virtual void late_barrier_analysis() const; + virtual int estimate_stub_size() const; + virtual void emit_stubs(CodeBuffer& cb) const; }; #endif // SHARE_GC_Z_C2_ZBARRIERSETC2_HPP diff --git a/src/hotspot/share/gc/z/zBarrierSetAssembler.hpp b/src/hotspot/share/gc/z/zBarrierSetAssembler.hpp index 52133c073..9d07f9e8c 100644 --- a/src/hotspot/share/gc/z/zBarrierSetAssembler.hpp +++ b/src/hotspot/share/gc/z/zBarrierSetAssembler.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -24,10 +24,7 @@ #ifndef SHARE_GC_Z_ZBARRIERSETASSEMBLER_HPP #define SHARE_GC_Z_ZBARRIERSETASSEMBLER_HPP -#include "asm/macroAssembler.hpp" #include "gc/shared/barrierSetAssembler.hpp" -#include "oops/accessDecorators.hpp" -#include "utilities/globalDefinitions.hpp" #include "utilities/macros.hpp" class ZBarrierSetAssemblerBase : public BarrierSetAssembler { diff --git a/src/hotspot/share/gc/z/zGlobals.hpp b/src/hotspot/share/gc/z/zGlobals.hpp index 080ea5c0e..0f9e9dcb4 100644 --- a/src/hotspot/share/gc/z/zGlobals.hpp +++ b/src/hotspot/share/gc/z/zGlobals.hpp @@ -117,11 +117,8 @@ extern uintptr_t ZAddressWeakBadMask; // Marked state extern uintptr_t ZAddressMetadataMarked; -// Address space for mark stack allocations -const size_t ZMarkStackSpaceSizeShift = 40; // 1TB -const size_t ZMarkStackSpaceSize = (size_t)1 << ZMarkStackSpaceSizeShift; -const uintptr_t ZMarkStackSpaceStart = ZAddressSpaceEnd + ZMarkStackSpaceSize; -const uintptr_t ZMarkStackSpaceEnd = ZMarkStackSpaceStart + ZMarkStackSpaceSize; +// Mark stack space +extern uintptr_t ZMarkStackSpaceStart; const size_t ZMarkStackSpaceExpandSize = (size_t)1 << 25; // 32M // Mark stack and magazine sizes diff --git a/src/hotspot/share/gc/z/zHeap.cpp b/src/hotspot/share/gc/z/zHeap.cpp index ff08a0759..7f0f0b0de 100644 --- a/src/hotspot/share/gc/z/zHeap.cpp +++ b/src/hotspot/share/gc/z/zHeap.cpp @@ -49,6 +49,7 @@ #include "runtime/thread.hpp" #include "utilities/align.hpp" #include "utilities/debug.hpp" +#include "prims/resolvedMethodTable.hpp" static const ZStatSampler ZSamplerHeapUsedBeforeMark("Memory", "Heap Used Before Mark", ZStatUnitBytes); static const ZStatSampler ZSamplerHeapUsedAfterMark("Memory", "Heap Used After Mark", ZStatUnitBytes); @@ -334,6 +335,10 @@ bool ZHeap::mark_end() { Universe::verify(); } + // Free unsed entries of ResolvedMethodTable and weakhandles + // avoid ResolvedMethodTable inflation and native memory leak + ResolvedMethodTable::unlink(); + return true; } diff --git a/src/hotspot/share/gc/z/zLiveMap.cpp b/src/hotspot/share/gc/z/zLiveMap.cpp index 7187b6166..c1d79b794 100644 --- a/src/hotspot/share/gc/z/zLiveMap.cpp +++ b/src/hotspot/share/gc/z/zLiveMap.cpp @@ -50,7 +50,9 @@ void ZLiveMap::reset(size_t index) { // Multiple threads can enter here, make sure only one of them // resets the marking information while the others busy wait. - for (uint32_t seqnum = _seqnum; seqnum != ZGlobalSeqNum; seqnum = _seqnum) { + for (uint32_t seqnum = OrderAccess::load_acquire(&_seqnum); + seqnum != ZGlobalSeqNum; + seqnum = OrderAccess::load_acquire(&_seqnum)) { if ((seqnum != seqnum_initializing) && (Atomic::cmpxchg(seqnum_initializing, &_seqnum, seqnum) == seqnum)) { // Reset marking information @@ -61,13 +63,13 @@ void ZLiveMap::reset(size_t index) { segment_live_bits().clear(); segment_claim_bits().clear(); - // Make sure the newly reset marking information is - // globally visible before updating the page seqnum. - OrderAccess::storestore(); - - // Update seqnum assert(_seqnum == seqnum_initializing, "Invalid"); - _seqnum = ZGlobalSeqNum; + + // Make sure the newly reset marking information is ordered + // before the update of the page seqnum, such that when the + // up-to-date seqnum is load acquired, the bit maps will not + // contain stale information. + OrderAccess::release_store(&_seqnum, ZGlobalSeqNum); break; } @@ -89,10 +91,6 @@ void ZLiveMap::reset_segment(BitMap::idx_t segment) { if (!claim_segment(segment)) { // Already claimed, wait for live bit to be set while (!is_segment_live(segment)) { - // Busy wait. The loadload barrier is needed to make - // sure we re-read the live bit every time we loop. - OrderAccess::loadload(); - // Mark reset contention if (!contention) { // Count contention once diff --git a/src/hotspot/share/gc/z/zLiveMap.inline.hpp b/src/hotspot/share/gc/z/zLiveMap.inline.hpp index 1e4d56f41..fb45a892c 100644 --- a/src/hotspot/share/gc/z/zLiveMap.inline.hpp +++ b/src/hotspot/share/gc/z/zLiveMap.inline.hpp @@ -30,6 +30,7 @@ #include "gc/z/zOop.inline.hpp" #include "gc/z/zUtils.inline.hpp" #include "runtime/atomic.hpp" +#include "runtime/orderAccess.hpp" #include "utilities/bitMap.inline.hpp" #include "utilities/debug.hpp" @@ -38,7 +39,7 @@ inline void ZLiveMap::reset() { } inline bool ZLiveMap::is_marked() const { - return _seqnum == ZGlobalSeqNum; + return OrderAccess::load_acquire(&_seqnum) == ZGlobalSeqNum; } inline uint32_t ZLiveMap::live_objects() const { @@ -68,15 +69,15 @@ inline BitMapView ZLiveMap::segment_claim_bits() { } inline bool ZLiveMap::is_segment_live(BitMap::idx_t segment) const { - return segment_live_bits().at(segment); + return segment_live_bits().par_at(segment); } inline bool ZLiveMap::set_segment_live_atomic(BitMap::idx_t segment) { - return segment_live_bits().par_set_bit(segment); + return segment_live_bits().par_set_bit(segment, memory_order_release); } inline bool ZLiveMap::claim_segment(BitMap::idx_t segment) { - return segment_claim_bits().par_set_bit(segment); + return segment_claim_bits().par_set_bit(segment, memory_order_acq_rel); } inline BitMap::idx_t ZLiveMap::first_live_segment() const { diff --git a/src/hotspot/share/gc/z/zMarkStack.cpp b/src/hotspot/share/gc/z/zMarkStack.cpp index 52fe51ece..9cc768956 100644 --- a/src/hotspot/share/gc/z/zMarkStack.cpp +++ b/src/hotspot/share/gc/z/zMarkStack.cpp @@ -28,58 +28,44 @@ #include "gc/z/zMarkStack.inline.hpp" #include "logging/log.hpp" #include "runtime/atomic.hpp" +#include "runtime/os.hpp" #include "utilities/debug.hpp" -#include <sys/mman.h> -#include <sys/types.h> +uintptr_t ZMarkStackSpaceStart; ZMarkStackSpace::ZMarkStackSpace() : _expand_lock(), + _start(0), _top(0), _end(0) { - assert(ZMarkStacksMax >= ZMarkStackSpaceExpandSize, "ZMarkStacksMax too small"); - assert(ZMarkStacksMax <= ZMarkStackSpaceSize, "ZMarkStacksMax too large"); - + assert(ZMarkStackSpaceLimit >= ZMarkStackSpaceExpandSize, "ZMarkStackSpaceLimit too small"); // Reserve address space - const void* res = mmap((void*)ZMarkStackSpaceStart, ZMarkStackSpaceSize, - PROT_NONE, MAP_ANONYMOUS|MAP_PRIVATE|MAP_NORESERVE, -1, 0); - if (res != (void*)ZMarkStackSpaceStart) { - log_error(gc, marking)("Failed to reserve address space for marking stacks"); + const size_t size = ZMarkStackSpaceLimit; + const size_t alignment = (size_t)os::vm_allocation_granularity(); + const uintptr_t addr = (uintptr_t)os::reserve_memory(size, NULL, alignment, mtGC); + if (addr == 0) { + log_error(gc, marking)("Failed to reserve address space for mark stacks"); return; } // Successfully initialized - _top = _end = ZMarkStackSpaceStart; -} + _start = _top = _end = addr; -bool ZMarkStackSpace::is_initialized() const { - return _top != 0; + // Register mark stack space start + ZMarkStackSpaceStart = _start; } -bool ZMarkStackSpace::expand() { - const size_t max = ZMarkStackSpaceStart + ZMarkStacksMax; - if (_end + ZMarkStackSpaceExpandSize > max) { - // Expansion limit reached - return false; - } - - void* const res = mmap((void*)_end, ZMarkStackSpaceExpandSize, - PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE|MAP_FIXED, -1, 0); - if (res == MAP_FAILED) { - ZErrno err; - log_error(gc, marking)("Failed to map memory for marking stacks (%s)", err.to_string()); - return false; - } - - return true; +bool ZMarkStackSpace::is_initialized() const { + return _start != 0; } uintptr_t ZMarkStackSpace::alloc_space(size_t size) { - uintptr_t top = _top; + uintptr_t top = Atomic::load(&_top); for (;;) { + const uintptr_t end = Atomic::load(&_end); const uintptr_t new_top = top + size; - if (new_top > _end) { + if (new_top > end) { // Not enough space left return 0; } @@ -104,24 +90,28 @@ uintptr_t ZMarkStackSpace::expand_and_alloc_space(size_t size) { return addr; } - // Expand stack space - if (!expand()) { - // We currently can't handle the situation where we - // are running out of mark stack space. - fatal("Mark stack overflow (allocated " SIZE_FORMAT "M, size " SIZE_FORMAT "M, max " SIZE_FORMAT "M)," - " use -XX:ZMarkStacksMax=? to increase this limit", - (_end - ZMarkStackSpaceStart) / M, size / M, ZMarkStacksMax / M); - return 0; + // Check expansion limit + const size_t expand_size = ZMarkStackSpaceExpandSize; + const size_t old_size = _end - _start; + const size_t new_size = old_size + expand_size; + if (new_size > ZMarkStackSpaceLimit) { + // Expansion limit reached. This is a fatal error since we + // currently can't recover from running out of mark stack space. + fatal("Mark stack space exhausted. Use -XX:ZMarkStackSpaceLimit=<size> to increase the " + "maximum number of bytes allocated for mark stacks. Current limit is " SIZE_FORMAT "M.", + ZMarkStackSpaceLimit / M); } log_debug(gc, marking)("Expanding mark stack space: " SIZE_FORMAT "M->" SIZE_FORMAT "M", - (_end - ZMarkStackSpaceStart) / M, - (_end - ZMarkStackSpaceStart + ZMarkStackSpaceExpandSize) / M); + old_size / M, new_size / M); + + // Expand + os::commit_memory_or_exit((char*)_end, expand_size, false /* executable */, "Mark stack space"); // Increment top before end to make sure another // thread can't steal out newly expanded space. addr = Atomic::add(size, &_top) - size; - _end += ZMarkStackSpaceExpandSize; + Atomic::add(expand_size, &_end); return addr; } diff --git a/src/hotspot/share/gc/z/zMarkStack.hpp b/src/hotspot/share/gc/z/zMarkStack.hpp index b68b9faa3..12f3e4eca 100644 --- a/src/hotspot/share/gc/z/zMarkStack.hpp +++ b/src/hotspot/share/gc/z/zMarkStack.hpp @@ -76,6 +76,7 @@ typedef ZStackList<ZMarkStackMagazine> ZMarkStackMagazineList; class ZMarkStackSpace { private: ZLock _expand_lock; + uintptr_t _start; volatile uintptr_t _top; volatile uintptr_t _end; diff --git a/src/hotspot/share/gc/z/zWorkers.cpp b/src/hotspot/share/gc/z/zWorkers.cpp index 0686ec7af..6a0c2561d 100644 --- a/src/hotspot/share/gc/z/zWorkers.cpp +++ b/src/hotspot/share/gc/z/zWorkers.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -22,14 +22,27 @@ */ #include "precompiled.hpp" +#include "gc/z/zGlobals.hpp" #include "gc/z/zTask.hpp" #include "gc/z/zWorkers.inline.hpp" #include "runtime/os.hpp" #include "runtime/mutexLocker.hpp" #include "runtime/safepoint.hpp" -uint ZWorkers::calculate_ncpus(double share_in_percent) { - return ceil(os::initial_active_processor_count() * share_in_percent / 100.0); +static uint calculate_nworkers_based_on_ncpus(double cpu_share_in_percent) { + return ceil(os::initial_active_processor_count() * cpu_share_in_percent / 100.0); +} + +static uint calculate_nworkers_based_on_heap_size(double reserve_share_in_percent) { + const int nworkers = ((MaxHeapSize * (reserve_share_in_percent / 100.0)) - ZPageSizeMedium) / ZPageSizeSmall; + return MAX2(nworkers, 1); +} + +static uint calculate_nworkers(double cpu_share_in_percent) { + // Cap number of workers so that we never use more than 10% of the max heap + // for the reserve. This is useful when using small heaps on large machines. + return MIN2(calculate_nworkers_based_on_ncpus(cpu_share_in_percent), + calculate_nworkers_based_on_heap_size(10.0)); } uint ZWorkers::calculate_nparallel() { @@ -38,7 +51,7 @@ uint ZWorkers::calculate_nparallel() { // close to the number of processors tends to lead to over-provisioning and // scheduling latency issues. Using 60% of the active processors appears to // be a fairly good balance. - return calculate_ncpus(60.0); + return calculate_nworkers(60.0); } uint ZWorkers::calculate_nconcurrent() { @@ -48,7 +61,7 @@ uint ZWorkers::calculate_nconcurrent() { // throughput, while using too few threads will prolong the GC-cycle and // we then risk being out-run by the application. Using 12.5% of the active // processors appears to be a fairly good balance. - return calculate_ncpus(12.5); + return calculate_nworkers(12.5); } class ZWorkersWarmupTask : public ZTask { diff --git a/src/hotspot/share/gc/z/zWorkers.hpp b/src/hotspot/share/gc/z/zWorkers.hpp index 8bd072ed4..663a5763b 100644 --- a/src/hotspot/share/gc/z/zWorkers.hpp +++ b/src/hotspot/share/gc/z/zWorkers.hpp @@ -35,8 +35,6 @@ private: bool _boost; WorkGang _workers; - static uint calculate_ncpus(double share_in_percent); - void run(ZTask* task, uint nworkers); public: diff --git a/src/hotspot/share/gc/z/z_globals.hpp b/src/hotspot/share/gc/z/z_globals.hpp index 9e0f8985b..8cee59be7 100644 --- a/src/hotspot/share/gc/z/z_globals.hpp +++ b/src/hotspot/share/gc/z/z_globals.hpp @@ -53,9 +53,9 @@ "Allow Java threads to stall and wait for GC to complete " \ "instead of immediately throwing an OutOfMemoryError") \ \ - product(size_t, ZMarkStacksMax, NOT_LP64(512*M) LP64_ONLY(8*G), \ - "Maximum number of bytes allocated for marking stacks") \ - range(32*M, NOT_LP64(512*M) LP64_ONLY(1024*G)) \ + product(size_t, ZMarkStackSpaceLimit, 8*G, \ + "Maximum number of bytes allocated for mark stacks") \ + range(32*M, 1024*G) \ \ product(uint, ZCollectionInterval, 0, \ "Force GC at a fixed time interval (in seconds)") \ diff --git a/src/hotspot/share/opto/c2compiler.cpp b/src/hotspot/share/opto/c2compiler.cpp index 7768615b7..5a842e31f 100644 --- a/src/hotspot/share/opto/c2compiler.cpp +++ b/src/hotspot/share/opto/c2compiler.cpp @@ -658,6 +658,7 @@ bool C2Compiler::is_intrinsic_supported(const methodHandle& method, bool is_virt case vmIntrinsics::_profileBoolean: case vmIntrinsics::_isCompileConstant: case vmIntrinsics::_Preconditions_checkIndex: + case vmIntrinsics::_nextInt: break; default: return false; diff --git a/src/hotspot/share/opto/classes.cpp b/src/hotspot/share/opto/classes.cpp index 75f070f7c..d1282ac78 100644 --- a/src/hotspot/share/opto/classes.cpp +++ b/src/hotspot/share/opto/classes.cpp @@ -48,9 +48,6 @@ #include "opto/subnode.hpp" #include "opto/vectornode.hpp" #include "utilities/macros.hpp" -#if INCLUDE_ZGC -#include "gc/z/c2/zBarrierSetC2.hpp" -#endif #if INCLUDE_SHENANDOAHGC #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp" #endif diff --git a/src/hotspot/share/opto/classes.hpp b/src/hotspot/share/opto/classes.hpp index c3a6dc55a..bffb5d1d6 100644 --- a/src/hotspot/share/opto/classes.hpp +++ b/src/hotspot/share/opto/classes.hpp @@ -189,17 +189,6 @@ macro(LoadP) macro(LoadN) macro(LoadRange) macro(LoadS) -#if INCLUDE_ZGC -#define zgcmacro(x) macro(x) -#else -#define zgcmacro(x) optionalmacro(x) -#endif -zgcmacro(LoadBarrier) -zgcmacro(LoadBarrierSlowReg) -zgcmacro(ZCompareAndSwapP) -zgcmacro(ZWeakCompareAndSwapP) -zgcmacro(ZCompareAndExchangeP) -zgcmacro(ZGetAndSetP) macro(Lock) macro(Loop) macro(LoopLimit) diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp index 7e743ee64..43c1dcbf9 100644 --- a/src/hotspot/share/opto/compile.cpp +++ b/src/hotspot/share/opto/compile.cpp @@ -79,9 +79,6 @@ #if INCLUDE_G1GC #include "gc/g1/g1ThreadLocalData.hpp" #endif // INCLUDE_G1GC -#if INCLUDE_ZGC -#include "gc/z/c2/zBarrierSetC2.hpp" -#endif #if INCLUDE_SHENANDOAHGC #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp" #endif @@ -647,9 +644,6 @@ Compile::Compile( ciEnv* ci_env, C2Compiler* compiler, ciMethod* target, int osr bool subsume_loads, bool do_escape_analysis, bool eliminate_boxing, bool do_locks_coarsening, DirectiveSet* directive) : Phase(Compiler), - _env(ci_env), - _directive(directive), - _log(ci_env->log()), _compile_id(ci_env->compile_id()), _save_argument_registers(false), _do_locks_coarsening(do_locks_coarsening), @@ -657,7 +651,6 @@ Compile::Compile( ciEnv* ci_env, C2Compiler* compiler, ciMethod* target, int osr _stub_function(NULL), _stub_entry_point(NULL), _method(target), - _barrier_set_state(BarrierSet::barrier_set()->barrier_set_c2()->create_barrier_state(comp_arena())), _entry_bci(osr_bci), _initial_gvn(NULL), _for_igvn(NULL), @@ -678,8 +671,6 @@ Compile::Compile( ciEnv* ci_env, C2Compiler* compiler, ciMethod* target, int osr _scratch_const_size(-1), _in_scratch_emit_size(false), _coarsened_locks (comp_arena(), 8, 0, NULL), - _dead_node_list(comp_arena()), - _dead_node_count(0), #ifndef PRODUCT _trace_opto_output(directive->TraceOptoOutputOption), _in_dump_cnt(0), @@ -687,9 +678,15 @@ Compile::Compile( ciEnv* ci_env, C2Compiler* compiler, ciMethod* target, int osr NOT_PRODUCT(_printer(NULL) COMMA) _congraph(NULL), _comp_arena(mtCompiler), + _barrier_set_state(BarrierSet::barrier_set()->barrier_set_c2()->create_barrier_state(comp_arena())), + _env(ci_env), + _directive(directive), + _log(ci_env->log()), _node_arena(mtCompiler), _old_arena(mtCompiler), _Compile_types(mtCompiler), + _dead_node_list(comp_arena()), + _dead_node_count(0), _replay_inline_data(NULL), _late_inlines(comp_arena(), 2, 0, NULL), _string_late_inlines(comp_arena(), 2, 0, NULL), @@ -967,9 +964,6 @@ Compile::Compile( ciEnv* ci_env, bool return_pc, DirectiveSet* directive) : Phase(Compiler), - _env(ci_env), - _directive(directive), - _log(ci_env->log()), _compile_id(0), _save_argument_registers(save_arg_registers), _do_locks_coarsening(false), @@ -999,6 +993,10 @@ Compile::Compile( ciEnv* ci_env, _in_dump_cnt(0), #endif NOT_PRODUCT(_printer(NULL) COMMA) + _barrier_set_state(BarrierSet::barrier_set()->barrier_set_c2()->create_barrier_state(comp_arena())), + _env(ci_env), + _directive(directive), + _log(ci_env->log()), _comp_arena(mtCompiler), _node_arena(mtCompiler), _old_arena(mtCompiler), @@ -2420,13 +2418,6 @@ void Compile::Optimize() { igvn.optimize(); } -#ifdef ASSERT - bs->verify_gc_barriers(false); -#endif - - bs->barrier_insertion_phase(C, igvn); - if (failing()) return; - #ifdef ASSERT bs->verify_gc_barriers(false); #endif @@ -3016,29 +3007,6 @@ void Compile::final_graph_reshaping_impl( Node *n, Final_Reshape_Counts &frc) { break; } -#if INCLUDE_ZGC - case Op_LoadBarrier: { - assert(0, "There should be no load barriers left"); - } - case Op_ZGetAndSetP: - case Op_ZCompareAndExchangeP: - case Op_ZCompareAndSwapP: - case Op_ZWeakCompareAndSwapP: - case Op_LoadBarrierSlowReg: { -#ifdef ASSERT - if (VerifyOptoOopOffsets) { - MemNode *mem = n->as_Mem(); - // Check to see if address types have grounded out somehow. - const TypeInstPtr *tp = mem->in(MemNode::Address)->bottom_type()->isa_instptr(); - ciInstanceKlass *k = tp->klass()->as_instance_klass(); - bool oop_offset_is_sane = k->contains_field_offset(tp->offset()); - assert(!tp || oop_offset_is_sane, ""); - } -#endif - break; - } -#endif - case Op_AddP: { // Assert sane base pointers Node *addp = n->in(AddPNode::Address); assert( !addp->is_AddP() || diff --git a/src/hotspot/share/opto/compile.hpp b/src/hotspot/share/opto/compile.hpp index a0ec7d496..0f51a0025 100644 --- a/src/hotspot/share/opto/compile.hpp +++ b/src/hotspot/share/opto/compile.hpp @@ -56,7 +56,6 @@ class ConnectionGraph; class IdealGraphPrinter; class InlineTree; class Int_Array; -class LoadBarrierNode; class Matcher; class MachConstantNode; class MachConstantBaseNode; @@ -1212,11 +1211,7 @@ class Compile : public Phase { bool in_scratch_emit_size() const { return _in_scratch_emit_size; } enum ScratchBufferBlob { -#if defined(PPC64) MAX_inst_size = 2048, -#else - MAX_inst_size = 1024, -#endif MAX_locs_size = 128, // number of relocInfo elements MAX_const_size = 128, MAX_stubs_size = 128 @@ -1292,14 +1287,30 @@ class Compile : public Phase { // Process an OopMap Element while emitting nodes void Process_OopMap_Node(MachNode *mach, int code_offset); + class BufferSizingData { + public: + int _stub; + int _code; + int _const; + int _reloc; + + BufferSizingData() : + _stub(0), + _code(0), + _const(0), + _reloc(0) + { }; + }; + // Initialize code buffer - CodeBuffer* init_buffer(uint* blk_starts); + void estimate_buffer_size(int& const_req); + CodeBuffer* init_buffer(BufferSizingData& buf_sizes); // Write out basic block data to code buffer void fill_buffer(CodeBuffer* cb, uint* blk_starts); // Determine which variable sized branches can be shortened - void shorten_branches(uint* blk_starts, int& code_size, int& reloc_size, int& stub_size); + void shorten_branches(uint* blk_starts, BufferSizingData& buf_sizes); // Compute the size of first NumberOfLoopInstrToAlign instructions // at the head of a loop. diff --git a/src/hotspot/share/opto/escape.cpp b/src/hotspot/share/opto/escape.cpp index 5da7a2f86..23334429e 100644 --- a/src/hotspot/share/opto/escape.cpp +++ b/src/hotspot/share/opto/escape.cpp @@ -506,13 +506,6 @@ void ConnectionGraph::add_node_to_connection_graph(Node *n, Unique_Node_List *de add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(0), delayed_worklist); } -#if INCLUDE_ZGC - else if (UseZGC) { - if (n->as_Proj()->_con == LoadBarrierNode::Oop && n->in(0)->is_LoadBarrier()) { - add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(0)->in(LoadBarrierNode::Oop), delayed_worklist); - } - } -#endif break; } case Op_Rethrow: // Exception object escapes @@ -747,14 +740,6 @@ void ConnectionGraph::add_final_edges(Node *n) { add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(0), NULL); break; } -#if INCLUDE_ZGC - else if (UseZGC) { - if (n->as_Proj()->_con == LoadBarrierNode::Oop && n->in(0)->is_LoadBarrier()) { - add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(0)->in(LoadBarrierNode::Oop), NULL); - break; - } - } -#endif ELSE_FAIL("Op_Proj"); } case Op_Rethrow: // Exception object escapes diff --git a/src/hotspot/share/opto/graphKit.cpp b/src/hotspot/share/opto/graphKit.cpp index b44bc313f..8dd81f7a1 100644 --- a/src/hotspot/share/opto/graphKit.cpp +++ b/src/hotspot/share/opto/graphKit.cpp @@ -43,6 +43,7 @@ #include "opto/runtime.hpp" #include "runtime/deoptimization.hpp" #include "runtime/sharedRuntime.hpp" +#include "utilities/bitMap.inline.hpp" #include "utilities/macros.hpp" #if INCLUDE_SHENANDOAHGC #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp" @@ -1519,18 +1520,19 @@ Node* GraphKit::make_load(Node* ctl, Node* adr, const Type* t, BasicType bt, bool require_atomic_access, bool unaligned, bool mismatched, - bool unsafe) { + bool unsafe, + uint8_t barrier_data) { assert(adr_idx != Compile::AliasIdxTop, "use other make_load factory" ); const TypePtr* adr_type = NULL; // debug-mode-only argument debug_only(adr_type = C->get_adr_type(adr_idx)); Node* mem = memory(adr_idx); Node* ld; if (require_atomic_access && bt == T_LONG) { - ld = LoadLNode::make_atomic(ctl, mem, adr, adr_type, t, mo, control_dependency, unaligned, mismatched, unsafe); + ld = LoadLNode::make_atomic(ctl, mem, adr, adr_type, t, mo, control_dependency, unaligned, mismatched, unsafe, barrier_data); } else if (require_atomic_access && bt == T_DOUBLE) { - ld = LoadDNode::make_atomic(ctl, mem, adr, adr_type, t, mo, control_dependency, unaligned, mismatched, unsafe); + ld = LoadDNode::make_atomic(ctl, mem, adr, adr_type, t, mo, control_dependency, unaligned, mismatched, unsafe, barrier_data); } else { - ld = LoadNode::make(_gvn, ctl, mem, adr, adr_type, t, bt, mo, control_dependency, unaligned, mismatched, unsafe); + ld = LoadNode::make(_gvn, ctl, mem, adr, adr_type, t, bt, mo, control_dependency, unaligned, mismatched, unsafe, barrier_data); } ld = _gvn.transform(ld); if (((bt == T_OBJECT) && C->do_escape_analysis()) || C->eliminate_boxing()) { diff --git a/src/hotspot/share/opto/graphKit.hpp b/src/hotspot/share/opto/graphKit.hpp index 3529cc239..1022fd09b 100644 --- a/src/hotspot/share/opto/graphKit.hpp +++ b/src/hotspot/share/opto/graphKit.hpp @@ -524,27 +524,27 @@ class GraphKit : public Phase { Node* make_load(Node* ctl, Node* adr, const Type* t, BasicType bt, MemNode::MemOrd mo, LoadNode::ControlDependency control_dependency = LoadNode::DependsOnlyOnTest, bool require_atomic_access = false, bool unaligned = false, - bool mismatched = false, bool unsafe = false) { + bool mismatched = false, bool unsafe = false, uint8_t barrier_data = 0) { // This version computes alias_index from bottom_type return make_load(ctl, adr, t, bt, adr->bottom_type()->is_ptr(), mo, control_dependency, require_atomic_access, - unaligned, mismatched, unsafe); + unaligned, mismatched, unsafe, barrier_data); } Node* make_load(Node* ctl, Node* adr, const Type* t, BasicType bt, const TypePtr* adr_type, MemNode::MemOrd mo, LoadNode::ControlDependency control_dependency = LoadNode::DependsOnlyOnTest, bool require_atomic_access = false, bool unaligned = false, - bool mismatched = false, bool unsafe = false) { + bool mismatched = false, bool unsafe = false, uint8_t barrier_data = 0) { // This version computes alias_index from an address type assert(adr_type != NULL, "use other make_load factory"); return make_load(ctl, adr, t, bt, C->get_alias_index(adr_type), mo, control_dependency, require_atomic_access, - unaligned, mismatched, unsafe); + unaligned, mismatched, unsafe, barrier_data); } // This is the base version which is given an alias index. Node* make_load(Node* ctl, Node* adr, const Type* t, BasicType bt, int adr_idx, MemNode::MemOrd mo, LoadNode::ControlDependency control_dependency = LoadNode::DependsOnlyOnTest, bool require_atomic_access = false, bool unaligned = false, - bool mismatched = false, bool unsafe = false); + bool mismatched = false, bool unsafe = false, uint8_t barrier_data = 0); // Create & transform a StoreNode and store the effect into the // parser's memory state. diff --git a/src/hotspot/share/opto/lcm.cpp b/src/hotspot/share/opto/lcm.cpp index e97a4437f..f3fea9965 100644 --- a/src/hotspot/share/opto/lcm.cpp +++ b/src/hotspot/share/opto/lcm.cpp @@ -169,7 +169,6 @@ void PhaseCFG::implicit_null_check(Block* block, Node *proj, Node *val, int allo case Op_LoadI: case Op_LoadL: case Op_LoadP: - case Op_LoadBarrierSlowReg: case Op_LoadN: case Op_LoadS: case Op_LoadKlass: diff --git a/src/hotspot/share/opto/library_call.cpp b/src/hotspot/share/opto/library_call.cpp index 515665ffa..2da775a6a 100644 --- a/src/hotspot/share/opto/library_call.cpp +++ b/src/hotspot/share/opto/library_call.cpp @@ -337,6 +337,7 @@ class LibraryCallKit : public GraphKit { bool inline_profileBoolean(); bool inline_isCompileConstant(); + bool inline_nextIntRuntime(); void clear_upper_avx() { #ifdef X86 if (UseAVX >= 2) { @@ -887,6 +888,9 @@ bool LibraryCallKit::try_to_inline(int predicate) { case vmIntrinsics::_isCompileConstant: return inline_isCompileConstant(); + case vmIntrinsics::_nextInt: + return SharedRuntime::_opt_for_aarch64 ? inline_nextIntRuntime() : false; + case vmIntrinsics::_hasNegatives: return inline_hasNegatives(); @@ -6989,3 +6993,16 @@ bool LibraryCallKit::inline_isCompileConstant() { set_result(n->is_Con() ? intcon(1) : intcon(0)); return true; } + +bool LibraryCallKit::inline_nextIntRuntime() { + Node* ctrl = control(); + Node* monotonical_incr_adr = makecon(TypeRawPtr::make(SharedRuntime::monotonical_incr_addr())); + int adr_type = Compile::AliasIdxRaw; + + Node* monotonical_incr = make_load(ctrl, monotonical_incr_adr, TypeInt::INT, T_INT, adr_type, MemNode::unordered); + Node* incr = _gvn.transform(new AddINode(monotonical_incr, _gvn.intcon(13))); + store_to_memory(ctrl, monotonical_incr_adr, incr, T_INT, adr_type, MemNode::unordered); + + set_result(incr); + return true; +} diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index 5ad560fdd..7a6436c62 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -4596,7 +4596,6 @@ void PhaseIdealLoop::build_loop_late_post( Node *n ) { case Op_LoadL: case Op_LoadS: case Op_LoadP: - case Op_LoadBarrierSlowReg: case Op_LoadN: case Op_LoadRange: case Op_LoadD_unaligned: diff --git a/src/hotspot/share/opto/loopopts.cpp b/src/hotspot/share/opto/loopopts.cpp index 27bf3a3c1..c170a0395 100644 --- a/src/hotspot/share/opto/loopopts.cpp +++ b/src/hotspot/share/opto/loopopts.cpp @@ -41,9 +41,6 @@ #include "opto/rootnode.hpp" #include "opto/subnode.hpp" #include "utilities/macros.hpp" -#if INCLUDE_ZGC -#include "gc/z/c2/zBarrierSetC2.hpp" -#endif //============================================================================= //------------------------------split_thru_phi--------------------------------- diff --git a/src/hotspot/share/opto/machnode.hpp b/src/hotspot/share/opto/machnode.hpp index a52325680..8fb75d0d6 100644 --- a/src/hotspot/share/opto/machnode.hpp +++ b/src/hotspot/share/opto/machnode.hpp @@ -197,7 +197,7 @@ public: // ADLC inherit from this class. class MachNode : public Node { public: - MachNode() : Node((uint)0), _num_opnds(0), _opnds(NULL) { + MachNode() : Node((uint)0), _barrier(0), _num_opnds(0), _opnds(NULL) { init_class_id(Class_Mach); } // Required boilerplate @@ -211,6 +211,10 @@ public: // no constant base node input. virtual uint mach_constant_base_node_input() const { return (uint)-1; } + uint8_t barrier_data() const { return _barrier; } + void set_barrier_data(uint data) { _barrier = data; } + + // Copy inputs and operands to new node of instruction. // Called from cisc_version() and short_branch_version(). // !!!! The method's body is defined in ad_<arch>.cpp file. @@ -255,6 +259,9 @@ public: // output have choices - but they must use the same choice. virtual uint two_adr( ) const { return 0; } + // The GC might require some barrier metadata for machine code emission. + uint8_t _barrier; + // Array of complex operand pointers. Each corresponds to zero or // more leafs. Must be set by MachNode constructor to point to an // internal array of MachOpers. The MachOper array is sized by diff --git a/src/hotspot/share/opto/matcher.cpp b/src/hotspot/share/opto/matcher.cpp index 2d3bafdd7..4cc7580a8 100644 --- a/src/hotspot/share/opto/matcher.cpp +++ b/src/hotspot/share/opto/matcher.cpp @@ -1754,6 +1754,13 @@ MachNode *Matcher::ReduceInst( State *s, int rule, Node *&mem ) { _shared_nodes.map(leaf->_idx, ex); } + // Have mach nodes inherit GC barrier data + if (leaf->is_LoadStore()) { + mach->set_barrier_data(leaf->as_LoadStore()->barrier_data()); + } else if (leaf->is_Mem()) { + mach->set_barrier_data(leaf->as_Mem()->barrier_data()); + } + return ex; } @@ -2173,17 +2180,6 @@ void Matcher::find_shared( Node *n ) { case Op_SafePoint: mem_op = true; break; -#if INCLUDE_ZGC - case Op_CallLeaf: - if (UseZGC) { - if (n->as_Call()->entry_point() == ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr() || - n->as_Call()->entry_point() == ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded_addr()) { - mem_op = true; - mem_addr_idx = TypeFunc::Parms+1; - } - break; - } -#endif default: if( n->is_Store() ) { // Do match stores, despite no ideal reg @@ -2296,33 +2292,6 @@ void Matcher::find_shared( Node *n ) { n->del_req(LoadStoreConditionalNode::ExpectedIn); break; } -#if INCLUDE_ZGC - case Op_ZCompareAndExchangeP: - case Op_ZCompareAndSwapP: - case Op_ZWeakCompareAndSwapP: { - Node *mem = n->in(MemNode::Address); - Node *keepalive = n->in(5); - Node *pair1 = new BinaryNode(mem, keepalive); - - Node *newval = n->in(MemNode::ValueIn); - Node *oldval = n->in(LoadStoreConditionalNode::ExpectedIn); - Node *pair2 = new BinaryNode(oldval, newval); - - n->set_req(MemNode::Address, pair1); - n->set_req(MemNode::ValueIn, pair2); - n->del_req(5); - n->del_req(LoadStoreConditionalNode::ExpectedIn); - break; - } - case Op_ZGetAndSetP: { - Node *keepalive = n->in(4); - Node *newval = n->in(MemNode::ValueIn); - Node *pair = new BinaryNode(newval, keepalive); - n->set_req(MemNode::ValueIn, pair); - n->del_req(4); - break; - } -#endif // INCLUDE_ZGC case Op_CMoveD: // Convert trinary to binary-tree case Op_CMoveF: case Op_CMoveI: diff --git a/src/hotspot/share/opto/memnode.cpp b/src/hotspot/share/opto/memnode.cpp index 8ffb5a708..32ce0f9bc 100644 --- a/src/hotspot/share/opto/memnode.cpp +++ b/src/hotspot/share/opto/memnode.cpp @@ -49,9 +49,6 @@ #include "utilities/copy.hpp" #include "utilities/macros.hpp" #include "utilities/vmError.hpp" -#if INCLUDE_ZGC -#include "gc/z/c2/zBarrierSetC2.hpp" -#endif #if INCLUDE_SHENANDOAHGC #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp" #endif @@ -858,7 +855,7 @@ bool LoadNode::is_immutable_value(Node* adr) { //----------------------------LoadNode::make----------------------------------- // Polymorphic factory method: Node *LoadNode::make(PhaseGVN& gvn, Node *ctl, Node *mem, Node *adr, const TypePtr* adr_type, const Type *rt, BasicType bt, MemOrd mo, - ControlDependency control_dependency, bool unaligned, bool mismatched, bool unsafe) { + ControlDependency control_dependency, bool unaligned, bool mismatched, bool unsafe, uint8_t barrier_data) { Compile* C = gvn.C; // sanity check the alias category against the created node type @@ -909,6 +906,7 @@ Node *LoadNode::make(PhaseGVN& gvn, Node *ctl, Node *mem, Node *adr, const TypeP if (unsafe) { load->set_unsafe_access(); } + load->set_barrier_data(barrier_data); if (load->Opcode() == Op_LoadN) { Node* ld = gvn.transform(load); return new DecodeNNode(ld, ld->bottom_type()->make_ptr()); @@ -918,7 +916,7 @@ Node *LoadNode::make(PhaseGVN& gvn, Node *ctl, Node *mem, Node *adr, const TypeP } LoadLNode* LoadLNode::make_atomic(Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, const Type* rt, MemOrd mo, - ControlDependency control_dependency, bool unaligned, bool mismatched, bool unsafe) { + ControlDependency control_dependency, bool unaligned, bool mismatched, bool unsafe, uint8_t barrier_data) { bool require_atomic = true; LoadLNode* load = new LoadLNode(ctl, mem, adr, adr_type, rt->is_long(), mo, control_dependency, require_atomic); if (unaligned) { @@ -930,11 +928,12 @@ LoadLNode* LoadLNode::make_atomic(Node* ctl, Node* mem, Node* adr, const TypePtr if (unsafe) { load->set_unsafe_access(); } + load->set_barrier_data(barrier_data); return load; } LoadDNode* LoadDNode::make_atomic(Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, const Type* rt, MemOrd mo, - ControlDependency control_dependency, bool unaligned, bool mismatched, bool unsafe) { + ControlDependency control_dependency, bool unaligned, bool mismatched, bool unsafe, uint8_t barrier_data) { bool require_atomic = true; LoadDNode* load = new LoadDNode(ctl, mem, adr, adr_type, rt, mo, control_dependency, require_atomic); if (unaligned) { @@ -946,6 +945,7 @@ LoadDNode* LoadDNode::make_atomic(Node* ctl, Node* mem, Node* adr, const TypePtr if (unsafe) { load->set_unsafe_access(); } + load->set_barrier_data(barrier_data); return load; } @@ -2891,7 +2891,7 @@ LoadStoreNode::LoadStoreNode( Node *c, Node *mem, Node *adr, Node *val, const Ty : Node(required), _type(rt), _adr_type(at), - _has_barrier(false) + _barrier(0) { init_req(MemNode::Control, c ); init_req(MemNode::Memory , mem); diff --git a/src/hotspot/share/opto/memnode.hpp b/src/hotspot/share/opto/memnode.hpp index d4c2895bf..259b4343f 100644 --- a/src/hotspot/share/opto/memnode.hpp +++ b/src/hotspot/share/opto/memnode.hpp @@ -43,6 +43,8 @@ private: bool _unaligned_access; // Unaligned access from unsafe bool _mismatched_access; // Mismatched access from unsafe: byte read in integer array for instance bool _unsafe_access; // Access of unsafe origin. + uint8_t _barrier; // Bit field with barrier information + protected: #ifdef ASSERT const TypePtr* _adr_type; // What kind of memory is being addressed? @@ -62,18 +64,30 @@ public: unset // The memory ordering is not set (used for testing) } MemOrd; protected: - MemNode( Node *c0, Node *c1, Node *c2, const TypePtr* at ) - : Node(c0,c1,c2 ), _unaligned_access(false), _mismatched_access(false), _unsafe_access(false) { + MemNode( Node *c0, Node *c1, Node *c2, const TypePtr* at ) : + Node(c0,c1,c2), + _unaligned_access(false), + _mismatched_access(false), + _unsafe_access(false), + _barrier(0) { init_class_id(Class_Mem); debug_only(_adr_type=at; adr_type();) } - MemNode( Node *c0, Node *c1, Node *c2, const TypePtr* at, Node *c3 ) - : Node(c0,c1,c2,c3), _unaligned_access(false), _mismatched_access(false), _unsafe_access(false) { + MemNode( Node *c0, Node *c1, Node *c2, const TypePtr* at, Node *c3 ) : + Node(c0,c1,c2,c3), + _unaligned_access(false), + _mismatched_access(false), + _unsafe_access(false), + _barrier(0) { init_class_id(Class_Mem); debug_only(_adr_type=at; adr_type();) } - MemNode( Node *c0, Node *c1, Node *c2, const TypePtr* at, Node *c3, Node *c4) - : Node(c0,c1,c2,c3,c4), _unaligned_access(false), _mismatched_access(false), _unsafe_access(false) { + MemNode( Node *c0, Node *c1, Node *c2, const TypePtr* at, Node *c3, Node *c4) : + Node(c0,c1,c2,c3,c4), + _unaligned_access(false), + _mismatched_access(false), + _unsafe_access(false), + _barrier(0) { init_class_id(Class_Mem); debug_only(_adr_type=at; adr_type();) } @@ -126,6 +140,9 @@ public: #endif } + uint8_t barrier_data() { return _barrier; } + void set_barrier_data(uint8_t barrier_data) { _barrier = barrier_data; } + // Search through memory states which precede this node (load or store). // Look for an exact match for the address, with no intervening // aliased stores. @@ -183,8 +200,6 @@ private: // this field. const MemOrd _mo; - uint _barrier; // Bit field with barrier information - protected: virtual uint cmp(const Node &n) const; virtual uint size_of() const; // Size is bigger @@ -196,7 +211,7 @@ protected: public: LoadNode(Node *c, Node *mem, Node *adr, const TypePtr* at, const Type *rt, MemOrd mo, ControlDependency control_dependency) - : MemNode(c,mem,adr,at), _type(rt), _mo(mo), _control_dependency(control_dependency), _barrier(0) { + : MemNode(c,mem,adr,at), _control_dependency(control_dependency), _mo(mo), _type(rt) { init_class_id(Class_Load); } inline bool is_unordered() const { return !is_acquire(); } @@ -213,7 +228,8 @@ public: static Node* make(PhaseGVN& gvn, Node *c, Node *mem, Node *adr, const TypePtr* at, const Type *rt, BasicType bt, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest, - bool unaligned = false, bool mismatched = false, bool unsafe = false); + bool unaligned = false, bool mismatched = false, bool unsafe = false, + uint8_t barrier_data = 0); virtual uint hash() const; // Check the type @@ -265,10 +281,6 @@ public: Node* convert_to_unsigned_load(PhaseGVN& gvn); Node* convert_to_signed_load(PhaseGVN& gvn); - void copy_barrier_info(const Node* src) { _barrier = src->as_Load()->_barrier; } - uint barrier_data() { return _barrier; } - void set_barrier_data(uint barrier_data) { _barrier |= barrier_data; } - #ifndef PRODUCT virtual void dump_spec(outputStream *st) const; #endif @@ -398,7 +410,7 @@ public: bool require_atomic_access() const { return _require_atomic_access; } static LoadLNode* make_atomic(Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, const Type* rt, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest, - bool unaligned = false, bool mismatched = false, bool unsafe = false); + bool unaligned = false, bool mismatched = false, bool unsafe = false, uint8_t barrier_data = 0); #ifndef PRODUCT virtual void dump_spec(outputStream *st) const { LoadNode::dump_spec(st); @@ -450,7 +462,7 @@ public: bool require_atomic_access() const { return _require_atomic_access; } static LoadDNode* make_atomic(Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, const Type* rt, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest, - bool unaligned = false, bool mismatched = false, bool unsafe = false); + bool unaligned = false, bool mismatched = false, bool unsafe = false, uint8_t barrier_data = 0); #ifndef PRODUCT virtual void dump_spec(outputStream *st) const { LoadNode::dump_spec(st); @@ -817,7 +829,7 @@ class LoadStoreNode : public Node { private: const Type* const _type; // What kind of value is loaded? const TypePtr* _adr_type; // What kind of memory is being addressed? - bool _has_barrier; + uint8_t _barrier; // Bit field with barrier information virtual uint size_of() const; // Size is bigger public: LoadStoreNode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at, const Type* rt, uint required ); @@ -831,8 +843,9 @@ public: bool result_not_used() const; MemBarNode* trailing_membar() const; - void set_has_barrier() { _has_barrier = true; }; - bool has_barrier() const { return _has_barrier; }; + + uint8_t barrier_data() { return _barrier; } + void set_barrier_data(uint8_t barrier_data) { _barrier = barrier_data; } }; class LoadStoreConditionalNode : public LoadStoreNode { @@ -885,6 +898,7 @@ public: MemNode::MemOrd order() const { return _mem_ord; } + virtual uint size_of() const { return sizeof(*this); } }; class CompareAndExchangeNode : public LoadStoreNode { @@ -902,6 +916,7 @@ public: MemNode::MemOrd order() const { return _mem_ord; } + virtual uint size_of() const { return sizeof(*this); } }; //------------------------------CompareAndSwapBNode--------------------------- diff --git a/src/hotspot/share/opto/node.cpp b/src/hotspot/share/opto/node.cpp index e439ebb93..84a56f8d0 100644 --- a/src/hotspot/share/opto/node.cpp +++ b/src/hotspot/share/opto/node.cpp @@ -550,9 +550,6 @@ Node *Node::clone() const { if (n->is_SafePoint()) { n->as_SafePoint()->clone_replaced_nodes(); } - if (n->is_Load()) { - n->as_Load()->copy_barrier_info(this); - } return n; // Return the clone } @@ -1478,10 +1475,6 @@ bool Node::needs_anti_dependence_check() const { if( req() < 2 || (_flags & Flag_needs_anti_dependence_check) == 0 ) { return false; } - BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); - if (!bs->needs_anti_dependence_check(this)) { - return false; - } return in(1)->bottom_type()->has_memory(); } diff --git a/src/hotspot/share/opto/node.hpp b/src/hotspot/share/opto/node.hpp index f3bd41d91..6efaa6fc7 100644 --- a/src/hotspot/share/opto/node.hpp +++ b/src/hotspot/share/opto/node.hpp @@ -83,8 +83,6 @@ class JVMState; class JumpNode; class JumpProjNode; class LoadNode; -class LoadBarrierNode; -class LoadBarrierSlowRegNode; class LoadStoreNode; class LoadStoreConditionalNode; class LockNode; @@ -648,7 +646,6 @@ public: DEFINE_CLASS_ID(MemBar, Multi, 3) DEFINE_CLASS_ID(Initialize, MemBar, 0) DEFINE_CLASS_ID(MemBarStoreStore, MemBar, 1) - DEFINE_CLASS_ID(LoadBarrier, Multi, 4) DEFINE_CLASS_ID(Mach, Node, 1) DEFINE_CLASS_ID(MachReturn, Mach, 0) @@ -700,7 +697,6 @@ public: DEFINE_CLASS_ID(Mem, Node, 4) DEFINE_CLASS_ID(Load, Mem, 0) DEFINE_CLASS_ID(LoadVector, Load, 0) - DEFINE_CLASS_ID(LoadBarrierSlowReg, Load, 1) DEFINE_CLASS_ID(Store, Mem, 1) DEFINE_CLASS_ID(StoreVector, Store, 0) DEFINE_CLASS_ID(LoadStore, Mem, 2) @@ -845,8 +841,6 @@ public: DEFINE_CLASS_QUERY(Load) DEFINE_CLASS_QUERY(LoadStore) DEFINE_CLASS_QUERY(LoadStoreConditional) - DEFINE_CLASS_QUERY(LoadBarrier) - DEFINE_CLASS_QUERY(LoadBarrierSlowReg) DEFINE_CLASS_QUERY(Lock) DEFINE_CLASS_QUERY(Loop) DEFINE_CLASS_QUERY(Mach) diff --git a/src/hotspot/share/opto/output.cpp b/src/hotspot/share/opto/output.cpp index b6540e06a..5b9873b4d 100644 --- a/src/hotspot/share/opto/output.cpp +++ b/src/hotspot/share/opto/output.cpp @@ -32,6 +32,8 @@ #include "compiler/compilerDirectives.hpp" #include "compiler/disassembler.hpp" #include "compiler/oopMap.hpp" +#include "gc/shared/barrierSet.hpp" +#include "gc/shared/c2/barrierSetC2.hpp" #include "memory/allocation.inline.hpp" #include "opto/ad.hpp" #include "opto/callnode.hpp" @@ -115,35 +117,35 @@ void Compile::Output() { } } + // Keeper of sizing aspects + BufferSizingData buf_sizes = BufferSizingData(); + + // Initialize code buffer + estimate_buffer_size(buf_sizes._const); + if (failing()) return; + + // Pre-compute the length of blocks and replace + // long branches with short if machine supports it. + // Must be done before ScheduleAndBundle due to SPARC delay slots + uint* blk_starts = NEW_RESOURCE_ARRAY(uint, _cfg->number_of_blocks() + 1); blk_starts[0] = 0; - // Initialize code buffer and process short branches. - CodeBuffer* cb = init_buffer(blk_starts); + shorten_branches(blk_starts, buf_sizes); - if (cb == NULL || failing()) { + ScheduleAndBundle(); + if (failing()) { return; } - ScheduleAndBundle(); - -#ifndef PRODUCT - if (trace_opto_output()) { - tty->print("\n---- After ScheduleAndBundle ----\n"); - for (uint i = 0; i < _cfg->number_of_blocks(); i++) { - tty->print("\nBB#%03d:\n", i); - Block* block = _cfg->get_block(i); - for (uint j = 0; j < block->number_of_nodes(); j++) { - Node* n = block->get_node(j); - OptoReg::Name reg = _regalloc->get_reg_first(n); - tty->print(" %-6s ", reg >= 0 && reg < REG_COUNT ? Matcher::regName[reg] : ""); - n->dump(); - } - } - } -#endif + // Late barrier analysis must be done after schedule and bundle + // Otherwise liveness based spilling will fail + BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); + bs->late_barrier_analysis(); - if (failing()) { + // Complete sizing of codebuffer + CodeBuffer* cb = init_buffer(buf_sizes); + if (cb == NULL || failing()) { return; } @@ -224,7 +226,7 @@ void Compile::compute_loop_first_inst_sizes() { // The architecture description provides short branch variants for some long // branch instructions. Replace eligible long branches with short branches. -void Compile::shorten_branches(uint* blk_starts, int& code_size, int& reloc_size, int& stub_size) { +void Compile::shorten_branches(uint* blk_starts, BufferSizingData& buf_sizes) { // Compute size of each block, method size, and relocation information size uint nblocks = _cfg->number_of_blocks(); @@ -242,11 +244,11 @@ void Compile::shorten_branches(uint* blk_starts, int& code_size, int& reloc_size bool has_short_branch_candidate = false; // Initialize the sizes to 0 - code_size = 0; // Size in bytes of generated code - stub_size = 0; // Size in bytes of all stub entries + int code_size = 0; // Size in bytes of generated code + int stub_size = 0; // Size in bytes of all stub entries // Size in bytes of all relocation entries, including those in local stubs. // Start with 2-bytes of reloc info for the unvalidated entry point - reloc_size = 1; // Number of relocation entries + int reloc_size = 1; // Number of relocation entries // Make three passes. The first computes pessimistic blk_starts, // relative jmp_offset and reloc_size information. The second performs @@ -480,6 +482,10 @@ void Compile::shorten_branches(uint* blk_starts, int& code_size, int& reloc_size // a relocation index. // The CodeBuffer will expand the locs array if this estimate is too low. reloc_size *= 10 / sizeof(relocInfo); + + buf_sizes._reloc = reloc_size; + buf_sizes._code = code_size; + buf_sizes._stub = stub_size; } //------------------------------FillLocArray----------------------------------- @@ -491,8 +497,8 @@ static LocationValue *new_loc_value( PhaseRegAlloc *ra, OptoReg::Name regnum, Lo // This should never have accepted Bad before assert(OptoReg::is_valid(regnum), "location must be valid"); return (OptoReg::is_reg(regnum)) - ? new LocationValue(Location::new_reg_loc(l_type, OptoReg::as_VMReg(regnum)) ) - : new LocationValue(Location::new_stk_loc(l_type, ra->reg2offset(regnum))); + ? new LocationValue(Location::new_reg_loc(l_type, OptoReg::as_VMReg(regnum)) ) + : new LocationValue(Location::new_stk_loc(l_type, ra->reg2offset(regnum))); } @@ -611,12 +617,12 @@ void Compile::FillLocArray( int idx, MachSafePointNode* sfpt, Node *local, } #endif //_LP64 else if( (t->base() == Type::FloatBot || t->base() == Type::FloatCon) && - OptoReg::is_reg(regnum) ) { + OptoReg::is_reg(regnum) ) { array->append(new_loc_value( _regalloc, regnum, Matcher::float_in_double() - ? Location::float_in_dbl : Location::normal )); + ? Location::float_in_dbl : Location::normal )); } else if( t->base() == Type::Int && OptoReg::is_reg(regnum) ) { array->append(new_loc_value( _regalloc, regnum, Matcher::int_in_long - ? Location::int_in_long : Location::normal )); + ? Location::int_in_long : Location::normal )); } else if( t->base() == Type::NarrowOop ) { array->append(new_loc_value( _regalloc, regnum, Location::narrowoop )); } else { @@ -627,48 +633,48 @@ void Compile::FillLocArray( int idx, MachSafePointNode* sfpt, Node *local, // No register. It must be constant data. switch (t->base()) { - case Type::Half: // Second half of a double - ShouldNotReachHere(); // Caller should skip 2nd halves - break; - case Type::AnyPtr: - array->append(new ConstantOopWriteValue(NULL)); - break; - case Type::AryPtr: - case Type::InstPtr: // fall through - array->append(new ConstantOopWriteValue(t->isa_oopptr()->const_oop()->constant_encoding())); - break; - case Type::NarrowOop: - if (t == TypeNarrowOop::NULL_PTR) { + case Type::Half: // Second half of a double + ShouldNotReachHere(); // Caller should skip 2nd halves + break; + case Type::AnyPtr: array->append(new ConstantOopWriteValue(NULL)); - } else { - array->append(new ConstantOopWriteValue(t->make_ptr()->isa_oopptr()->const_oop()->constant_encoding())); - } - break; - case Type::Int: - array->append(new ConstantIntValue(t->is_int()->get_con())); - break; - case Type::RawPtr: - // A return address (T_ADDRESS). - assert((intptr_t)t->is_ptr()->get_con() < (intptr_t)0x10000, "must be a valid BCI"); + break; + case Type::AryPtr: + case Type::InstPtr: // fall through + array->append(new ConstantOopWriteValue(t->isa_oopptr()->const_oop()->constant_encoding())); + break; + case Type::NarrowOop: + if (t == TypeNarrowOop::NULL_PTR) { + array->append(new ConstantOopWriteValue(NULL)); + } else { + array->append(new ConstantOopWriteValue(t->make_ptr()->isa_oopptr()->const_oop()->constant_encoding())); + } + break; + case Type::Int: + array->append(new ConstantIntValue(t->is_int()->get_con())); + break; + case Type::RawPtr: + // A return address (T_ADDRESS). + assert((intptr_t)t->is_ptr()->get_con() < (intptr_t)0x10000, "must be a valid BCI"); #ifdef _LP64 - // Must be restored to the full-width 64-bit stack slot. - array->append(new ConstantLongValue(t->is_ptr()->get_con())); + // Must be restored to the full-width 64-bit stack slot. + array->append(new ConstantLongValue(t->is_ptr()->get_con())); #else - array->append(new ConstantIntValue(t->is_ptr()->get_con())); + array->append(new ConstantIntValue(t->is_ptr()->get_con())); #endif - break; - case Type::FloatCon: { - float f = t->is_float_constant()->getf(); - array->append(new ConstantIntValue(jint_cast(f))); - break; - } - case Type::DoubleCon: { - jdouble d = t->is_double_constant()->getd(); + break; + case Type::FloatCon: { + float f = t->is_float_constant()->getf(); + array->append(new ConstantIntValue(jint_cast(f))); + break; + } + case Type::DoubleCon: { + jdouble d = t->is_double_constant()->getd(); #ifdef _LP64 - array->append(new ConstantIntValue((jint)0)); - array->append(new ConstantDoubleValue(d)); + array->append(new ConstantIntValue((jint)0)); + array->append(new ConstantDoubleValue(d)); #else - // Repack the double as two jints. + // Repack the double as two jints. // The convention the interpreter uses is that the second local // holds the first raw word of the native double representation. // This is actually reasonable, since locals and stack arrays @@ -680,15 +686,15 @@ void Compile::FillLocArray( int idx, MachSafePointNode* sfpt, Node *local, array->append(new ConstantIntValue(acc.words[1])); array->append(new ConstantIntValue(acc.words[0])); #endif - break; - } - case Type::Long: { - jlong d = t->is_long()->get_con(); + break; + } + case Type::Long: { + jlong d = t->is_long()->get_con(); #ifdef _LP64 - array->append(new ConstantIntValue((jint)0)); - array->append(new ConstantLongValue(d)); + array->append(new ConstantIntValue((jint)0)); + array->append(new ConstantLongValue(d)); #else - // Repack the long as two jints. + // Repack the long as two jints. // The convention the interpreter uses is that the second local // holds the first raw word of the native double representation. // This is actually reasonable, since locals and stack arrays @@ -700,14 +706,14 @@ void Compile::FillLocArray( int idx, MachSafePointNode* sfpt, Node *local, array->append(new ConstantIntValue(acc.words[1])); array->append(new ConstantIntValue(acc.words[0])); #endif - break; - } - case Type::Top: // Add an illegal value here - array->append(new LocationValue(Location())); - break; - default: - ShouldNotReachHere(); - break; + break; + } + case Type::Top: // Add an illegal value here + array->append(new LocationValue(Location())); + break; + default: + ShouldNotReachHere(); + break; } } @@ -872,58 +878,58 @@ void Compile::Process_OopMap_Node(MachNode *mach, int current_offset) { // A simplified version of Process_OopMap_Node, to handle non-safepoints. class NonSafepointEmitter { - Compile* C; - JVMState* _pending_jvms; - int _pending_offset; + Compile* C; + JVMState* _pending_jvms; + int _pending_offset; - void emit_non_safepoint(); + void emit_non_safepoint(); public: - NonSafepointEmitter(Compile* compile) { - this->C = compile; - _pending_jvms = NULL; - _pending_offset = 0; - } + NonSafepointEmitter(Compile* compile) { + this->C = compile; + _pending_jvms = NULL; + _pending_offset = 0; + } - void observe_instruction(Node* n, int pc_offset) { - if (!C->debug_info()->recording_non_safepoints()) return; + void observe_instruction(Node* n, int pc_offset) { + if (!C->debug_info()->recording_non_safepoints()) return; - Node_Notes* nn = C->node_notes_at(n->_idx); - if (nn == NULL || nn->jvms() == NULL) return; - if (_pending_jvms != NULL && - _pending_jvms->same_calls_as(nn->jvms())) { - // Repeated JVMS? Stretch it up here. - _pending_offset = pc_offset; - } else { + Node_Notes* nn = C->node_notes_at(n->_idx); + if (nn == NULL || nn->jvms() == NULL) return; if (_pending_jvms != NULL && - _pending_offset < pc_offset) { - emit_non_safepoint(); - } - _pending_jvms = NULL; - if (pc_offset > C->debug_info()->last_pc_offset()) { - // This is the only way _pending_jvms can become non-NULL: - _pending_jvms = nn->jvms(); + _pending_jvms->same_calls_as(nn->jvms())) { + // Repeated JVMS? Stretch it up here. _pending_offset = pc_offset; + } else { + if (_pending_jvms != NULL && + _pending_offset < pc_offset) { + emit_non_safepoint(); + } + _pending_jvms = NULL; + if (pc_offset > C->debug_info()->last_pc_offset()) { + // This is the only way _pending_jvms can become non-NULL: + _pending_jvms = nn->jvms(); + _pending_offset = pc_offset; + } } } - } // Stay out of the way of real safepoints: - void observe_safepoint(JVMState* jvms, int pc_offset) { - if (_pending_jvms != NULL && - !_pending_jvms->same_calls_as(jvms) && - _pending_offset < pc_offset) { - emit_non_safepoint(); + void observe_safepoint(JVMState* jvms, int pc_offset) { + if (_pending_jvms != NULL && + !_pending_jvms->same_calls_as(jvms) && + _pending_offset < pc_offset) { + emit_non_safepoint(); + } + _pending_jvms = NULL; } - _pending_jvms = NULL; - } - void flush_at_end() { - if (_pending_jvms != NULL) { - emit_non_safepoint(); + void flush_at_end() { + if (_pending_jvms != NULL) { + emit_non_safepoint(); + } + _pending_jvms = NULL; } - _pending_jvms = NULL; - } }; void NonSafepointEmitter::emit_non_safepoint() { @@ -953,15 +959,11 @@ void NonSafepointEmitter::emit_non_safepoint() { } //------------------------------init_buffer------------------------------------ -CodeBuffer* Compile::init_buffer(uint* blk_starts) { +void Compile::estimate_buffer_size(int& const_req) { // Set the initially allocated size - int code_req = initial_code_capacity; - int locs_req = initial_locs_capacity; - int stub_req = initial_stub_capacity; - int const_req = initial_const_capacity; + const_req = initial_const_capacity; - int pad_req = NativeCall::instruction_size; // The extra spacing after the code is necessary on some platforms. // Sometimes we need to patch in a jump after the last instruction, // if the nmethod has been deoptimized. (See 4932387, 4894843.) @@ -973,7 +975,7 @@ CodeBuffer* Compile::init_buffer(uint* blk_starts) { // Compute prolog code size _method_size = 0; - _frame_slots = OptoReg::reg2stack(_matcher->_old_SP)+_regalloc->_framesize; + _frame_slots = OptoReg::reg2stack(_matcher->_old_SP) + _regalloc->_framesize; #if defined(IA64) && !defined(AIX) if (save_argument_registers()) { // 4815101: this is a stub with implicit and unknown precision fp args. @@ -1022,12 +1024,18 @@ CodeBuffer* Compile::init_buffer(uint* blk_starts) { // Initialize the space for the BufferBlob used to find and verify // instruction size in MachNode::emit_size() init_scratch_buffer_blob(const_req); - if (failing()) return NULL; // Out of memory +} - // Pre-compute the length of blocks and replace - // long branches with short if machine supports it. - shorten_branches(blk_starts, code_req, locs_req, stub_req); +CodeBuffer* Compile::init_buffer(BufferSizingData& buf_sizes) { + int stub_req = buf_sizes._stub; + int code_req = buf_sizes._code; + int const_req = buf_sizes._const; + + int pad_req = NativeCall::instruction_size; + + BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); + stub_req += bs->estimate_stub_size(); // nmethod and CodeBuffer count stubs & constants as part of method's code. // class HandlerImpl is platform-specific and defined in the *.ad files. int exception_handler_req = HandlerImpl::size_exception_handler() + MAX_stubs_size; // add marginal slop for handler @@ -1039,18 +1047,18 @@ CodeBuffer* Compile::init_buffer(uint* blk_starts) { code_req = const_req = stub_req = exception_handler_req = deopt_handler_req = 0x10; // force expansion int total_req = - const_req + - code_req + - pad_req + - stub_req + - exception_handler_req + - deopt_handler_req; // deopt handler + const_req + + code_req + + pad_req + + stub_req + + exception_handler_req + + deopt_handler_req; // deopt handler if (has_method_handle_invokes()) total_req += deopt_handler_req; // deopt MH handler CodeBuffer* cb = code_buffer(); - cb->initialize(total_req, locs_req); + cb->initialize(total_req, buf_sizes._reloc); // Have we run out of code space? if ((cb->blob() == NULL) || (!CompileBroker::should_compile_new_jobs())) { @@ -1265,12 +1273,12 @@ void Compile::fill_buffer(CodeBuffer* cb, uint* blk_starts) { Process_OopMap_Node(mach, current_offset); } // End if safepoint - // If this is a null check, then add the start of the previous instruction to the list + // If this is a null check, then add the start of the previous instruction to the list else if( mach->is_MachNullCheck() ) { inct_starts[inct_cnt++] = previous_offset; } - // If this is a branch, then fill in the label with the target BB's label + // If this is a branch, then fill in the label with the target BB's label else if (mach->is_MachBranch()) { // This requires the TRUE branch target be in succs[0] uint block_num = block->non_connector_successor(0)->_pre_order; @@ -1281,8 +1289,8 @@ void Compile::fill_buffer(CodeBuffer* cb, uint* blk_starts) { bool delay_slot_is_used = valid_bundle_info(n) && node_bundling(n)->use_unconditional_delay(); if (!delay_slot_is_used && mach->may_be_short_branch()) { - assert(delay_slot == NULL, "not expecting delay slot node"); - int br_size = n->size(_regalloc); + assert(delay_slot == NULL, "not expecting delay slot node"); + int br_size = n->size(_regalloc); int offset = blk_starts[block_num] - current_offset; if (block_num >= i) { // Current and following block's offset are not @@ -1340,7 +1348,7 @@ void Compile::fill_buffer(CodeBuffer* cb, uint* blk_starts) { } } #ifdef ASSERT - // Check that oop-store precedes the card-mark + // Check that oop-store precedes the card-mark else if (mach->ideal_Opcode() == Op_StoreCM) { uint storeCM_idx = j; int count = 0; @@ -1519,6 +1527,10 @@ void Compile::fill_buffer(CodeBuffer* cb, uint* blk_starts) { } #endif + BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); + bs->emit_stubs(*cb); + if (failing()) return; + #ifndef PRODUCT // Information on the size of the method, without the extraneous code Scheduling::increment_method_size(cb->insts_size()); @@ -1682,20 +1694,20 @@ uint Scheduling::_total_instructions_per_bundle[Pipeline::_max_instrs_per_cycle+ // Initializer for class Scheduling Scheduling::Scheduling(Arena *arena, Compile &compile) - : _arena(arena), - _cfg(compile.cfg()), - _regalloc(compile.regalloc()), - _reg_node(arena), - _bundle_instr_count(0), - _bundle_cycle_number(0), - _scheduled(arena), - _available(arena), - _next_node(NULL), - _bundle_use(0, 0, resource_count, &_bundle_use_elements[0]), - _pinch_free_list(arena) + : _arena(arena), + _cfg(compile.cfg()), + _regalloc(compile.regalloc()), + _reg_node(arena), + _bundle_instr_count(0), + _bundle_cycle_number(0), + _scheduled(arena), + _available(arena), + _next_node(NULL), + _bundle_use(0, 0, resource_count, &_bundle_use_elements[0]), + _pinch_free_list(arena) #ifndef PRODUCT - , _branches(0) - , _unconditional_delays(0) + , _branches(0) + , _unconditional_delays(0) #endif { // Create a MachNopNode @@ -1774,8 +1786,8 @@ void Scheduling::step_and_clear() { _bundle_use.reset(); memcpy(_bundle_use_elements, - Pipeline_Use::elaborated_elements, - sizeof(Pipeline_Use::elaborated_elements)); + Pipeline_Use::elaborated_elements, + sizeof(Pipeline_Use::elaborated_elements)); } // Perform instruction scheduling and bundling over the sequence of @@ -1802,6 +1814,22 @@ void Compile::ScheduleAndBundle() { // Walk backwards over each basic block, computing the needed alignment // Walk over all the basic blocks scheduling.DoScheduling(); + +#ifndef PRODUCT + if (trace_opto_output()) { + tty->print("\n---- After ScheduleAndBundle ----\n"); + for (uint i = 0; i < _cfg->number_of_blocks(); i++) { + tty->print("\nBB#%03d:\n", i); + Block* block = _cfg->get_block(i); + for (uint j = 0; j < block->number_of_nodes(); j++) { + Node* n = block->get_node(j); + OptoReg::Name reg = _regalloc->get_reg_first(n); + tty->print(" %-6s ", reg >= 0 && reg < REG_COUNT ? Matcher::regName[reg] : ""); + n->dump(); + } + } + } +#endif } // Compute the latency of all the instructions. This is fairly simple, @@ -1870,7 +1898,7 @@ bool Scheduling::NodeFitsInBundle(Node *n) { #ifndef PRODUCT if (_cfg->C->trace_opto_output()) tty->print("# NodeFitsInBundle [%4d]: FALSE; latency %4d > %d\n", - n->_idx, _current_latency[n_idx], _bundle_cycle_number); + n->_idx, _current_latency[n_idx], _bundle_cycle_number); #endif return (false); } @@ -2095,12 +2123,12 @@ void Scheduling::AddNodeToBundle(Node *n, const Block *bb) { // Don't allow safepoints in the branch shadow, that will // cause a number of difficulties if ( avail_pipeline->instructionCount() == 1 && - !avail_pipeline->hasMultipleBundles() && - !avail_pipeline->hasBranchDelay() && - Pipeline::instr_has_unit_size() && - d->size(_regalloc) == Pipeline::instr_unit_size() && - NodeFitsInBundle(d) && - !node_bundling(d)->used_in_delay()) { + !avail_pipeline->hasMultipleBundles() && + !avail_pipeline->hasBranchDelay() && + Pipeline::instr_has_unit_size() && + d->size(_regalloc) == Pipeline::instr_unit_size() && + NodeFitsInBundle(d) && + !node_bundling(d)->used_in_delay()) { if (d->is_Mach() && !d->is_MachSafePoint()) { // A node that fits in the delay slot was found, so we need to @@ -2145,13 +2173,13 @@ void Scheduling::AddNodeToBundle(Node *n, const Block *bb) { // step of the bundles if (!NodeFitsInBundle(n)) { #ifndef PRODUCT - if (_cfg->C->trace_opto_output()) - tty->print("# *** STEP(branch won't fit) ***\n"); + if (_cfg->C->trace_opto_output()) + tty->print("# *** STEP(branch won't fit) ***\n"); #endif // Update the state information - _bundle_instr_count = 0; - _bundle_cycle_number += 1; - _bundle_use.step(1); + _bundle_instr_count = 0; + _bundle_cycle_number += 1; + _bundle_use.step(1); } } @@ -2197,8 +2225,8 @@ void Scheduling::AddNodeToBundle(Node *n, const Block *bb) { #ifndef PRODUCT if (_cfg->C->trace_opto_output()) tty->print("# *** STEP(%d >= %d instructions) ***\n", - instruction_count + _bundle_instr_count, - Pipeline::_max_instrs_per_cycle); + instruction_count + _bundle_instr_count, + Pipeline::_max_instrs_per_cycle); #endif step(1); } @@ -2404,7 +2432,7 @@ void Scheduling::DoScheduling() { } assert(!last->is_Mach() || last->as_Mach()->ideal_Opcode() != Op_Con, ""); if( last->is_Catch() || - (last->is_Mach() && last->as_Mach()->ideal_Opcode() == Op_Halt) ) { + (last->is_Mach() && last->as_Mach()->ideal_Opcode() == Op_Halt) ) { // There might be a prior call. Skip it. while (_bb_start < _bb_end && bb->get_node(--_bb_end)->is_MachProj()); } else if( last->is_MachNullCheck() ) { @@ -2474,7 +2502,7 @@ void Scheduling::DoScheduling() { } #endif #ifdef ASSERT - verify_good_schedule(bb,"after block local scheduling"); + verify_good_schedule(bb,"after block local scheduling"); #endif } @@ -2822,31 +2850,31 @@ void Scheduling::ComputeRegisterAntidependencies(Block *b) { // void Scheduling::garbage_collect_pinch_nodes() { #ifndef PRODUCT - if (_cfg->C->trace_opto_output()) tty->print("Reclaimed pinch nodes:"); -#endif - int trace_cnt = 0; - for (uint k = 0; k < _reg_node.Size(); k++) { - Node* pinch = _reg_node[k]; - if ((pinch != NULL) && pinch->Opcode() == Op_Node && - // no predecence input edges - (pinch->req() == pinch->len() || pinch->in(pinch->req()) == NULL) ) { - cleanup_pinch(pinch); - _pinch_free_list.push(pinch); - _reg_node.map(k, NULL); + if (_cfg->C->trace_opto_output()) tty->print("Reclaimed pinch nodes:"); +#endif + int trace_cnt = 0; + for (uint k = 0; k < _reg_node.Size(); k++) { + Node* pinch = _reg_node[k]; + if ((pinch != NULL) && pinch->Opcode() == Op_Node && + // no predecence input edges + (pinch->req() == pinch->len() || pinch->in(pinch->req()) == NULL) ) { + cleanup_pinch(pinch); + _pinch_free_list.push(pinch); + _reg_node.map(k, NULL); #ifndef PRODUCT - if (_cfg->C->trace_opto_output()) { - trace_cnt++; - if (trace_cnt > 40) { - tty->print("\n"); - trace_cnt = 0; - } - tty->print(" %d", pinch->_idx); + if (_cfg->C->trace_opto_output()) { + trace_cnt++; + if (trace_cnt > 40) { + tty->print("\n"); + trace_cnt = 0; } -#endif + tty->print(" %d", pinch->_idx); } +#endif } + } #ifndef PRODUCT - if (_cfg->C->trace_opto_output()) tty->print("\n"); + if (_cfg->C->trace_opto_output()) tty->print("\n"); #endif } @@ -2883,19 +2911,19 @@ void Scheduling::dump_available() const { void Scheduling::print_statistics() { // Print the size added by nops for bundling tty->print("Nops added %d bytes to total of %d bytes", - _total_nop_size, _total_method_size); + _total_nop_size, _total_method_size); if (_total_method_size > 0) tty->print(", for %.2f%%", - ((double)_total_nop_size) / ((double) _total_method_size) * 100.0); + ((double)_total_nop_size) / ((double) _total_method_size) * 100.0); tty->print("\n"); // Print the number of branch shadows filled if (Pipeline::_branch_has_delay_slot) { tty->print("Of %d branches, %d had unconditional delay slots filled", - _total_branches, _total_unconditional_delays); + _total_branches, _total_unconditional_delays); if (_total_branches > 0) tty->print(", for %.2f%%", - ((double)_total_unconditional_delays) / ((double)_total_branches) * 100.0); + ((double)_total_unconditional_delays) / ((double)_total_branches) * 100.0); tty->print("\n"); } @@ -2909,6 +2937,6 @@ void Scheduling::print_statistics() { if (total_bundles > 0) tty->print("Average ILP (excluding nops) is %.2f\n", - ((double)total_instructions) / ((double)total_bundles)); + ((double)total_instructions) / ((double)total_bundles)); } #endif diff --git a/src/hotspot/share/opto/output.hpp b/src/hotspot/share/opto/output.hpp index ab3c1a304..ec3cc2981 100644 --- a/src/hotspot/share/opto/output.hpp +++ b/src/hotspot/share/opto/output.hpp @@ -49,10 +49,7 @@ class Pipeline_Use; // Define the initial sizes for allocation of the resizable code buffer enum { - initial_code_capacity = 16 * 1024, - initial_stub_capacity = 4 * 1024, - initial_const_capacity = 4 * 1024, - initial_locs_capacity = 3 * 1024 + initial_const_capacity = 4 * 1024 }; //------------------------------Scheduling---------------------------------- diff --git a/src/hotspot/share/opto/parse1.cpp b/src/hotspot/share/opto/parse1.cpp index 8286f8c4d..78149369d 100644 --- a/src/hotspot/share/opto/parse1.cpp +++ b/src/hotspot/share/opto/parse1.cpp @@ -41,6 +41,7 @@ #include "runtime/handles.inline.hpp" #include "runtime/safepointMechanism.hpp" #include "runtime/sharedRuntime.hpp" +#include "utilities/bitMap.inline.hpp" #include "utilities/copy.hpp" // Static array so we can figure out which bytecodes stop us from compiling diff --git a/src/hotspot/share/opto/phaseX.cpp b/src/hotspot/share/opto/phaseX.cpp index 41971513f..3d71d941c 100644 --- a/src/hotspot/share/opto/phaseX.cpp +++ b/src/hotspot/share/opto/phaseX.cpp @@ -1726,14 +1726,14 @@ void PhaseIterGVN::add_users_to_worklist( Node *n ) { // of the mirror load depends on the type of 'n'. See LoadNode::Value(). // LoadBarrier?(LoadP(LoadP(AddP(foo:Klass, #java_mirror)))) BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); - bool has_load_barriers = bs->has_load_barriers(); + bool has_load_barrier_nodes = bs->has_load_barrier_nodes(); if (use_op == Op_LoadP && use->bottom_type()->isa_rawptr()) { for (DUIterator_Fast i2max, i2 = use->fast_outs(i2max); i2 < i2max; i2++) { Node* u = use->fast_out(i2); const Type* ut = u->bottom_type(); if (u->Opcode() == Op_LoadP && ut->isa_instptr()) { - if (has_load_barriers) { + if (has_load_barrier_nodes) { // Search for load barriers behind the load for (DUIterator_Fast i3max, i3 = u->fast_outs(i3max); i3 < i3max; i3++) { Node* b = u->fast_out(i3); @@ -1927,14 +1927,14 @@ void PhaseCCP::analyze() { // Loading the java mirror from a Klass requires two loads and the type // of the mirror load depends on the type of 'n'. See LoadNode::Value(). BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); - bool has_load_barriers = bs->has_load_barriers(); + bool has_load_barrier_nodes = bs->has_load_barrier_nodes(); if (m_op == Op_LoadP && m->bottom_type()->isa_rawptr()) { for (DUIterator_Fast i2max, i2 = m->fast_outs(i2max); i2 < i2max; i2++) { Node* u = m->fast_out(i2); const Type* ut = u->bottom_type(); if (u->Opcode() == Op_LoadP && ut->isa_instptr() && ut != type(u)) { - if (has_load_barriers) { + if (has_load_barrier_nodes) { // Search for load barriers behind the load for (DUIterator_Fast i3max, i3 = u->fast_outs(i3max); i3 < i3max; i3++) { Node* b = u->fast_out(i3); diff --git a/src/hotspot/share/opto/vectornode.cpp b/src/hotspot/share/opto/vectornode.cpp index 1ee9db8f0..1f2cf2c64 100644 --- a/src/hotspot/share/opto/vectornode.cpp +++ b/src/hotspot/share/opto/vectornode.cpp @@ -286,7 +286,6 @@ void VectorNode::vector_operands(Node* n, uint* start, uint* end) { case Op_LoadI: case Op_LoadL: case Op_LoadF: case Op_LoadD: case Op_LoadP: case Op_LoadN: - case Op_LoadBarrierSlowReg: *start = 0; *end = 0; // no vector operands break; diff --git a/src/hotspot/share/runtime/sharedRuntime.cpp b/src/hotspot/share/runtime/sharedRuntime.cpp index 595ff7495..3f366633e 100644 --- a/src/hotspot/share/runtime/sharedRuntime.cpp +++ b/src/hotspot/share/runtime/sharedRuntime.cpp @@ -205,6 +205,8 @@ void SharedRuntime::print_ic_miss_histogram() { } #endif // PRODUCT +int SharedRuntime::_monotonical_incr = 0; +bool SharedRuntime::_opt_for_aarch64 = false; JRT_LEAF(jlong, SharedRuntime::lmul(jlong y, jlong x)) return x * y; diff --git a/src/hotspot/share/runtime/sharedRuntime.hpp b/src/hotspot/share/runtime/sharedRuntime.hpp index a11009837..eba82d453 100644 --- a/src/hotspot/share/runtime/sharedRuntime.hpp +++ b/src/hotspot/share/runtime/sharedRuntime.hpp @@ -596,6 +596,11 @@ class SharedRuntime: AllStatic { static void print_ic_miss_histogram(); #endif // PRODUCT + + static int _monotonical_incr; + static bool _opt_for_aarch64; + + static address monotonical_incr_addr() { return (address)&_monotonical_incr; } }; diff --git a/src/hotspot/share/utilities/bitMap.hpp b/src/hotspot/share/utilities/bitMap.hpp index c671535c9..e26f34687 100644 --- a/src/hotspot/share/utilities/bitMap.hpp +++ b/src/hotspot/share/utilities/bitMap.hpp @@ -26,6 +26,7 @@ #define SHARE_VM_UTILITIES_BITMAP_HPP #include "memory/allocation.hpp" +#include "runtime/atomic.hpp" #include "utilities/align.hpp" #include "utilities/globalDefinitions.hpp" @@ -95,6 +96,8 @@ class BitMap { void set_word (idx_t word) { set_word(word, ~(bm_word_t)0); } void clear_word(idx_t word) { _map[word] = 0; } + static inline const bm_word_t load_word_ordered(const volatile bm_word_t* const addr, atomic_memory_order memory_order); + // Utilities for ranges of bits. Ranges are half-open [beg, end). // Ranges within a single word. @@ -194,6 +197,9 @@ class BitMap { return (*word_addr(index) & bit_mask(index)) != 0; } + // memory_order must be memory_order_relaxed or memory_order_acquire. + bool par_at(idx_t index, atomic_memory_order memory_order = memory_order_acquire) const; + // Align bit index up or down to the next bitmap word boundary, or check // alignment. static idx_t word_align_up(idx_t bit) { @@ -210,9 +216,14 @@ class BitMap { inline void set_bit(idx_t bit); inline void clear_bit(idx_t bit); - // Atomically set or clear the specified bit. - inline bool par_set_bit(idx_t bit); - inline bool par_clear_bit(idx_t bit); + // Attempts to change a bit to a desired value. The operation returns true if + // this thread changed the value of the bit. It was changed with a RMW operation + // using the specified memory_order. The operation returns false if the change + // could not be set due to the bit already being observed in the desired state. + // The atomic access that observed the bit in the desired state has acquire + // semantics, unless memory_order is memory_order_relaxed or memory_order_release. + inline bool par_set_bit(idx_t bit, atomic_memory_order memory_order = memory_order_conservative); + inline bool par_clear_bit(idx_t bit, atomic_memory_order memory_order = memory_order_conservative); // Put the given value at the given offset. The parallel version // will CAS the value into the bitmap and is quite a bit slower. diff --git a/src/hotspot/share/utilities/bitMap.inline.hpp b/src/hotspot/share/utilities/bitMap.inline.hpp index b10726d18..7a7e2ad43 100644 --- a/src/hotspot/share/utilities/bitMap.inline.hpp +++ b/src/hotspot/share/utilities/bitMap.inline.hpp @@ -26,6 +26,7 @@ #define SHARE_VM_UTILITIES_BITMAP_INLINE_HPP #include "runtime/atomic.hpp" +#include "runtime/orderAccess.hpp" #include "utilities/bitMap.hpp" inline void BitMap::set_bit(idx_t bit) { @@ -38,18 +39,39 @@ inline void BitMap::clear_bit(idx_t bit) { *word_addr(bit) &= ~bit_mask(bit); } -inline bool BitMap::par_set_bit(idx_t bit) { +inline const BitMap::bm_word_t BitMap::load_word_ordered(const volatile bm_word_t* const addr, atomic_memory_order memory_order) { + if (memory_order == memory_order_relaxed || memory_order == memory_order_release) { + return Atomic::load(addr); + } else { + assert(memory_order == memory_order_acq_rel || + memory_order == memory_order_acquire || + memory_order == memory_order_conservative, + "unexpected memory ordering"); + return OrderAccess::load_acquire(addr); + } +} + +inline bool BitMap::par_at(idx_t index, atomic_memory_order memory_order) const { + verify_index(index); + assert(memory_order == memory_order_acquire || + memory_order == memory_order_relaxed, + "unexpected memory ordering"); + const volatile bm_word_t* const addr = word_addr(index); + return (load_word_ordered(addr, memory_order) & bit_mask(index)) != 0; +} + +inline bool BitMap::par_set_bit(idx_t bit, atomic_memory_order memory_order) { verify_index(bit); volatile bm_word_t* const addr = word_addr(bit); const bm_word_t mask = bit_mask(bit); - bm_word_t old_val = *addr; + bm_word_t old_val = load_word_ordered(addr, memory_order); do { const bm_word_t new_val = old_val | mask; if (new_val == old_val) { return false; // Someone else beat us to it. } - const bm_word_t cur_val = Atomic::cmpxchg(new_val, addr, old_val); + const bm_word_t cur_val = Atomic::cmpxchg(new_val, addr, old_val, memory_order); if (cur_val == old_val) { return true; // Success. } @@ -57,18 +79,18 @@ inline bool BitMap::par_set_bit(idx_t bit) { } while (true); } -inline bool BitMap::par_clear_bit(idx_t bit) { +inline bool BitMap::par_clear_bit(idx_t bit, atomic_memory_order memory_order) { verify_index(bit); volatile bm_word_t* const addr = word_addr(bit); const bm_word_t mask = ~bit_mask(bit); - bm_word_t old_val = *addr; + bm_word_t old_val = load_word_ordered(addr, memory_order); do { const bm_word_t new_val = old_val & mask; if (new_val == old_val) { return false; // Someone else beat us to it. } - const bm_word_t cur_val = Atomic::cmpxchg(new_val, addr, old_val); + const bm_word_t cur_val = Atomic::cmpxchg(new_val, addr, old_val, memory_order); if (cur_val == old_val) { return true; // Success. } diff --git a/src/java.base/share/classes/java/util/Random.java b/src/java.base/share/classes/java/util/Random.java index 92c1193cb..aaf3da581 100644 --- a/src/java.base/share/classes/java/util/Random.java +++ b/src/java.base/share/classes/java/util/Random.java @@ -35,6 +35,7 @@ import java.util.stream.LongStream; import java.util.stream.StreamSupport; import jdk.internal.misc.Unsafe; +import jdk.internal.HotSpotIntrinsicCandidate; /** * An instance of this class is used to generate a stream of @@ -325,6 +326,7 @@ class Random implements java.io.Serializable { * @return the next pseudorandom, uniformly distributed {@code int} * value from this random number generator's sequence */ + @HotSpotIntrinsicCandidate public int nextInt() { return next(32); } diff --git a/test/hotspot/jtreg/runtime/MemberName/MemberNameLeak.java b/test/hotspot/jtreg/runtime/MemberName/MemberNameLeak.java index a8aff4775..afadfd68a 100644 --- a/test/hotspot/jtreg/runtime/MemberName/MemberNameLeak.java +++ b/test/hotspot/jtreg/runtime/MemberName/MemberNameLeak.java @@ -75,6 +75,7 @@ public class MemberNameLeak { test("-XX:+UseG1GC"); test("-XX:+UseParallelGC"); test("-XX:+UseSerialGC"); + test("-XX:+UseZGC"); if (!Compiler.isGraalEnabled()) { // Graal does not support CMS and Shenandoah test("-XX:+UseConcMarkSweepGC"); if (GC.Shenandoah.isSupported()) { -- 2.19.1
Locations
Projects
Search
Status Monitor
Help
Open Build Service
OBS Manuals
API Documentation
OBS Portal
Reporting a Bug
Contact
Mailing List
Forums
Chat (IRC)
Twitter
Open Build Service (OBS)
is an
openSUSE project
.
浙ICP备2022010568号-2