File _service:tar_scm:0019-8040213-C2-does-not-put-a... of Package openjdk-1.8.0

File _service:tar_scm:0019-8040213-C2-does-not-put-all-modified-nodes-on-IGVN-w.patch of Package openjdk-1.8.0 (Revision 617086e2b925dc07422add99d6975b23)

Currently displaying revision 617086e2b925dc07422add99d6975b23 , Show latest

Date: Mon, 5 Jun 2023 20:33:14 +0800
Subject: [PATCH 19/59] 8040213: C2 does not put all modified nodes on IGVN worklist

Bug url: https://bugs.openjdk.org/browse/JDK-8040213
---
 hotspot/src/share/vm/opto/cfgnode.cpp       |  7 +++-
 hotspot/src/share/vm/opto/compile.cpp       | 31 +++++++++++++---
 hotspot/src/share/vm/opto/compile.hpp       |  6 ++++
 hotspot/src/share/vm/opto/divnode.cpp       | 10 ++++--
 hotspot/src/share/vm/opto/loopPredicate.cpp |  9 ++---
 hotspot/src/share/vm/opto/loopTransform.cpp | 27 +++++---------
 hotspot/src/share/vm/opto/loopnode.cpp      | 23 ++++++------
 hotspot/src/share/vm/opto/loopopts.cpp      |  2 +-
 hotspot/src/share/vm/opto/macro.cpp         |  2 ++
 hotspot/src/share/vm/opto/memnode.cpp       | 10 ++++--
 hotspot/src/share/vm/opto/node.cpp          |  4 +++
 hotspot/src/share/vm/opto/node.hpp          |  2 ++
 hotspot/src/share/vm/opto/phaseX.cpp        | 39 +++++++++++++++++----
 hotspot/src/share/vm/opto/rootnode.cpp      |  4 ++-
 14 files changed, 123 insertions(+), 53 deletions(-)

diff --git a/hotspot/src/share/vm/opto/cfgnode.cpp b/hotspot/src/share/vm/opto/cfgnode.cpp
index 82fc957d5..752b603e8 100644
--- a/hotspot/src/share/vm/opto/cfgnode.cpp
+++ b/hotspot/src/share/vm/opto/cfgnode.cpp
@@ -105,6 +105,7 @@ static Node *merge_region(RegionNode *region, PhaseGVN *phase) {
 
         rreq++;                 // One more input to Region
       } // Found a region to merge into Region
+      igvn->_worklist.push(r);
       // Clobber pointer to the now dead 'r'
       region->set_req(i, phase->C->top());
     }
@@ -446,6 +447,7 @@ Node *RegionNode::Ideal(PhaseGVN *phase, bool can_reshape) {
   // Remove TOP or NULL input paths. If only 1 input path remains, this Region
   // degrades to a copy.
   bool add_to_worklist = false;
+  bool modified = false;
   int cnt = 0;                  // Count of values merging
   DEBUG_ONLY( int cnt_orig = req(); ) // Save original inputs count
   int del_it = 0;               // The last input path we delete
@@ -456,6 +458,7 @@ Node *RegionNode::Ideal(PhaseGVN *phase, bool can_reshape) {
       // Remove useless control copy inputs
       if( n->is_Region() && n->as_Region()->is_copy() ) {
         set_req(i, n->nonnull_req());
+        modified = true;
         i--;
         continue;
       }
@@ -463,12 +466,14 @@ Node *RegionNode::Ideal(PhaseGVN *phase, bool can_reshape) {
         Node *call = n->in(0);
         if (call->is_Call() && call->as_Call()->entry_point() == OptoRuntime::rethrow_stub()) {
           set_req(i, call->in(0));
+          modified = true;
           i--;
           continue;
         }
       }
       if( phase->type(n) == Type::TOP ) {
         set_req(i, NULL);       // Ignore TOP inputs
+        modified = true;
         i--;
         continue;
       }
@@ -688,7 +693,7 @@ Node *RegionNode::Ideal(PhaseGVN *phase, bool can_reshape) {
     }
   }
 
-  return NULL;
+  return modified ? this : NULL;
 }
 
 
diff --git a/hotspot/src/share/vm/opto/compile.cpp b/hotspot/src/share/vm/opto/compile.cpp
index 4a32e8a9f..f55437eb3 100644
--- a/hotspot/src/share/vm/opto/compile.cpp
+++ b/hotspot/src/share/vm/opto/compile.cpp
@@ -1071,6 +1071,7 @@ void Compile::Init(int aliaslevel) {
 
   _node_note_array = NULL;
   _default_node_notes = NULL;
+  DEBUG_ONLY( _modified_nodes = NULL; ) // Used in Optimize()
 
   _immutable_memory = NULL; // filled in at first inquiry
 
@@ -1283,6 +1284,19 @@ void Compile::print_missing_nodes() {
     }
   }
 }
+
+void Compile::record_modified_node(Node* n) const {
+  if (_modified_nodes != NULL && !_inlining_incrementally &&
+      n->outcnt() != 0 && !n->is_Con()) {
+    _modified_nodes->push(n);
+  }
+}
+
+void Compile::remove_modified_node(Node* n) const {
+  if (_modified_nodes != NULL) {
+    _modified_nodes->remove(n);
+  }
+}
 #endif
 
 #ifndef PRODUCT
@@ -2123,6 +2137,9 @@ void Compile::Optimize() {
   // Iterative Global Value Numbering, including ideal transforms
   // Initialize IterGVN with types and values from parse-time GVN
   PhaseIterGVN igvn(initial_gvn());
+#ifdef ASSERT
+  _modified_nodes = new (comp_arena()) Unique_Node_List(comp_arena());
+#endif
   {
     NOT_PRODUCT( TracePhase t2("iterGVN", &_t_iterGVN, TimeCompiler); )
     igvn.optimize();
@@ -2308,7 +2325,7 @@ void Compile::Optimize() {
       return;
     }
   }
-
+  DEBUG_ONLY( _modified_nodes = NULL; )
  } // (End scope of igvn; run destructor if necessary for asserts.)
 
   dump_inlining();
@@ -4059,6 +4076,7 @@ void Compile::cleanup_expensive_nodes(PhaseIterGVN &igvn) {
   int j = 0;
   int identical = 0;
   int i = 0;
+  bool modified = false;
   for (; i < _expensive_nodes->length()-1; i++) {
     assert(j <= i, "can't write beyond current index");
     if (_expensive_nodes->at(i)->Opcode() == _expensive_nodes->at(i+1)->Opcode()) {
@@ -4071,20 +4089,23 @@ void Compile::cleanup_expensive_nodes(PhaseIterGVN &igvn) {
       identical = 0;
     } else {
       Node* n = _expensive_nodes->at(i);
-      igvn.hash_delete(n);
-      n->set_req(0, NULL);
+      igvn.replace_input_of(n, 0, NULL);
       igvn.hash_insert(n);
+      modified = true;
     }
   }
   if (identical > 0) {
     _expensive_nodes->at_put(j++, _expensive_nodes->at(i));
   } else if (_expensive_nodes->length() >= 1) {
     Node* n = _expensive_nodes->at(i);
-    igvn.hash_delete(n);
-    n->set_req(0, NULL);
+    igvn.replace_input_of(n, 0, NULL);
     igvn.hash_insert(n);
+    modified = true;
   }
   _expensive_nodes->trunc_to(j);
+  if (modified) {
+    igvn.optimize();
+  }
 }
 
 void Compile::add_expensive_node(Node * n) {
diff --git a/hotspot/src/share/vm/opto/compile.hpp b/hotspot/src/share/vm/opto/compile.hpp
index fb12b6874..20d8324ff 100644
--- a/hotspot/src/share/vm/opto/compile.hpp
+++ b/hotspot/src/share/vm/opto/compile.hpp
@@ -358,6 +358,7 @@ class Compile : public Phase {
   VectorSet             _dead_node_list;        // Set of dead nodes
   uint                  _dead_node_count;       // Number of dead nodes; VectorSet::Size() is O(N).
                                                 // So use this to keep count and make the call O(1).
+  DEBUG_ONLY( Unique_Node_List* _modified_nodes; )  // List of nodes which inputs were modified
   debug_only(static int _debug_idx;)            // Monotonic counter (not reset), use -XX:BreakAtNode=<idx>
   Arena                 _node_arena;            // Arena for new-space Nodes
   Arena                 _old_arena;             // Arena for old-space Nodes, lifetime during xform
@@ -795,6 +796,11 @@ class Compile : public Phase {
   void         print_missing_nodes();
 #endif
 
+  // Record modified nodes to check that they are put on IGVN worklist
+  void   record_modified_node(Node* /* n */) const NOT_DEBUG_RETURN;
+  void   remove_modified_node(Node* /* n */) const NOT_DEBUG_RETURN;
+  DEBUG_ONLY( Unique_Node_List*   modified_nodes() const { return _modified_nodes; } )
+
   // Constant table
   ConstantTable&   constant_table() { return _constant_table; }
 
diff --git a/hotspot/src/share/vm/opto/divnode.cpp b/hotspot/src/share/vm/opto/divnode.cpp
index e1a8ad558..6a893ff3a 100644
--- a/hotspot/src/share/vm/opto/divnode.cpp
+++ b/hotspot/src/share/vm/opto/divnode.cpp
@@ -477,7 +477,10 @@ Node *DivINode::Ideal(PhaseGVN *phase, bool can_reshape) {
 
   if (i == 0) return NULL;      // Dividing by zero constant does not idealize
 
-  set_req(0,NULL);              // Dividing by a not-zero constant; no faulting
+  if (in(0) != NULL) {
+    phase->igvn_rehash_node_delayed(this);
+    set_req(0, NULL);           // Dividing by a not-zero constant; no faulting
+  }
 
   // Dividing by MININT does not optimize as a power-of-2 shift.
   if( i == min_jint ) return NULL;
@@ -576,7 +579,10 @@ Node *DivLNode::Ideal( PhaseGVN *phase, bool can_reshape) {
 
   if (l == 0) return NULL;      // Dividing by zero constant does not idealize
 
-  set_req(0,NULL);              // Dividing by a not-zero constant; no faulting
+  if (in(0) != NULL) {
+    phase->igvn_rehash_node_delayed(this);
+    set_req(0, NULL);           // Dividing by a not-zero constant; no faulting
+  }
 
   // Dividing by MINLONG does not optimize as a power-of-2 shift.
   if( l == min_jlong ) return NULL;
diff --git a/hotspot/src/share/vm/opto/loopPredicate.cpp b/hotspot/src/share/vm/opto/loopPredicate.cpp
index c54f10358..a21702e98 100644
--- a/hotspot/src/share/vm/opto/loopPredicate.cpp
+++ b/hotspot/src/share/vm/opto/loopPredicate.cpp
@@ -106,8 +106,7 @@ ProjNode* PhaseIdealLoop::create_new_if_for_predicate(ProjNode* cont_proj, Node*
     rgn = new (C) RegionNode(1);
     rgn->add_req(uncommon_proj);
     register_control(rgn, loop, uncommon_proj);
-    _igvn.hash_delete(call);
-    call->set_req(0, rgn);
+    _igvn.replace_input_of(call, 0, rgn);
     // When called from beautify_loops() idom is not constructed yet.
     if (_idom != NULL) {
       set_idom(call, rgn, dom_depth(rgn));
@@ -165,8 +164,7 @@ ProjNode* PhaseIdealLoop::create_new_if_for_predicate(ProjNode* cont_proj, Node*
 
   if (new_entry == NULL) {
     // Attach if_cont to iff
-    _igvn.hash_delete(iff);
-    iff->set_req(0, if_cont);
+    _igvn.replace_input_of(iff, 0, if_cont);
     if (_idom != NULL) {
       set_idom(iff, if_cont, dom_depth(iff));
     }
@@ -193,8 +191,7 @@ ProjNode* PhaseIterGVN::create_new_if_for_predicate(ProjNode* cont_proj, Node* n
     rgn = new (C) RegionNode(1);
     register_new_node_with_optimizer(rgn);
     rgn->add_req(uncommon_proj);
-    hash_delete(call);
-    call->set_req(0, rgn);
+    replace_input_of(call, 0, rgn);
   } else {
     // Find region's edge corresponding to uncommon_proj
     for (; proj_index < rgn->req(); proj_index++)
diff --git a/hotspot/src/share/vm/opto/loopTransform.cpp b/hotspot/src/share/vm/opto/loopTransform.cpp
index 7b3f3abe8..414edc26b 100644
--- a/hotspot/src/share/vm/opto/loopTransform.cpp
+++ b/hotspot/src/share/vm/opto/loopTransform.cpp
@@ -934,15 +934,13 @@ void PhaseIdealLoop::insert_pre_post_loops( IdealLoopTree *loop, Node_List &old_
   if( bol->outcnt() != 1 ) {
     bol = bol->clone();
     register_new_node(bol,main_end->in(CountedLoopEndNode::TestControl));
-    _igvn.hash_delete(main_end);
-    main_end->set_req(CountedLoopEndNode::TestValue, bol);
+    _igvn.replace_input_of(main_end, CountedLoopEndNode::TestValue, bol);
   }
   // Need only 1 user of 'cmp' because I will be hacking the loop bounds.
   if( cmp->outcnt() != 1 ) {
     cmp = cmp->clone();
     register_new_node(cmp,main_end->in(CountedLoopEndNode::TestControl));
-    _igvn.hash_delete(bol);
-    bol->set_req(1, cmp);
+    _igvn.replace_input_of(bol, 1, cmp);
   }
 
   //------------------------------
@@ -1146,8 +1144,7 @@ void PhaseIdealLoop::insert_pre_post_loops( IdealLoopTree *loop, Node_List &old_
     Node* pre_bol = pre_end->in(CountedLoopEndNode::TestValue)->as_Bool();
     BoolNode* new_bol0 = new (C) BoolNode(pre_bol->in(1), new_test);
     register_new_node( new_bol0, pre_head->in(0) );
-    _igvn.hash_delete(pre_end);
-    pre_end->set_req(CountedLoopEndNode::TestValue, new_bol0);
+    _igvn.replace_input_of(pre_end, CountedLoopEndNode::TestValue, new_bol0);
     // Modify main loop guard condition
     assert(min_iff->in(CountedLoopEndNode::TestValue) == min_bol, "guard okay");
     BoolNode* new_bol1 = new (C) BoolNode(min_bol->in(1), new_test);
@@ -1158,8 +1155,7 @@ void PhaseIdealLoop::insert_pre_post_loops( IdealLoopTree *loop, Node_List &old_
     BoolNode* main_bol = main_end->in(CountedLoopEndNode::TestValue)->as_Bool();
     BoolNode* new_bol2 = new (C) BoolNode(main_bol->in(1), new_test);
     register_new_node( new_bol2, main_end->in(CountedLoopEndNode::TestControl) );
-    _igvn.hash_delete(main_end);
-    main_end->set_req(CountedLoopEndNode::TestValue, new_bol2);
+    _igvn.replace_input_of(main_end, CountedLoopEndNode::TestValue, new_bol2);
   }
 
   // Flag main loop
@@ -1368,8 +1364,7 @@ void PhaseIdealLoop::do_unroll( IdealLoopTree *loop, Node_List &old_new, bool ad
         Node* bol2 = loop_end->in(1)->clone();
         bol2->set_req(1, cmp2);
         register_new_node(bol2, ctrl2);
-        _igvn.hash_delete(loop_end);
-        loop_end->set_req(1, bol2);
+        _igvn.replace_input_of(loop_end, 1, bol2);
       }
       // Step 3: Find the min-trip test guaranteed before a 'main' loop.
       // Make it a 1-trip test (means at least 2 trips).
@@ -1378,8 +1373,7 @@ void PhaseIdealLoop::do_unroll( IdealLoopTree *loop, Node_List &old_new, bool ad
       // can edit it's inputs directly.  Hammer in the new limit for the
       // minimum-trip guard.
       assert(opaq->outcnt() == 1, "");
-      _igvn.hash_delete(opaq);
-      opaq->set_req(1, new_limit);
+      _igvn.replace_input_of(opaq, 1, new_limit);
     }
 
     // Adjust max trip count. The trip count is intentionally rounded
@@ -1429,8 +1423,7 @@ void PhaseIdealLoop::do_unroll( IdealLoopTree *loop, Node_List &old_new, bool ad
     register_new_node( cmp2, ctrl2 );
     Node *bol2 = new (C) BoolNode( cmp2, loop_end->test_trip() );
     register_new_node( bol2, ctrl2 );
-    _igvn.hash_delete(loop_end);
-    loop_end->set_req(CountedLoopEndNode::TestValue, bol2);
+    _igvn.replace_input_of(loop_end, CountedLoopEndNode::TestValue, bol2);
 
     // Step 3: Find the min-trip test guaranteed before a 'main' loop.
     // Make it a 1-trip test (means at least 2 trips).
@@ -1978,8 +1971,7 @@ void PhaseIdealLoop::do_range_check( IdealLoopTree *loop, Node_List &old_new ) {
                                  : (Node*)new (C) MaxINode(pre_limit, orig_limit);
     register_new_node(pre_limit, pre_ctrl);
   }
-  _igvn.hash_delete(pre_opaq);
-  pre_opaq->set_req(1, pre_limit);
+  _igvn.replace_input_of(pre_opaq, 1, pre_limit);
 
   // Note:: we are making the main loop limit no longer precise;
   // need to round up based on stride.
@@ -2008,10 +2000,9 @@ void PhaseIdealLoop::do_range_check( IdealLoopTree *loop, Node_List &old_new ) {
   Node *main_bol = main_cle->in(1);
   // Hacking loop bounds; need private copies of exit test
   if( main_bol->outcnt() > 1 ) {// BoolNode shared?
-    _igvn.hash_delete(main_cle);
     main_bol = main_bol->clone();// Clone a private BoolNode
     register_new_node( main_bol, main_cle->in(0) );
-    main_cle->set_req(1,main_bol);
+    _igvn.replace_input_of(main_cle, 1, main_bol);
   }
   Node *main_cmp = main_bol->in(1);
   if( main_cmp->outcnt() > 1 ) { // CmpNode shared?
diff --git a/hotspot/src/share/vm/opto/loopnode.cpp b/hotspot/src/share/vm/opto/loopnode.cpp
index d8143821b..1b7c768df 100644
--- a/hotspot/src/share/vm/opto/loopnode.cpp
+++ b/hotspot/src/share/vm/opto/loopnode.cpp
@@ -132,7 +132,7 @@ Node *PhaseIdealLoop::get_early_ctrl( Node *n ) {
   // Return earliest legal location
   assert(early == find_non_split_ctrl(early), "unexpected early control");
 
-  if (n->is_expensive()) {
+  if (n->is_expensive() && !_verify_only && !_verify_me) {
     assert(n->in(0), "should have control input");
     early = get_early_ctrl_for_expensive(n, early);
   }
@@ -225,8 +225,7 @@ Node *PhaseIdealLoop::get_early_ctrl_for_expensive(Node *n, Node* earliest) {
   }
 
   if (ctl != n->in(0)) {
-    _igvn.hash_delete(n);
-    n->set_req(0, ctl);
+    _igvn.replace_input_of(n, 0, ctl);
     _igvn.hash_insert(n);
   }
 
@@ -712,7 +710,7 @@ bool PhaseIdealLoop::is_counted_loop( Node *x, IdealLoopTree *loop ) {
   incr->set_req(2,stride);
   incr = _igvn.register_new_node_with_optimizer(incr);
   set_early_ctrl( incr );
-  _igvn.hash_delete(phi);
+  _igvn.rehash_node_delayed(phi);
   phi->set_req_X( LoopNode::LoopBackControl, incr, &_igvn );
 
   // If phi type is more restrictive than Int, raise to
@@ -765,8 +763,8 @@ bool PhaseIdealLoop::is_counted_loop( Node *x, IdealLoopTree *loop ) {
     iffalse = iff2;
     iftrue  = ift2;
   } else {
-    _igvn.hash_delete(iffalse);
-    _igvn.hash_delete(iftrue);
+    _igvn.rehash_node_delayed(iffalse);
+    _igvn.rehash_node_delayed(iftrue);
     iffalse->set_req_X( 0, le, &_igvn );
     iftrue ->set_req_X( 0, le, &_igvn );
   }
@@ -1282,6 +1280,7 @@ void IdealLoopTree::split_fall_in( PhaseIdealLoop *phase, int fall_in_cnt ) {
       _head->del_req(i);
     }
   }
+  igvn.rehash_node_delayed(_head);
   // Transform landing pad
   igvn.register_new_node_with_optimizer(landing_pad, _head);
   // Insert landing pad into the header
@@ -1422,7 +1421,7 @@ void IdealLoopTree::merge_many_backedges( PhaseIdealLoop *phase ) {
   igvn.register_new_node_with_optimizer(r, _head);
   // Plug region into end of loop _head, followed by hot_tail
   while( _head->req() > 3 ) _head->del_req( _head->req()-1 );
-  _head->set_req(2, r);
+  igvn.replace_input_of(_head, 2, r);
   if( hot_idx ) _head->add_req(hot_tail);
 
   // Split all the Phis up between '_head' loop and the Region 'r'
@@ -1444,7 +1443,7 @@ void IdealLoopTree::merge_many_backedges( PhaseIdealLoop *phase ) {
       igvn.register_new_node_with_optimizer(phi, n);
       // Add the merge phi to the old Phi
       while( n->req() > 3 ) n->del_req( n->req()-1 );
-      n->set_req(2, phi);
+      igvn.replace_input_of(n, 2, phi);
       if( hot_idx ) n->add_req(hot_phi);
     }
   }
@@ -1520,13 +1519,14 @@ bool IdealLoopTree::beautify_loops( PhaseIdealLoop *phase ) {
   if( fall_in_cnt > 1 ) {
     // Since I am just swapping inputs I do not need to update def-use info
     Node *tmp = _head->in(1);
+    igvn.rehash_node_delayed(_head);
     _head->set_req( 1, _head->in(fall_in_cnt) );
     _head->set_req( fall_in_cnt, tmp );
     // Swap also all Phis
     for (DUIterator_Fast imax, i = _head->fast_outs(imax); i < imax; i++) {
       Node* phi = _head->fast_out(i);
       if( phi->is_Phi() ) {
-        igvn.hash_delete(phi); // Yank from hash before hacking edges
+        igvn.rehash_node_delayed(phi); // Yank from hash before hacking edges
         tmp = phi->in(1);
         phi->set_req( 1, phi->in(fall_in_cnt) );
         phi->set_req( fall_in_cnt, tmp );
@@ -2970,6 +2970,7 @@ int PhaseIdealLoop::build_loop_tree_impl( Node *n, int pre_order ) {
           uint k = 0;             // Probably cfg->in(0)
           while( cfg->in(k) != m ) k++; // But check incase cfg is a Region
           cfg->set_req( k, if_t ); // Now point to NeverBranch
+          _igvn._worklist.push(cfg);
 
           // Now create the never-taken loop exit
           Node *if_f = new (C) CProjNode( iff, 1 );
diff --git a/hotspot/src/share/vm/opto/loopopts.cpp b/hotspot/src/share/vm/opto/loopopts.cpp
index 28bfcb75b..006b1a203 100644
--- a/hotspot/src/share/vm/opto/loopopts.cpp
+++ b/hotspot/src/share/vm/opto/loopopts.cpp
@@ -2825,7 +2825,7 @@ bool PhaseIdealLoop::partial_peel( IdealLoopTree *loop, Node_List &old_new ) {
   new_head->set_unswitch_count(head->unswitch_count()); // Preserve
   _igvn.register_new_node_with_optimizer(new_head);
   assert(first_not_peeled->in(0) == last_peel, "last_peel <- first_not_peeled");
-  first_not_peeled->set_req(0, new_head);
+  _igvn.replace_input_of(first_not_peeled, 0, new_head);
   set_loop(new_head, loop);
   loop->_body.push(new_head);
   not_peel.set(new_head->_idx);
diff --git a/hotspot/src/share/vm/opto/macro.cpp b/hotspot/src/share/vm/opto/macro.cpp
index bd8b528e1..86c5b7aa0 100644
--- a/hotspot/src/share/vm/opto/macro.cpp
+++ b/hotspot/src/share/vm/opto/macro.cpp
@@ -853,6 +853,7 @@ bool PhaseMacroExpand::scalar_replacement(AllocateNode *alloc, GrowableArray <Sa
     int start = jvms->debug_start();
     int end   = jvms->debug_end();
     sfpt->replace_edges_in_range(res, sobj, start, end);
+    _igvn._worklist.push(sfpt);
     safepoints_done.append_if_missing(sfpt); // keep it for rollback
   }
   return true;
@@ -1793,6 +1794,7 @@ Node* PhaseMacroExpand::prefetch_allocation(Node* i_o, Node*& needgc_false,
       Node *pf_region = new (C) RegionNode(3);
       Node *pf_phi_rawmem = new (C) PhiNode( pf_region, Type::MEMORY,
                                              TypeRawPtr::BOTTOM );
+      transform_later(pf_region);
 
       // Generate several prefetch instructions.
       uint lines = (length != NULL) ? AllocatePrefetchLines : AllocateInstancePrefetchLines;
diff --git a/hotspot/src/share/vm/opto/memnode.cpp b/hotspot/src/share/vm/opto/memnode.cpp
index 8d62fcc7e..db867c95a 100644
--- a/hotspot/src/share/vm/opto/memnode.cpp
+++ b/hotspot/src/share/vm/opto/memnode.cpp
@@ -1521,6 +1521,7 @@ Node *LoadNode::Ideal(PhaseGVN *phase, bool can_reshape) {
 
   Node* ctrl    = in(MemNode::Control);
   Node* address = in(MemNode::Address);
+  bool progress = false;
 
   bool addr_mark = ((phase->type(address)->isa_oopptr() || phase->type(address)->isa_narrowoop()) &&
          phase->type(address)->is_ptr()->offset() == oopDesc::mark_offset_in_bytes());
@@ -1532,6 +1533,7 @@ Node *LoadNode::Ideal(PhaseGVN *phase, bool can_reshape) {
       !addr_mark ) {
     ctrl = ctrl->in(0);
     set_req(MemNode::Control,ctrl);
+    progress = true;
   }
 
   intptr_t ignore = 0;
@@ -1545,6 +1547,7 @@ Node *LoadNode::Ideal(PhaseGVN *phase, bool can_reshape) {
         && all_controls_dominate(base, phase->C->start())) {
       // A method-invariant, non-null address (constant or 'this' argument).
       set_req(MemNode::Control, NULL);
+      progress = true;
     }
   }
 
@@ -1605,7 +1608,7 @@ Node *LoadNode::Ideal(PhaseGVN *phase, bool can_reshape) {
     }
   }
 
-  return NULL;                  // No further progress
+  return progress ? this : NULL;
 }
 
 // Helper to recognize certain Klass fields which are invariant across
@@ -3151,6 +3154,7 @@ Node *MemBarNode::Ideal(PhaseGVN *phase, bool can_reshape) {
     return NULL;
   }
 
+  bool progress = false;
   // Eliminate volatile MemBars for scalar replaced objects.
   if (can_reshape && req() == (Precedent+1)) {
     bool eliminate = false;
@@ -3173,6 +3177,7 @@ Node *MemBarNode::Ideal(PhaseGVN *phase, bool can_reshape) {
           phase->is_IterGVN()->_worklist.push(my_mem); // remove dead node later
           my_mem = NULL;
         }
+        progress = true;
       }
       if (my_mem != NULL && my_mem->is_Mem()) {
         const TypeOopPtr* t_oop = my_mem->in(MemNode::Address)->bottom_type()->isa_oopptr();
@@ -3203,7 +3208,7 @@ Node *MemBarNode::Ideal(PhaseGVN *phase, bool can_reshape) {
       return new (phase->C) ConINode(TypeInt::ZERO);
     }
   }
-  return NULL;
+  return progress ? this : NULL;
 }
 
 //------------------------------Value------------------------------------------
@@ -3840,6 +3845,7 @@ Node* InitializeNode::capture_store(StoreNode* st, intptr_t start,
   // if it redundantly stored the same value (or zero to fresh memory).
 
   // In any case, wire it in:
+  phase->igvn_rehash_node_delayed(this);
   set_req(i, new_st);
 
   // The caller may now kill the old guy.
diff --git a/hotspot/src/share/vm/opto/node.cpp b/hotspot/src/share/vm/opto/node.cpp
index 7ea783e54..2e7a74b83 100644
--- a/hotspot/src/share/vm/opto/node.cpp
+++ b/hotspot/src/share/vm/opto/node.cpp
@@ -675,6 +675,7 @@ void Node::destruct() {
   *(address*)this = badAddress;  // smash the C++ vtbl, probably
   _in = _out = (Node**) badAddress;
   _max = _cnt = _outmax = _outcnt = 0;
+  compile->remove_modified_node(this);
 #endif
 }
 
@@ -821,6 +822,7 @@ void Node::del_req( uint idx ) {
   _in[idx] = in(--_cnt); // Compact the array
   // Avoid spec violation: Gap in prec edges.
   close_prec_gap_at(_cnt);
+  Compile::current()->record_modified_node(this);
 }
 
 //------------------------------del_req_ordered--------------------------------
@@ -837,6 +839,7 @@ void Node::del_req_ordered( uint idx ) {
   }
   // Avoid spec violation: Gap in prec edges.
   close_prec_gap_at(_cnt);
+  Compile::current()->record_modified_node(this);
 }
 
 //------------------------------ins_req----------------------------------------
@@ -1376,6 +1379,7 @@ static void kill_dead_code( Node *dead, PhaseIterGVN *igvn ) {
       // Done with outputs.
       igvn->hash_delete(dead);
       igvn->_worklist.remove(dead);
+      igvn->C->remove_modified_node(dead);
       igvn->set_type(dead, Type::TOP);
       if (dead->is_macro()) {
         igvn->C->remove_macro_node(dead);
diff --git a/hotspot/src/share/vm/opto/node.hpp b/hotspot/src/share/vm/opto/node.hpp
index f0a6ee061..0a4b94acc 100644
--- a/hotspot/src/share/vm/opto/node.hpp
+++ b/hotspot/src/share/vm/opto/node.hpp
@@ -409,6 +409,7 @@ protected:
     if (*p != NULL)  (*p)->del_out((Node *)this);
     (*p) = n;
     if (n != NULL)      n->add_out((Node *)this);
+    Compile::current()->record_modified_node(this);
   }
   // Light version of set_req() to init inputs after node creation.
   void init_req( uint i, Node *n ) {
@@ -420,6 +421,7 @@ protected:
     assert( _in[i] == NULL, "sanity");
     _in[i] = n;
     if (n != NULL)      n->add_out((Node *)this);
+    Compile::current()->record_modified_node(this);
   }
   // Find first occurrence of n among my edges:
   int find_edge(Node* n);
diff --git a/hotspot/src/share/vm/opto/phaseX.cpp b/hotspot/src/share/vm/opto/phaseX.cpp
index ae1031d31..67c06317b 100644
--- a/hotspot/src/share/vm/opto/phaseX.cpp
+++ b/hotspot/src/share/vm/opto/phaseX.cpp
@@ -950,6 +950,17 @@ void PhaseIterGVN::optimize() {
     for ( int i = 0; i < _verify_window_size; i++ ) {
       _verify_window[i] = NULL;
     }
+#ifdef ASSERT
+    // Verify that all modified nodes are on _worklist
+    Unique_Node_List* modified_list = C->modified_nodes();
+    while (modified_list != NULL && modified_list->size()) {
+      Node* n = modified_list->pop();
+      if (n->outcnt() != 0 && !n->is_Con() && !_worklist.member(n)) {
+        n->dump();
+        assert(false, "modified node is not on IGVN._worklist");
+      }
+    }
+#endif
   }
 #endif
 
@@ -1048,6 +1059,17 @@ void PhaseIterGVN::optimize() {
   }
 
 #ifndef PRODUCT
+#ifdef ASSERT
+  // Verify nodes with changed inputs.
+  Unique_Node_List* modified_list = C->modified_nodes();
+  while (modified_list != NULL && modified_list->size()) {
+    Node* n = modified_list->pop();
+    if (n->outcnt() != 0 && !n->is_Con()) { // skip dead and Con nodes
+      n->dump();
+      assert(false, "modified node was not processed by IGVN.transform_old()");
+    }
+  }
+#endif
   C->verify_graph_edges();
   if( VerifyOpto && allow_progress() ) {
     // Must turn off allow_progress to enable assert and break recursion
@@ -1075,6 +1097,13 @@ void PhaseIterGVN::optimize() {
       tty->print_cr("VerifyIterativeGVN: %d transforms, %d full verify passes",
                   (int) _verify_counter, (int) _verify_full_passes);
   }
+#ifdef ASSERT
+  while (modified_list->size()) {
+    Node* n = modified_list->pop();
+    n->dump();
+    assert(false, "VerifyIterativeGVN: new modified node was added");
+  }
+#endif
 #endif
 }
 
@@ -1123,6 +1152,7 @@ Node *PhaseIterGVN::transform_old( Node *n ) {
   Node *k = n;
   DEBUG_ONLY(dead_loop_check(k);)
   DEBUG_ONLY(bool is_new = (k->outcnt() == 0);)
+  C->remove_modified_node(k);
   Node *i = k->Ideal(this, /*can_reshape=*/true);
   assert(i != k || is_new || i->outcnt() > 0, "don't return dead nodes");
 #ifndef PRODUCT
@@ -1163,6 +1193,7 @@ Node *PhaseIterGVN::transform_old( Node *n ) {
     DEBUG_ONLY(dead_loop_check(k);)
     // Try idealizing again
     DEBUG_ONLY(is_new = (k->outcnt() == 0);)
+    C->remove_modified_node(k);
     i = k->Ideal(this, /*can_reshape=*/true);
     assert(i != k || is_new || i->outcnt() > 0, "don't return dead nodes");
 #ifndef PRODUCT
@@ -1316,6 +1347,7 @@ void PhaseIterGVN::remove_globally_dead_node( Node *dead ) {
       _stack.pop();
       // Remove dead node from iterative worklist
       _worklist.remove(dead);
+      C->remove_modified_node(dead);
       // Constant node that has no out-edges and has only one in-edge from
       // root is usually dead. However, sometimes reshaping walk makes
       // it reachable by adding use edges. So, we will NOT count Con nodes
@@ -1352,7 +1384,7 @@ void PhaseIterGVN::subsume_node( Node *old, Node *nn ) {
   for (DUIterator_Last imin, i = old->last_outs(imin); i >= imin; ) {
     Node* use = old->last_out(i);  // for each use...
     // use might need re-hashing (but it won't if it's a new node)
-    bool is_in_table = _table.hash_delete( use );
+    rehash_node_delayed(use);
     // Update use-def info as well
     // We remove all occurrences of old within use->in,
     // so as to avoid rehashing any node more than once.
@@ -1364,11 +1396,6 @@ void PhaseIterGVN::subsume_node( Node *old, Node *nn ) {
         ++num_edges;
       }
     }
-    // Insert into GVN hash table if unique
-    // If a duplicate, 'use' will be cleaned up when pulled off worklist
-    if( is_in_table ) {
-      hash_find_insert(use);
-    }
     i -= num_edges;    // we deleted 1 or more copies of this edge
   }
 
diff --git a/hotspot/src/share/vm/opto/rootnode.cpp b/hotspot/src/share/vm/opto/rootnode.cpp
index 56775ed7e..4cf51528d 100644
--- a/hotspot/src/share/vm/opto/rootnode.cpp
+++ b/hotspot/src/share/vm/opto/rootnode.cpp
@@ -35,10 +35,12 @@
 //------------------------------Ideal------------------------------------------
 // Remove dead inputs
 Node *RootNode::Ideal(PhaseGVN *phase, bool can_reshape) {
+  bool modified = false;
   for( uint i = 1; i < req(); i++ ) { // For all inputs
     // Check for and remove dead inputs
     if( phase->type(in(i)) == Type::TOP ) {
       del_req(i--);             // Delete TOP inputs
+      modified = true;
     }
   }
 
@@ -56,7 +58,7 @@ Node *RootNode::Ideal(PhaseGVN *phase, bool can_reshape) {
   // If we want to get the rest of the win later, we should pattern match
   // simple recursive call trees to closed-form solutions.
 
-  return NULL;                  // No further opportunities exposed
+  return modified ? this : NULL;
 }
 
 //=============================================================================
-- 
2.22.0