8372779: C2: Disambiguate Node::adr_type for the IR graph

src/hotspot/share/gc/shared/c2/barrierSetC2.cpp

@@ -744,7 +744,7 @@ void BarrierSetC2::clone(GraphKit* kit, Node* src_base, Node* dst_base, Node* si
   Node* n = kit->gvn().transform(ac);
   if (n == ac) {
     const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
-    ac->set_adr_type(TypeRawPtr::BOTTOM);
+    ac->set_out_adr_type(TypeRawPtr::BOTTOM);
     kit->set_predefined_output_for_runtime_call(ac, ac->in(TypeFunc::Memory), raw_adr_type);
   } else {
     kit->set_all_memory(n);

src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp

@@ -886,7 +886,7 @@ void ShenandoahBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCo
 
     enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT };
     Node* region = new RegionNode(PATH_LIMIT);
-    Node* mem_phi = new PhiNode(region, Type::MEMORY, TypeRawPtr::BOTTOM);
+    Node* mem_phi = new PhiNode(region, Type::MEMORY, TypePtr::BOTTOM);
 
     Node* thread = phase->transform_later(new ThreadLocalNode());
     Node* offset = phase->igvn().MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
@@ -914,7 +914,7 @@ void ShenandoahBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCo
                                        ShenandoahBarrierSetC2::clone_barrier_Type(),
                                        CAST_FROM_FN_PTR(address, ShenandoahRuntime::clone_barrier),
                                        "shenandoah_clone",
-                                       TypeRawPtr::BOTTOM,
+                                       TypePtr::BOTTOM,
                                        src_base);
     call = phase->transform_later(call);
 

src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp

@@ -38,6 +38,7 @@
 #include "opto/callnode.hpp"
 #include "opto/castnode.hpp"
 #include "opto/movenode.hpp"
+#include "opto/opcodes.hpp"
 #include "opto/phaseX.hpp"
 #include "opto/rootnode.hpp"
 #include "opto/runtime.hpp"
@@ -802,7 +803,7 @@ Node* ShenandoahBarrierC2Support::find_bottom_mem(Node* ctrl, PhaseIdealLoop* ph
         }
       }
     } else {
-      if (c->is_Call() && c->as_Call()->adr_type() != nullptr) {
+      if (c->is_Call() && c->as_Call()->out_adr_type() != nullptr) {
         CallProjections projs;
         c->as_Call()->extract_projections(&projs, true, false);
         if (projs.fallthrough_memproj != nullptr) {
@@ -836,7 +837,7 @@ Node* ShenandoahBarrierC2Support::find_bottom_mem(Node* ctrl, PhaseIdealLoop* ph
               mem = u;
           }
         }
-        assert(!c->is_Call() || c->as_Call()->adr_type() != nullptr || mem == nullptr, "no mem projection expected");
+        assert(!c->is_Call() || c->as_Call()->out_adr_type() != nullptr || mem == nullptr, "no mem projection expected");
       }
     }
     c = phase->idom(c);
@@ -1003,7 +1004,7 @@ void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node* lo
       name = "load_reference_barrier_phantom";
     }
   }
-  Node* call = new CallLeafNode(ShenandoahBarrierSetC2::load_reference_barrier_Type(), calladdr, name, TypeRawPtr::BOTTOM);
+  Node* call = new CallLeafNode(ShenandoahBarrierSetC2::load_reference_barrier_Type(), calladdr, name, nullptr, nullptr);
 
   call->init_req(TypeFunc::Control, ctrl);
   call->init_req(TypeFunc::I_O, phase->C->top());
@@ -1601,6 +1602,9 @@ void MemoryGraphFixer::collect_memory_nodes() {
           assert(_alias == Compile::AliasIdxRaw, "");
           stack.push(mem, mem->req());
           mem = mem->in(MemNode::Memory);
+        } else if (mem->Opcode() == Op_StrInflatedCopy || mem->Opcode() == Op_StrCompressedCopy || mem->Opcode() == Op_EncodeISOArray) {
+          stack.push(mem, mem->req());
+          mem = mem->in(1);
         } else {
 #ifdef ASSERT
           mem->dump();
@@ -2145,29 +2149,30 @@ void MemoryGraphFixer::fix_mem(Node* ctrl, Node* new_ctrl, Node* mem, Node* mem_
             --i;
           }
         }
-      } else if ((u->adr_type() == TypePtr::BOTTOM && u->Opcode() != Op_StrInflatedCopy) ||
-                 u->adr_type() == nullptr) {
-        assert(u->adr_type() != nullptr ||
+      } else if ((u->in_adr_type() == TypePtr::BOTTOM &&
+                  u->Opcode() != Op_StrInflatedCopy && mem->Opcode() != Op_StrCompressedCopy && mem->Opcode() != Op_EncodeISOArray) ||
+                 u->in_adr_type() == nullptr) {
+        assert(u->in_adr_type() != nullptr ||
                u->Opcode() == Op_Rethrow ||
                u->Opcode() == Op_Return ||
                u->Opcode() == Op_SafePoint ||
                (u->is_CallStaticJava() && u->as_CallStaticJava()->uncommon_trap_request() != 0) ||
                (u->is_CallStaticJava() && u->as_CallStaticJava()->_entry_point == OptoRuntime::rethrow_stub()) ||
-               u->Opcode() == Op_CallLeaf, "");
+               u->Opcode() == Op_CallLeaf || u->Opcode() == Op_CallLeafNoFP, "");
         Node* m = find_mem(_phase->ctrl_or_self(u), u);
         if (m != mem) {
           mm = allocate_merge_mem(mem, m, _phase->get_ctrl(m));
           _phase->igvn().replace_input_of(u, u->find_edge(mem), mm);
           --i;
         }
-      } else if (_phase->C->get_alias_index(u->adr_type()) == alias) {
+      } else if (_phase->C->get_alias_index(u->in_adr_type()) == alias) {
         Node* m = find_mem(_phase->ctrl_or_self(u), u);
         if (m != mem) {
           DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of use"); u->dump(); });
           _phase->igvn().replace_input_of(u, u->find_edge(mem), m);
           --i;
         }
-      } else if (u->adr_type() != TypePtr::BOTTOM &&
+      } else if (u->in_adr_type() != TypePtr::BOTTOM &&
                  _memory_nodes[_phase->ctrl_or_self(u)->_idx] == u) {
         Node* m = find_mem(_phase->ctrl_or_self(u), u);
         assert(m != mem, "");
@@ -2331,23 +2336,24 @@ void MemoryGraphFixer::fix_memory_uses(Node* mem, Node* replacement, Node* rep_p
           }
 
         }
-      } else if ((u->adr_type() == TypePtr::BOTTOM && u->Opcode() != Op_StrInflatedCopy) ||
-                 u->adr_type() == nullptr) {
-        assert(u->adr_type() != nullptr ||
+      } else if ((u->in_adr_type() == TypePtr::BOTTOM &&
+                  u->Opcode() != Op_StrInflatedCopy && mem->Opcode() != Op_StrCompressedCopy && mem->Opcode() != Op_EncodeISOArray) ||
+                 u->in_adr_type() == nullptr) {
+        assert(u->in_adr_type() != nullptr ||
                u->Opcode() == Op_Rethrow ||
                u->Opcode() == Op_Return ||
                u->Opcode() == Op_SafePoint ||
                (u->is_CallStaticJava() && u->as_CallStaticJava()->uncommon_trap_request() != 0) ||
                (u->is_CallStaticJava() && u->as_CallStaticJava()->_entry_point == OptoRuntime::rethrow_stub()) ||
-               u->Opcode() == Op_CallLeaf, "%s", u->Name());
+               u->Opcode() == Op_CallLeaf || u->Opcode() == Op_CallLeafNoFP, "%s", u->Name());
         if (ShenandoahBarrierC2Support::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) {
           if (mm == nullptr) {
             mm = allocate_merge_mem(mem, rep_proj, rep_ctrl);
           }
           _phase->igvn().replace_input_of(u, u->find_edge(mem), mm);
           --i;
         }
-      } else if (_phase->C->get_alias_index(u->adr_type()) == _alias) {
+      } else if (_phase->C->get_alias_index(u->in_adr_type()) == _alias) {
         if (ShenandoahBarrierC2Support::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) {
           _phase->igvn().replace_input_of(u, u->find_edge(mem), rep_proj);
           --i;

src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp

@@ -230,7 +230,6 @@ class ShenandoahLoadReferenceBarrierNode : public Node {
   virtual int Opcode() const;
   virtual const Type* bottom_type() const;
   virtual const Type* Value(PhaseGVN* phase) const;
-  virtual const class TypePtr *adr_type() const { return TypeOopPtr::BOTTOM; }
   virtual uint match_edge(uint idx) const {
     return idx >= ValueIn;
   }

src/hotspot/share/opto/arraycopynode.cpp

@@ -33,7 +33,7 @@
 const TypeFunc* ArrayCopyNode::_arraycopy_type_Type = nullptr;
 
 ArrayCopyNode::ArrayCopyNode(Compile* C, bool alloc_tightly_coupled, bool has_negative_length_guard)
-  : CallNode(arraycopy_type(), nullptr, TypePtr::BOTTOM),
+  : CallNode(arraycopy_type(), nullptr, TypePtr::BOTTOM, TypePtr::BOTTOM),
     _kind(None),
     _alloc_tightly_coupled(alloc_tightly_coupled),
     _has_negative_length_guard(has_negative_length_guard),

src/hotspot/share/opto/callnode.cpp

@@ -759,6 +759,10 @@ const Type* CallNode::Value(PhaseGVN* phase) const {
   if (in(0) == nullptr || phase->type(in(0)) == Type::TOP) {
     return Type::TOP;
   }
+  if (in_adr_type() != nullptr && in(TypeFunc::Memory)->is_top()) {
+    // The memory input died, this node should be dead, too
+    return Type::TOP;
+  }
   return tf()->range();
 }
 
@@ -1689,10 +1693,12 @@ void SafePointScalarMergeNode::dump_spec(outputStream *st) const {
 //=============================================================================
 uint AllocateNode::size_of() const { return sizeof(*this); }
 
+// The entire memory state is needed for slow path of the allocation since GC and deoptimization
+// can happened, so in_adr_type = TypePtr::BOTTOM
 AllocateNode::AllocateNode(Compile* C, const TypeFunc *atype,
                            Node *ctrl, Node *mem, Node *abio,
                            Node *size, Node *klass_node, Node *initial_test)
-  : CallNode(atype, nullptr, TypeRawPtr::BOTTOM)
+  : CallNode(atype, nullptr, TypeRawPtr::BOTTOM, TypePtr::BOTTOM)
 {
   init_class_id(Class_Allocate);
   init_flags(Flag_is_macro);

src/hotspot/share/opto/callnode.hpp

@@ -71,7 +71,6 @@ class StartNode : public MultiNode {
   virtual int Opcode() const;
   virtual bool pinned() const { return true; };
   virtual const Type *bottom_type() const;
-  virtual const TypePtr *adr_type() const { return TypePtr::BOTTOM; }
   virtual const Type* Value(PhaseGVN* phase) const;
   virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
   virtual void  calling_convention( BasicType* sig_bt, VMRegPair *parm_reg, uint length ) const;
@@ -82,6 +81,10 @@ class StartNode : public MultiNode {
   virtual void  dump_spec(outputStream *st) const;
   virtual void  dump_compact_spec(outputStream *st) const;
 #endif
+
+private:
+  virtual const TypePtr* in_adr_type_impl() const { return nullptr; }
+  virtual const TypePtr* out_adr_type_impl() const { return TypePtr::BOTTOM; }
 };
 
 //------------------------------StartOSRNode-----------------------------------
@@ -341,10 +344,10 @@ class SafePointNode : public MultiNode {
 
 protected:
   JVMState* const _jvms;      // Pointer to list of JVM State objects
-  // Many calls take *all* of memory as input,
-  // but some produce a limited subset of that memory as output.
-  // The adr_type reports the call's behavior as a store, not a load.
-  const TypePtr*  _adr_type;  // What type of memory does this node produce?
+  // Java calls consume and produce all memory, pure calls do not consume or produce memory,
+  // runtime calls have various memory effects
+  const TypePtr*  _out_adr_type;
+  const TypePtr*  _in_adr_type;
   ReplacedNodes   _replaced_nodes; // During parsing: list of pair of nodes from calls to GraphKit::replace_in_map()
   bool            _has_ea_local_in_scope; // NoEscape or ArgEscape objects in JVM States
 
@@ -355,10 +358,11 @@ class SafePointNode : public MultiNode {
 public:
   SafePointNode(uint edges, JVMState* jvms,
                 // A plain safepoint advertises no memory effects (null):
-                const TypePtr* adr_type = nullptr)
+                const TypePtr* out_adr_type = nullptr, const TypePtr* in_adr_type = nullptr)
     : MultiNode( edges ),
       _jvms(jvms),
-      _adr_type(adr_type),
+      _out_adr_type(out_adr_type),
+      _in_adr_type(in_adr_type),
       _has_ea_local_in_scope(false)
   {
     init_class_id(Class_SafePoint);
@@ -512,8 +516,7 @@ class SafePointNode : public MultiNode {
   virtual bool           pinned() const { return true; }
   virtual const Type*    Value(PhaseGVN* phase) const;
   virtual const Type*    bottom_type() const { return Type::CONTROL; }
-  virtual const TypePtr* adr_type() const { return _adr_type; }
-  void set_adr_type(const TypePtr* adr_type) { _adr_type = adr_type; }
+  void set_out_adr_type(const TypePtr* adr_type) { _out_adr_type = adr_type; }
   virtual Node          *Ideal(PhaseGVN *phase, bool can_reshape);
   virtual Node*          Identity(PhaseGVN* phase);
   virtual uint           ideal_reg() const { return 0; }
@@ -524,6 +527,10 @@ class SafePointNode : public MultiNode {
 #ifndef PRODUCT
   virtual void           dump_spec(outputStream *st) const;
 #endif
+
+private:
+  virtual const TypePtr* out_adr_type_impl() const { return _out_adr_type; }
+  virtual const TypePtr* in_adr_type_impl() const { return _in_adr_type; }
 };
 
 //------------------------------SafePointScalarObjectNode----------------------
@@ -696,8 +703,8 @@ class CallNode : public SafePointNode {
   CallGenerator*  _generator;   // corresponding CallGenerator for some late inline calls
   const char*     _name;        // Printable name, if _method is null
 
-  CallNode(const TypeFunc* tf, address addr, const TypePtr* adr_type, JVMState* jvms = nullptr)
-    : SafePointNode(tf->domain()->cnt(), jvms, adr_type),
+  CallNode(const TypeFunc* tf, address addr, const TypePtr* out_adr_type, const TypePtr* in_adr_type, JVMState* jvms = nullptr)
+    : SafePointNode(tf->domain()->cnt(), jvms, out_adr_type, in_adr_type),
       _tf(tf),
       _entry_point(addr),
       _cnt(COUNT_UNKNOWN),
@@ -783,7 +790,7 @@ class CallJavaNode : public CallNode {
   bool    _arg_escape;             // ArgEscape in parameter list
 public:
   CallJavaNode(const TypeFunc* tf , address addr, ciMethod* method)
-    : CallNode(tf, addr, TypePtr::BOTTOM),
+    : CallNode(tf, addr, TypePtr::BOTTOM, TypePtr::BOTTOM),
       _method(method),
       _optimized_virtual(false),
       _override_symbolic_info(false),
@@ -828,11 +835,12 @@ class CallStaticJavaNode : public CallJavaNode {
       C->add_macro_node(this);
     }
   }
-  CallStaticJavaNode(const TypeFunc* tf, address addr, const char* name, const TypePtr* adr_type)
+  CallStaticJavaNode(const TypeFunc* tf, address addr, const char* name, const TypePtr* out_adr_type, const TypePtr* in_adr_type)
     : CallJavaNode(tf, addr, nullptr) {
     init_class_id(Class_CallStaticJava);
     // This node calls a runtime stub, which often has narrow memory effects.
-    _adr_type = adr_type;
+    _out_adr_type = out_adr_type;
+    _in_adr_type = in_adr_type;
     _name = name;
   }
 
@@ -892,8 +900,8 @@ class CallRuntimeNode : public CallNode {
   virtual uint size_of() const; // Size is bigger
 public:
   CallRuntimeNode(const TypeFunc* tf, address addr, const char* name,
-                  const TypePtr* adr_type, JVMState* jvms = nullptr)
-    : CallNode(tf, addr, adr_type, jvms)
+                  const TypePtr* out_adr_type, const TypePtr* in_adr_type, JVMState* jvms = nullptr)
+    : CallNode(tf, addr, out_adr_type, in_adr_type, jvms)
   {
     init_class_id(Class_CallRuntime);
     _name = name;
@@ -913,8 +921,8 @@ class CallRuntimeNode : public CallNode {
 class CallLeafNode : public CallRuntimeNode {
 public:
   CallLeafNode(const TypeFunc* tf, address addr, const char* name,
-               const TypePtr* adr_type)
-    : CallRuntimeNode(tf, addr, name, adr_type)
+               const TypePtr* out_adr_type, const TypePtr* in_adr_type)
+    : CallRuntimeNode(tf, addr, name, out_adr_type, in_adr_type)
   {
     init_class_id(Class_CallLeaf);
   }
@@ -943,9 +951,7 @@ class CallLeafPureNode : public CallLeafNode {
   TupleNode* make_tuple_of_input_state_and_top_return_values(const Compile* C) const;
 
 public:
-  CallLeafPureNode(const TypeFunc* tf, address addr, const char* name,
-                   const TypePtr* adr_type)
-      : CallLeafNode(tf, addr, name, adr_type) {
+  CallLeafPureNode(const TypeFunc* tf, address addr, const char* name) : CallLeafNode(tf, addr, name, nullptr, nullptr) {
     init_class_id(Class_CallLeafPure);
   }
   int Opcode() const override;
@@ -958,8 +964,8 @@ class CallLeafPureNode : public CallLeafNode {
 class CallLeafNoFPNode : public CallLeafNode {
 public:
   CallLeafNoFPNode(const TypeFunc* tf, address addr, const char* name,
-                   const TypePtr* adr_type)
-    : CallLeafNode(tf, addr, name, adr_type)
+                   const TypePtr* out_adr_type, const TypePtr* in_adr_type)
+    : CallLeafNode(tf, addr, name, out_adr_type, in_adr_type)
   {
     init_class_id(Class_CallLeafNoFP);
   }
@@ -976,8 +982,8 @@ class CallLeafVectorNode : public CallLeafNode {
   virtual uint size_of() const; // Size is bigger
 public:
   CallLeafVectorNode(const TypeFunc* tf, address addr, const char* name,
-                   const TypePtr* adr_type, uint num_bits)
-    : CallLeafNode(tf, addr, name, adr_type), _num_bits(num_bits)
+                   const TypePtr* out_adr_type, const TypePtr* in_adr_type, uint num_bits)
+    : CallLeafNode(tf, addr, name, out_adr_type, in_adr_type), _num_bits(num_bits)
   {
   }
   virtual int   Opcode() const;
@@ -1173,8 +1179,8 @@ class AbstractLockNode: public CallNode {
   void set_eliminated_lock_counter() PRODUCT_RETURN;
 
 public:
-  AbstractLockNode(const TypeFunc *tf)
-    : CallNode(tf, nullptr, TypeRawPtr::BOTTOM),
+  AbstractLockNode(const TypeFunc *tf, const TypePtr* out_adr_type, const TypePtr* in_adr_type)
+    : CallNode(tf, nullptr, out_adr_type, in_adr_type),
       _kind(Regular)
   {
 #ifndef PRODUCT
@@ -1255,7 +1261,7 @@ class LockNode : public AbstractLockNode {
 
   virtual int Opcode() const;
   virtual uint size_of() const; // Size is bigger
-  LockNode(Compile* C, const TypeFunc *tf) : AbstractLockNode( tf ) {
+  LockNode(Compile* C, const TypeFunc *tf) : AbstractLockNode(tf, TypeRawPtr::BOTTOM, TypePtr::BOTTOM) {
     init_class_id(Class_Lock);
     init_flags(Flag_is_macro);
     C->add_macro_node(this);
@@ -1280,7 +1286,7 @@ class UnlockNode : public AbstractLockNode {
 public:
   virtual int Opcode() const;
   virtual uint size_of() const; // Size is bigger
-  UnlockNode(Compile* C, const TypeFunc *tf) : AbstractLockNode( tf )
+  UnlockNode(Compile* C, const TypeFunc *tf) : AbstractLockNode(tf, TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM)
 #ifdef ASSERT
     , _dbg_jvms(nullptr)
 #endif

src/hotspot/share/opto/cfgnode.cpp

@@ -1075,13 +1075,13 @@ PhiNode* PhiNode::make(Node* r, Node* x, const Type *t, const TypePtr* at) {
 PhiNode* PhiNode::make(Node* r, Node* x) {
   const Type*    t  = x->bottom_type();
   const TypePtr* at = nullptr;
-  if (t == Type::MEMORY)  at = flatten_phi_adr_type(x->adr_type());
+  if (t == Type::MEMORY)  at = flatten_phi_adr_type(x->out_adr_type());
   return make(r, x, t, at);
 }
 PhiNode* PhiNode::make_blank(Node* r, Node* x) {
   const Type*    t  = x->bottom_type();
   const TypePtr* at = nullptr;
-  if (t == Type::MEMORY)  at = flatten_phi_adr_type(x->adr_type());
+  if (t == Type::MEMORY)  at = flatten_phi_adr_type(x->out_adr_type());
   return new PhiNode(r, t, at);
 }