8359200: Memory corruption in MStack::push

Reviewed-by: epeter, shade
Backport-of: ed39e17

Author: TobiHartmann

src/hotspot/share/libadt/vectset.cpp

@@ -47,7 +47,6 @@ void VectorSet::init(Arena* arena) {
 
 // Expand the existing set to a bigger size
 void VectorSet::grow(uint new_word_capacity) {
-  _nesting.check(_set_arena); // Check if a potential reallocation in the arena is safe
   assert(new_word_capacity >= _size, "Should have been checked before, use maybe_grow?");
   assert(new_word_capacity < (1U << 30), "");
   uint x = next_power_of_2(new_word_capacity);

src/hotspot/share/libadt/vectset.hpp

@@ -52,6 +52,7 @@ class VectorSet : public AnyObj {
 
   // Grow vector to required word capacity
   void maybe_grow(uint new_word_capacity) {
+    _nesting.check(_set_arena); // Check if a potential reallocation in the arena is safe
     if (new_word_capacity >= _size) {
       grow(new_word_capacity);
     }

src/hotspot/share/opto/block.cpp

@@ -37,11 +37,8 @@
 #include "utilities/copy.hpp"
 #include "utilities/powerOfTwo.hpp"
 
-void Block_Array::grow( uint i ) {
-  _nesting.check(_arena); // Check if a potential reallocation in the arena is safe
-  if (i < Max()) {
-    return; // No need to grow
-  }
+void Block_Array::grow(uint i) {
+  assert(i >= Max(), "Should have been checked before, use maybe_grow?");
   DEBUG_ONLY(_limit = i+1);
   if( i < _size )  return;
   if( !_size ) {

src/hotspot/share/opto/block.hpp

@@ -53,7 +53,13 @@ class Block_Array : public ArenaObj {
   ReallocMark _nesting;         // Safety checks for arena reallocation
 protected:
   Block **_blocks;
-  void grow( uint i );          // Grow array node to fit
+  void maybe_grow(uint i) {
+    _nesting.check(_arena);     // Check if a potential reallocation in the arena is safe
+    if (i >= Max()) {
+      grow(i);
+    }
+  }
+  void grow(uint i);            // Grow array node to fit
 
 public:
   Block_Array(Arena *a) : _size(OptoBlockListSize), _arena(a) {
@@ -68,7 +74,7 @@ class Block_Array : public ArenaObj {
   Block *operator[] ( uint i ) const // Lookup, or assert for not mapped
   { assert( i < Max(), "oob" ); return _blocks[i]; }
   // Extend the mapping: index i maps to Block *n.
-  void map( uint i, Block *n ) { grow(i); _blocks[i] = n; }
+  void map( uint i, Block *n ) { maybe_grow(i); _blocks[i] = n; }
   uint Max() const { DEBUG_ONLY(return _limit); return _size; }
 };
 

src/hotspot/share/opto/matcher.hpp

@@ -65,13 +65,8 @@ class Matcher : public PhaseTransform {
       Node_Stack::push(n, (uint)ns);
     }
     void push(Node *n, Node_State ns, Node *parent, int indx) {
-      ++_inode_top;
-      if ((_inode_top + 1) >= _inode_max) grow();
-      _inode_top->node = parent;
-      _inode_top->indx = (uint)indx;
-      ++_inode_top;
-      _inode_top->node = n;
-      _inode_top->indx = (uint)ns;
+      Node_Stack::push(parent, (uint)indx);
+      Node_Stack::push(n, (uint)ns);
     }
     Node *parent() {
       pop();

src/hotspot/share/opto/node.cpp

@@ -2798,7 +2798,6 @@ const RegMask &Node::in_RegMask(uint) const {
 }
 
 void Node_Array::grow(uint i) {
-  _nesting.check(_a); // Check if a potential reallocation in the arena is safe
   assert(i >= _max, "Should have been checked before, use maybe_grow?");
   assert(_max > 0, "invariant");
   uint old = _max;
@@ -3038,10 +3037,6 @@ void Unique_Node_List::remove_useless_nodes(VectorSet &useful) {
 
 //=============================================================================
 void Node_Stack::grow() {
-  _nesting.check(_a); // Check if a potential reallocation in the arena is safe
-  if (_inode_top < _inode_max) {
-    return; // No need to grow
-  }
   size_t old_top = pointer_delta(_inode_top,_inodes,sizeof(INode)); // save _top
   size_t old_max = pointer_delta(_inode_max,_inodes,sizeof(INode));
   size_t max = old_max << 1;             // max * 2

src/hotspot/share/opto/node.hpp

@@ -1633,6 +1633,7 @@ class Node_Array : public AnyObj {
 
   // Grow array to required capacity
   void maybe_grow(uint i) {
+    _nesting.check(_a); // Check if a potential reallocation in the arena is safe
     if (i >= _max) {
       grow(i);
     }
@@ -1884,7 +1885,15 @@ class Node_Stack {
   INode *_inodes;    // Array storage for the stack
   Arena *_a;         // Arena to allocate in
   ReallocMark _nesting; // Safety checks for arena reallocation
+
+  void maybe_grow() {
+    _nesting.check(_a); // Check if a potential reallocation in the arena is safe
+    if (_inode_top >= _inode_max) {
+      grow();
+    }
+  }
   void grow();
+
 public:
   Node_Stack(int size) {
     size_t max = (size > OptoNodeListSize) ? size : OptoNodeListSize;
@@ -1907,7 +1916,7 @@ class Node_Stack {
   }
   void push(Node *n, uint i) {
     ++_inode_top;
-    grow();
+    maybe_grow();
     INode *top = _inode_top; // optimization
     top->node = n;
     top->indx = i;

test/hotspot/jtreg/compiler/arguments/TestOptoNodeListSize.java

@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+ /*
+ * @test
+ * @library /test/lib /
+ * @bug 8359200
+ * @key randomness
+ * @requires vm.flagless & vm.compiler2.enabled & vm.debug == true
+ * @summary Test that -XX:OptoNodeListSize does not crash the VM.
+ * @run driver compiler.arguments.TestOptoNodeListSize
+ */
+
+package compiler.arguments;
+
+import java.io.IOException;
+import java.util.Random;
+
+import jdk.test.lib.process.ProcessTools;
+import jdk.test.lib.process.OutputAnalyzer;
+import jdk.test.lib.Utils;
+
+public class TestOptoNodeListSize {
+    private static final Random RANDOM = Utils.getRandomInstance();
+
+    public static void main(String[] args) throws IOException {
+        if (args.length == 0) {
+            int size = RANDOM.nextInt(1000) + 1;
+            ProcessBuilder pb = ProcessTools.createLimitedTestJavaProcessBuilder("-XX:OptoNodeListSize=" + size,
+                "-Xcomp", "-XX:-TieredCompilation", "compiler.arguments.TestOptoNodeListSize", "run");
+            OutputAnalyzer output = new OutputAnalyzer(pb.start());
+            output.shouldHaveExitValue(0);
+        } else {
+            System.out.println("Test passed.");
+        }
+    }
+}