8339661: ZGC: Move some page resets and verification to callsites

Reviewed-by: stefank, eosterlund

src/hotspot/share/gc/z/zForwarding.cpp

@@ -73,7 +73,7 @@ void ZForwarding::in_place_relocation_finish() {
   if (_from_age == ZPageAge::old || _to_age != ZPageAge::old) {
     // Only do this for non-promoted pages, that still need to reset live map.
     // Done with iterating over the "from-page" view, so can now drop the _livemap.
-    _page->finalize_reset_for_in_place_relocation();
+    _page->reset_livemap();
   }
 
   // Disable relaxed ZHeap::is_in checks

src/hotspot/share/gc/z/zPage.cpp

@@ -57,14 +57,9 @@ ZPage::ZPage(ZPageType type, const ZVirtualMemory& vmem, const ZPhysicalMemory&
 }
 
 ZPage* ZPage::clone_limited() const {
-  // Only copy type and memory layouts. Let the rest be lazily reconstructed when needed.
-  return new ZPage(_type, _virtual, _physical);
-}
-
-ZPage* ZPage::clone_limited_promote_flipped() const {
+  // Only copy type and memory layouts, and also update _top. Let the rest be
+  // lazily reconstructed when needed.
   ZPage* const page = new ZPage(_type, _virtual, _physical);
-
-  // The page is still filled with the same objects, need to retain the top pointer.
   page->_top = _top;
 
   return page;
@@ -83,58 +78,30 @@ void ZPage::reset_seqnum() {
   Atomic::store(&_seqnum_other, ZGeneration::generation(_generation_id == ZGenerationId::young ? ZGenerationId::old : ZGenerationId::young)->seqnum());
 }
 
-void ZPage::remset_clear() {
-  _remembered_set.clear_all();
-}
-
-void ZPage::verify_remset_after_reset(ZPageAge prev_age, ZPageResetType type) {
-  // Young-to-old reset
-  if (prev_age != ZPageAge::old) {
-    verify_remset_cleared_previous();
-    verify_remset_cleared_current();
-    return;
-  }
+void ZPage::remset_initialize() {
+  // Remsets should only be initialized once and only for old pages.
+  assert(!_remembered_set.is_initialized(), "Should not be initialized");
+  assert(is_old(), "Only old pages need a remset");
 
-  // Old-to-old reset
-  switch (type) {
-  case ZPageResetType::InPlaceRelocation:
-    // Relocation failed and page is being compacted in-place.
-    // The remset bits are flipped each young mark start, so
-    // the verification code below needs to use the right remset.
-    if (ZGeneration::old()->active_remset_is_current()) {
-      verify_remset_cleared_previous();
-    } else {
-      verify_remset_cleared_current();
-    }
-    break;
+  _remembered_set.initialize(size());
+}
 
-  case ZPageResetType::FlipAging:
-    fatal("Should not have called this for old-to-old flipping");
-    break;
+void ZPage::remset_initialize_or_verify_cleared() {
+  assert(is_old(), "Only old pages need a remset");
 
-  case ZPageResetType::Allocation:
-    verify_remset_cleared_previous();
+  if (_remembered_set.is_initialized()) {
     verify_remset_cleared_current();
-    break;
-  };
-}
-
-void ZPage::reset_remembered_set() {
-  if (is_young()) {
-    // Remset not needed
-    return;
+    verify_remset_cleared_previous();
+  } else {
+    remset_initialize();
   }
+}
 
-  // Clearing of remsets is done when freeing a page, so this code only
-  // needs to ensure the remset is initialized the first time a page
-  // becomes old.
-  if (!_remembered_set.is_initialized()) {
-    _remembered_set.initialize(size());
-  }
+void ZPage::remset_clear() {
+  _remembered_set.clear_all();
 }
 
-void ZPage::reset(ZPageAge age, ZPageResetType type) {
-  const ZPageAge prev_age = _age;
+void ZPage::reset(ZPageAge age) {
   _age = age;
   _last_used = 0;
 
@@ -143,27 +110,16 @@ void ZPage::reset(ZPageAge age, ZPageResetType type) {
       : ZGenerationId::young;
 
   reset_seqnum();
-
-  // Flip aged pages are still filled with the same objects, need to retain the top pointer.
-  if (type != ZPageResetType::FlipAging) {
-    _top = to_zoffset_end(start());
-  }
-
-  reset_remembered_set();
-  verify_remset_after_reset(prev_age, type);
-
-  if (type != ZPageResetType::InPlaceRelocation || (prev_age != ZPageAge::old && age == ZPageAge::old)) {
-    // Promoted in-place relocations reset the live map,
-    // because they clone the page.
-    _livemap.reset();
-  }
 }
 
-void ZPage::finalize_reset_for_in_place_relocation() {
-  // Now we're done iterating over the livemaps
+void ZPage::reset_livemap() {
   _livemap.reset();
 }
 
+void ZPage::reset_top_for_allocation() {
+  _top = to_zoffset_end(start());
+}
+
 void ZPage::reset_type_and_size(ZPageType type) {
   _type = type;
   _livemap.resize(object_max_count());
@@ -261,11 +217,11 @@ void ZPage::verify_remset_cleared_previous() const {
 }
 
 void ZPage::clear_remset_current() {
- _remembered_set.clear_current();
+  _remembered_set.clear_current();
 }
 
 void ZPage::clear_remset_previous() {
- _remembered_set.clear_previous();
+  _remembered_set.clear_previous();
 }
 
 void ZPage::swap_remset_bitmaps() {

src/hotspot/share/gc/z/zPage.hpp

@@ -36,16 +36,6 @@
 
 class ZGeneration;
 
-enum class ZPageResetType {
-  // Normal allocation path
-  Allocation,
-  // Relocation failed and started to relocate in-place
-  InPlaceRelocation,
-  // Page was not selected for relocation, all objects
-  // stayed, but the page aged.
-  FlipAging,
-};
-
 class ZPage : public CHeapObj<mtGC> {
   friend class VMStructs;
   friend class ZList<ZPage>;
@@ -80,17 +70,13 @@ class ZPage : public CHeapObj<mtGC> {
   const ZGeneration* generation() const;
 
   void reset_seqnum();
-  void reset_remembered_set();
 
   ZPage* split_with_pmem(ZPageType type, const ZPhysicalMemory& pmem);
 
-  void verify_remset_after_reset(ZPageAge prev_age, ZPageResetType type);
-
 public:
   ZPage(ZPageType type, const ZVirtualMemory& vmem, const ZPhysicalMemory& pmem);
 
   ZPage* clone_limited() const;
-  ZPage* clone_limited_promote_flipped() const;
 
   uint32_t object_max_count() const;
   size_t object_alignment_shift() const;
@@ -126,10 +112,9 @@ class ZPage : public CHeapObj<mtGC> {
   uint64_t last_used() const;
   void set_last_used();
 
-  void reset(ZPageAge age, ZPageResetType type);
-
-  void finalize_reset_for_in_place_relocation();
-
+  void reset(ZPageAge age);
+  void reset_livemap();
+  void reset_top_for_allocation();
   void reset_type_and_size(ZPageType type);
 
   ZPage* retype(ZPageType type);
@@ -170,6 +155,8 @@ class ZPage : public CHeapObj<mtGC> {
   void clear_remset_range_non_par_current(uintptr_t l_offset, size_t size);
   void swap_remset_bitmaps();
 
+  void remset_initialize();
+  void remset_initialize_or_verify_cleared();
   void remset_clear();
 
   ZBitMap::ReverseIterator remset_reverse_iterator_previous();

src/hotspot/share/gc/z/zPageAllocator.cpp

@@ -729,7 +729,12 @@ ZPage* ZPageAllocator::alloc_page(ZPageType type, size_t size, ZAllocationFlags
   // Reset page. This updates the page's sequence number and must
   // be done after we potentially blocked in a safepoint (stalled)
   // where the global sequence number was updated.
-  page->reset(age, ZPageResetType::Allocation);
+  page->reset(age);
+  page->reset_top_for_allocation();
+  page->reset_livemap();
+  if (age == ZPageAge::old) {
+    page->remset_initialize_or_verify_cleared();
+  }
 
   // Update allocation statistics. Exclude gc relocations to avoid
   // artificial inflation of the allocation rate during relocation.

src/hotspot/share/gc/z/zRelocate.cpp

@@ -843,7 +843,23 @@ class ZRelocateWork : public StackObj {
 
     // Promotions happen through a new cloned page
     ZPage* const to_page = promotion ? from_page->clone_limited() : from_page;
-    to_page->reset(to_age, ZPageResetType::InPlaceRelocation);
+
+    // Reset page for in-place relocation
+    to_page->reset(to_age);
+    to_page->reset_top_for_allocation();
+    if (promotion) {
+      to_page->remset_initialize();
+    }
+
+    // Verify that the inactive remset is clear when resetting the page for
+    // in-place relocation.
+    if (from_page->age() == ZPageAge::old) {
+      if (ZGeneration::old()->active_remset_is_current()) {
+        to_page->verify_remset_cleared_previous();
+      } else {
+        to_page->verify_remset_cleared_current();
+      }
+    }
 
     // Clear remset bits for all objects that were relocated
     // before this page became an in-place relocated page.
@@ -1270,8 +1286,14 @@ class ZFlipAgePagesTask : public ZTask {
       prev_page->log_msg(promotion ? " (flip promoted)" : " (flip survived)");
 
       // Setup to-space page
-      ZPage* const new_page = promotion ? prev_page->clone_limited_promote_flipped() : prev_page;
-      new_page->reset(to_age, ZPageResetType::FlipAging);
+      ZPage* const new_page = promotion ? prev_page->clone_limited() : prev_page;
+
+      // Reset page for flip aging
+      new_page->reset(to_age);
+      new_page->reset_livemap();
+      if (promotion) {
+        new_page->remset_initialize();
+      }
 
       if (promotion) {
         ZGeneration::young()->flip_promote(prev_page, new_page);

test/hotspot/gtest/gc/z/test_zForwarding.cpp

@@ -225,7 +225,7 @@ class ZForwardingTest : public Test {
     const ZPhysicalMemory pmem(ZPhysicalMemorySegment(zoffset(0), ZPageSizeSmall, true));
     ZPage page(ZPageType::small, vmem, pmem);
 
-    page.reset(ZPageAge::eden, ZPageResetType::Allocation);
+    page.reset(ZPageAge::eden);
 
     const size_t object_size = 16;
     const zaddress object = page.alloc_object(object_size);