Merge "Clear (madvise) card table for CC"
diff --git a/runtime/gc/accounting/card_table-inl.h b/runtime/gc/accounting/card_table-inl.h
index f72f219..6ff5359 100644
--- a/runtime/gc/accounting/card_table-inl.h
+++ b/runtime/gc/accounting/card_table-inl.h
@@ -50,13 +50,17 @@
}
template <bool kClearCard, typename Visitor>
-inline size_t CardTable::Scan(ContinuousSpaceBitmap* bitmap, uint8_t* scan_begin, uint8_t* scan_end,
- const Visitor& visitor, const uint8_t minimum_age) const {
+inline size_t CardTable::Scan(ContinuousSpaceBitmap* bitmap,
+ uint8_t* const scan_begin,
+ uint8_t* const scan_end,
+ const Visitor& visitor,
+ const uint8_t minimum_age) {
DCHECK_GE(scan_begin, reinterpret_cast<uint8_t*>(bitmap->HeapBegin()));
// scan_end is the byte after the last byte we scan.
DCHECK_LE(scan_end, reinterpret_cast<uint8_t*>(bitmap->HeapLimit()));
- uint8_t* card_cur = CardFromAddr(scan_begin);
- uint8_t* card_end = CardFromAddr(AlignUp(scan_end, kCardSize));
+ uint8_t* const card_begin = CardFromAddr(scan_begin);
+ uint8_t* const card_end = CardFromAddr(AlignUp(scan_end, kCardSize));
+ uint8_t* card_cur = card_begin;
CheckCardValid(card_cur);
CheckCardValid(card_end);
size_t cards_scanned = 0;
@@ -67,9 +71,6 @@
uintptr_t start = reinterpret_cast<uintptr_t>(AddrFromCard(card_cur));
bitmap->VisitMarkedRange(start, start + kCardSize, visitor);
++cards_scanned;
- if (kClearCard) {
- *card_cur = 0;
- }
}
++card_cur;
}
@@ -99,9 +100,6 @@
<< "card " << static_cast<size_t>(*card) << " intptr_t " << (start_word & 0xFF);
bitmap->VisitMarkedRange(start, start + kCardSize, visitor);
++cards_scanned;
- if (kClearCard) {
- *card = 0;
- }
}
start_word >>= 8;
start += kCardSize;
@@ -116,13 +114,14 @@
uintptr_t start = reinterpret_cast<uintptr_t>(AddrFromCard(card_cur));
bitmap->VisitMarkedRange(start, start + kCardSize, visitor);
++cards_scanned;
- if (kClearCard) {
- *card_cur = 0;
- }
}
++card_cur;
}
+ if (kClearCard) {
+ ClearCardRange(scan_begin, scan_end);
+ }
+
return cards_scanned;
}
@@ -135,7 +134,9 @@
* us to know which cards got cleared.
*/
template <typename Visitor, typename ModifiedVisitor>
-inline void CardTable::ModifyCardsAtomic(uint8_t* scan_begin, uint8_t* scan_end, const Visitor& visitor,
+inline void CardTable::ModifyCardsAtomic(uint8_t* scan_begin,
+ uint8_t* scan_end,
+ const Visitor& visitor,
const ModifiedVisitor& modified) {
uint8_t* card_cur = CardFromAddr(scan_begin);
uint8_t* card_end = CardFromAddr(AlignUp(scan_end, kCardSize));
diff --git a/runtime/gc/accounting/card_table.cc b/runtime/gc/accounting/card_table.cc
index 121da37..4506597 100644
--- a/runtime/gc/accounting/card_table.cc
+++ b/runtime/gc/accounting/card_table.cc
@@ -97,36 +97,18 @@
// Destroys MemMap via std::unique_ptr<>.
}
-void CardTable::ClearSpaceCards(space::ContinuousSpace* space) {
- // TODO: clear just the range of the table that has been modified
- uint8_t* card_start = CardFromAddr(space->Begin());
- uint8_t* card_end = CardFromAddr(space->End()); // Make sure to round up.
- memset(reinterpret_cast<void*>(card_start), kCardClean, card_end - card_start);
-}
-
void CardTable::ClearCardTable() {
static_assert(kCardClean == 0, "kCardClean must be 0");
mem_map_->MadviseDontNeedAndZero();
}
void CardTable::ClearCardRange(uint8_t* start, uint8_t* end) {
- if (!kMadviseZeroes) {
- memset(start, 0, end - start);
- return;
- }
CHECK_ALIGNED(reinterpret_cast<uintptr_t>(start), kCardSize);
CHECK_ALIGNED(reinterpret_cast<uintptr_t>(end), kCardSize);
static_assert(kCardClean == 0, "kCardClean must be 0");
uint8_t* start_card = CardFromAddr(start);
uint8_t* end_card = CardFromAddr(end);
- uint8_t* round_start = AlignUp(start_card, kPageSize);
- uint8_t* round_end = AlignDown(end_card, kPageSize);
- if (round_start < round_end) {
- madvise(round_start, round_end - round_start, MADV_DONTNEED);
- }
- // Handle unaligned regions at start / end.
- memset(start_card, 0, std::min(round_start, end_card) - start_card);
- memset(std::max(round_end, start_card), 0, end_card - std::max(round_end, start_card));
+ ZeroAndReleasePages(start_card, end_card - start_card);
}
bool CardTable::AddrIsInCardTable(const void* addr) const {
diff --git a/runtime/gc/accounting/card_table.h b/runtime/gc/accounting/card_table.h
index 969bfb7..68ef15d 100644
--- a/runtime/gc/accounting/card_table.h
+++ b/runtime/gc/accounting/card_table.h
@@ -98,15 +98,19 @@
* us to know which cards got cleared.
*/
template <typename Visitor, typename ModifiedVisitor>
- void ModifyCardsAtomic(uint8_t* scan_begin, uint8_t* scan_end, const Visitor& visitor,
+ void ModifyCardsAtomic(uint8_t* scan_begin,
+ uint8_t* scan_end,
+ const Visitor& visitor,
const ModifiedVisitor& modified);
// For every dirty at least minumum age between begin and end invoke the visitor with the
// specified argument. Returns how many cards the visitor was run on.
template <bool kClearCard, typename Visitor>
- size_t Scan(SpaceBitmap<kObjectAlignment>* bitmap, uint8_t* scan_begin, uint8_t* scan_end,
+ size_t Scan(SpaceBitmap<kObjectAlignment>* bitmap,
+ uint8_t* scan_begin,
+ uint8_t* scan_end,
const Visitor& visitor,
- const uint8_t minimum_age = kCardDirty) const
+ const uint8_t minimum_age = kCardDirty)
REQUIRES(Locks::heap_bitmap_lock_)
REQUIRES_SHARED(Locks::mutator_lock_);
@@ -119,9 +123,6 @@
// Clear a range of cards that covers start to end, start and end must be aligned to kCardSize.
void ClearCardRange(uint8_t* start, uint8_t* end);
- // Resets all of the bytes in the card table which do not map to the image space.
- void ClearSpaceCards(space::ContinuousSpace* space);
-
// Returns the first address in the heap which maps to this card.
void* AddrFromCard(const uint8_t *card_addr) const ALWAYS_INLINE;
diff --git a/runtime/gc/accounting/mod_union_table.cc b/runtime/gc/accounting/mod_union_table.cc
index 14f5997..0325535 100644
--- a/runtime/gc/accounting/mod_union_table.cc
+++ b/runtime/gc/accounting/mod_union_table.cc
@@ -168,7 +168,7 @@
bool* const contains_reference_to_other_space_;
};
-void ModUnionTableReferenceCache::ClearCards() {
+void ModUnionTableReferenceCache::ProcessCards() {
CardTable* card_table = GetHeap()->GetCardTable();
ModUnionAddToCardSetVisitor visitor(&cleared_cards_);
// Clear dirty cards in the this space and update the corresponding mod-union bits.
@@ -525,7 +525,7 @@
ModUnionTable::CardBitmap* const card_bitmap_;
};
-void ModUnionTableCardCache::ClearCards() {
+void ModUnionTableCardCache::ProcessCards() {
CardTable* const card_table = GetHeap()->GetCardTable();
ModUnionAddToCardBitmapVisitor visitor(card_bitmap_.get(), card_table);
// Clear dirty cards in the this space and update the corresponding mod-union bits.
diff --git a/runtime/gc/accounting/mod_union_table.h b/runtime/gc/accounting/mod_union_table.h
index b6792c4..591365f 100644
--- a/runtime/gc/accounting/mod_union_table.h
+++ b/runtime/gc/accounting/mod_union_table.h
@@ -55,10 +55,10 @@
virtual ~ModUnionTable() {}
- // Clear cards which map to a memory range of a space. This doesn't immediately update the
- // mod-union table, as updating the mod-union table may have an associated cost, such as
- // determining references to track.
- virtual void ClearCards() = 0;
+ // Process cards for a memory range of a space. This doesn't immediately update the mod-union
+ // table, as updating the mod-union table may have an associated cost, such as determining
+ // references to track.
+ virtual void ProcessCards() = 0;
// Set all the cards.
virtual void SetCards() = 0;
@@ -66,9 +66,9 @@
// Clear all of the table.
virtual void ClearTable() = 0;
- // Update the mod-union table using data stored by ClearCards. There may be multiple ClearCards
- // before a call to update, for example, back-to-back sticky GCs. Also mark references to other
- // spaces which are stored in the mod-union table.
+ // Update the mod-union table using data stored by ProcessCards. There may be multiple
+ // ProcessCards before a call to update, for example, back-to-back sticky GCs. Also mark
+ // references to other spaces which are stored in the mod-union table.
virtual void UpdateAndMarkReferences(MarkObjectVisitor* visitor) = 0;
// Visit all of the objects that may contain references to other spaces.
@@ -117,7 +117,7 @@
virtual ~ModUnionTableReferenceCache() {}
// Clear and store cards for a space.
- void ClearCards() OVERRIDE;
+ void ProcessCards() OVERRIDE;
// Update table based on cleared cards and mark all references to the other spaces.
void UpdateAndMarkReferences(MarkObjectVisitor* visitor) OVERRIDE
@@ -164,7 +164,7 @@
virtual ~ModUnionTableCardCache() {}
// Clear and store cards for a space.
- virtual void ClearCards() OVERRIDE;
+ virtual void ProcessCards() OVERRIDE;
// Mark all references to the alloc space(s).
virtual void UpdateAndMarkReferences(MarkObjectVisitor* visitor) OVERRIDE
diff --git a/runtime/gc/accounting/mod_union_table_test.cc b/runtime/gc/accounting/mod_union_table_test.cc
index 2810f58..cf63b30 100644
--- a/runtime/gc/accounting/mod_union_table_test.cc
+++ b/runtime/gc/accounting/mod_union_table_test.cc
@@ -214,7 +214,7 @@
ASSERT_TRUE(other_space_ref2 != nullptr);
obj1->Set(1, other_space_ref1);
obj2->Set(3, other_space_ref2);
- table->ClearCards();
+ table->ProcessCards();
std::set<mirror::Object*> visited_before;
CollectVisitedVisitor collector_before(&visited_before);
table->UpdateAndMarkReferences(&collector_before);
diff --git a/runtime/gc/accounting/space_bitmap.cc b/runtime/gc/accounting/space_bitmap.cc
index e2f5a1d..f4d0bc7 100644
--- a/runtime/gc/accounting/space_bitmap.cc
+++ b/runtime/gc/accounting/space_bitmap.cc
@@ -118,31 +118,8 @@
}
const uintptr_t start_index = OffsetToIndex(begin_offset);
const uintptr_t end_index = OffsetToIndex(end_offset);
- Atomic<uintptr_t>* const mem_begin = &bitmap_begin_[start_index];
- Atomic<uintptr_t>* const mem_end = &bitmap_begin_[end_index];
- Atomic<uintptr_t>* const page_begin = AlignUp(mem_begin, kPageSize);
- Atomic<uintptr_t>* const page_end = AlignDown(mem_end, kPageSize);
- if (!kMadviseZeroes || page_begin >= page_end) {
- // No possible area to madvise.
- std::fill(reinterpret_cast<uint8_t*>(mem_begin),
- reinterpret_cast<uint8_t*>(mem_end),
- 0);
- } else {
- // Spans one or more pages.
- DCHECK_LE(mem_begin, page_begin);
- DCHECK_LE(page_begin, page_end);
- DCHECK_LE(page_end, mem_end);
- std::fill(reinterpret_cast<uint8_t*>(mem_begin),
- reinterpret_cast<uint8_t*>(page_begin),
- 0);
- CHECK_NE(madvise(page_begin,
- reinterpret_cast<uint8_t*>(page_end) - reinterpret_cast<uint8_t*>(page_begin),
- MADV_DONTNEED),
- -1) << "madvise failed";
- std::fill(reinterpret_cast<uint8_t*>(page_end),
- reinterpret_cast<uint8_t*>(mem_end),
- 0);
- }
+ ZeroAndReleasePages(reinterpret_cast<uint8_t*>(&bitmap_begin_[start_index]),
+ (end_index - start_index) * sizeof(*bitmap_begin_));
}
template<size_t kAlignment>
diff --git a/runtime/gc/collector/concurrent_copying.cc b/runtime/gc/collector/concurrent_copying.cc
index 13af67e..c43b048 100644
--- a/runtime/gc/collector/concurrent_copying.cc
+++ b/runtime/gc/collector/concurrent_copying.cc
@@ -486,9 +486,14 @@
// Table is non null for boot image and zygote spaces. It is only null for application image
// spaces.
if (table != nullptr) {
- // TODO: Add preclean outside the pause.
- table->ClearCards();
+ // TODO: Consider adding precleaning outside the pause.
+ table->ProcessCards();
table->VisitObjects(GrayImmuneObjectVisitor::Callback, &visitor);
+ // Since the cards are recorded in the mod-union table and this is paused, we can clear
+ // the cards for the space (to madvise).
+ TimingLogger::ScopedTiming split2("(Paused)ClearCards", GetTimings());
+ card_table->ClearCardRange(space->Begin(),
+ AlignDown(space->End(), accounting::CardTable::kCardSize));
} else {
// TODO: Consider having a mark bitmap for app image spaces and avoid scanning during the
// pause because app image spaces are all dirty pages anyways.
@@ -2325,9 +2330,14 @@
MutexLock mu(self, mark_stack_lock_);
CHECK_EQ(pooled_mark_stacks_.size(), kMarkStackPoolSize);
}
- region_space_ = nullptr;
{
- MutexLock mu(Thread::Current(), skipped_blocks_lock_);
+ TimingLogger::ScopedTiming split("ClearRegionSpaceCards", GetTimings());
+ // We do not currently use the region space cards at all, madvise them away to save ram.
+ heap_->GetCardTable()->ClearCardRange(region_space_->Begin(), region_space_->Limit());
+ region_space_ = nullptr;
+ }
+ {
+ MutexLock mu(self, skipped_blocks_lock_);
skipped_blocks_map_.clear();
}
{
@@ -2339,10 +2349,9 @@
if (kUseBakerReadBarrier && kFilterModUnionCards) {
TimingLogger::ScopedTiming split("FilterModUnionCards", GetTimings());
ReaderMutexLock mu2(self, *Locks::heap_bitmap_lock_);
- gc::Heap* const heap = Runtime::Current()->GetHeap();
for (space::ContinuousSpace* space : immune_spaces_.GetSpaces()) {
DCHECK(space->IsImageSpace() || space->IsZygoteSpace());
- accounting::ModUnionTable* table = heap->FindModUnionTableFromSpace(space);
+ accounting::ModUnionTable* table = heap_->FindModUnionTableFromSpace(space);
// Filter out cards that don't need to be set.
if (table != nullptr) {
table->FilterCards();
@@ -2351,7 +2360,7 @@
}
if (kUseBakerReadBarrier) {
TimingLogger::ScopedTiming split("EmptyRBMarkBitStack", GetTimings());
- DCHECK(rb_mark_bit_stack_.get() != nullptr);
+ DCHECK(rb_mark_bit_stack_ != nullptr);
const auto* limit = rb_mark_bit_stack_->End();
for (StackReference<mirror::Object>* it = rb_mark_bit_stack_->Begin(); it != limit; ++it) {
CHECK(it->AsMirrorPtr()->AtomicSetMarkBit(1, 0));
diff --git a/runtime/gc/heap.cc b/runtime/gc/heap.cc
index 3b9abd2..ffad80d 100644
--- a/runtime/gc/heap.cc
+++ b/runtime/gc/heap.cc
@@ -3327,7 +3327,7 @@
const char* name = space->IsZygoteSpace() ? "ZygoteModUnionClearCards" :
"ImageModUnionClearCards";
TimingLogger::ScopedTiming t2(name, timings);
- table->ClearCards();
+ table->ProcessCards();
} else if (use_rem_sets && rem_set != nullptr) {
DCHECK(collector::SemiSpace::kUseRememberedSet && collector_type_ == kCollectorTypeGSS)
<< static_cast<int>(collector_type_);
diff --git a/runtime/mem_map.cc b/runtime/mem_map.cc
index bb07fcb..e4c67e0 100644
--- a/runtime/mem_map.cc
+++ b/runtime/mem_map.cc
@@ -918,4 +918,23 @@
}
}
+void ZeroAndReleasePages(void* address, size_t length) {
+ uint8_t* const mem_begin = reinterpret_cast<uint8_t*>(address);
+ uint8_t* const mem_end = mem_begin + length;
+ uint8_t* const page_begin = AlignUp(mem_begin, kPageSize);
+ uint8_t* const page_end = AlignDown(mem_end, kPageSize);
+ if (!kMadviseZeroes || page_begin >= page_end) {
+ // No possible area to madvise.
+ std::fill(mem_begin, mem_end, 0);
+ } else {
+ // Spans one or more pages.
+ DCHECK_LE(mem_begin, page_begin);
+ DCHECK_LE(page_begin, page_end);
+ DCHECK_LE(page_end, mem_end);
+ std::fill(mem_begin, page_begin, 0);
+ CHECK_NE(madvise(page_begin, page_end - page_begin, MADV_DONTNEED), -1) << "madvise failed";
+ std::fill(page_end, mem_end, 0);
+ }
+}
+
} // namespace art
diff --git a/runtime/mem_map.h b/runtime/mem_map.h
index 597f0d4..049ae12 100644
--- a/runtime/mem_map.h
+++ b/runtime/mem_map.h
@@ -241,9 +241,13 @@
friend class MemMapTest; // To allow access to base_begin_ and base_size_.
};
+
std::ostream& operator<<(std::ostream& os, const MemMap& mem_map);
std::ostream& operator<<(std::ostream& os, const MemMap::Maps& mem_maps);
+// Zero and release pages if possible, no requirements on alignments.
+void ZeroAndReleasePages(void* address, size_t length);
+
} // namespace art
#endif // ART_RUNTIME_MEM_MAP_H_