Avoid CAS for marking region space bitmap for baker
Only have the GC thread mark it. This occurs when popping from the
mark stack. The race where an object may be pushed to the mark
stack twice is handled by not scanning if it is already marked.
Also avoid checking is_active when marking from the GC.
EAAC: 1263 -> 1253 (average of 30 runs)
GC time: 7.21s -> 6.83s (average of 18 runs)
Timings on 960 mhz N6P.
Bug: 12687968
Change-Id: I47e98c3e258829d2ba0babd803a219c82a36168c
Test: test-art-host, debug N6P booting with baker CC.
diff --git a/runtime/gc/collector/concurrent_copying.cc b/runtime/gc/collector/concurrent_copying.cc
index 42816a0..651669e 100644
--- a/runtime/gc/collector/concurrent_copying.cc
+++ b/runtime/gc/collector/concurrent_copying.cc
@@ -1302,8 +1302,19 @@
<< " " << to_ref << " " << to_ref->GetReadBarrierPointer()
<< " is_marked=" << IsMarked(to_ref);
}
- // Scan ref fields.
- Scan(to_ref);
+ bool add_to_live_bytes = false;
+ if (region_space_->IsInUnevacFromSpace(to_ref)) {
+ // Mark the bitmap only in the GC thread here so that we don't need a CAS.
+ if (!kUseBakerReadBarrier || !region_space_bitmap_->Set(to_ref)) {
+ // It may be already marked if we accidentally pushed the same object twice due to the racy
+ // bitmap read in MarkUnevacFromSpaceRegion.
+ Scan(to_ref);
+ // Only add to the live bytes if the object was not already marked.
+ add_to_live_bytes = true;
+ }
+ } else {
+ Scan(to_ref);
+ }
if (kUseBakerReadBarrier) {
DCHECK(to_ref->GetReadBarrierPointer() == ReadBarrier::GrayPtr())
<< " " << to_ref << " " << to_ref->GetReadBarrierPointer()
@@ -1332,7 +1343,7 @@
DCHECK(!kUseBakerReadBarrier);
#endif
- if (region_space_->IsInUnevacFromSpace(to_ref)) {
+ if (add_to_live_bytes) {
// Add to the live bytes per unevacuated from space. Note this code is always run by the
// GC-running thread (no synchronization required).
DCHECK(region_space_bitmap_->Test(to_ref));
@@ -1567,7 +1578,7 @@
// OK.
return;
} else if (region_space_->IsInUnevacFromSpace(ref)) {
- CHECK(region_space_bitmap_->Test(ref)) << ref;
+ CHECK(IsMarkedInUnevacFromSpace(ref)) << ref;
} else if (region_space_->IsInFromSpace(ref)) {
// Not OK. Do extra logging.
if (obj != nullptr) {
@@ -1614,7 +1625,7 @@
// OK.
return;
} else if (region_space_->IsInUnevacFromSpace(ref)) {
- CHECK(region_space_bitmap_->Test(ref)) << ref;
+ CHECK(IsMarkedInUnevacFromSpace(ref)) << ref;
} else if (region_space_->IsInFromSpace(ref)) {
// Not OK. Do extra logging.
if (gc_root_source == nullptr) {
@@ -1654,7 +1665,7 @@
LOG(INFO) << "holder is in the to-space.";
} else if (region_space_->IsInUnevacFromSpace(obj)) {
LOG(INFO) << "holder is in the unevac from-space.";
- if (region_space_bitmap_->Test(obj)) {
+ if (IsMarkedInUnevacFromSpace(obj)) {
LOG(INFO) << "holder is marked in the region space bitmap.";
} else {
LOG(INFO) << "holder is not marked in the region space bitmap.";
@@ -1783,7 +1794,7 @@
DCHECK_EQ(Thread::Current(), thread_running_gc_);
mirror::Object* ref = obj->GetFieldObject<
mirror::Object, kVerifyNone, kWithoutReadBarrier, false>(offset);
- mirror::Object* to_ref = Mark</*kGrayImmuneObject*/false>(ref);
+ mirror::Object* to_ref = Mark</*kGrayImmuneObject*/false, /*kFromGCThread*/true>(ref);
if (to_ref == ref) {
return;
}
@@ -2126,7 +2137,7 @@
heap_->non_moving_space_->HasAddress(to_ref))
<< "from_ref=" << from_ref << " to_ref=" << to_ref;
} else if (rtype == space::RegionSpace::RegionType::kRegionTypeUnevacFromSpace) {
- if (region_space_bitmap_->Test(from_ref)) {
+ if (IsMarkedInUnevacFromSpace(from_ref)) {
to_ref = from_ref;
} else {
to_ref = nullptr;