Revert "Revert "Gray only immune objects mutators access.""
To reduce image/zygote dirty pages.
GC doesn't gray immune space objects except when visiting the thread
GC roots of suspended threads during the thread flip and in
FillWithDummyObject(). GC updates the fields of immune space objects
without pushing/popping them through the mark stack. GC sets a bool
flag after updating the fields of immune space objects. After this
point, mutators don't need gray to immune space objects. Removed the
mark bitmaps for immune spaces.
This reverts commit ddeb172eeedb58ab96e074a55a0d1578b5df4110.
Bug: 29516465
Bug: 12687968
Test: art tests, libartd device boot, ritzperf, jdwp test.
Change-Id: If272373aea3d41b2719e40a6a41f44d9299ba309
diff --git a/runtime/gc/collector/concurrent_copying.h b/runtime/gc/collector/concurrent_copying.h
index a986a7a..166a1f0 100644
--- a/runtime/gc/collector/concurrent_copying.h
+++ b/runtime/gc/collector/concurrent_copying.h
@@ -61,10 +61,12 @@
ConcurrentCopying(Heap* heap, const std::string& name_prefix = "");
~ConcurrentCopying();
- virtual void RunPhases() OVERRIDE REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_);
- void InitializePhase() SHARED_REQUIRES(Locks::mutator_lock_) REQUIRES(!mark_stack_lock_);
+ virtual void RunPhases() OVERRIDE
+ REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_, !immune_gray_stack_lock_);
+ void InitializePhase() SHARED_REQUIRES(Locks::mutator_lock_)
+ REQUIRES(!mark_stack_lock_, !immune_gray_stack_lock_);
void MarkingPhase() SHARED_REQUIRES(Locks::mutator_lock_)
- REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_);
+ REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_, !immune_gray_stack_lock_);
void ReclaimPhase() SHARED_REQUIRES(Locks::mutator_lock_) REQUIRES(!mark_stack_lock_);
void FinishPhase() REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_);
@@ -92,8 +94,9 @@
DCHECK(ref != nullptr);
return IsMarked(ref) == ref;
}
+ template<bool kGrayImmuneObject = true>
ALWAYS_INLINE mirror::Object* Mark(mirror::Object* from_ref) SHARED_REQUIRES(Locks::mutator_lock_)
- REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_);
+ REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_, !immune_gray_stack_lock_);
bool IsMarking() const {
return is_marking_;
}
@@ -117,16 +120,19 @@
void Scan(mirror::Object* to_ref) SHARED_REQUIRES(Locks::mutator_lock_)
REQUIRES(!mark_stack_lock_);
void Process(mirror::Object* obj, MemberOffset offset)
- SHARED_REQUIRES(Locks::mutator_lock_) REQUIRES(!mark_stack_lock_ , !skipped_blocks_lock_);
+ SHARED_REQUIRES(Locks::mutator_lock_)
+ REQUIRES(!mark_stack_lock_ , !skipped_blocks_lock_, !immune_gray_stack_lock_);
virtual void VisitRoots(mirror::Object*** roots, size_t count, const RootInfo& info)
OVERRIDE SHARED_REQUIRES(Locks::mutator_lock_)
- REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_);
+ REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_, !immune_gray_stack_lock_);
+ template<bool kGrayImmuneObject>
void MarkRoot(mirror::CompressedReference<mirror::Object>* root)
- SHARED_REQUIRES(Locks::mutator_lock_) REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_);
+ SHARED_REQUIRES(Locks::mutator_lock_)
+ REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_, !immune_gray_stack_lock_);
virtual void VisitRoots(mirror::CompressedReference<mirror::Object>** roots, size_t count,
const RootInfo& info)
OVERRIDE SHARED_REQUIRES(Locks::mutator_lock_)
- REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_);
+ REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_, !immune_gray_stack_lock_);
void VerifyNoFromSpaceReferences() REQUIRES(Locks::mutator_lock_);
accounting::ObjectStack* GetAllocationStack();
accounting::ObjectStack* GetLiveStack();
@@ -146,9 +152,11 @@
SHARED_REQUIRES(Locks::mutator_lock_);
void ProcessReferences(Thread* self) SHARED_REQUIRES(Locks::mutator_lock_);
virtual mirror::Object* MarkObject(mirror::Object* from_ref) OVERRIDE
- SHARED_REQUIRES(Locks::mutator_lock_) REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_);
+ SHARED_REQUIRES(Locks::mutator_lock_)
+ REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_, !immune_gray_stack_lock_);
virtual void MarkHeapReference(mirror::HeapReference<mirror::Object>* from_ref) OVERRIDE
- SHARED_REQUIRES(Locks::mutator_lock_) REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_);
+ SHARED_REQUIRES(Locks::mutator_lock_)
+ REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_, !immune_gray_stack_lock_);
virtual mirror::Object* IsMarked(mirror::Object* from_ref) OVERRIDE
SHARED_REQUIRES(Locks::mutator_lock_);
virtual bool IsMarkedHeapReference(mirror::HeapReference<mirror::Object>* field) OVERRIDE
@@ -182,14 +190,19 @@
void ExpandGcMarkStack() SHARED_REQUIRES(Locks::mutator_lock_);
mirror::Object* MarkNonMoving(mirror::Object* from_ref) SHARED_REQUIRES(Locks::mutator_lock_)
REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_);
- ALWAYS_INLINE mirror::Object* MarkUnevacFromSpaceRegionOrImmuneSpace(mirror::Object* from_ref,
+ ALWAYS_INLINE mirror::Object* MarkUnevacFromSpaceRegion(mirror::Object* from_ref,
accounting::SpaceBitmap<kObjectAlignment>* bitmap)
SHARED_REQUIRES(Locks::mutator_lock_)
REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_);
+ template<bool kGrayImmuneObject>
+ ALWAYS_INLINE mirror::Object* MarkImmuneSpace(mirror::Object* from_ref)
+ SHARED_REQUIRES(Locks::mutator_lock_) REQUIRES(!immune_gray_stack_lock_);
void PushOntoFalseGrayStack(mirror::Object* obj) SHARED_REQUIRES(Locks::mutator_lock_)
REQUIRES(!mark_stack_lock_);
void ProcessFalseGrayStack() SHARED_REQUIRES(Locks::mutator_lock_)
REQUIRES(!mark_stack_lock_);
+ void ScanImmuneObject(mirror::Object* obj)
+ SHARED_REQUIRES(Locks::mutator_lock_) REQUIRES(!mark_stack_lock_);
space::RegionSpace* region_space_; // The underlying region space.
std::unique_ptr<Barrier> gc_barrier_;
@@ -207,8 +220,6 @@
bool is_active_; // True while the collection is ongoing.
bool is_asserting_to_space_invariant_; // True while asserting the to-space invariant.
ImmuneSpaces immune_spaces_;
- std::unique_ptr<accounting::HeapBitmap> cc_heap_bitmap_;
- std::vector<accounting::SpaceBitmap<kObjectAlignment>*> cc_bitmaps_;
accounting::SpaceBitmap<kObjectAlignment>* region_space_bitmap_;
// A cache of Heap::GetMarkBitmap().
accounting::HeapBitmap* heap_mark_bitmap_;
@@ -242,6 +253,10 @@
accounting::ReadBarrierTable* rb_table_;
bool force_evacuate_all_; // True if all regions are evacuated.
+ Atomic<bool> updated_all_immune_objects_;
+ bool gc_grays_immune_objects_;
+ Mutex immune_gray_stack_lock_ DEFAULT_MUTEX_ACQUIRED_AFTER;
+ std::vector<mirror::Object*> immune_gray_stack_ GUARDED_BY(immune_gray_stack_lock_);
class AssertToSpaceInvariantFieldVisitor;
class AssertToSpaceInvariantObjectVisitor;
@@ -250,14 +265,15 @@
class ComputeUnevacFromSpaceLiveRatioVisitor;
class DisableMarkingCheckpoint;
class FlipCallback;
- class ImmuneSpaceObjVisitor;
+ class ImmuneSpaceScanObjVisitor;
class LostCopyVisitor;
class RefFieldsVisitor;
class RevokeThreadLocalMarkStackCheckpoint;
+ class ScopedGcGraysImmuneObjects;
+ class ThreadFlipVisitor;
class VerifyNoFromSpaceRefsFieldVisitor;
class VerifyNoFromSpaceRefsObjectVisitor;
class VerifyNoFromSpaceRefsVisitor;
- class ThreadFlipVisitor;
DISALLOW_IMPLICIT_CONSTRUCTORS(ConcurrentCopying);
};