Rosalloc thread local allocation path without a cas.
Speedup on N4:
MemAllocTest 3044 -> 2396 (~21% reduction)
BinaryTrees 4101 -> 2929 (~26% reduction)
Bug: 9986565
Change-Id: Ia1d1a37b9e001f903c3c056e8ec68fc8c623a78b
diff --git a/runtime/gc/space/bump_pointer_space-inl.h b/runtime/gc/space/bump_pointer_space-inl.h
index 9f1f953..14a93d1 100644
--- a/runtime/gc/space/bump_pointer_space-inl.h
+++ b/runtime/gc/space/bump_pointer_space-inl.h
@@ -24,7 +24,8 @@
namespace space {
inline mirror::Object* BumpPointerSpace::Alloc(Thread*, size_t num_bytes, size_t* bytes_allocated,
- size_t* usable_size) {
+ size_t* usable_size,
+ size_t* bytes_tl_bulk_allocated) {
num_bytes = RoundUp(num_bytes, kAlignment);
mirror::Object* ret = AllocNonvirtual(num_bytes);
if (LIKELY(ret != nullptr)) {
@@ -32,13 +33,15 @@
if (usable_size != nullptr) {
*usable_size = num_bytes;
}
+ *bytes_tl_bulk_allocated = num_bytes;
}
return ret;
}
inline mirror::Object* BumpPointerSpace::AllocThreadUnsafe(Thread* self, size_t num_bytes,
size_t* bytes_allocated,
- size_t* usable_size) {
+ size_t* usable_size,
+ size_t* bytes_tl_bulk_allocated) {
Locks::mutator_lock_->AssertExclusiveHeld(self);
num_bytes = RoundUp(num_bytes, kAlignment);
uint8_t* end = end_.LoadRelaxed();
@@ -54,6 +57,7 @@
if (UNLIKELY(usable_size != nullptr)) {
*usable_size = num_bytes;
}
+ *bytes_tl_bulk_allocated = num_bytes;
return obj;
}
diff --git a/runtime/gc/space/bump_pointer_space.cc b/runtime/gc/space/bump_pointer_space.cc
index fbfc449..1303d77 100644
--- a/runtime/gc/space/bump_pointer_space.cc
+++ b/runtime/gc/space/bump_pointer_space.cc
@@ -93,12 +93,13 @@
return reinterpret_cast<mirror::Object*>(RoundUp(position, kAlignment));
}
-void BumpPointerSpace::RevokeThreadLocalBuffers(Thread* thread) {
+size_t BumpPointerSpace::RevokeThreadLocalBuffers(Thread* thread) {
MutexLock mu(Thread::Current(), block_lock_);
RevokeThreadLocalBuffersLocked(thread);
+ return 0U;
}
-void BumpPointerSpace::RevokeAllThreadLocalBuffers() {
+size_t BumpPointerSpace::RevokeAllThreadLocalBuffers() {
Thread* self = Thread::Current();
MutexLock mu(self, *Locks::runtime_shutdown_lock_);
MutexLock mu2(self, *Locks::thread_list_lock_);
@@ -107,6 +108,7 @@
for (Thread* thread : thread_list) {
RevokeThreadLocalBuffers(thread);
}
+ return 0U;
}
void BumpPointerSpace::AssertThreadLocalBuffersAreRevoked(Thread* thread) {
diff --git a/runtime/gc/space/bump_pointer_space.h b/runtime/gc/space/bump_pointer_space.h
index 089ede4..c496a42 100644
--- a/runtime/gc/space/bump_pointer_space.h
+++ b/runtime/gc/space/bump_pointer_space.h
@@ -47,10 +47,10 @@
// Allocate num_bytes, returns nullptr if the space is full.
mirror::Object* Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated,
- size_t* usable_size) OVERRIDE;
+ size_t* usable_size, size_t* bytes_tl_bulk_allocated) OVERRIDE;
// Thread-unsafe allocation for when mutators are suspended, used by the semispace collector.
mirror::Object* AllocThreadUnsafe(Thread* self, size_t num_bytes, size_t* bytes_allocated,
- size_t* usable_size)
+ size_t* usable_size, size_t* bytes_tl_bulk_allocated)
OVERRIDE EXCLUSIVE_LOCKS_REQUIRED(Locks::mutator_lock_);
mirror::Object* AllocNonvirtual(size_t num_bytes);
@@ -103,9 +103,9 @@
void Dump(std::ostream& os) const;
- void RevokeThreadLocalBuffers(Thread* thread) LOCKS_EXCLUDED(block_lock_);
- void RevokeAllThreadLocalBuffers() LOCKS_EXCLUDED(Locks::runtime_shutdown_lock_,
- Locks::thread_list_lock_);
+ size_t RevokeThreadLocalBuffers(Thread* thread) LOCKS_EXCLUDED(block_lock_);
+ size_t RevokeAllThreadLocalBuffers() LOCKS_EXCLUDED(Locks::runtime_shutdown_lock_,
+ Locks::thread_list_lock_);
void AssertThreadLocalBuffersAreRevoked(Thread* thread) LOCKS_EXCLUDED(block_lock_);
void AssertAllThreadLocalBuffersAreRevoked() LOCKS_EXCLUDED(Locks::runtime_shutdown_lock_,
Locks::thread_list_lock_);
diff --git a/runtime/gc/space/dlmalloc_space-inl.h b/runtime/gc/space/dlmalloc_space-inl.h
index 4c8a35e..9eace89 100644
--- a/runtime/gc/space/dlmalloc_space-inl.h
+++ b/runtime/gc/space/dlmalloc_space-inl.h
@@ -27,11 +27,13 @@
inline mirror::Object* DlMallocSpace::AllocNonvirtual(Thread* self, size_t num_bytes,
size_t* bytes_allocated,
- size_t* usable_size) {
+ size_t* usable_size,
+ size_t* bytes_tl_bulk_allocated) {
mirror::Object* obj;
{
MutexLock mu(self, lock_);
- obj = AllocWithoutGrowthLocked(self, num_bytes, bytes_allocated, usable_size);
+ obj = AllocWithoutGrowthLocked(self, num_bytes, bytes_allocated, usable_size,
+ bytes_tl_bulk_allocated);
}
if (LIKELY(obj != NULL)) {
// Zero freshly allocated memory, done while not holding the space's lock.
@@ -49,9 +51,11 @@
return size + kChunkOverhead;
}
-inline mirror::Object* DlMallocSpace::AllocWithoutGrowthLocked(Thread* /*self*/, size_t num_bytes,
- size_t* bytes_allocated,
- size_t* usable_size) {
+inline mirror::Object* DlMallocSpace::AllocWithoutGrowthLocked(
+ Thread* /*self*/, size_t num_bytes,
+ size_t* bytes_allocated,
+ size_t* usable_size,
+ size_t* bytes_tl_bulk_allocated) {
mirror::Object* result = reinterpret_cast<mirror::Object*>(mspace_malloc(mspace_, num_bytes));
if (LIKELY(result != NULL)) {
if (kDebugSpaces) {
@@ -61,6 +65,7 @@
size_t allocation_size = AllocationSizeNonvirtual(result, usable_size);
DCHECK(bytes_allocated != NULL);
*bytes_allocated = allocation_size;
+ *bytes_tl_bulk_allocated = allocation_size;
}
return result;
}
diff --git a/runtime/gc/space/dlmalloc_space.cc b/runtime/gc/space/dlmalloc_space.cc
index b8a9dd6..225861d 100644
--- a/runtime/gc/space/dlmalloc_space.cc
+++ b/runtime/gc/space/dlmalloc_space.cc
@@ -123,7 +123,8 @@
}
mirror::Object* DlMallocSpace::AllocWithGrowth(Thread* self, size_t num_bytes,
- size_t* bytes_allocated, size_t* usable_size) {
+ size_t* bytes_allocated, size_t* usable_size,
+ size_t* bytes_tl_bulk_allocated) {
mirror::Object* result;
{
MutexLock mu(self, lock_);
@@ -131,7 +132,8 @@
size_t max_allowed = Capacity();
mspace_set_footprint_limit(mspace_, max_allowed);
// Try the allocation.
- result = AllocWithoutGrowthLocked(self, num_bytes, bytes_allocated, usable_size);
+ result = AllocWithoutGrowthLocked(self, num_bytes, bytes_allocated, usable_size,
+ bytes_tl_bulk_allocated);
// Shrink back down as small as possible.
size_t footprint = mspace_footprint(mspace_);
mspace_set_footprint_limit(mspace_, footprint);
diff --git a/runtime/gc/space/dlmalloc_space.h b/runtime/gc/space/dlmalloc_space.h
index 6ce138c..1f80f1f 100644
--- a/runtime/gc/space/dlmalloc_space.h
+++ b/runtime/gc/space/dlmalloc_space.h
@@ -48,11 +48,15 @@
// Virtual to allow ValgrindMallocSpace to intercept.
virtual mirror::Object* AllocWithGrowth(Thread* self, size_t num_bytes, size_t* bytes_allocated,
- size_t* usable_size) OVERRIDE LOCKS_EXCLUDED(lock_);
+ size_t* usable_size,
+ size_t* bytes_tl_bulk_allocated)
+ OVERRIDE LOCKS_EXCLUDED(lock_);
// Virtual to allow ValgrindMallocSpace to intercept.
virtual mirror::Object* Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated,
- size_t* usable_size) OVERRIDE LOCKS_EXCLUDED(lock_) {
- return AllocNonvirtual(self, num_bytes, bytes_allocated, usable_size);
+ size_t* usable_size, size_t* bytes_tl_bulk_allocated)
+ OVERRIDE LOCKS_EXCLUDED(lock_) {
+ return AllocNonvirtual(self, num_bytes, bytes_allocated, usable_size,
+ bytes_tl_bulk_allocated);
}
// Virtual to allow ValgrindMallocSpace to intercept.
virtual size_t AllocationSize(mirror::Object* obj, size_t* usable_size) OVERRIDE {
@@ -67,15 +71,22 @@
LOCKS_EXCLUDED(lock_)
SHARED_LOCKS_REQUIRED(Locks::mutator_lock_);
- // DlMallocSpaces don't have thread local state.
- void RevokeThreadLocalBuffers(art::Thread*) OVERRIDE {
+ size_t MaxBytesBulkAllocatedFor(size_t num_bytes) OVERRIDE {
+ return num_bytes;
}
- void RevokeAllThreadLocalBuffers() OVERRIDE {
+
+ // DlMallocSpaces don't have thread local state.
+ size_t RevokeThreadLocalBuffers(art::Thread*) OVERRIDE {
+ return 0U;
+ }
+ size_t RevokeAllThreadLocalBuffers() OVERRIDE {
+ return 0U;
}
// Faster non-virtual allocation path.
mirror::Object* AllocNonvirtual(Thread* self, size_t num_bytes, size_t* bytes_allocated,
- size_t* usable_size) LOCKS_EXCLUDED(lock_);
+ size_t* usable_size, size_t* bytes_tl_bulk_allocated)
+ LOCKS_EXCLUDED(lock_);
// Faster non-virtual allocation size path.
size_t AllocationSizeNonvirtual(mirror::Object* obj, size_t* usable_size);
@@ -134,7 +145,8 @@
private:
mirror::Object* AllocWithoutGrowthLocked(Thread* self, size_t num_bytes, size_t* bytes_allocated,
- size_t* usable_size)
+ size_t* usable_size,
+ size_t* bytes_tl_bulk_allocated)
EXCLUSIVE_LOCKS_REQUIRED(lock_);
void* CreateAllocator(void* base, size_t morecore_start, size_t initial_size,
diff --git a/runtime/gc/space/large_object_space.cc b/runtime/gc/space/large_object_space.cc
index 7523de5..5c8e4b9 100644
--- a/runtime/gc/space/large_object_space.cc
+++ b/runtime/gc/space/large_object_space.cc
@@ -38,10 +38,11 @@
}
virtual mirror::Object* Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated,
- size_t* usable_size) OVERRIDE {
+ size_t* usable_size, size_t* bytes_tl_bulk_allocated)
+ OVERRIDE {
mirror::Object* obj =
LargeObjectMapSpace::Alloc(self, num_bytes + kValgrindRedZoneBytes * 2, bytes_allocated,
- usable_size);
+ usable_size, bytes_tl_bulk_allocated);
mirror::Object* object_without_rdz = reinterpret_cast<mirror::Object*>(
reinterpret_cast<uintptr_t>(obj) + kValgrindRedZoneBytes);
VALGRIND_MAKE_MEM_NOACCESS(reinterpret_cast<void*>(obj), kValgrindRedZoneBytes);
@@ -108,7 +109,8 @@
}
mirror::Object* LargeObjectMapSpace::Alloc(Thread* self, size_t num_bytes,
- size_t* bytes_allocated, size_t* usable_size) {
+ size_t* bytes_allocated, size_t* usable_size,
+ size_t* bytes_tl_bulk_allocated) {
std::string error_msg;
MemMap* mem_map = MemMap::MapAnonymous("large object space allocation", nullptr, num_bytes,
PROT_READ | PROT_WRITE, true, false, &error_msg);
@@ -131,6 +133,8 @@
if (usable_size != nullptr) {
*usable_size = allocation_size;
}
+ DCHECK(bytes_tl_bulk_allocated != nullptr);
+ *bytes_tl_bulk_allocated = allocation_size;
num_bytes_allocated_ += allocation_size;
total_bytes_allocated_ += allocation_size;
++num_objects_allocated_;
@@ -413,7 +417,7 @@
}
mirror::Object* FreeListSpace::Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated,
- size_t* usable_size) {
+ size_t* usable_size, size_t* bytes_tl_bulk_allocated) {
MutexLock mu(self, lock_);
const size_t allocation_size = RoundUp(num_bytes, kAlignment);
AllocationInfo temp_info;
@@ -451,6 +455,8 @@
if (usable_size != nullptr) {
*usable_size = allocation_size;
}
+ DCHECK(bytes_tl_bulk_allocated != nullptr);
+ *bytes_tl_bulk_allocated = allocation_size;
// Need to do these inside of the lock.
++num_objects_allocated_;
++total_objects_allocated_;
diff --git a/runtime/gc/space/large_object_space.h b/runtime/gc/space/large_object_space.h
index 847f575..d1f9386 100644
--- a/runtime/gc/space/large_object_space.h
+++ b/runtime/gc/space/large_object_space.h
@@ -62,9 +62,11 @@
}
size_t FreeList(Thread* self, size_t num_ptrs, mirror::Object** ptrs) OVERRIDE;
// LargeObjectSpaces don't have thread local state.
- void RevokeThreadLocalBuffers(art::Thread*) OVERRIDE {
+ size_t RevokeThreadLocalBuffers(art::Thread*) OVERRIDE {
+ return 0U;
}
- void RevokeAllThreadLocalBuffers() OVERRIDE {
+ size_t RevokeAllThreadLocalBuffers() OVERRIDE {
+ return 0U;
}
bool IsAllocSpace() const OVERRIDE {
return true;
@@ -124,7 +126,7 @@
// Return the storage space required by obj.
size_t AllocationSize(mirror::Object* obj, size_t* usable_size);
mirror::Object* Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated,
- size_t* usable_size);
+ size_t* usable_size, size_t* bytes_tl_bulk_allocated);
size_t Free(Thread* self, mirror::Object* ptr);
void Walk(DlMallocSpace::WalkCallback, void* arg) OVERRIDE LOCKS_EXCLUDED(lock_);
// TODO: disabling thread safety analysis as this may be called when we already hold lock_.
@@ -153,7 +155,7 @@
size_t AllocationSize(mirror::Object* obj, size_t* usable_size) OVERRIDE
EXCLUSIVE_LOCKS_REQUIRED(lock_);
mirror::Object* Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated,
- size_t* usable_size) OVERRIDE;
+ size_t* usable_size, size_t* bytes_tl_bulk_allocated) OVERRIDE;
size_t Free(Thread* self, mirror::Object* obj) OVERRIDE;
void Walk(DlMallocSpace::WalkCallback callback, void* arg) OVERRIDE LOCKS_EXCLUDED(lock_);
void Dump(std::ostream& os) const;
diff --git a/runtime/gc/space/large_object_space_test.cc b/runtime/gc/space/large_object_space_test.cc
index e17bad8..a261663 100644
--- a/runtime/gc/space/large_object_space_test.cc
+++ b/runtime/gc/space/large_object_space_test.cc
@@ -49,11 +49,13 @@
while (requests.size() < num_allocations) {
size_t request_size = test_rand(&rand_seed) % max_allocation_size;
size_t allocation_size = 0;
+ size_t bytes_tl_bulk_allocated;
mirror::Object* obj = los->Alloc(Thread::Current(), request_size, &allocation_size,
- nullptr);
+ nullptr, &bytes_tl_bulk_allocated);
ASSERT_TRUE(obj != nullptr);
ASSERT_EQ(allocation_size, los->AllocationSize(obj, nullptr));
ASSERT_GE(allocation_size, request_size);
+ ASSERT_EQ(allocation_size, bytes_tl_bulk_allocated);
// Fill in our magic value.
uint8_t magic = (request_size & 0xFF) | 1;
memset(obj, magic, request_size);
@@ -83,9 +85,10 @@
// Test that dump doesn't crash.
los->Dump(LOG(INFO));
- size_t bytes_allocated = 0;
+ size_t bytes_allocated = 0, bytes_tl_bulk_allocated;
// Checks that the coalescing works.
- mirror::Object* obj = los->Alloc(Thread::Current(), 100 * MB, &bytes_allocated, nullptr);
+ mirror::Object* obj = los->Alloc(Thread::Current(), 100 * MB, &bytes_allocated, nullptr,
+ &bytes_tl_bulk_allocated);
EXPECT_TRUE(obj != nullptr);
los->Free(Thread::Current(), obj);
@@ -102,8 +105,9 @@
void Run(Thread* self) {
for (size_t i = 0; i < iterations_ ; ++i) {
- size_t alloc_size;
- mirror::Object* ptr = los_->Alloc(self, size_, &alloc_size, nullptr);
+ size_t alloc_size, bytes_tl_bulk_allocated;
+ mirror::Object* ptr = los_->Alloc(self, size_, &alloc_size, nullptr,
+ &bytes_tl_bulk_allocated);
NanoSleep((id_ + 3) * 1000); // (3+id) mu s
diff --git a/runtime/gc/space/malloc_space.h b/runtime/gc/space/malloc_space.h
index 06239e5..bbf1bbb 100644
--- a/runtime/gc/space/malloc_space.h
+++ b/runtime/gc/space/malloc_space.h
@@ -55,10 +55,11 @@
// Allocate num_bytes allowing the underlying space to grow.
virtual mirror::Object* AllocWithGrowth(Thread* self, size_t num_bytes,
- size_t* bytes_allocated, size_t* usable_size) = 0;
+ size_t* bytes_allocated, size_t* usable_size,
+ size_t* bytes_tl_bulk_allocated) = 0;
// Allocate num_bytes without allowing the underlying space to grow.
virtual mirror::Object* Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated,
- size_t* usable_size) = 0;
+ size_t* usable_size, size_t* bytes_tl_bulk_allocated) = 0;
// Return the storage space required by obj. If usable_size isn't nullptr then it is set to the
// amount of the storage space that may be used by obj.
virtual size_t AllocationSize(mirror::Object* obj, size_t* usable_size) = 0;
@@ -67,6 +68,11 @@
virtual size_t FreeList(Thread* self, size_t num_ptrs, mirror::Object** ptrs)
SHARED_LOCKS_REQUIRED(Locks::mutator_lock_) = 0;
+ // Returns the maximum bytes that could be allocated for the given
+ // size in bulk, that is the maximum value for the
+ // bytes_allocated_bulk out param returned by MallocSpace::Alloc().
+ virtual size_t MaxBytesBulkAllocatedFor(size_t num_bytes) = 0;
+
#ifndef NDEBUG
virtual void CheckMoreCoreForPrecondition() {} // to be overridden in the debug build.
#else
diff --git a/runtime/gc/space/region_space-inl.h b/runtime/gc/space/region_space-inl.h
index a4ed718..1cdf69d 100644
--- a/runtime/gc/space/region_space-inl.h
+++ b/runtime/gc/space/region_space-inl.h
@@ -24,30 +24,36 @@
namespace space {
inline mirror::Object* RegionSpace::Alloc(Thread*, size_t num_bytes, size_t* bytes_allocated,
- size_t* usable_size) {
+ size_t* usable_size,
+ size_t* bytes_tl_bulk_allocated) {
num_bytes = RoundUp(num_bytes, kAlignment);
- return AllocNonvirtual<false>(num_bytes, bytes_allocated, usable_size);
+ return AllocNonvirtual<false>(num_bytes, bytes_allocated, usable_size,
+ bytes_tl_bulk_allocated);
}
inline mirror::Object* RegionSpace::AllocThreadUnsafe(Thread* self, size_t num_bytes,
size_t* bytes_allocated,
- size_t* usable_size) {
+ size_t* usable_size,
+ size_t* bytes_tl_bulk_allocated) {
Locks::mutator_lock_->AssertExclusiveHeld(self);
- return Alloc(self, num_bytes, bytes_allocated, usable_size);
+ return Alloc(self, num_bytes, bytes_allocated, usable_size, bytes_tl_bulk_allocated);
}
template<bool kForEvac>
inline mirror::Object* RegionSpace::AllocNonvirtual(size_t num_bytes, size_t* bytes_allocated,
- size_t* usable_size) {
+ size_t* usable_size,
+ size_t* bytes_tl_bulk_allocated) {
DCHECK(IsAligned<kAlignment>(num_bytes));
mirror::Object* obj;
if (LIKELY(num_bytes <= kRegionSize)) {
// Non-large object.
if (!kForEvac) {
- obj = current_region_->Alloc(num_bytes, bytes_allocated, usable_size);
+ obj = current_region_->Alloc(num_bytes, bytes_allocated, usable_size,
+ bytes_tl_bulk_allocated);
} else {
DCHECK(evac_region_ != nullptr);
- obj = evac_region_->Alloc(num_bytes, bytes_allocated, usable_size);
+ obj = evac_region_->Alloc(num_bytes, bytes_allocated, usable_size,
+ bytes_tl_bulk_allocated);
}
if (LIKELY(obj != nullptr)) {
return obj;
@@ -55,9 +61,11 @@
MutexLock mu(Thread::Current(), region_lock_);
// Retry with current region since another thread may have updated it.
if (!kForEvac) {
- obj = current_region_->Alloc(num_bytes, bytes_allocated, usable_size);
+ obj = current_region_->Alloc(num_bytes, bytes_allocated, usable_size,
+ bytes_tl_bulk_allocated);
} else {
- obj = evac_region_->Alloc(num_bytes, bytes_allocated, usable_size);
+ obj = evac_region_->Alloc(num_bytes, bytes_allocated, usable_size,
+ bytes_tl_bulk_allocated);
}
if (LIKELY(obj != nullptr)) {
return obj;
@@ -73,7 +81,7 @@
r->Unfree(time_);
r->SetNewlyAllocated();
++num_non_free_regions_;
- obj = r->Alloc(num_bytes, bytes_allocated, usable_size);
+ obj = r->Alloc(num_bytes, bytes_allocated, usable_size, bytes_tl_bulk_allocated);
CHECK(obj != nullptr);
current_region_ = r;
return obj;
@@ -85,7 +93,7 @@
if (r->IsFree()) {
r->Unfree(time_);
++num_non_free_regions_;
- obj = r->Alloc(num_bytes, bytes_allocated, usable_size);
+ obj = r->Alloc(num_bytes, bytes_allocated, usable_size, bytes_tl_bulk_allocated);
CHECK(obj != nullptr);
evac_region_ = r;
return obj;
@@ -94,7 +102,8 @@
}
} else {
// Large object.
- obj = AllocLarge<kForEvac>(num_bytes, bytes_allocated, usable_size);
+ obj = AllocLarge<kForEvac>(num_bytes, bytes_allocated, usable_size,
+ bytes_tl_bulk_allocated);
if (LIKELY(obj != nullptr)) {
return obj;
}
@@ -103,7 +112,8 @@
}
inline mirror::Object* RegionSpace::Region::Alloc(size_t num_bytes, size_t* bytes_allocated,
- size_t* usable_size) {
+ size_t* usable_size,
+ size_t* bytes_tl_bulk_allocated) {
DCHECK(IsAllocated() && IsInToSpace());
DCHECK(IsAligned<kAlignment>(num_bytes));
Atomic<uint8_t*>* atomic_top = reinterpret_cast<Atomic<uint8_t*>*>(&top_);
@@ -124,6 +134,7 @@
if (usable_size != nullptr) {
*usable_size = num_bytes;
}
+ *bytes_tl_bulk_allocated = num_bytes;
return reinterpret_cast<mirror::Object*>(old_top);
}
@@ -253,7 +264,8 @@
template<bool kForEvac>
mirror::Object* RegionSpace::AllocLarge(size_t num_bytes, size_t* bytes_allocated,
- size_t* usable_size) {
+ size_t* usable_size,
+ size_t* bytes_tl_bulk_allocated) {
DCHECK(IsAligned<kAlignment>(num_bytes));
DCHECK_GT(num_bytes, kRegionSize);
size_t num_regs = RoundUp(num_bytes, kRegionSize) / kRegionSize;
@@ -300,6 +312,7 @@
if (usable_size != nullptr) {
*usable_size = num_regs * kRegionSize;
}
+ *bytes_tl_bulk_allocated = num_bytes;
return reinterpret_cast<mirror::Object*>(first_reg->Begin());
} else {
// right points to the non-free region. Start with the one after it.
diff --git a/runtime/gc/space/region_space.cc b/runtime/gc/space/region_space.cc
index 8bb73d6..814ab6c 100644
--- a/runtime/gc/space/region_space.cc
+++ b/runtime/gc/space/region_space.cc
@@ -76,7 +76,7 @@
current_region_ = &full_region_;
evac_region_ = nullptr;
size_t ignored;
- DCHECK(full_region_.Alloc(kAlignment, &ignored, nullptr) == nullptr);
+ DCHECK(full_region_.Alloc(kAlignment, &ignored, nullptr, &ignored) == nullptr);
}
size_t RegionSpace::FromSpaceSize() {
@@ -356,9 +356,10 @@
return false;
}
-void RegionSpace::RevokeThreadLocalBuffers(Thread* thread) {
+size_t RegionSpace::RevokeThreadLocalBuffers(Thread* thread) {
MutexLock mu(Thread::Current(), region_lock_);
RevokeThreadLocalBuffersLocked(thread);
+ return 0U;
}
void RegionSpace::RevokeThreadLocalBuffersLocked(Thread* thread) {
@@ -377,7 +378,7 @@
thread->SetTlab(nullptr, nullptr);
}
-void RegionSpace::RevokeAllThreadLocalBuffers() {
+size_t RegionSpace::RevokeAllThreadLocalBuffers() {
Thread* self = Thread::Current();
MutexLock mu(self, *Locks::runtime_shutdown_lock_);
MutexLock mu2(self, *Locks::thread_list_lock_);
@@ -385,6 +386,7 @@
for (Thread* thread : thread_list) {
RevokeThreadLocalBuffers(thread);
}
+ return 0U;
}
void RegionSpace::AssertThreadLocalBuffersAreRevoked(Thread* thread) {
diff --git a/runtime/gc/space/region_space.h b/runtime/gc/space/region_space.h
index 4160547..b88ce24 100644
--- a/runtime/gc/space/region_space.h
+++ b/runtime/gc/space/region_space.h
@@ -42,18 +42,20 @@
// Allocate num_bytes, returns nullptr if the space is full.
mirror::Object* Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated,
- size_t* usable_size) OVERRIDE;
+ size_t* usable_size, size_t* bytes_tl_bulk_allocated) OVERRIDE;
// Thread-unsafe allocation for when mutators are suspended, used by the semispace collector.
mirror::Object* AllocThreadUnsafe(Thread* self, size_t num_bytes, size_t* bytes_allocated,
- size_t* usable_size)
+ size_t* usable_size, size_t* bytes_tl_bulk_allocated)
OVERRIDE EXCLUSIVE_LOCKS_REQUIRED(Locks::mutator_lock_);
// The main allocation routine.
template<bool kForEvac>
ALWAYS_INLINE mirror::Object* AllocNonvirtual(size_t num_bytes, size_t* bytes_allocated,
- size_t* usable_size);
+ size_t* usable_size,
+ size_t* bytes_tl_bulk_allocated);
// Allocate/free large objects (objects that are larger than the region size.)
template<bool kForEvac>
- mirror::Object* AllocLarge(size_t num_bytes, size_t* bytes_allocated, size_t* usable_size);
+ mirror::Object* AllocLarge(size_t num_bytes, size_t* bytes_allocated, size_t* usable_size,
+ size_t* bytes_tl_bulk_allocated);
void FreeLarge(mirror::Object* large_obj, size_t bytes_allocated);
// Return the storage space required by obj.
@@ -87,10 +89,10 @@
void DumpRegions(std::ostream& os);
void DumpNonFreeRegions(std::ostream& os);
- void RevokeThreadLocalBuffers(Thread* thread) LOCKS_EXCLUDED(region_lock_);
+ size_t RevokeThreadLocalBuffers(Thread* thread) LOCKS_EXCLUDED(region_lock_);
void RevokeThreadLocalBuffersLocked(Thread* thread) EXCLUSIVE_LOCKS_REQUIRED(region_lock_);
- void RevokeAllThreadLocalBuffers() LOCKS_EXCLUDED(Locks::runtime_shutdown_lock_,
- Locks::thread_list_lock_);
+ size_t RevokeAllThreadLocalBuffers() LOCKS_EXCLUDED(Locks::runtime_shutdown_lock_,
+ Locks::thread_list_lock_);
void AssertThreadLocalBuffersAreRevoked(Thread* thread) LOCKS_EXCLUDED(region_lock_);
void AssertAllThreadLocalBuffersAreRevoked() LOCKS_EXCLUDED(Locks::runtime_shutdown_lock_,
Locks::thread_list_lock_);
@@ -269,7 +271,8 @@
}
ALWAYS_INLINE mirror::Object* Alloc(size_t num_bytes, size_t* bytes_allocated,
- size_t* usable_size);
+ size_t* usable_size,
+ size_t* bytes_tl_bulk_allocated);
bool IsFree() const {
bool is_free = state_ == RegionState::kRegionStateFree;
diff --git a/runtime/gc/space/rosalloc_space-inl.h b/runtime/gc/space/rosalloc_space-inl.h
index 5d6642d..9d582a3 100644
--- a/runtime/gc/space/rosalloc_space-inl.h
+++ b/runtime/gc/space/rosalloc_space-inl.h
@@ -26,13 +26,19 @@
namespace gc {
namespace space {
+template<bool kMaybeRunningOnValgrind>
inline size_t RosAllocSpace::AllocationSizeNonvirtual(mirror::Object* obj, size_t* usable_size) {
// obj is a valid object. Use its class in the header to get the size.
// Don't use verification since the object may be dead if we are sweeping.
size_t size = obj->SizeOf<kVerifyNone>();
- bool running_on_valgrind = RUNNING_ON_VALGRIND != 0;
- if (running_on_valgrind) {
- size += 2 * kDefaultValgrindRedZoneBytes;
+ bool running_on_valgrind = false;
+ if (kMaybeRunningOnValgrind) {
+ running_on_valgrind = RUNNING_ON_VALGRIND != 0;
+ if (running_on_valgrind) {
+ size += 2 * kDefaultValgrindRedZoneBytes;
+ }
+ } else {
+ DCHECK_EQ(RUNNING_ON_VALGRIND, 0U);
}
size_t size_by_size = rosalloc_->UsableSize(size);
if (kIsDebugBuild) {
@@ -55,28 +61,50 @@
template<bool kThreadSafe>
inline mirror::Object* RosAllocSpace::AllocCommon(Thread* self, size_t num_bytes,
- size_t* bytes_allocated, size_t* usable_size) {
- size_t rosalloc_size = 0;
+ size_t* bytes_allocated, size_t* usable_size,
+ size_t* bytes_tl_bulk_allocated) {
+ size_t rosalloc_bytes_allocated = 0;
+ size_t rosalloc_usable_size = 0;
+ size_t rosalloc_bytes_tl_bulk_allocated = 0;
if (!kThreadSafe) {
Locks::mutator_lock_->AssertExclusiveHeld(self);
}
mirror::Object* result = reinterpret_cast<mirror::Object*>(
- rosalloc_->Alloc<kThreadSafe>(self, num_bytes, &rosalloc_size));
+ rosalloc_->Alloc<kThreadSafe>(self, num_bytes, &rosalloc_bytes_allocated,
+ &rosalloc_usable_size,
+ &rosalloc_bytes_tl_bulk_allocated));
if (LIKELY(result != NULL)) {
if (kDebugSpaces) {
CHECK(Contains(result)) << "Allocation (" << reinterpret_cast<void*>(result)
<< ") not in bounds of allocation space " << *this;
}
DCHECK(bytes_allocated != NULL);
- *bytes_allocated = rosalloc_size;
- DCHECK_EQ(rosalloc_size, rosalloc_->UsableSize(result));
+ *bytes_allocated = rosalloc_bytes_allocated;
+ DCHECK_EQ(rosalloc_usable_size, rosalloc_->UsableSize(result));
if (usable_size != nullptr) {
- *usable_size = rosalloc_size;
+ *usable_size = rosalloc_usable_size;
}
+ DCHECK(bytes_tl_bulk_allocated != NULL);
+ *bytes_tl_bulk_allocated = rosalloc_bytes_tl_bulk_allocated;
}
return result;
}
+inline bool RosAllocSpace::CanAllocThreadLocal(Thread* self, size_t num_bytes) {
+ return rosalloc_->CanAllocFromThreadLocalRun(self, num_bytes);
+}
+
+inline mirror::Object* RosAllocSpace::AllocThreadLocal(Thread* self, size_t num_bytes,
+ size_t* bytes_allocated) {
+ DCHECK(bytes_allocated != nullptr);
+ return reinterpret_cast<mirror::Object*>(
+ rosalloc_->AllocFromThreadLocalRun(self, num_bytes, bytes_allocated));
+}
+
+inline size_t RosAllocSpace::MaxBytesBulkAllocatedForNonvirtual(size_t num_bytes) {
+ return rosalloc_->MaxBytesBulkAllocatedFor(num_bytes);
+}
+
} // namespace space
} // namespace gc
} // namespace art
diff --git a/runtime/gc/space/rosalloc_space.cc b/runtime/gc/space/rosalloc_space.cc
index ced25a4..f140021 100644
--- a/runtime/gc/space/rosalloc_space.cc
+++ b/runtime/gc/space/rosalloc_space.cc
@@ -154,7 +154,8 @@
}
mirror::Object* RosAllocSpace::AllocWithGrowth(Thread* self, size_t num_bytes,
- size_t* bytes_allocated, size_t* usable_size) {
+ size_t* bytes_allocated, size_t* usable_size,
+ size_t* bytes_tl_bulk_allocated) {
mirror::Object* result;
{
MutexLock mu(self, lock_);
@@ -162,7 +163,8 @@
size_t max_allowed = Capacity();
rosalloc_->SetFootprintLimit(max_allowed);
// Try the allocation.
- result = AllocCommon(self, num_bytes, bytes_allocated, usable_size);
+ result = AllocCommon(self, num_bytes, bytes_allocated, usable_size,
+ bytes_tl_bulk_allocated);
// Shrink back down as small as possible.
size_t footprint = rosalloc_->Footprint();
rosalloc_->SetFootprintLimit(footprint);
@@ -209,7 +211,7 @@
__builtin_prefetch(reinterpret_cast<char*>(ptrs[i + kPrefetchLookAhead]));
}
if (kVerifyFreedBytes) {
- verify_bytes += AllocationSizeNonvirtual(ptrs[i], nullptr);
+ verify_bytes += AllocationSizeNonvirtual<true>(ptrs[i], nullptr);
}
}
@@ -338,12 +340,12 @@
}
}
-void RosAllocSpace::RevokeThreadLocalBuffers(Thread* thread) {
- rosalloc_->RevokeThreadLocalRuns(thread);
+size_t RosAllocSpace::RevokeThreadLocalBuffers(Thread* thread) {
+ return rosalloc_->RevokeThreadLocalRuns(thread);
}
-void RosAllocSpace::RevokeAllThreadLocalBuffers() {
- rosalloc_->RevokeAllThreadLocalRuns();
+size_t RosAllocSpace::RevokeAllThreadLocalBuffers() {
+ return rosalloc_->RevokeAllThreadLocalRuns();
}
void RosAllocSpace::AssertThreadLocalBuffersAreRevoked(Thread* thread) {
diff --git a/runtime/gc/space/rosalloc_space.h b/runtime/gc/space/rosalloc_space.h
index c856e95..36268f7 100644
--- a/runtime/gc/space/rosalloc_space.h
+++ b/runtime/gc/space/rosalloc_space.h
@@ -47,18 +47,21 @@
bool low_memory_mode, bool can_move_objects);
mirror::Object* AllocWithGrowth(Thread* self, size_t num_bytes, size_t* bytes_allocated,
- size_t* usable_size) OVERRIDE LOCKS_EXCLUDED(lock_);
+ size_t* usable_size, size_t* bytes_tl_bulk_allocated)
+ OVERRIDE LOCKS_EXCLUDED(lock_);
mirror::Object* Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated,
- size_t* usable_size) OVERRIDE {
- return AllocNonvirtual(self, num_bytes, bytes_allocated, usable_size);
+ size_t* usable_size, size_t* bytes_tl_bulk_allocated) OVERRIDE {
+ return AllocNonvirtual(self, num_bytes, bytes_allocated, usable_size,
+ bytes_tl_bulk_allocated);
}
mirror::Object* AllocThreadUnsafe(Thread* self, size_t num_bytes, size_t* bytes_allocated,
- size_t* usable_size)
+ size_t* usable_size, size_t* bytes_tl_bulk_allocated)
OVERRIDE EXCLUSIVE_LOCKS_REQUIRED(Locks::mutator_lock_) {
- return AllocNonvirtualThreadUnsafe(self, num_bytes, bytes_allocated, usable_size);
+ return AllocNonvirtualThreadUnsafe(self, num_bytes, bytes_allocated, usable_size,
+ bytes_tl_bulk_allocated);
}
size_t AllocationSize(mirror::Object* obj, size_t* usable_size) OVERRIDE {
- return AllocationSizeNonvirtual(obj, usable_size);
+ return AllocationSizeNonvirtual<true>(obj, usable_size);
}
size_t Free(Thread* self, mirror::Object* ptr) OVERRIDE
SHARED_LOCKS_REQUIRED(Locks::mutator_lock_);
@@ -66,17 +69,33 @@
SHARED_LOCKS_REQUIRED(Locks::mutator_lock_);
mirror::Object* AllocNonvirtual(Thread* self, size_t num_bytes, size_t* bytes_allocated,
- size_t* usable_size) {
+ size_t* usable_size, size_t* bytes_tl_bulk_allocated) {
// RosAlloc zeroes memory internally.
- return AllocCommon(self, num_bytes, bytes_allocated, usable_size);
+ return AllocCommon(self, num_bytes, bytes_allocated, usable_size,
+ bytes_tl_bulk_allocated);
}
mirror::Object* AllocNonvirtualThreadUnsafe(Thread* self, size_t num_bytes,
- size_t* bytes_allocated, size_t* usable_size) {
+ size_t* bytes_allocated, size_t* usable_size,
+ size_t* bytes_tl_bulk_allocated) {
// RosAlloc zeroes memory internally. Pass in false for thread unsafe.
- return AllocCommon<false>(self, num_bytes, bytes_allocated, usable_size);
+ return AllocCommon<false>(self, num_bytes, bytes_allocated, usable_size,
+ bytes_tl_bulk_allocated);
}
+ // Returns true if the given allocation request can be allocated in
+ // an existing thread local run without allocating a new run.
+ ALWAYS_INLINE bool CanAllocThreadLocal(Thread* self, size_t num_bytes);
+ // Allocate the given allocation request in an existing thread local
+ // run without allocating a new run.
+ ALWAYS_INLINE mirror::Object* AllocThreadLocal(Thread* self, size_t num_bytes,
+ size_t* bytes_allocated);
+ size_t MaxBytesBulkAllocatedFor(size_t num_bytes) OVERRIDE {
+ return MaxBytesBulkAllocatedForNonvirtual(num_bytes);
+ }
+ ALWAYS_INLINE size_t MaxBytesBulkAllocatedForNonvirtual(size_t num_bytes);
+
// TODO: NO_THREAD_SAFETY_ANALYSIS because SizeOf() requires that mutator_lock is held.
+ template<bool kMaybeRunningOnValgrind>
size_t AllocationSizeNonvirtual(mirror::Object* obj, size_t* usable_size)
NO_THREAD_SAFETY_ANALYSIS;
@@ -99,8 +118,8 @@
uint64_t GetBytesAllocated() OVERRIDE;
uint64_t GetObjectsAllocated() OVERRIDE;
- void RevokeThreadLocalBuffers(Thread* thread);
- void RevokeAllThreadLocalBuffers();
+ size_t RevokeThreadLocalBuffers(Thread* thread);
+ size_t RevokeAllThreadLocalBuffers();
void AssertThreadLocalBuffersAreRevoked(Thread* thread);
void AssertAllThreadLocalBuffersAreRevoked();
@@ -134,7 +153,7 @@
private:
template<bool kThreadSafe = true>
mirror::Object* AllocCommon(Thread* self, size_t num_bytes, size_t* bytes_allocated,
- size_t* usable_size);
+ size_t* usable_size, size_t* bytes_tl_bulk_allocated);
void* CreateAllocator(void* base, size_t morecore_start, size_t initial_size,
size_t maximum_size, bool low_memory_mode) OVERRIDE {
diff --git a/runtime/gc/space/space.h b/runtime/gc/space/space.h
index d24650b..f2378d9 100644
--- a/runtime/gc/space/space.h
+++ b/runtime/gc/space/space.h
@@ -203,14 +203,24 @@
// succeeds, the output parameter bytes_allocated will be set to the
// actually allocated bytes which is >= num_bytes.
// Alloc can be called from multiple threads at the same time and must be thread-safe.
+ //
+ // bytes_tl_bulk_allocated - bytes allocated in bulk ahead of time for a thread local allocation,
+ // if applicable. It can be
+ // 1) equal to bytes_allocated if it's not a thread local allocation,
+ // 2) greater than bytes_allocated if it's a thread local
+ // allocation that required a new buffer, or
+ // 3) zero if it's a thread local allocation in an existing
+ // buffer.
+ // This is what is to be added to Heap::num_bytes_allocated_.
virtual mirror::Object* Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated,
- size_t* usable_size) = 0;
+ size_t* usable_size, size_t* bytes_tl_bulk_allocated) = 0;
// Thread-unsafe allocation for when mutators are suspended, used by the semispace collector.
virtual mirror::Object* AllocThreadUnsafe(Thread* self, size_t num_bytes, size_t* bytes_allocated,
- size_t* usable_size)
+ size_t* usable_size,
+ size_t* bytes_tl_bulk_allocated)
EXCLUSIVE_LOCKS_REQUIRED(Locks::mutator_lock_) {
- return Alloc(self, num_bytes, bytes_allocated, usable_size);
+ return Alloc(self, num_bytes, bytes_allocated, usable_size, bytes_tl_bulk_allocated);
}
// Return the storage space required by obj.
@@ -224,11 +234,15 @@
// Revoke any sort of thread-local buffers that are used to speed up allocations for the given
// thread, if the alloc space implementation uses any.
- virtual void RevokeThreadLocalBuffers(Thread* thread) = 0;
+ // Returns the total free bytes in the revoked thread local runs that's to be subtracted
+ // from Heap::num_bytes_allocated_ or zero if unnecessary.
+ virtual size_t RevokeThreadLocalBuffers(Thread* thread) = 0;
// Revoke any sort of thread-local buffers that are used to speed up allocations for all the
// threads, if the alloc space implementation uses any.
- virtual void RevokeAllThreadLocalBuffers() = 0;
+ // Returns the total free bytes in the revoked thread local runs that's to be subtracted
+ // from Heap::num_bytes_allocated_ or zero if unnecessary.
+ virtual size_t RevokeAllThreadLocalBuffers() = 0;
virtual void LogFragmentationAllocFailure(std::ostream& os, size_t failed_alloc_bytes) = 0;
diff --git a/runtime/gc/space/space_test.h b/runtime/gc/space/space_test.h
index 09d10dd..3e9e9f7 100644
--- a/runtime/gc/space/space_test.h
+++ b/runtime/gc/space/space_test.h
@@ -61,11 +61,13 @@
}
mirror::Object* Alloc(space::MallocSpace* alloc_space, Thread* self, size_t bytes,
- size_t* bytes_allocated, size_t* usable_size)
+ size_t* bytes_allocated, size_t* usable_size,
+ size_t* bytes_tl_bulk_allocated)
SHARED_LOCKS_REQUIRED(Locks::mutator_lock_) {
StackHandleScope<1> hs(self);
Handle<mirror::Class> byte_array_class(hs.NewHandle(GetByteArrayClass(self)));
- mirror::Object* obj = alloc_space->Alloc(self, bytes, bytes_allocated, usable_size);
+ mirror::Object* obj = alloc_space->Alloc(self, bytes, bytes_allocated, usable_size,
+ bytes_tl_bulk_allocated);
if (obj != nullptr) {
InstallClass(obj, byte_array_class.Get(), bytes);
}
@@ -73,11 +75,13 @@
}
mirror::Object* AllocWithGrowth(space::MallocSpace* alloc_space, Thread* self, size_t bytes,
- size_t* bytes_allocated, size_t* usable_size)
+ size_t* bytes_allocated, size_t* usable_size,
+ size_t* bytes_tl_bulk_allocated)
SHARED_LOCKS_REQUIRED(Locks::mutator_lock_) {
StackHandleScope<1> hs(self);
Handle<mirror::Class> byte_array_class(hs.NewHandle(GetByteArrayClass(self)));
- mirror::Object* obj = alloc_space->AllocWithGrowth(self, bytes, bytes_allocated, usable_size);
+ mirror::Object* obj = alloc_space->AllocWithGrowth(self, bytes, bytes_allocated, usable_size,
+ bytes_tl_bulk_allocated);
if (obj != nullptr) {
InstallClass(obj, byte_array_class.Get(), bytes);
}
@@ -182,34 +186,38 @@
ScopedObjectAccess soa(self);
// Succeeds, fits without adjusting the footprint limit.
- size_t ptr1_bytes_allocated, ptr1_usable_size;
+ size_t ptr1_bytes_allocated, ptr1_usable_size, ptr1_bytes_tl_bulk_allocated;
StackHandleScope<3> hs(soa.Self());
MutableHandle<mirror::Object> ptr1(
- hs.NewHandle(Alloc(space, self, 1 * MB, &ptr1_bytes_allocated, &ptr1_usable_size)));
+ hs.NewHandle(Alloc(space, self, 1 * MB, &ptr1_bytes_allocated, &ptr1_usable_size,
+ &ptr1_bytes_tl_bulk_allocated)));
EXPECT_TRUE(ptr1.Get() != nullptr);
EXPECT_LE(1U * MB, ptr1_bytes_allocated);
EXPECT_LE(1U * MB, ptr1_usable_size);
EXPECT_LE(ptr1_usable_size, ptr1_bytes_allocated);
+ EXPECT_EQ(ptr1_bytes_tl_bulk_allocated, ptr1_bytes_allocated);
// Fails, requires a higher footprint limit.
- mirror::Object* ptr2 = Alloc(space, self, 8 * MB, &dummy, nullptr);
+ mirror::Object* ptr2 = Alloc(space, self, 8 * MB, &dummy, nullptr, &dummy);
EXPECT_TRUE(ptr2 == nullptr);
// Succeeds, adjusts the footprint.
- size_t ptr3_bytes_allocated, ptr3_usable_size;
+ size_t ptr3_bytes_allocated, ptr3_usable_size, ptr3_bytes_tl_bulk_allocated;
MutableHandle<mirror::Object> ptr3(
- hs.NewHandle(AllocWithGrowth(space, self, 8 * MB, &ptr3_bytes_allocated, &ptr3_usable_size)));
+ hs.NewHandle(AllocWithGrowth(space, self, 8 * MB, &ptr3_bytes_allocated, &ptr3_usable_size,
+ &ptr3_bytes_tl_bulk_allocated)));
EXPECT_TRUE(ptr3.Get() != nullptr);
EXPECT_LE(8U * MB, ptr3_bytes_allocated);
EXPECT_LE(8U * MB, ptr3_usable_size);
EXPECT_LE(ptr3_usable_size, ptr3_bytes_allocated);
+ EXPECT_EQ(ptr3_bytes_tl_bulk_allocated, ptr3_bytes_allocated);
// Fails, requires a higher footprint limit.
- mirror::Object* ptr4 = space->Alloc(self, 8 * MB, &dummy, nullptr);
+ mirror::Object* ptr4 = space->Alloc(self, 8 * MB, &dummy, nullptr, &dummy);
EXPECT_TRUE(ptr4 == nullptr);
// Also fails, requires a higher allowed footprint.
- mirror::Object* ptr5 = space->AllocWithGrowth(self, 8 * MB, &dummy, nullptr);
+ mirror::Object* ptr5 = space->AllocWithGrowth(self, 8 * MB, &dummy, nullptr, &dummy);
EXPECT_TRUE(ptr5 == nullptr);
// Release some memory.
@@ -219,13 +227,15 @@
EXPECT_LE(8U * MB, free3);
// Succeeds, now that memory has been freed.
- size_t ptr6_bytes_allocated, ptr6_usable_size;
+ size_t ptr6_bytes_allocated, ptr6_usable_size, ptr6_bytes_tl_bulk_allocated;
Handle<mirror::Object> ptr6(
- hs.NewHandle(AllocWithGrowth(space, self, 9 * MB, &ptr6_bytes_allocated, &ptr6_usable_size)));
+ hs.NewHandle(AllocWithGrowth(space, self, 9 * MB, &ptr6_bytes_allocated, &ptr6_usable_size,
+ &ptr6_bytes_tl_bulk_allocated)));
EXPECT_TRUE(ptr6.Get() != nullptr);
EXPECT_LE(9U * MB, ptr6_bytes_allocated);
EXPECT_LE(9U * MB, ptr6_usable_size);
EXPECT_LE(ptr6_usable_size, ptr6_bytes_allocated);
+ EXPECT_EQ(ptr6_bytes_tl_bulk_allocated, ptr6_bytes_allocated);
// Final clean up.
size_t free1 = space->AllocationSize(ptr1.Get(), nullptr);
@@ -233,7 +243,7 @@
EXPECT_LE(1U * MB, free1);
// Make sure that the zygote space isn't directly at the start of the space.
- EXPECT_TRUE(space->Alloc(self, 1U * MB, &dummy, nullptr) != nullptr);
+ EXPECT_TRUE(space->Alloc(self, 1U * MB, &dummy, nullptr, &dummy) != nullptr);
gc::Heap* heap = Runtime::Current()->GetHeap();
space::Space* old_space = space;
@@ -250,22 +260,26 @@
AddSpace(space, false);
// Succeeds, fits without adjusting the footprint limit.
- ptr1.Assign(Alloc(space, self, 1 * MB, &ptr1_bytes_allocated, &ptr1_usable_size));
+ ptr1.Assign(Alloc(space, self, 1 * MB, &ptr1_bytes_allocated, &ptr1_usable_size,
+ &ptr1_bytes_tl_bulk_allocated));
EXPECT_TRUE(ptr1.Get() != nullptr);
EXPECT_LE(1U * MB, ptr1_bytes_allocated);
EXPECT_LE(1U * MB, ptr1_usable_size);
EXPECT_LE(ptr1_usable_size, ptr1_bytes_allocated);
+ EXPECT_EQ(ptr1_bytes_tl_bulk_allocated, ptr1_bytes_allocated);
// Fails, requires a higher footprint limit.
- ptr2 = Alloc(space, self, 8 * MB, &dummy, nullptr);
+ ptr2 = Alloc(space, self, 8 * MB, &dummy, nullptr, &dummy);
EXPECT_TRUE(ptr2 == nullptr);
// Succeeds, adjusts the footprint.
- ptr3.Assign(AllocWithGrowth(space, self, 2 * MB, &ptr3_bytes_allocated, &ptr3_usable_size));
+ ptr3.Assign(AllocWithGrowth(space, self, 2 * MB, &ptr3_bytes_allocated, &ptr3_usable_size,
+ &ptr3_bytes_tl_bulk_allocated));
EXPECT_TRUE(ptr3.Get() != nullptr);
EXPECT_LE(2U * MB, ptr3_bytes_allocated);
EXPECT_LE(2U * MB, ptr3_usable_size);
EXPECT_LE(ptr3_usable_size, ptr3_bytes_allocated);
+ EXPECT_EQ(ptr3_bytes_tl_bulk_allocated, ptr3_bytes_allocated);
space->Free(self, ptr3.Assign(nullptr));
// Final clean up.
@@ -285,34 +299,38 @@
AddSpace(space);
// Succeeds, fits without adjusting the footprint limit.
- size_t ptr1_bytes_allocated, ptr1_usable_size;
+ size_t ptr1_bytes_allocated, ptr1_usable_size, ptr1_bytes_tl_bulk_allocated;
StackHandleScope<3> hs(soa.Self());
MutableHandle<mirror::Object> ptr1(
- hs.NewHandle(Alloc(space, self, 1 * MB, &ptr1_bytes_allocated, &ptr1_usable_size)));
+ hs.NewHandle(Alloc(space, self, 1 * MB, &ptr1_bytes_allocated, &ptr1_usable_size,
+ &ptr1_bytes_tl_bulk_allocated)));
EXPECT_TRUE(ptr1.Get() != nullptr);
EXPECT_LE(1U * MB, ptr1_bytes_allocated);
EXPECT_LE(1U * MB, ptr1_usable_size);
EXPECT_LE(ptr1_usable_size, ptr1_bytes_allocated);
+ EXPECT_EQ(ptr1_bytes_tl_bulk_allocated, ptr1_bytes_allocated);
// Fails, requires a higher footprint limit.
- mirror::Object* ptr2 = Alloc(space, self, 8 * MB, &dummy, nullptr);
+ mirror::Object* ptr2 = Alloc(space, self, 8 * MB, &dummy, nullptr, &dummy);
EXPECT_TRUE(ptr2 == nullptr);
// Succeeds, adjusts the footprint.
- size_t ptr3_bytes_allocated, ptr3_usable_size;
+ size_t ptr3_bytes_allocated, ptr3_usable_size, ptr3_bytes_tl_bulk_allocated;
MutableHandle<mirror::Object> ptr3(
- hs.NewHandle(AllocWithGrowth(space, self, 8 * MB, &ptr3_bytes_allocated, &ptr3_usable_size)));
+ hs.NewHandle(AllocWithGrowth(space, self, 8 * MB, &ptr3_bytes_allocated, &ptr3_usable_size,
+ &ptr3_bytes_tl_bulk_allocated)));
EXPECT_TRUE(ptr3.Get() != nullptr);
EXPECT_LE(8U * MB, ptr3_bytes_allocated);
EXPECT_LE(8U * MB, ptr3_usable_size);
EXPECT_LE(ptr3_usable_size, ptr3_bytes_allocated);
+ EXPECT_EQ(ptr3_bytes_tl_bulk_allocated, ptr3_bytes_allocated);
// Fails, requires a higher footprint limit.
- mirror::Object* ptr4 = Alloc(space, self, 8 * MB, &dummy, nullptr);
+ mirror::Object* ptr4 = Alloc(space, self, 8 * MB, &dummy, nullptr, &dummy);
EXPECT_TRUE(ptr4 == nullptr);
// Also fails, requires a higher allowed footprint.
- mirror::Object* ptr5 = AllocWithGrowth(space, self, 8 * MB, &dummy, nullptr);
+ mirror::Object* ptr5 = AllocWithGrowth(space, self, 8 * MB, &dummy, nullptr, &dummy);
EXPECT_TRUE(ptr5 == nullptr);
// Release some memory.
@@ -322,13 +340,15 @@
EXPECT_LE(8U * MB, free3);
// Succeeds, now that memory has been freed.
- size_t ptr6_bytes_allocated, ptr6_usable_size;
+ size_t ptr6_bytes_allocated, ptr6_usable_size, ptr6_bytes_tl_bulk_allocated;
Handle<mirror::Object> ptr6(
- hs.NewHandle(AllocWithGrowth(space, self, 9 * MB, &ptr6_bytes_allocated, &ptr6_usable_size)));
+ hs.NewHandle(AllocWithGrowth(space, self, 9 * MB, &ptr6_bytes_allocated, &ptr6_usable_size,
+ &ptr6_bytes_tl_bulk_allocated)));
EXPECT_TRUE(ptr6.Get() != nullptr);
EXPECT_LE(9U * MB, ptr6_bytes_allocated);
EXPECT_LE(9U * MB, ptr6_usable_size);
EXPECT_LE(ptr6_usable_size, ptr6_bytes_allocated);
+ EXPECT_EQ(ptr6_bytes_tl_bulk_allocated, ptr6_bytes_allocated);
// Final clean up.
size_t free1 = space->AllocationSize(ptr1.Get(), nullptr);
@@ -348,14 +368,16 @@
// Succeeds, fits without adjusting the max allowed footprint.
mirror::Object* lots_of_objects[1024];
for (size_t i = 0; i < arraysize(lots_of_objects); i++) {
- size_t allocation_size, usable_size;
+ size_t allocation_size, usable_size, bytes_tl_bulk_allocated;
size_t size_of_zero_length_byte_array = SizeOfZeroLengthByteArray();
lots_of_objects[i] = Alloc(space, self, size_of_zero_length_byte_array, &allocation_size,
- &usable_size);
+ &usable_size, &bytes_tl_bulk_allocated);
EXPECT_TRUE(lots_of_objects[i] != nullptr);
size_t computed_usable_size;
EXPECT_EQ(allocation_size, space->AllocationSize(lots_of_objects[i], &computed_usable_size));
EXPECT_EQ(usable_size, computed_usable_size);
+ EXPECT_TRUE(bytes_tl_bulk_allocated == 0 ||
+ bytes_tl_bulk_allocated >= allocation_size);
}
// Release memory.
@@ -363,12 +385,15 @@
// Succeeds, fits by adjusting the max allowed footprint.
for (size_t i = 0; i < arraysize(lots_of_objects); i++) {
- size_t allocation_size, usable_size;
- lots_of_objects[i] = AllocWithGrowth(space, self, 1024, &allocation_size, &usable_size);
+ size_t allocation_size, usable_size, bytes_tl_bulk_allocated;
+ lots_of_objects[i] = AllocWithGrowth(space, self, 1024, &allocation_size, &usable_size,
+ &bytes_tl_bulk_allocated);
EXPECT_TRUE(lots_of_objects[i] != nullptr);
size_t computed_usable_size;
EXPECT_EQ(allocation_size, space->AllocationSize(lots_of_objects[i], &computed_usable_size));
EXPECT_EQ(usable_size, computed_usable_size);
+ EXPECT_TRUE(bytes_tl_bulk_allocated == 0 ||
+ bytes_tl_bulk_allocated >= allocation_size);
}
// Release memory.
@@ -425,10 +450,13 @@
StackHandleScope<1> hs(soa.Self());
auto object(hs.NewHandle<mirror::Object>(nullptr));
size_t bytes_allocated = 0;
+ size_t bytes_tl_bulk_allocated;
if (round <= 1) {
- object.Assign(Alloc(space, self, alloc_size, &bytes_allocated, nullptr));
+ object.Assign(Alloc(space, self, alloc_size, &bytes_allocated, nullptr,
+ &bytes_tl_bulk_allocated));
} else {
- object.Assign(AllocWithGrowth(space, self, alloc_size, &bytes_allocated, nullptr));
+ object.Assign(AllocWithGrowth(space, self, alloc_size, &bytes_allocated, nullptr,
+ &bytes_tl_bulk_allocated));
}
footprint = space->GetFootprint();
EXPECT_GE(space->Size(), footprint); // invariant
@@ -441,6 +469,8 @@
} else {
EXPECT_GE(allocation_size, 8u);
}
+ EXPECT_TRUE(bytes_tl_bulk_allocated == 0 ||
+ bytes_tl_bulk_allocated >= allocation_size);
amount_allocated += allocation_size;
break;
}
@@ -518,11 +548,13 @@
auto large_object(hs.NewHandle<mirror::Object>(nullptr));
size_t three_quarters_space = (growth_limit / 2) + (growth_limit / 4);
size_t bytes_allocated = 0;
+ size_t bytes_tl_bulk_allocated;
if (round <= 1) {
- large_object.Assign(Alloc(space, self, three_quarters_space, &bytes_allocated, nullptr));
+ large_object.Assign(Alloc(space, self, three_quarters_space, &bytes_allocated, nullptr,
+ &bytes_tl_bulk_allocated));
} else {
large_object.Assign(AllocWithGrowth(space, self, three_quarters_space, &bytes_allocated,
- nullptr));
+ nullptr, &bytes_tl_bulk_allocated));
}
EXPECT_TRUE(large_object.Get() != nullptr);
diff --git a/runtime/gc/space/valgrind_malloc_space-inl.h b/runtime/gc/space/valgrind_malloc_space-inl.h
index ae8e892..bc329e1 100644
--- a/runtime/gc/space/valgrind_malloc_space-inl.h
+++ b/runtime/gc/space/valgrind_malloc_space-inl.h
@@ -32,10 +32,15 @@
template <size_t kValgrindRedZoneBytes, bool kUseObjSizeForUsable>
inline mirror::Object* AdjustForValgrind(void* obj_with_rdz, size_t num_bytes,
size_t bytes_allocated, size_t usable_size,
- size_t* bytes_allocated_out, size_t* usable_size_out) {
+ size_t bytes_tl_bulk_allocated,
+ size_t* bytes_allocated_out, size_t* usable_size_out,
+ size_t* bytes_tl_bulk_allocated_out) {
if (bytes_allocated_out != nullptr) {
*bytes_allocated_out = bytes_allocated;
}
+ if (bytes_tl_bulk_allocated_out != nullptr) {
+ *bytes_tl_bulk_allocated_out = bytes_tl_bulk_allocated;
+ }
// This cuts over-provision and is a trade-off between testing the over-provisioning code paths
// vs checking overflows in the regular paths.
@@ -82,20 +87,25 @@
kValgrindRedZoneBytes,
kAdjustForRedzoneInAllocSize,
kUseObjSizeForUsable>::AllocWithGrowth(
- Thread* self, size_t num_bytes, size_t* bytes_allocated_out, size_t* usable_size_out) {
+ Thread* self, size_t num_bytes, size_t* bytes_allocated_out, size_t* usable_size_out,
+ size_t* bytes_tl_bulk_allocated_out) {
size_t bytes_allocated;
size_t usable_size;
+ size_t bytes_tl_bulk_allocated;
void* obj_with_rdz = S::AllocWithGrowth(self, num_bytes + 2 * kValgrindRedZoneBytes,
- &bytes_allocated, &usable_size);
+ &bytes_allocated, &usable_size,
+ &bytes_tl_bulk_allocated);
if (obj_with_rdz == nullptr) {
return nullptr;
}
- return valgrind_details::AdjustForValgrind<kValgrindRedZoneBytes,
- kUseObjSizeForUsable>(obj_with_rdz, num_bytes,
- bytes_allocated, usable_size,
- bytes_allocated_out,
- usable_size_out);
+ return valgrind_details::AdjustForValgrind<kValgrindRedZoneBytes, kUseObjSizeForUsable>(
+ obj_with_rdz, num_bytes,
+ bytes_allocated, usable_size,
+ bytes_tl_bulk_allocated,
+ bytes_allocated_out,
+ usable_size_out,
+ bytes_tl_bulk_allocated_out);
}
template <typename S,
@@ -106,11 +116,13 @@
kValgrindRedZoneBytes,
kAdjustForRedzoneInAllocSize,
kUseObjSizeForUsable>::Alloc(
- Thread* self, size_t num_bytes, size_t* bytes_allocated_out, size_t* usable_size_out) {
+ Thread* self, size_t num_bytes, size_t* bytes_allocated_out, size_t* usable_size_out,
+ size_t* bytes_tl_bulk_allocated_out) {
size_t bytes_allocated;
size_t usable_size;
+ size_t bytes_tl_bulk_allocated;
void* obj_with_rdz = S::Alloc(self, num_bytes + 2 * kValgrindRedZoneBytes,
- &bytes_allocated, &usable_size);
+ &bytes_allocated, &usable_size, &bytes_tl_bulk_allocated);
if (obj_with_rdz == nullptr) {
return nullptr;
}
@@ -118,8 +130,10 @@
return valgrind_details::AdjustForValgrind<kValgrindRedZoneBytes,
kUseObjSizeForUsable>(obj_with_rdz, num_bytes,
bytes_allocated, usable_size,
+ bytes_tl_bulk_allocated,
bytes_allocated_out,
- usable_size_out);
+ usable_size_out,
+ bytes_tl_bulk_allocated_out);
}
template <typename S,
@@ -130,20 +144,25 @@
kValgrindRedZoneBytes,
kAdjustForRedzoneInAllocSize,
kUseObjSizeForUsable>::AllocThreadUnsafe(
- Thread* self, size_t num_bytes, size_t* bytes_allocated_out, size_t* usable_size_out) {
+ Thread* self, size_t num_bytes, size_t* bytes_allocated_out, size_t* usable_size_out,
+ size_t* bytes_tl_bulk_allocated_out) {
size_t bytes_allocated;
size_t usable_size;
+ size_t bytes_tl_bulk_allocated;
void* obj_with_rdz = S::AllocThreadUnsafe(self, num_bytes + 2 * kValgrindRedZoneBytes,
- &bytes_allocated, &usable_size);
+ &bytes_allocated, &usable_size,
+ &bytes_tl_bulk_allocated);
if (obj_with_rdz == nullptr) {
return nullptr;
}
- return valgrind_details::AdjustForValgrind<kValgrindRedZoneBytes,
- kUseObjSizeForUsable>(obj_with_rdz, num_bytes,
- bytes_allocated, usable_size,
- bytes_allocated_out,
- usable_size_out);
+ return valgrind_details::AdjustForValgrind<kValgrindRedZoneBytes, kUseObjSizeForUsable>(
+ obj_with_rdz, num_bytes,
+ bytes_allocated, usable_size,
+ bytes_tl_bulk_allocated,
+ bytes_allocated_out,
+ usable_size_out,
+ bytes_tl_bulk_allocated_out);
}
template <typename S,
@@ -226,6 +245,17 @@
mem_map->Size() - initial_size);
}
+template <typename S,
+ size_t kValgrindRedZoneBytes,
+ bool kAdjustForRedzoneInAllocSize,
+ bool kUseObjSizeForUsable>
+size_t ValgrindMallocSpace<S,
+ kValgrindRedZoneBytes,
+ kAdjustForRedzoneInAllocSize,
+ kUseObjSizeForUsable>::MaxBytesBulkAllocatedFor(size_t num_bytes) {
+ return S::MaxBytesBulkAllocatedFor(num_bytes + 2 * kValgrindRedZoneBytes);
+}
+
} // namespace space
} // namespace gc
} // namespace art
diff --git a/runtime/gc/space/valgrind_malloc_space.h b/runtime/gc/space/valgrind_malloc_space.h
index 707ea69..a6b010a 100644
--- a/runtime/gc/space/valgrind_malloc_space.h
+++ b/runtime/gc/space/valgrind_malloc_space.h
@@ -34,12 +34,13 @@
class ValgrindMallocSpace FINAL : public BaseMallocSpaceType {
public:
mirror::Object* AllocWithGrowth(Thread* self, size_t num_bytes, size_t* bytes_allocated,
- size_t* usable_size) OVERRIDE;
+ size_t* usable_size, size_t* bytes_tl_bulk_allocated)
+ OVERRIDE;
mirror::Object* Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated,
- size_t* usable_size) OVERRIDE;
+ size_t* usable_size, size_t* bytes_tl_bulk_allocated) OVERRIDE;
mirror::Object* AllocThreadUnsafe(Thread* self, size_t num_bytes, size_t* bytes_allocated,
- size_t* usable_size) OVERRIDE
- EXCLUSIVE_LOCKS_REQUIRED(Locks::mutator_lock_);
+ size_t* usable_size, size_t* bytes_tl_bulk_allocated)
+ OVERRIDE EXCLUSIVE_LOCKS_REQUIRED(Locks::mutator_lock_);
size_t AllocationSize(mirror::Object* obj, size_t* usable_size) OVERRIDE;
@@ -53,6 +54,8 @@
UNUSED(ptr);
}
+ size_t MaxBytesBulkAllocatedFor(size_t num_bytes) OVERRIDE;
+
template <typename... Params>
explicit ValgrindMallocSpace(MemMap* mem_map, size_t initial_size, Params... params);
virtual ~ValgrindMallocSpace() {}
diff --git a/runtime/gc/space/zygote_space.cc b/runtime/gc/space/zygote_space.cc
index a868e68..9e882a8 100644
--- a/runtime/gc/space/zygote_space.cc
+++ b/runtime/gc/space/zygote_space.cc
@@ -77,7 +77,7 @@
<< ",name=\"" << GetName() << "\"]";
}
-mirror::Object* ZygoteSpace::Alloc(Thread*, size_t, size_t*, size_t*) {
+mirror::Object* ZygoteSpace::Alloc(Thread*, size_t, size_t*, size_t*, size_t*) {
UNIMPLEMENTED(FATAL);
UNREACHABLE();
}
diff --git a/runtime/gc/space/zygote_space.h b/runtime/gc/space/zygote_space.h
index 0cf4bb1..934a234 100644
--- a/runtime/gc/space/zygote_space.h
+++ b/runtime/gc/space/zygote_space.h
@@ -46,7 +46,7 @@
}
mirror::Object* Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated,
- size_t* usable_size) OVERRIDE;
+ size_t* usable_size, size_t* bytes_tl_bulk_allocated) OVERRIDE;
size_t AllocationSize(mirror::Object* obj, size_t* usable_size) OVERRIDE;
@@ -55,9 +55,11 @@
size_t FreeList(Thread* self, size_t num_ptrs, mirror::Object** ptrs) OVERRIDE;
// ZygoteSpaces don't have thread local state.
- void RevokeThreadLocalBuffers(art::Thread*) OVERRIDE {
+ size_t RevokeThreadLocalBuffers(art::Thread*) OVERRIDE {
+ return 0U;
}
- void RevokeAllThreadLocalBuffers() OVERRIDE {
+ size_t RevokeAllThreadLocalBuffers() OVERRIDE {
+ return 0U;
}
uint64_t GetBytesAllocated() {