Improve the allocation speed.
- Improves the Ritz MemAllocTest benchmark result by ~500 ms (or ~5%) on Nexus 4.
- Move the memset() call that zeroes the allocated memory out of the lock region.
- De-virtualize/Inline the allocation call chains into Heap::AllocObject().
- Turn Heap::measure_allocation_time_ into a static const variable.
- Surround the VerifyObject() call with kIsDebugBuild.
Bug: 9986565
Change-Id: Ib70b6d051a80ec329788b30256565561f031da2a
diff --git a/runtime/gc/space/dlmalloc_space-inl.h b/runtime/gc/space/dlmalloc_space-inl.h
new file mode 100644
index 0000000..849157f
--- /dev/null
+++ b/runtime/gc/space/dlmalloc_space-inl.h
@@ -0,0 +1,63 @@
+/*
+ * Copyright (C) 2011 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ART_RUNTIME_GC_SPACE_DLMALLOC_SPACE_INL_H_
+#define ART_RUNTIME_GC_SPACE_DLMALLOC_SPACE_INL_H_
+
+#include "dlmalloc_space.h"
+
+namespace art {
+namespace gc {
+namespace space {
+
+inline mirror::Object* DlMallocSpace::AllocNonvirtual(Thread* self, size_t num_bytes,
+ size_t* bytes_allocated) {
+ mirror::Object* obj;
+ {
+ MutexLock mu(self, lock_);
+ obj = AllocWithoutGrowthLocked(num_bytes, bytes_allocated);
+ }
+ if (obj != NULL) {
+ // Zero freshly allocated memory, done while not holding the space's lock.
+ memset(obj, 0, num_bytes);
+ }
+ return obj;
+}
+
+inline mirror::Object* DlMallocSpace::AllocWithoutGrowthLocked(size_t num_bytes, size_t* bytes_allocated) {
+ mirror::Object* result = reinterpret_cast<mirror::Object*>(mspace_malloc(mspace_, num_bytes));
+ if (result != NULL) {
+ if (kDebugSpaces) {
+ CHECK(Contains(result)) << "Allocation (" << reinterpret_cast<void*>(result)
+ << ") not in bounds of allocation space " << *this;
+ }
+ size_t allocation_size = AllocationSizeNonvirtual(result);
+ if (bytes_allocated != NULL) {
+ *bytes_allocated = allocation_size;
+ }
+ num_bytes_allocated_ += allocation_size;
+ total_bytes_allocated_ += allocation_size;
+ ++total_objects_allocated_;
+ ++num_objects_allocated_;
+ }
+ return result;
+}
+
+} // namespace space
+} // namespace gc
+} // namespace art
+
+#endif // ART_RUNTIME_GC_SPACE_DLMALLOC_SPACE_INL_H_
diff --git a/runtime/gc/space/dlmalloc_space.cc b/runtime/gc/space/dlmalloc_space.cc
index d539aa2..8b99e96 100644
--- a/runtime/gc/space/dlmalloc_space.cc
+++ b/runtime/gc/space/dlmalloc_space.cc
@@ -14,6 +14,7 @@
* limitations under the License.
*/
#include "dlmalloc_space.h"
+#include "dlmalloc_space-inl.h"
#include "gc/accounting/card_table.h"
#include "gc/heap.h"
#include "runtime.h"
@@ -46,8 +47,9 @@
// A specialization of DlMallocSpace that provides information to valgrind wrt allocations.
class ValgrindDlMallocSpace : public DlMallocSpace {
public:
- virtual mirror::Object* AllocWithGrowth(Thread* self, size_t num_bytes) {
- void* obj_with_rdz = DlMallocSpace::AllocWithGrowth(self, num_bytes + 2 * kValgrindRedZoneBytes);
+ virtual mirror::Object* AllocWithGrowth(Thread* self, size_t num_bytes, size_t* bytes_allocated) {
+ void* obj_with_rdz = DlMallocSpace::AllocWithGrowth(self, num_bytes + 2 * kValgrindRedZoneBytes,
+ bytes_allocated);
if (obj_with_rdz == NULL) {
return NULL;
}
@@ -59,8 +61,9 @@
return result;
}
- virtual mirror::Object* Alloc(Thread* self, size_t num_bytes) {
- void* obj_with_rdz = DlMallocSpace::Alloc(self, num_bytes + 2 * kValgrindRedZoneBytes);
+ virtual mirror::Object* Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated) {
+ void* obj_with_rdz = DlMallocSpace::Alloc(self, num_bytes + 2 * kValgrindRedZoneBytes,
+ bytes_allocated);
if (obj_with_rdz == NULL) {
return NULL;
}
@@ -234,37 +237,27 @@
mark_bitmap_->SetName(temp_name);
}
-mirror::Object* DlMallocSpace::AllocWithoutGrowthLocked(size_t num_bytes) {
- mirror::Object* result = reinterpret_cast<mirror::Object*>(mspace_calloc(mspace_, 1, num_bytes));
- if (result != NULL) {
- if (kDebugSpaces) {
- CHECK(Contains(result)) << "Allocation (" << reinterpret_cast<void*>(result)
- << ") not in bounds of allocation space " << *this;
- }
- size_t allocation_size = InternalAllocationSize(result);
- num_bytes_allocated_ += allocation_size;
- total_bytes_allocated_ += allocation_size;
- ++total_objects_allocated_;
- ++num_objects_allocated_;
+mirror::Object* DlMallocSpace::Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated) {
+ return AllocNonvirtual(self, num_bytes, bytes_allocated);
+}
+
+mirror::Object* DlMallocSpace::AllocWithGrowth(Thread* self, size_t num_bytes, size_t* bytes_allocated) {
+ mirror::Object* result;
+ {
+ MutexLock mu(self, lock_);
+ // Grow as much as possible within the mspace.
+ size_t max_allowed = Capacity();
+ mspace_set_footprint_limit(mspace_, max_allowed);
+ // Try the allocation.
+ result = AllocWithoutGrowthLocked(num_bytes, bytes_allocated);
+ // Shrink back down as small as possible.
+ size_t footprint = mspace_footprint(mspace_);
+ mspace_set_footprint_limit(mspace_, footprint);
}
- return result;
-}
-
-mirror::Object* DlMallocSpace::Alloc(Thread* self, size_t num_bytes) {
- MutexLock mu(self, lock_);
- return AllocWithoutGrowthLocked(num_bytes);
-}
-
-mirror::Object* DlMallocSpace::AllocWithGrowth(Thread* self, size_t num_bytes) {
- MutexLock mu(self, lock_);
- // Grow as much as possible within the mspace.
- size_t max_allowed = Capacity();
- mspace_set_footprint_limit(mspace_, max_allowed);
- // Try the allocation.
- mirror::Object* result = AllocWithoutGrowthLocked(num_bytes);
- // Shrink back down as small as possible.
- size_t footprint = mspace_footprint(mspace_);
- mspace_set_footprint_limit(mspace_, footprint);
+ if (result != NULL) {
+ // Zero freshly allocated memory, done while not holding the space's lock.
+ memset(result, 0, num_bytes);
+ }
// Return the new allocation or NULL.
CHECK(!kDebugSpaces || result == NULL || Contains(result));
return result;
@@ -415,8 +408,7 @@
// Virtual functions can't get inlined.
inline size_t DlMallocSpace::InternalAllocationSize(const mirror::Object* obj) {
- return mspace_usable_size(const_cast<void*>(reinterpret_cast<const void*>(obj))) +
- kChunkOverhead;
+ return AllocationSizeNonvirtual(obj);
}
size_t DlMallocSpace::AllocationSize(const mirror::Object* obj) {
diff --git a/runtime/gc/space/dlmalloc_space.h b/runtime/gc/space/dlmalloc_space.h
index c15d0ba..c498ef8 100644
--- a/runtime/gc/space/dlmalloc_space.h
+++ b/runtime/gc/space/dlmalloc_space.h
@@ -50,16 +50,24 @@
size_t capacity, byte* requested_begin);
// Allocate num_bytes without allowing the underlying mspace to grow.
- virtual mirror::Object* AllocWithGrowth(Thread* self, size_t num_bytes);
+ virtual mirror::Object* AllocWithGrowth(Thread* self, size_t num_bytes,
+ size_t* bytes_allocated) LOCKS_EXCLUDED(lock_);
// Allocate num_bytes allowing the underlying mspace to grow.
- virtual mirror::Object* Alloc(Thread* self, size_t num_bytes);
+ virtual mirror::Object* Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated);
// Return the storage space required by obj.
virtual size_t AllocationSize(const mirror::Object* obj);
virtual size_t Free(Thread* self, mirror::Object* ptr);
virtual size_t FreeList(Thread* self, size_t num_ptrs, mirror::Object** ptrs);
+ mirror::Object* AllocNonvirtual(Thread* self, size_t num_bytes, size_t* bytes_allocated);
+
+ size_t AllocationSizeNonvirtual(const mirror::Object* obj) {
+ return mspace_usable_size(const_cast<void*>(reinterpret_cast<const void*>(obj))) +
+ kChunkOverhead;
+ }
+
void* MoreCore(intptr_t increment);
void* GetMspace() const {
@@ -141,7 +149,8 @@
private:
size_t InternalAllocationSize(const mirror::Object* obj);
- mirror::Object* AllocWithoutGrowthLocked(size_t num_bytes) EXCLUSIVE_LOCKS_REQUIRED(lock_);
+ mirror::Object* AllocWithoutGrowthLocked(size_t num_bytes, size_t* bytes_allocated)
+ EXCLUSIVE_LOCKS_REQUIRED(lock_);
bool Init(size_t initial_size, size_t maximum_size, size_t growth_size, byte* requested_base);
diff --git a/runtime/gc/space/large_object_space.cc b/runtime/gc/space/large_object_space.cc
index d7db561..832c6fa 100644
--- a/runtime/gc/space/large_object_space.cc
+++ b/runtime/gc/space/large_object_space.cc
@@ -55,7 +55,7 @@
return new LargeObjectMapSpace(name);
}
-mirror::Object* LargeObjectMapSpace::Alloc(Thread* self, size_t num_bytes) {
+mirror::Object* LargeObjectMapSpace::Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated) {
MemMap* mem_map = MemMap::MapAnonymous("large object space allocation", NULL, num_bytes,
PROT_READ | PROT_WRITE);
if (mem_map == NULL) {
@@ -66,6 +66,9 @@
large_objects_.push_back(obj);
mem_maps_.Put(obj, mem_map);
size_t allocation_size = mem_map->Size();
+ if (bytes_allocated != NULL) {
+ *bytes_allocated = allocation_size;
+ }
num_bytes_allocated_ += allocation_size;
total_bytes_allocated_ += allocation_size;
++num_objects_allocated_;
@@ -239,7 +242,7 @@
return chunk->GetSize();
}
-mirror::Object* FreeListSpace::Alloc(Thread* self, size_t num_bytes) {
+mirror::Object* FreeListSpace::Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated) {
MutexLock mu(self, lock_);
num_bytes = RoundUp(num_bytes, kAlignment);
Chunk temp;
@@ -262,6 +265,9 @@
AddFreeChunk(AddrFromChunk(new_chunk), chunk_size - num_bytes, chunk);
}
+ if (bytes_allocated != NULL) {
+ *bytes_allocated = num_bytes;
+ }
num_objects_allocated_++;
total_objects_allocated_++;
num_bytes_allocated_ += num_bytes;
diff --git a/runtime/gc/space/large_object_space.h b/runtime/gc/space/large_object_space.h
index 8cd5088..31659db 100644
--- a/runtime/gc/space/large_object_space.h
+++ b/runtime/gc/space/large_object_space.h
@@ -83,7 +83,7 @@
// Return the storage space required by obj.
size_t AllocationSize(const mirror::Object* obj);
- mirror::Object* Alloc(Thread* self, size_t num_bytes);
+ mirror::Object* Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated);
size_t Free(Thread* self, mirror::Object* ptr);
void Walk(DlMallocSpace::WalkCallback, void* arg);
// TODO: disabling thread safety analysis as this may be called when we already hold lock_.
@@ -110,7 +110,7 @@
static FreeListSpace* Create(const std::string& name, byte* requested_begin, size_t capacity);
size_t AllocationSize(const mirror::Object* obj) EXCLUSIVE_LOCKS_REQUIRED(lock_);
- mirror::Object* Alloc(Thread* self, size_t num_bytes);
+ mirror::Object* Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated);
size_t Free(Thread* self, mirror::Object* obj);
bool Contains(const mirror::Object* obj) const;
void Walk(DlMallocSpace::WalkCallback callback, void* arg);
diff --git a/runtime/gc/space/space.h b/runtime/gc/space/space.h
index bc6e818..231cabc 100644
--- a/runtime/gc/space/space.h
+++ b/runtime/gc/space/space.h
@@ -154,8 +154,10 @@
// Number of objects allocated since the space was created.
virtual uint64_t GetTotalObjectsAllocated() const = 0;
- // Allocate num_bytes without allowing growth.
- virtual mirror::Object* Alloc(Thread* self, size_t num_bytes) = 0;
+ // Allocate num_bytes without allowing growth. If the allocation
+ // succeeds, the output parameter bytes_allocated will be set to the
+ // actually allocated bytes which is >= num_bytes.
+ virtual mirror::Object* Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated) = 0;
// Return the storage space required by obj.
virtual size_t AllocationSize(const mirror::Object* obj) = 0;
diff --git a/runtime/gc/space/space_test.cc b/runtime/gc/space/space_test.cc
index 3003140..66b8c11 100644
--- a/runtime/gc/space/space_test.cc
+++ b/runtime/gc/space/space_test.cc
@@ -88,32 +88,35 @@
Thread* self = Thread::Current();
// Succeeds, fits without adjusting the footprint limit.
- mirror::Object* ptr1 = space->Alloc(self, 1 * MB);
+ mirror::Object* ptr1 = space->Alloc(self, 1 * MB, NULL);
EXPECT_TRUE(ptr1 != NULL);
// Fails, requires a higher footprint limit.
- mirror::Object* ptr2 = space->Alloc(self, 8 * MB);
+ mirror::Object* ptr2 = space->Alloc(self, 8 * MB, NULL);
EXPECT_TRUE(ptr2 == NULL);
// Succeeds, adjusts the footprint.
- mirror::Object* ptr3 = space->AllocWithGrowth(self, 8 * MB);
+ size_t ptr3_bytes_allocated;
+ mirror::Object* ptr3 = space->AllocWithGrowth(self, 8 * MB, &ptr3_bytes_allocated);
EXPECT_TRUE(ptr3 != NULL);
+ EXPECT_LE(8U * MB, ptr3_bytes_allocated);
// Fails, requires a higher footprint limit.
- mirror::Object* ptr4 = space->Alloc(self, 8 * MB);
+ mirror::Object* ptr4 = space->Alloc(self, 8 * MB, NULL);
EXPECT_TRUE(ptr4 == NULL);
// Also fails, requires a higher allowed footprint.
- mirror::Object* ptr5 = space->AllocWithGrowth(self, 8 * MB);
+ mirror::Object* ptr5 = space->AllocWithGrowth(self, 8 * MB, NULL);
EXPECT_TRUE(ptr5 == NULL);
// Release some memory.
size_t free3 = space->AllocationSize(ptr3);
+ EXPECT_EQ(free3, ptr3_bytes_allocated);
EXPECT_EQ(free3, space->Free(self, ptr3));
EXPECT_LE(8U * MB, free3);
// Succeeds, now that memory has been freed.
- void* ptr6 = space->AllocWithGrowth(self, 9 * MB);
+ void* ptr6 = space->AllocWithGrowth(self, 9 * MB, NULL);
EXPECT_TRUE(ptr6 != NULL);
// Final clean up.
@@ -122,22 +125,22 @@
EXPECT_LE(1U * MB, free1);
// Make sure that the zygote space isn't directly at the start of the space.
- space->Alloc(self, 1U * MB);
+ space->Alloc(self, 1U * MB, NULL);
space = space->CreateZygoteSpace("alloc space");
// Make space findable to the heap, will also delete space when runtime is cleaned up
AddContinuousSpace(space);
// Succeeds, fits without adjusting the footprint limit.
- ptr1 = space->Alloc(self, 1 * MB);
+ ptr1 = space->Alloc(self, 1 * MB, NULL);
EXPECT_TRUE(ptr1 != NULL);
// Fails, requires a higher footprint limit.
- ptr2 = space->Alloc(self, 8 * MB);
+ ptr2 = space->Alloc(self, 8 * MB, NULL);
EXPECT_TRUE(ptr2 == NULL);
// Succeeds, adjusts the footprint.
- ptr3 = space->AllocWithGrowth(self, 2 * MB);
+ ptr3 = space->AllocWithGrowth(self, 2 * MB, NULL);
EXPECT_TRUE(ptr3 != NULL);
space->Free(self, ptr3);
@@ -156,32 +159,35 @@
AddContinuousSpace(space);
// Succeeds, fits without adjusting the footprint limit.
- mirror::Object* ptr1 = space->Alloc(self, 1 * MB);
+ mirror::Object* ptr1 = space->Alloc(self, 1 * MB, NULL);
EXPECT_TRUE(ptr1 != NULL);
// Fails, requires a higher footprint limit.
- mirror::Object* ptr2 = space->Alloc(self, 8 * MB);
+ mirror::Object* ptr2 = space->Alloc(self, 8 * MB, NULL);
EXPECT_TRUE(ptr2 == NULL);
// Succeeds, adjusts the footprint.
- mirror::Object* ptr3 = space->AllocWithGrowth(self, 8 * MB);
+ size_t ptr3_bytes_allocated;
+ mirror::Object* ptr3 = space->AllocWithGrowth(self, 8 * MB, &ptr3_bytes_allocated);
EXPECT_TRUE(ptr3 != NULL);
+ EXPECT_LE(8U * MB, ptr3_bytes_allocated);
// Fails, requires a higher footprint limit.
- mirror::Object* ptr4 = space->Alloc(self, 8 * MB);
+ mirror::Object* ptr4 = space->Alloc(self, 8 * MB, NULL);
EXPECT_TRUE(ptr4 == NULL);
// Also fails, requires a higher allowed footprint.
- mirror::Object* ptr5 = space->AllocWithGrowth(self, 8 * MB);
+ mirror::Object* ptr5 = space->AllocWithGrowth(self, 8 * MB, NULL);
EXPECT_TRUE(ptr5 == NULL);
// Release some memory.
size_t free3 = space->AllocationSize(ptr3);
+ EXPECT_EQ(free3, ptr3_bytes_allocated);
space->Free(self, ptr3);
EXPECT_LE(8U * MB, free3);
// Succeeds, now that memory has been freed.
- void* ptr6 = space->AllocWithGrowth(self, 9 * MB);
+ void* ptr6 = space->AllocWithGrowth(self, 9 * MB, NULL);
EXPECT_TRUE(ptr6 != NULL);
// Final clean up.
@@ -201,7 +207,7 @@
// Succeeds, fits without adjusting the max allowed footprint.
mirror::Object* lots_of_objects[1024];
for (size_t i = 0; i < arraysize(lots_of_objects); i++) {
- lots_of_objects[i] = space->Alloc(self, 16);
+ lots_of_objects[i] = space->Alloc(self, 16, NULL);
EXPECT_TRUE(lots_of_objects[i] != NULL);
}
@@ -213,7 +219,7 @@
// Succeeds, fits by adjusting the max allowed footprint.
for (size_t i = 0; i < arraysize(lots_of_objects); i++) {
- lots_of_objects[i] = space->AllocWithGrowth(self, 1024);
+ lots_of_objects[i] = space->AllocWithGrowth(self, 1024, NULL);
EXPECT_TRUE(lots_of_objects[i] != NULL);
}
@@ -276,9 +282,9 @@
}
mirror::Object* object;
if (round <= 1) {
- object = space->Alloc(self, alloc_size);
+ object = space->Alloc(self, alloc_size, NULL);
} else {
- object = space->AllocWithGrowth(self, alloc_size);
+ object = space->AllocWithGrowth(self, alloc_size, NULL);
}
footprint = mspace_footprint(mspace);
EXPECT_GE(space->Size(), footprint); // invariant
@@ -355,9 +361,9 @@
mirror::Object* large_object;
size_t three_quarters_space = (growth_limit / 2) + (growth_limit / 4);
if (round <= 1) {
- large_object = space->Alloc(self, three_quarters_space);
+ large_object = space->Alloc(self, three_quarters_space, NULL);
} else {
- large_object = space->AllocWithGrowth(self, three_quarters_space);
+ large_object = space->AllocWithGrowth(self, three_quarters_space, NULL);
}
EXPECT_TRUE(large_object != NULL);