A custom 'runs-of-slots' memory allocator.
Bug: 9986565
Change-Id: I0eb73b9458752113f519483616536d219d5f798b
diff --git a/runtime/gc/space/dlmalloc_space-inl.h b/runtime/gc/space/dlmalloc_space-inl.h
index fb2c66b..fbbba1f 100644
--- a/runtime/gc/space/dlmalloc_space-inl.h
+++ b/runtime/gc/space/dlmalloc_space-inl.h
@@ -18,6 +18,7 @@
#define ART_RUNTIME_GC_SPACE_DLMALLOC_SPACE_INL_H_
#include "dlmalloc_space.h"
+#include "thread.h"
namespace art {
namespace gc {
@@ -28,7 +29,7 @@
mirror::Object* obj;
{
MutexLock mu(self, lock_);
- obj = AllocWithoutGrowthLocked(num_bytes, bytes_allocated);
+ obj = AllocWithoutGrowthLocked(self, num_bytes, bytes_allocated);
}
if (LIKELY(obj != NULL)) {
// Zero freshly allocated memory, done while not holding the space's lock.
@@ -37,7 +38,8 @@
return obj;
}
-inline mirror::Object* DlMallocSpace::AllocWithoutGrowthLocked(size_t num_bytes, size_t* bytes_allocated) {
+inline mirror::Object* DlMallocSpace::AllocWithoutGrowthLocked(Thread* /*self*/, size_t num_bytes,
+ size_t* bytes_allocated) {
mirror::Object* result = reinterpret_cast<mirror::Object*>(mspace_malloc(mspace_, num_bytes));
if (LIKELY(result != NULL)) {
if (kDebugSpaces) {
diff --git a/runtime/gc/space/dlmalloc_space.cc b/runtime/gc/space/dlmalloc_space.cc
index 1c7aa22..fcac588 100644
--- a/runtime/gc/space/dlmalloc_space.cc
+++ b/runtime/gc/space/dlmalloc_space.cc
@@ -13,13 +13,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
#include "dlmalloc_space.h"
+
#include "dlmalloc_space-inl.h"
#include "gc/accounting/card_table.h"
#include "gc/heap.h"
+#include "mirror/class-inl.h"
#include "mirror/object-inl.h"
#include "runtime.h"
#include "thread.h"
+#include "thread_list.h"
#include "utils.h"
#include <valgrind.h>
@@ -29,16 +33,6 @@
namespace gc {
namespace space {
-// TODO: Remove define macro
-#define CHECK_MEMORY_CALL(call, args, what) \
- do { \
- int rc = call args; \
- if (UNLIKELY(rc != 0)) { \
- errno = rc; \
- PLOG(FATAL) << # call << " failed for " << what; \
- } \
- } while (false)
-
static const bool kPrefetchDuringDlMallocFreeList = true;
// Number of bytes to use as a red zone (rdz). A red zone of this size will be placed before and
@@ -114,81 +108,38 @@
DISALLOW_COPY_AND_ASSIGN(ValgrindDlMallocSpace);
};
-size_t DlMallocSpace::bitmap_index_ = 0;
-
DlMallocSpace::DlMallocSpace(const std::string& name, MemMap* mem_map, void* mspace, byte* begin,
- byte* end, byte* limit, size_t growth_limit)
- : ContinuousMemMapAllocSpace(name, mem_map, begin, end, limit, kGcRetentionPolicyAlwaysCollect),
- recent_free_pos_(0), total_bytes_freed_(0), total_objects_freed_(0),
- lock_("allocation space lock", kAllocSpaceLock), mspace_(mspace),
- growth_limit_(growth_limit) {
+ byte* end, byte* limit, size_t growth_limit)
+ : MallocSpace(name, mem_map, begin, end, limit, growth_limit),
+ total_bytes_freed_(0), total_objects_freed_(0), mspace_(mspace) {
CHECK(mspace != NULL);
- size_t bitmap_index = bitmap_index_++;
- static const uintptr_t kGcCardSize = static_cast<uintptr_t>(accounting::CardTable::kCardSize);
- CHECK(IsAligned<kGcCardSize>(reinterpret_cast<uintptr_t>(mem_map->Begin())));
- CHECK(IsAligned<kGcCardSize>(reinterpret_cast<uintptr_t>(mem_map->End())));
- live_bitmap_.reset(accounting::SpaceBitmap::Create(
- StringPrintf("allocspace %s live-bitmap %d", name.c_str(), static_cast<int>(bitmap_index)),
- Begin(), Capacity()));
- DCHECK(live_bitmap_.get() != NULL) << "could not create allocspace live bitmap #" << bitmap_index;
- mark_bitmap_.reset(accounting::SpaceBitmap::Create(
- StringPrintf("allocspace %s mark-bitmap %d", name.c_str(), static_cast<int>(bitmap_index)),
- Begin(), Capacity()));
- DCHECK(live_bitmap_.get() != NULL) << "could not create allocspace mark bitmap #" << bitmap_index;
- for (auto& freed : recent_freed_objects_) {
- freed.first = nullptr;
- freed.second = nullptr;
- }
}
-DlMallocSpace* DlMallocSpace::Create(const std::string& name, size_t initial_size, size_t
- growth_limit, size_t capacity, byte* requested_begin) {
- // Memory we promise to dlmalloc before it asks for morecore.
- // Note: making this value large means that large allocations are unlikely to succeed as dlmalloc
- // will ask for this memory from sys_alloc which will fail as the footprint (this value plus the
- // size of the large allocation) will be greater than the footprint limit.
- size_t starting_size = kPageSize;
+DlMallocSpace* DlMallocSpace::Create(const std::string& name, size_t initial_size, size_t growth_limit,
+ size_t capacity, byte* requested_begin) {
uint64_t start_time = 0;
if (VLOG_IS_ON(heap) || VLOG_IS_ON(startup)) {
start_time = NanoTime();
- VLOG(startup) << "Space::CreateAllocSpace entering " << name
+ VLOG(startup) << "DlMallocSpace::Create entering " << name
<< " initial_size=" << PrettySize(initial_size)
<< " growth_limit=" << PrettySize(growth_limit)
<< " capacity=" << PrettySize(capacity)
<< " requested_begin=" << reinterpret_cast<void*>(requested_begin);
}
- // Sanity check arguments
- if (starting_size > initial_size) {
- initial_size = starting_size;
- }
- if (initial_size > growth_limit) {
- LOG(ERROR) << "Failed to create alloc space (" << name << ") where the initial size ("
- << PrettySize(initial_size) << ") is larger than its capacity ("
- << PrettySize(growth_limit) << ")";
+ // Memory we promise to dlmalloc before it asks for morecore.
+ // Note: making this value large means that large allocations are unlikely to succeed as dlmalloc
+ // will ask for this memory from sys_alloc which will fail as the footprint (this value plus the
+ // size of the large allocation) will be greater than the footprint limit.
+ size_t starting_size = kPageSize;
+ MemMap* mem_map = CreateMemMap(name, starting_size, &initial_size, &growth_limit, &capacity,
+ requested_begin);
+ if (mem_map == NULL) {
+ LOG(ERROR) << "Failed to create mem map for alloc space (" << name << ") of size "
+ << PrettySize(capacity);
return NULL;
}
- if (growth_limit > capacity) {
- LOG(ERROR) << "Failed to create alloc space (" << name << ") where the growth limit capacity ("
- << PrettySize(growth_limit) << ") is larger than the capacity ("
- << PrettySize(capacity) << ")";
- return NULL;
- }
-
- // Page align growth limit and capacity which will be used to manage mmapped storage
- growth_limit = RoundUp(growth_limit, kPageSize);
- capacity = RoundUp(capacity, kPageSize);
-
- std::string error_msg;
- UniquePtr<MemMap> mem_map(MemMap::MapAnonymous(name.c_str(), requested_begin, capacity,
- PROT_READ | PROT_WRITE, &error_msg));
- if (mem_map.get() == NULL) {
- LOG(ERROR) << "Failed to allocate pages for alloc space (" << name << ") of size "
- << PrettySize(capacity) << ": " << error_msg;
- return NULL;
- }
-
- void* mspace = CreateMallocSpace(mem_map->Begin(), starting_size, initial_size);
+ void* mspace = CreateMspace(mem_map->Begin(), starting_size, initial_size);
if (mspace == NULL) {
LOG(ERROR) << "Failed to initialize mspace for alloc space (" << name << ")";
return NULL;
@@ -201,24 +152,23 @@
}
// Everything is set so record in immutable structure and leave
- MemMap* mem_map_ptr = mem_map.release();
DlMallocSpace* space;
- byte* begin = mem_map_ptr->Begin();
+ byte* begin = mem_map->Begin();
if (RUNNING_ON_VALGRIND > 0) {
- space = new ValgrindDlMallocSpace(name, mem_map_ptr, mspace, begin, end, begin + capacity,
+ space = new ValgrindDlMallocSpace(name, mem_map, mspace, begin, end, begin + capacity,
growth_limit, initial_size);
} else {
- space = new DlMallocSpace(name, mem_map_ptr, mspace, begin, end, begin + capacity, growth_limit);
+ space = new DlMallocSpace(name, mem_map, mspace, begin, end, begin + capacity, growth_limit);
}
// We start out with only the initial size possibly containing objects.
if (VLOG_IS_ON(heap) || VLOG_IS_ON(startup)) {
- LOG(INFO) << "Space::CreateAllocSpace exiting (" << PrettyDuration(NanoTime() - start_time)
+ LOG(INFO) << "DlMallocSpace::Create exiting (" << PrettyDuration(NanoTime() - start_time)
<< " ) " << *space;
}
return space;
}
-void* DlMallocSpace::CreateMallocSpace(void* begin, size_t morecore_start, size_t initial_size) {
+void* DlMallocSpace::CreateMspace(void* begin, size_t morecore_start, size_t initial_size) {
// clear errno to allow PLOG on error
errno = 0;
// create mspace using our backing storage starting at begin and with a footprint of
@@ -234,14 +184,6 @@
return msp;
}
-void DlMallocSpace::SwapBitmaps() {
- live_bitmap_.swap(mark_bitmap_);
- // Swap names to get more descriptive diagnostics.
- std::string temp_name(live_bitmap_->GetName());
- live_bitmap_->SetName(mark_bitmap_->GetName());
- mark_bitmap_->SetName(temp_name);
-}
-
mirror::Object* DlMallocSpace::Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated) {
return AllocNonvirtual(self, num_bytes, bytes_allocated);
}
@@ -250,11 +192,11 @@
mirror::Object* result;
{
MutexLock mu(self, lock_);
- // Grow as much as possible within the mspace.
+ // Grow as much as possible within the space.
size_t max_allowed = Capacity();
mspace_set_footprint_limit(mspace_, max_allowed);
// Try the allocation.
- result = AllocWithoutGrowthLocked(num_bytes, bytes_allocated);
+ result = AllocWithoutGrowthLocked(self, num_bytes, bytes_allocated);
// Shrink back down as small as possible.
size_t footprint = mspace_footprint(mspace_);
mspace_set_footprint_limit(mspace_, footprint);
@@ -268,83 +210,9 @@
return result;
}
-void DlMallocSpace::SetGrowthLimit(size_t growth_limit) {
- growth_limit = RoundUp(growth_limit, kPageSize);
- growth_limit_ = growth_limit;
- if (Size() > growth_limit_) {
- end_ = begin_ + growth_limit;
- }
-}
-
-DlMallocSpace* DlMallocSpace::CreateZygoteSpace(const char* alloc_space_name) {
- end_ = reinterpret_cast<byte*>(RoundUp(reinterpret_cast<uintptr_t>(end_), kPageSize));
- DCHECK(IsAligned<accounting::CardTable::kCardSize>(begin_));
- DCHECK(IsAligned<accounting::CardTable::kCardSize>(end_));
- DCHECK(IsAligned<kPageSize>(begin_));
- DCHECK(IsAligned<kPageSize>(end_));
- size_t size = RoundUp(Size(), kPageSize);
- // Trim the heap so that we minimize the size of the Zygote space.
- Trim();
- // TODO: Not hardcode these in?
- const size_t starting_size = kPageSize;
- const size_t initial_size = 2 * MB;
- // Remaining size is for the new alloc space.
- const size_t growth_limit = growth_limit_ - size;
- const size_t capacity = Capacity() - size;
- VLOG(heap) << "Begin " << reinterpret_cast<const void*>(begin_) << "\n"
- << "End " << reinterpret_cast<const void*>(end_) << "\n"
- << "Size " << size << "\n"
- << "GrowthLimit " << growth_limit_ << "\n"
- << "Capacity " << Capacity();
- SetGrowthLimit(RoundUp(size, kPageSize));
- SetFootprintLimit(RoundUp(size, kPageSize));
- // FIXME: Do we need reference counted pointers here?
- // Make the two spaces share the same mark bitmaps since the bitmaps span both of the spaces.
- VLOG(heap) << "Creating new AllocSpace: ";
- VLOG(heap) << "Size " << GetMemMap()->Size();
- VLOG(heap) << "GrowthLimit " << PrettySize(growth_limit);
- VLOG(heap) << "Capacity " << PrettySize(capacity);
- // Remap the tail.
- std::string error_msg;
- UniquePtr<MemMap> mem_map(GetMemMap()->RemapAtEnd(end_, alloc_space_name,
- PROT_READ | PROT_WRITE, &error_msg));
- CHECK(mem_map.get() != nullptr) << error_msg;
- void* mspace = CreateMallocSpace(end_, starting_size, initial_size);
- // Protect memory beyond the initial size.
- byte* end = mem_map->Begin() + starting_size;
- if (capacity - initial_size > 0) {
- CHECK_MEMORY_CALL(mprotect, (end, capacity - initial_size, PROT_NONE), alloc_space_name);
- }
- DlMallocSpace* alloc_space =
- new DlMallocSpace(alloc_space_name, mem_map.release(), mspace, end_, end, limit_,
- growth_limit);
- SetLimit(End());
- live_bitmap_->SetHeapLimit(reinterpret_cast<uintptr_t>(End()));
- CHECK_EQ(live_bitmap_->HeapLimit(), reinterpret_cast<uintptr_t>(End()));
- mark_bitmap_->SetHeapLimit(reinterpret_cast<uintptr_t>(End()));
- CHECK_EQ(mark_bitmap_->HeapLimit(), reinterpret_cast<uintptr_t>(End()));
- VLOG(heap) << "zygote space creation done";
- return alloc_space;
-}
-
-mirror::Class* DlMallocSpace::FindRecentFreedObject(const mirror::Object* obj) {
- size_t pos = recent_free_pos_;
- // Start at the most recently freed object and work our way back since there may be duplicates
- // caused by dlmalloc reusing memory.
- if (kRecentFreeCount > 0) {
- for (size_t i = 0; i + 1 < kRecentFreeCount + 1; ++i) {
- pos = pos != 0 ? pos - 1 : kRecentFreeMask;
- if (recent_freed_objects_[pos].first == obj) {
- return recent_freed_objects_[pos].second;
- }
- }
- }
- return nullptr;
-}
-
-void DlMallocSpace::RegisterRecentFree(mirror::Object* ptr) {
- recent_freed_objects_[recent_free_pos_] = std::make_pair(ptr, ptr->GetClass());
- recent_free_pos_ = (recent_free_pos_ + 1) & kRecentFreeMask;
+MallocSpace* DlMallocSpace::CreateInstance(const std::string& name, MemMap* mem_map, void* allocator, byte* begin, byte* end,
+ byte* limit, size_t growth_limit) {
+ return new DlMallocSpace(name, mem_map, allocator, begin, end, limit, growth_limit);
}
size_t DlMallocSpace::Free(Thread* self, mirror::Object* ptr) {
@@ -411,40 +279,11 @@
// Callback from dlmalloc when it needs to increase the footprint
extern "C" void* art_heap_morecore(void* mspace, intptr_t increment) {
Heap* heap = Runtime::Current()->GetHeap();
- DCHECK_EQ(heap->GetNonMovingSpace()->GetMspace(), mspace);
+ DCHECK(heap->GetNonMovingSpace()->IsDlMallocSpace());
+ DCHECK_EQ(heap->GetNonMovingSpace()->AsDlMallocSpace()->GetMspace(), mspace);
return heap->GetNonMovingSpace()->MoreCore(increment);
}
-void* DlMallocSpace::MoreCore(intptr_t increment) {
- lock_.AssertHeld(Thread::Current());
- byte* original_end = end_;
- if (increment != 0) {
- VLOG(heap) << "DlMallocSpace::MoreCore " << PrettySize(increment);
- byte* new_end = original_end + increment;
- if (increment > 0) {
- // Should never be asked to increase the allocation beyond the capacity of the space. Enforced
- // by mspace_set_footprint_limit.
- CHECK_LE(new_end, Begin() + Capacity());
- CHECK_MEMORY_CALL(mprotect, (original_end, increment, PROT_READ | PROT_WRITE), GetName());
- } else {
- // Should never be asked for negative footprint (ie before begin)
- CHECK_GT(original_end + increment, Begin());
- // Advise we don't need the pages and protect them
- // TODO: by removing permissions to the pages we may be causing TLB shoot-down which can be
- // expensive (note the same isn't true for giving permissions to a page as the protected
- // page shouldn't be in a TLB). We should investigate performance impact of just
- // removing ignoring the memory protection change here and in Space::CreateAllocSpace. It's
- // likely just a useful debug feature.
- size_t size = -increment;
- CHECK_MEMORY_CALL(madvise, (new_end, size, MADV_DONTNEED), GetName());
- CHECK_MEMORY_CALL(mprotect, (new_end, size, PROT_NONE), GetName());
- }
- // Update end_
- end_ = new_end;
- }
- return original_end;
-}
-
// Virtual functions can't get inlined.
inline size_t DlMallocSpace::InternalAllocationSize(const mirror::Object* obj) {
return AllocationSizeNonvirtual(obj);
@@ -481,32 +320,9 @@
return mspace_footprint_limit(mspace_);
}
-// Returns the old mark bitmap.
-accounting::SpaceBitmap* DlMallocSpace::BindLiveToMarkBitmap() {
- accounting::SpaceBitmap* live_bitmap = GetLiveBitmap();
- accounting::SpaceBitmap* mark_bitmap = mark_bitmap_.release();
- temp_bitmap_.reset(mark_bitmap);
- mark_bitmap_.reset(live_bitmap);
- return mark_bitmap;
-}
-
-bool DlMallocSpace::HasBoundBitmaps() const {
- return temp_bitmap_.get() != nullptr;
-}
-
-void DlMallocSpace::UnBindBitmaps() {
- CHECK(HasBoundBitmaps());
- // At this point, the temp_bitmap holds our old mark bitmap.
- accounting::SpaceBitmap* new_bitmap = temp_bitmap_.release();
- CHECK_EQ(mark_bitmap_.release(), live_bitmap_.get());
- mark_bitmap_.reset(new_bitmap);
- DCHECK(temp_bitmap_.get() == NULL);
-}
-
-
void DlMallocSpace::SetFootprintLimit(size_t new_size) {
MutexLock mu(Thread::Current(), lock_);
- VLOG(heap) << "DLMallocSpace::SetFootprintLimit " << PrettySize(new_size);
+ VLOG(heap) << "DlMallocSpace::SetFootprintLimit " << PrettySize(new_size);
// Compare against the actual footprint, rather than the Size(), because the heap may not have
// grown all the way to the allowed size yet.
size_t current_space_size = mspace_footprint(mspace_);
@@ -517,14 +333,6 @@
mspace_set_footprint_limit(mspace_, new_size);
}
-void DlMallocSpace::Dump(std::ostream& os) const {
- os << GetType()
- << " begin=" << reinterpret_cast<void*>(Begin())
- << ",end=" << reinterpret_cast<void*>(End())
- << ",size=" << PrettySize(Size()) << ",capacity=" << PrettySize(Capacity())
- << ",name=\"" << GetName() << "\"]";
-}
-
uint64_t DlMallocSpace::GetBytesAllocated() {
if (mspace_ != nullptr) {
MutexLock mu(Thread::Current(), lock_);
@@ -547,6 +355,12 @@
}
}
+#ifndef NDEBUG
+void DlMallocSpace::CheckMoreCoreForPrecondition() {
+ lock_.AssertHeld(Thread::Current());
+}
+#endif
+
} // namespace space
} // namespace gc
} // namespace art
diff --git a/runtime/gc/space/dlmalloc_space.h b/runtime/gc/space/dlmalloc_space.h
index 59dafe3..63b1c21 100644
--- a/runtime/gc/space/dlmalloc_space.h
+++ b/runtime/gc/space/dlmalloc_space.h
@@ -18,6 +18,7 @@
#define ART_RUNTIME_GC_SPACE_DLMALLOC_SPACE_H_
#include "gc/allocator/dlmalloc.h"
+#include "malloc_space.h"
#include "space.h"
namespace art {
@@ -30,33 +31,19 @@
namespace space {
// An alloc space is a space where objects may be allocated and garbage collected.
-class DlMallocSpace : public ContinuousMemMapAllocSpace {
+class DlMallocSpace : public MallocSpace {
public:
- typedef void(*WalkCallback)(void *start, void *end, size_t num_bytes, void* callback_arg);
- SpaceType GetType() const {
- if (GetGcRetentionPolicy() == kGcRetentionPolicyFullCollect) {
- return kSpaceTypeZygoteSpace;
- } else {
- return kSpaceTypeAllocSpace;
- }
- }
-
- // Create a AllocSpace with the requested sizes. The requested
+ // Create a DlMallocSpace with the requested sizes. The requested
// base address is not guaranteed to be granted, if it is required,
- // the caller should call Begin on the returned space to confirm
- // the request was granted.
+ // the caller should call Begin on the returned space to confirm the
+ // request was granted.
static DlMallocSpace* Create(const std::string& name, size_t initial_size, size_t growth_limit,
size_t capacity, byte* requested_begin);
- // Allocate num_bytes without allowing the underlying mspace to grow.
virtual mirror::Object* AllocWithGrowth(Thread* self, size_t num_bytes,
size_t* bytes_allocated) LOCKS_EXCLUDED(lock_);
-
- // Allocate num_bytes allowing the underlying mspace to grow.
virtual mirror::Object* Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated);
-
- // Return the storage space required by obj.
virtual size_t AllocationSize(const mirror::Object* obj);
virtual size_t Free(Thread* self, mirror::Object* ptr);
virtual size_t FreeList(Thread* self, size_t num_ptrs, mirror::Object** ptrs);
@@ -64,17 +51,19 @@
mirror::Object* AllocNonvirtual(Thread* self, size_t num_bytes, size_t* bytes_allocated);
size_t AllocationSizeNonvirtual(const mirror::Object* obj) {
- return mspace_usable_size(const_cast<void*>(reinterpret_cast<const void*>(obj))) +
- kChunkOverhead;
+ void* obj_ptr = const_cast<void*>(reinterpret_cast<const void*>(obj));
+ return mspace_usable_size(obj_ptr) + kChunkOverhead;
}
- void* MoreCore(intptr_t increment);
+#ifndef NDEBUG
+ // Override only in the debug build.
+ void CheckMoreCoreForPrecondition();
+#endif
void* GetMspace() const {
return mspace_;
}
- // Hands unused pages back to the system.
size_t Trim();
// Perform a mspace_inspect_all which calls back for each allocation chunk. The chunk may not be
@@ -93,39 +82,8 @@
// allocations fail we GC before increasing the footprint limit and allowing the mspace to grow.
void SetFootprintLimit(size_t limit);
- // Removes the fork time growth limit on capacity, allowing the application to allocate up to the
- // maximum reserved size of the heap.
- void ClearGrowthLimit() {
- growth_limit_ = NonGrowthLimitCapacity();
- }
-
- // Override capacity so that we only return the possibly limited capacity
- size_t Capacity() const {
- return growth_limit_;
- }
-
- // The total amount of memory reserved for the alloc space.
- size_t NonGrowthLimitCapacity() const {
- return GetMemMap()->Size();
- }
-
- accounting::SpaceBitmap* GetLiveBitmap() const {
- return live_bitmap_.get();
- }
-
- accounting::SpaceBitmap* GetMarkBitmap() const {
- return mark_bitmap_.get();
- }
-
- void Dump(std::ostream& os) const;
-
- void SetGrowthLimit(size_t growth_limit);
-
- // Swap the live and mark bitmaps of this space. This is used by the GC for concurrent sweeping.
- void SwapBitmaps();
-
- // Turn ourself into a zygote space and return a new alloc space which has our unused memory.
- DlMallocSpace* CreateZygoteSpace(const char* alloc_space_name);
+ MallocSpace* CreateInstance(const std::string& name, MemMap* mem_map, void* allocator,
+ byte* begin, byte* end, byte* limit, size_t growth_limit);
uint64_t GetBytesAllocated();
uint64_t GetObjectsAllocated();
@@ -136,66 +94,45 @@
return GetObjectsAllocated() + total_objects_freed_;
}
- // Returns the old mark bitmap.
- accounting::SpaceBitmap* BindLiveToMarkBitmap();
- bool HasBoundBitmaps() const;
- void UnBindBitmaps();
-
// Returns the class of a recently freed object.
mirror::Class* FindRecentFreedObject(const mirror::Object* obj);
- // Used to ensure that failure happens when you free / allocate into an invalidated space. If we
- // don't do this we may get heap corruption instead of a segfault at null.
- void InvalidateMSpace() {
+ virtual void InvalidateAllocator() {
mspace_ = nullptr;
}
+ virtual bool IsDlMallocSpace() const {
+ return true;
+ }
+ virtual DlMallocSpace* AsDlMallocSpace() {
+ return this;
+ }
+
protected:
DlMallocSpace(const std::string& name, MemMap* mem_map, void* mspace, byte* begin, byte* end,
byte* limit, size_t growth_limit);
private:
size_t InternalAllocationSize(const mirror::Object* obj);
- mirror::Object* AllocWithoutGrowthLocked(size_t num_bytes, size_t* bytes_allocated)
+
+ mirror::Object* AllocWithoutGrowthLocked(Thread* self, size_t num_bytes, size_t* bytes_allocated)
EXCLUSIVE_LOCKS_REQUIRED(lock_);
- bool Init(size_t initial_size, size_t maximum_size, size_t growth_size, byte* requested_base);
- void RegisterRecentFree(mirror::Object* ptr) EXCLUSIVE_LOCKS_REQUIRED(lock_);
- static void* CreateMallocSpace(void* base, size_t morecore_start, size_t initial_size);
- UniquePtr<accounting::SpaceBitmap> live_bitmap_;
- UniquePtr<accounting::SpaceBitmap> mark_bitmap_;
- UniquePtr<accounting::SpaceBitmap> temp_bitmap_;
-
- // Recent allocation buffer.
- static constexpr size_t kRecentFreeCount = kDebugSpaces ? (1 << 16) : 0;
- static constexpr size_t kRecentFreeMask = kRecentFreeCount - 1;
- std::pair<const mirror::Object*, mirror::Class*> recent_freed_objects_[kRecentFreeCount];
- size_t recent_free_pos_;
+ void* CreateAllocator(void* base, size_t morecore_start, size_t initial_size) {
+ return CreateMspace(base, morecore_start, initial_size);
+ }
+ static void* CreateMspace(void* base, size_t morecore_start, size_t initial_size);
// Approximate number of bytes and objects which have been deallocated in the space.
size_t total_bytes_freed_;
size_t total_objects_freed_;
- static size_t bitmap_index_;
-
// The boundary tag overhead.
static const size_t kChunkOverhead = kWordSize;
- // Used to ensure mutual exclusion when the allocation spaces data structures are being modified.
- Mutex lock_ DEFAULT_MUTEX_ACQUIRED_AFTER;
-
// Underlying malloc space
void* mspace_;
- // The capacity of the alloc space until such time that ClearGrowthLimit is called.
- // The underlying mem_map_ controls the maximum size we allow the heap to grow to. The growth
- // limit is a value <= to the mem_map_ capacity used for ergonomic reasons because of the zygote.
- // Prior to forking the zygote the heap will have a maximally sized mem_map_ but the growth_limit_
- // will be set to a lower value. The growth_limit_ is used as the capacity of the alloc_space_,
- // however, capacity normally can't vary. In the case of the growth_limit_ it can be cleared
- // one time by a call to ClearGrowthLimit.
- size_t growth_limit_;
-
friend class collector::MarkSweep;
DISALLOW_COPY_AND_ASSIGN(DlMallocSpace);
diff --git a/runtime/gc/space/large_object_space.h b/runtime/gc/space/large_object_space.h
index 07fb288..d374ad3 100644
--- a/runtime/gc/space/large_object_space.h
+++ b/runtime/gc/space/large_object_space.h
@@ -116,7 +116,8 @@
virtual ~FreeListSpace();
static FreeListSpace* Create(const std::string& name, byte* requested_begin, size_t capacity);
- size_t AllocationSize(const mirror::Object* obj) EXCLUSIVE_LOCKS_REQUIRED(lock_);
+ size_t AllocationSize(const mirror::Object* obj)
+ EXCLUSIVE_LOCKS_REQUIRED(lock_);
mirror::Object* Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated);
size_t Free(Thread* self, mirror::Object* obj);
bool Contains(const mirror::Object* obj) const;
diff --git a/runtime/gc/space/malloc_space.cc b/runtime/gc/space/malloc_space.cc
new file mode 100644
index 0000000..785b5ed
--- /dev/null
+++ b/runtime/gc/space/malloc_space.cc
@@ -0,0 +1,243 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "malloc_space.h"
+
+#include "gc/accounting/card_table.h"
+#include "gc/heap.h"
+#include "mirror/class-inl.h"
+#include "mirror/object-inl.h"
+#include "runtime.h"
+#include "thread.h"
+#include "thread_list.h"
+#include "utils.h"
+
+namespace art {
+namespace gc {
+namespace space {
+
+size_t MallocSpace::bitmap_index_ = 0;
+
+MallocSpace::MallocSpace(const std::string& name, MemMap* mem_map,
+ byte* begin, byte* end, byte* limit, size_t growth_limit)
+ : ContinuousMemMapAllocSpace(name, mem_map, begin, end, limit, kGcRetentionPolicyAlwaysCollect),
+ recent_free_pos_(0), lock_("allocation space lock", kAllocSpaceLock),
+ growth_limit_(growth_limit) {
+ size_t bitmap_index = bitmap_index_++;
+ static const uintptr_t kGcCardSize = static_cast<uintptr_t>(accounting::CardTable::kCardSize);
+ CHECK(IsAligned<kGcCardSize>(reinterpret_cast<uintptr_t>(mem_map->Begin())));
+ CHECK(IsAligned<kGcCardSize>(reinterpret_cast<uintptr_t>(mem_map->End())));
+ live_bitmap_.reset(accounting::SpaceBitmap::Create(
+ StringPrintf("allocspace %s live-bitmap %d", name.c_str(), static_cast<int>(bitmap_index)),
+ Begin(), Capacity()));
+ DCHECK(live_bitmap_.get() != NULL) << "could not create allocspace live bitmap #" << bitmap_index;
+ mark_bitmap_.reset(accounting::SpaceBitmap::Create(
+ StringPrintf("allocspace %s mark-bitmap %d", name.c_str(), static_cast<int>(bitmap_index)),
+ Begin(), Capacity()));
+ DCHECK(live_bitmap_.get() != NULL) << "could not create allocspace mark bitmap #" << bitmap_index;
+ for (auto& freed : recent_freed_objects_) {
+ freed.first = nullptr;
+ freed.second = nullptr;
+ }
+}
+
+MemMap* MallocSpace::CreateMemMap(const std::string& name, size_t starting_size, size_t* initial_size,
+ size_t* growth_limit, size_t* capacity, byte* requested_begin) {
+ // Sanity check arguments
+ if (starting_size > *initial_size) {
+ *initial_size = starting_size;
+ }
+ if (*initial_size > *growth_limit) {
+ LOG(ERROR) << "Failed to create alloc space (" << name << ") where the initial size ("
+ << PrettySize(*initial_size) << ") is larger than its capacity ("
+ << PrettySize(*growth_limit) << ")";
+ return NULL;
+ }
+ if (*growth_limit > *capacity) {
+ LOG(ERROR) << "Failed to create alloc space (" << name << ") where the growth limit capacity ("
+ << PrettySize(*growth_limit) << ") is larger than the capacity ("
+ << PrettySize(*capacity) << ")";
+ return NULL;
+ }
+
+ // Page align growth limit and capacity which will be used to manage mmapped storage
+ *growth_limit = RoundUp(*growth_limit, kPageSize);
+ *capacity = RoundUp(*capacity, kPageSize);
+
+ std::string error_msg;
+ MemMap* mem_map = MemMap::MapAnonymous(name.c_str(), requested_begin, *capacity,
+ PROT_READ | PROT_WRITE, &error_msg);
+ if (mem_map == NULL) {
+ LOG(ERROR) << "Failed to allocate pages for alloc space (" << name << ") of size "
+ << PrettySize(*capacity) << ": " << error_msg;
+ return NULL;
+ }
+ return mem_map;
+}
+
+void MallocSpace::SwapBitmaps() {
+ live_bitmap_.swap(mark_bitmap_);
+ // Swap names to get more descriptive diagnostics.
+ std::string temp_name(live_bitmap_->GetName());
+ live_bitmap_->SetName(mark_bitmap_->GetName());
+ mark_bitmap_->SetName(temp_name);
+}
+
+mirror::Class* MallocSpace::FindRecentFreedObject(const mirror::Object* obj) {
+ size_t pos = recent_free_pos_;
+ // Start at the most recently freed object and work our way back since there may be duplicates
+ // caused by dlmalloc reusing memory.
+ if (kRecentFreeCount > 0) {
+ for (size_t i = 0; i + 1 < kRecentFreeCount + 1; ++i) {
+ pos = pos != 0 ? pos - 1 : kRecentFreeMask;
+ if (recent_freed_objects_[pos].first == obj) {
+ return recent_freed_objects_[pos].second;
+ }
+ }
+ }
+ return nullptr;
+}
+
+void MallocSpace::RegisterRecentFree(mirror::Object* ptr) {
+ recent_freed_objects_[recent_free_pos_] = std::make_pair(ptr, ptr->GetClass());
+ recent_free_pos_ = (recent_free_pos_ + 1) & kRecentFreeMask;
+}
+
+void MallocSpace::SetGrowthLimit(size_t growth_limit) {
+ growth_limit = RoundUp(growth_limit, kPageSize);
+ growth_limit_ = growth_limit;
+ if (Size() > growth_limit_) {
+ end_ = begin_ + growth_limit;
+ }
+}
+
+void* MallocSpace::MoreCore(intptr_t increment) {
+ CheckMoreCoreForPrecondition();
+ byte* original_end = end_;
+ if (increment != 0) {
+ VLOG(heap) << "MallocSpace::MoreCore " << PrettySize(increment);
+ byte* new_end = original_end + increment;
+ if (increment > 0) {
+ // Should never be asked to increase the allocation beyond the capacity of the space. Enforced
+ // by mspace_set_footprint_limit.
+ CHECK_LE(new_end, Begin() + Capacity());
+ CHECK_MEMORY_CALL(mprotect, (original_end, increment, PROT_READ | PROT_WRITE), GetName());
+ } else {
+ // Should never be asked for negative footprint (ie before begin). Zero footprint is ok.
+ CHECK_GE(original_end + increment, Begin());
+ // Advise we don't need the pages and protect them
+ // TODO: by removing permissions to the pages we may be causing TLB shoot-down which can be
+ // expensive (note the same isn't true for giving permissions to a page as the protected
+ // page shouldn't be in a TLB). We should investigate performance impact of just
+ // removing ignoring the memory protection change here and in Space::CreateAllocSpace. It's
+ // likely just a useful debug feature.
+ size_t size = -increment;
+ CHECK_MEMORY_CALL(madvise, (new_end, size, MADV_DONTNEED), GetName());
+ CHECK_MEMORY_CALL(mprotect, (new_end, size, PROT_NONE), GetName());
+ }
+ // Update end_
+ end_ = new_end;
+ }
+ return original_end;
+}
+
+// Returns the old mark bitmap.
+accounting::SpaceBitmap* MallocSpace::BindLiveToMarkBitmap() {
+ accounting::SpaceBitmap* live_bitmap = GetLiveBitmap();
+ accounting::SpaceBitmap* mark_bitmap = mark_bitmap_.release();
+ temp_bitmap_.reset(mark_bitmap);
+ mark_bitmap_.reset(live_bitmap);
+ return mark_bitmap;
+}
+
+bool MallocSpace::HasBoundBitmaps() const {
+ return temp_bitmap_.get() != nullptr;
+}
+
+void MallocSpace::UnBindBitmaps() {
+ CHECK(HasBoundBitmaps());
+ // At this point, the temp_bitmap holds our old mark bitmap.
+ accounting::SpaceBitmap* new_bitmap = temp_bitmap_.release();
+ CHECK_EQ(mark_bitmap_.release(), live_bitmap_.get());
+ mark_bitmap_.reset(new_bitmap);
+ DCHECK(temp_bitmap_.get() == NULL);
+}
+
+MallocSpace* MallocSpace::CreateZygoteSpace(const char* alloc_space_name) {
+ // For RosAlloc, revoke thread local runs before creating a new
+ // alloc space so that we won't mix thread local runs from different
+ // alloc spaces.
+ RevokeAllThreadLocalBuffers();
+ end_ = reinterpret_cast<byte*>(RoundUp(reinterpret_cast<uintptr_t>(end_), kPageSize));
+ DCHECK(IsAligned<accounting::CardTable::kCardSize>(begin_));
+ DCHECK(IsAligned<accounting::CardTable::kCardSize>(end_));
+ DCHECK(IsAligned<kPageSize>(begin_));
+ DCHECK(IsAligned<kPageSize>(end_));
+ size_t size = RoundUp(Size(), kPageSize);
+ // Trim the heap so that we minimize the size of the Zygote space.
+ Trim();
+ // TODO: Not hardcode these in?
+ const size_t starting_size = kPageSize;
+ const size_t initial_size = 2 * MB;
+ // Remaining size is for the new alloc space.
+ const size_t growth_limit = growth_limit_ - size;
+ const size_t capacity = Capacity() - size;
+ VLOG(heap) << "Begin " << reinterpret_cast<const void*>(begin_) << "\n"
+ << "End " << reinterpret_cast<const void*>(end_) << "\n"
+ << "Size " << size << "\n"
+ << "GrowthLimit " << growth_limit_ << "\n"
+ << "Capacity " << Capacity();
+ SetGrowthLimit(RoundUp(size, kPageSize));
+ SetFootprintLimit(RoundUp(size, kPageSize));
+ // FIXME: Do we need reference counted pointers here?
+ // Make the two spaces share the same mark bitmaps since the bitmaps span both of the spaces.
+ VLOG(heap) << "Creating new AllocSpace: ";
+ VLOG(heap) << "Size " << GetMemMap()->Size();
+ VLOG(heap) << "GrowthLimit " << PrettySize(growth_limit);
+ VLOG(heap) << "Capacity " << PrettySize(capacity);
+ // Remap the tail.
+ std::string error_msg;
+ UniquePtr<MemMap> mem_map(GetMemMap()->RemapAtEnd(end_, alloc_space_name,
+ PROT_READ | PROT_WRITE, &error_msg));
+ CHECK(mem_map.get() != nullptr) << error_msg;
+ void* allocator = CreateAllocator(end_, starting_size, initial_size);
+ // Protect memory beyond the initial size.
+ byte* end = mem_map->Begin() + starting_size;
+ if (capacity - initial_size > 0) {
+ CHECK_MEMORY_CALL(mprotect, (end, capacity - initial_size, PROT_NONE), alloc_space_name);
+ }
+ MallocSpace* alloc_space = CreateInstance(alloc_space_name, mem_map.release(), allocator,
+ end_, end, limit_, growth_limit);
+ SetLimit(End());
+ live_bitmap_->SetHeapLimit(reinterpret_cast<uintptr_t>(End()));
+ CHECK_EQ(live_bitmap_->HeapLimit(), reinterpret_cast<uintptr_t>(End()));
+ mark_bitmap_->SetHeapLimit(reinterpret_cast<uintptr_t>(End()));
+ CHECK_EQ(mark_bitmap_->HeapLimit(), reinterpret_cast<uintptr_t>(End()));
+ VLOG(heap) << "zygote space creation done";
+ return alloc_space;
+}
+
+void MallocSpace::Dump(std::ostream& os) const {
+ os << GetType()
+ << " begin=" << reinterpret_cast<void*>(Begin())
+ << ",end=" << reinterpret_cast<void*>(End())
+ << ",size=" << PrettySize(Size()) << ",capacity=" << PrettySize(Capacity())
+ << ",name=\"" << GetName() << "\"]";
+}
+
+} // namespace space
+} // namespace gc
+} // namespace art
diff --git a/runtime/gc/space/malloc_space.h b/runtime/gc/space/malloc_space.h
new file mode 100644
index 0000000..bd21a4d
--- /dev/null
+++ b/runtime/gc/space/malloc_space.h
@@ -0,0 +1,191 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ART_RUNTIME_GC_SPACE_MALLOC_SPACE_H_
+#define ART_RUNTIME_GC_SPACE_MALLOC_SPACE_H_
+
+#include "space.h"
+
+namespace art {
+namespace gc {
+
+namespace collector {
+ class MarkSweep;
+} // namespace collector
+
+namespace space {
+
+// TODO: Remove define macro
+#define CHECK_MEMORY_CALL(call, args, what) \
+ do { \
+ int rc = call args; \
+ if (UNLIKELY(rc != 0)) { \
+ errno = rc; \
+ PLOG(FATAL) << # call << " failed for " << what; \
+ } \
+ } while (false)
+
+// const bool kUseRosAlloc = true;
+
+// A common parent of DlMallocSpace and RosAllocSpace.
+class MallocSpace : public ContinuousMemMapAllocSpace {
+ public:
+ typedef void(*WalkCallback)(void *start, void *end, size_t num_bytes, void* callback_arg);
+
+ SpaceType GetType() const {
+ if (GetGcRetentionPolicy() == kGcRetentionPolicyFullCollect) {
+ return kSpaceTypeZygoteSpace;
+ } else {
+ return kSpaceTypeAllocSpace;
+ }
+ }
+
+ // Allocate num_bytes without allowing the underlying space to grow.
+ virtual mirror::Object* AllocWithGrowth(Thread* self, size_t num_bytes,
+ size_t* bytes_allocated) = 0;
+ // Allocate num_bytes allowing the underlying space to grow.
+ virtual mirror::Object* Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated) = 0;
+ // Return the storage space required by obj.
+ virtual size_t AllocationSize(const mirror::Object* obj) = 0;
+ virtual size_t Free(Thread* self, mirror::Object* ptr) = 0;
+ virtual size_t FreeList(Thread* self, size_t num_ptrs, mirror::Object** ptrs) = 0;
+
+#ifndef NDEBUG
+ virtual void CheckMoreCoreForPrecondition() {} // to be overridden in the debug build.
+#else
+ void CheckMoreCoreForPrecondition() {} // no-op in the non-debug build.
+#endif
+
+ void* MoreCore(intptr_t increment);
+
+ // Hands unused pages back to the system.
+ virtual size_t Trim() = 0;
+
+ // Perform a mspace_inspect_all which calls back for each allocation chunk. The chunk may not be
+ // in use, indicated by num_bytes equaling zero.
+ virtual void Walk(WalkCallback callback, void* arg) = 0;
+
+ // Returns the number of bytes that the space has currently obtained from the system. This is
+ // greater or equal to the amount of live data in the space.
+ virtual size_t GetFootprint() = 0;
+
+ // Returns the number of bytes that the heap is allowed to obtain from the system via MoreCore.
+ virtual size_t GetFootprintLimit() = 0;
+
+ // Set the maximum number of bytes that the heap is allowed to obtain from the system via
+ // MoreCore. Note this is used to stop the mspace growing beyond the limit to Capacity. When
+ // allocations fail we GC before increasing the footprint limit and allowing the mspace to grow.
+ virtual void SetFootprintLimit(size_t limit) = 0;
+
+ // Removes the fork time growth limit on capacity, allowing the application to allocate up to the
+ // maximum reserved size of the heap.
+ void ClearGrowthLimit() {
+ growth_limit_ = NonGrowthLimitCapacity();
+ }
+
+ // Override capacity so that we only return the possibly limited capacity
+ size_t Capacity() const {
+ return growth_limit_;
+ }
+
+ // The total amount of memory reserved for the alloc space.
+ size_t NonGrowthLimitCapacity() const {
+ return GetMemMap()->Size();
+ }
+
+ accounting::SpaceBitmap* GetLiveBitmap() const {
+ return live_bitmap_.get();
+ }
+
+ accounting::SpaceBitmap* GetMarkBitmap() const {
+ return mark_bitmap_.get();
+ }
+
+ void Dump(std::ostream& os) const;
+
+ void SetGrowthLimit(size_t growth_limit);
+
+ // Swap the live and mark bitmaps of this space. This is used by the GC for concurrent sweeping.
+ void SwapBitmaps();
+
+ virtual MallocSpace* CreateInstance(const std::string& name, MemMap* mem_map, void* allocator,
+ byte* begin, byte* end, byte* limit, size_t growth_limit) = 0;
+
+ // Turn ourself into a zygote space and return a new alloc space which has our unused memory.
+ MallocSpace* CreateZygoteSpace(const char* alloc_space_name);
+
+ virtual uint64_t GetBytesAllocated() = 0;
+ virtual uint64_t GetObjectsAllocated() = 0;
+ virtual uint64_t GetTotalBytesAllocated() = 0;
+ virtual uint64_t GetTotalObjectsAllocated() = 0;
+
+ // Returns the old mark bitmap.
+ accounting::SpaceBitmap* BindLiveToMarkBitmap();
+ bool HasBoundBitmaps() const;
+ void UnBindBitmaps();
+
+ // Returns the class of a recently freed object.
+ mirror::Class* FindRecentFreedObject(const mirror::Object* obj);
+
+ // Used to ensure that failure happens when you free / allocate into an invalidated space. If we
+ // don't do this we may get heap corruption instead of a segfault at null.
+ virtual void InvalidateAllocator() = 0;
+
+ protected:
+ MallocSpace(const std::string& name, MemMap* mem_map, byte* begin, byte* end,
+ byte* limit, size_t growth_limit);
+
+ static MemMap* CreateMemMap(const std::string& name, size_t starting_size, size_t* initial_size,
+ size_t* growth_limit, size_t* capacity, byte* requested_begin);
+
+ virtual void* CreateAllocator(void* base, size_t morecore_start, size_t initial_size) = 0;
+
+ void RegisterRecentFree(mirror::Object* ptr) EXCLUSIVE_LOCKS_REQUIRED(lock_);
+
+ UniquePtr<accounting::SpaceBitmap> live_bitmap_;
+ UniquePtr<accounting::SpaceBitmap> mark_bitmap_;
+ UniquePtr<accounting::SpaceBitmap> temp_bitmap_;
+
+ // Recent allocation buffer.
+ static constexpr size_t kRecentFreeCount = kDebugSpaces ? (1 << 16) : 0;
+ static constexpr size_t kRecentFreeMask = kRecentFreeCount - 1;
+ std::pair<const mirror::Object*, mirror::Class*> recent_freed_objects_[kRecentFreeCount];
+ size_t recent_free_pos_;
+
+ static size_t bitmap_index_;
+
+ // Used to ensure mutual exclusion when the allocation spaces data structures are being modified.
+ Mutex lock_ DEFAULT_MUTEX_ACQUIRED_AFTER;
+
+ // The capacity of the alloc space until such time that ClearGrowthLimit is called.
+ // The underlying mem_map_ controls the maximum size we allow the heap to grow to. The growth
+ // limit is a value <= to the mem_map_ capacity used for ergonomic reasons because of the zygote.
+ // Prior to forking the zygote the heap will have a maximally sized mem_map_ but the growth_limit_
+ // will be set to a lower value. The growth_limit_ is used as the capacity of the alloc_space_,
+ // however, capacity normally can't vary. In the case of the growth_limit_ it can be cleared
+ // one time by a call to ClearGrowthLimit.
+ size_t growth_limit_;
+
+ friend class collector::MarkSweep;
+
+ DISALLOW_COPY_AND_ASSIGN(MallocSpace);
+};
+
+} // namespace space
+} // namespace gc
+} // namespace art
+
+#endif // ART_RUNTIME_GC_SPACE_DLMALLOC_SPACE_H_
diff --git a/runtime/gc/space/rosalloc_space-inl.h b/runtime/gc/space/rosalloc_space-inl.h
new file mode 100644
index 0000000..92c69af
--- /dev/null
+++ b/runtime/gc/space/rosalloc_space-inl.h
@@ -0,0 +1,55 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ART_RUNTIME_GC_SPACE_ROSALLOC_SPACE_INL_H_
+#define ART_RUNTIME_GC_SPACE_ROSALLOC_SPACE_INL_H_
+
+#include "rosalloc_space.h"
+#include "thread.h"
+
+namespace art {
+namespace gc {
+namespace space {
+
+inline mirror::Object* RosAllocSpace::AllocNonvirtual(Thread* self, size_t num_bytes,
+ size_t* bytes_allocated) {
+ mirror::Object* obj;
+ obj = AllocWithoutGrowthLocked(self, num_bytes, bytes_allocated);
+ // RosAlloc zeroes memory internally.
+ return obj;
+}
+
+inline mirror::Object* RosAllocSpace::AllocWithoutGrowthLocked(Thread* self, size_t num_bytes,
+ size_t* bytes_allocated) {
+ size_t rosalloc_size = 0;
+ mirror::Object* result = reinterpret_cast<mirror::Object*>(rosalloc_->Alloc(self, num_bytes,
+ &rosalloc_size));
+ if (LIKELY(result != NULL)) {
+ if (kDebugSpaces) {
+ CHECK(Contains(result)) << "Allocation (" << reinterpret_cast<void*>(result)
+ << ") not in bounds of allocation space " << *this;
+ }
+ DCHECK(bytes_allocated != NULL);
+ *bytes_allocated = rosalloc_size;
+ }
+ return result;
+}
+
+} // namespace space
+} // namespace gc
+} // namespace art
+
+#endif // ART_RUNTIME_GC_SPACE_ROSALLOC_SPACE_INL_H_
diff --git a/runtime/gc/space/rosalloc_space.cc b/runtime/gc/space/rosalloc_space.cc
new file mode 100644
index 0000000..72692d6
--- /dev/null
+++ b/runtime/gc/space/rosalloc_space.cc
@@ -0,0 +1,306 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "rosalloc_space.h"
+
+#include "rosalloc_space-inl.h"
+#include "gc/accounting/card_table.h"
+#include "gc/heap.h"
+#include "mirror/class-inl.h"
+#include "mirror/object-inl.h"
+#include "runtime.h"
+#include "thread.h"
+#include "thread_list.h"
+#include "utils.h"
+
+#include <valgrind.h>
+#include <memcheck/memcheck.h>
+
+namespace art {
+namespace gc {
+namespace space {
+
+static const bool kPrefetchDuringRosAllocFreeList = true;
+
+RosAllocSpace::RosAllocSpace(const std::string& name, MemMap* mem_map,
+ art::gc::allocator::RosAlloc* rosalloc, byte* begin, byte* end,
+ byte* limit, size_t growth_limit)
+ : MallocSpace(name, mem_map, begin, end, limit, growth_limit), rosalloc_(rosalloc) {
+ CHECK(rosalloc != NULL);
+}
+
+RosAllocSpace* RosAllocSpace::Create(const std::string& name, size_t initial_size, size_t growth_limit,
+ size_t capacity, byte* requested_begin) {
+ uint64_t start_time = 0;
+ if (VLOG_IS_ON(heap) || VLOG_IS_ON(startup)) {
+ start_time = NanoTime();
+ VLOG(startup) << "RosAllocSpace::Create entering " << name
+ << " initial_size=" << PrettySize(initial_size)
+ << " growth_limit=" << PrettySize(growth_limit)
+ << " capacity=" << PrettySize(capacity)
+ << " requested_begin=" << reinterpret_cast<void*>(requested_begin);
+ }
+
+ // Memory we promise to rosalloc before it asks for morecore.
+ // Note: making this value large means that large allocations are unlikely to succeed as rosalloc
+ // will ask for this memory from sys_alloc which will fail as the footprint (this value plus the
+ // size of the large allocation) will be greater than the footprint limit.
+ size_t starting_size = kPageSize;
+ MemMap* mem_map = CreateMemMap(name, starting_size, &initial_size, &growth_limit, &capacity,
+ requested_begin);
+ if (mem_map == NULL) {
+ LOG(ERROR) << "Failed to create mem map for alloc space (" << name << ") of size "
+ << PrettySize(capacity);
+ return NULL;
+ }
+ allocator::RosAlloc* rosalloc = CreateRosAlloc(mem_map->Begin(), starting_size, initial_size);
+ if (rosalloc == NULL) {
+ LOG(ERROR) << "Failed to initialize rosalloc for alloc space (" << name << ")";
+ return NULL;
+ }
+
+ // Protect memory beyond the initial size.
+ byte* end = mem_map->Begin() + starting_size;
+ if (capacity - initial_size > 0) {
+ CHECK_MEMORY_CALL(mprotect, (end, capacity - initial_size, PROT_NONE), name);
+ }
+
+ // Everything is set so record in immutable structure and leave
+ RosAllocSpace* space;
+ byte* begin = mem_map->Begin();
+ if (RUNNING_ON_VALGRIND > 0) {
+ // TODO: support valgrind.
+ LOG(FATAL) << "Unimplemented";
+ space = NULL;
+ } else {
+ space = new RosAllocSpace(name, mem_map, rosalloc, begin, end, begin + capacity, growth_limit);
+ }
+ // We start out with only the initial size possibly containing objects.
+ if (VLOG_IS_ON(heap) || VLOG_IS_ON(startup)) {
+ LOG(INFO) << "RosAllocSpace::Create exiting (" << PrettyDuration(NanoTime() - start_time)
+ << " ) " << *space;
+ }
+ return space;
+}
+
+allocator::RosAlloc* RosAllocSpace::CreateRosAlloc(void* begin, size_t morecore_start, size_t initial_size) {
+ // clear errno to allow PLOG on error
+ errno = 0;
+ // create rosalloc using our backing storage starting at begin and
+ // with a footprint of morecore_start. When morecore_start bytes of
+ // memory is exhaused morecore will be called.
+ allocator::RosAlloc* rosalloc = new art::gc::allocator::RosAlloc(begin, morecore_start);
+ if (rosalloc != NULL) {
+ rosalloc->SetFootprintLimit(initial_size);
+ } else {
+ PLOG(ERROR) << "RosAlloc::Create failed";
+ }
+ return rosalloc;
+}
+
+mirror::Object* RosAllocSpace::Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated) {
+ return AllocNonvirtual(self, num_bytes, bytes_allocated);
+}
+
+mirror::Object* RosAllocSpace::AllocWithGrowth(Thread* self, size_t num_bytes, size_t* bytes_allocated) {
+ mirror::Object* result;
+ {
+ MutexLock mu(self, lock_);
+ // Grow as much as possible within the space.
+ size_t max_allowed = Capacity();
+ rosalloc_->SetFootprintLimit(max_allowed);
+ // Try the allocation.
+ result = AllocWithoutGrowthLocked(self, num_bytes, bytes_allocated);
+ // Shrink back down as small as possible.
+ size_t footprint = rosalloc_->Footprint();
+ rosalloc_->SetFootprintLimit(footprint);
+ }
+ // Note RosAlloc zeroes memory internally.
+ // Return the new allocation or NULL.
+ CHECK(!kDebugSpaces || result == NULL || Contains(result));
+ return result;
+}
+
+MallocSpace* RosAllocSpace::CreateInstance(const std::string& name, MemMap* mem_map, void* allocator,
+ byte* begin, byte* end, byte* limit, size_t growth_limit) {
+ return new RosAllocSpace(name, mem_map, reinterpret_cast<allocator::RosAlloc*>(allocator),
+ begin, end, limit, growth_limit);
+}
+
+size_t RosAllocSpace::Free(Thread* self, mirror::Object* ptr) {
+ if (kDebugSpaces) {
+ CHECK(ptr != NULL);
+ CHECK(Contains(ptr)) << "Free (" << ptr << ") not in bounds of heap " << *this;
+ }
+ const size_t bytes_freed = InternalAllocationSize(ptr);
+ total_bytes_freed_atomic_.fetch_add(bytes_freed);
+ ++total_objects_freed_atomic_;
+ if (kRecentFreeCount > 0) {
+ MutexLock mu(self, lock_);
+ RegisterRecentFree(ptr);
+ }
+ rosalloc_->Free(self, ptr);
+ return bytes_freed;
+}
+
+size_t RosAllocSpace::FreeList(Thread* self, size_t num_ptrs, mirror::Object** ptrs) {
+ DCHECK(ptrs != NULL);
+
+ // Don't need the lock to calculate the size of the freed pointers.
+ size_t bytes_freed = 0;
+ for (size_t i = 0; i < num_ptrs; i++) {
+ mirror::Object* ptr = ptrs[i];
+ const size_t look_ahead = 8;
+ if (kPrefetchDuringRosAllocFreeList && i + look_ahead < num_ptrs) {
+ __builtin_prefetch(reinterpret_cast<char*>(ptrs[i + look_ahead]));
+ }
+ bytes_freed += InternalAllocationSize(ptr);
+ }
+
+ if (kRecentFreeCount > 0) {
+ MutexLock mu(self, lock_);
+ for (size_t i = 0; i < num_ptrs; i++) {
+ RegisterRecentFree(ptrs[i]);
+ }
+ }
+
+ if (kDebugSpaces) {
+ size_t num_broken_ptrs = 0;
+ for (size_t i = 0; i < num_ptrs; i++) {
+ if (!Contains(ptrs[i])) {
+ num_broken_ptrs++;
+ LOG(ERROR) << "FreeList[" << i << "] (" << ptrs[i] << ") not in bounds of heap " << *this;
+ } else {
+ size_t size = rosalloc_->UsableSize(ptrs[i]);
+ memset(ptrs[i], 0xEF, size);
+ }
+ }
+ CHECK_EQ(num_broken_ptrs, 0u);
+ }
+
+ rosalloc_->BulkFree(self, reinterpret_cast<void**>(ptrs), num_ptrs);
+ total_bytes_freed_atomic_.fetch_add(bytes_freed);
+ total_objects_freed_atomic_.fetch_add(num_ptrs);
+ return bytes_freed;
+}
+
+// Callback from rosalloc when it needs to increase the footprint
+extern "C" void* art_heap_rosalloc_morecore(allocator::RosAlloc* rosalloc, intptr_t increment) {
+ Heap* heap = Runtime::Current()->GetHeap();
+ DCHECK(heap->GetNonMovingSpace()->IsRosAllocSpace());
+ DCHECK_EQ(heap->GetNonMovingSpace()->AsRosAllocSpace()->GetRosAlloc(), rosalloc);
+ return heap->GetNonMovingSpace()->MoreCore(increment);
+}
+
+// Virtual functions can't get inlined.
+inline size_t RosAllocSpace::InternalAllocationSize(const mirror::Object* obj) {
+ return AllocationSizeNonvirtual(obj);
+}
+
+size_t RosAllocSpace::AllocationSize(const mirror::Object* obj) {
+ return InternalAllocationSize(obj);
+}
+
+size_t RosAllocSpace::Trim() {
+ MutexLock mu(Thread::Current(), lock_);
+ // Trim to release memory at the end of the space.
+ rosalloc_->Trim();
+ // No inspect_all necessary here as trimming of pages is built-in.
+ return 0;
+}
+
+void RosAllocSpace::Walk(void(*callback)(void *start, void *end, size_t num_bytes, void* callback_arg),
+ void* arg) {
+ InspectAllRosAlloc(callback, arg);
+ callback(NULL, NULL, 0, arg); // Indicate end of a space.
+}
+
+size_t RosAllocSpace::GetFootprint() {
+ MutexLock mu(Thread::Current(), lock_);
+ return rosalloc_->Footprint();
+}
+
+size_t RosAllocSpace::GetFootprintLimit() {
+ MutexLock mu(Thread::Current(), lock_);
+ return rosalloc_->FootprintLimit();
+}
+
+void RosAllocSpace::SetFootprintLimit(size_t new_size) {
+ MutexLock mu(Thread::Current(), lock_);
+ VLOG(heap) << "RosAllocSpace::SetFootprintLimit " << PrettySize(new_size);
+ // Compare against the actual footprint, rather than the Size(), because the heap may not have
+ // grown all the way to the allowed size yet.
+ size_t current_space_size = rosalloc_->Footprint();
+ if (new_size < current_space_size) {
+ // Don't let the space grow any more.
+ new_size = current_space_size;
+ }
+ rosalloc_->SetFootprintLimit(new_size);
+}
+
+uint64_t RosAllocSpace::GetBytesAllocated() {
+ if (rosalloc_ != NULL) {
+ size_t bytes_allocated = 0;
+ InspectAllRosAlloc(art::gc::allocator::RosAlloc::BytesAllocatedCallback, &bytes_allocated);
+ return bytes_allocated;
+ } else {
+ return Size();
+ }
+}
+
+uint64_t RosAllocSpace::GetObjectsAllocated() {
+ if (rosalloc_ != NULL) {
+ size_t objects_allocated = 0;
+ InspectAllRosAlloc(art::gc::allocator::RosAlloc::ObjectsAllocatedCallback, &objects_allocated);
+ return objects_allocated;
+ } else {
+ return 0;
+ }
+}
+
+void RosAllocSpace::InspectAllRosAlloc(void (*callback)(void *start, void *end, size_t num_bytes, void* callback_arg),
+ void* arg) NO_THREAD_SAFETY_ANALYSIS {
+ // TODO: NO_THREAD_SAFETY_ANALYSIS.
+ Thread* self = Thread::Current();
+ if (Locks::mutator_lock_->IsExclusiveHeld(self)) {
+ // The mutators are already suspended. For example, a call path
+ // from SignalCatcher::HandleSigQuit().
+ rosalloc_->InspectAll(callback, arg);
+ } else {
+ // The mutators are not suspended yet.
+ DCHECK(!Locks::mutator_lock_->IsSharedHeld(self));
+ ThreadList* tl = Runtime::Current()->GetThreadList();
+ tl->SuspendAll();
+ {
+ MutexLock mu(self, *Locks::runtime_shutdown_lock_);
+ MutexLock mu2(self, *Locks::thread_list_lock_);
+ rosalloc_->InspectAll(callback, arg);
+ }
+ tl->ResumeAll();
+ }
+}
+
+void RosAllocSpace::RevokeThreadLocalBuffers(Thread* thread) {
+ rosalloc_->RevokeThreadLocalRuns(thread);
+}
+
+void RosAllocSpace::RevokeAllThreadLocalBuffers() {
+ rosalloc_->RevokeAllThreadLocalRuns();
+}
+
+} // namespace space
+} // namespace gc
+} // namespace art
diff --git a/runtime/gc/space/rosalloc_space.h b/runtime/gc/space/rosalloc_space.h
new file mode 100644
index 0000000..8191ffb
--- /dev/null
+++ b/runtime/gc/space/rosalloc_space.h
@@ -0,0 +1,144 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ART_RUNTIME_GC_SPACE_ROSALLOC_SPACE_H_
+#define ART_RUNTIME_GC_SPACE_ROSALLOC_SPACE_H_
+
+#include "gc/allocator/rosalloc.h"
+#include "malloc_space.h"
+#include "space.h"
+
+namespace art {
+namespace gc {
+
+namespace collector {
+ class MarkSweep;
+} // namespace collector
+
+namespace space {
+
+// An alloc space is a space where objects may be allocated and garbage collected.
+class RosAllocSpace : public MallocSpace {
+ public:
+ // Create a RosAllocSpace with the requested sizes. The requested
+ // base address is not guaranteed to be granted, if it is required,
+ // the caller should call Begin on the returned space to confirm the
+ // request was granted.
+ static RosAllocSpace* Create(const std::string& name, size_t initial_size, size_t growth_limit,
+ size_t capacity, byte* requested_begin);
+
+ virtual mirror::Object* AllocWithGrowth(Thread* self, size_t num_bytes,
+ size_t* bytes_allocated) LOCKS_EXCLUDED(lock_);
+ virtual mirror::Object* Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated);
+ virtual size_t AllocationSize(const mirror::Object* obj);
+ virtual size_t Free(Thread* self, mirror::Object* ptr);
+ virtual size_t FreeList(Thread* self, size_t num_ptrs, mirror::Object** ptrs);
+
+ mirror::Object* AllocNonvirtual(Thread* self, size_t num_bytes, size_t* bytes_allocated);
+
+ size_t AllocationSizeNonvirtual(const mirror::Object* obj)
+ NO_THREAD_SAFETY_ANALYSIS {
+ // TODO: NO_THREAD_SAFETY_ANALYSIS because SizeOf() requires that mutator_lock is held.
+ void* obj_ptr = const_cast<void*>(reinterpret_cast<const void*>(obj));
+ // obj is a valid object. Use its class in the header to get the size.
+ size_t size = obj->SizeOf();
+ size_t size_by_size = rosalloc_->UsableSize(size);
+ if (kIsDebugBuild) {
+ size_t size_by_ptr = rosalloc_->UsableSize(obj_ptr);
+ if (size_by_size != size_by_ptr) {
+ LOG(INFO) << "Found a bad sized obj of size " << size
+ << " at " << std::hex << reinterpret_cast<intptr_t>(obj_ptr) << std::dec
+ << " size_by_size=" << size_by_size << " size_by_ptr=" << size_by_ptr;
+ }
+ DCHECK_EQ(size_by_size, size_by_ptr);
+ }
+ return size_by_size;
+ }
+
+ art::gc::allocator::RosAlloc* GetRosAlloc() {
+ return rosalloc_;
+ }
+
+ size_t Trim();
+ void Walk(WalkCallback callback, void* arg) LOCKS_EXCLUDED(lock_);
+ size_t GetFootprint();
+ size_t GetFootprintLimit();
+ void SetFootprintLimit(size_t limit);
+
+ MallocSpace* CreateInstance(const std::string& name, MemMap* mem_map, void* allocator,
+ byte* begin, byte* end, byte* limit, size_t growth_limit);
+
+ uint64_t GetBytesAllocated();
+ uint64_t GetObjectsAllocated();
+ uint64_t GetTotalBytesAllocated() {
+ return GetBytesAllocated() + total_bytes_freed_atomic_;
+ }
+ uint64_t GetTotalObjectsAllocated() {
+ return GetObjectsAllocated() + total_objects_freed_atomic_;
+ }
+
+ void RevokeThreadLocalBuffers(Thread* thread);
+ void RevokeAllThreadLocalBuffers();
+
+ // Returns the class of a recently freed object.
+ mirror::Class* FindRecentFreedObject(const mirror::Object* obj);
+
+ virtual void InvalidateAllocator() {
+ rosalloc_ = NULL;
+ }
+
+ virtual bool IsRosAllocSpace() const {
+ return true;
+ }
+ virtual RosAllocSpace* AsRosAllocSpace() {
+ return this;
+ }
+
+ protected:
+ RosAllocSpace(const std::string& name, MemMap* mem_map, allocator::RosAlloc* rosalloc,
+ byte* begin, byte* end, byte* limit, size_t growth_limit);
+
+ private:
+ size_t InternalAllocationSize(const mirror::Object* obj);
+ mirror::Object* AllocWithoutGrowthLocked(Thread* self, size_t num_bytes, size_t* bytes_allocated);
+
+ void* CreateAllocator(void* base, size_t morecore_start, size_t initial_size) {
+ return CreateRosAlloc(base, morecore_start, initial_size);
+ }
+ static allocator::RosAlloc* CreateRosAlloc(void* base, size_t morecore_start, size_t initial_size);
+
+
+ void InspectAllRosAlloc(void (*callback)(void *start, void *end, size_t num_bytes, void* callback_arg),
+ void* arg)
+ LOCKS_EXCLUDED(Locks::runtime_shutdown_lock_, Locks::thread_list_lock_);
+
+ // Approximate number of bytes and objects which have been deallocated in the space.
+ AtomicInteger total_bytes_freed_atomic_;
+ AtomicInteger total_objects_freed_atomic_;
+
+ // Underlying rosalloc.
+ art::gc::allocator::RosAlloc* rosalloc_;
+
+ friend class collector::MarkSweep;
+
+ DISALLOW_COPY_AND_ASSIGN(RosAllocSpace);
+};
+
+} // namespace space
+} // namespace gc
+} // namespace art
+
+#endif // ART_RUNTIME_GC_SPACE_ROSALLOC_SPACE_H_
diff --git a/runtime/gc/space/space-inl.h b/runtime/gc/space/space-inl.h
index f1031ff..0c1d7a2 100644
--- a/runtime/gc/space/space-inl.h
+++ b/runtime/gc/space/space-inl.h
@@ -31,9 +31,10 @@
return down_cast<ImageSpace*>(down_cast<MemMapSpace*>(this));
}
-inline DlMallocSpace* Space::AsDlMallocSpace() {
- DCHECK(IsDlMallocSpace());
- return down_cast<DlMallocSpace*>(down_cast<MemMapSpace*>(this));
+inline MallocSpace* Space::AsMallocSpace() {
+ DCHECK(GetType() == kSpaceTypeAllocSpace || GetType() == kSpaceTypeZygoteSpace);
+ DCHECK(IsDlMallocSpace() || IsRosAllocSpace());
+ return down_cast<MallocSpace*>(down_cast<MemMapSpace*>(this));
}
inline LargeObjectSpace* Space::AsLargeObjectSpace() {
diff --git a/runtime/gc/space/space.h b/runtime/gc/space/space.h
index 4c05dde..38b602e 100644
--- a/runtime/gc/space/space.h
+++ b/runtime/gc/space/space.h
@@ -44,8 +44,10 @@
class AllocSpace;
class ContinuousSpace;
-class DlMallocSpace;
class DiscontinuousSpace;
+class MallocSpace;
+class DlMallocSpace;
+class RosAllocSpace;
class ImageSpace;
class LargeObjectSpace;
@@ -106,11 +108,26 @@
ImageSpace* AsImageSpace();
// Is this a dlmalloc backed allocation space?
- bool IsDlMallocSpace() const {
+ bool IsMallocSpace() const {
SpaceType type = GetType();
return type == kSpaceTypeAllocSpace || type == kSpaceTypeZygoteSpace;
}
- DlMallocSpace* AsDlMallocSpace();
+ MallocSpace* AsMallocSpace();
+
+ virtual bool IsDlMallocSpace() const {
+ return false;
+ }
+ virtual DlMallocSpace* AsDlMallocSpace() {
+ LOG(FATAL) << "Unreachable";
+ return NULL;
+ }
+ virtual bool IsRosAllocSpace() const {
+ return false;
+ }
+ virtual RosAllocSpace* AsRosAllocSpace() {
+ LOG(FATAL) << "Unreachable";
+ return NULL;
+ }
// Is this the space allocated into by the Zygote and no-longer in use?
bool IsZygoteSpace() const {
@@ -195,6 +212,16 @@
// Returns how many bytes were freed.
virtual size_t FreeList(Thread* self, size_t num_ptrs, mirror::Object** ptrs) = 0;
+ // Revoke any sort of thread-local buffers that are used to speed up
+ // allocations for the given thread, if the alloc space
+ // implementation uses any. No-op by default.
+ virtual void RevokeThreadLocalBuffers(Thread* /*thread*/) {}
+
+ // Revoke any sort of thread-local buffers that are used to speed up
+ // allocations for all the threads, if the alloc space
+ // implementation uses any. No-op by default.
+ virtual void RevokeAllThreadLocalBuffers() {}
+
protected:
AllocSpace() {}
virtual ~AllocSpace() {}
diff --git a/runtime/gc/space/space_test.cc b/runtime/gc/space/space_test.cc
index 383714b..6b597ae 100644
--- a/runtime/gc/space/space_test.cc
+++ b/runtime/gc/space/space_test.cc
@@ -20,6 +20,8 @@
#include "common_test.h"
#include "globals.h"
#include "UniquePtr.h"
+#include "mirror/array-inl.h"
+#include "mirror/object-inl.h"
#include <stdint.h>
@@ -34,8 +36,25 @@
void SizeFootPrintGrowthLimitAndTrimDriver(size_t object_size);
void AddSpace(ContinuousSpace* space) {
+ // For RosAlloc, revoke the thread local runs before moving onto a
+ // new alloc space.
+ Runtime::Current()->GetHeap()->RevokeAllThreadLocalBuffers();
Runtime::Current()->GetHeap()->AddSpace(space);
}
+ void InstallClass(mirror::Object* o, size_t size) NO_THREAD_SAFETY_ANALYSIS {
+ // Note the minimum size, which is the size of a zero-length byte array, is 12.
+ EXPECT_GE(size, static_cast<size_t>(12));
+ SirtRef<mirror::ClassLoader> null_loader(Thread::Current(), NULL);
+ mirror::Class* byte_array_class = Runtime::Current()->GetClassLinker()->FindClass("[B", null_loader);
+ EXPECT_TRUE(byte_array_class != NULL);
+ o->SetClass(byte_array_class);
+ mirror::Array* arr = o->AsArray();
+ // size_t header_size = sizeof(mirror::Object) + 4;
+ size_t header_size = arr->DataOffset(1).Uint32Value();
+ int32_t length = size - header_size;
+ arr->SetLength(length);
+ EXPECT_EQ(arr->SizeOf(), size);
+ }
};
static size_t test_rand(size_t* seed) {
@@ -87,7 +106,7 @@
// the GC works with the ZygoteSpace.
TEST_F(SpaceTest, ZygoteSpace) {
size_t dummy = 0;
- DlMallocSpace* space(DlMallocSpace::Create("test", 4 * MB, 16 * MB, 16 * MB, NULL));
+ MallocSpace* space(DlMallocSpace::Create("test", 4 * MB, 16 * MB, 16 * MB, NULL));
ASSERT_TRUE(space != NULL);
// Make space findable to the heap, will also delete space when runtime is cleaned up
@@ -97,6 +116,7 @@
// Succeeds, fits without adjusting the footprint limit.
mirror::Object* ptr1 = space->Alloc(self, 1 * MB, &dummy);
EXPECT_TRUE(ptr1 != NULL);
+ InstallClass(ptr1, 1 * MB);
// Fails, requires a higher footprint limit.
mirror::Object* ptr2 = space->Alloc(self, 8 * MB, &dummy);
@@ -107,6 +127,7 @@
mirror::Object* ptr3 = space->AllocWithGrowth(self, 8 * MB, &ptr3_bytes_allocated);
EXPECT_TRUE(ptr3 != NULL);
EXPECT_LE(8U * MB, ptr3_bytes_allocated);
+ InstallClass(ptr3, 8 * MB);
// Fails, requires a higher footprint limit.
mirror::Object* ptr4 = space->Alloc(self, 8 * MB, &dummy);
@@ -123,8 +144,9 @@
EXPECT_LE(8U * MB, free3);
// Succeeds, now that memory has been freed.
- void* ptr6 = space->AllocWithGrowth(self, 9 * MB, &dummy);
+ mirror::Object* ptr6 = space->AllocWithGrowth(self, 9 * MB, &dummy);
EXPECT_TRUE(ptr6 != NULL);
+ InstallClass(ptr6, 9 * MB);
// Final clean up.
size_t free1 = space->AllocationSize(ptr1);
@@ -141,6 +163,7 @@
// Succeeds, fits without adjusting the footprint limit.
ptr1 = space->Alloc(self, 1 * MB, &dummy);
EXPECT_TRUE(ptr1 != NULL);
+ InstallClass(ptr1, 1 * MB);
// Fails, requires a higher footprint limit.
ptr2 = space->Alloc(self, 8 * MB, &dummy);
@@ -149,6 +172,7 @@
// Succeeds, adjusts the footprint.
ptr3 = space->AllocWithGrowth(self, 2 * MB, &dummy);
EXPECT_TRUE(ptr3 != NULL);
+ InstallClass(ptr3, 2 * MB);
space->Free(self, ptr3);
// Final clean up.
@@ -169,6 +193,7 @@
// Succeeds, fits without adjusting the footprint limit.
mirror::Object* ptr1 = space->Alloc(self, 1 * MB, &dummy);
EXPECT_TRUE(ptr1 != NULL);
+ InstallClass(ptr1, 1 * MB);
// Fails, requires a higher footprint limit.
mirror::Object* ptr2 = space->Alloc(self, 8 * MB, &dummy);
@@ -179,6 +204,7 @@
mirror::Object* ptr3 = space->AllocWithGrowth(self, 8 * MB, &ptr3_bytes_allocated);
EXPECT_TRUE(ptr3 != NULL);
EXPECT_LE(8U * MB, ptr3_bytes_allocated);
+ InstallClass(ptr3, 8 * MB);
// Fails, requires a higher footprint limit.
mirror::Object* ptr4 = space->Alloc(self, 8 * MB, &dummy);
@@ -195,8 +221,9 @@
EXPECT_LE(8U * MB, free3);
// Succeeds, now that memory has been freed.
- void* ptr6 = space->AllocWithGrowth(self, 9 * MB, &dummy);
+ mirror::Object* ptr6 = space->AllocWithGrowth(self, 9 * MB, &dummy);
EXPECT_TRUE(ptr6 != NULL);
+ InstallClass(ptr6, 9 * MB);
// Final clean up.
size_t free1 = space->AllocationSize(ptr1);
@@ -278,8 +305,9 @@
for (size_t i = 0; i < arraysize(lots_of_objects); i++) {
size_t allocation_size = 0;
lots_of_objects[i] = space->Alloc(self, 16, &allocation_size);
- EXPECT_EQ(allocation_size, space->AllocationSize(lots_of_objects[i]));
EXPECT_TRUE(lots_of_objects[i] != NULL);
+ InstallClass(lots_of_objects[i], 16);
+ EXPECT_EQ(allocation_size, space->AllocationSize(lots_of_objects[i]));
}
// Release memory and check pointers are NULL
@@ -292,8 +320,9 @@
for (size_t i = 0; i < arraysize(lots_of_objects); i++) {
size_t allocation_size = 0;
lots_of_objects[i] = space->AllocWithGrowth(self, 1024, &allocation_size);
- EXPECT_EQ(allocation_size, space->AllocationSize(lots_of_objects[i]));
EXPECT_TRUE(lots_of_objects[i] != NULL);
+ InstallClass(lots_of_objects[i], 1024);
+ EXPECT_EQ(allocation_size, space->AllocationSize(lots_of_objects[i]));
}
// Release memory and check pointers are NULL
@@ -310,22 +339,20 @@
// No allocation can succeed
return;
}
- // Mspace for raw dlmalloc operations
- void* mspace = space->GetMspace();
- // mspace's footprint equals amount of resources requested from system
- size_t footprint = mspace_footprint(mspace);
+ // The space's footprint equals amount of resources requested from system
+ size_t footprint = space->GetFootprint();
- // mspace must at least have its book keeping allocated
+ // The space must at least have its book keeping allocated
EXPECT_GT(footprint, 0u);
- // mspace but it shouldn't exceed the initial size
+ // But it shouldn't exceed the initial size
EXPECT_LE(footprint, growth_limit);
// space's size shouldn't exceed the initial size
EXPECT_LE(space->Size(), growth_limit);
- // this invariant should always hold or else the mspace has grown to be larger than what the
+ // this invariant should always hold or else the space has grown to be larger than what the
// space believes its size is (which will break invariants)
EXPECT_GE(space->Size(), footprint);
@@ -345,8 +372,9 @@
alloc_size = object_size;
} else {
alloc_size = test_rand(&rand_seed) % static_cast<size_t>(-object_size);
- if (alloc_size < 8) {
- alloc_size = 8;
+ // Note the minimum size, which is the size of a zero-length byte array, is 12.
+ if (alloc_size < 12) {
+ alloc_size = 12;
}
}
mirror::Object* object;
@@ -356,9 +384,10 @@
} else {
object = space->AllocWithGrowth(self, alloc_size, &bytes_allocated);
}
- footprint = mspace_footprint(mspace);
+ footprint = space->GetFootprint();
EXPECT_GE(space->Size(), footprint); // invariant
if (object != NULL) { // allocation succeeded
+ InstallClass(object, alloc_size);
lots_of_objects.get()[i] = object;
size_t allocation_size = space->AllocationSize(object);
EXPECT_EQ(bytes_allocated, allocation_size);
@@ -395,7 +424,7 @@
space->Trim();
// Bounds sanity
- footprint = mspace_footprint(mspace);
+ footprint = space->GetFootprint();
EXPECT_LE(amount_allocated, growth_limit);
EXPECT_GE(footprint, amount_allocated);
EXPECT_LE(footprint, growth_limit);
@@ -421,13 +450,21 @@
space->Free(self, object);
lots_of_objects.get()[i] = NULL;
amount_allocated -= allocation_size;
- footprint = mspace_footprint(mspace);
+ footprint = space->GetFootprint();
EXPECT_GE(space->Size(), footprint); // invariant
}
free_increment >>= 1;
}
+ // The space has become empty here before allocating a large object
+ // below. For RosAlloc, revoke thread-local runs, which are kept
+ // even when empty for a performance reason, so that they won't
+ // cause the following large object allocation to fail due to
+ // potential fragmentation. Note they are normally revoked at each
+ // GC (but no GC here.)
+ space->RevokeAllThreadLocalBuffers();
+
// All memory was released, try a large allocation to check freed memory is being coalesced
mirror::Object* large_object;
size_t three_quarters_space = (growth_limit / 2) + (growth_limit / 4);
@@ -438,9 +475,10 @@
large_object = space->AllocWithGrowth(self, three_quarters_space, &bytes_allocated);
}
EXPECT_TRUE(large_object != NULL);
+ InstallClass(large_object, three_quarters_space);
// Sanity check footprint
- footprint = mspace_footprint(mspace);
+ footprint = space->GetFootprint();
EXPECT_LE(footprint, growth_limit);
EXPECT_GE(space->Size(), footprint);
EXPECT_LE(space->Size(), growth_limit);
@@ -449,7 +487,7 @@
space->Free(self, large_object);
// Sanity check footprint
- footprint = mspace_footprint(mspace);
+ footprint = space->GetFootprint();
EXPECT_LE(footprint, growth_limit);
EXPECT_GE(space->Size(), footprint);
EXPECT_LE(space->Size(), growth_limit);
@@ -488,8 +526,8 @@
}
// Each size test is its own test so that we get a fresh heap each time
-TEST_F(SpaceTest, SizeFootPrintGrowthLimitAndTrim_AllocationsOf_8B) {
- SizeFootPrintGrowthLimitAndTrimDriver(8);
+TEST_F(SpaceTest, SizeFootPrintGrowthLimitAndTrim_AllocationsOf_12B) {
+ SizeFootPrintGrowthLimitAndTrimDriver(12);
}
TEST_SizeFootPrintGrowthLimitAndTrim(16B, 16)
TEST_SizeFootPrintGrowthLimitAndTrim(24B, 24)