blob: af1b26b87067bd43b4e1ed050bfbb690fada3ade [file] [log] [blame]
Hiroshi Yamauchi3b4c1892013-09-12 21:33:12 -07001/*
2 * Copyright (C) 2013 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#ifndef ART_RUNTIME_GC_HEAP_INL_H_
18#define ART_RUNTIME_GC_HEAP_INL_H_
19
20#include "heap.h"
21
22#include "debugger.h"
Mathieu Chartier590fee92013-09-13 13:46:47 -070023#include "gc/space/bump_pointer_space-inl.h"
Hiroshi Yamauchi3b4c1892013-09-12 21:33:12 -070024#include "gc/space/dlmalloc_space-inl.h"
25#include "gc/space/large_object_space.h"
Hiroshi Yamauchicf58d4a2013-09-26 14:21:22 -070026#include "gc/space/rosalloc_space-inl.h"
Hiroshi Yamauchi3b4c1892013-09-12 21:33:12 -070027#include "object_utils.h"
28#include "runtime.h"
29#include "thread.h"
30#include "thread-inl.h"
31
32namespace art {
33namespace gc {
34
Mathieu Chartier692fafd2013-11-29 17:24:40 -080035template <bool kInstrumented, bool kCheckLargeObject, typename PreFenceVisitor>
Mathieu Chartier1febddf2013-11-20 12:33:14 -080036inline mirror::Object* Heap::AllocObjectWithAllocator(Thread* self, mirror::Class* klass,
37 size_t byte_count, AllocatorType allocator,
38 const PreFenceVisitor& pre_fence_visitor) {
39 DebugCheckPreconditionsForAllocObject(klass, byte_count);
Hiroshi Yamauchi3b4c1892013-09-12 21:33:12 -070040 // Since allocation can cause a GC which will need to SuspendAll, make sure all allocations are
41 // done in the runnable state where suspension is expected.
42 DCHECK_EQ(self->GetState(), kRunnable);
43 self->AssertThreadSuspensionIsAllowable();
Mathieu Chartierc528dba2013-11-26 12:00:11 -080044 // Need to check that we arent the large object allocator since the large object allocation code
45 // path this function. If we didn't check we would have an infinite loop.
Mathieu Chartier692fafd2013-11-29 17:24:40 -080046 if (kCheckLargeObject && UNLIKELY(ShouldAllocLargeObject(klass, byte_count))) {
Mathieu Chartierc528dba2013-11-26 12:00:11 -080047 return AllocLargeObject<kInstrumented, PreFenceVisitor>(self, klass, byte_count,
48 pre_fence_visitor);
49 }
Mathieu Chartiercbb2d202013-11-14 17:45:16 -080050 mirror::Object* obj;
Mathieu Chartiercbb2d202013-11-14 17:45:16 -080051 AllocationTimer alloc_timer(this, &obj);
Mathieu Chartier692fafd2013-11-29 17:24:40 -080052 size_t bytes_allocated;
Mathieu Chartierc528dba2013-11-26 12:00:11 -080053 obj = TryToAllocate<kInstrumented, false>(self, allocator, byte_count, &bytes_allocated);
Mathieu Chartiercbb2d202013-11-14 17:45:16 -080054 if (UNLIKELY(obj == nullptr)) {
Mathieu Chartierc528dba2013-11-26 12:00:11 -080055 obj = AllocateInternalWithGc(self, allocator, byte_count, &bytes_allocated, &klass);
Mathieu Chartiercbb2d202013-11-14 17:45:16 -080056 if (obj == nullptr) {
57 return nullptr;
Mathieu Chartiercbb2d202013-11-14 17:45:16 -080058 }
59 }
Mathieu Chartier1febddf2013-11-20 12:33:14 -080060 obj->SetClass(klass);
61 pre_fence_visitor(obj);
Mathieu Chartiercbb2d202013-11-14 17:45:16 -080062 DCHECK_GT(bytes_allocated, 0u);
63 const size_t new_num_bytes_allocated =
Ian Rogersb122a4b2013-11-19 18:00:50 -080064 static_cast<size_t>(num_bytes_allocated_.FetchAndAdd(bytes_allocated)) + bytes_allocated;
Mathieu Chartiercbb2d202013-11-14 17:45:16 -080065 // TODO: Deprecate.
66 if (kInstrumented) {
67 if (Runtime::Current()->HasStatsEnabled()) {
68 RuntimeStats* thread_stats = self->GetStats();
69 ++thread_stats->allocated_objects;
70 thread_stats->allocated_bytes += bytes_allocated;
71 RuntimeStats* global_stats = Runtime::Current()->GetStats();
72 ++global_stats->allocated_objects;
73 global_stats->allocated_bytes += bytes_allocated;
74 }
75 } else {
76 DCHECK(!Runtime::Current()->HasStatsEnabled());
77 }
78 if (AllocatorHasAllocationStack(allocator)) {
79 // This is safe to do since the GC will never free objects which are neither in the allocation
80 // stack or the live bitmap.
81 while (!allocation_stack_->AtomicPushBack(obj)) {
82 CollectGarbageInternal(collector::kGcTypeSticky, kGcCauseForAlloc, false);
83 }
84 }
85 if (kInstrumented) {
86 if (Dbg::IsAllocTrackingEnabled()) {
Mathieu Chartier1febddf2013-11-20 12:33:14 -080087 Dbg::RecordAllocation(klass, bytes_allocated);
Mathieu Chartiercbb2d202013-11-14 17:45:16 -080088 }
89 } else {
90 DCHECK(!Dbg::IsAllocTrackingEnabled());
91 }
Mathieu Chartier692fafd2013-11-29 17:24:40 -080092 // concurrent_gc_ isn't known at compile time so we can optimize by not checking it for
93 // the BumpPointer or TLAB allocators. This is nice since it allows the entire if statement to be
94 // optimized out. And for the other allocators, AllocatorMayHaveConcurrentGC is a constant since
95 // the allocator_type should be constant propagated.
96 if (AllocatorMayHaveConcurrentGC(allocator) && concurrent_gc_) {
Mathieu Chartiercbb2d202013-11-14 17:45:16 -080097 CheckConcurrentGC(self, new_num_bytes_allocated, obj);
98 }
99 if (kIsDebugBuild) {
100 if (kDesiredHeapVerification > kNoHeapVerification) {
101 VerifyObject(obj);
102 }
103 self->VerifyStack();
104 }
105 return obj;
Hiroshi Yamauchi3b4c1892013-09-12 21:33:12 -0700106}
107
Mathieu Chartierc528dba2013-11-26 12:00:11 -0800108template <bool kInstrumented, typename PreFenceVisitor>
109inline mirror::Object* Heap::AllocLargeObject(Thread* self, mirror::Class* klass,
110 size_t byte_count,
111 const PreFenceVisitor& pre_fence_visitor) {
Mathieu Chartier692fafd2013-11-29 17:24:40 -0800112 return AllocObjectWithAllocator<kInstrumented, false, PreFenceVisitor>(self, klass, byte_count,
113 kAllocatorTypeLOS,
114 pre_fence_visitor);
Mathieu Chartierc528dba2013-11-26 12:00:11 -0800115}
116
117template <const bool kInstrumented, const bool kGrow>
Mathieu Chartiercbb2d202013-11-14 17:45:16 -0800118inline mirror::Object* Heap::TryToAllocate(Thread* self, AllocatorType allocator_type,
Mathieu Chartierc528dba2013-11-26 12:00:11 -0800119 size_t alloc_size, size_t* bytes_allocated) {
Mathieu Chartier692fafd2013-11-29 17:24:40 -0800120 if (UNLIKELY(IsOutOfMemoryOnAllocation<kGrow>(allocator_type, alloc_size))) {
Mathieu Chartiercbb2d202013-11-14 17:45:16 -0800121 return nullptr;
Hiroshi Yamauchi3b4c1892013-09-12 21:33:12 -0700122 }
Mathieu Chartiercbb2d202013-11-14 17:45:16 -0800123 if (kInstrumented) {
124 if (UNLIKELY(running_on_valgrind_ && allocator_type == kAllocatorTypeFreeList)) {
125 return non_moving_space_->Alloc(self, alloc_size, bytes_allocated);
126 }
Hiroshi Yamauchie4e23c02013-12-06 13:38:43 -0800127 } else {
128 // If running on valgrind, we should be using the instrumented path.
129 DCHECK(!running_on_valgrind_);
Mathieu Chartiercbb2d202013-11-14 17:45:16 -0800130 }
131 mirror::Object* ret;
132 switch (allocator_type) {
133 case kAllocatorTypeBumpPointer: {
134 DCHECK(bump_pointer_space_ != nullptr);
135 alloc_size = RoundUp(alloc_size, space::BumpPointerSpace::kAlignment);
136 ret = bump_pointer_space_->AllocNonvirtual(alloc_size);
137 if (LIKELY(ret != nullptr)) {
138 *bytes_allocated = alloc_size;
139 }
140 break;
141 }
142 case kAllocatorTypeFreeList: {
143 if (kUseRosAlloc) {
144 ret = reinterpret_cast<space::RosAllocSpace*>(non_moving_space_)->AllocNonvirtual(
145 self, alloc_size, bytes_allocated);
146 } else {
147 ret = reinterpret_cast<space::DlMallocSpace*>(non_moving_space_)->AllocNonvirtual(
148 self, alloc_size, bytes_allocated);
149 }
150 break;
151 }
152 case kAllocatorTypeLOS: {
153 ret = large_object_space_->Alloc(self, alloc_size, bytes_allocated);
Hiroshi Yamauchi95a659f2013-11-22 14:43:45 -0800154 // Note that the bump pointer spaces aren't necessarily next to
155 // the other continuous spaces like the non-moving alloc space or
156 // the zygote space.
157 DCHECK(ret == nullptr || large_object_space_->Contains(ret));
Mathieu Chartiercbb2d202013-11-14 17:45:16 -0800158 break;
159 }
Mathieu Chartier692fafd2013-11-29 17:24:40 -0800160 case kAllocatorTypeTLAB: {
161 alloc_size = RoundUp(alloc_size, space::BumpPointerSpace::kAlignment);
162 if (UNLIKELY(self->TLABSize() < alloc_size)) {
163 // Try allocating a new thread local buffer, if the allocaiton fails the space must be
164 // full so return nullptr.
165 if (!bump_pointer_space_->AllocNewTLAB(self, alloc_size + kDefaultTLABSize)) {
166 return nullptr;
167 }
168 }
169 // The allocation can't fail.
170 ret = self->AllocTLAB(alloc_size);
171 DCHECK(ret != nullptr);
172 *bytes_allocated = alloc_size;
173 break;
174 }
Mathieu Chartiercbb2d202013-11-14 17:45:16 -0800175 default: {
176 LOG(FATAL) << "Invalid allocator type";
177 ret = nullptr;
178 }
179 }
180 return ret;
Hiroshi Yamauchi3b4c1892013-09-12 21:33:12 -0700181}
182
Mathieu Chartier590fee92013-09-13 13:46:47 -0700183inline void Heap::DebugCheckPreconditionsForAllocObject(mirror::Class* c, size_t byte_count) {
Hiroshi Yamauchi3b4c1892013-09-12 21:33:12 -0700184 DCHECK(c == NULL || (c->IsClassClass() && byte_count >= sizeof(mirror::Class)) ||
185 (c->IsVariableSize() || c->GetObjectSize() == byte_count) ||
Ian Rogersdfb325e2013-10-30 01:00:44 -0700186 strlen(ClassHelper(c).GetDescriptor()) == 0);
Hiroshi Yamauchi3b4c1892013-09-12 21:33:12 -0700187 DCHECK_GE(byte_count, sizeof(mirror::Object));
188}
189
190inline Heap::AllocationTimer::AllocationTimer(Heap* heap, mirror::Object** allocated_obj_ptr)
191 : heap_(heap), allocated_obj_ptr_(allocated_obj_ptr) {
192 if (kMeasureAllocationTime) {
193 allocation_start_time_ = NanoTime() / kTimeAdjust;
194 }
195}
196
197inline Heap::AllocationTimer::~AllocationTimer() {
198 if (kMeasureAllocationTime) {
199 mirror::Object* allocated_obj = *allocated_obj_ptr_;
200 // Only if the allocation succeeded, record the time.
Mathieu Chartiercbb2d202013-11-14 17:45:16 -0800201 if (allocated_obj != nullptr) {
Hiroshi Yamauchi3b4c1892013-09-12 21:33:12 -0700202 uint64_t allocation_end_time = NanoTime() / kTimeAdjust;
Ian Rogersb122a4b2013-11-19 18:00:50 -0800203 heap_->total_allocation_time_.FetchAndAdd(allocation_end_time - allocation_start_time_);
Hiroshi Yamauchi3b4c1892013-09-12 21:33:12 -0700204 }
205 }
206};
207
Mathieu Chartiercbb2d202013-11-14 17:45:16 -0800208inline bool Heap::ShouldAllocLargeObject(mirror::Class* c, size_t byte_count) const {
Hiroshi Yamauchi3b4c1892013-09-12 21:33:12 -0700209 // We need to have a zygote space or else our newly allocated large object can end up in the
210 // Zygote resulting in it being prematurely freed.
211 // We can only do this for primitive objects since large objects will not be within the card table
212 // range. This also means that we rely on SetClass not dirtying the object's card.
213 return byte_count >= kLargeObjectThreshold && have_zygote_space_ && c->IsPrimitiveArray();
214}
215
Mathieu Chartier692fafd2013-11-29 17:24:40 -0800216template <bool kGrow>
217inline bool Heap::IsOutOfMemoryOnAllocation(AllocatorType allocator_type, size_t alloc_size) {
Hiroshi Yamauchi3b4c1892013-09-12 21:33:12 -0700218 size_t new_footprint = num_bytes_allocated_ + alloc_size;
219 if (UNLIKELY(new_footprint > max_allowed_footprint_)) {
220 if (UNLIKELY(new_footprint > growth_limit_)) {
221 return true;
222 }
Mathieu Chartier692fafd2013-11-29 17:24:40 -0800223 if (!AllocatorMayHaveConcurrentGC(allocator_type) || !concurrent_gc_) {
Mathieu Chartierc528dba2013-11-26 12:00:11 -0800224 if (!kGrow) {
Hiroshi Yamauchi3b4c1892013-09-12 21:33:12 -0700225 return true;
Hiroshi Yamauchi3b4c1892013-09-12 21:33:12 -0700226 }
Mathieu Chartier7bf82af2013-12-06 16:51:45 -0800227 // TODO: Grow for allocation is racy, fix it.
228 VLOG(heap) << "Growing heap from " << PrettySize(max_allowed_footprint_) << " to "
229 << PrettySize(new_footprint) << " for a " << PrettySize(alloc_size) << " allocation";
230 max_allowed_footprint_ = new_footprint;
Hiroshi Yamauchi3b4c1892013-09-12 21:33:12 -0700231 }
232 }
233 return false;
234}
235
Mathieu Chartiercbb2d202013-11-14 17:45:16 -0800236inline void Heap::CheckConcurrentGC(Thread* self, size_t new_num_bytes_allocated,
237 mirror::Object* obj) {
Hiroshi Yamauchi3b4c1892013-09-12 21:33:12 -0700238 if (UNLIKELY(new_num_bytes_allocated >= concurrent_start_bytes_)) {
239 // The SirtRef is necessary since the calls in RequestConcurrentGC are a safepoint.
240 SirtRef<mirror::Object> ref(self, obj);
241 RequestConcurrentGC(self);
242 }
243}
244
245} // namespace gc
246} // namespace art
247
248#endif // ART_RUNTIME_GC_HEAP_INL_H_