Hiroshi Yamauchi | 3b4c189 | 2013-09-12 21:33:12 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2013 The Android Open Source Project |
| 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | |
| 17 | #ifndef ART_RUNTIME_GC_HEAP_INL_H_ |
| 18 | #define ART_RUNTIME_GC_HEAP_INL_H_ |
| 19 | |
| 20 | #include "heap.h" |
| 21 | |
| 22 | #include "debugger.h" |
Mathieu Chartier | 590fee9 | 2013-09-13 13:46:47 -0700 | [diff] [blame^] | 23 | #include "gc/space/bump_pointer_space-inl.h" |
Hiroshi Yamauchi | 3b4c189 | 2013-09-12 21:33:12 -0700 | [diff] [blame] | 24 | #include "gc/space/dlmalloc_space-inl.h" |
| 25 | #include "gc/space/large_object_space.h" |
| 26 | #include "object_utils.h" |
| 27 | #include "runtime.h" |
| 28 | #include "thread.h" |
| 29 | #include "thread-inl.h" |
| 30 | |
| 31 | namespace art { |
| 32 | namespace gc { |
| 33 | |
Mathieu Chartier | 590fee9 | 2013-09-13 13:46:47 -0700 | [diff] [blame^] | 34 | inline mirror::Object* Heap::AllocNonMovableObjectUninstrumented(Thread* self, mirror::Class* c, |
| 35 | size_t byte_count) { |
| 36 | DebugCheckPreconditionsForAllocObject(c, byte_count); |
Hiroshi Yamauchi | 3b4c189 | 2013-09-12 21:33:12 -0700 | [diff] [blame] | 37 | mirror::Object* obj; |
| 38 | size_t bytes_allocated; |
| 39 | AllocationTimer alloc_timer(this, &obj); |
| 40 | bool large_object_allocation = TryAllocLargeObjectUninstrumented(self, c, byte_count, |
| 41 | &obj, &bytes_allocated); |
| 42 | if (LIKELY(!large_object_allocation)) { |
| 43 | // Non-large object allocation. |
Mathieu Chartier | 590fee9 | 2013-09-13 13:46:47 -0700 | [diff] [blame^] | 44 | obj = AllocateUninstrumented(self, non_moving_space_, byte_count, &bytes_allocated); |
Hiroshi Yamauchi | 3b4c189 | 2013-09-12 21:33:12 -0700 | [diff] [blame] | 45 | // Ensure that we did not allocate into a zygote space. |
| 46 | DCHECK(obj == NULL || !have_zygote_space_ || !FindSpaceFromObject(obj, false)->IsZygoteSpace()); |
| 47 | } |
| 48 | if (LIKELY(obj != NULL)) { |
| 49 | obj->SetClass(c); |
| 50 | // Record allocation after since we want to use the atomic add for the atomic fence to guard |
| 51 | // the SetClass since we do not want the class to appear NULL in another thread. |
| 52 | size_t new_num_bytes_allocated = RecordAllocationUninstrumented(bytes_allocated, obj); |
| 53 | DCHECK(!Dbg::IsAllocTrackingEnabled()); |
| 54 | CheckConcurrentGC(self, new_num_bytes_allocated, obj); |
| 55 | if (kDesiredHeapVerification > kNoHeapVerification) { |
| 56 | VerifyObject(obj); |
| 57 | } |
Mathieu Chartier | 590fee9 | 2013-09-13 13:46:47 -0700 | [diff] [blame^] | 58 | } else { |
| 59 | ThrowOutOfMemoryError(self, byte_count, large_object_allocation); |
Hiroshi Yamauchi | 3b4c189 | 2013-09-12 21:33:12 -0700 | [diff] [blame] | 60 | } |
Mathieu Chartier | 590fee9 | 2013-09-13 13:46:47 -0700 | [diff] [blame^] | 61 | if (kIsDebugBuild) { |
| 62 | self->VerifyStack(); |
| 63 | } |
| 64 | return obj; |
| 65 | } |
| 66 | |
| 67 | inline mirror::Object* Heap::AllocMovableObjectUninstrumented(Thread* self, mirror::Class* c, |
| 68 | size_t byte_count) { |
| 69 | DebugCheckPreconditionsForAllocObject(c, byte_count); |
| 70 | mirror::Object* obj; |
| 71 | AllocationTimer alloc_timer(this, &obj); |
| 72 | byte_count = (byte_count + 7) & ~7; |
| 73 | if (UNLIKELY(IsOutOfMemoryOnAllocation(byte_count, false))) { |
| 74 | CollectGarbageInternal(collector::kGcTypeFull, kGcCauseForAlloc, false); |
| 75 | if (UNLIKELY(IsOutOfMemoryOnAllocation(byte_count, true))) { |
| 76 | CollectGarbageInternal(collector::kGcTypeFull, kGcCauseForAlloc, true); |
| 77 | } |
| 78 | } |
| 79 | obj = bump_pointer_space_->AllocNonvirtual(byte_count); |
| 80 | if (LIKELY(obj != NULL)) { |
| 81 | obj->SetClass(c); |
| 82 | DCHECK(!obj->IsClass()); |
| 83 | // Record allocation after since we want to use the atomic add for the atomic fence to guard |
| 84 | // the SetClass since we do not want the class to appear NULL in another thread. |
| 85 | num_bytes_allocated_.fetch_add(byte_count); |
| 86 | DCHECK(!Dbg::IsAllocTrackingEnabled()); |
| 87 | if (kDesiredHeapVerification > kNoHeapVerification) { |
| 88 | VerifyObject(obj); |
| 89 | } |
| 90 | } else { |
| 91 | ThrowOutOfMemoryError(self, byte_count, false); |
| 92 | } |
| 93 | if (kIsDebugBuild) { |
| 94 | self->VerifyStack(); |
| 95 | } |
| 96 | return obj; |
Hiroshi Yamauchi | 3b4c189 | 2013-09-12 21:33:12 -0700 | [diff] [blame] | 97 | } |
| 98 | |
| 99 | inline size_t Heap::RecordAllocationUninstrumented(size_t size, mirror::Object* obj) { |
| 100 | DCHECK(obj != NULL); |
| 101 | DCHECK_GT(size, 0u); |
| 102 | size_t old_num_bytes_allocated = static_cast<size_t>(num_bytes_allocated_.fetch_add(size)); |
| 103 | |
| 104 | DCHECK(!Runtime::Current()->HasStatsEnabled()); |
| 105 | |
| 106 | // This is safe to do since the GC will never free objects which are neither in the allocation |
| 107 | // stack or the live bitmap. |
| 108 | while (!allocation_stack_->AtomicPushBack(obj)) { |
| 109 | CollectGarbageInternal(collector::kGcTypeSticky, kGcCauseForAlloc, false); |
| 110 | } |
| 111 | |
| 112 | return old_num_bytes_allocated + size; |
| 113 | } |
| 114 | |
| 115 | inline mirror::Object* Heap::TryToAllocateUninstrumented(Thread* self, space::AllocSpace* space, size_t alloc_size, |
| 116 | bool grow, size_t* bytes_allocated) { |
| 117 | if (UNLIKELY(IsOutOfMemoryOnAllocation(alloc_size, grow))) { |
| 118 | return NULL; |
| 119 | } |
| 120 | DCHECK(!running_on_valgrind_); |
| 121 | return space->Alloc(self, alloc_size, bytes_allocated); |
| 122 | } |
| 123 | |
| 124 | // DlMallocSpace-specific version. |
| 125 | inline mirror::Object* Heap::TryToAllocateUninstrumented(Thread* self, space::DlMallocSpace* space, size_t alloc_size, |
| 126 | bool grow, size_t* bytes_allocated) { |
| 127 | if (UNLIKELY(IsOutOfMemoryOnAllocation(alloc_size, grow))) { |
| 128 | return NULL; |
| 129 | } |
| 130 | DCHECK(!running_on_valgrind_); |
| 131 | return space->AllocNonvirtual(self, alloc_size, bytes_allocated); |
| 132 | } |
| 133 | |
| 134 | template <class T> |
| 135 | inline mirror::Object* Heap::AllocateUninstrumented(Thread* self, T* space, size_t alloc_size, |
| 136 | size_t* bytes_allocated) { |
| 137 | // Since allocation can cause a GC which will need to SuspendAll, make sure all allocations are |
| 138 | // done in the runnable state where suspension is expected. |
| 139 | DCHECK_EQ(self->GetState(), kRunnable); |
| 140 | self->AssertThreadSuspensionIsAllowable(); |
| 141 | |
| 142 | mirror::Object* ptr = TryToAllocateUninstrumented(self, space, alloc_size, false, bytes_allocated); |
| 143 | if (LIKELY(ptr != NULL)) { |
| 144 | return ptr; |
| 145 | } |
| 146 | return AllocateInternalWithGc(self, space, alloc_size, bytes_allocated); |
| 147 | } |
| 148 | |
| 149 | inline bool Heap::TryAllocLargeObjectUninstrumented(Thread* self, mirror::Class* c, size_t byte_count, |
| 150 | mirror::Object** obj_ptr, size_t* bytes_allocated) { |
| 151 | bool large_object_allocation = ShouldAllocLargeObject(c, byte_count); |
| 152 | if (UNLIKELY(large_object_allocation)) { |
| 153 | mirror::Object* obj = AllocateUninstrumented(self, large_object_space_, byte_count, bytes_allocated); |
| 154 | // Make sure that our large object didn't get placed anywhere within the space interval or else |
| 155 | // it breaks the immune range. |
| 156 | DCHECK(obj == NULL || |
| 157 | reinterpret_cast<byte*>(obj) < continuous_spaces_.front()->Begin() || |
| 158 | reinterpret_cast<byte*>(obj) >= continuous_spaces_.back()->End()); |
| 159 | *obj_ptr = obj; |
| 160 | } |
| 161 | return large_object_allocation; |
| 162 | } |
| 163 | |
Mathieu Chartier | 590fee9 | 2013-09-13 13:46:47 -0700 | [diff] [blame^] | 164 | inline void Heap::DebugCheckPreconditionsForAllocObject(mirror::Class* c, size_t byte_count) { |
Hiroshi Yamauchi | 3b4c189 | 2013-09-12 21:33:12 -0700 | [diff] [blame] | 165 | DCHECK(c == NULL || (c->IsClassClass() && byte_count >= sizeof(mirror::Class)) || |
| 166 | (c->IsVariableSize() || c->GetObjectSize() == byte_count) || |
Ian Rogers | dfb325e | 2013-10-30 01:00:44 -0700 | [diff] [blame] | 167 | strlen(ClassHelper(c).GetDescriptor()) == 0); |
Hiroshi Yamauchi | 3b4c189 | 2013-09-12 21:33:12 -0700 | [diff] [blame] | 168 | DCHECK_GE(byte_count, sizeof(mirror::Object)); |
| 169 | } |
| 170 | |
| 171 | inline Heap::AllocationTimer::AllocationTimer(Heap* heap, mirror::Object** allocated_obj_ptr) |
| 172 | : heap_(heap), allocated_obj_ptr_(allocated_obj_ptr) { |
| 173 | if (kMeasureAllocationTime) { |
| 174 | allocation_start_time_ = NanoTime() / kTimeAdjust; |
| 175 | } |
| 176 | } |
| 177 | |
| 178 | inline Heap::AllocationTimer::~AllocationTimer() { |
| 179 | if (kMeasureAllocationTime) { |
| 180 | mirror::Object* allocated_obj = *allocated_obj_ptr_; |
| 181 | // Only if the allocation succeeded, record the time. |
| 182 | if (allocated_obj != NULL) { |
| 183 | uint64_t allocation_end_time = NanoTime() / kTimeAdjust; |
| 184 | heap_->total_allocation_time_.fetch_add(allocation_end_time - allocation_start_time_); |
| 185 | } |
| 186 | } |
| 187 | }; |
| 188 | |
| 189 | inline bool Heap::ShouldAllocLargeObject(mirror::Class* c, size_t byte_count) { |
| 190 | // We need to have a zygote space or else our newly allocated large object can end up in the |
| 191 | // Zygote resulting in it being prematurely freed. |
| 192 | // We can only do this for primitive objects since large objects will not be within the card table |
| 193 | // range. This also means that we rely on SetClass not dirtying the object's card. |
| 194 | return byte_count >= kLargeObjectThreshold && have_zygote_space_ && c->IsPrimitiveArray(); |
| 195 | } |
| 196 | |
| 197 | inline bool Heap::IsOutOfMemoryOnAllocation(size_t alloc_size, bool grow) { |
| 198 | size_t new_footprint = num_bytes_allocated_ + alloc_size; |
| 199 | if (UNLIKELY(new_footprint > max_allowed_footprint_)) { |
| 200 | if (UNLIKELY(new_footprint > growth_limit_)) { |
| 201 | return true; |
| 202 | } |
| 203 | if (!concurrent_gc_) { |
| 204 | if (!grow) { |
| 205 | return true; |
| 206 | } else { |
| 207 | max_allowed_footprint_ = new_footprint; |
| 208 | } |
| 209 | } |
| 210 | } |
| 211 | return false; |
| 212 | } |
| 213 | |
| 214 | inline void Heap::CheckConcurrentGC(Thread* self, size_t new_num_bytes_allocated, mirror::Object* obj) { |
| 215 | if (UNLIKELY(new_num_bytes_allocated >= concurrent_start_bytes_)) { |
| 216 | // The SirtRef is necessary since the calls in RequestConcurrentGC are a safepoint. |
| 217 | SirtRef<mirror::Object> ref(self, obj); |
| 218 | RequestConcurrentGC(self); |
| 219 | } |
| 220 | } |
| 221 | |
| 222 | } // namespace gc |
| 223 | } // namespace art |
| 224 | |
| 225 | #endif // ART_RUNTIME_GC_HEAP_INL_H_ |