Move image intern table into image
Previously we recreated this intern table during runtime startup.
This added 50-100ms of boot time.
Fixed bug where we didn't copy over hashcodes into the image.
Deleted some stale code.
Bug: 20727525
Bug: 19569780
Change-Id: I08959e9aa2a73cedb52f393033e2ffea3a26e76b
diff --git a/runtime/base/hash_set.h b/runtime/base/hash_set.h
index ab63ddd..8daf6d4 100644
--- a/runtime/base/hash_set.h
+++ b/runtime/base/hash_set.h
@@ -22,6 +22,7 @@
#include <stdint.h>
#include <utility>
+#include "bit_utils.h"
#include "logging.h"
namespace art {
@@ -121,6 +122,7 @@
typedef BaseIterator<T, HashSet> Iterator;
typedef BaseIterator<const T, const HashSet> ConstIterator;
+ // If we don't own the data, this will create a new array which owns the data.
void Clear() {
DeallocateStorage();
AllocateStorage(1);
@@ -128,19 +130,70 @@
elements_until_expand_ = 0;
}
- HashSet() : num_elements_(0), num_buckets_(0), data_(nullptr),
+ HashSet() : num_elements_(0), num_buckets_(0), owns_data_(false), data_(nullptr),
min_load_factor_(kDefaultMinLoadFactor), max_load_factor_(kDefaultMaxLoadFactor) {
Clear();
}
- HashSet(const HashSet& other) : num_elements_(0), num_buckets_(0), data_(nullptr) {
+ HashSet(const HashSet& other) : num_elements_(0), num_buckets_(0), owns_data_(false),
+ data_(nullptr) {
*this = other;
}
- HashSet(HashSet&& other) : num_elements_(0), num_buckets_(0), data_(nullptr) {
+ HashSet(HashSet&& other) : num_elements_(0), num_buckets_(0), owns_data_(false),
+ data_(nullptr) {
*this = std::move(other);
}
+ // Construct from existing data.
+ // Read from a block of memory, if make_copy_of_data is false, then data_ points to within the
+ // passed in ptr_.
+ HashSet(const uint8_t* ptr, bool make_copy_of_data, size_t* read_count) {
+ uint64_t temp;
+ size_t offset = 0;
+ offset = ReadFromBytes(ptr, offset, &temp);
+ num_elements_ = static_cast<uint64_t>(temp);
+ offset = ReadFromBytes(ptr, offset, &temp);
+ num_buckets_ = static_cast<uint64_t>(temp);
+ CHECK_LE(num_elements_, num_buckets_);
+ offset = ReadFromBytes(ptr, offset, &temp);
+ elements_until_expand_ = static_cast<uint64_t>(temp);
+ offset = ReadFromBytes(ptr, offset, &min_load_factor_);
+ offset = ReadFromBytes(ptr, offset, &max_load_factor_);
+ if (!make_copy_of_data) {
+ owns_data_ = false;
+ data_ = const_cast<T*>(reinterpret_cast<const T*>(ptr + offset));
+ offset += sizeof(*data_) * num_buckets_;
+ } else {
+ AllocateStorage(num_buckets_);
+ // Write elements, not that this may not be safe for cross compilation if the elements are
+ // pointer sized.
+ for (size_t i = 0; i < num_buckets_; ++i) {
+ offset = ReadFromBytes(ptr, offset, &data_[i]);
+ }
+ }
+ // Caller responsible for aligning.
+ *read_count = offset;
+ }
+
+ // Returns how large the table is after being written. If target is null, then no writing happens
+ // but the size is still returned. Target must be 8 byte aligned.
+ size_t WriteToMemory(uint8_t* ptr) {
+ size_t offset = 0;
+ offset = WriteToBytes(ptr, offset, static_cast<uint64_t>(num_elements_));
+ offset = WriteToBytes(ptr, offset, static_cast<uint64_t>(num_buckets_));
+ offset = WriteToBytes(ptr, offset, static_cast<uint64_t>(elements_until_expand_));
+ offset = WriteToBytes(ptr, offset, min_load_factor_);
+ offset = WriteToBytes(ptr, offset, max_load_factor_);
+ // Write elements, not that this may not be safe for cross compilation if the elements are
+ // pointer sized.
+ for (size_t i = 0; i < num_buckets_; ++i) {
+ offset = WriteToBytes(ptr, offset, data_[i]);
+ }
+ // Caller responsible for aligning.
+ return offset;
+ }
+
~HashSet() {
DeallocateStorage();
}
@@ -152,6 +205,7 @@
std::swap(elements_until_expand_, other.elements_until_expand_);
std::swap(min_load_factor_, other.min_load_factor_);
std::swap(max_load_factor_, other.max_load_factor_);
+ std::swap(owns_data_, other.owns_data_);
return *this;
}
@@ -386,6 +440,7 @@
void AllocateStorage(size_t num_buckets) {
num_buckets_ = num_buckets;
data_ = allocfn_.allocate(num_buckets_);
+ owns_data_ = true;
for (size_t i = 0; i < num_buckets_; ++i) {
allocfn_.construct(allocfn_.address(data_[i]));
emptyfn_.MakeEmpty(data_[i]);
@@ -394,10 +449,13 @@
void DeallocateStorage() {
if (num_buckets_ != 0) {
- for (size_t i = 0; i < NumBuckets(); ++i) {
- allocfn_.destroy(allocfn_.address(data_[i]));
+ if (owns_data_) {
+ for (size_t i = 0; i < NumBuckets(); ++i) {
+ allocfn_.destroy(allocfn_.address(data_[i]));
+ }
+ allocfn_.deallocate(data_, NumBuckets());
+ owns_data_ = false;
}
- allocfn_.deallocate(data_, NumBuckets());
data_ = nullptr;
num_buckets_ = 0;
}
@@ -418,18 +476,23 @@
// Expand / shrink the table to the new specified size.
void Resize(size_t new_size) {
DCHECK_GE(new_size, Size());
- T* old_data = data_;
+ T* const old_data = data_;
size_t old_num_buckets = num_buckets_;
// Reinsert all of the old elements.
+ const bool owned_data = owns_data_;
AllocateStorage(new_size);
for (size_t i = 0; i < old_num_buckets; ++i) {
T& element = old_data[i];
if (!emptyfn_.IsEmpty(element)) {
data_[FirstAvailableSlot(IndexForHash(hashfn_(element)))] = std::move(element);
}
- allocfn_.destroy(allocfn_.address(element));
+ if (owned_data) {
+ allocfn_.destroy(allocfn_.address(element));
+ }
}
- allocfn_.deallocate(old_data, old_num_buckets);
+ if (owned_data) {
+ allocfn_.deallocate(old_data, old_num_buckets);
+ }
}
ALWAYS_INLINE size_t FirstAvailableSlot(size_t index) const {
@@ -439,6 +502,24 @@
return index;
}
+ // Return new offset.
+ template <typename Elem>
+ static size_t WriteToBytes(uint8_t* ptr, size_t offset, Elem n) {
+ DCHECK_ALIGNED(ptr + offset, sizeof(n));
+ if (ptr != nullptr) {
+ *reinterpret_cast<Elem*>(ptr + offset) = n;
+ }
+ return offset + sizeof(n);
+ }
+
+ template <typename Elem>
+ static size_t ReadFromBytes(const uint8_t* ptr, size_t offset, Elem* out) {
+ DCHECK(ptr != nullptr);
+ DCHECK_ALIGNED(ptr + offset, sizeof(*out));
+ *out = *reinterpret_cast<const Elem*>(ptr + offset);
+ return offset + sizeof(*out);
+ }
+
Alloc allocfn_; // Allocator function.
HashFn hashfn_; // Hashing function.
EmptyFn emptyfn_; // IsEmpty/SetEmpty function.
@@ -446,6 +527,7 @@
size_t num_elements_; // Number of inserted elements.
size_t num_buckets_; // Number of hash table buckets.
size_t elements_until_expand_; // Maxmimum number of elements until we expand the table.
+ bool owns_data_; // If we own data_ and are responsible for freeing it.
T* data_; // Backing storage.
double min_load_factor_;
double max_load_factor_;
diff --git a/runtime/class_linker.cc b/runtime/class_linker.cc
index 2a0e4e8..91812e7 100644
--- a/runtime/class_linker.cc
+++ b/runtime/class_linker.cc
@@ -1055,7 +1055,7 @@
static void SanityCheckObjectsCallback(mirror::Object* obj, void* arg ATTRIBUTE_UNUSED)
SHARED_LOCKS_REQUIRED(Locks::mutator_lock_) {
DCHECK(obj != nullptr);
- CHECK(obj->GetClass() != nullptr) << "Null class " << obj;
+ CHECK(obj->GetClass() != nullptr) << "Null class in object " << obj;
CHECK(obj->GetClass()->GetClass() != nullptr) << "Null class class " << obj;
if (obj->IsClass()) {
auto klass = obj->AsClass();
diff --git a/runtime/gc/accounting/space_bitmap-inl.h b/runtime/gc/accounting/space_bitmap-inl.h
index c16f5d3..006d2c7 100644
--- a/runtime/gc/accounting/space_bitmap-inl.h
+++ b/runtime/gc/accounting/space_bitmap-inl.h
@@ -159,6 +159,7 @@
inline bool SpaceBitmap<kAlignment>::Modify(const mirror::Object* obj) {
uintptr_t addr = reinterpret_cast<uintptr_t>(obj);
DCHECK_GE(addr, heap_begin_);
+ DCHECK(HasAddress(obj)) << obj;
const uintptr_t offset = addr - heap_begin_;
const size_t index = OffsetToIndex(offset);
const uintptr_t mask = OffsetToMask(offset);
diff --git a/runtime/gc/accounting/space_bitmap.cc b/runtime/gc/accounting/space_bitmap.cc
index fe2b284..6546eb4 100644
--- a/runtime/gc/accounting/space_bitmap.cc
+++ b/runtime/gc/accounting/space_bitmap.cc
@@ -35,6 +35,11 @@
}
template<size_t kAlignment>
+size_t SpaceBitmap<kAlignment>::ComputeHeapSize(uint64_t bitmap_bytes) {
+ return bitmap_bytes * kBitsPerByte * kAlignment;
+}
+
+template<size_t kAlignment>
SpaceBitmap<kAlignment>* SpaceBitmap<kAlignment>::CreateFromMemMap(
const std::string& name, MemMap* mem_map, uint8_t* heap_begin, size_t heap_capacity) {
CHECK(mem_map != nullptr);
diff --git a/runtime/gc/accounting/space_bitmap.h b/runtime/gc/accounting/space_bitmap.h
index d6b3ed4..35faff3 100644
--- a/runtime/gc/accounting/space_bitmap.h
+++ b/runtime/gc/accounting/space_bitmap.h
@@ -188,15 +188,16 @@
std::string Dump() const;
+ // Helper function for computing bitmap size based on a 64 bit capacity.
+ static size_t ComputeBitmapSize(uint64_t capacity);
+ static size_t ComputeHeapSize(uint64_t bitmap_bytes);
+
private:
// TODO: heap_end_ is initialized so that the heap bitmap is empty, this doesn't require the -1,
// however, we document that this is expected on heap_end_
SpaceBitmap(const std::string& name, MemMap* mem_map, uintptr_t* bitmap_begin, size_t bitmap_size,
const void* heap_begin);
- // Helper function for computing bitmap size based on a 64 bit capacity.
- static size_t ComputeBitmapSize(uint64_t capacity);
-
template<bool kSetBit>
bool Modify(const mirror::Object* obj);
diff --git a/runtime/gc/space/image_space.cc b/runtime/gc/space/image_space.cc
index 437fd8c..f7ceb84 100644
--- a/runtime/gc/space/image_space.cc
+++ b/runtime/gc/space/image_space.cc
@@ -694,7 +694,7 @@
const auto section_idx = static_cast<ImageHeader::ImageSections>(i);
auto& section = image_header.GetImageSection(section_idx);
LOG(INFO) << section_idx << " start="
- << reinterpret_cast<void*>(image_header.GetImageBegin() + section.Offset())
+ << reinterpret_cast<void*>(image_header.GetImageBegin() + section.Offset()) << " "
<< section;
}
}
@@ -730,9 +730,9 @@
std::string bitmap_name(StringPrintf("imagespace %s live-bitmap %u", image_filename,
bitmap_index));
std::unique_ptr<accounting::ContinuousSpaceBitmap> bitmap(
- accounting::ContinuousSpaceBitmap::CreateFromMemMap(bitmap_name, image_map.release(),
- reinterpret_cast<uint8_t*>(map->Begin()),
- map->Size()));
+ accounting::ContinuousSpaceBitmap::CreateFromMemMap(
+ bitmap_name, image_map.release(), reinterpret_cast<uint8_t*>(map->Begin()),
+ accounting::ContinuousSpaceBitmap::ComputeHeapSize(bitmap_section.Size())));
if (bitmap.get() == nullptr) {
*error_msg = StringPrintf("Could not create bitmap '%s'", bitmap_name.c_str());
return nullptr;
diff --git a/runtime/image.cc b/runtime/image.cc
index 947c914..44193da 100644
--- a/runtime/image.cc
+++ b/runtime/image.cc
@@ -24,7 +24,7 @@
namespace art {
const uint8_t ImageHeader::kImageMagic[] = { 'a', 'r', 't', '\n' };
-const uint8_t ImageHeader::kImageVersion[] = { '0', '1', '6', '\0' };
+const uint8_t ImageHeader::kImageVersion[] = { '0', '1', '7', '\0' };
ImageHeader::ImageHeader(uint32_t image_begin,
uint32_t image_size,
diff --git a/runtime/image.h b/runtime/image.h
index c6be7ef..d856f21 100644
--- a/runtime/image.h
+++ b/runtime/image.h
@@ -142,6 +142,7 @@
kSectionObjects,
kSectionArtFields,
kSectionArtMethods,
+ kSectionInternedStrings,
kSectionImageBitmap,
kSectionCount, // Number of elements in enum.
};
diff --git a/runtime/intern_table.cc b/runtime/intern_table.cc
index 9abbca8..2a96278 100644
--- a/runtime/intern_table.cc
+++ b/runtime/intern_table.cc
@@ -152,20 +152,28 @@
CHECK(image_space != nullptr);
MutexLock mu(Thread::Current(), *Locks::intern_table_lock_);
if (!image_added_to_intern_table_) {
- mirror::Object* root = image_space->GetImageHeader().GetImageRoot(ImageHeader::kDexCaches);
- mirror::ObjectArray<mirror::DexCache>* dex_caches = root->AsObjectArray<mirror::DexCache>();
- for (int32_t i = 0; i < dex_caches->GetLength(); ++i) {
- mirror::DexCache* dex_cache = dex_caches->Get(i);
- const DexFile* dex_file = dex_cache->GetDexFile();
- const size_t num_strings = dex_file->NumStringIds();
- for (size_t j = 0; j < num_strings; ++j) {
- mirror::String* image_string = dex_cache->GetResolvedString(j);
- if (image_string != nullptr) {
- mirror::String* found = LookupStrong(image_string);
- if (found == nullptr) {
- InsertStrong(image_string);
- } else {
- DCHECK_EQ(found, image_string);
+ const ImageHeader* const header = &image_space->GetImageHeader();
+ // Check if we have the interned strings section.
+ const ImageSection& section = header->GetImageSection(ImageHeader::kSectionInternedStrings);
+ if (section.Size() > 0) {
+ ReadFromMemoryLocked(image_space->Begin() + section.Offset());
+ } else {
+ // TODO: Delete this logic?
+ mirror::Object* root = header->GetImageRoot(ImageHeader::kDexCaches);
+ mirror::ObjectArray<mirror::DexCache>* dex_caches = root->AsObjectArray<mirror::DexCache>();
+ for (int32_t i = 0; i < dex_caches->GetLength(); ++i) {
+ mirror::DexCache* dex_cache = dex_caches->Get(i);
+ const DexFile* dex_file = dex_cache->GetDexFile();
+ const size_t num_strings = dex_file->NumStringIds();
+ for (size_t j = 0; j < num_strings; ++j) {
+ mirror::String* image_string = dex_cache->GetResolvedString(j);
+ if (image_string != nullptr) {
+ mirror::String* found = LookupStrong(image_string);
+ if (found == nullptr) {
+ InsertStrong(image_string);
+ } else {
+ DCHECK_EQ(found, image_string);
+ }
}
}
}
@@ -285,6 +293,29 @@
weak_interns_.SweepWeaks(callback, arg);
}
+void InternTable::AddImageInternTable(gc::space::ImageSpace* image_space) {
+ const ImageSection& intern_section = image_space->GetImageHeader().GetImageSection(
+ ImageHeader::kSectionInternedStrings);
+ // Read the string tables from the image.
+ const uint8_t* ptr = image_space->Begin() + intern_section.Offset();
+ const size_t offset = ReadFromMemory(ptr);
+ CHECK_LE(offset, intern_section.Size());
+}
+
+size_t InternTable::ReadFromMemory(const uint8_t* ptr) {
+ MutexLock mu(Thread::Current(), *Locks::intern_table_lock_);
+ return ReadFromMemoryLocked(ptr);
+}
+
+size_t InternTable::ReadFromMemoryLocked(const uint8_t* ptr) {
+ return strong_interns_.ReadIntoPreZygoteTable(ptr);
+}
+
+size_t InternTable::WriteToMemory(uint8_t* ptr) {
+ MutexLock mu(Thread::Current(), *Locks::intern_table_lock_);
+ return strong_interns_.WriteFromPostZygoteTable(ptr);
+}
+
std::size_t InternTable::StringHashEquals::operator()(const GcRoot<mirror::String>& root) const {
if (kIsDebugBuild) {
Locks::mutator_lock_->AssertSharedHeld(Thread::Current());
@@ -300,6 +331,17 @@
return a.Read()->Equals(b.Read());
}
+size_t InternTable::Table::ReadIntoPreZygoteTable(const uint8_t* ptr) {
+ CHECK_EQ(pre_zygote_table_.Size(), 0u);
+ size_t read_count = 0;
+ pre_zygote_table_ = UnorderedSet(ptr, false /* make copy */, &read_count);
+ return read_count;
+}
+
+size_t InternTable::Table::WriteFromPostZygoteTable(uint8_t* ptr) {
+ return post_zygote_table_.WriteToMemory(ptr);
+}
+
void InternTable::Table::Remove(mirror::String* s) {
auto it = post_zygote_table_.Find(GcRoot<mirror::String>(s));
if (it != post_zygote_table_.end()) {
@@ -325,9 +367,13 @@
}
void InternTable::Table::SwapPostZygoteWithPreZygote() {
- CHECK(pre_zygote_table_.Empty());
- std::swap(pre_zygote_table_, post_zygote_table_);
- VLOG(heap) << "Swapping " << pre_zygote_table_.Size() << " interns to the pre zygote table";
+ if (pre_zygote_table_.Empty()) {
+ std::swap(pre_zygote_table_, post_zygote_table_);
+ VLOG(heap) << "Swapping " << pre_zygote_table_.Size() << " interns to the pre zygote table";
+ } else {
+ // This case happens if read the intern table from the image.
+ VLOG(heap) << "Not swapping due to non-empty pre_zygote_table_";
+ }
}
void InternTable::Table::Insert(mirror::String* s) {
diff --git a/runtime/intern_table.h b/runtime/intern_table.h
index 1e5d3c2..97ce73c 100644
--- a/runtime/intern_table.h
+++ b/runtime/intern_table.h
@@ -97,6 +97,20 @@
void SwapPostZygoteWithPreZygote()
SHARED_LOCKS_REQUIRED(Locks::mutator_lock_) LOCKS_EXCLUDED(Locks::intern_table_lock_);
+ // Add an intern table which was serialized to the image.
+ void AddImageInternTable(gc::space::ImageSpace* image_space)
+ SHARED_LOCKS_REQUIRED(Locks::mutator_lock_) LOCKS_EXCLUDED(Locks::intern_table_lock_);
+
+ // Read the intern table from memory. The elements aren't copied, the intern hash set data will
+ // point to somewhere within ptr. Only reads the strong interns.
+ size_t ReadFromMemory(const uint8_t* ptr) LOCKS_EXCLUDED(Locks::intern_table_lock_)
+ SHARED_LOCKS_REQUIRED(Locks::mutator_lock_);
+
+ // Write the post zygote intern table to a pointer. Only writes the strong interns since it is
+ // expected that there is no weak interns since this is called from the image writer.
+ size_t WriteToMemory(uint8_t* ptr) SHARED_LOCKS_REQUIRED(Locks::mutator_lock_)
+ LOCKS_EXCLUDED(Locks::intern_table_lock_);
+
private:
class StringHashEquals {
public:
@@ -133,6 +147,16 @@
EXCLUSIVE_LOCKS_REQUIRED(Locks::intern_table_lock_);
void SwapPostZygoteWithPreZygote() EXCLUSIVE_LOCKS_REQUIRED(Locks::intern_table_lock_);
size_t Size() const EXCLUSIVE_LOCKS_REQUIRED(Locks::intern_table_lock_);
+ // Read pre zygote table is called from ReadFromMemory which happens during runtime creation
+ // when we load the image intern table. Returns how many bytes were read.
+ size_t ReadIntoPreZygoteTable(const uint8_t* ptr)
+ EXCLUSIVE_LOCKS_REQUIRED(Locks::intern_table_lock_)
+ SHARED_LOCKS_REQUIRED(Locks::mutator_lock_);
+ // The image writer calls WritePostZygoteTable through WriteToMemory, it writes the interns in
+ // the post zygote table. Returns how many bytes were written.
+ size_t WriteFromPostZygoteTable(uint8_t* ptr)
+ EXCLUSIVE_LOCKS_REQUIRED(Locks::intern_table_lock_)
+ SHARED_LOCKS_REQUIRED(Locks::mutator_lock_);
private:
typedef HashSet<GcRoot<mirror::String>, GcRootEmptyFn, StringHashEquals, StringHashEquals,
@@ -192,6 +216,10 @@
EXCLUSIVE_LOCKS_REQUIRED(Locks::intern_table_lock_);
friend class Transaction;
+ size_t ReadFromMemoryLocked(const uint8_t* ptr)
+ EXCLUSIVE_LOCKS_REQUIRED(Locks::intern_table_lock_)
+ SHARED_LOCKS_REQUIRED(Locks::mutator_lock_);
+
bool image_added_to_intern_table_ GUARDED_BY(Locks::intern_table_lock_);
bool log_new_roots_ GUARDED_BY(Locks::intern_table_lock_);
bool allow_new_interns_ GUARDED_BY(Locks::intern_table_lock_);