runtime/jit/jit_code_cache.cc

/*
 * Copyright 2014 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "jit_code_cache.h"

#include <sstream>

#include "art_method-inl.h"
#include "entrypoints/runtime_asm_entrypoints.h"
#include "gc/accounting/bitmap-inl.h"
#include "jit/profiling_info.h"
#include "linear_alloc.h"
#include "mem_map.h"
#include "oat_file-inl.h"
#include "scoped_thread_state_change.h"
#include "thread_list.h"

namespace art {
namespace jit {

static constexpr int kProtAll = PROT_READ | PROT_WRITE | PROT_EXEC;
static constexpr int kProtData = PROT_READ | PROT_WRITE;
static constexpr int kProtCode = PROT_READ | PROT_EXEC;

#define CHECKED_MPROTECT(memory, size, prot)                \
  do {                                                      \
    int rc = mprotect(memory, size, prot);                  \
    if (UNLIKELY(rc != 0)) {                                \
      errno = rc;                                           \
      PLOG(FATAL) << "Failed to mprotect jit code cache";   \
    }                                                       \
  } while (false)                                           \

JitCodeCache* JitCodeCache::Create(size_t capacity, std::string* error_msg) {
  CHECK_GT(capacity, 0U);
  CHECK_LT(capacity, kMaxCapacity);
  std::string error_str;
  // Map name specific for android_os_Debug.cpp accounting.
  MemMap* data_map = MemMap::MapAnonymous(
    "data-code-cache", nullptr, capacity, kProtAll, false, false, &error_str);
  if (data_map == nullptr) {
    std::ostringstream oss;
    oss << "Failed to create read write execute cache: " << error_str << " size=" << capacity;
    *error_msg = oss.str();
    return nullptr;
  }

  // Data cache is 1 / 2 of the map.
  // TODO: Make this variable?
  size_t data_size = RoundUp(data_map->Size() / 2, kPageSize);
  size_t code_size = data_map->Size() - data_size;
  uint8_t* divider = data_map->Begin() + data_size;

  // We need to have 32 bit offsets from method headers in code cache which point to things
  // in the data cache. If the maps are more than 4G apart, having multiple maps wouldn't work.
  MemMap* code_map = data_map->RemapAtEnd(divider, "jit-code-cache", kProtAll, &error_str);
  if (code_map == nullptr) {
    std::ostringstream oss;
    oss << "Failed to create read write execute cache: " << error_str << " size=" << capacity;
    *error_msg = oss.str();
    return nullptr;
  }
  DCHECK_EQ(code_map->Size(), code_size);
  DCHECK_EQ(code_map->Begin(), divider);
  return new JitCodeCache(code_map, data_map);
}

JitCodeCache::JitCodeCache(MemMap* code_map, MemMap* data_map)
    : lock_("Jit code cache", kJitCodeCacheLock),
      lock_cond_("Jit code cache variable", lock_),
      collection_in_progress_(false),
      code_map_(code_map),
      data_map_(data_map) {

  code_mspace_ = create_mspace_with_base(code_map_->Begin(), code_map_->Size(), false /*locked*/);
  data_mspace_ = create_mspace_with_base(data_map_->Begin(), data_map_->Size(), false /*locked*/);

  if (code_mspace_ == nullptr || data_mspace_ == nullptr) {
    PLOG(FATAL) << "create_mspace_with_base failed";
  }

  // Prevent morecore requests from the mspace.
  mspace_set_footprint_limit(code_mspace_, code_map_->Size());
  mspace_set_footprint_limit(data_mspace_, data_map_->Size());

  CHECKED_MPROTECT(code_map_->Begin(), code_map_->Size(), kProtCode);
  CHECKED_MPROTECT(data_map_->Begin(), data_map_->Size(), kProtData);

  live_bitmap_.reset(CodeCacheBitmap::Create("code-cache-bitmap",
                                             reinterpret_cast<uintptr_t>(code_map_->Begin()),
                                             reinterpret_cast<uintptr_t>(code_map_->End())));

  if (live_bitmap_.get() == nullptr) {
    PLOG(FATAL) << "creating bitmaps for the JIT code cache failed";
  }

  VLOG(jit) << "Created jit code cache: data size="
            << PrettySize(data_map_->Size())
            << ", code size="
            << PrettySize(code_map_->Size());
}

bool JitCodeCache::ContainsPc(const void* ptr) const {
  return code_map_->Begin() <= ptr && ptr < code_map_->End();
}

class ScopedCodeCacheWrite {
 public:
  explicit ScopedCodeCacheWrite(MemMap* code_map) : code_map_(code_map) {
    CHECKED_MPROTECT(code_map_->Begin(), code_map_->Size(), kProtAll);
  }
  ~ScopedCodeCacheWrite() {
    CHECKED_MPROTECT(code_map_->Begin(), code_map_->Size(), kProtCode);
  }
 private:
  MemMap* const code_map_;

  DISALLOW_COPY_AND_ASSIGN(ScopedCodeCacheWrite);
};

uint8_t* JitCodeCache::CommitCode(Thread* self,
                                  ArtMethod* method,
                                  const uint8_t* mapping_table,
                                  const uint8_t* vmap_table,
                                  const uint8_t* gc_map,
                                  size_t frame_size_in_bytes,
                                  size_t core_spill_mask,
                                  size_t fp_spill_mask,
                                  const uint8_t* code,
                                  size_t code_size) {
  uint8_t* result = CommitCodeInternal(self,
                                       method,
                                       mapping_table,
                                       vmap_table,
                                       gc_map,
                                       frame_size_in_bytes,
                                       core_spill_mask,
                                       fp_spill_mask,
                                       code,
                                       code_size);
  if (result == nullptr) {
    // Retry.
    GarbageCollectCache(self);
    result = CommitCodeInternal(self,
                                method,
                                mapping_table,
                                vmap_table,
                                gc_map,
                                frame_size_in_bytes,
                                core_spill_mask,
                                fp_spill_mask,
                                code,
                                code_size);
  }
  return result;
}

bool JitCodeCache::WaitForPotentialCollectionToComplete(Thread* self) {
  bool in_collection = false;
  while (collection_in_progress_) {
    in_collection = true;
    lock_cond_.Wait(self);
  }
  return in_collection;
}

static uintptr_t FromCodeToAllocation(const void* code) {
  size_t alignment = GetInstructionSetAlignment(kRuntimeISA);
  return reinterpret_cast<uintptr_t>(code) - RoundUp(sizeof(OatQuickMethodHeader), alignment);
}

void JitCodeCache::FreeCode(const void* code_ptr, ArtMethod* method ATTRIBUTE_UNUSED) {
  uintptr_t allocation = FromCodeToAllocation(code_ptr);
  const OatQuickMethodHeader* method_header = OatQuickMethodHeader::FromCodePointer(code_ptr);
  const uint8_t* data = method_header->GetNativeGcMap();
  if (data != nullptr) {
    mspace_free(data_mspace_, const_cast<uint8_t*>(data));
  }
  data = method_header->GetMappingTable();
  if (data != nullptr) {
    mspace_free(data_mspace_, const_cast<uint8_t*>(data));
  }
  // Use the offset directly to prevent sanity check that the method is
  // compiled with optimizing.
  // TODO(ngeoffray): Clean up.
  if (method_header->vmap_table_offset_ != 0) {
    data = method_header->code_ - method_header->vmap_table_offset_;
    mspace_free(data_mspace_, const_cast<uint8_t*>(data));
  }
  mspace_free(code_mspace_, reinterpret_cast<uint8_t*>(allocation));
}

void JitCodeCache::RemoveMethodsIn(Thread* self, const LinearAlloc& alloc) {
  MutexLock mu(self, lock_);
  // We do not check if a code cache GC is in progress, as this method comes
  // with the classlinker_classes_lock_ held, and suspending ourselves could
  // lead to a deadlock.
  {
    ScopedCodeCacheWrite scc(code_map_.get());
    for (auto it = method_code_map_.begin(); it != method_code_map_.end();) {
      if (alloc.ContainsUnsafe(it->second)) {
        FreeCode(it->first, it->second);
        it = method_code_map_.erase(it);
      } else {
        ++it;
      }
    }
  }
  for (auto it = profiling_infos_.begin(); it != profiling_infos_.end();) {
    ProfilingInfo* info = *it;
    if (alloc.ContainsUnsafe(info->GetMethod())) {
      info->GetMethod()->SetProfilingInfo(nullptr);
      mspace_free(data_mspace_, reinterpret_cast<uint8_t*>(info));
      it = profiling_infos_.erase(it);
    } else {
      ++it;
    }
  }
}

uint8_t* JitCodeCache::CommitCodeInternal(Thread* self,
                                          ArtMethod* method,
                                          const uint8_t* mapping_table,
                                          const uint8_t* vmap_table,
                                          const uint8_t* gc_map,
                                          size_t frame_size_in_bytes,
                                          size_t core_spill_mask,
                                          size_t fp_spill_mask,
                                          const uint8_t* code,
                                          size_t code_size) {
  size_t alignment = GetInstructionSetAlignment(kRuntimeISA);
  // Ensure the header ends up at expected instruction alignment.
  size_t header_size = RoundUp(sizeof(OatQuickMethodHeader), alignment);
  size_t total_size = header_size + code_size;

  OatQuickMethodHeader* method_header = nullptr;
  uint8_t* code_ptr = nullptr;
  {
    ScopedThreadSuspension sts(self, kSuspended);
    MutexLock mu(self, lock_);
    WaitForPotentialCollectionToComplete(self);
    {
      ScopedCodeCacheWrite scc(code_map_.get());
      uint8_t* result = reinterpret_cast<uint8_t*>(
          mspace_memalign(code_mspace_, alignment, total_size));
      if (result == nullptr) {
        return nullptr;
      }
      code_ptr = result + header_size;
      DCHECK_ALIGNED_PARAM(reinterpret_cast<uintptr_t>(code_ptr), alignment);

      std::copy(code, code + code_size, code_ptr);
      method_header = OatQuickMethodHeader::FromCodePointer(code_ptr);
      new (method_header) OatQuickMethodHeader(
          (mapping_table == nullptr) ? 0 : code_ptr - mapping_table,
          (vmap_table == nullptr) ? 0 : code_ptr - vmap_table,
          (gc_map == nullptr) ? 0 : code_ptr - gc_map,
          frame_size_in_bytes,
          core_spill_mask,
          fp_spill_mask,
          code_size);
    }

    __builtin___clear_cache(reinterpret_cast<char*>(code_ptr),
                            reinterpret_cast<char*>(code_ptr + code_size));
    method_code_map_.Put(code_ptr, method);
    // We have checked there was no collection in progress earlier. If we
    // were, setting the entry point of a method would be unsafe, as the collection
    // could delete it.
    DCHECK(!collection_in_progress_);
    method->SetEntryPointFromQuickCompiledCode(method_header->GetEntryPoint());
  }
  VLOG(jit)
      << "JIT added "
      << PrettyMethod(method) << "@" << method
      << " ccache_size=" << PrettySize(CodeCacheSize()) << ": "
      << " dcache_size=" << PrettySize(DataCacheSize()) << ": "
      << reinterpret_cast<const void*>(method_header->GetEntryPoint()) << ","
      << reinterpret_cast<const void*>(method_header->GetEntryPoint() + method_header->code_size_);

  return reinterpret_cast<uint8_t*>(method_header);
}

size_t JitCodeCache::CodeCacheSize() {
  MutexLock mu(Thread::Current(), lock_);
  size_t bytes_allocated = 0;
  mspace_inspect_all(code_mspace_, DlmallocBytesAllocatedCallback, &bytes_allocated);
  return bytes_allocated;
}

size_t JitCodeCache::DataCacheSize() {
  MutexLock mu(Thread::Current(), lock_);
  size_t bytes_allocated = 0;
  mspace_inspect_all(data_mspace_, DlmallocBytesAllocatedCallback, &bytes_allocated);
  return bytes_allocated;
}

size_t JitCodeCache::NumberOfCompiledCode() {
  MutexLock mu(Thread::Current(), lock_);
  return method_code_map_.size();
}

void JitCodeCache::ClearData(Thread* self, void* data) {
  MutexLock mu(self, lock_);
  mspace_free(data_mspace_, data);
}

uint8_t* JitCodeCache::ReserveData(Thread* self, size_t size) {
  size = RoundUp(size, sizeof(void*));
  uint8_t* result = nullptr;

  {
    ScopedThreadSuspension sts(self, kSuspended);
    MutexLock mu(self, lock_);
    WaitForPotentialCollectionToComplete(self);
    result = reinterpret_cast<uint8_t*>(mspace_malloc(data_mspace_, size));
  }

  if (result == nullptr) {
    // Retry.
    GarbageCollectCache(self);
    ScopedThreadSuspension sts(self, kSuspended);
    MutexLock mu(self, lock_);
    WaitForPotentialCollectionToComplete(self);
    result = reinterpret_cast<uint8_t*>(mspace_malloc(data_mspace_, size));
  }

  return result;
}

uint8_t* JitCodeCache::AddDataArray(Thread* self, const uint8_t* begin, const uint8_t* end) {
  uint8_t* result = ReserveData(self, end - begin);
  if (result == nullptr) {
    return nullptr;  // Out of space in the data cache.
  }
  std::copy(begin, end, result);
  return result;
}

class MarkCodeVisitor FINAL : public StackVisitor {
 public:
  MarkCodeVisitor(Thread* thread_in, JitCodeCache* code_cache_in)
      : StackVisitor(thread_in, nullptr, StackVisitor::StackWalkKind::kSkipInlinedFrames),
        code_cache_(code_cache_in),
        bitmap_(code_cache_->GetLiveBitmap()) {}

  bool VisitFrame() OVERRIDE SHARED_REQUIRES(Locks::mutator_lock_) {
    const OatQuickMethodHeader* method_header = GetCurrentOatQuickMethodHeader();
    if (method_header == nullptr) {
      return true;
    }
    const void* code = method_header->GetCode();
    if (code_cache_->ContainsPc(code)) {
      // Use the atomic set version, as multiple threads are executing this code.
      bitmap_->AtomicTestAndSet(FromCodeToAllocation(code));
    }
    return true;
  }

 private:
  JitCodeCache* const code_cache_;
  CodeCacheBitmap* const bitmap_;
};

class MarkCodeClosure FINAL : public Closure {
 public:
  MarkCodeClosure(JitCodeCache* code_cache, Barrier* barrier)
      : code_cache_(code_cache), barrier_(barrier) {}

  void Run(Thread* thread) OVERRIDE SHARED_REQUIRES(Locks::mutator_lock_) {
    DCHECK(thread == Thread::Current() || thread->IsSuspended());
    MarkCodeVisitor visitor(thread, code_cache_);
    visitor.WalkStack();
    if (kIsDebugBuild) {
      // The stack walking code queries the side instrumentation stack if it
      // sees an instrumentation exit pc, so the JIT code of methods in that stack
      // must have been seen. We sanity check this below.
      for (const instrumentation::InstrumentationStackFrame& frame
              : *thread->GetInstrumentationStack()) {
        // The 'method_' in InstrumentationStackFrame is the one that has return_pc_ in
        // its stack frame, it is not the method owning return_pc_. We just pass null to
        // LookupMethodHeader: the method is only checked against in debug builds.
        OatQuickMethodHeader* method_header =
            code_cache_->LookupMethodHeader(frame.return_pc_, nullptr);
        if (method_header != nullptr) {
          const void* code = method_header->GetCode();
          CHECK(code_cache_->GetLiveBitmap()->Test(FromCodeToAllocation(code)));
        }
      }
    }
    barrier_->Pass(Thread::Current());
  }

 private:
  JitCodeCache* const code_cache_;
  Barrier* const barrier_;
};

void JitCodeCache::GarbageCollectCache(Thread* self) {
  if (!kIsDebugBuild || VLOG_IS_ON(jit)) {
    LOG(INFO) << "Clearing code cache, code="
              << PrettySize(CodeCacheSize())
              << ", data=" << PrettySize(DataCacheSize());
  }

  size_t map_size = 0;
  ScopedThreadSuspension sts(self, kSuspended);

  // Walk over all compiled methods and set the entry points of these
  // methods to interpreter.
  {
    MutexLock mu(self, lock_);
    if (WaitForPotentialCollectionToComplete(self)) {
      return;
    }
    collection_in_progress_ = true;
    map_size = method_code_map_.size();
    for (auto& it : method_code_map_) {
      it.second->SetEntryPointFromQuickCompiledCode(GetQuickToInterpreterBridge());
    }
    for (ProfilingInfo* info : profiling_infos_) {
      info->GetMethod()->SetProfilingInfo(nullptr);
    }
  }

  // Run a checkpoint on all threads to mark the JIT compiled code they are running.
  {
    Barrier barrier(0);
    size_t threads_running_checkpoint = 0;
    {
      // Walking the stack requires the mutator lock.
      // We only take the lock when running the checkpoint and not waiting so that
      // when we go back to suspended, we can execute checkpoints that were requested
      // concurrently, and then move to waiting for our own checkpoint to finish.
      ScopedObjectAccess soa(self);
      MarkCodeClosure closure(this, &barrier);
      threads_running_checkpoint =
          Runtime::Current()->GetThreadList()->RunCheckpoint(&closure);
    }
    if (threads_running_checkpoint != 0) {
      barrier.Increment(self, threads_running_checkpoint);
    }
  }

  {
    MutexLock mu(self, lock_);
    DCHECK_EQ(map_size, method_code_map_.size());
    // Free unused compiled code, and restore the entry point of used compiled code.
    {
      ScopedCodeCacheWrite scc(code_map_.get());
      for (auto it = method_code_map_.begin(); it != method_code_map_.end();) {
        const void* code_ptr = it->first;
        ArtMethod* method = it->second;
        uintptr_t allocation = FromCodeToAllocation(code_ptr);
        const OatQuickMethodHeader* method_header = OatQuickMethodHeader::FromCodePointer(code_ptr);
        if (GetLiveBitmap()->Test(allocation)) {
          method->SetEntryPointFromQuickCompiledCode(method_header->GetEntryPoint());
          ++it;
        } else {
          method->ClearCounter();
          DCHECK_NE(method->GetEntryPointFromQuickCompiledCode(), method_header->GetEntryPoint());
          FreeCode(code_ptr, method);
          it = method_code_map_.erase(it);
        }
      }
    }
    GetLiveBitmap()->Bitmap::Clear();

    // Free all profiling info.
    for (ProfilingInfo* info : profiling_infos_) {
      DCHECK(info->GetMethod()->GetProfilingInfo(sizeof(void*)) == nullptr);
      mspace_free(data_mspace_, reinterpret_cast<uint8_t*>(info));
    }
    profiling_infos_.clear();

    collection_in_progress_ = false;
    lock_cond_.Broadcast(self);
  }

  if (!kIsDebugBuild || VLOG_IS_ON(jit)) {
    LOG(INFO) << "After clearing code cache, code="
              << PrettySize(CodeCacheSize())
              << ", data=" << PrettySize(DataCacheSize());
  }
}


OatQuickMethodHeader* JitCodeCache::LookupMethodHeader(uintptr_t pc, ArtMethod* method) {
  static_assert(kRuntimeISA != kThumb2, "kThumb2 cannot be a runtime ISA");
  if (kRuntimeISA == kArm) {
    // On Thumb-2, the pc is offset by one.
    --pc;
  }
  if (!ContainsPc(reinterpret_cast<const void*>(pc))) {
    return nullptr;
  }

  MutexLock mu(Thread::Current(), lock_);
  if (method_code_map_.empty()) {
    return nullptr;
  }
  auto it = method_code_map_.lower_bound(reinterpret_cast<const void*>(pc));
  --it;

  const void* code_ptr = it->first;
  OatQuickMethodHeader* method_header = OatQuickMethodHeader::FromCodePointer(code_ptr);
  if (!method_header->Contains(pc)) {
    return nullptr;
  }
  if (kIsDebugBuild && method != nullptr) {
    DCHECK_EQ(it->second, method)
        << PrettyMethod(method) << " " << PrettyMethod(it->second) << " " << std::hex << pc;
  }
  return method_header;
}

ProfilingInfo* JitCodeCache::AddProfilingInfo(Thread* self,
                                              ArtMethod* method,
                                              const std::vector<uint32_t>& entries,
                                              bool retry_allocation) {
  ProfilingInfo* info = AddProfilingInfoInternal(self, method, entries);

  if (info == nullptr && retry_allocation) {
    GarbageCollectCache(self);
    info = AddProfilingInfoInternal(self, method, entries);
  }
  return info;
}

ProfilingInfo* JitCodeCache::AddProfilingInfoInternal(Thread* self,
                                                      ArtMethod* method,
                                                      const std::vector<uint32_t>& entries) {
  size_t profile_info_size = RoundUp(
      sizeof(ProfilingInfo) + sizeof(ProfilingInfo::InlineCache) * entries.size(),
      sizeof(void*));
  ScopedThreadSuspension sts(self, kSuspended);
  MutexLock mu(self, lock_);
  WaitForPotentialCollectionToComplete(self);

  // Check whether some other thread has concurrently created it.
  ProfilingInfo* info = method->GetProfilingInfo(sizeof(void*));
  if (info != nullptr) {
    return info;
  }

  uint8_t* data = reinterpret_cast<uint8_t*>(mspace_malloc(data_mspace_, profile_info_size));
  if (data == nullptr) {
    return nullptr;
  }
  info = new (data) ProfilingInfo(method, entries);
  method->SetProfilingInfo(info);
  profiling_infos_.push_back(info);
  return info;
}

}  // namespace jit
}  // namespace art