ART: Add FlushInstructionPipeline()
Use membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED), where available, to
flush CPU instruction pipelines after JIT code cache updates. This is
needed on architectures where TLB updates do not require a TLB
shootdown.
Bug: 65312375
Bug: 66095511
Bug: 111199492
Test: manual (requires kernel >= 4.16).
Change-Id: I96811c611133ba765a546a09432c0c951ad39e10
diff --git a/runtime/jit/jit_code_cache.cc b/runtime/jit/jit_code_cache.cc
index 2b2898c..461eb81 100644
--- a/runtime/jit/jit_code_cache.cc
+++ b/runtime/jit/jit_code_cache.cc
@@ -28,6 +28,7 @@
#include "base/stl_util.h"
#include "base/systrace.h"
#include "base/time_utils.h"
+#include "base/utils.h"
#include "cha.h"
#include "debugger_interface.h"
#include "dex/dex_file_loader.h"
@@ -53,8 +54,9 @@
namespace art {
namespace jit {
-static constexpr int kProtData = PROT_READ | PROT_WRITE;
static constexpr int kProtCode = PROT_READ | PROT_EXEC;
+static constexpr int kProtData = PROT_READ | PROT_WRITE;
+static constexpr int kProtProfile = PROT_READ;
static constexpr size_t kCodeSizeLogThreshold = 50 * KB;
static constexpr size_t kStackMapSizeLogThreshold = 50 * KB;
@@ -192,7 +194,7 @@
// to profile system server.
// NOTE 2: We could just not create the code section at all but we will need to
// special case too many cases.
- int memmap_flags_prot_code = used_only_for_profile_data ? (kProtCode & ~PROT_EXEC) : kProtCode;
+ int memmap_flags_prot_code = used_only_for_profile_data ? kProtProfile : kProtCode;
std::string error_str;
// Map name specific for android_os_Debug.cpp accounting.
@@ -801,6 +803,16 @@
// https://android.googlesource.com/kernel/msm/+/3fbe6bc28a6b9939d0650f2f17eb5216c719950c
FlushInstructionCache(reinterpret_cast<char*>(code_ptr),
reinterpret_cast<char*>(code_ptr + code_size));
+
+ // Ensure CPU instruction pipelines are flushed for all cores. This is necessary for
+ // correctness as code may still be in instruction pipelines despite the i-cache flush. It is
+ // not safe to assume that changing permissions with mprotect (RX->RWX->RX) will cause a TLB
+ // shootdown (incidentally invalidating the CPU pipelines by sending an IPI to all cores to
+ // notify them of the TLB invalidation). Some architectures, notably ARM and ARM64, have
+ // hardware support that broadcasts TLB invalidations and so their kernels have no software
+ // based TLB shootdown. FlushInstructionPipeline() is a wrapper around the Linux
+ // membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED) syscall which does the appropriate flushing.
+ FlushInstructionPipeline();
DCHECK(!Runtime::Current()->IsAotCompiler());
if (has_should_deoptimize_flag) {
method_header->SetHasShouldDeoptimizeFlag();