Introduce VDEX file, use it for DEX files
This patch introduces a new output file called VDEX. In the future,
VDEX files will store pre-validated DEX files which do not need to be
re-extracted and re-verified when recompiling, e.g. due to new
profiling information or after a system update.
With this CL, the OatWriter writes DEX files into the VDEX and the
rest of its output into OAT. The OatFile class and related classes
are updated to load the VDEX at runtime and mmap the DEX file section
from it. Patchoat creates symlinks to the source VDEX files in the
target directory or copies the files if passed in as file descriptors.
The feature can be disabled by setting the environment variable
ART_ENABLE_VDEX to false.
Test: m test-art-host
Bug: 30937355
Change-Id: I54dcaececf6814c258c80524ec15e2e2ef69c8dd
diff --git a/runtime/Android.bp b/runtime/Android.bp
index 59e4a15..a884505 100644
--- a/runtime/Android.bp
+++ b/runtime/Android.bp
@@ -190,6 +190,7 @@
"type_lookup_table.cc",
"utf.cc",
"utils.cc",
+ "vdex_file.cc",
"verifier/instruction_flags.cc",
"verifier/method_verifier.cc",
"verifier/reg_type.cc",
diff --git a/runtime/globals.h b/runtime/globals.h
index aba5661..691bf55 100644
--- a/runtime/globals.h
+++ b/runtime/globals.h
@@ -166,6 +166,12 @@
static constexpr bool kArm32QuickCodeUseSoftFloat = false;
+#ifdef ART_ENABLE_VDEX
+static constexpr bool kIsVdexEnabled = true;
+#else
+static constexpr bool kIsVdexEnabled = false;
+#endif
+
} // namespace art
#endif // ART_RUNTIME_GLOBALS_H_
diff --git a/runtime/image.h b/runtime/image.h
index 3a4fa79..da9976a 100644
--- a/runtime/image.h
+++ b/runtime/image.h
@@ -168,13 +168,11 @@
}
static std::string GetOatLocationFromImageLocation(const std::string& image) {
- std::string oat_filename = image;
- if (oat_filename.length() <= 3) {
- oat_filename += ".oat";
- } else {
- oat_filename.replace(oat_filename.length() - 3, 3, "oat");
- }
- return oat_filename;
+ return GetLocationFromImageLocation(image, "oat");
+ }
+
+ static std::string GetVdexLocationFromImageLocation(const std::string& image) {
+ return GetLocationFromImageLocation(image, "vdex");
}
enum ImageMethod {
@@ -299,6 +297,17 @@
static const uint8_t kImageMagic[4];
static const uint8_t kImageVersion[4];
+ static std::string GetLocationFromImageLocation(const std::string& image,
+ const std::string& extension) {
+ std::string filename = image;
+ if (filename.length() <= 3) {
+ filename += "." + extension;
+ } else {
+ filename.replace(filename.length() - 3, 3, extension);
+ }
+ return filename;
+ }
+
uint8_t magic_[4];
uint8_t version_[4];
diff --git a/runtime/oat_file.cc b/runtime/oat_file.cc
index 5752fd9..cbc5d3c 100644
--- a/runtime/oat_file.cc
+++ b/runtime/oat_file.cc
@@ -83,7 +83,8 @@
virtual ~OatFileBase() {}
template <typename kOatFileBaseSubType>
- static OatFileBase* OpenOatFile(const std::string& elf_filename,
+ static OatFileBase* OpenOatFile(const std::string& vdex_filename,
+ const std::string& elf_filename,
const std::string& location,
uint8_t* requested_base,
uint8_t* oat_file_begin,
@@ -101,6 +102,11 @@
virtual void PreLoad() = 0;
+ bool LoadVdex(const std::string& vdex_filename,
+ bool writable,
+ bool low_4gb,
+ std::string* error_msg);
+
virtual bool Load(const std::string& elf_filename,
uint8_t* oat_file_begin,
bool writable,
@@ -131,7 +137,8 @@
};
template <typename kOatFileBaseSubType>
-OatFileBase* OatFileBase::OpenOatFile(const std::string& elf_filename,
+OatFileBase* OatFileBase::OpenOatFile(const std::string& vdex_filename,
+ const std::string& elf_filename,
const std::string& location,
uint8_t* requested_base,
uint8_t* oat_file_begin,
@@ -144,6 +151,10 @@
ret->PreLoad();
+ if (kIsVdexEnabled && !ret->LoadVdex(vdex_filename, writable, low_4gb, error_msg)) {
+ return nullptr;
+ }
+
if (!ret->Load(elf_filename,
oat_file_begin,
writable,
@@ -166,6 +177,20 @@
return ret.release();
}
+bool OatFileBase::LoadVdex(const std::string& vdex_filename,
+ bool writable,
+ bool low_4gb,
+ std::string* error_msg) {
+ vdex_.reset(VdexFile::Open(vdex_filename, writable, low_4gb, error_msg));
+ if (vdex_.get() == nullptr) {
+ *error_msg = StringPrintf("Failed to load vdex file '%s' %s",
+ vdex_filename.c_str(),
+ error_msg->c_str());
+ return false;
+ }
+ return true;
+}
+
bool OatFileBase::ComputeFields(uint8_t* requested_base,
const std::string& file_path,
std::string* error_msg) {
@@ -321,29 +346,29 @@
dex_file_location.c_str());
return false;
}
- if (UNLIKELY(dex_file_offset > Size())) {
+ if (UNLIKELY(dex_file_offset > DexSize())) {
*error_msg = StringPrintf("In oat file '%s' found OatDexFile #%zu for '%s' with dex file "
"offset %u > %zu",
GetLocation().c_str(),
i,
dex_file_location.c_str(),
dex_file_offset,
- Size());
+ DexSize());
return false;
}
- if (UNLIKELY(Size() - dex_file_offset < sizeof(DexFile::Header))) {
+ if (UNLIKELY(DexSize() - dex_file_offset < sizeof(DexFile::Header))) {
*error_msg = StringPrintf("In oat file '%s' found OatDexFile #%zu for '%s' with dex file "
"offset %u of %zu but the size of dex file header is %zu",
GetLocation().c_str(),
i,
dex_file_location.c_str(),
dex_file_offset,
- Size(),
+ DexSize(),
sizeof(DexFile::Header));
return false;
}
- const uint8_t* dex_file_pointer = Begin() + dex_file_offset;
+ const uint8_t* dex_file_pointer = DexBegin() + dex_file_offset;
if (UNLIKELY(!DexFile::IsMagicValid(dex_file_pointer))) {
*error_msg = StringPrintf("In oat file '%s' found OatDexFile #%zu for '%s' with invalid "
"dex file magic '%s'",
@@ -363,7 +388,7 @@
return false;
}
const DexFile::Header* header = reinterpret_cast<const DexFile::Header*>(dex_file_pointer);
- if (Size() - dex_file_offset < header->file_size_) {
+ if (DexSize() - dex_file_offset < header->file_size_) {
*error_msg = StringPrintf("In oat file '%s' found OatDexFile #%zu for '%s' with dex file "
"offset %u and size %u truncated at %zu",
GetLocation().c_str(),
@@ -371,7 +396,7 @@
dex_file_location.c_str(),
dex_file_offset,
header->file_size_,
- Size());
+ DexSize());
return false;
}
@@ -942,31 +967,37 @@
: nullptr;
}
-OatFile* OatFile::Open(const std::string& filename,
- const std::string& location,
+OatFile* OatFile::Open(const std::string& oat_filename,
+ const std::string& oat_location,
uint8_t* requested_base,
uint8_t* oat_file_begin,
bool executable,
bool low_4gb,
const char* abs_dex_location,
std::string* error_msg) {
- ScopedTrace trace("Open oat file " + location);
- CHECK(!filename.empty()) << location;
- CheckLocation(location);
+ ScopedTrace trace("Open oat file " + oat_location);
+ CHECK(!oat_filename.empty()) << oat_location;
+ CheckLocation(oat_location);
- // Check that the file even exists, fast-fail.
- if (!OS::FileExists(filename.c_str())) {
- *error_msg = StringPrintf("File %s does not exist.", filename.c_str());
+ std::string vdex_filename = ReplaceFileExtension(oat_filename, "vdex");
+
+ // Check that the files even exist, fast-fail.
+ if (kIsVdexEnabled && !OS::FileExists(vdex_filename.c_str())) {
+ *error_msg = StringPrintf("File %s does not exist.", vdex_filename.c_str());
+ return nullptr;
+ } else if (!OS::FileExists(oat_filename.c_str())) {
+ *error_msg = StringPrintf("File %s does not exist.", oat_filename.c_str());
return nullptr;
}
// Try dlopen first, as it is required for native debuggability. This will fail fast if dlopen is
// disabled.
- OatFile* with_dlopen = OatFileBase::OpenOatFile<DlOpenOatFile>(filename,
- location,
+ OatFile* with_dlopen = OatFileBase::OpenOatFile<DlOpenOatFile>(vdex_filename,
+ oat_filename,
+ oat_location,
requested_base,
oat_file_begin,
- false,
+ false /* writable */,
executable,
low_4gb,
abs_dex_location,
@@ -975,7 +1006,7 @@
return with_dlopen;
}
if (kPrintDlOpenErrorMessage) {
- LOG(ERROR) << "Failed to dlopen: " << filename << " with error " << *error_msg;
+ LOG(ERROR) << "Failed to dlopen: " << oat_filename << " with error " << *error_msg;
}
// If we aren't trying to execute, we just use our own ElfFile loader for a couple reasons:
//
@@ -990,11 +1021,12 @@
//
// Another independent reason is the absolute placement of boot.oat. dlopen on the host usually
// does honor the virtual address encoded in the ELF file only for ET_EXEC files, not ET_DYN.
- OatFile* with_internal = OatFileBase::OpenOatFile<ElfOatFile>(filename,
- location,
+ OatFile* with_internal = OatFileBase::OpenOatFile<ElfOatFile>(vdex_filename,
+ oat_filename,
+ oat_location,
requested_base,
oat_file_begin,
- false,
+ false /* writable */,
executable,
low_4gb,
abs_dex_location,
@@ -1036,6 +1068,7 @@
OatFile::OatFile(const std::string& location, bool is_executable)
: location_(location),
+ vdex_(nullptr),
begin_(nullptr),
end_(nullptr),
bss_begin_(nullptr),
@@ -1071,6 +1104,14 @@
return bss_end_;
}
+const uint8_t* OatFile::DexBegin() const {
+ return kIsVdexEnabled ? vdex_->Begin() : Begin();
+}
+
+const uint8_t* OatFile::DexEnd() const {
+ return kIsVdexEnabled ? vdex_->End() : End();
+}
+
const OatFile::OatDexFile* OatFile::GetOatDexFile(const char* dex_location,
const uint32_t* dex_location_checksum,
std::string* error_msg) const {
diff --git a/runtime/oat_file.h b/runtime/oat_file.h
index f5ab9dc..96e651e 100644
--- a/runtime/oat_file.h
+++ b/runtime/oat_file.h
@@ -30,6 +30,7 @@
#include "oat.h"
#include "os.h"
#include "utils.h"
+#include "vdex_file.h"
namespace art {
@@ -46,6 +47,14 @@
} // namespace collector
} // namespace gc
+// Runtime representation of the OAT file format which holds compiler output.
+// The class opens an OAT file from storage and maps it to memory, typically with
+// dlopen and provides access to its internal data structures (see OatWriter for
+// for more details about the OAT format).
+// In the process of loading OAT, the class also loads the associated VDEX file
+// with the input DEX files (see VdexFile for details about the VDEX format).
+// The raw DEX data are accessible transparently through the OatDexFile objects.
+
class OatFile {
public:
// Special classpath that skips shared library check.
@@ -240,12 +249,19 @@
return BssEnd() - BssBegin();
}
+ size_t DexSize() const {
+ return DexEnd() - DexBegin();
+ }
+
const uint8_t* Begin() const;
const uint8_t* End() const;
const uint8_t* BssBegin() const;
const uint8_t* BssEnd() const;
+ const uint8_t* DexBegin() const;
+ const uint8_t* DexEnd() const;
+
// Returns the absolute dex location for the encoded relative dex location.
//
// If not null, abs_dex_location is used to resolve the absolute dex
@@ -279,6 +295,9 @@
// The image will embed this to link its associated oat file.
const std::string location_;
+ // Pointer to the Vdex file with the Dex files for this Oat file.
+ std::unique_ptr<VdexFile> vdex_;
+
// Pointer to OatHeader.
const uint8_t* begin_;
diff --git a/runtime/oat_file_assistant.cc b/runtime/oat_file_assistant.cc
index fe6332d..415f991 100644
--- a/runtime/oat_file_assistant.cc
+++ b/runtime/oat_file_assistant.cc
@@ -345,17 +345,6 @@
return odex_.CompilerFilter();
}
-static std::string ArtFileName(const OatFile* oat_file) {
- const std::string oat_file_location = oat_file->GetLocation();
- // Replace extension with .art
- const size_t last_ext = oat_file_location.find_last_of('.');
- if (last_ext == std::string::npos) {
- LOG(ERROR) << "No extension in oat file " << oat_file_location;
- return std::string();
- }
- return oat_file_location.substr(0, last_ext) + ".art";
-}
-
const std::string* OatFileAssistant::OatFileName() {
return oat_.Filename();
}
@@ -565,6 +554,7 @@
return kUpdateNotAttempted;
}
const std::string& oat_file_name = *oat_.Filename();
+ const std::string& vdex_file_name = ReplaceFileExtension(oat_file_name, "vdex");
// dex2oat ignores missing dex files and doesn't report an error.
// Check explicitly here so we can detect the error properly.
@@ -574,8 +564,22 @@
return kUpdateNotAttempted;
}
- std::unique_ptr<File> oat_file;
- oat_file.reset(OS::CreateEmptyFile(oat_file_name.c_str()));
+ std::unique_ptr<File> vdex_file(OS::CreateEmptyFile(vdex_file_name.c_str()));
+ if (vdex_file.get() == nullptr) {
+ *error_msg = "Generation of oat file " + oat_file_name
+ + " not attempted because the vdex file " + vdex_file_name
+ + " could not be opened.";
+ return kUpdateNotAttempted;
+ }
+
+ if (fchmod(vdex_file->Fd(), 0644) != 0) {
+ *error_msg = "Generation of oat file " + oat_file_name
+ + " not attempted because the vdex file " + vdex_file_name
+ + " could not be made world readable.";
+ return kUpdateNotAttempted;
+ }
+
+ std::unique_ptr<File> oat_file(OS::CreateEmptyFile(oat_file_name.c_str()));
if (oat_file.get() == nullptr) {
*error_msg = "Generation of oat file " + oat_file_name
+ " not attempted because the oat file could not be created.";
@@ -591,17 +595,26 @@
std::vector<std::string> args;
args.push_back("--dex-file=" + dex_location_);
+ args.push_back("--vdex-fd=" + std::to_string(vdex_file->Fd()));
args.push_back("--oat-fd=" + std::to_string(oat_file->Fd()));
args.push_back("--oat-location=" + oat_file_name);
if (!Dex2Oat(args, error_msg)) {
- // Manually delete the file. This ensures there is no garbage left over if
- // the process unexpectedly died.
+ // Manually delete the oat and vdex files. This ensures there is no garbage
+ // left over if the process unexpectedly died.
+ vdex_file->Erase();
+ unlink(vdex_file_name.c_str());
oat_file->Erase();
unlink(oat_file_name.c_str());
return kUpdateFailed;
}
+ if (vdex_file->FlushCloseOrErase() != 0) {
+ *error_msg = "Unable to close vdex file " + vdex_file_name;
+ unlink(vdex_file_name.c_str());
+ return kUpdateFailed;
+ }
+
if (oat_file->FlushCloseOrErase() != 0) {
*error_msg = "Unable to close oat file " + oat_file_name;
unlink(oat_file_name.c_str());
@@ -830,7 +843,7 @@
std::unique_ptr<gc::space::ImageSpace> OatFileAssistant::OpenImageSpace(const OatFile* oat_file) {
DCHECK(oat_file != nullptr);
- std::string art_file = ArtFileName(oat_file);
+ std::string art_file = ReplaceFileExtension(oat_file->GetLocation(), "art");
if (art_file.empty()) {
return nullptr;
}
diff --git a/runtime/utils.cc b/runtime/utils.cc
index 313190c..d48edcf 100644
--- a/runtime/utils.cc
+++ b/runtime/utils.cc
@@ -1213,6 +1213,15 @@
return buffer.st_size > 0;
}
+std::string ReplaceFileExtension(const std::string& filename, const std::string& new_extension) {
+ const size_t last_ext = filename.find_last_of('.');
+ if (last_ext == std::string::npos) {
+ return filename + "." + new_extension;
+ } else {
+ return filename.substr(0, last_ext + 1) + new_extension;
+ }
+}
+
std::string PrettyDescriptor(Primitive::Type type) {
return PrettyDescriptor(Primitive::Descriptor(type));
}
diff --git a/runtime/utils.h b/runtime/utils.h
index 958f0a3..f3284e8 100644
--- a/runtime/utils.h
+++ b/runtime/utils.h
@@ -279,6 +279,13 @@
bool FileExists(const std::string& filename);
bool FileExistsAndNotEmpty(const std::string& filename);
+// Returns `filename` with the text after the last occurrence of '.' replaced with
+// `extension`. If `filename` does not contain a period, returns a string containing `filename`,
+// a period, and `new_extension`.
+// Example: ReplaceFileExtension("foo.bar", "abc") == "foo.abc"
+// ReplaceFileExtension("foo", "abc") == "foo.abc"
+std::string ReplaceFileExtension(const std::string& filename, const std::string& new_extension);
+
class VoidFunctor {
public:
template <typename A>
diff --git a/runtime/vdex_file.cc b/runtime/vdex_file.cc
new file mode 100644
index 0000000..12bc451
--- /dev/null
+++ b/runtime/vdex_file.cc
@@ -0,0 +1,87 @@
+/*
+ * Copyright (C) 2016 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "vdex_file.h"
+
+#include <memory>
+
+#include "base/logging.h"
+
+namespace art {
+
+constexpr uint8_t VdexFile::Header::kVdexMagic[4];
+constexpr uint8_t VdexFile::Header::kVdexVersion[4];
+
+bool VdexFile::Header::IsMagicValid() const {
+ return (memcmp(magic_, kVdexMagic, sizeof(kVdexMagic)) == 0);
+}
+
+bool VdexFile::Header::IsVersionValid() const {
+ return (memcmp(version_, kVdexVersion, sizeof(kVdexVersion)) == 0);
+}
+
+VdexFile::Header::Header() {
+ memcpy(magic_, kVdexMagic, sizeof(kVdexMagic));
+ memcpy(version_, kVdexVersion, sizeof(kVdexVersion));
+ DCHECK(IsMagicValid());
+ DCHECK(IsVersionValid());
+}
+
+VdexFile* VdexFile::Open(const std::string& vdex_filename,
+ bool writable,
+ bool low_4gb,
+ std::string* error_msg) {
+ if (!OS::FileExists(vdex_filename.c_str())) {
+ *error_msg = "File " + vdex_filename + " does not exist.";
+ return nullptr;
+ }
+
+ std::unique_ptr<File> vdex_file;
+ if (writable) {
+ vdex_file.reset(OS::OpenFileReadWrite(vdex_filename.c_str()));
+ } else {
+ vdex_file.reset(OS::OpenFileForReading(vdex_filename.c_str()));
+ }
+ if (vdex_file == nullptr) {
+ *error_msg = "Could not open file " + vdex_filename +
+ (writable ? " for read/write" : "for reading");
+ return nullptr;
+ }
+
+ int64_t vdex_length = vdex_file->GetLength();
+ if (vdex_length == -1) {
+ *error_msg = "Could not read the length of file " + vdex_filename;
+ return nullptr;
+ }
+
+ std::unique_ptr<MemMap> mmap(MemMap::MapFile(vdex_length,
+ writable ? PROT_READ | PROT_WRITE : PROT_READ,
+ MAP_SHARED,
+ vdex_file->Fd(),
+ 0 /* start offset */,
+ low_4gb,
+ vdex_filename.c_str(),
+ error_msg));
+ if (mmap == nullptr) {
+ *error_msg = "Failed to mmap file " + vdex_filename + " : " + *error_msg;
+ return nullptr;
+ }
+
+ *error_msg = "Success";
+ return new VdexFile(vdex_file.release(), mmap.release());
+}
+
+} // namespace art
diff --git a/runtime/vdex_file.h b/runtime/vdex_file.h
new file mode 100644
index 0000000..e381eb7
--- /dev/null
+++ b/runtime/vdex_file.h
@@ -0,0 +1,79 @@
+/*
+ * Copyright (C) 2016 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ART_RUNTIME_VDEX_FILE_H_
+#define ART_RUNTIME_VDEX_FILE_H_
+
+#include <stdint.h>
+#include <string>
+
+#include "base/macros.h"
+#include "base/unix_file/fd_file.h"
+#include "mem_map.h"
+#include "os.h"
+
+namespace art {
+
+// VDEX files contain extracted DEX files. The VdexFile class maps the file to
+// memory and provides tools for accessing its individual sections.
+//
+// File format:
+// VdexFile::Header fixed-length header
+//
+// DEX[0] array of the input DEX files
+// DEX[1] the bytecode may have been quickened
+// ...
+// DEX[D]
+//
+
+class VdexFile {
+ public:
+ struct Header {
+ public:
+ Header();
+
+ bool IsMagicValid() const;
+ bool IsVersionValid() const;
+
+ private:
+ static constexpr uint8_t kVdexMagic[] = { 'v', 'd', 'e', 'x' };
+ static constexpr uint8_t kVdexVersion[] = { '0', '0', '0', '\0' };
+
+ uint8_t magic_[4];
+ uint8_t version_[4];
+ };
+
+ static VdexFile* Open(const std::string& vdex_filename,
+ bool writable,
+ bool low_4gb,
+ std::string* error_msg);
+
+ const uint8_t* Begin() const { return mmap_->Begin(); }
+ const uint8_t* End() const { return mmap_->End(); }
+ size_t Size() const { return mmap_->Size(); }
+
+ private:
+ VdexFile(File* file, MemMap* mmap) : file_(file), mmap_(mmap) {}
+
+ std::unique_ptr<File> file_;
+ std::unique_ptr<MemMap> mmap_;
+
+ DISALLOW_COPY_AND_ASSIGN(VdexFile);
+};
+
+} // namespace art
+
+#endif // ART_RUNTIME_VDEX_FILE_H_