Introduce VDEX file, use it for DEX files
This patch introduces a new output file called VDEX. In the future,
VDEX files will store pre-validated DEX files which do not need to be
re-extracted and re-verified when recompiling, e.g. due to new
profiling information or after a system update.
With this CL, the OatWriter writes DEX files into the VDEX and the
rest of its output into OAT. The OatFile class and related classes
are updated to load the VDEX at runtime and mmap the DEX file section
from it. Patchoat creates symlinks to the source VDEX files in the
target directory or copies the files if passed in as file descriptors.
The feature can be disabled by setting the environment variable
ART_ENABLE_VDEX to false.
Test: m test-art-host
Bug: 30937355
Change-Id: I54dcaececf6814c258c80524ec15e2e2ef69c8dd
diff --git a/patchoat/patchoat.cc b/patchoat/patchoat.cc
index b7ce02c..5240011 100644
--- a/patchoat/patchoat.cc
+++ b/patchoat/patchoat.cc
@@ -31,6 +31,7 @@
#include "base/stringpiece.h"
#include "base/stringprintf.h"
#include "base/unix_file/fd_file.h"
+#include "base/unix_file/random_access_file_utils.h"
#include "elf_utils.h"
#include "elf_file.h"
#include "elf_file_impl.h"
@@ -151,6 +152,28 @@
}
}
+static bool SymlinkFile(const std::string& input_filename, const std::string& output_filename) {
+ if (input_filename == output_filename) {
+ // Input and output are the same, nothing to do.
+ return true;
+ }
+
+ // Unlink the original filename, since we are overwriting it.
+ unlink(output_filename.c_str());
+
+ // Create a symlink from the source file to the target path.
+ if (symlink(input_filename.c_str(), output_filename.c_str()) < 0) {
+ PLOG(ERROR) << "Failed to create symlink " << output_filename << " -> " << input_filename;
+ return false;
+ }
+
+ if (kIsDebugBuild) {
+ LOG(INFO) << "Created symlink " << output_filename << " -> " << input_filename;
+ }
+
+ return true;
+}
+
bool PatchOat::Patch(const std::string& image_location,
off_t delta,
const std::string& output_directory,
@@ -230,9 +253,13 @@
space_to_memmap_map.emplace(space, std::move(image));
}
+ // Do a first pass over the image spaces. Symlink PIC oat and vdex files, and
+ // prepare PatchOat instances for the rest.
for (size_t i = 0; i < spaces.size(); ++i) {
gc::space::ImageSpace* space = spaces[i];
std::string input_image_filename = space->GetImageFilename();
+ std::string input_vdex_filename =
+ ImageHeader::GetVdexLocationFromImageLocation(input_image_filename);
std::string input_oat_filename =
ImageHeader::GetOatLocationFromImageLocation(input_image_filename);
std::unique_ptr<File> input_oat_file(OS::OpenFileForReading(input_oat_filename.c_str()));
@@ -261,13 +288,16 @@
std::string output_image_filename = output_directory +
(StartsWith(converted_image_filename, "/") ? "" : "/") +
converted_image_filename;
+ std::string output_vdex_filename =
+ ImageHeader::GetVdexLocationFromImageLocation(output_image_filename);
std::string output_oat_filename =
ImageHeader::GetOatLocationFromImageLocation(output_image_filename);
if (!ReplaceOatFileWithSymlink(input_oat_file->GetPath(),
output_oat_filename,
false,
- true)) {
+ true) ||
+ !SymlinkFile(input_vdex_filename, output_vdex_filename)) {
// Errors already logged by above call.
return false;
}
@@ -301,9 +331,13 @@
space_to_skip_patching_map.emplace(space, skip_patching_oat);
}
+ // Do a second pass over the image spaces. Patch image files, non-PIC oat files
+ // and symlink their corresponding vdex files.
for (size_t i = 0; i < spaces.size(); ++i) {
gc::space::ImageSpace* space = spaces[i];
std::string input_image_filename = space->GetImageFilename();
+ std::string input_vdex_filename =
+ ImageHeader::GetVdexLocationFromImageLocation(input_image_filename);
t.NewTiming("Writing files");
std::string converted_image_filename = space->GetImageLocation();
@@ -329,8 +363,11 @@
bool skip_patching_oat = space_to_skip_patching_map.find(space)->second;
if (!skip_patching_oat) {
+ std::string output_vdex_filename =
+ ImageHeader::GetVdexLocationFromImageLocation(output_image_filename);
std::string output_oat_filename =
ImageHeader::GetOatLocationFromImageLocation(output_image_filename);
+
std::unique_ptr<File>
output_oat_file(CreateOrOpen(output_oat_filename.c_str(), &new_oat_out));
if (output_oat_file.get() == nullptr) {
@@ -339,6 +376,9 @@
}
success = p.WriteElf(output_oat_file.get());
success = FinishFile(output_oat_file.get(), success);
+ if (success) {
+ success = SymlinkFile(input_vdex_filename, output_vdex_filename);
+ }
if (!success) {
return false;
}
@@ -921,6 +961,9 @@
UsageError(" --input-oat-fd=<file-descriptor>: Specifies the file-descriptor of the oat file");
UsageError(" to be patched.");
UsageError("");
+ UsageError(" --input-vdex-fd=<file-descriptor>: Specifies the file-descriptor of the vdex file");
+ UsageError(" associated with the oat file.");
+ UsageError("");
UsageError(" --input-oat-location=<file.oat>: Specifies the 'location' to read the patched");
UsageError(" oat file from. If used one must also supply the --instruction-set");
UsageError("");
@@ -932,7 +975,10 @@
UsageError(" file to.");
UsageError("");
UsageError(" --output-oat-fd=<file-descriptor>: Specifies the file-descriptor to write the");
- UsageError(" the patched oat file to.");
+ UsageError(" patched oat file to.");
+ UsageError("");
+ UsageError(" --output-vdex-fd=<file-descriptor>: Specifies the file-descriptor to copy the");
+ UsageError(" the vdex file associated with the patch oat file to.");
UsageError("");
UsageError(" --output-image-file=<file.art>: Specifies the exact file to write the patched");
UsageError(" image file to.");
@@ -1029,10 +1075,12 @@
off_t base_delta,
bool base_delta_set,
int input_oat_fd,
+ int input_vdex_fd,
const std::string& input_oat_location,
std::string input_oat_filename,
bool have_input_oat,
int output_oat_fd,
+ int output_vdex_fd,
std::string output_oat_filename,
bool have_output_oat,
bool lock_output,
@@ -1062,6 +1110,12 @@
}
}
+ if ((input_oat_fd == -1) != (input_vdex_fd == -1)) {
+ Usage("Either both input oat and vdex have to be passed as file descriptors or none of them");
+ } else if ((output_oat_fd == -1) != (output_vdex_fd == -1)) {
+ Usage("Either both output oat and vdex have to be passed as file descriptors or none of them");
+ }
+
bool match_delta = false;
if (!patched_image_location.empty()) {
std::string system_filename;
@@ -1102,8 +1156,24 @@
Usage("Base offset/delta must be alligned to a pagesize (0x%08x) boundary.", kPageSize);
}
+ // We can symlink VDEX only if we have both input and output specified as filenames.
+ // Store that piece of information before we possibly create bogus filenames for
+ // files passed as file descriptors.
+ bool symlink_vdex = !input_oat_filename.empty() && !output_oat_filename.empty();
+
+ // Infer names of VDEX files.
+ std::string input_vdex_filename;
+ std::string output_vdex_filename;
+ if (!input_oat_filename.empty()) {
+ input_vdex_filename = ReplaceFileExtension(input_oat_filename, "vdex");
+ }
+ if (!output_oat_filename.empty()) {
+ output_vdex_filename = ReplaceFileExtension(output_oat_filename, "vdex");
+ }
+
// Do we need to cleanup output files if we fail?
bool new_oat_out = false;
+ bool new_vdex_out = false;
std::unique_ptr<File> input_oat;
std::unique_ptr<File> output_oat;
@@ -1162,13 +1232,52 @@
}
}
+ // Open VDEX files if we are not symlinking them.
+ std::unique_ptr<File> input_vdex;
+ std::unique_ptr<File> output_vdex;
+ if (symlink_vdex) {
+ new_vdex_out = !OS::FileExists(output_vdex_filename.c_str());
+ } else {
+ if (input_vdex_fd != -1) {
+ input_vdex.reset(new File(input_vdex_fd, input_vdex_filename, true));
+ if (input_vdex == nullptr) {
+ // Unlikely, but ensure exhaustive logging in non-0 exit code case
+ LOG(ERROR) << "Failed to open input vdex file by its FD" << input_vdex_fd;
+ }
+ } else {
+ input_vdex.reset(OS::OpenFileForReading(input_vdex_filename.c_str()));
+ if (input_vdex == nullptr) {
+ PLOG(ERROR) << "Failed to open input vdex file " << input_vdex_filename;
+ return EXIT_FAILURE;
+ }
+ }
+ if (output_vdex_fd != -1) {
+ output_vdex.reset(new File(output_vdex_fd, output_vdex_filename, true));
+ if (output_vdex == nullptr) {
+ // Unlikely, but ensure exhaustive logging in non-0 exit code case
+ LOG(ERROR) << "Failed to open output vdex file by its FD" << output_vdex_fd;
+ }
+ } else {
+ output_vdex.reset(CreateOrOpen(output_vdex_filename.c_str(), &new_vdex_out));
+ if (output_vdex == nullptr) {
+ PLOG(ERROR) << "Failed to open output vdex file " << output_vdex_filename;
+ return EXIT_FAILURE;
+ }
+ }
+ }
+
// TODO: get rid of this.
- auto cleanup = [&output_oat_filename, &new_oat_out](bool success) {
+ auto cleanup = [&output_oat_filename, &output_vdex_filename, &new_oat_out, &new_vdex_out]
+ (bool success) {
if (!success) {
if (new_oat_out) {
CHECK(!output_oat_filename.empty());
unlink(output_oat_filename.c_str());
}
+ if (new_vdex_out) {
+ CHECK(!output_vdex_filename.empty());
+ unlink(output_vdex_filename.c_str());
+ }
}
if (kIsDebugBuild) {
@@ -1220,6 +1329,14 @@
new_oat_out);
ret = FinishFile(output_oat.get(), ret);
+ if (ret) {
+ if (symlink_vdex) {
+ ret = SymlinkFile(input_vdex_filename, output_vdex_filename);
+ } else {
+ ret = unix_file::CopyFile(*input_vdex.get(), output_vdex.get());
+ }
+ }
+
if (kIsDebugBuild) {
LOG(INFO) << "Exiting with return ... " << ret;
}
@@ -1227,6 +1344,18 @@
return ret ? EXIT_SUCCESS : EXIT_FAILURE;
}
+static int ParseFd(const StringPiece& option, const char* cmdline_arg) {
+ int fd;
+ const char* fd_str = option.substr(strlen(cmdline_arg)).data();
+ if (!ParseInt(fd_str, &fd)) {
+ Usage("Failed to parse %d argument '%s' as an integer", cmdline_arg, fd_str);
+ }
+ if (fd < 0) {
+ Usage("%s pass a negative value %d", cmdline_arg, fd);
+ }
+ return fd;
+}
+
static int patchoat(int argc, char **argv) {
InitLogging(argv);
MemMap::Init();
@@ -1253,10 +1382,12 @@
std::string input_oat_filename;
std::string input_oat_location;
int input_oat_fd = -1;
+ int input_vdex_fd = -1;
bool have_input_oat = false;
std::string input_image_location;
std::string output_oat_filename;
int output_oat_fd = -1;
+ int output_vdex_fd = -1;
bool have_output_oat = false;
std::string output_image_filename;
off_t base_delta = 0;
@@ -1296,13 +1427,9 @@
Usage("Only one of --input-oat-file, --input-oat-location and --input-oat-fd may be used.");
}
have_input_oat = true;
- const char* oat_fd_str = option.substr(strlen("--input-oat-fd=")).data();
- if (!ParseInt(oat_fd_str, &input_oat_fd)) {
- Usage("Failed to parse --input-oat-fd argument '%s' as an integer", oat_fd_str);
- }
- if (input_oat_fd < 0) {
- Usage("--input-oat-fd pass a negative value %d", input_oat_fd);
- }
+ input_oat_fd = ParseFd(option, "--input-oat-fd=");
+ } else if (option.starts_with("--input-vdex-fd=")) {
+ input_vdex_fd = ParseFd(option, "--input-vdex-fd=");
} else if (option.starts_with("--input-image-location=")) {
input_image_location = option.substr(strlen("--input-image-location=")).data();
} else if (option.starts_with("--output-oat-file=")) {
@@ -1316,13 +1443,9 @@
Usage("Only one of --output-oat-file, --output-oat-fd may be used.");
}
have_output_oat = true;
- const char* oat_fd_str = option.substr(strlen("--output-oat-fd=")).data();
- if (!ParseInt(oat_fd_str, &output_oat_fd)) {
- Usage("Failed to parse --output-oat-fd argument '%s' as an integer", oat_fd_str);
- }
- if (output_oat_fd < 0) {
- Usage("--output-oat-fd pass a negative value %d", output_oat_fd);
- }
+ output_oat_fd = ParseFd(option, "--output-oat-fd=");
+ } else if (option.starts_with("--output-vdex-fd=")) {
+ output_vdex_fd = ParseFd(option, "--output-vdex-fd=");
} else if (option.starts_with("--output-image-file=")) {
output_image_filename = option.substr(strlen("--output-image-file=")).data();
} else if (option.starts_with("--base-offset-delta=")) {
@@ -1367,10 +1490,12 @@
base_delta,
base_delta_set,
input_oat_fd,
+ input_vdex_fd,
input_oat_location,
input_oat_filename,
have_input_oat,
output_oat_fd,
+ output_vdex_fd,
output_oat_filename,
have_output_oat,
lock_output,