Add profman tool: responsible to process profiles
This pulls out profile parsing from dex2oat into a separate tool.
Some additional refactoring:
- better return codes
- dex2oat now accepts only one profile file
This is the first step towards support secondary dex files and
extracting profiles out of the code_cache directory.
Bug: 26719109
Bug: 26563023
(cherry picked from commit d81c289e58f4ced8ec7674fd377c356669f97227)
Change-Id: I34d408faa318e866e1a23d4d04c369131ee5012b
diff --git a/compiler/Android.mk b/compiler/Android.mk
index b164942..3dfb4b1 100644
--- a/compiler/Android.mk
+++ b/compiler/Android.mk
@@ -108,8 +108,7 @@
elf_writer.cc \
elf_writer_quick.cc \
image_writer.cc \
- oat_writer.cc \
- profile_assistant.cc
+ oat_writer.cc
LIBART_COMPILER_SRC_FILES_arm := \
dex/quick/arm/assemble_arm.cc \
diff --git a/compiler/profile_assistant.cc b/compiler/profile_assistant.cc
deleted file mode 100644
index 85335ef..0000000
--- a/compiler/profile_assistant.cc
+++ /dev/null
@@ -1,169 +0,0 @@
-/*
- * Copyright (C) 2015 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "profile_assistant.h"
-
-#include "base/unix_file/fd_file.h"
-#include "os.h"
-
-namespace art {
-
-// Minimum number of new methods that profiles must contain to enable recompilation.
-static constexpr const uint32_t kMinNewMethodsForCompilation = 10;
-
-bool ProfileAssistant::ProcessProfilesInternal(
- const std::vector<ScopedFlock>& profile_files,
- const std::vector<ScopedFlock>& reference_profile_files,
- /*out*/ ProfileCompilationInfo** profile_compilation_info) {
- DCHECK(!profile_files.empty());
- DCHECK(!reference_profile_files.empty() ||
- (profile_files.size() == reference_profile_files.size()));
-
- std::vector<ProfileCompilationInfo> new_info(profile_files.size());
- bool should_compile = false;
- // Read the main profile files.
- for (size_t i = 0; i < new_info.size(); i++) {
- if (!new_info[i].Load(profile_files[i].GetFile()->Fd())) {
- LOG(WARNING) << "Could not load profile file at index " << i;
- return false;
- }
- // Do we have enough new profiled methods that will make the compilation worthwhile?
- should_compile |= (new_info[i].GetNumberOfMethods() > kMinNewMethodsForCompilation);
- }
-
- if (!should_compile) {
- return true;
- }
-
- std::unique_ptr<ProfileCompilationInfo> result(new ProfileCompilationInfo());
- // Merge information.
- for (size_t i = 0; i < new_info.size(); i++) {
- if (!reference_profile_files.empty()) {
- if (!new_info[i].Load(reference_profile_files[i].GetFile()->Fd())) {
- LOG(WARNING) << "Could not load reference profile file at index " << i;
- return false;
- }
- }
- // Merge all data into a single object.
- if (!result->Load(new_info[i])) {
- LOG(WARNING) << "Could not merge profile data at index " << i;
- return false;
- }
- }
- // We were successful in merging all profile information. Update the files.
- for (size_t i = 0; i < new_info.size(); i++) {
- if (!reference_profile_files.empty()) {
- if (!reference_profile_files[i].GetFile()->ClearContent()) {
- PLOG(WARNING) << "Could not clear reference profile file at index " << i;
- return false;
- }
- if (!new_info[i].Save(reference_profile_files[i].GetFile()->Fd())) {
- LOG(WARNING) << "Could not save reference profile file at index " << i;
- return false;
- }
- if (!profile_files[i].GetFile()->ClearContent()) {
- PLOG(WARNING) << "Could not clear profile file at index " << i;
- return false;
- }
- }
- }
-
- *profile_compilation_info = result.release();
- return true;
-}
-
-class ScopedCollectionFlock {
- public:
- explicit ScopedCollectionFlock(size_t size) : flocks_(size) {}
-
- // Will block until all the locks are acquired.
- bool Init(const std::vector<std::string>& filenames, /* out */ std::string* error) {
- for (size_t i = 0; i < filenames.size(); i++) {
- if (!flocks_[i].Init(filenames[i].c_str(), O_RDWR, /* block */ true, error)) {
- *error += " (index=" + std::to_string(i) + ")";
- return false;
- }
- }
- return true;
- }
-
- // Will block until all the locks are acquired.
- bool Init(const std::vector<uint32_t>& fds, /* out */ std::string* error) {
- for (size_t i = 0; i < fds.size(); i++) {
- // We do not own the descriptor, so disable auto-close and don't check usage.
- File file(fds[i], false);
- file.DisableAutoClose();
- if (!flocks_[i].Init(&file, error)) {
- *error += " (index=" + std::to_string(i) + ")";
- return false;
- }
- }
- return true;
- }
-
- const std::vector<ScopedFlock>& Get() const { return flocks_; }
-
- private:
- std::vector<ScopedFlock> flocks_;
-};
-
-bool ProfileAssistant::ProcessProfiles(
- const std::vector<uint32_t>& profile_files_fd,
- const std::vector<uint32_t>& reference_profile_files_fd,
- /*out*/ ProfileCompilationInfo** profile_compilation_info) {
- *profile_compilation_info = nullptr;
-
- std::string error;
- ScopedCollectionFlock profile_files_flocks(profile_files_fd.size());
- if (!profile_files_flocks.Init(profile_files_fd, &error)) {
- LOG(WARNING) << "Could not lock profile files: " << error;
- return false;
- }
- ScopedCollectionFlock reference_profile_files_flocks(reference_profile_files_fd.size());
- if (!reference_profile_files_flocks.Init(reference_profile_files_fd, &error)) {
- LOG(WARNING) << "Could not lock reference profile files: " << error;
- return false;
- }
-
- return ProcessProfilesInternal(profile_files_flocks.Get(),
- reference_profile_files_flocks.Get(),
- profile_compilation_info);
-}
-
-bool ProfileAssistant::ProcessProfiles(
- const std::vector<std::string>& profile_files,
- const std::vector<std::string>& reference_profile_files,
- /*out*/ ProfileCompilationInfo** profile_compilation_info) {
- *profile_compilation_info = nullptr;
-
- std::string error;
- ScopedCollectionFlock profile_files_flocks(profile_files.size());
- if (!profile_files_flocks.Init(profile_files, &error)) {
- LOG(WARNING) << "Could not lock profile files: " << error;
- return false;
- }
- ScopedCollectionFlock reference_profile_files_flocks(reference_profile_files.size());
- if (!reference_profile_files_flocks.Init(reference_profile_files, &error)) {
- LOG(WARNING) << "Could not lock reference profile files: " << error;
- return false;
- }
-
- return ProcessProfilesInternal(profile_files_flocks.Get(),
- reference_profile_files_flocks.Get(),
- profile_compilation_info);
-}
-
-} // namespace art
diff --git a/compiler/profile_assistant.h b/compiler/profile_assistant.h
deleted file mode 100644
index ad5e216..0000000
--- a/compiler/profile_assistant.h
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Copyright (C) 2015 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef ART_COMPILER_PROFILE_ASSISTANT_H_
-#define ART_COMPILER_PROFILE_ASSISTANT_H_
-
-#include <string>
-#include <vector>
-
-#include "base/scoped_flock.h"
-#include "jit/offline_profiling_info.cc"
-
-namespace art {
-
-class ProfileAssistant {
- public:
- // Process the profile information present in the given files. Returns true
- // if the analysis ended up successfully (i.e. no errors during reading,
- // merging or writing of profile files).
- //
- // If the returned value is true and there is a significant difference between
- // profile_files and reference_profile_files:
- // - profile_compilation_info is set to a not null object that
- // can be used to drive compilation. It will be the merge of all the data
- // found in profile_files and reference_profile_files.
- // - the data from profile_files[i] is merged into
- // reference_profile_files[i] and the corresponding backing file is
- // updated.
- //
- // If the returned value is false or the difference is insignificant,
- // profile_compilation_info will be set to null.
- //
- // Additional notes:
- // - as mentioned above, this function may update the content of the files
- // passed with the reference_profile_files.
- // - if reference_profile_files is not empty it must be the same size as
- // profile_files.
- static bool ProcessProfiles(
- const std::vector<std::string>& profile_files,
- const std::vector<std::string>& reference_profile_files,
- /*out*/ ProfileCompilationInfo** profile_compilation_info);
-
- static bool ProcessProfiles(
- const std::vector<uint32_t>& profile_files_fd_,
- const std::vector<uint32_t>& reference_profile_files_fd_,
- /*out*/ ProfileCompilationInfo** profile_compilation_info);
-
- private:
- static bool ProcessProfilesInternal(
- const std::vector<ScopedFlock>& profile_files,
- const std::vector<ScopedFlock>& reference_profile_files,
- /*out*/ ProfileCompilationInfo** profile_compilation_info);
-
- DISALLOW_COPY_AND_ASSIGN(ProfileAssistant);
-};
-
-} // namespace art
-
-#endif // ART_COMPILER_PROFILE_ASSISTANT_H_
diff --git a/compiler/profile_assistant_test.cc b/compiler/profile_assistant_test.cc
deleted file mode 100644
index 58b7513..0000000
--- a/compiler/profile_assistant_test.cc
+++ /dev/null
@@ -1,279 +0,0 @@
-/*
- * Copyright (C) 2016 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <gtest/gtest.h>
-
-#include "base/unix_file/fd_file.h"
-#include "common_runtime_test.h"
-#include "compiler/profile_assistant.h"
-#include "jit/offline_profiling_info.h"
-
-namespace art {
-
-class ProfileAssistantTest : public CommonRuntimeTest {
- protected:
- void SetupProfile(const std::string& id,
- uint32_t checksum,
- uint16_t number_of_methods,
- const ScratchFile& profile,
- ProfileCompilationInfo* info,
- uint16_t start_method_index = 0) {
- std::string dex_location1 = "location1" + id;
- uint32_t dex_location_checksum1 = checksum;
- std::string dex_location2 = "location2" + id;
- uint32_t dex_location_checksum2 = 10 * checksum;
- for (uint16_t i = start_method_index; i < start_method_index + number_of_methods; i++) {
- ASSERT_TRUE(info->AddData(dex_location1, dex_location_checksum1, i));
- ASSERT_TRUE(info->AddData(dex_location2, dex_location_checksum2, i));
- }
- ASSERT_TRUE(info->Save(GetFd(profile)));
- ASSERT_EQ(0, profile.GetFile()->Flush());
- ASSERT_TRUE(profile.GetFile()->ResetOffset());
- }
-
- uint32_t GetFd(const ScratchFile& file) const {
- return static_cast<uint32_t>(file.GetFd());
- }
-};
-
-TEST_F(ProfileAssistantTest, AdviseCompilationEmptyReferences) {
- ScratchFile profile1;
- ScratchFile profile2;
- ScratchFile reference_profile1;
- ScratchFile reference_profile2;
-
- std::vector<uint32_t> profile_fds({
- GetFd(profile1),
- GetFd(profile2)});
- std::vector<uint32_t> reference_profile_fds({
- GetFd(reference_profile1),
- GetFd(reference_profile2)});
-
- const uint16_t kNumberOfMethodsToEnableCompilation = 100;
- ProfileCompilationInfo info1;
- SetupProfile("p1", 1, kNumberOfMethodsToEnableCompilation, profile1, &info1);
- ProfileCompilationInfo info2;
- SetupProfile("p2", 2, kNumberOfMethodsToEnableCompilation, profile2, &info2);
-
- // We should advise compilation.
- ProfileCompilationInfo* result;
- ASSERT_TRUE(ProfileAssistant::ProcessProfiles(profile_fds, reference_profile_fds, &result));
- ASSERT_TRUE(result != nullptr);
-
- // The resulting compilation info must be equal to the merge of the inputs.
- ProfileCompilationInfo expected;
- ASSERT_TRUE(expected.Load(info1));
- ASSERT_TRUE(expected.Load(info2));
- ASSERT_TRUE(expected.Equals(*result));
-
- // The information from profiles must be transfered to the reference profiles.
- ProfileCompilationInfo file_info1;
- ASSERT_TRUE(reference_profile1.GetFile()->ResetOffset());
- ASSERT_TRUE(file_info1.Load(GetFd(reference_profile1)));
- ASSERT_TRUE(file_info1.Equals(info1));
-
- ProfileCompilationInfo file_info2;
- ASSERT_TRUE(reference_profile2.GetFile()->ResetOffset());
- ASSERT_TRUE(file_info2.Load(GetFd(reference_profile2)));
- ASSERT_TRUE(file_info2.Equals(info2));
-
- // Initial profiles must be cleared.
- ASSERT_EQ(0, profile1.GetFile()->GetLength());
- ASSERT_EQ(0, profile2.GetFile()->GetLength());
-}
-
-TEST_F(ProfileAssistantTest, AdviseCompilationNonEmptyReferences) {
- ScratchFile profile1;
- ScratchFile profile2;
- ScratchFile reference_profile1;
- ScratchFile reference_profile2;
-
- std::vector<uint32_t> profile_fds({
- GetFd(profile1),
- GetFd(profile2)});
- std::vector<uint32_t> reference_profile_fds({
- GetFd(reference_profile1),
- GetFd(reference_profile2)});
-
- // The new profile info will contain the methods with indices 0-100.
- const uint16_t kNumberOfMethodsToEnableCompilation = 100;
- ProfileCompilationInfo info1;
- SetupProfile("p1", 1, kNumberOfMethodsToEnableCompilation, profile1, &info1);
- ProfileCompilationInfo info2;
- SetupProfile("p2", 2, kNumberOfMethodsToEnableCompilation, profile2, &info2);
-
-
- // The reference profile info will contain the methods with indices 50-150.
- const uint16_t kNumberOfMethodsAlreadyCompiled = 100;
- ProfileCompilationInfo reference_info1;
- SetupProfile("p1", 1, kNumberOfMethodsAlreadyCompiled, reference_profile1,
- &reference_info1, kNumberOfMethodsToEnableCompilation / 2);
- ProfileCompilationInfo reference_info2;
- SetupProfile("p2", 2, kNumberOfMethodsAlreadyCompiled, reference_profile2,
- &reference_info2, kNumberOfMethodsToEnableCompilation / 2);
-
- // We should advise compilation.
- ProfileCompilationInfo* result;
- ASSERT_TRUE(ProfileAssistant::ProcessProfiles(profile_fds, reference_profile_fds, &result));
- ASSERT_TRUE(result != nullptr);
-
- // The resulting compilation info must be equal to the merge of the inputs
- ProfileCompilationInfo expected;
- ASSERT_TRUE(expected.Load(info1));
- ASSERT_TRUE(expected.Load(info2));
- ASSERT_TRUE(expected.Load(reference_info1));
- ASSERT_TRUE(expected.Load(reference_info2));
- ASSERT_TRUE(expected.Equals(*result));
-
- // The information from profiles must be transfered to the reference profiles.
- ProfileCompilationInfo file_info1;
- ProfileCompilationInfo merge1;
- ASSERT_TRUE(merge1.Load(info1));
- ASSERT_TRUE(merge1.Load(reference_info1));
- ASSERT_TRUE(reference_profile1.GetFile()->ResetOffset());
- ASSERT_TRUE(file_info1.Load(GetFd(reference_profile1)));
- ASSERT_TRUE(file_info1.Equals(merge1));
-
- ProfileCompilationInfo file_info2;
- ProfileCompilationInfo merge2;
- ASSERT_TRUE(merge2.Load(info2));
- ASSERT_TRUE(merge2.Load(reference_info2));
- ASSERT_TRUE(reference_profile2.GetFile()->ResetOffset());
- ASSERT_TRUE(file_info2.Load(GetFd(reference_profile2)));
- ASSERT_TRUE(file_info2.Equals(merge2));
-
- // Initial profiles must be cleared.
- ASSERT_EQ(0, profile1.GetFile()->GetLength());
- ASSERT_EQ(0, profile2.GetFile()->GetLength());
-}
-
-TEST_F(ProfileAssistantTest, DoNotAdviseCompilation) {
- ScratchFile profile1;
- ScratchFile profile2;
- ScratchFile reference_profile1;
- ScratchFile reference_profile2;
-
- std::vector<uint32_t> profile_fds({
- GetFd(profile1),
- GetFd(profile2)});
- std::vector<uint32_t> reference_profile_fds({
- GetFd(reference_profile1),
- GetFd(reference_profile2)});
-
- const uint16_t kNumberOfMethodsToSkipCompilation = 1;
- ProfileCompilationInfo info1;
- SetupProfile("p1", 1, kNumberOfMethodsToSkipCompilation, profile1, &info1);
- ProfileCompilationInfo info2;
- SetupProfile("p2", 2, kNumberOfMethodsToSkipCompilation, profile2, &info2);
-
- // We should not advise compilation.
- ProfileCompilationInfo* result = nullptr;
- ASSERT_TRUE(ProfileAssistant::ProcessProfiles(profile_fds, reference_profile_fds, &result));
- ASSERT_TRUE(result == nullptr);
-
- // The information from profiles must remain the same.
- ProfileCompilationInfo file_info1;
- ASSERT_TRUE(profile1.GetFile()->ResetOffset());
- ASSERT_TRUE(file_info1.Load(GetFd(profile1)));
- ASSERT_TRUE(file_info1.Equals(info1));
-
- ProfileCompilationInfo file_info2;
- ASSERT_TRUE(profile2.GetFile()->ResetOffset());
- ASSERT_TRUE(file_info2.Load(GetFd(profile2)));
- ASSERT_TRUE(file_info2.Equals(info2));
-
- // Reference profile files must remain empty.
- ASSERT_EQ(0, reference_profile1.GetFile()->GetLength());
- ASSERT_EQ(0, reference_profile2.GetFile()->GetLength());
-}
-
-TEST_F(ProfileAssistantTest, FailProcessingBecauseOfProfiles) {
- ScratchFile profile1;
- ScratchFile profile2;
- ScratchFile reference_profile1;
- ScratchFile reference_profile2;
-
- std::vector<uint32_t> profile_fds({
- GetFd(profile1),
- GetFd(profile2)});
- std::vector<uint32_t> reference_profile_fds({
- GetFd(reference_profile1),
- GetFd(reference_profile2)});
-
- const uint16_t kNumberOfMethodsToEnableCompilation = 100;
- // Assign different hashes for the same dex file. This will make merging of information to fail.
- ProfileCompilationInfo info1;
- SetupProfile("p1", 1, kNumberOfMethodsToEnableCompilation, profile1, &info1);
- ProfileCompilationInfo info2;
- SetupProfile("p1", 2, kNumberOfMethodsToEnableCompilation, profile2, &info2);
-
- // We should fail processing.
- ProfileCompilationInfo* result = nullptr;
- ASSERT_FALSE(ProfileAssistant::ProcessProfiles(profile_fds, reference_profile_fds, &result));
- ASSERT_TRUE(result == nullptr);
-
- // The information from profiles must still remain the same.
- ProfileCompilationInfo file_info1;
- ASSERT_TRUE(profile1.GetFile()->ResetOffset());
- ASSERT_TRUE(file_info1.Load(GetFd(profile1)));
- ASSERT_TRUE(file_info1.Equals(info1));
-
- ProfileCompilationInfo file_info2;
- ASSERT_TRUE(profile2.GetFile()->ResetOffset());
- ASSERT_TRUE(file_info2.Load(GetFd(profile2)));
- ASSERT_TRUE(file_info2.Equals(info2));
-
- // Reference profile files must still remain empty.
- ASSERT_EQ(0, reference_profile1.GetFile()->GetLength());
- ASSERT_EQ(0, reference_profile2.GetFile()->GetLength());
-}
-
-TEST_F(ProfileAssistantTest, FailProcessingBecauseOfReferenceProfiles) {
- ScratchFile profile1;
- ScratchFile reference_profile;
-
- std::vector<uint32_t> profile_fds({
- GetFd(profile1)});
- std::vector<uint32_t> reference_profile_fds({
- GetFd(reference_profile)});
-
- const uint16_t kNumberOfMethodsToEnableCompilation = 100;
- // Assign different hashes for the same dex file. This will make merging of information to fail.
- ProfileCompilationInfo info1;
- SetupProfile("p1", 1, kNumberOfMethodsToEnableCompilation, profile1, &info1);
- ProfileCompilationInfo reference_info;
- SetupProfile("p1", 2, kNumberOfMethodsToEnableCompilation, reference_profile, &reference_info);
-
- // We should not advise compilation.
- ProfileCompilationInfo* result = nullptr;
- ASSERT_TRUE(profile1.GetFile()->ResetOffset());
- ASSERT_TRUE(reference_profile.GetFile()->ResetOffset());
- ASSERT_FALSE(ProfileAssistant::ProcessProfiles(profile_fds, reference_profile_fds, &result));
- ASSERT_TRUE(result == nullptr);
-
- // The information from profiles must still remain the same.
- ProfileCompilationInfo file_info1;
- ASSERT_TRUE(profile1.GetFile()->ResetOffset());
- ASSERT_TRUE(file_info1.Load(GetFd(profile1)));
- ASSERT_TRUE(file_info1.Equals(info1));
-
- ProfileCompilationInfo file_info2;
- ASSERT_TRUE(reference_profile.GetFile()->ResetOffset());
- ASSERT_TRUE(file_info2.Load(GetFd(reference_profile)));
- ASSERT_TRUE(file_info2.Equals(reference_info));
-}
-
-} // namespace art