Healthcheck: proper job allocation and test.
Bug: 153874006
Test: atest PackageManagerShellCommandTest PackageManagerShellCommandIncrementalTest IncrementalServiceTest
Change-Id: Iede1f2297cc4f8e3c3f0acd43cee597f75dff179
diff --git a/services/incremental/test/IncrementalServiceTest.cpp b/services/incremental/test/IncrementalServiceTest.cpp
index 84ec7d3..26b5094 100644
--- a/services/incremental/test/IncrementalServiceTest.cpp
+++ b/services/incremental/test/IncrementalServiceTest.cpp
@@ -22,6 +22,7 @@
#include <gtest/gtest.h>
#include <utils/Log.h>
+#include <chrono>
#include <future>
#include "IncrementalService.h"
@@ -295,9 +296,21 @@
void openMountSuccess() {
ON_CALL(*this, openMount(_)).WillByDefault(Invoke(this, &MockIncFs::openMountForHealth));
}
- void waitForPendingReadsSuccess() {
+
+ // 1000ms
+ void waitForPendingReadsSuccess(uint64_t ts = 0) {
ON_CALL(*this, waitForPendingReads(_, _, _))
- .WillByDefault(Invoke(this, &MockIncFs::waitForPendingReadsForHealth));
+ .WillByDefault(
+ Invoke([ts](const Control& control, std::chrono::milliseconds timeout,
+ std::vector<incfs::ReadInfo>* pendingReadsBuffer) {
+ pendingReadsBuffer->push_back({.bootClockTsUs = ts});
+ return android::incfs::WaitResult::HaveData;
+ }));
+ }
+
+ void waitForPendingReadsTimeout() {
+ ON_CALL(*this, waitForPendingReads(_, _, _))
+ .WillByDefault(Return(android::incfs::WaitResult::Timeout));
}
static constexpr auto kPendingReadsFd = 42;
@@ -305,13 +318,6 @@
return UniqueControl(IncFs_CreateControl(-1, kPendingReadsFd, -1));
}
- WaitResult waitForPendingReadsForHealth(
- const Control& control, std::chrono::milliseconds timeout,
- std::vector<incfs::ReadInfo>* pendingReadsBuffer) const {
- pendingReadsBuffer->push_back({.bootClockTsUs = 0});
- return android::incfs::WaitResult::HaveData;
- }
-
RawMetadata getMountInfoMetadata(const Control& control, std::string_view path) {
metadata::Mount m;
m.mutable_storage()->set_id(100);
@@ -371,7 +377,7 @@
public:
MOCK_CONST_METHOD0(initializeForCurrentThread, void());
- MockJniWrapper() { EXPECT_CALL(*this, initializeForCurrentThread()).Times(3); }
+ MockJniWrapper() { EXPECT_CALL(*this, initializeForCurrentThread()).Times(2); }
};
class MockLooperWrapper : public LooperWrapper {
@@ -385,7 +391,7 @@
ON_CALL(*this, addFd(_, _, _, _, _))
.WillByDefault(Invoke(this, &MockLooperWrapper::storeCallback));
ON_CALL(*this, removeFd(_)).WillByDefault(Invoke(this, &MockLooperWrapper::clearCallback));
- ON_CALL(*this, pollAll(_)).WillByDefault(Invoke(this, &MockLooperWrapper::sleepFor));
+ ON_CALL(*this, pollAll(_)).WillByDefault(Invoke(this, &MockLooperWrapper::wait10Ms));
}
int storeCallback(int, int, int, android::Looper_callbackFunc callback, void* data) {
@@ -400,8 +406,10 @@
return 0;
}
- int sleepFor(int timeoutMillis) {
- std::this_thread::sleep_for(std::chrono::milliseconds(timeoutMillis));
+ int wait10Ms(int) {
+ // This is called from a loop in runCmdLooper.
+ // Sleeping for 10ms only to avoid busy looping.
+ std::this_thread::sleep_for(10ms);
return 0;
}
@@ -409,6 +417,55 @@
void* mCallbackData = nullptr;
};
+class MockTimedQueueWrapper : public TimedQueueWrapper {
+public:
+ MOCK_METHOD3(addJob, void(MountId, Milliseconds, Job));
+ MOCK_METHOD1(removeJobs, void(MountId));
+ MOCK_METHOD0(stop, void());
+
+ MockTimedQueueWrapper() {
+ ON_CALL(*this, addJob(_, _, _))
+ .WillByDefault(Invoke(this, &MockTimedQueueWrapper::storeJob));
+ ON_CALL(*this, removeJobs(_)).WillByDefault(Invoke(this, &MockTimedQueueWrapper::clearJob));
+ }
+
+ void storeJob(MountId id, Milliseconds after, Job what) {
+ mId = id;
+ mAfter = after;
+ mWhat = std::move(what);
+ }
+
+ void clearJob(MountId id) {
+ if (mId == id) {
+ mAfter = {};
+ mWhat = {};
+ }
+ }
+
+ MountId mId = -1;
+ Milliseconds mAfter;
+ Job mWhat;
+};
+
+class MockStorageHealthListener : public os::incremental::BnStorageHealthListener {
+public:
+ MOCK_METHOD2(onHealthStatus, binder::Status(int32_t storageId, int32_t status));
+
+ MockStorageHealthListener() {
+ ON_CALL(*this, onHealthStatus(_, _))
+ .WillByDefault(Invoke(this, &MockStorageHealthListener::storeStorageIdAndStatus));
+ }
+
+ binder::Status storeStorageIdAndStatus(int32_t storageId, int32_t status) {
+ mStorageId = storageId;
+ mStatus = status;
+ return binder::Status::ok();
+ }
+
+ int32_t mStorageId = -1;
+ int32_t mStatus = -1;
+};
+
class MockServiceManager : public ServiceManagerWrapper {
public:
MockServiceManager(std::unique_ptr<MockVoldService> vold,
@@ -416,13 +473,15 @@
std::unique_ptr<MockIncFs> incfs,
std::unique_ptr<MockAppOpsManager> appOpsManager,
std::unique_ptr<MockJniWrapper> jni,
- std::unique_ptr<MockLooperWrapper> looper)
+ std::unique_ptr<MockLooperWrapper> looper,
+ std::unique_ptr<MockTimedQueueWrapper> timedQueue)
: mVold(std::move(vold)),
mDataLoaderManager(std::move(dataLoaderManager)),
mIncFs(std::move(incfs)),
mAppOpsManager(std::move(appOpsManager)),
mJni(std::move(jni)),
- mLooper(std::move(looper)) {}
+ mLooper(std::move(looper)),
+ mTimedQueue(std::move(timedQueue)) {}
std::unique_ptr<VoldServiceWrapper> getVoldService() final { return std::move(mVold); }
std::unique_ptr<DataLoaderManagerWrapper> getDataLoaderManager() final {
return std::move(mDataLoaderManager);
@@ -431,6 +490,7 @@
std::unique_ptr<AppOpsManagerWrapper> getAppOpsManager() final { return std::move(mAppOpsManager); }
std::unique_ptr<JniWrapper> getJni() final { return std::move(mJni); }
std::unique_ptr<LooperWrapper> getLooper() final { return std::move(mLooper); }
+ std::unique_ptr<TimedQueueWrapper> getTimedQueue() final { return std::move(mTimedQueue); }
private:
std::unique_ptr<MockVoldService> mVold;
@@ -439,6 +499,7 @@
std::unique_ptr<MockAppOpsManager> mAppOpsManager;
std::unique_ptr<MockJniWrapper> mJni;
std::unique_ptr<MockLooperWrapper> mLooper;
+ std::unique_ptr<MockTimedQueueWrapper> mTimedQueue;
};
// --- IncrementalServiceTest ---
@@ -460,6 +521,8 @@
mJni = jni.get();
auto looper = std::make_unique<NiceMock<MockLooperWrapper>>();
mLooper = looper.get();
+ auto timedQueue = std::make_unique<NiceMock<MockTimedQueueWrapper>>();
+ mTimedQueue = timedQueue.get();
mIncrementalService =
std::make_unique<IncrementalService>(MockServiceManager(std::move(vold),
std::move(
@@ -467,7 +530,8 @@
std::move(incFs),
std::move(appOps),
std::move(jni),
- std::move(looper)),
+ std::move(looper),
+ std::move(timedQueue)),
mRootDir.path);
mDataLoaderParcel.packageName = "com.test";
mDataLoaderParcel.arguments = "uri";
@@ -503,6 +567,7 @@
NiceMock<MockAppOpsManager>* mAppOpsManager = nullptr;
NiceMock<MockJniWrapper>* mJni = nullptr;
NiceMock<MockLooperWrapper>* mLooper = nullptr;
+ NiceMock<MockTimedQueueWrapper>* mTimedQueue = nullptr;
NiceMock<MockDataLoader>* mDataLoader = nullptr;
std::unique_ptr<IncrementalService> mIncrementalService;
TemporaryDir mRootDir;
@@ -710,6 +775,136 @@
mLooper->mCallback(-1, -1, mLooper->mCallbackData);
}
+TEST_F(IncrementalServiceTest, testStartDataLoaderUnhealthyStorage) {
+ mVold->mountIncFsSuccess();
+ mIncFs->makeFileSuccess();
+ mIncFs->openMountSuccess();
+ mVold->bindMountSuccess();
+ mDataLoaderManager->bindToDataLoaderSuccess();
+ mDataLoaderManager->getDataLoaderSuccess();
+ EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _)).Times(1);
+ EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_)).Times(1);
+ EXPECT_CALL(*mDataLoader, create(_, _, _, _)).Times(1);
+ EXPECT_CALL(*mDataLoader, start(_)).Times(1);
+ EXPECT_CALL(*mDataLoader, destroy(_)).Times(1);
+ EXPECT_CALL(*mVold, unmountIncFs(_)).Times(2);
+ EXPECT_CALL(*mLooper, addFd(MockIncFs::kPendingReadsFd, _, _, _, _)).Times(2);
+ EXPECT_CALL(*mLooper, removeFd(MockIncFs::kPendingReadsFd)).Times(2);
+ EXPECT_CALL(*mTimedQueue, addJob(_, _, _)).Times(4);
+
+ sp<NiceMock<MockStorageHealthListener>> listener{new NiceMock<MockStorageHealthListener>};
+ NiceMock<MockStorageHealthListener>* listenerMock = listener.get();
+ EXPECT_CALL(*listenerMock, onHealthStatus(_, IStorageHealthListener::HEALTH_STATUS_OK))
+ .Times(2);
+ EXPECT_CALL(*listenerMock,
+ onHealthStatus(_, IStorageHealthListener::HEALTH_STATUS_READS_PENDING))
+ .Times(1);
+ EXPECT_CALL(*listenerMock, onHealthStatus(_, IStorageHealthListener::HEALTH_STATUS_BLOCKED))
+ .Times(1);
+ EXPECT_CALL(*listenerMock, onHealthStatus(_, IStorageHealthListener::HEALTH_STATUS_UNHEALTHY))
+ .Times(2);
+
+ StorageHealthCheckParams params;
+ params.blockedTimeoutMs = 10000;
+ params.unhealthyTimeoutMs = 20000;
+ params.unhealthyMonitoringMs = 30000;
+
+ using MS = std::chrono::milliseconds;
+ using MCS = std::chrono::microseconds;
+
+ const auto blockedTimeout = MS(params.blockedTimeoutMs);
+ const auto unhealthyTimeout = MS(params.unhealthyTimeoutMs);
+ const auto unhealthyMonitoring = MS(params.unhealthyMonitoringMs);
+
+ const uint64_t kFirstTimestampUs = 1000000000ll;
+ const uint64_t kBlockedTimestampUs =
+ kFirstTimestampUs - std::chrono::duration_cast<MCS>(blockedTimeout).count();
+ const uint64_t kUnhealthyTimestampUs =
+ kFirstTimestampUs - std::chrono::duration_cast<MCS>(unhealthyTimeout).count();
+
+ TemporaryDir tempDir;
+ int storageId = mIncrementalService->createStorage(tempDir.path, std::move(mDataLoaderParcel),
+ IncrementalService::CreateOptions::CreateNew,
+ {}, std::move(params), listener);
+ ASSERT_GE(storageId, 0);
+
+ // Healthy state, registered for pending reads.
+ ASSERT_NE(nullptr, mLooper->mCallback);
+ ASSERT_NE(nullptr, mLooper->mCallbackData);
+ ASSERT_EQ(storageId, listener->mStorageId);
+ ASSERT_EQ(IStorageHealthListener::HEALTH_STATUS_OK, listener->mStatus);
+
+ // Looper/epoll callback.
+ mIncFs->waitForPendingReadsSuccess(kFirstTimestampUs);
+ mLooper->mCallback(-1, -1, mLooper->mCallbackData);
+
+ // Unregister from pending reads and wait.
+ ASSERT_EQ(nullptr, mLooper->mCallback);
+ ASSERT_EQ(nullptr, mLooper->mCallbackData);
+ ASSERT_EQ(storageId, listener->mStorageId);
+ ASSERT_EQ(IStorageHealthListener::HEALTH_STATUS_READS_PENDING, listener->mStatus);
+ // Timed callback present.
+ ASSERT_EQ(storageId, mTimedQueue->mId);
+ ASSERT_GE(mTimedQueue->mAfter, blockedTimeout);
+ auto timedCallback = mTimedQueue->mWhat;
+ mTimedQueue->clearJob(storageId);
+
+ // Timed job callback for blocked.
+ mIncFs->waitForPendingReadsSuccess(kBlockedTimestampUs);
+ timedCallback();
+
+ // Still not registered, and blocked.
+ ASSERT_EQ(nullptr, mLooper->mCallback);
+ ASSERT_EQ(nullptr, mLooper->mCallbackData);
+ ASSERT_EQ(storageId, listener->mStorageId);
+ ASSERT_EQ(IStorageHealthListener::HEALTH_STATUS_BLOCKED, listener->mStatus);
+ // Timed callback present.
+ ASSERT_EQ(storageId, mTimedQueue->mId);
+ ASSERT_GE(mTimedQueue->mAfter, 1000ms);
+ timedCallback = mTimedQueue->mWhat;
+ mTimedQueue->clearJob(storageId);
+
+ // Timed job callback for unhealthy.
+ mIncFs->waitForPendingReadsSuccess(kUnhealthyTimestampUs);
+ timedCallback();
+
+ // Still not registered, and blocked.
+ ASSERT_EQ(nullptr, mLooper->mCallback);
+ ASSERT_EQ(nullptr, mLooper->mCallbackData);
+ ASSERT_EQ(storageId, listener->mStorageId);
+ ASSERT_EQ(IStorageHealthListener::HEALTH_STATUS_UNHEALTHY, listener->mStatus);
+ // Timed callback present.
+ ASSERT_EQ(storageId, mTimedQueue->mId);
+ ASSERT_GE(mTimedQueue->mAfter, unhealthyMonitoring);
+ timedCallback = mTimedQueue->mWhat;
+ mTimedQueue->clearJob(storageId);
+
+ // One more unhealthy.
+ mIncFs->waitForPendingReadsSuccess(kUnhealthyTimestampUs);
+ timedCallback();
+
+ // Still not registered, and blocked.
+ ASSERT_EQ(nullptr, mLooper->mCallback);
+ ASSERT_EQ(nullptr, mLooper->mCallbackData);
+ ASSERT_EQ(storageId, listener->mStorageId);
+ ASSERT_EQ(IStorageHealthListener::HEALTH_STATUS_UNHEALTHY, listener->mStatus);
+ // Timed callback present.
+ ASSERT_EQ(storageId, mTimedQueue->mId);
+ ASSERT_GE(mTimedQueue->mAfter, unhealthyMonitoring);
+ timedCallback = mTimedQueue->mWhat;
+ mTimedQueue->clearJob(storageId);
+
+ // And now healthy.
+ mIncFs->waitForPendingReadsTimeout();
+ timedCallback();
+
+ // Healthy state, registered for pending reads.
+ ASSERT_NE(nullptr, mLooper->mCallback);
+ ASSERT_NE(nullptr, mLooper->mCallbackData);
+ ASSERT_EQ(storageId, listener->mStorageId);
+ ASSERT_EQ(IStorageHealthListener::HEALTH_STATUS_OK, listener->mStatus);
+}
+
TEST_F(IncrementalServiceTest, testSetIncFsMountOptionsSuccess) {
mVold->mountIncFsSuccess();
mIncFs->makeFileSuccess();