Updates uidmap to update snapshots and upload.
We send a snapshot of all installed apps with their uids every time
a user is added or removed and when statsd is started.
We keep track of the latest timestamp when a config key has retrieved
the UID map data. This allows us to remove older data when we're
guaranteed that all config sources have retrieved the old data.
Test: Added more unit tests to UidMap_test and passed on marlin-eng.
Change-Id: I34a3d61e75eedec44b98d896d7f6db0bc383f46a
diff --git a/cmds/statsd/Android.mk b/cmds/statsd/Android.mk
index db634d4..1c88a24 100644
--- a/cmds/statsd/Android.mk
+++ b/cmds/statsd/Android.mk
@@ -103,7 +103,8 @@
LOCAL_AIDL_INCLUDES := $(statsd_common_aidl_includes)
LOCAL_C_INCLUDES += $(statsd_common_c_includes)
-LOCAL_SHARED_LIBRARIES := $(statsd_common_shared_libraries)
+LOCAL_SHARED_LIBRARIES := $(statsd_common_shared_libraries) \
+ libgtest_prod
LOCAL_MODULE_CLASS := EXECUTABLES
diff --git a/cmds/statsd/src/StatsLogProcessor.cpp b/cmds/statsd/src/StatsLogProcessor.cpp
index cdaca1b..c0cedb1 100644
--- a/cmds/statsd/src/StatsLogProcessor.cpp
+++ b/cmds/statsd/src/StatsLogProcessor.cpp
@@ -60,6 +60,7 @@
unique_ptr<MetricsManager> newMetricsManager = std::make_unique<MetricsManager>(config);
if (newMetricsManager->isConfigValid()) {
+ mUidMap->OnConfigUpdated(key);
mMetricsManagers[key] = std::move(newMetricsManager);
// Why doesn't this work? mMetricsManagers.insert({key, std::move(newMetricsManager)});
ALOGD("StatsdConfig valid");
@@ -69,14 +70,27 @@
}
}
-vector<StatsLogReport> StatsLogProcessor::onDumpReport(const ConfigKey& key) {
+ConfigMetricsReport StatsLogProcessor::onDumpReport(const ConfigKey& key) {
+ ConfigMetricsReport report;
+
auto it = mMetricsManagers.find(key);
if (it == mMetricsManagers.end()) {
ALOGW("Config source %s does not exist", key.ToString().c_str());
- return vector<StatsLogReport>();
+ return report;
}
- return it->second->onDumpReport();
+ auto set_key = report.mutable_config_key();
+ set_key->set_uid(key.GetUid());
+ set_key->set_name(key.GetName());
+ for (auto m : it->second->onDumpReport()) {
+ // Transfer the vector of StatsLogReport into a field
+ // TODO: perhaps we just have bytes being returned from onDumpReport and transfer bytes
+ auto dest = report.add_metrics();
+ *dest = m;
+ }
+ auto temp = mUidMap->getOutput(key);
+ report.set_allocated_uid_map(&temp);
+ return report;
}
void StatsLogProcessor::OnConfigRemoved(const ConfigKey& key) {
@@ -84,6 +98,7 @@
if (it != mMetricsManagers.end()) {
it->second->finish();
mMetricsManagers.erase(it);
+ mUidMap->OnConfigRemoved(key);
}
auto flushTime = mLastFlushTimes.find(key);
if (flushTime != mLastFlushTimes.end()) {
diff --git a/cmds/statsd/src/StatsLogProcessor.h b/cmds/statsd/src/StatsLogProcessor.h
index 6463441..0083827 100644
--- a/cmds/statsd/src/StatsLogProcessor.h
+++ b/cmds/statsd/src/StatsLogProcessor.h
@@ -42,7 +42,7 @@
void OnConfigRemoved(const ConfigKey& key);
// TODO: Once we have the ProtoOutputStream in c++, we can just return byte array.
- std::vector<StatsLogReport> onDumpReport(const ConfigKey& key);
+ ConfigMetricsReport onDumpReport(const ConfigKey& key);
/* Request a flush through a binder call. */
void flush();
diff --git a/cmds/statsd/src/metrics/CountMetricProducer.h b/cmds/statsd/src/metrics/CountMetricProducer.h
index be77e47..8ed2234 100644
--- a/cmds/statsd/src/metrics/CountMetricProducer.h
+++ b/cmds/statsd/src/metrics/CountMetricProducer.h
@@ -53,6 +53,8 @@
// TODO: Implement this later.
virtual void notifyAppUpgrade(const string& apk, const int uid, const int version) override{};
+ // TODO: Implement this later.
+ virtual void notifyAppRemoved(const string& apk, const int uid) override{};
protected:
void onMatchedLogEventInternal(const size_t matcherIndex, const HashableDimensionKey& eventKey,
diff --git a/cmds/statsd/src/metrics/DurationMetricProducer.h b/cmds/statsd/src/metrics/DurationMetricProducer.h
index 8820403..29fc828 100644
--- a/cmds/statsd/src/metrics/DurationMetricProducer.h
+++ b/cmds/statsd/src/metrics/DurationMetricProducer.h
@@ -73,6 +73,8 @@
// TODO: Implement this later.
virtual void notifyAppUpgrade(const string& apk, const int uid, const int version) override{};
+ // TODO: Implement this later.
+ virtual void notifyAppRemoved(const string& apk, const int uid) override{};
protected:
void onMatchedLogEventInternal(const size_t matcherIndex, const HashableDimensionKey& eventKey,
diff --git a/cmds/statsd/src/metrics/EventMetricProducer.h b/cmds/statsd/src/metrics/EventMetricProducer.h
index 14fa31c..de1102b 100644
--- a/cmds/statsd/src/metrics/EventMetricProducer.h
+++ b/cmds/statsd/src/metrics/EventMetricProducer.h
@@ -55,6 +55,8 @@
// TODO: Implement this later.
virtual void notifyAppUpgrade(const string& apk, const int uid, const int version) override{};
+ // TODO: Implement this later.
+ virtual void notifyAppRemoved(const string& apk, const int uid) override{};
private:
const EventMetric mMetric;
diff --git a/cmds/statsd/src/packages/PackageInfoListener.h b/cmds/statsd/src/packages/PackageInfoListener.h
index 8b948de..13e776f 100644
--- a/cmds/statsd/src/packages/PackageInfoListener.h
+++ b/cmds/statsd/src/packages/PackageInfoListener.h
@@ -29,6 +29,9 @@
// Uid map will notify this listener that the app with apk name and uid has been upgraded to
// the specified version.
virtual void notifyAppUpgrade(const std::string& apk, const int uid, const int version) = 0;
+
+ // Notify interested listeners that the given apk and uid combination no longer exits.
+ virtual void notifyAppRemoved(const std::string& apk, const int uid) = 0;
};
} // namespace statsd
diff --git a/cmds/statsd/src/packages/UidMap.cpp b/cmds/statsd/src/packages/UidMap.cpp
index f4621ee..d83c3a4 100644
--- a/cmds/statsd/src/packages/UidMap.cpp
+++ b/cmds/statsd/src/packages/UidMap.cpp
@@ -52,25 +52,35 @@
void UidMap::updateMap(const vector<int32_t>& uid, const vector<int32_t>& versionCode,
const vector<String16>& packageName) {
+ updateMap(time(nullptr) * 1000000000, uid, versionCode, packageName);
+}
+
+void UidMap::updateMap(const int64_t& timestamp, const vector<int32_t>& uid,
+ const vector<int32_t>& versionCode, const vector<String16>& packageName) {
lock_guard<mutex> lock(mMutex); // Exclusively lock for updates.
mMap.clear();
- for (unsigned long j = 0; j < uid.size(); j++) {
+ for (size_t j = 0; j < uid.size(); j++) {
mMap.insert(make_pair(uid[j],
AppData(string(String8(packageName[j]).string()), versionCode[j])));
}
- if (mOutput.initial_size() == 0) { // Provide the initial states in the mOutput proto
- for (unsigned long j = 0; j < uid.size(); j++) {
- auto t = mOutput.add_initial();
- t->set_app(string(String8(packageName[j]).string()));
- t->set_version(int(versionCode[j]));
- t->set_uid(uid[j]);
- }
+ auto snapshot = mOutput.add_snapshots();
+ snapshot->set_timestamp_nanos(timestamp);
+ for (size_t j = 0; j < uid.size(); j++) {
+ auto t = snapshot->add_package_info();
+ t->set_name(string(String8(packageName[j]).string()));
+ t->set_version(int(versionCode[j]));
+ t->set_uid(uid[j]);
}
}
void UidMap::updateApp(const String16& app_16, const int32_t& uid, const int32_t& versionCode) {
+ updateApp(time(nullptr) * 1000000000, app_16, uid, versionCode);
+}
+
+void UidMap::updateApp(const int64_t& timestamp, const String16& app_16, const int32_t& uid,
+ const int32_t& versionCode) {
lock_guard<mutex> lock(mMutex);
string app = string(String8(app_16).string());
@@ -82,7 +92,7 @@
auto log = mOutput.add_changes();
log->set_deletion(false);
- // log.timestamp = TODO: choose how timestamps are computed
+ log->set_timestamp_nanos(timestamp);
log->set_app(app);
log->set_uid(uid);
log->set_version(versionCode);
@@ -102,13 +112,20 @@
}
void UidMap::removeApp(const String16& app_16, const int32_t& uid) {
+ removeApp(time(nullptr) * 1000000000, app_16, uid);
+}
+void UidMap::removeApp(const int64_t& timestamp, const String16& app_16, const int32_t& uid) {
lock_guard<mutex> lock(mMutex);
string app = string(String8(app_16).string());
+ for (auto it : mSubscribers) {
+ it->notifyAppRemoved(app, uid);
+ }
+
auto log = mOutput.add_changes();
log->set_deletion(true);
- // log.timestamp = TODO: choose how timestamps are computed
+ log->set_timestamp_nanos(timestamp);
log->set_app(app);
log->set_uid(uid);
@@ -133,21 +150,67 @@
mSubscribers.erase(producer);
}
-UidMapping UidMap::getAndClearOutput() {
- lock_guard<mutex> lock(mMutex); // Lock for updates
-
- auto ret = UidMapping(mOutput); // Copy that will be returned.
+void UidMap::clearOutput() {
mOutput.Clear();
// Re-initialize the initial state for the outputs. This results in extra data being uploaded
- // but helps ensure we can't re-construct the UID->app name, versionCode mapping in server.
+ // but helps ensure we can re-construct the UID->app name, versionCode mapping in server.
+ auto snapshot = mOutput.add_snapshots();
for (auto it : mMap) {
- auto t = mOutput.add_initial();
- t->set_app(it.second.packageName);
+ auto t = snapshot->add_package_info();
+ t->set_name(it.second.packageName);
t->set_version(it.second.versionCode);
t->set_uid(it.first);
}
+}
+int64_t UidMap::getMinimumTimestampNs() {
+ int64_t m = 0;
+ for (auto it : mLastUpdatePerConfigKey) {
+ if (m == 0) {
+ m = it.second;
+ } else if (it.second < m) {
+ m = it.second;
+ }
+ }
+ return m;
+}
+
+UidMapping UidMap::getOutput(const ConfigKey& key) {
+ return getOutput(time(nullptr) * 1000000000, key);
+}
+
+UidMapping UidMap::getOutput(const int64_t& timestamp, const ConfigKey& key) {
+ lock_guard<mutex> lock(mMutex); // Lock for updates
+
+ auto ret = UidMapping(mOutput); // Copy that will be returned.
+ int64_t prevMin = getMinimumTimestampNs();
+ mLastUpdatePerConfigKey[key] = timestamp;
+ int64_t newMin = getMinimumTimestampNs();
+
+ if (newMin > prevMin) {
+ int64_t cutoff_nanos = newMin;
+ auto snapshots = mOutput.mutable_snapshots();
+ auto it_snapshots = snapshots->cbegin();
+ while (it_snapshots != snapshots->cend()) {
+ if (it_snapshots->timestamp_nanos() < cutoff_nanos) {
+ // it_snapshots now points to the following element.
+ it_snapshots = snapshots->erase(it_snapshots);
+ } else {
+ ++it_snapshots;
+ }
+ }
+ auto deltas = mOutput.mutable_changes();
+ auto it_deltas = deltas->cbegin();
+ while (it_deltas != deltas->cend()) {
+ if (it_deltas->timestamp_nanos() < cutoff_nanos) {
+ // it_deltas now points to the following element.
+ it_deltas = deltas->erase(it_deltas);
+ } else {
+ ++it_deltas;
+ }
+ }
+ }
return ret;
}
@@ -160,6 +223,14 @@
}
}
+void UidMap::OnConfigUpdated(const ConfigKey& key) {
+ mLastUpdatePerConfigKey[key] = -1;
+}
+
+void UidMap::OnConfigRemoved(const ConfigKey& key) {
+ mLastUpdatePerConfigKey.erase(key);
+}
+
} // namespace statsd
} // namespace os
} // namespace android
diff --git a/cmds/statsd/src/packages/UidMap.h b/cmds/statsd/src/packages/UidMap.h
index d550372..bf120e0 100644
--- a/cmds/statsd/src/packages/UidMap.h
+++ b/cmds/statsd/src/packages/UidMap.h
@@ -17,11 +17,14 @@
#ifndef STATSD_UIDMAP_H
#define STATSD_UIDMAP_H
+#include "config/ConfigKey.h"
+#include "config/ConfigListener.h"
#include "frameworks/base/cmds/statsd/src/stats_log.pb.h"
#include "packages/PackageInfoListener.h"
#include <binder/IResultReceiver.h>
#include <binder/IShellCallback.h>
+#include <gtest/gtest_prod.h>
#include <log/logprint.h>
#include <stdio.h>
#include <utils/RefBase.h>
@@ -51,17 +54,19 @@
* All three inputs must be the same size, and the jth element in each array refers to the same
* tuple, ie. uid[j] corresponds to packageName[j] with versionCode[j].
*/
+ // TODO: Add safeguards to call clearOutput if there's too much data already stored.
void updateMap(const vector<int32_t>& uid, const vector<int32_t>& versionCode,
const vector<String16>& packageName);
+ // TODO: Add safeguards to call clearOutput if there's too much data already stored.
+ void updateApp(const String16& packageName, const int32_t& uid, const int32_t& versionCode);
+ void removeApp(const String16& packageName, const int32_t& uid);
+
// Returns true if the given uid contains the specified app (eg. com.google.android.gms).
bool hasApp(int uid, const string& packageName) const;
int getAppVersion(int uid, const string& packageName) const;
- void updateApp(const String16& packageName, const int32_t& uid, const int32_t& versionCode);
- void removeApp(const String16& packageName, const int32_t& uid);
-
// Helper for debugging contents of this uid map. Can be triggered with:
// adb shell cmd stats print-uid-map
void printUidMap(FILE* out);
@@ -73,13 +78,36 @@
// Remove the listener from the set of metric producers that subscribe to updates.
void removeListener(sp<PackageInfoListener> producer);
- // Grabs the current output contents and then clears it.
- UidMapping getAndClearOutput();
+ // Informs uid map that a config is added/updated. Used for keeping mConfigKeys up to date.
+ void OnConfigUpdated(const ConfigKey& key);
+
+ // Informs uid map that a config is removed. Used for keeping mConfigKeys up to date.
+ void OnConfigRemoved(const ConfigKey& key);
+
+ // Gets the output. If every config key has received the output, then the output is cleared.
+ UidMapping getOutput(const ConfigKey& key);
+
+ // Forces the output to be cleared. We still generate a snapshot based on the current state.
+ // This results in extra data uploaded but helps us reconstruct the uid mapping on the server
+ // in case we lose a previous upload.
+ void clearOutput();
private:
+ void updateMap(const int64_t& timestamp, const vector<int32_t>& uid,
+ const vector<int32_t>& versionCode, const vector<String16>& packageName);
+
+ // TODO: Add safeguards to call clearOutput if there's too much data already stored.
+ void updateApp(const int64_t& timestamp, const String16& packageName, const int32_t& uid,
+ const int32_t& versionCode);
+ void removeApp(const int64_t& timestamp, const String16& packageName, const int32_t& uid);
+
+ UidMapping getOutput(const int64_t& timestamp, const ConfigKey& key);
+
// TODO: Use shared_mutex for improved read-locking if a library can be found in Android.
mutable mutex mMutex;
+ // Maps uid to application data. This must be multimap since there is a feature in Android for
+ // multiple apps to share the same uid.
std::unordered_multimap<int, AppData> mMap;
// We prepare the output proto as apps are updated, so that we can grab the current output.
@@ -87,6 +115,17 @@
// Metric producers that should be notified if there's an upgrade in any app.
set<sp<PackageInfoListener>> mSubscribers;
+
+ // Mapping of config keys we're aware of to the epoch time they last received an update. This
+ // lets us know it's safe to delete events older than the oldest update. The value is nanosec.
+ // Value of -1 denotes this config key has never received an upload.
+ std::unordered_map<ConfigKey, int64_t> mLastUpdatePerConfigKey;
+
+ // Returns the minimum value from mConfigKeys.
+ int64_t getMinimumTimestampNs();
+
+ // Allows unit-test to access private methods.
+ FRIEND_TEST(UidMapTest, TestClearingOutput);
};
} // namespace statsd
diff --git a/cmds/statsd/src/stats_log.proto b/cmds/statsd/src/stats_log.proto
index ec91509..66a31a5 100644
--- a/cmds/statsd/src/stats_log.proto
+++ b/cmds/statsd/src/stats_log.proto
@@ -98,15 +98,19 @@
}
message UidMapping {
- message AppInfo {
- optional string app = 1;
+ message PackageInfoSnapshot {
+ message PackageInfo {
+ optional string name = 1;
- optional int32 version = 2;
+ optional int32 version = 2;
- optional int32 uid = 3;
+ optional int32 uid = 3;
+ }
+ optional int64 timestamp_nanos = 1;
+
+ repeated PackageInfo package_info = 2;
}
-
- repeated AppInfo initial = 1;
+ repeated PackageInfoSnapshot snapshots = 1;
message Change {
optional bool deletion = 1;
diff --git a/cmds/statsd/src/statsd_config.proto b/cmds/statsd/src/statsd_config.proto
index a4d2421..72bde1f 100644
--- a/cmds/statsd/src/statsd_config.proto
+++ b/cmds/statsd/src/statsd_config.proto
@@ -210,14 +210,7 @@
repeated EventConditionLink links = 8;
enum Operation {
- SUM_DIFF = 1;
- MIN_DIFF = 2;
- MAX_DIFF = 3;
- SUM = 4;
- MIN = 5;
- MAX = 6;
- FIRST = 7;
- LAST = 8;
+ SUM = 1;
}
optional Operation operation = 9 [default = SUM];
}
diff --git a/cmds/statsd/tests/UidMap_test.cpp b/cmds/statsd/tests/UidMap_test.cpp
index f9a90e4..671f6d4 100644
--- a/cmds/statsd/tests/UidMap_test.cpp
+++ b/cmds/statsd/tests/UidMap_test.cpp
@@ -13,13 +13,17 @@
// limitations under the License.
#include "packages/UidMap.h"
+#include "config/ConfigKey.h"
#include <gtest/gtest.h>
#include <stdio.h>
using namespace android;
-using namespace android::os::statsd;
+
+namespace android {
+namespace os {
+namespace statsd {
#ifdef __ANDROID__
const string kApp1 = "app1.sharing.1";
@@ -64,6 +68,57 @@
EXPECT_FALSE(m.hasApp(1000, kApp1));
EXPECT_TRUE(m.hasApp(1000, kApp2));
}
+
+TEST(UidMapTest, TestClearingOutput) {
+ UidMap m;
+
+ ConfigKey config1(1, "config1");
+ ConfigKey config2(1, "config2");
+
+ m.OnConfigUpdated(config1);
+
+ vector<int32_t> uids;
+ vector<int32_t> versions;
+ vector<String16> apps;
+ uids.push_back(1000);
+ uids.push_back(1000);
+ apps.push_back(String16(kApp1.c_str()));
+ apps.push_back(String16(kApp2.c_str()));
+ versions.push_back(4);
+ versions.push_back(5);
+ m.updateMap(1, uids, versions, apps);
+
+ UidMapping results = m.getOutput(2, config1);
+ EXPECT_EQ(1, results.snapshots_size());
+
+ // It should be cleared now
+ results = m.getOutput(3, config1);
+ EXPECT_EQ(0, results.snapshots_size());
+
+ // Now add another configuration.
+ m.OnConfigUpdated(config2);
+ m.updateApp(5, String16(kApp1.c_str()), 1000, 40);
+ results = m.getOutput(6, config1);
+ EXPECT_EQ(0, results.snapshots_size());
+ EXPECT_EQ(1, results.changes_size());
+
+ // Now we still haven't been able to delete anything
+ m.updateApp(7, String16(kApp2.c_str()), 1001, 41);
+ results = m.getOutput(8, config1);
+ EXPECT_EQ(0, results.snapshots_size());
+ EXPECT_EQ(2, results.changes_size());
+
+ results = m.getOutput(9, config2);
+ EXPECT_EQ(0, results.snapshots_size());
+ EXPECT_EQ(2, results.changes_size());
+ // At this point both should be cleared.
+ EXPECT_EQ(0, m.mOutput.snapshots_size());
+ EXPECT_EQ(0, m.mOutput.changes_size());
+}
#else
GTEST_LOG_(INFO) << "This test does nothing.\n";
#endif
+
+} // namespace statsd
+} // namespace os
+} // namespace android
diff --git a/services/core/java/com/android/server/stats/StatsCompanionService.java b/services/core/java/com/android/server/stats/StatsCompanionService.java
index 22d2bcf..2219de8 100644
--- a/services/core/java/com/android/server/stats/StatsCompanionService.java
+++ b/services/core/java/com/android/server/stats/StatsCompanionService.java
@@ -144,15 +144,16 @@
public final static class AppUpdateReceiver extends BroadcastReceiver {
@Override
public void onReceive(Context context, Intent intent) {
- Slog.i(TAG, "StatsCompanionService noticed an app was updated.");
/**
* App updates actually consist of REMOVE, ADD, and then REPLACE broadcasts. To avoid
* waste, we ignore the REMOVE and ADD broadcasts that contain the replacing flag.
+ * If we can't find the value for EXTRA_REPLACING, we default to false.
*/
- if (!intent.getAction().equals(Intent.ACTION_PACKAGE_REPLACED) &&
- intent.getBooleanExtra(Intent.EXTRA_REPLACING, false)) {
+ if (!intent.getAction().equals(Intent.ACTION_PACKAGE_REPLACED)
+ && intent.getBooleanExtra(Intent.EXTRA_REPLACING, false)) {
return; // Keep only replacing or normal add and remove.
}
+ Slog.i(TAG, "StatsCompanionService noticed an app was updated.");
synchronized (sStatsdLock) {
if (sStatsd == null) {
Slog.w(TAG, "Could not access statsd to inform it of anomaly alarm firing");