Reduce statsd log data size.
1. Hash the strings in metric dimensions.
2. Optimize the timestamp encoding in bucket.
Use bucket num for full bucket and millis for
partial bucket.
3. Encode the dimension path per metric and avoid
deduping it across dimensons.
Test: statsd test
Change-Id: I18f69654de85edb21a9c835c73edead756295e05
BUG: b/77813755
diff --git a/cmds/statsd/Android.mk b/cmds/statsd/Android.mk
index b085a09..7198bad 100644
--- a/cmds/statsd/Android.mk
+++ b/cmds/statsd/Android.mk
@@ -19,6 +19,7 @@
../../core/java/android/os/IStatsManager.aidl \
src/statsd_config.proto \
src/FieldValue.cpp \
+ src/hash.cpp \
src/stats_log_util.cpp \
src/anomaly/AlarmMonitor.cpp \
src/anomaly/AlarmTracker.cpp \
diff --git a/cmds/statsd/src/FieldValue.cpp b/cmds/statsd/src/FieldValue.cpp
index dfd8705..f150f07 100644
--- a/cmds/statsd/src/FieldValue.cpp
+++ b/cmds/statsd/src/FieldValue.cpp
@@ -237,6 +237,18 @@
return false;
}
+bool HasPositionALL(const FieldMatcher& matcher) {
+ if (matcher.has_position() && matcher.position() == Position::ALL) {
+ return true;
+ }
+ for (const auto& child : matcher.child()) {
+ if (HasPositionALL(child)) {
+ return true;
+ }
+ }
+ return false;
+}
+
} // namespace statsd
} // namespace os
} // namespace android
\ No newline at end of file
diff --git a/cmds/statsd/src/FieldValue.h b/cmds/statsd/src/FieldValue.h
index f7ce23b..02c49b9 100644
--- a/cmds/statsd/src/FieldValue.h
+++ b/cmds/statsd/src/FieldValue.h
@@ -351,6 +351,7 @@
};
bool HasPositionANY(const FieldMatcher& matcher);
+bool HasPositionALL(const FieldMatcher& matcher);
bool isAttributionUidField(const FieldValue& value);
diff --git a/cmds/statsd/src/StatsLogProcessor.cpp b/cmds/statsd/src/StatsLogProcessor.cpp
index d548c0a..5219885 100644
--- a/cmds/statsd/src/StatsLogProcessor.cpp
+++ b/cmds/statsd/src/StatsLogProcessor.cpp
@@ -65,6 +65,7 @@
const int FIELD_ID_LAST_REPORT_WALL_CLOCK_NANOS = 5;
const int FIELD_ID_CURRENT_REPORT_WALL_CLOCK_NANOS = 6;
const int FIELD_ID_DUMP_REPORT_REASON = 8;
+const int FIELD_ID_STRINGS = 9;
#define NS_PER_HOUR 3600 * NS_PER_SEC
@@ -293,6 +294,7 @@
*/
void StatsLogProcessor::onDumpReport(const ConfigKey& key, const int64_t dumpTimeStampNs,
const bool include_current_partial_bucket,
+ const bool include_string,
const DumpReportReason dumpReportReason,
vector<uint8_t>* outData) {
std::lock_guard<std::mutex> lock(mMetricsMutex);
@@ -320,7 +322,7 @@
uint64_t reportsToken =
proto.start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_REPORTS);
onConfigMetricsReportLocked(key, dumpTimeStampNs, include_current_partial_bucket,
- dumpReportReason, &proto);
+ include_string, dumpReportReason, &proto);
proto.end(reportsToken);
// End of ConfigMetricsReport (reports).
} else {
@@ -349,6 +351,7 @@
void StatsLogProcessor::onConfigMetricsReportLocked(const ConfigKey& key,
const int64_t dumpTimeStampNs,
const bool include_current_partial_bucket,
+ const bool include_string,
const DumpReportReason dumpReportReason,
ProtoOutputStream* proto) {
// We already checked whether key exists in mMetricsManagers in
@@ -360,13 +363,16 @@
int64_t lastReportTimeNs = it->second->getLastReportTimeNs();
int64_t lastReportWallClockNs = it->second->getLastReportWallClockNs();
+ std::set<string> str_set;
+
// First, fill in ConfigMetricsReport using current data on memory, which
// starts from filling in StatsLogReport's.
- it->second->onDumpReport(dumpTimeStampNs, include_current_partial_bucket, proto);
+ it->second->onDumpReport(dumpTimeStampNs, include_current_partial_bucket,
+ &str_set, proto);
// Fill in UidMap.
uint64_t uidMapToken = proto->start(FIELD_TYPE_MESSAGE | FIELD_ID_UID_MAP);
- mUidMap->appendUidMap(dumpTimeStampNs, key, proto);
+ mUidMap->appendUidMap(dumpTimeStampNs, key, &str_set, proto);
proto->end(uidMapToken);
// Fill in the timestamps.
@@ -380,6 +386,12 @@
(long long)getWallClockNs());
// Dump report reason
proto->write(FIELD_TYPE_INT32 | FIELD_ID_DUMP_REPORT_REASON, dumpReportReason);
+
+ if (include_string) {
+ for (const auto& str : str_set) {
+ proto->write(FIELD_TYPE_STRING | FIELD_COUNT_REPEATED | FIELD_ID_STRINGS, str);
+ }
+ }
}
void StatsLogProcessor::resetConfigsLocked(const int64_t timestampNs,
@@ -465,7 +477,8 @@
const DumpReportReason dumpReportReason) {
ProtoOutputStream proto;
onConfigMetricsReportLocked(key, getElapsedRealtimeNs(),
- true /* include_current_partial_bucket*/, dumpReportReason, &proto);
+ true /* include_current_partial_bucket*/,
+ false /* include strings */, dumpReportReason, &proto);
string file_name = StringPrintf("%s/%ld_%d_%lld", STATS_DATA_DIR,
(long)getWallClockSec(), key.GetUid(), (long long)key.GetId());
android::base::unique_fd fd(open(file_name.c_str(),
diff --git a/cmds/statsd/src/StatsLogProcessor.h b/cmds/statsd/src/StatsLogProcessor.h
index c2337c1..c3c4663 100644
--- a/cmds/statsd/src/StatsLogProcessor.h
+++ b/cmds/statsd/src/StatsLogProcessor.h
@@ -63,7 +63,7 @@
size_t GetMetricsSize(const ConfigKey& key) const;
void onDumpReport(const ConfigKey& key, const int64_t dumpTimeNs,
- const bool include_current_partial_bucket,
+ const bool include_current_partial_bucket, const bool include_string,
const DumpReportReason dumpReportReason, vector<uint8_t>* outData);
/* Tells MetricsManager that the alarms in alarmSet have fired. Modifies anomaly alarmSet. */
@@ -126,6 +126,7 @@
void onConfigMetricsReportLocked(const ConfigKey& key, const int64_t dumpTimeStampNs,
const bool include_current_partial_bucket,
+ const bool include_string,
const DumpReportReason dumpReportReason,
util::ProtoOutputStream* proto);
diff --git a/cmds/statsd/src/StatsService.cpp b/cmds/statsd/src/StatsService.cpp
index 5229071..0e7b4f9 100644
--- a/cmds/statsd/src/StatsService.cpp
+++ b/cmds/statsd/src/StatsService.cpp
@@ -592,7 +592,8 @@
if (good) {
vector<uint8_t> data;
mProcessor->onDumpReport(ConfigKey(uid, StrToInt64(name)), getElapsedRealtimeNs(),
- false /* include_current_bucket*/, ADB_DUMP, &data);
+ false /* include_current_bucket*/,
+ true /* include strings */, ADB_DUMP, &data);
// TODO: print the returned StatsLogReport to file instead of printing to logcat.
if (proto) {
for (size_t i = 0; i < data.size(); i ++) {
@@ -865,8 +866,9 @@
IPCThreadState* ipc = IPCThreadState::self();
VLOG("StatsService::getData with Pid %i, Uid %i", ipc->getCallingPid(), ipc->getCallingUid());
ConfigKey configKey(ipc->getCallingUid(), key);
- mProcessor->onDumpReport(configKey, getElapsedRealtimeNs(), false /* include_current_bucket*/,
- GET_DATA_CALLED, output);
+ mProcessor->onDumpReport(configKey, getElapsedRealtimeNs(),
+ false /* include_current_bucket*/, true /* include strings */,
+ GET_DATA_CALLED, output);
return Status::ok();
}
diff --git a/cmds/statsd/src/hash.cpp b/cmds/statsd/src/hash.cpp
new file mode 100644
index 0000000..c501c9f
--- /dev/null
+++ b/cmds/statsd/src/hash.cpp
@@ -0,0 +1,130 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hash.h"
+
+namespace android {
+namespace os {
+namespace statsd {
+
+namespace {
+// Lower-level versions of Get... that read directly from a character buffer
+// without any bounds checking.
+inline uint32_t DecodeFixed32(const char *ptr) {
+ return ((static_cast<uint32_t>(static_cast<unsigned char>(ptr[0]))) |
+ (static_cast<uint32_t>(static_cast<unsigned char>(ptr[1])) << 8) |
+ (static_cast<uint32_t>(static_cast<unsigned char>(ptr[2])) << 16) |
+ (static_cast<uint32_t>(static_cast<unsigned char>(ptr[3])) << 24));
+}
+
+inline uint64_t DecodeFixed64(const char* ptr) {
+ uint64_t lo = DecodeFixed32(ptr);
+ uint64_t hi = DecodeFixed32(ptr + 4);
+ return (hi << 32) | lo;
+}
+
+// 0xff is in case char is signed.
+static inline uint32_t ByteAs32(char c) { return static_cast<uint32_t>(c) & 0xff; }
+static inline uint64_t ByteAs64(char c) { return static_cast<uint64_t>(c) & 0xff; }
+
+} // namespace
+
+uint32_t Hash32(const char *data, size_t n, uint32_t seed) {
+ // 'm' and 'r' are mixing constants generated offline.
+ // They're not really 'magic', they just happen to work well.
+ const uint32_t m = 0x5bd1e995;
+ const int r = 24;
+
+ // Initialize the hash to a 'random' value
+ uint32_t h = static_cast<uint32_t>(seed ^ n);
+
+ // Mix 4 bytes at a time into the hash
+ while (n >= 4) {
+ uint32_t k = DecodeFixed32(data);
+ k *= m;
+ k ^= k >> r;
+ k *= m;
+ h *= m;
+ h ^= k;
+ data += 4;
+ n -= 4;
+ }
+
+ // Handle the last few bytes of the input array
+ switch (n) {
+ case 3:
+ h ^= ByteAs32(data[2]) << 16;
+ case 2:
+ h ^= ByteAs32(data[1]) << 8;
+ case 1:
+ h ^= ByteAs32(data[0]);
+ h *= m;
+ }
+
+ // Do a few final mixes of the hash to ensure the last few
+ // bytes are well-incorporated.
+ h ^= h >> 13;
+ h *= m;
+ h ^= h >> 15;
+ return h;
+}
+
+uint64_t Hash64(const char* data, size_t n, uint64_t seed) {
+ const uint64_t m = 0xc6a4a7935bd1e995;
+ const int r = 47;
+
+ uint64_t h = seed ^ (n * m);
+
+ while (n >= 8) {
+ uint64_t k = DecodeFixed64(data);
+ data += 8;
+ n -= 8;
+
+ k *= m;
+ k ^= k >> r;
+ k *= m;
+
+ h ^= k;
+ h *= m;
+ }
+
+ switch (n) {
+ case 7:
+ h ^= ByteAs64(data[6]) << 48;
+ case 6:
+ h ^= ByteAs64(data[5]) << 40;
+ case 5:
+ h ^= ByteAs64(data[4]) << 32;
+ case 4:
+ h ^= ByteAs64(data[3]) << 24;
+ case 3:
+ h ^= ByteAs64(data[2]) << 16;
+ case 2:
+ h ^= ByteAs64(data[1]) << 8;
+ case 1:
+ h ^= ByteAs64(data[0]);
+ h *= m;
+ }
+
+ h ^= h >> r;
+ h *= m;
+ h ^= h >> r;
+
+ return h;
+}
+} // namespace statsd
+} // namespace os
+} // namespace android
diff --git a/cmds/statsd/src/hash.h b/cmds/statsd/src/hash.h
new file mode 100644
index 0000000..cfe869f
--- /dev/null
+++ b/cmds/statsd/src/hash.h
@@ -0,0 +1,46 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <string>
+
+namespace android {
+namespace os {
+namespace statsd {
+
+extern uint32_t Hash32(const char *data, size_t n, uint32_t seed);
+extern uint64_t Hash64(const char* data, size_t n, uint64_t seed);
+
+inline uint32_t Hash32(const char *data, size_t n) {
+ return Hash32(data, n, 0xBEEF);
+}
+
+inline uint32_t Hash32(const std::string &input) {
+ return Hash32(input.data(), input.size());
+}
+
+inline uint64_t Hash64(const char* data, size_t n) {
+ return Hash64(data, n, 0xDECAFCAFFE);
+}
+
+inline uint64_t Hash64(const std::string& str) {
+ return Hash64(str.data(), str.size());
+}
+
+} // namespace statsd
+} // namespace os
+} // namespace android
diff --git a/cmds/statsd/src/metrics/CountMetricProducer.cpp b/cmds/statsd/src/metrics/CountMetricProducer.cpp
index e21392c..a020fb1 100644
--- a/cmds/statsd/src/metrics/CountMetricProducer.cpp
+++ b/cmds/statsd/src/metrics/CountMetricProducer.cpp
@@ -45,16 +45,23 @@
// for StatsLogReport
const int FIELD_ID_ID = 1;
const int FIELD_ID_COUNT_METRICS = 5;
+const int FIELD_ID_TIME_BASE = 9;
+const int FIELD_ID_BUCKET_SIZE = 10;
+const int FIELD_ID_DIMENSION_PATH_IN_WHAT = 11;
+const int FIELD_ID_DIMENSION_PATH_IN_CONDITION = 12;
// for CountMetricDataWrapper
const int FIELD_ID_DATA = 1;
// for CountMetricData
const int FIELD_ID_DIMENSION_IN_WHAT = 1;
const int FIELD_ID_DIMENSION_IN_CONDITION = 2;
const int FIELD_ID_BUCKET_INFO = 3;
+const int FIELD_ID_DIMENSION_LEAF_IN_WHAT = 4;
+const int FIELD_ID_DIMENSION_LEAF_IN_CONDITION = 5;
// for CountBucketInfo
-const int FIELD_ID_START_BUCKET_ELAPSED_NANOS = 1;
-const int FIELD_ID_END_BUCKET_ELAPSED_NANOS = 2;
const int FIELD_ID_COUNT = 3;
+const int FIELD_ID_BUCKET_NUM = 4;
+const int FIELD_ID_START_BUCKET_ELAPSED_MILLIS = 5;
+const int FIELD_ID_END_BUCKET_ELAPSED_MILLIS = 6;
CountMetricProducer::CountMetricProducer(const ConfigKey& key, const CountMetric& metric,
const int conditionIndex,
@@ -74,6 +81,9 @@
mContainANYPositionInDimensionsInWhat = HasPositionANY(metric.dimensions_in_what());
}
+ mSliceByPositionALL = HasPositionALL(metric.dimensions_in_what()) ||
+ HasPositionALL(metric.dimensions_in_condition());
+
if (metric.has_dimensions_in_condition()) {
translateFieldMatcher(metric.dimensions_in_condition(), &mDimensionsInCondition);
}
@@ -124,6 +134,7 @@
void CountMetricProducer::onDumpReportLocked(const int64_t dumpTimeNs,
const bool include_current_partial_bucket,
+ std::set<string> *str_set,
ProtoOutputStream* protoOutput) {
if (include_current_partial_bucket) {
flushLocked(dumpTimeNs);
@@ -134,6 +145,26 @@
return;
}
protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_ID, (long long)mMetricId);
+ protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_TIME_BASE, (long long)mTimeBaseNs);
+ protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_BUCKET_SIZE, (long long)mBucketSizeNs);
+
+ // Fills the dimension path if not slicing by ALL.
+ if (!mSliceByPositionALL) {
+ if (!mDimensionsInWhat.empty()) {
+ uint64_t dimenPathToken = protoOutput->start(
+ FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_PATH_IN_WHAT);
+ writeDimensionPathToProto(mDimensionsInWhat, protoOutput);
+ protoOutput->end(dimenPathToken);
+ }
+ if (!mDimensionsInCondition.empty()) {
+ uint64_t dimenPathToken = protoOutput->start(
+ FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_PATH_IN_CONDITION);
+ writeDimensionPathToProto(mDimensionsInCondition, protoOutput);
+ protoOutput->end(dimenPathToken);
+ }
+
+ }
+
uint64_t protoToken = protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_ID_COUNT_METRICS);
for (const auto& counter : mPastBuckets) {
@@ -144,27 +175,42 @@
protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_DATA);
// First fill dimension.
- uint64_t dimensionInWhatToken = protoOutput->start(
- FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_IN_WHAT);
- writeDimensionToProto(dimensionKey.getDimensionKeyInWhat(), protoOutput);
- protoOutput->end(dimensionInWhatToken);
+ if (mSliceByPositionALL) {
+ uint64_t dimensionToken = protoOutput->start(
+ FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_IN_WHAT);
+ writeDimensionToProto(dimensionKey.getDimensionKeyInWhat(), str_set, protoOutput);
+ protoOutput->end(dimensionToken);
- if (dimensionKey.hasDimensionKeyInCondition()) {
- uint64_t dimensionInConditionToken = protoOutput->start(
- FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_IN_CONDITION);
- writeDimensionToProto(dimensionKey.getDimensionKeyInCondition(), protoOutput);
- protoOutput->end(dimensionInConditionToken);
+ if (dimensionKey.hasDimensionKeyInCondition()) {
+ uint64_t dimensionInConditionToken = protoOutput->start(
+ FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_IN_CONDITION);
+ writeDimensionToProto(dimensionKey.getDimensionKeyInCondition(),
+ str_set, protoOutput);
+ protoOutput->end(dimensionInConditionToken);
+ }
+ } else {
+ writeDimensionLeafNodesToProto(dimensionKey.getDimensionKeyInWhat(),
+ FIELD_ID_DIMENSION_LEAF_IN_WHAT, str_set, protoOutput);
+ if (dimensionKey.hasDimensionKeyInCondition()) {
+ writeDimensionLeafNodesToProto(dimensionKey.getDimensionKeyInCondition(),
+ FIELD_ID_DIMENSION_LEAF_IN_CONDITION,
+ str_set, protoOutput);
+ }
}
-
// Then fill bucket_info (CountBucketInfo).
-
for (const auto& bucket : counter.second) {
uint64_t bucketInfoToken = protoOutput->start(
FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_BUCKET_INFO);
- protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_START_BUCKET_ELAPSED_NANOS,
- (long long)bucket.mBucketStartNs);
- protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_END_BUCKET_ELAPSED_NANOS,
- (long long)bucket.mBucketEndNs);
+ // Partial bucket.
+ if (bucket.mBucketEndNs - bucket.mBucketStartNs != mBucketSizeNs) {
+ protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_START_BUCKET_ELAPSED_MILLIS,
+ (long long)NanoToMillis(bucket.mBucketStartNs));
+ protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_END_BUCKET_ELAPSED_MILLIS,
+ (long long)NanoToMillis(bucket.mBucketEndNs));
+ } else {
+ protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_BUCKET_NUM,
+ (long long)(getBucketNumFromEndTimeNs(bucket.mBucketEndNs)));
+ }
protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_COUNT, (long long)bucket.mCount);
protoOutput->end(bucketInfoToken);
VLOG("\t bucket [%lld - %lld] count: %lld", (long long)bucket.mBucketStartNs,
diff --git a/cmds/statsd/src/metrics/CountMetricProducer.h b/cmds/statsd/src/metrics/CountMetricProducer.h
index cafc882..9c62cfc 100644
--- a/cmds/statsd/src/metrics/CountMetricProducer.h
+++ b/cmds/statsd/src/metrics/CountMetricProducer.h
@@ -57,6 +57,7 @@
void onDumpReportLocked(const int64_t dumpTimeNs,
const bool include_current_partial_bucket,
+ std::set<string> *str_set,
android::util::ProtoOutputStream* protoOutput) override;
// Internal interface to handle condition change.
diff --git a/cmds/statsd/src/metrics/DurationMetricProducer.cpp b/cmds/statsd/src/metrics/DurationMetricProducer.cpp
index 3661b31..05516c5 100644
--- a/cmds/statsd/src/metrics/DurationMetricProducer.cpp
+++ b/cmds/statsd/src/metrics/DurationMetricProducer.cpp
@@ -44,16 +44,23 @@
// for StatsLogReport
const int FIELD_ID_ID = 1;
const int FIELD_ID_DURATION_METRICS = 6;
+const int FIELD_ID_TIME_BASE = 9;
+const int FIELD_ID_BUCKET_SIZE = 10;
+const int FIELD_ID_DIMENSION_PATH_IN_WHAT = 11;
+const int FIELD_ID_DIMENSION_PATH_IN_CONDITION = 12;
// for DurationMetricDataWrapper
const int FIELD_ID_DATA = 1;
// for DurationMetricData
const int FIELD_ID_DIMENSION_IN_WHAT = 1;
const int FIELD_ID_DIMENSION_IN_CONDITION = 2;
const int FIELD_ID_BUCKET_INFO = 3;
+const int FIELD_ID_DIMENSION_LEAF_IN_WHAT = 4;
+const int FIELD_ID_DIMENSION_LEAF_IN_CONDITION = 5;
// for DurationBucketInfo
-const int FIELD_ID_START_BUCKET_ELAPSED_NANOS = 1;
-const int FIELD_ID_END_BUCKET_ELAPSED_NANOS = 2;
const int FIELD_ID_DURATION = 3;
+const int FIELD_ID_BUCKET_NUM = 4;
+const int FIELD_ID_START_BUCKET_ELAPSED_MILLIS = 5;
+const int FIELD_ID_END_BUCKET_ELAPSED_MILLIS = 6;
DurationMetricProducer::DurationMetricProducer(const ConfigKey& key, const DurationMetric& metric,
const int conditionIndex, const size_t startIndex,
@@ -99,6 +106,9 @@
translateFieldMatcher(metric.dimensions_in_condition(), &mDimensionsInCondition);
}
+ mSliceByPositionALL = HasPositionALL(metric.dimensions_in_what()) ||
+ HasPositionALL(metric.dimensions_in_condition());
+
if (metric.links().size() > 0) {
for (const auto& link : metric.links()) {
Metric2Condition mc;
@@ -440,6 +450,7 @@
void DurationMetricProducer::onDumpReportLocked(const int64_t dumpTimeNs,
const bool include_current_partial_bucket,
+ std::set<string> *str_set,
ProtoOutputStream* protoOutput) {
if (include_current_partial_bucket) {
flushLocked(dumpTimeNs);
@@ -452,6 +463,24 @@
}
protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_ID, (long long)mMetricId);
+ protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_TIME_BASE, (long long)mTimeBaseNs);
+ protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_BUCKET_SIZE, (long long)mBucketSizeNs);
+
+ if (!mSliceByPositionALL) {
+ if (!mDimensionsInWhat.empty()) {
+ uint64_t dimenPathToken = protoOutput->start(
+ FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_PATH_IN_WHAT);
+ writeDimensionPathToProto(mDimensionsInWhat, protoOutput);
+ protoOutput->end(dimenPathToken);
+ }
+ if (!mDimensionsInCondition.empty()) {
+ uint64_t dimenPathToken = protoOutput->start(
+ FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_PATH_IN_CONDITION);
+ writeDimensionPathToProto(mDimensionsInCondition, protoOutput);
+ protoOutput->end(dimenPathToken);
+ }
+ }
+
uint64_t protoToken = protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_ID_DURATION_METRICS);
VLOG("Duration metric %lld dump report now...", (long long)mMetricId);
@@ -464,26 +493,41 @@
protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_DATA);
// First fill dimension.
- uint64_t dimensionToken = protoOutput->start(
- FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_IN_WHAT);
- writeDimensionToProto(dimensionKey.getDimensionKeyInWhat(), protoOutput);
- protoOutput->end(dimensionToken);
+ if (mSliceByPositionALL) {
+ uint64_t dimensionToken = protoOutput->start(
+ FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_IN_WHAT);
+ writeDimensionToProto(dimensionKey.getDimensionKeyInWhat(), str_set, protoOutput);
+ protoOutput->end(dimensionToken);
- if (dimensionKey.hasDimensionKeyInCondition()) {
- uint64_t dimensionInConditionToken = protoOutput->start(
- FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_IN_CONDITION);
- writeDimensionToProto(dimensionKey.getDimensionKeyInCondition(), protoOutput);
- protoOutput->end(dimensionInConditionToken);
+ if (dimensionKey.hasDimensionKeyInCondition()) {
+ uint64_t dimensionInConditionToken = protoOutput->start(
+ FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_IN_CONDITION);
+ writeDimensionToProto(dimensionKey.getDimensionKeyInCondition(),
+ str_set, protoOutput);
+ protoOutput->end(dimensionInConditionToken);
+ }
+ } else {
+ writeDimensionLeafNodesToProto(dimensionKey.getDimensionKeyInWhat(),
+ FIELD_ID_DIMENSION_LEAF_IN_WHAT, str_set, protoOutput);
+ if (dimensionKey.hasDimensionKeyInCondition()) {
+ writeDimensionLeafNodesToProto(dimensionKey.getDimensionKeyInCondition(),
+ FIELD_ID_DIMENSION_LEAF_IN_CONDITION,
+ str_set, protoOutput);
+ }
}
-
// Then fill bucket_info (DurationBucketInfo).
for (const auto& bucket : pair.second) {
uint64_t bucketInfoToken = protoOutput->start(
FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_BUCKET_INFO);
- protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_START_BUCKET_ELAPSED_NANOS,
- (long long)bucket.mBucketStartNs);
- protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_END_BUCKET_ELAPSED_NANOS,
- (long long)bucket.mBucketEndNs);
+ if (bucket.mBucketEndNs - bucket.mBucketStartNs != mBucketSizeNs) {
+ protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_START_BUCKET_ELAPSED_MILLIS,
+ (long long)NanoToMillis(bucket.mBucketStartNs));
+ protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_END_BUCKET_ELAPSED_MILLIS,
+ (long long)NanoToMillis(bucket.mBucketEndNs));
+ } else {
+ protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_BUCKET_NUM,
+ (long long)(getBucketNumFromEndTimeNs(bucket.mBucketEndNs)));
+ }
protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_DURATION, (long long)bucket.mDuration);
protoOutput->end(bucketInfoToken);
VLOG("\t bucket [%lld - %lld] duration: %lld", (long long)bucket.mBucketStartNs,
diff --git a/cmds/statsd/src/metrics/DurationMetricProducer.h b/cmds/statsd/src/metrics/DurationMetricProducer.h
index 80fbdde..8e05edb 100644
--- a/cmds/statsd/src/metrics/DurationMetricProducer.h
+++ b/cmds/statsd/src/metrics/DurationMetricProducer.h
@@ -63,6 +63,7 @@
void onDumpReportLocked(const int64_t dumpTimeNs,
const bool include_current_partial_bucket,
+ std::set<string> *str_set,
android::util::ProtoOutputStream* protoOutput) override;
// Internal interface to handle condition change.
diff --git a/cmds/statsd/src/metrics/EventMetricProducer.cpp b/cmds/statsd/src/metrics/EventMetricProducer.cpp
index 2f2679e..f278aa3 100644
--- a/cmds/statsd/src/metrics/EventMetricProducer.cpp
+++ b/cmds/statsd/src/metrics/EventMetricProducer.cpp
@@ -102,6 +102,7 @@
void EventMetricProducer::onDumpReportLocked(const int64_t dumpTimeNs,
const bool include_current_partial_bucket,
+ std::set<string> *str_set,
ProtoOutputStream* protoOutput) {
if (mProto->size() <= 0) {
return;
diff --git a/cmds/statsd/src/metrics/EventMetricProducer.h b/cmds/statsd/src/metrics/EventMetricProducer.h
index 5c29174..0cab611 100644
--- a/cmds/statsd/src/metrics/EventMetricProducer.h
+++ b/cmds/statsd/src/metrics/EventMetricProducer.h
@@ -48,6 +48,7 @@
void onDumpReportLocked(const int64_t dumpTimeNs,
const bool include_current_partial_bucket,
+ std::set<string> *str_set,
android::util::ProtoOutputStream* protoOutput) override;
// Internal interface to handle condition change.
diff --git a/cmds/statsd/src/metrics/GaugeMetricProducer.cpp b/cmds/statsd/src/metrics/GaugeMetricProducer.cpp
index 1270856..2461a97 100644
--- a/cmds/statsd/src/metrics/GaugeMetricProducer.cpp
+++ b/cmds/statsd/src/metrics/GaugeMetricProducer.cpp
@@ -45,21 +45,28 @@
// for StatsLogReport
const int FIELD_ID_ID = 1;
const int FIELD_ID_GAUGE_METRICS = 8;
+const int FIELD_ID_TIME_BASE = 9;
+const int FIELD_ID_BUCKET_SIZE = 10;
+const int FIELD_ID_DIMENSION_PATH_IN_WHAT = 11;
+const int FIELD_ID_DIMENSION_PATH_IN_CONDITION = 12;
// for GaugeMetricDataWrapper
const int FIELD_ID_DATA = 1;
const int FIELD_ID_SKIPPED = 2;
-const int FIELD_ID_SKIPPED_START = 1;
-const int FIELD_ID_SKIPPED_END = 2;
+const int FIELD_ID_SKIPPED_START_MILLIS = 3;
+const int FIELD_ID_SKIPPED_END_MILLIS = 4;
// for GaugeMetricData
const int FIELD_ID_DIMENSION_IN_WHAT = 1;
const int FIELD_ID_DIMENSION_IN_CONDITION = 2;
const int FIELD_ID_BUCKET_INFO = 3;
+const int FIELD_ID_DIMENSION_LEAF_IN_WHAT = 4;
+const int FIELD_ID_DIMENSION_LEAF_IN_CONDITION = 5;
// for GaugeBucketInfo
-const int FIELD_ID_START_BUCKET_ELAPSED_NANOS = 1;
-const int FIELD_ID_END_BUCKET_ELAPSED_NANOS = 2;
const int FIELD_ID_ATOM = 3;
const int FIELD_ID_ELAPSED_ATOM_TIMESTAMP = 4;
const int FIELD_ID_WALL_CLOCK_ATOM_TIMESTAMP = 5;
+const int FIELD_ID_BUCKET_NUM = 6;
+const int FIELD_ID_START_BUCKET_ELAPSED_MILLIS = 7;
+const int FIELD_ID_END_BUCKET_ELAPSED_MILLIS = 8;
GaugeMetricProducer::GaugeMetricProducer(const ConfigKey& key, const GaugeMetric& metric,
const int conditionIndex,
@@ -113,6 +120,8 @@
}
}
mConditionSliced = (metric.links().size() > 0) || (mDimensionsInCondition.size() > 0);
+ mSliceByPositionALL = HasPositionALL(metric.dimensions_in_what()) ||
+ HasPositionALL(metric.dimensions_in_condition());
flushIfNeededLocked(startTimeNs);
// Kicks off the puller immediately.
@@ -162,6 +171,7 @@
void GaugeMetricProducer::onDumpReportLocked(const int64_t dumpTimeNs,
const bool include_current_partial_bucket,
+ std::set<string> *str_set,
ProtoOutputStream* protoOutput) {
VLOG("Gauge metric %lld report now...", (long long)mMetricId);
if (include_current_partial_bucket) {
@@ -176,13 +186,34 @@
}
protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_ID, (long long)mMetricId);
+ protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_TIME_BASE, (long long)mTimeBaseNs);
+ protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_BUCKET_SIZE, (long long)mBucketSizeNs);
+
+ // Fills the dimension path if not slicing by ALL.
+ if (!mSliceByPositionALL) {
+ if (!mDimensionsInWhat.empty()) {
+ uint64_t dimenPathToken = protoOutput->start(
+ FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_PATH_IN_WHAT);
+ writeDimensionPathToProto(mDimensionsInWhat, protoOutput);
+ protoOutput->end(dimenPathToken);
+ }
+ if (!mDimensionsInCondition.empty()) {
+ uint64_t dimenPathToken = protoOutput->start(
+ FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_PATH_IN_CONDITION);
+ writeDimensionPathToProto(mDimensionsInCondition, protoOutput);
+ protoOutput->end(dimenPathToken);
+ }
+ }
+
uint64_t protoToken = protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_ID_GAUGE_METRICS);
for (const auto& pair : mSkippedBuckets) {
uint64_t wrapperToken =
protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_SKIPPED);
- protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_SKIPPED_START, (long long)pair.first);
- protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_SKIPPED_END, (long long)pair.second);
+ protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_SKIPPED_START_MILLIS,
+ (long long)(NanoToMillis(pair.first)));
+ protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_SKIPPED_END_MILLIS,
+ (long long)(NanoToMillis(pair.second)));
protoOutput->end(wrapperToken);
}
mSkippedBuckets.clear();
@@ -195,26 +226,43 @@
protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_DATA);
// First fill dimension.
- uint64_t dimensionToken = protoOutput->start(
- FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_IN_WHAT);
- writeDimensionToProto(dimensionKey.getDimensionKeyInWhat(), protoOutput);
- protoOutput->end(dimensionToken);
+ if (mSliceByPositionALL) {
+ uint64_t dimensionToken = protoOutput->start(
+ FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_IN_WHAT);
+ writeDimensionToProto(dimensionKey.getDimensionKeyInWhat(), str_set, protoOutput);
+ protoOutput->end(dimensionToken);
- if (dimensionKey.hasDimensionKeyInCondition()) {
- uint64_t dimensionInConditionToken = protoOutput->start(
- FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_IN_CONDITION);
- writeDimensionToProto(dimensionKey.getDimensionKeyInCondition(), protoOutput);
- protoOutput->end(dimensionInConditionToken);
+ if (dimensionKey.hasDimensionKeyInCondition()) {
+ uint64_t dimensionInConditionToken = protoOutput->start(
+ FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_IN_CONDITION);
+ writeDimensionToProto(dimensionKey.getDimensionKeyInCondition(),
+ str_set, protoOutput);
+ protoOutput->end(dimensionInConditionToken);
+ }
+ } else {
+ writeDimensionLeafNodesToProto(dimensionKey.getDimensionKeyInWhat(),
+ FIELD_ID_DIMENSION_LEAF_IN_WHAT, str_set, protoOutput);
+ if (dimensionKey.hasDimensionKeyInCondition()) {
+ writeDimensionLeafNodesToProto(dimensionKey.getDimensionKeyInCondition(),
+ FIELD_ID_DIMENSION_LEAF_IN_CONDITION,
+ str_set, protoOutput);
+ }
}
// Then fill bucket_info (GaugeBucketInfo).
for (const auto& bucket : pair.second) {
uint64_t bucketInfoToken = protoOutput->start(
FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_BUCKET_INFO);
- protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_START_BUCKET_ELAPSED_NANOS,
- (long long)bucket.mBucketStartNs);
- protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_END_BUCKET_ELAPSED_NANOS,
- (long long)bucket.mBucketEndNs);
+
+ if (bucket.mBucketEndNs - bucket.mBucketStartNs != mBucketSizeNs) {
+ protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_START_BUCKET_ELAPSED_MILLIS,
+ (long long)NanoToMillis(bucket.mBucketStartNs));
+ protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_END_BUCKET_ELAPSED_MILLIS,
+ (long long)NanoToMillis(bucket.mBucketEndNs));
+ } else {
+ protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_BUCKET_NUM,
+ (long long)(getBucketNumFromEndTimeNs(bucket.mBucketEndNs)));
+ }
if (!bucket.mGaugeAtoms.empty()) {
for (const auto& atom : bucket.mGaugeAtoms) {
diff --git a/cmds/statsd/src/metrics/GaugeMetricProducer.h b/cmds/statsd/src/metrics/GaugeMetricProducer.h
index 71d5912..84a6a58 100644
--- a/cmds/statsd/src/metrics/GaugeMetricProducer.h
+++ b/cmds/statsd/src/metrics/GaugeMetricProducer.h
@@ -90,6 +90,7 @@
private:
void onDumpReportLocked(const int64_t dumpTimeNs,
const bool include_current_partial_bucket,
+ std::set<string> *str_set,
android::util::ProtoOutputStream* protoOutput) override;
// for testing
diff --git a/cmds/statsd/src/metrics/MetricProducer.h b/cmds/statsd/src/metrics/MetricProducer.h
index 532ecbf..94b8c60 100644
--- a/cmds/statsd/src/metrics/MetricProducer.h
+++ b/cmds/statsd/src/metrics/MetricProducer.h
@@ -52,6 +52,7 @@
mWizard(wizard),
mConditionTrackerIndex(conditionIndex),
mContainANYPositionInDimensionsInWhat(false),
+ mSliceByPositionALL(false),
mSameConditionDimensionsInTracker(false),
mHasLinksToAllConditionDimensionsInTracker(false) {
}
@@ -114,9 +115,10 @@
// This method clears all the past buckets.
void onDumpReport(const int64_t dumpTimeNs,
const bool include_current_partial_bucket,
+ std::set<string> *str_set,
android::util::ProtoOutputStream* protoOutput) {
std::lock_guard<std::mutex> lock(mMutex);
- return onDumpReportLocked(dumpTimeNs, include_current_partial_bucket, protoOutput);
+ return onDumpReportLocked(dumpTimeNs, include_current_partial_bucket, str_set, protoOutput);
}
void dumpStates(FILE* out, bool verbose) const {
@@ -176,6 +178,7 @@
const int64_t eventTime) = 0;
virtual void onDumpReportLocked(const int64_t dumpTimeNs,
const bool include_current_partial_bucket,
+ std::set<string> *str_set,
android::util::ProtoOutputStream* protoOutput) = 0;
virtual size_t byteSizeLocked() const = 0;
virtual void dumpStatesLocked(FILE* out, bool verbose) const = 0;
@@ -212,6 +215,10 @@
return mTimeBaseNs + (mCurrentBucketNum + 1) * mBucketSizeNs;
}
+ int64_t getBucketNumFromEndTimeNs(const int64_t endNs) {
+ return (endNs - mTimeBaseNs) / mBucketSizeNs - 1;
+ }
+
virtual void dropDataLocked(const int64_t dropTimeNs) = 0;
const int64_t mMetricId;
@@ -244,6 +251,7 @@
vector<Matcher> mDimensionsInCondition; // The dimensions_in_condition defined in statsd_config
bool mContainANYPositionInDimensionsInWhat;
+ bool mSliceByPositionALL;
// True iff the condition dimensions equal to the sliced dimensions in the simple condition
// tracker. This field is always false for combinational condition trackers.
diff --git a/cmds/statsd/src/metrics/MetricsManager.cpp b/cmds/statsd/src/metrics/MetricsManager.cpp
index 47a1a86..e8c123b 100644
--- a/cmds/statsd/src/metrics/MetricsManager.cpp
+++ b/cmds/statsd/src/metrics/MetricsManager.cpp
@@ -36,6 +36,7 @@
using android::util::FIELD_TYPE_INT32;
using android::util::FIELD_TYPE_INT64;
using android::util::FIELD_TYPE_MESSAGE;
+using android::util::FIELD_TYPE_STRING;
using android::util::ProtoOutputStream;
using std::make_unique;
@@ -192,14 +193,16 @@
void MetricsManager::onDumpReport(const int64_t dumpTimeStampNs,
const bool include_current_partial_bucket,
+ std::set<string> *str_set,
ProtoOutputStream* protoOutput) {
VLOG("=========================Metric Reports Start==========================");
// one StatsLogReport per MetricProduer
for (const auto& producer : mAllMetricProducers) {
if (mNoReportMetricIds.find(producer->getMetricId()) == mNoReportMetricIds.end()) {
- uint64_t token =
- protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_METRICS);
- producer->onDumpReport(dumpTimeStampNs, include_current_partial_bucket, protoOutput);
+ uint64_t token = protoOutput->start(
+ FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_METRICS);
+ producer->onDumpReport(dumpTimeStampNs, include_current_partial_bucket, str_set,
+ protoOutput);
protoOutput->end(token);
}
}
@@ -211,6 +214,7 @@
protoOutput->write(FIELD_TYPE_INT32 | FIELD_ID_ANNOTATIONS_INT32, annotation.second);
protoOutput->end(token);
}
+
mLastReportTimeNs = dumpTimeStampNs;
mLastReportWallClockNs = getWallClockNs();
VLOG("=========================Metric Reports End==========================");
diff --git a/cmds/statsd/src/metrics/MetricsManager.h b/cmds/statsd/src/metrics/MetricsManager.h
index 3d2c595..170d6a7 100644
--- a/cmds/statsd/src/metrics/MetricsManager.h
+++ b/cmds/statsd/src/metrics/MetricsManager.h
@@ -92,6 +92,7 @@
virtual void onDumpReport(const int64_t dumpTimeNs,
const bool include_current_partial_bucket,
+ std::set<string> *str_set,
android::util::ProtoOutputStream* protoOutput);
// Computes the total byte size of all metrics managed by a single config source.
diff --git a/cmds/statsd/src/metrics/ValueMetricProducer.cpp b/cmds/statsd/src/metrics/ValueMetricProducer.cpp
index 27fd78f..c505d22 100644
--- a/cmds/statsd/src/metrics/ValueMetricProducer.cpp
+++ b/cmds/statsd/src/metrics/ValueMetricProducer.cpp
@@ -48,19 +48,26 @@
// for StatsLogReport
const int FIELD_ID_ID = 1;
const int FIELD_ID_VALUE_METRICS = 7;
+const int FIELD_ID_TIME_BASE = 9;
+const int FIELD_ID_BUCKET_SIZE = 10;
+const int FIELD_ID_DIMENSION_PATH_IN_WHAT = 11;
+const int FIELD_ID_DIMENSION_PATH_IN_CONDITION = 12;
// for ValueMetricDataWrapper
const int FIELD_ID_DATA = 1;
const int FIELD_ID_SKIPPED = 2;
-const int FIELD_ID_SKIPPED_START = 1;
-const int FIELD_ID_SKIPPED_END = 2;
+const int FIELD_ID_SKIPPED_START_MILLIS = 3;
+const int FIELD_ID_SKIPPED_END_MILLIS = 4;
// for ValueMetricData
const int FIELD_ID_DIMENSION_IN_WHAT = 1;
const int FIELD_ID_DIMENSION_IN_CONDITION = 2;
const int FIELD_ID_BUCKET_INFO = 3;
+const int FIELD_ID_DIMENSION_LEAF_IN_WHAT = 4;
+const int FIELD_ID_DIMENSION_LEAF_IN_CONDITION = 5;
// for ValueBucketInfo
-const int FIELD_ID_START_BUCKET_NANOS = 1;
-const int FIELD_ID_END_BUCKET_NANOS = 2;
const int FIELD_ID_VALUE = 3;
+const int FIELD_ID_BUCKET_NUM = 4;
+const int FIELD_ID_START_BUCKET_ELAPSED_MILLIS = 5;
+const int FIELD_ID_END_BUCKET_ELAPSED_MILLIS = 6;
// ValueMetric has a minimum bucket size of 10min so that we don't pull too frequently
ValueMetricProducer::ValueMetricProducer(const ConfigKey& key, const ValueMetric& metric,
@@ -113,6 +120,8 @@
mField = mValueField.child(0).field();
}
mConditionSliced = (metric.links().size() > 0) || (mDimensionsInCondition.size() > 0);
+ mSliceByPositionALL = HasPositionALL(metric.dimensions_in_what()) ||
+ HasPositionALL(metric.dimensions_in_condition());
// Kicks off the puller immediately.
flushIfNeededLocked(startTimestampNs);
@@ -153,6 +162,7 @@
void ValueMetricProducer::onDumpReportLocked(const int64_t dumpTimeNs,
const bool include_current_partial_bucket,
+ std::set<string> *str_set,
ProtoOutputStream* protoOutput) {
VLOG("metric %lld dump report now...", (long long)mMetricId);
if (include_current_partial_bucket) {
@@ -164,13 +174,33 @@
return;
}
protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_ID, (long long)mMetricId);
+ protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_TIME_BASE, (long long)mTimeBaseNs);
+ protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_BUCKET_SIZE, (long long)mBucketSizeNs);
+ // Fills the dimension path if not slicing by ALL.
+ if (!mSliceByPositionALL) {
+ if (!mDimensionsInWhat.empty()) {
+ uint64_t dimenPathToken = protoOutput->start(
+ FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_PATH_IN_WHAT);
+ writeDimensionPathToProto(mDimensionsInWhat, protoOutput);
+ protoOutput->end(dimenPathToken);
+ }
+ if (!mDimensionsInCondition.empty()) {
+ uint64_t dimenPathToken = protoOutput->start(
+ FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_PATH_IN_CONDITION);
+ writeDimensionPathToProto(mDimensionsInCondition, protoOutput);
+ protoOutput->end(dimenPathToken);
+ }
+ }
+
uint64_t protoToken = protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_ID_VALUE_METRICS);
for (const auto& pair : mSkippedBuckets) {
uint64_t wrapperToken =
protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_SKIPPED);
- protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_SKIPPED_START, (long long)pair.first);
- protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_SKIPPED_END, (long long)pair.second);
+ protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_SKIPPED_START_MILLIS,
+ (long long)(NanoToMillis(pair.first)));
+ protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_SKIPPED_END_MILLIS,
+ (long long)(NanoToMillis(pair.second)));
protoOutput->end(wrapperToken);
}
mSkippedBuckets.clear();
@@ -182,25 +212,43 @@
protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_DATA);
// First fill dimension.
- uint64_t dimensionToken = protoOutput->start(
- FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_IN_WHAT);
- writeDimensionToProto(dimensionKey.getDimensionKeyInWhat(), protoOutput);
- protoOutput->end(dimensionToken);
- if (dimensionKey.hasDimensionKeyInCondition()) {
- uint64_t dimensionInConditionToken = protoOutput->start(
- FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_IN_CONDITION);
- writeDimensionToProto(dimensionKey.getDimensionKeyInCondition(), protoOutput);
- protoOutput->end(dimensionInConditionToken);
+ if (mSliceByPositionALL) {
+ uint64_t dimensionToken = protoOutput->start(
+ FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_IN_WHAT);
+ writeDimensionToProto(dimensionKey.getDimensionKeyInWhat(), str_set, protoOutput);
+ protoOutput->end(dimensionToken);
+ if (dimensionKey.hasDimensionKeyInCondition()) {
+ uint64_t dimensionInConditionToken = protoOutput->start(
+ FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_IN_CONDITION);
+ writeDimensionToProto(dimensionKey.getDimensionKeyInCondition(),
+ str_set, protoOutput);
+ protoOutput->end(dimensionInConditionToken);
+ }
+ } else {
+ writeDimensionLeafNodesToProto(dimensionKey.getDimensionKeyInWhat(),
+ FIELD_ID_DIMENSION_LEAF_IN_WHAT, str_set, protoOutput);
+ if (dimensionKey.hasDimensionKeyInCondition()) {
+ writeDimensionLeafNodesToProto(dimensionKey.getDimensionKeyInCondition(),
+ FIELD_ID_DIMENSION_LEAF_IN_CONDITION,
+ str_set, protoOutput);
+ }
}
// Then fill bucket_info (ValueBucketInfo).
for (const auto& bucket : pair.second) {
uint64_t bucketInfoToken = protoOutput->start(
FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_BUCKET_INFO);
- protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_START_BUCKET_NANOS,
- (long long)bucket.mBucketStartNs);
- protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_END_BUCKET_NANOS,
- (long long)bucket.mBucketEndNs);
+
+ if (bucket.mBucketEndNs - bucket.mBucketStartNs != mBucketSizeNs) {
+ protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_START_BUCKET_ELAPSED_MILLIS,
+ (long long)NanoToMillis(bucket.mBucketStartNs));
+ protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_END_BUCKET_ELAPSED_MILLIS,
+ (long long)NanoToMillis(bucket.mBucketEndNs));
+ } else {
+ protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_BUCKET_NUM,
+ (long long)(getBucketNumFromEndTimeNs(bucket.mBucketEndNs)));
+ }
+
protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_VALUE, (long long)bucket.mValue);
protoOutput->end(bucketInfoToken);
VLOG("\t bucket [%lld - %lld] count: %lld", (long long)bucket.mBucketStartNs,
diff --git a/cmds/statsd/src/metrics/ValueMetricProducer.h b/cmds/statsd/src/metrics/ValueMetricProducer.h
index 8df30d3..4d02b43 100644
--- a/cmds/statsd/src/metrics/ValueMetricProducer.h
+++ b/cmds/statsd/src/metrics/ValueMetricProducer.h
@@ -88,6 +88,7 @@
private:
void onDumpReportLocked(const int64_t dumpTimeNs,
const bool include_current_partial_bucket,
+ std::set<string> *str_set,
android::util::ProtoOutputStream* protoOutput) override;
// Internal interface to handle condition change.
diff --git a/cmds/statsd/src/packages/UidMap.cpp b/cmds/statsd/src/packages/UidMap.cpp
index 2674171..fff909c 100644
--- a/cmds/statsd/src/packages/UidMap.cpp
+++ b/cmds/statsd/src/packages/UidMap.cpp
@@ -16,6 +16,7 @@
#define DEBUG false // STOPSHIP if true
#include "Log.h"
+#include "hash.h"
#include "stats_log_util.h"
#include "guardrail/StatsdStats.h"
#include "packages/UidMap.h"
@@ -34,6 +35,7 @@
using android::util::FIELD_TYPE_FLOAT;
using android::util::FIELD_TYPE_INT32;
using android::util::FIELD_TYPE_INT64;
+using android::util::FIELD_TYPE_UINT64;
using android::util::FIELD_TYPE_MESSAGE;
using android::util::FIELD_TYPE_STRING;
using android::util::ProtoOutputStream;
@@ -46,6 +48,7 @@
const int FIELD_ID_SNAPSHOT_PACKAGE_VERSION = 2;
const int FIELD_ID_SNAPSHOT_PACKAGE_UID = 3;
const int FIELD_ID_SNAPSHOT_PACKAGE_DELETED = 4;
+const int FIELD_ID_SNAPSHOT_PACKAGE_NAME_HASH = 5;
const int FIELD_ID_SNAPSHOT_TIMESTAMP = 1;
const int FIELD_ID_SNAPSHOT_PACKAGE_INFO = 2;
const int FIELD_ID_SNAPSHOTS = 1;
@@ -56,6 +59,7 @@
const int FIELD_ID_CHANGE_UID = 4;
const int FIELD_ID_CHANGE_NEW_VERSION = 5;
const int FIELD_ID_CHANGE_PREV_VERSION = 6;
+const int FIELD_ID_CHANGE_PACKAGE_HASH = 7;
UidMap::UidMap() : mBytesUsed(0) {}
@@ -312,7 +316,7 @@
}
void UidMap::appendUidMap(const int64_t& timestamp, const ConfigKey& key,
- ProtoOutputStream* proto) {
+ std::set<string> *str_set, ProtoOutputStream* proto) {
lock_guard<mutex> lock(mMutex); // Lock for updates
for (const ChangeRecord& record : mChanges) {
@@ -322,7 +326,14 @@
proto->write(FIELD_TYPE_BOOL | FIELD_ID_CHANGE_DELETION, (bool)record.deletion);
proto->write(FIELD_TYPE_INT64 | FIELD_ID_CHANGE_TIMESTAMP,
(long long)record.timestampNs);
- proto->write(FIELD_TYPE_STRING | FIELD_ID_CHANGE_PACKAGE, record.package);
+ if (str_set != nullptr) {
+ str_set->insert(record.package);
+ proto->write(FIELD_TYPE_UINT64 | FIELD_ID_CHANGE_PACKAGE_HASH,
+ (long long)Hash64(record.package));
+ } else {
+ proto->write(FIELD_TYPE_STRING | FIELD_ID_CHANGE_PACKAGE, record.package);
+ }
+
proto->write(FIELD_TYPE_INT32 | FIELD_ID_CHANGE_UID, (int)record.uid);
proto->write(FIELD_TYPE_INT64 | FIELD_ID_CHANGE_NEW_VERSION, (long long)record.version);
proto->write(FIELD_TYPE_INT64 | FIELD_ID_CHANGE_PREV_VERSION,
@@ -338,7 +349,15 @@
for (const auto& kv : mMap) {
uint64_t token = proto->start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED |
FIELD_ID_SNAPSHOT_PACKAGE_INFO);
- proto->write(FIELD_TYPE_STRING | FIELD_ID_SNAPSHOT_PACKAGE_NAME, kv.first.second);
+
+ if (str_set != nullptr) {
+ str_set->insert(kv.first.second);
+ proto->write(FIELD_TYPE_UINT64 | FIELD_ID_SNAPSHOT_PACKAGE_NAME_HASH,
+ (long long)Hash64(kv.first.second));
+ } else {
+ proto->write(FIELD_TYPE_STRING | FIELD_ID_SNAPSHOT_PACKAGE_NAME, kv.first.second);
+ }
+
proto->write(FIELD_TYPE_INT64 | FIELD_ID_SNAPSHOT_PACKAGE_VERSION,
(long long)kv.second.versionCode);
proto->write(FIELD_TYPE_INT32 | FIELD_ID_SNAPSHOT_PACKAGE_UID, kv.first.first);
diff --git a/cmds/statsd/src/packages/UidMap.h b/cmds/statsd/src/packages/UidMap.h
index 755b707..5e42cd1 100644
--- a/cmds/statsd/src/packages/UidMap.h
+++ b/cmds/statsd/src/packages/UidMap.h
@@ -128,7 +128,7 @@
// If every config key has received a change or snapshot record, then this
// record is deleted.
void appendUidMap(const int64_t& timestamp, const ConfigKey& key,
- util::ProtoOutputStream* proto);
+ std::set<string> *str_set, util::ProtoOutputStream* proto);
// Forces the output to be cleared. We still generate a snapshot based on the current state.
// This results in extra data uploaded but helps us reconstruct the uid mapping on the server
diff --git a/cmds/statsd/src/stats_log.proto b/cmds/statsd/src/stats_log.proto
index 447e4b7..9236864 100644
--- a/cmds/statsd/src/stats_log.proto
+++ b/cmds/statsd/src/stats_log.proto
@@ -33,6 +33,7 @@
bool value_bool = 5;
float value_float = 6;
DimensionsValueTuple value_tuple = 7;
+ uint64 value_str_hash = 8;
}
}
@@ -54,6 +55,12 @@
optional int64 end_bucket_elapsed_nanos = 2;
optional int64 count = 3;
+
+ optional int64 bucket_num = 4;
+
+ optional int64 start_bucket_elapsed_millis = 5;
+
+ optional int64 end_bucket_elapsed_millis = 6;
}
message CountMetricData {
@@ -62,6 +69,10 @@
optional DimensionsValue dimensions_in_condition = 2;
repeated CountBucketInfo bucket_info = 3;
+
+ repeated DimensionsValue dimension_leaf_values_in_what = 4;
+
+ repeated DimensionsValue dimension_leaf_values_in_condition = 5;
}
message DurationBucketInfo {
@@ -70,6 +81,12 @@
optional int64 end_bucket_elapsed_nanos = 2;
optional int64 duration_nanos = 3;
+
+ optional int64 bucket_num = 4;
+
+ optional int64 start_bucket_elapsed_millis = 5;
+
+ optional int64 end_bucket_elapsed_millis = 6;
}
message DurationMetricData {
@@ -78,6 +95,10 @@
optional DimensionsValue dimensions_in_condition = 2;
repeated DurationBucketInfo bucket_info = 3;
+
+ repeated DimensionsValue dimension_leaf_values_in_what = 4;
+
+ repeated DimensionsValue dimension_leaf_values_in_condition = 5;
}
message ValueBucketInfo {
@@ -86,6 +107,12 @@
optional int64 end_bucket_elapsed_nanos = 2;
optional int64 value = 3;
+
+ optional int64 bucket_num = 4;
+
+ optional int64 start_bucket_elapsed_millis = 5;
+
+ optional int64 end_bucket_elapsed_millis = 6;
}
message ValueMetricData {
@@ -94,6 +121,10 @@
optional DimensionsValue dimensions_in_condition = 2;
repeated ValueBucketInfo bucket_info = 3;
+
+ repeated DimensionsValue dimension_leaf_values_in_what = 4;
+
+ repeated DimensionsValue dimension_leaf_values_in_condition = 5;
}
message GaugeBucketInfo {
@@ -106,6 +137,12 @@
repeated int64 elapsed_timestamp_nanos = 4;
repeated int64 wall_clock_timestamp_nanos = 5;
+
+ optional int64 bucket_num = 6;
+
+ optional int64 start_bucket_elapsed_millis = 7;
+
+ optional int64 end_bucket_elapsed_millis = 8;
}
message GaugeMetricData {
@@ -114,6 +151,10 @@
optional DimensionsValue dimensions_in_condition = 2;
repeated GaugeBucketInfo bucket_info = 3;
+
+ repeated DimensionsValue dimension_leaf_values_in_what = 4;
+
+ repeated DimensionsValue dimension_leaf_values_in_condition = 5;
}
message StatsLogReport {
@@ -122,8 +163,10 @@
// Fields 2 and 3 are reserved.
message SkippedBuckets {
- optional int64 start_elapsed_nanos = 1;
- optional int64 end_elapsed_nanos = 2;
+ optional int64 start_bucket_elapsed_nanos = 1;
+ optional int64 end_bucket_elapsed_nanos = 2;
+ optional int64 start_bucket_elapsed_millis = 3;
+ optional int64 end_bucket_elapsed_millis = 4;
}
message EventMetricDataWrapper {
@@ -152,6 +195,14 @@
ValueMetricDataWrapper value_metrics = 7;
GaugeMetricDataWrapper gauge_metrics = 8;
}
+
+ optional int64 time_base_elapsed_nano_seconds = 9;
+
+ optional int64 bucket_size_nano_seconds = 10;
+
+ optional DimensionsValue dimensions_path_in_what = 11;
+
+ optional DimensionsValue dimensions_path_in_condition = 12;
}
message UidMapping {
@@ -164,6 +215,8 @@
optional int32 uid = 3;
optional bool deleted = 4;
+
+ optional uint64 name_hash = 5;
}
optional int64 elapsed_timestamp_nanos = 1;
@@ -180,6 +233,7 @@
optional int64 new_version = 5;
optional int64 prev_version = 6;
+ optional uint64 app_hash = 7;
}
repeated Change changes = 2;
}
@@ -197,6 +251,12 @@
optional int64 current_report_wall_clock_nanos = 6;
+ message Annotation {
+ optional int64 field_int64 = 1;
+ optional int32 field_int32 = 2;
+ }
+ repeated Annotation annotation = 7;
+
enum DumpReportReason {
DEVICE_SHUTDOWN = 1;
CONFIG_UPDATED = 2;
@@ -208,11 +268,7 @@
}
optional DumpReportReason dump_report_reason = 8;
- message Annotation {
- optional int64 field_int64 = 1;
- optional int32 field_int32 = 2;
- }
- repeated Annotation annotation = 7;
+ repeated string strings = 9;
}
message ConfigMetricsReportList {
diff --git a/cmds/statsd/src/stats_log_util.cpp b/cmds/statsd/src/stats_log_util.cpp
index efd810f..a0ab3e4 100644
--- a/cmds/statsd/src/stats_log_util.cpp
+++ b/cmds/statsd/src/stats_log_util.cpp
@@ -14,6 +14,7 @@
* limitations under the License.
*/
+#include "hash.h"
#include "stats_log_util.h"
#include <logd/LogEvent.h>
@@ -29,6 +30,8 @@
using android::util::FIELD_TYPE_FLOAT;
using android::util::FIELD_TYPE_INT32;
using android::util::FIELD_TYPE_INT64;
+using android::util::FIELD_TYPE_UINT64;
+using android::util::FIELD_TYPE_FIXED64;
using android::util::FIELD_TYPE_MESSAGE;
using android::util::FIELD_TYPE_STRING;
using android::util::ProtoOutputStream;
@@ -45,6 +48,7 @@
// const int DIMENSIONS_VALUE_VALUE_BOOL = 5; // logd doesn't have bool data type.
const int DIMENSIONS_VALUE_VALUE_FLOAT = 6;
const int DIMENSIONS_VALUE_VALUE_TUPLE = 7;
+const int DIMENSIONS_VALUE_VALUE_STR_HASH = 8;
const int DIMENSIONS_VALUE_TUPLE_VALUE = 1;
@@ -54,10 +58,12 @@
const int FIELD_ID_TOTAL_PULL = 2;
const int FIELD_ID_TOTAL_PULL_FROM_CACHE = 3;
const int FIELD_ID_MIN_PULL_INTERVAL_SEC = 4;
+
namespace {
void writeDimensionToProtoHelper(const std::vector<FieldValue>& dims, size_t* index, int depth,
- int prefix, ProtoOutputStream* protoOutput) {
+ int prefix, std::set<string> *str_set,
+ ProtoOutputStream* protoOutput) {
size_t count = dims.size();
while (*index < count) {
const auto& dim = dims[*index];
@@ -87,8 +93,15 @@
dim.mValue.float_value);
break;
case STRING:
- protoOutput->write(FIELD_TYPE_STRING | DIMENSIONS_VALUE_VALUE_STR,
- dim.mValue.str_value);
+ if (str_set == nullptr) {
+ protoOutput->write(FIELD_TYPE_STRING | DIMENSIONS_VALUE_VALUE_STR,
+ dim.mValue.str_value);
+ } else {
+ str_set->insert(dim.mValue.str_value);
+ protoOutput->write(
+ FIELD_TYPE_UINT64 | DIMENSIONS_VALUE_VALUE_STR_HASH,
+ (long long)Hash64(dim.mValue.str_value));
+ }
break;
default:
break;
@@ -105,7 +118,107 @@
uint64_t tupleToken =
protoOutput->start(FIELD_TYPE_MESSAGE | DIMENSIONS_VALUE_VALUE_TUPLE);
writeDimensionToProtoHelper(dims, index, valueDepth, dim.mField.getPrefix(valueDepth),
- protoOutput);
+ str_set, protoOutput);
+ protoOutput->end(tupleToken);
+ protoOutput->end(dimensionToken);
+ } else {
+ // Done with the prev sub tree
+ return;
+ }
+ }
+}
+
+void writeDimensionLeafToProtoHelper(const std::vector<FieldValue>& dims,
+ const int dimensionLeafField,
+ size_t* index, int depth,
+ int prefix, std::set<string> *str_set,
+ ProtoOutputStream* protoOutput) {
+ size_t count = dims.size();
+ while (*index < count) {
+ const auto& dim = dims[*index];
+ const int valueDepth = dim.mField.getDepth();
+ const int valuePrefix = dim.mField.getPrefix(depth);
+ if (valueDepth > 2) {
+ ALOGE("Depth > 2 not supported");
+ return;
+ }
+
+ if (depth == valueDepth && valuePrefix == prefix) {
+ uint64_t token = protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED |
+ dimensionLeafField);
+ switch (dim.mValue.getType()) {
+ case INT:
+ protoOutput->write(FIELD_TYPE_INT32 | DIMENSIONS_VALUE_VALUE_INT,
+ dim.mValue.int_value);
+ break;
+ case LONG:
+ protoOutput->write(FIELD_TYPE_INT64 | DIMENSIONS_VALUE_VALUE_LONG,
+ (long long)dim.mValue.long_value);
+ break;
+ case FLOAT:
+ protoOutput->write(FIELD_TYPE_FLOAT | DIMENSIONS_VALUE_VALUE_FLOAT,
+ dim.mValue.float_value);
+ break;
+ case STRING:
+ if (str_set == nullptr) {
+ protoOutput->write(FIELD_TYPE_STRING | DIMENSIONS_VALUE_VALUE_STR,
+ dim.mValue.str_value);
+ } else {
+ str_set->insert(dim.mValue.str_value);
+ protoOutput->write(
+ FIELD_TYPE_UINT64 | DIMENSIONS_VALUE_VALUE_STR_HASH,
+ (long long)Hash64(dim.mValue.str_value));
+ }
+ break;
+ default:
+ break;
+ }
+ if (token != 0) {
+ protoOutput->end(token);
+ }
+ (*index)++;
+ } else if (valueDepth > depth && valuePrefix == prefix) {
+ writeDimensionLeafToProtoHelper(dims, dimensionLeafField,
+ index, valueDepth, dim.mField.getPrefix(valueDepth),
+ str_set, protoOutput);
+ } else {
+ // Done with the prev sub tree
+ return;
+ }
+ }
+}
+
+void writeDimensionPathToProtoHelper(const std::vector<Matcher>& fieldMatchers,
+ size_t* index, int depth, int prefix,
+ ProtoOutputStream* protoOutput) {
+ size_t count = fieldMatchers.size();
+ while (*index < count) {
+ const Field& field = fieldMatchers[*index].mMatcher;
+ const int valueDepth = field.getDepth();
+ const int valuePrefix = field.getPrefix(depth);
+ const int fieldNum = field.getPosAtDepth(depth);
+ if (valueDepth > 2) {
+ ALOGE("Depth > 2 not supported");
+ return;
+ }
+
+ if (depth == valueDepth && valuePrefix == prefix) {
+ uint64_t token = protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED |
+ DIMENSIONS_VALUE_TUPLE_VALUE);
+ protoOutput->write(FIELD_TYPE_INT32 | DIMENSIONS_VALUE_FIELD, fieldNum);
+ if (token != 0) {
+ protoOutput->end(token);
+ }
+ (*index)++;
+ } else if (valueDepth > depth && valuePrefix == prefix) {
+ // Writing the sub tree
+ uint64_t dimensionToken = protoOutput->start(
+ FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | DIMENSIONS_VALUE_TUPLE_VALUE);
+ protoOutput->write(FIELD_TYPE_INT32 | DIMENSIONS_VALUE_FIELD, fieldNum);
+ uint64_t tupleToken =
+ protoOutput->start(FIELD_TYPE_MESSAGE | DIMENSIONS_VALUE_VALUE_TUPLE);
+ writeDimensionPathToProtoHelper(fieldMatchers, index, valueDepth,
+ field.getPrefix(valueDepth), protoOutput);
protoOutput->end(tupleToken);
protoOutput->end(dimensionToken);
} else {
@@ -117,7 +230,8 @@
} // namespace
-void writeDimensionToProto(const HashableDimensionKey& dimension, ProtoOutputStream* protoOutput) {
+void writeDimensionToProto(const HashableDimensionKey& dimension, std::set<string> *str_set,
+ ProtoOutputStream* protoOutput) {
if (dimension.getValues().size() == 0) {
return;
}
@@ -125,7 +239,32 @@
dimension.getValues()[0].mField.getTag());
uint64_t topToken = protoOutput->start(FIELD_TYPE_MESSAGE | DIMENSIONS_VALUE_VALUE_TUPLE);
size_t index = 0;
- writeDimensionToProtoHelper(dimension.getValues(), &index, 0, 0, protoOutput);
+ writeDimensionToProtoHelper(dimension.getValues(), &index, 0, 0, str_set, protoOutput);
+ protoOutput->end(topToken);
+}
+
+void writeDimensionLeafNodesToProto(const HashableDimensionKey& dimension,
+ const int dimensionLeafFieldId,
+ std::set<string> *str_set,
+ ProtoOutputStream* protoOutput) {
+ if (dimension.getValues().size() == 0) {
+ return;
+ }
+ size_t index = 0;
+ writeDimensionLeafToProtoHelper(dimension.getValues(), dimensionLeafFieldId,
+ &index, 0, 0, str_set, protoOutput);
+}
+
+void writeDimensionPathToProto(const std::vector<Matcher>& fieldMatchers,
+ ProtoOutputStream* protoOutput) {
+ if (fieldMatchers.size() == 0) {
+ return;
+ }
+ protoOutput->write(FIELD_TYPE_INT32 | DIMENSIONS_VALUE_FIELD,
+ fieldMatchers[0].mMatcher.getTag());
+ uint64_t topToken = protoOutput->start(FIELD_TYPE_MESSAGE | DIMENSIONS_VALUE_VALUE_TUPLE);
+ size_t index = 0;
+ writeDimensionPathToProtoHelper(fieldMatchers, &index, 0, 0, protoOutput);
protoOutput->end(topToken);
}
@@ -297,6 +436,14 @@
return timestampNs / NS_PER_SEC / (5 * 60) * NS_PER_SEC * (5 * 60);
}
+int64_t NanoToMillis(const int64_t nano) {
+ return nano / 1000000;
+}
+
+int64_t MillisToNano(const int64_t millis) {
+ return millis * 1000000;
+}
+
} // namespace statsd
} // namespace os
} // namespace android
diff --git a/cmds/statsd/src/stats_log_util.h b/cmds/statsd/src/stats_log_util.h
index 9722050..b8f6850 100644
--- a/cmds/statsd/src/stats_log_util.h
+++ b/cmds/statsd/src/stats_log_util.h
@@ -28,9 +28,17 @@
void writeFieldValueTreeToStream(int tagId, const std::vector<FieldValue>& values,
util::ProtoOutputStream* protoOutput);
-void writeDimensionToProto(const HashableDimensionKey& dimension,
+void writeDimensionToProto(const HashableDimensionKey& dimension, std::set<string> *str_set,
util::ProtoOutputStream* protoOutput);
+void writeDimensionLeafNodesToProto(const HashableDimensionKey& dimension,
+ const int dimensionLeafFieldId,
+ std::set<string> *str_set,
+ util::ProtoOutputStream* protoOutput);
+
+void writeDimensionPathToProto(const std::vector<Matcher>& fieldMatchers,
+ util::ProtoOutputStream* protoOutput);
+
// Convert the TimeUnit enum to the bucket size in millis with a guardrail on
// bucket size.
int64_t TimeUnitToBucketSizeInMillisGuardrailed(int uid, TimeUnit unit);
@@ -56,6 +64,10 @@
// Gets the wall clock timestamp in seconds.
int64_t getWallClockSec();
+int64_t NanoToMillis(const int64_t nano);
+
+int64_t MillisToNano(const int64_t millis);
+
// Helper function to write PulledAtomStats to ProtoOutputStream
void writePullerStatsToStream(const std::pair<int, StatsdStats::PulledAtomStats>& pair,
util::ProtoOutputStream* protoOutput);
diff --git a/cmds/statsd/tests/FieldValue_test.cpp b/cmds/statsd/tests/FieldValue_test.cpp
index 5a6aba6..c253bc1 100644
--- a/cmds/statsd/tests/FieldValue_test.cpp
+++ b/cmds/statsd/tests/FieldValue_test.cpp
@@ -226,6 +226,68 @@
EXPECT_EQ((int32_t)27, link.conditionFields[0].mMatcher.getTag());
}
+TEST(AtomMatcherTest, TestWriteDimensionPath) {
+ for (auto position : {Position::ANY, Position::ALL, Position::FIRST, Position::LAST}) {
+ FieldMatcher matcher1;
+ matcher1.set_field(10);
+ FieldMatcher* child = matcher1.add_child();
+ child->set_field(2);
+ child->set_position(position);
+ child->add_child()->set_field(1);
+ child->add_child()->set_field(3);
+
+ child = matcher1.add_child();
+ child->set_field(4);
+
+ child = matcher1.add_child();
+ child->set_field(6);
+ child->add_child()->set_field(2);
+
+ vector<Matcher> matchers;
+ translateFieldMatcher(matcher1, &matchers);
+
+ android::util::ProtoOutputStream protoOut;
+ writeDimensionPathToProto(matchers, &protoOut);
+
+ vector<uint8_t> outData;
+ outData.resize(protoOut.size());
+ size_t pos = 0;
+ auto iter = protoOut.data();
+ while (iter.readBuffer() != NULL) {
+ size_t toRead = iter.currentToRead();
+ std::memcpy(&(outData[pos]), iter.readBuffer(), toRead);
+ pos += toRead;
+ iter.rp()->move(toRead);
+ }
+
+ DimensionsValue result;
+ EXPECT_EQ(true, result.ParseFromArray(&outData[0], outData.size()));
+
+ EXPECT_EQ(10, result.field());
+ EXPECT_EQ(DimensionsValue::ValueCase::kValueTuple, result.value_case());
+ EXPECT_EQ(3, result.value_tuple().dimensions_value_size());
+
+ const auto& dim1 = result.value_tuple().dimensions_value(0);
+ EXPECT_EQ(2, dim1.field());
+ EXPECT_EQ(2, dim1.value_tuple().dimensions_value_size());
+
+ const auto& dim11 = dim1.value_tuple().dimensions_value(0);
+ EXPECT_EQ(1, dim11.field());
+
+ const auto& dim12 = dim1.value_tuple().dimensions_value(1);
+ EXPECT_EQ(3, dim12.field());
+
+ const auto& dim2 = result.value_tuple().dimensions_value(1);
+ EXPECT_EQ(4, dim2.field());
+
+ const auto& dim3 = result.value_tuple().dimensions_value(2);
+ EXPECT_EQ(6, dim3.field());
+ EXPECT_EQ(1, dim3.value_tuple().dimensions_value_size());
+ const auto& dim31 = dim3.value_tuple().dimensions_value(0);
+ EXPECT_EQ(2, dim31.field());
+ }
+}
+
TEST(AtomMatcherTest, TestSubscriberDimensionWrite) {
HashableDimensionKey dim;
@@ -275,7 +337,7 @@
dim.addValue(FieldValue(field4, value4));
android::util::ProtoOutputStream protoOut;
- writeDimensionToProto(dim, &protoOut);
+ writeDimensionToProto(dim, nullptr /* include strings */, &protoOut);
vector<uint8_t> outData;
outData.resize(protoOut.size());
@@ -315,6 +377,62 @@
EXPECT_EQ(99999, dim2.value_int());
}
+TEST(AtomMatcherTest, TestWriteDimensionLeafNodesToProto) {
+ HashableDimensionKey dim;
+ int pos1[] = {1, 1, 1};
+ int pos2[] = {1, 1, 2};
+ int pos3[] = {1, 1, 3};
+ int pos4[] = {2, 0, 0};
+ Field field1(10, pos1, 2);
+ Field field2(10, pos2, 2);
+ Field field3(10, pos3, 2);
+ Field field4(10, pos4, 0);
+
+ Value value1((int32_t)10025);
+ Value value2("tag");
+ Value value3((int32_t)987654);
+ Value value4((int64_t)99999);
+
+ dim.addValue(FieldValue(field1, value1));
+ dim.addValue(FieldValue(field2, value2));
+ dim.addValue(FieldValue(field3, value3));
+ dim.addValue(FieldValue(field4, value4));
+
+ android::util::ProtoOutputStream protoOut;
+ writeDimensionLeafNodesToProto(dim, 1, nullptr /* include strings */, &protoOut);
+
+ vector<uint8_t> outData;
+ outData.resize(protoOut.size());
+ size_t pos = 0;
+ auto iter = protoOut.data();
+ while (iter.readBuffer() != NULL) {
+ size_t toRead = iter.currentToRead();
+ std::memcpy(&(outData[pos]), iter.readBuffer(), toRead);
+ pos += toRead;
+ iter.rp()->move(toRead);
+ }
+
+ DimensionsValueTuple result;
+ EXPECT_EQ(true, result.ParseFromArray(&outData[0], outData.size()));
+ EXPECT_EQ(4, result.dimensions_value_size());
+
+ const auto& dim1 = result.dimensions_value(0);
+ EXPECT_EQ(DimensionsValue::ValueCase::kValueInt, dim1.value_case());
+ EXPECT_EQ(10025, dim1.value_int());
+
+ const auto& dim2 = result.dimensions_value(1);
+ EXPECT_EQ(DimensionsValue::ValueCase::kValueStr, dim2.value_case());
+ EXPECT_EQ("tag", dim2.value_str());
+
+ const auto& dim3 = result.dimensions_value(2);
+ EXPECT_EQ(DimensionsValue::ValueCase::kValueInt, dim3.value_case());
+ EXPECT_EQ(987654, dim3.value_int());
+
+ const auto& dim4 = result.dimensions_value(3);
+ EXPECT_EQ(DimensionsValue::ValueCase::kValueLong, dim4.value_case());
+ EXPECT_EQ(99999, dim4.value_long());
+}
+
TEST(AtomMatcherTest, TestWriteAtomToProto) {
AttributionNodeInternal attribution_node1;
attribution_node1.set_uid(1111);
diff --git a/cmds/statsd/tests/StatsLogProcessor_test.cpp b/cmds/statsd/tests/StatsLogProcessor_test.cpp
index 004b235..9fdf7a3 100644
--- a/cmds/statsd/tests/StatsLogProcessor_test.cpp
+++ b/cmds/statsd/tests/StatsLogProcessor_test.cpp
@@ -139,7 +139,7 @@
// Expect to get no metrics, but snapshot specified above in uidmap.
vector<uint8_t> bytes;
- p.onDumpReport(key, 1, false, ADB_DUMP, &bytes);
+ p.onDumpReport(key, 1, false, true, ADB_DUMP, &bytes);
ConfigMetricsReportList output;
output.ParseFromArray(bytes.data(), bytes.size());
@@ -167,7 +167,7 @@
// Expect to get no metrics, but snapshot specified above in uidmap.
vector<uint8_t> bytes;
- p.onDumpReport(key, 1, false, ADB_DUMP, &bytes);
+ p.onDumpReport(key, 1, false, true, ADB_DUMP, &bytes);
ConfigMetricsReportList output;
output.ParseFromArray(bytes.data(), bytes.size());
diff --git a/cmds/statsd/tests/UidMap_test.cpp b/cmds/statsd/tests/UidMap_test.cpp
index 2fab975..dde50c2 100644
--- a/cmds/statsd/tests/UidMap_test.cpp
+++ b/cmds/statsd/tests/UidMap_test.cpp
@@ -17,6 +17,7 @@
#include "config/ConfigKey.h"
#include "guardrail/StatsdStats.h"
#include "logd/LogEvent.h"
+#include "hash.h"
#include "statslog.h"
#include "statsd_test_util.h"
@@ -192,7 +193,7 @@
m.mLastUpdatePerConfigKey[config1] = 2;
ProtoOutputStream proto;
- m.appendUidMap(3, config1, &proto);
+ m.appendUidMap(3, config1, nullptr, &proto);
// Check there's still a uidmap attached this one.
UidMapping results;
@@ -215,7 +216,7 @@
m.removeApp(2, String16(kApp2.c_str()), 1000);
ProtoOutputStream proto;
- m.appendUidMap(3, config1, &proto);
+ m.appendUidMap(3, config1, nullptr, &proto);
// Snapshot should still contain this item as deleted.
UidMapping results;
@@ -243,7 +244,7 @@
// First, verify that we have the expected number of items.
UidMapping results;
ProtoOutputStream proto;
- m.appendUidMap(3, config1, &proto);
+ m.appendUidMap(3, config1, nullptr, &proto);
protoOutputStreamToUidMapping(&proto, &results);
EXPECT_EQ(maxDeletedApps + 10, results.snapshots(0).package_info_size());
@@ -254,7 +255,7 @@
}
proto.clear();
- m.appendUidMap(5, config1, &proto);
+ m.appendUidMap(5, config1, nullptr, &proto);
// Snapshot drops the first nine items.
protoOutputStreamToUidMapping(&proto, &results);
EXPECT_EQ(maxDeletedApps, results.snapshots(0).package_info_size());
@@ -280,14 +281,14 @@
m.updateMap(1, uids, versions, apps);
ProtoOutputStream proto;
- m.appendUidMap(2, config1, &proto);
+ m.appendUidMap(2, config1, nullptr, &proto);
UidMapping results;
protoOutputStreamToUidMapping(&proto, &results);
EXPECT_EQ(1, results.snapshots_size());
// We have to keep at least one snapshot in memory at all times.
proto.clear();
- m.appendUidMap(2, config1, &proto);
+ m.appendUidMap(2, config1, nullptr, &proto);
protoOutputStreamToUidMapping(&proto, &results);
EXPECT_EQ(1, results.snapshots_size());
@@ -296,7 +297,7 @@
m.updateApp(5, String16(kApp1.c_str()), 1000, 40);
EXPECT_EQ(1U, m.mChanges.size());
proto.clear();
- m.appendUidMap(6, config1, &proto);
+ m.appendUidMap(6, config1, nullptr, &proto);
protoOutputStreamToUidMapping(&proto, &results);
EXPECT_EQ(1, results.snapshots_size());
EXPECT_EQ(1, results.changes_size());
@@ -308,14 +309,14 @@
// We still can't remove anything.
proto.clear();
- m.appendUidMap(8, config1, &proto);
+ m.appendUidMap(8, config1, nullptr, &proto);
protoOutputStreamToUidMapping(&proto, &results);
EXPECT_EQ(1, results.snapshots_size());
EXPECT_EQ(1, results.changes_size());
EXPECT_EQ(2U, m.mChanges.size());
proto.clear();
- m.appendUidMap(9, config2, &proto);
+ m.appendUidMap(9, config2, nullptr, &proto);
protoOutputStreamToUidMapping(&proto, &results);
EXPECT_EQ(1, results.snapshots_size());
EXPECT_EQ(2, results.changes_size());
@@ -342,10 +343,10 @@
ProtoOutputStream proto;
vector<uint8_t> bytes;
- m.appendUidMap(2, config1, &proto);
+ m.appendUidMap(2, config1, nullptr, &proto);
size_t prevBytes = m.mBytesUsed;
- m.appendUidMap(4, config1, &proto);
+ m.appendUidMap(4, config1, nullptr, &proto);
EXPECT_TRUE(m.mBytesUsed < prevBytes);
}
@@ -376,6 +377,7 @@
m.updateApp(5, String16("EXTREMELY_LONG_STRING_FOR_APP_TO_WASTE_MEMORY.0"), 1000, 4);
EXPECT_EQ(1U, m.mChanges.size());
}
+
#else
GTEST_LOG_(INFO) << "This test does nothing.\n";
#endif
diff --git a/cmds/statsd/tests/e2e/Attribution_e2e_test.cpp b/cmds/statsd/tests/e2e/Attribution_e2e_test.cpp
index 3b24341..5c47af7 100644
--- a/cmds/statsd/tests/e2e/Attribution_e2e_test.cpp
+++ b/cmds/statsd/tests/e2e/Attribution_e2e_test.cpp
@@ -144,10 +144,13 @@
}
ConfigMetricsReportList reports;
vector<uint8_t> buffer;
- processor->onDumpReport(cfgKey, bucketStartTimeNs + 4 * bucketSizeNs + 1, false, ADB_DUMP,
- &buffer);
+ processor->onDumpReport(cfgKey, bucketStartTimeNs + 4 * bucketSizeNs + 1, false, true,
+ ADB_DUMP, &buffer);
EXPECT_TRUE(buffer.size() > 0);
EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size()));
+ backfillDimensionPath(&reports);
+ backfillStringInReport(&reports);
+ backfillStartEndTimestamp(&reports);
EXPECT_EQ(reports.reports_size(), 1);
EXPECT_EQ(reports.reports(0).metrics_size(), 1);
@@ -287,10 +290,13 @@
}
ConfigMetricsReportList reports;
vector<uint8_t> buffer;
- processor->onDumpReport(cfgKey, bucketStartTimeNs + 4 * bucketSizeNs + 1, false, ADB_DUMP,
- &buffer);
+ processor->onDumpReport(cfgKey, bucketStartTimeNs + 4 * bucketSizeNs + 1, false, true,
+ ADB_DUMP, &buffer);
EXPECT_TRUE(buffer.size() > 0);
EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size()));
+ backfillDimensionPath(&reports);
+ backfillStringInReport(&reports);
+ backfillStartEndTimestamp(&reports);
EXPECT_EQ(reports.reports_size(), 1);
EXPECT_EQ(reports.reports(0).metrics_size(), 1);
diff --git a/cmds/statsd/tests/e2e/DimensionInCondition_e2e_combination_AND_cond_test.cpp b/cmds/statsd/tests/e2e/DimensionInCondition_e2e_combination_AND_cond_test.cpp
index 934b951..8a74f2d 100644
--- a/cmds/statsd/tests/e2e/DimensionInCondition_e2e_combination_AND_cond_test.cpp
+++ b/cmds/statsd/tests/e2e/DimensionInCondition_e2e_combination_AND_cond_test.cpp
@@ -172,10 +172,13 @@
ConfigMetricsReportList reports;
vector<uint8_t> buffer;
- processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, false,
- ADB_DUMP, &buffer);
+ processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1,
+ false, true, ADB_DUMP, &buffer);
EXPECT_TRUE(buffer.size() > 0);
EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size()));
+ backfillDimensionPath(&reports);
+ backfillStringInReport(&reports);
+ backfillStartEndTimestamp(&reports);
EXPECT_EQ(reports.reports_size(), 1);
EXPECT_EQ(reports.reports(0).metrics_size(), 1);
@@ -489,10 +492,13 @@
ConfigMetricsReportList reports;
vector<uint8_t> buffer;
- processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, false,
+ processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, false, true,
ADB_DUMP, &buffer);
EXPECT_TRUE(buffer.size() > 0);
EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size()));
+ backfillDimensionPath(&reports);
+ backfillStringInReport(&reports);
+ backfillStartEndTimestamp(&reports);
EXPECT_EQ(reports.reports_size(), 1);
EXPECT_EQ(reports.reports(0).metrics_size(), 1);
@@ -733,10 +739,13 @@
ConfigMetricsReportList reports;
vector<uint8_t> buffer;
- processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, false, ADB_DUMP,
- &buffer);
+ processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, false, true,
+ ADB_DUMP, &buffer);
EXPECT_TRUE(buffer.size() > 0);
EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size()));
+ backfillDimensionPath(&reports);
+ backfillStringInReport(&reports);
+ backfillStartEndTimestamp(&reports);
EXPECT_EQ(reports.reports_size(), 1);
EXPECT_EQ(reports.reports(0).metrics_size(), 1);
diff --git a/cmds/statsd/tests/e2e/DimensionInCondition_e2e_combination_OR_cond_test.cpp b/cmds/statsd/tests/e2e/DimensionInCondition_e2e_combination_OR_cond_test.cpp
index 9f20754..d4fe712 100644
--- a/cmds/statsd/tests/e2e/DimensionInCondition_e2e_combination_OR_cond_test.cpp
+++ b/cmds/statsd/tests/e2e/DimensionInCondition_e2e_combination_OR_cond_test.cpp
@@ -130,10 +130,13 @@
ConfigMetricsReportList reports;
vector<uint8_t> buffer;
- processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, false, ADB_DUMP,
- &buffer);
+ processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, false, true,
+ ADB_DUMP, &buffer);
EXPECT_TRUE(buffer.size() > 0);
EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size()));
+ backfillDimensionPath(&reports);
+ backfillStringInReport(&reports);
+ backfillStartEndTimestamp(&reports);
EXPECT_EQ(reports.reports_size(), 1);
EXPECT_EQ(reports.reports(0).metrics_size(), 1);
@@ -343,10 +346,13 @@
ConfigMetricsReportList reports;
vector<uint8_t> buffer;
- processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, false, ADB_DUMP,
- &buffer);
+ processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, false, true,
+ ADB_DUMP, &buffer);
EXPECT_TRUE(buffer.size() > 0);
EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size()));
+ backfillDimensionPath(&reports);
+ backfillStringInReport(&reports);
+ backfillStartEndTimestamp(&reports);
EXPECT_EQ(reports.reports_size(), 1);
EXPECT_EQ(reports.reports(0).metrics_size(), 1);
@@ -524,10 +530,13 @@
ConfigMetricsReportList reports;
vector<uint8_t> buffer;
- processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, false, ADB_DUMP,
- &buffer);
+ processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, false, true,
+ ADB_DUMP, &buffer);
EXPECT_TRUE(buffer.size() > 0);
EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size()));
+ backfillDimensionPath(&reports);
+ backfillStringInReport(&reports);
+ backfillStartEndTimestamp(&reports);
EXPECT_EQ(reports.reports_size(), 1);
EXPECT_EQ(reports.reports(0).metrics_size(), 1);
@@ -723,10 +732,13 @@
ConfigMetricsReportList reports;
vector<uint8_t> buffer;
- processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, false, ADB_DUMP,
- &buffer);
+ processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, false, true,
+ ADB_DUMP, &buffer);
EXPECT_TRUE(buffer.size() > 0);
EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size()));
+ backfillDimensionPath(&reports);
+ backfillStringInReport(&reports);
+ backfillStartEndTimestamp(&reports);
EXPECT_EQ(reports.reports_size(), 1);
EXPECT_EQ(reports.reports(0).metrics_size(), 1);
diff --git a/cmds/statsd/tests/e2e/DimensionInCondition_e2e_simple_cond_test.cpp b/cmds/statsd/tests/e2e/DimensionInCondition_e2e_simple_cond_test.cpp
index 3f193ac..97089ca 100644
--- a/cmds/statsd/tests/e2e/DimensionInCondition_e2e_simple_cond_test.cpp
+++ b/cmds/statsd/tests/e2e/DimensionInCondition_e2e_simple_cond_test.cpp
@@ -142,10 +142,13 @@
ConfigMetricsReportList reports;
vector<uint8_t> buffer;
- processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, false,
+ processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, false, true,
ADB_DUMP, &buffer);
EXPECT_TRUE(buffer.size() > 0);
EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size()));
+ backfillDimensionPath(&reports);
+ backfillStringInReport(&reports);
+ backfillStartEndTimestamp(&reports);
EXPECT_EQ(reports.reports_size(), 1);
EXPECT_EQ(reports.reports(0).metrics_size(), 1);
@@ -434,10 +437,13 @@
ConfigMetricsReportList reports;
vector<uint8_t> buffer;
- processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, false,
+ processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, false, true,
ADB_DUMP, &buffer);
EXPECT_TRUE(buffer.size() > 0);
EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size()));
+ backfillDimensionPath(&reports);
+ backfillStringInReport(&reports);
+ backfillStartEndTimestamp(&reports);
EXPECT_EQ(reports.reports_size(), 1);
EXPECT_EQ(reports.reports(0).metrics_size(), 1);
@@ -652,10 +658,13 @@
ConfigMetricsReportList reports;
vector<uint8_t> buffer;
- processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, false, ADB_DUMP,
- &buffer);
+ processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, false, true,
+ ADB_DUMP, &buffer);
EXPECT_TRUE(buffer.size() > 0);
EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size()));
+ backfillDimensionPath(&reports);
+ backfillStringInReport(&reports);
+ backfillStartEndTimestamp(&reports);
EXPECT_EQ(reports.reports_size(), 1);
EXPECT_EQ(reports.reports(0).metrics_size(), 1);
diff --git a/cmds/statsd/tests/e2e/GaugeMetric_e2e_pull_test.cpp b/cmds/statsd/tests/e2e/GaugeMetric_e2e_pull_test.cpp
index f4ad0ce..6a69100 100644
--- a/cmds/statsd/tests/e2e/GaugeMetric_e2e_pull_test.cpp
+++ b/cmds/statsd/tests/e2e/GaugeMetric_e2e_pull_test.cpp
@@ -122,10 +122,13 @@
ConfigMetricsReportList reports;
vector<uint8_t> buffer;
- processor->onDumpReport(cfgKey, configAddedTimeNs + 7 * bucketSizeNs + 10, false, ADB_DUMP,
- &buffer);
+ processor->onDumpReport(cfgKey, configAddedTimeNs + 7 * bucketSizeNs + 10, false, true,
+ ADB_DUMP, &buffer);
EXPECT_TRUE(buffer.size() > 0);
EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size()));
+ backfillDimensionPath(&reports);
+ backfillStringInReport(&reports);
+ backfillStartEndTimestamp(&reports);
EXPECT_EQ(1, reports.reports_size());
EXPECT_EQ(1, reports.reports(0).metrics_size());
StatsLogReport::GaugeMetricDataWrapper gaugeMetrics;
@@ -241,10 +244,13 @@
ConfigMetricsReportList reports;
vector<uint8_t> buffer;
- processor->onDumpReport(cfgKey, configAddedTimeNs + 8 * bucketSizeNs + 10, false, ADB_DUMP,
- &buffer);
+ processor->onDumpReport(cfgKey, configAddedTimeNs + 8 * bucketSizeNs + 10, false, true,
+ ADB_DUMP, &buffer);
EXPECT_TRUE(buffer.size() > 0);
EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size()));
+ backfillDimensionPath(&reports);
+ backfillStringInReport(&reports);
+ backfillStartEndTimestamp(&reports);
EXPECT_EQ(1, reports.reports_size());
EXPECT_EQ(1, reports.reports(0).metrics_size());
StatsLogReport::GaugeMetricDataWrapper gaugeMetrics;
@@ -342,10 +348,13 @@
ConfigMetricsReportList reports;
vector<uint8_t> buffer;
- processor->onDumpReport(cfgKey, configAddedTimeNs + 7 * bucketSizeNs + 10, false, ADB_DUMP,
- &buffer);
+ processor->onDumpReport(cfgKey, configAddedTimeNs + 7 * bucketSizeNs + 10, false, true,
+ ADB_DUMP, &buffer);
EXPECT_TRUE(buffer.size() > 0);
EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size()));
+ backfillDimensionPath(&reports);
+ backfillStringInReport(&reports);
+ backfillStartEndTimestamp(&reports);
EXPECT_EQ(1, reports.reports_size());
EXPECT_EQ(1, reports.reports(0).metrics_size());
StatsLogReport::GaugeMetricDataWrapper gaugeMetrics;
diff --git a/cmds/statsd/tests/e2e/GaugeMetric_e2e_push_test.cpp b/cmds/statsd/tests/e2e/GaugeMetric_e2e_push_test.cpp
index 98372ff..f1052f6 100644
--- a/cmds/statsd/tests/e2e/GaugeMetric_e2e_push_test.cpp
+++ b/cmds/statsd/tests/e2e/GaugeMetric_e2e_push_test.cpp
@@ -149,10 +149,13 @@
}
ConfigMetricsReportList reports;
vector<uint8_t> buffer;
- processor->onDumpReport(cfgKey, bucketStartTimeNs + 3 * bucketSizeNs, false, ADB_DUMP,
- &buffer);
+ processor->onDumpReport(cfgKey, bucketStartTimeNs + 3 * bucketSizeNs, false, true,
+ ADB_DUMP, &buffer);
EXPECT_TRUE(buffer.size() > 0);
EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size()));
+ backfillDimensionPath(&reports);
+ backfillStringInReport(&reports);
+ backfillStartEndTimestamp(&reports);
EXPECT_EQ(1, reports.reports_size());
EXPECT_EQ(1, reports.reports(0).metrics_size());
StatsLogReport::GaugeMetricDataWrapper gaugeMetrics;
diff --git a/cmds/statsd/tests/e2e/MetricConditionLink_e2e_test.cpp b/cmds/statsd/tests/e2e/MetricConditionLink_e2e_test.cpp
index 8020787..eca35c5 100644
--- a/cmds/statsd/tests/e2e/MetricConditionLink_e2e_test.cpp
+++ b/cmds/statsd/tests/e2e/MetricConditionLink_e2e_test.cpp
@@ -200,10 +200,13 @@
}
ConfigMetricsReportList reports;
vector<uint8_t> buffer;
- processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs - 1, false, ADB_DUMP,
- &buffer);
+ processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs - 1, false, true,
+ ADB_DUMP, &buffer);
EXPECT_TRUE(buffer.size() > 0);
EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size()));
+ backfillDimensionPath(&reports);
+ backfillStringInReport(&reports);
+ backfillStartEndTimestamp(&reports);
EXPECT_EQ(reports.reports_size(), 1);
EXPECT_EQ(reports.reports(0).metrics_size(), 1);
EXPECT_EQ(reports.reports(0).metrics(0).count_metrics().data_size(), 1);
@@ -316,10 +319,13 @@
ConfigMetricsReportList reports;
vector<uint8_t> buffer;
- processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, false, ADB_DUMP,
- &buffer);
+ processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, false, true,
+ ADB_DUMP, &buffer);
EXPECT_TRUE(buffer.size() > 0);
EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size()));
+ backfillDimensionPath(&reports);
+ backfillStringInReport(&reports);
+ backfillStartEndTimestamp(&reports);
EXPECT_EQ(reports.reports_size(), 1);
EXPECT_EQ(reports.reports(0).metrics_size(), 1);
EXPECT_EQ(reports.reports(0).metrics(0).count_metrics().data_size(), 1);
diff --git a/cmds/statsd/tests/e2e/PartialBucket_e2e_test.cpp b/cmds/statsd/tests/e2e/PartialBucket_e2e_test.cpp
index d646f73..545fa01 100644
--- a/cmds/statsd/tests/e2e/PartialBucket_e2e_test.cpp
+++ b/cmds/statsd/tests/e2e/PartialBucket_e2e_test.cpp
@@ -46,7 +46,7 @@
IPCThreadState* ipc = IPCThreadState::self();
ConfigKey configKey(ipc->getCallingUid(), kConfigKey);
processor->onDumpReport(configKey, timestamp, include_current /* include_current_bucket*/,
- ADB_DUMP, &output);
+ true/* include strings*/, ADB_DUMP, &output);
ConfigMetricsReportList reports;
reports.ParseFromArray(output.data(), output.size());
EXPECT_EQ(1, reports.reports_size());
@@ -153,7 +153,12 @@
service.mProcessor->OnLogEvent(CreateAppCrashEvent(100, start + 3).get());
ConfigMetricsReport report = GetReports(service.mProcessor, start + 4);
+ backfillStartEndTimestamp(&report);
EXPECT_EQ(1, report.metrics_size());
+ EXPECT_TRUE(report.metrics(0).count_metrics().data(0).bucket_info(0).
+ has_start_bucket_elapsed_nanos());
+ EXPECT_TRUE(report.metrics(0).count_metrics().data(0).bucket_info(0).
+ has_end_bucket_elapsed_nanos());
EXPECT_EQ(1, report.metrics(0).count_metrics().data(0).bucket_info(0).count());
}
@@ -171,7 +176,12 @@
service.mProcessor->OnLogEvent(CreateAppCrashEvent(100, start + 3).get());
ConfigMetricsReport report = GetReports(service.mProcessor, start + 4);
+ backfillStartEndTimestamp(&report);
EXPECT_EQ(1, report.metrics_size());
+ EXPECT_TRUE(report.metrics(0).count_metrics().data(0).bucket_info(0).
+ has_start_bucket_elapsed_nanos());
+ EXPECT_TRUE(report.metrics(0).count_metrics().data(0).bucket_info(0).
+ has_end_bucket_elapsed_nanos());
EXPECT_EQ(1, report.metrics(0).count_metrics().data(0).bucket_info(0).count());
}
@@ -206,10 +216,13 @@
ConfigMetricsReport report =
GetReports(service.mProcessor, 5 * 60 * NS_PER_SEC + start + 100 * NS_PER_SEC, true);
+ backfillStartEndTimestamp(&report);
EXPECT_EQ(1, report.metrics_size());
EXPECT_EQ(1, report.metrics(0).value_metrics().skipped_size());
+ EXPECT_TRUE(report.metrics(0).value_metrics().skipped(0).has_start_bucket_elapsed_nanos());
// Can't test the start time since it will be based on the actual time when the pulling occurs.
- EXPECT_EQ(endSkipped, report.metrics(0).value_metrics().skipped(0).end_elapsed_nanos());
+ EXPECT_EQ(MillisToNano(NanoToMillis(endSkipped)),
+ report.metrics(0).value_metrics().skipped(0).end_bucket_elapsed_nanos());
}
TEST(PartialBucketE2eTest, TestGaugeMetricWithoutMinPartialBucket) {
@@ -243,10 +256,13 @@
ConfigMetricsReport report =
GetReports(service.mProcessor, 5 * 60 * NS_PER_SEC + start + 100 * NS_PER_SEC, true);
+ backfillStartEndTimestamp(&report);
EXPECT_EQ(1, report.metrics_size());
EXPECT_EQ(1, report.metrics(0).gauge_metrics().skipped_size());
// Can't test the start time since it will be based on the actual time when the pulling occurs.
- EXPECT_EQ(endSkipped, report.metrics(0).gauge_metrics().skipped(0).end_elapsed_nanos());
+ EXPECT_TRUE(report.metrics(0).gauge_metrics().skipped(0).has_start_bucket_elapsed_nanos());
+ EXPECT_EQ(MillisToNano(NanoToMillis(endSkipped)),
+ report.metrics(0).gauge_metrics().skipped(0).end_bucket_elapsed_nanos());
}
#else
diff --git a/cmds/statsd/tests/e2e/ValueMetric_pull_e2e_test.cpp b/cmds/statsd/tests/e2e/ValueMetric_pull_e2e_test.cpp
index a01e91f..98a312f 100644
--- a/cmds/statsd/tests/e2e/ValueMetric_pull_e2e_test.cpp
+++ b/cmds/statsd/tests/e2e/ValueMetric_pull_e2e_test.cpp
@@ -117,10 +117,13 @@
ConfigMetricsReportList reports;
vector<uint8_t> buffer;
- processor->onDumpReport(cfgKey, configAddedTimeNs + 7 * bucketSizeNs + 10, false, ADB_DUMP,
- &buffer);
+ processor->onDumpReport(cfgKey, configAddedTimeNs + 7 * bucketSizeNs + 10, false, true,
+ ADB_DUMP, &buffer);
EXPECT_TRUE(buffer.size() > 0);
EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size()));
+ backfillDimensionPath(&reports);
+ backfillStringInReport(&reports);
+ backfillStartEndTimestamp(&reports);
EXPECT_EQ(1, reports.reports_size());
EXPECT_EQ(1, reports.reports(0).metrics_size());
StatsLogReport::ValueMetricDataWrapper valueMetrics;
@@ -221,10 +224,13 @@
ConfigMetricsReportList reports;
vector<uint8_t> buffer;
- processor->onDumpReport(cfgKey, configAddedTimeNs + 9 * bucketSizeNs + 10, false, ADB_DUMP,
- &buffer);
+ processor->onDumpReport(cfgKey, configAddedTimeNs + 9 * bucketSizeNs + 10, false, true,
+ ADB_DUMP, &buffer);
EXPECT_TRUE(buffer.size() > 0);
EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size()));
+ backfillDimensionPath(&reports);
+ backfillStringInReport(&reports);
+ backfillStartEndTimestamp(&reports);
EXPECT_EQ(1, reports.reports_size());
EXPECT_EQ(1, reports.reports(0).metrics_size());
StatsLogReport::ValueMetricDataWrapper valueMetrics;
diff --git a/cmds/statsd/tests/e2e/WakelockDuration_e2e_test.cpp b/cmds/statsd/tests/e2e/WakelockDuration_e2e_test.cpp
index 974e442..6d1317c 100644
--- a/cmds/statsd/tests/e2e/WakelockDuration_e2e_test.cpp
+++ b/cmds/statsd/tests/e2e/WakelockDuration_e2e_test.cpp
@@ -127,10 +127,13 @@
FeedEvents(config, processor);
vector<uint8_t> buffer;
ConfigMetricsReportList reports;
- processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs - 1, false, ADB_DUMP,
- &buffer);
+ processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs - 1, false, true,
+ ADB_DUMP, &buffer);
EXPECT_TRUE(buffer.size() > 0);
EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size()));
+ backfillDimensionPath(&reports);
+ backfillStringInReport(&reports);
+ backfillStartEndTimestamp(&reports);
EXPECT_EQ(reports.reports_size(), 1);
EXPECT_EQ(reports.reports(0).metrics_size(), 1);
@@ -161,11 +164,13 @@
FeedEvents(config, processor);
vector<uint8_t> buffer;
ConfigMetricsReportList reports;
-
- processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, false, ADB_DUMP,
- &buffer);
+ processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, false, true,
+ ADB_DUMP, &buffer);
EXPECT_TRUE(buffer.size() > 0);
EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size()));
+ backfillDimensionPath(&reports);
+ backfillStringInReport(&reports);
+ backfillStartEndTimestamp(&reports);
EXPECT_EQ(reports.reports_size(), 1);
EXPECT_EQ(reports.reports(0).metrics_size(), 1);
EXPECT_EQ(reports.reports(0).metrics(0).duration_metrics().data_size(), 1);
@@ -210,10 +215,13 @@
processor->OnLogEvent(event.get());
}
- processor->onDumpReport(cfgKey, bucketStartTimeNs + 6 * bucketSizeNs + 1, false, ADB_DUMP,
- &buffer);
+ processor->onDumpReport(cfgKey, bucketStartTimeNs + 6 * bucketSizeNs + 1, false, true,
+ ADB_DUMP, &buffer);
EXPECT_TRUE(buffer.size() > 0);
EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size()));
+ backfillDimensionPath(&reports);
+ backfillStringInReport(&reports);
+ backfillStartEndTimestamp(&reports);
EXPECT_EQ(reports.reports_size(), 1);
EXPECT_EQ(reports.reports(0).metrics_size(), 1);
EXPECT_EQ(reports.reports(0).metrics(0).duration_metrics().data_size(), 1);
@@ -240,11 +248,14 @@
FeedEvents(config, processor);
ConfigMetricsReportList reports;
vector<uint8_t> buffer;
- processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs - 1, false, ADB_DUMP,
- &buffer);
+ processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs - 1, false, true,
+ ADB_DUMP, &buffer);
EXPECT_TRUE(buffer.size() > 0);
EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size()));
+ backfillDimensionPath(&reports);
+ backfillStringInReport(&reports);
+ backfillStartEndTimestamp(&reports);
EXPECT_EQ(reports.reports_size(), 1);
@@ -266,10 +277,13 @@
FeedEvents(config, processor);
ConfigMetricsReportList reports;
vector<uint8_t> buffer;
- processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, false, ADB_DUMP,
- &buffer);
+ processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, false, true,
+ ADB_DUMP, &buffer);
EXPECT_TRUE(buffer.size() > 0);
EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size()));
+ backfillDimensionPath(&reports);
+ backfillStringInReport(&reports);
+ backfillStartEndTimestamp(&reports);
EXPECT_EQ(reports.reports_size(), 1);
EXPECT_EQ(reports.reports(0).metrics_size(), 1);
EXPECT_EQ(reports.reports(0).metrics(0).duration_metrics().data_size(), 1);
@@ -309,10 +323,13 @@
processor->OnLogEvent(event.get());
}
- processor->onDumpReport(cfgKey, bucketStartTimeNs + 6 * bucketSizeNs + 1, false, ADB_DUMP,
- &buffer);
+ processor->onDumpReport(cfgKey, bucketStartTimeNs + 6 * bucketSizeNs + 1, false, true,
+ ADB_DUMP, &buffer);
EXPECT_TRUE(buffer.size() > 0);
EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size()));
+ backfillDimensionPath(&reports);
+ backfillStringInReport(&reports);
+ backfillStartEndTimestamp(&reports);
EXPECT_EQ(reports.reports_size(), 1);
EXPECT_EQ(reports.reports(0).metrics_size(), 1);
EXPECT_EQ(reports.reports(0).metrics(0).duration_metrics().data_size(), 1);
diff --git a/cmds/statsd/tests/statsd_test_util.cpp b/cmds/statsd/tests/statsd_test_util.cpp
index 1264909..5903993 100644
--- a/cmds/statsd/tests/statsd_test_util.cpp
+++ b/cmds/statsd/tests/statsd_test_util.cpp
@@ -645,6 +645,183 @@
return LessThan(s1.dimInCondition, s2.dimInCondition);
}
+void backfillStringInDimension(const std::map<uint64_t, string>& str_map,
+ DimensionsValue* dimension) {
+ if (dimension->has_value_str_hash()) {
+ auto it = str_map.find((uint64_t)(dimension->value_str_hash()));
+ if (it != str_map.end()) {
+ dimension->clear_value_str_hash();
+ dimension->set_value_str(it->second);
+ } else {
+ ALOGE("Can not find the string hash: %llu",
+ (unsigned long long)dimension->value_str_hash());
+ }
+ } else if (dimension->has_value_tuple()) {
+ auto value_tuple = dimension->mutable_value_tuple();
+ for (int i = 0; i < value_tuple->dimensions_value_size(); ++i) {
+ backfillStringInDimension(str_map, value_tuple->mutable_dimensions_value(i));
+ }
+ }
+}
+
+void backfillStringInReport(ConfigMetricsReport *config_report) {
+ std::map<uint64_t, string> str_map;
+ for (const auto& str : config_report->strings()) {
+ uint64_t hash = Hash64(str);
+ if (str_map.find(hash) != str_map.end()) {
+ ALOGE("String hash conflicts: %s %s", str.c_str(), str_map[hash].c_str());
+ }
+ str_map[hash] = str;
+ }
+ for (int i = 0; i < config_report->metrics_size(); ++i) {
+ auto metric_report = config_report->mutable_metrics(i);
+ if (metric_report->has_count_metrics()) {
+ backfillStringInDimension(str_map, metric_report->mutable_count_metrics());
+ } else if (metric_report->has_duration_metrics()) {
+ backfillStringInDimension(str_map, metric_report->mutable_duration_metrics());
+ } else if (metric_report->has_gauge_metrics()) {
+ backfillStringInDimension(str_map, metric_report->mutable_gauge_metrics());
+ } else if (metric_report->has_value_metrics()) {
+ backfillStringInDimension(str_map, metric_report->mutable_value_metrics());
+ }
+ }
+ // Backfill the package names.
+ for (int i = 0 ; i < config_report->uid_map().snapshots_size(); ++i) {
+ auto snapshot = config_report->mutable_uid_map()->mutable_snapshots(i);
+ for (int j = 0 ; j < snapshot->package_info_size(); ++j) {
+ auto package_info = snapshot->mutable_package_info(j);
+ if (package_info->has_name_hash()) {
+ auto it = str_map.find((uint64_t)(package_info->name_hash()));
+ if (it != str_map.end()) {
+ package_info->clear_name_hash();
+ package_info->set_name(it->second);
+ } else {
+ ALOGE("Can not find the string package name hash: %llu",
+ (unsigned long long)package_info->name_hash());
+ }
+
+ }
+ }
+ }
+ // Backfill the app name in app changes.
+ for (int i = 0 ; i < config_report->uid_map().changes_size(); ++i) {
+ auto change = config_report->mutable_uid_map()->mutable_changes(i);
+ if (change->has_app_hash()) {
+ auto it = str_map.find((uint64_t)(change->app_hash()));
+ if (it != str_map.end()) {
+ change->clear_app_hash();
+ change->set_app(it->second);
+ } else {
+ ALOGE("Can not find the string change app name hash: %llu",
+ (unsigned long long)change->app_hash());
+ }
+ }
+ }
+}
+
+void backfillStringInReport(ConfigMetricsReportList *config_report_list) {
+ for (int i = 0; i < config_report_list->reports_size(); ++i) {
+ backfillStringInReport(config_report_list->mutable_reports(i));
+ }
+}
+
+bool backfillDimensionPath(const DimensionsValue& path,
+ const google::protobuf::RepeatedPtrField<DimensionsValue>& leafValues,
+ int* leafIndex,
+ DimensionsValue* dimension) {
+ dimension->set_field(path.field());
+ if (path.has_value_tuple()) {
+ for (int i = 0; i < path.value_tuple().dimensions_value_size(); ++i) {
+ if (!backfillDimensionPath(
+ path.value_tuple().dimensions_value(i), leafValues, leafIndex,
+ dimension->mutable_value_tuple()->add_dimensions_value())) {
+ return false;
+ }
+ }
+ } else {
+ if (*leafIndex < 0 || *leafIndex >= leafValues.size()) {
+ return false;
+ }
+ dimension->MergeFrom(leafValues.Get(*leafIndex));
+ (*leafIndex)++;
+ }
+ return true;
+}
+
+bool backfillDimensionPath(const DimensionsValue& path,
+ const google::protobuf::RepeatedPtrField<DimensionsValue>& leafValues,
+ DimensionsValue* dimension) {
+ int leafIndex = 0;
+ return backfillDimensionPath(path, leafValues, &leafIndex, dimension);
+}
+
+void backfillDimensionPath(ConfigMetricsReportList *config_report_list) {
+ for (int i = 0; i < config_report_list->reports_size(); ++i) {
+ auto report = config_report_list->mutable_reports(i);
+ for (int j = 0; j < report->metrics_size(); ++j) {
+ auto metric_report = report->mutable_metrics(j);
+ if (metric_report->has_dimensions_path_in_what() ||
+ metric_report->has_dimensions_path_in_condition()) {
+ auto whatPath = metric_report->dimensions_path_in_what();
+ auto conditionPath = metric_report->dimensions_path_in_condition();
+ if (metric_report->has_count_metrics()) {
+ backfillDimensionPath(whatPath, conditionPath,
+ metric_report->mutable_count_metrics());
+ } else if (metric_report->has_duration_metrics()) {
+ backfillDimensionPath(whatPath, conditionPath,
+ metric_report->mutable_duration_metrics());
+ } else if (metric_report->has_gauge_metrics()) {
+ backfillDimensionPath(whatPath, conditionPath,
+ metric_report->mutable_gauge_metrics());
+ } else if (metric_report->has_value_metrics()) {
+ backfillDimensionPath(whatPath, conditionPath,
+ metric_report->mutable_value_metrics());
+ }
+ metric_report->clear_dimensions_path_in_what();
+ metric_report->clear_dimensions_path_in_condition();
+ }
+ }
+ }
+}
+
+void backfillStartEndTimestamp(StatsLogReport *report) {
+ const int64_t timeBaseNs = report->time_base_elapsed_nano_seconds();
+ const int64_t bucketSizeNs = report->bucket_size_nano_seconds();
+ if (report->has_count_metrics()) {
+ backfillStartEndTimestampForMetrics(
+ timeBaseNs, bucketSizeNs, report->mutable_count_metrics());
+ } else if (report->has_duration_metrics()) {
+ backfillStartEndTimestampForMetrics(
+ timeBaseNs, bucketSizeNs, report->mutable_duration_metrics());
+ } else if (report->has_gauge_metrics()) {
+ backfillStartEndTimestampForMetrics(
+ timeBaseNs, bucketSizeNs, report->mutable_gauge_metrics());
+ if (report->gauge_metrics().skipped_size() > 0) {
+ backfillStartEndTimestampForSkippedBuckets(
+ timeBaseNs, report->mutable_gauge_metrics());
+ }
+ } else if (report->has_value_metrics()) {
+ backfillStartEndTimestampForMetrics(
+ timeBaseNs, bucketSizeNs, report->mutable_value_metrics());
+ if (report->value_metrics().skipped_size() > 0) {
+ backfillStartEndTimestampForSkippedBuckets(
+ timeBaseNs, report->mutable_value_metrics());
+ }
+ }
+}
+
+void backfillStartEndTimestamp(ConfigMetricsReport *config_report) {
+ for (int j = 0; j < config_report->metrics_size(); ++j) {
+ backfillStartEndTimestamp(config_report->mutable_metrics(j));
+ }
+}
+
+void backfillStartEndTimestamp(ConfigMetricsReportList *config_report_list) {
+ for (int i = 0; i < config_report_list->reports_size(); ++i) {
+ backfillStartEndTimestamp(config_report_list->mutable_reports(i));
+ }
+}
+
} // namespace statsd
} // namespace os
} // namespace android
\ No newline at end of file
diff --git a/cmds/statsd/tests/statsd_test_util.h b/cmds/statsd/tests/statsd_test_util.h
index 6ecca46..635c583 100644
--- a/cmds/statsd/tests/statsd_test_util.h
+++ b/cmds/statsd/tests/statsd_test_util.h
@@ -19,12 +19,16 @@
#include "frameworks/base/cmds/statsd/src/statsd_config.pb.h"
#include "src/StatsLogProcessor.h"
#include "src/logd/LogEvent.h"
+#include "src/hash.h"
+#include "src/stats_log_util.h"
#include "statslog.h"
namespace android {
namespace os {
namespace statsd {
+using google::protobuf::RepeatedPtrField;
+
// Create AtomMatcher proto to simply match a specific atom type.
AtomMatcher CreateSimpleAtomMatcher(const string& name, int atomId);
@@ -201,6 +205,53 @@
bool LessThan(const DimensionsValue& s1, const DimensionsValue& s2);
bool LessThan(const DimensionsPair& s1, const DimensionsPair& s2);
+
+void backfillStartEndTimestamp(ConfigMetricsReport *config_report);
+void backfillStartEndTimestamp(ConfigMetricsReportList *config_report_list);
+
+void backfillStringInReport(ConfigMetricsReportList *config_report_list);
+void backfillStringInDimension(const std::map<uint64_t, string>& str_map,
+ DimensionsValue* dimension);
+
+template <typename T>
+void backfillStringInDimension(const std::map<uint64_t, string>& str_map,
+ T* metrics) {
+ for (int i = 0; i < metrics->data_size(); ++i) {
+ auto data = metrics->mutable_data(i);
+ if (data->has_dimensions_in_what()) {
+ backfillStringInDimension(str_map, data->mutable_dimensions_in_what());
+ }
+ if (data->has_dimensions_in_condition()) {
+ backfillStringInDimension(str_map, data->mutable_dimensions_in_condition());
+ }
+ }
+}
+
+void backfillDimensionPath(ConfigMetricsReportList* config_report_list);
+
+bool backfillDimensionPath(const DimensionsValue& path,
+ const google::protobuf::RepeatedPtrField<DimensionsValue>& leafValues,
+ DimensionsValue* dimension);
+
+template <typename T>
+void backfillDimensionPath(const DimensionsValue& whatPath,
+ const DimensionsValue& conditionPath,
+ T* metricData) {
+ for (int i = 0; i < metricData->data_size(); ++i) {
+ auto data = metricData->mutable_data(i);
+ if (data->dimension_leaf_values_in_what_size() > 0) {
+ backfillDimensionPath(whatPath, data->dimension_leaf_values_in_what(),
+ data->mutable_dimensions_in_what());
+ data->clear_dimension_leaf_values_in_what();
+ }
+ if (data->dimension_leaf_values_in_condition_size() > 0) {
+ backfillDimensionPath(conditionPath, data->dimension_leaf_values_in_condition(),
+ data->mutable_dimensions_in_condition());
+ data->clear_dimension_leaf_values_in_condition();
+ }
+ }
+}
+
struct DimensionCompare {
bool operator()(const DimensionsPair& s1, const DimensionsPair& s2) const {
return LessThan(s1, s2);
@@ -221,6 +272,51 @@
}
}
+template <typename T>
+void backfillStartEndTimestampForFullBucket(
+ const int64_t timeBaseNs, const int64_t bucketSizeNs, T* bucket) {
+ bucket->set_start_bucket_elapsed_nanos(timeBaseNs + bucketSizeNs * bucket->bucket_num());
+ bucket->set_end_bucket_elapsed_nanos(
+ timeBaseNs + bucketSizeNs * bucket->bucket_num() + bucketSizeNs);
+ bucket->clear_bucket_num();
+}
+
+template <typename T>
+void backfillStartEndTimestampForPartialBucket(const int64_t timeBaseNs, T* bucket) {
+ if (bucket->has_start_bucket_elapsed_millis()) {
+ bucket->set_start_bucket_elapsed_nanos(
+ MillisToNano(bucket->start_bucket_elapsed_millis()));
+ bucket->clear_start_bucket_elapsed_millis();
+ }
+ if (bucket->has_end_bucket_elapsed_millis()) {
+ bucket->set_end_bucket_elapsed_nanos(
+ MillisToNano(bucket->end_bucket_elapsed_millis()));
+ bucket->clear_end_bucket_elapsed_millis();
+ }
+}
+
+template <typename T>
+void backfillStartEndTimestampForMetrics(const int64_t timeBaseNs, const int64_t bucketSizeNs,
+ T* metrics) {
+ for (int i = 0; i < metrics->data_size(); ++i) {
+ auto data = metrics->mutable_data(i);
+ for (int j = 0; j < data->bucket_info_size(); ++j) {
+ auto bucket = data->mutable_bucket_info(j);
+ if (bucket->has_bucket_num()) {
+ backfillStartEndTimestampForFullBucket(timeBaseNs, bucketSizeNs, bucket);
+ } else {
+ backfillStartEndTimestampForPartialBucket(timeBaseNs, bucket);
+ }
+ }
+ }
+}
+
+template <typename T>
+void backfillStartEndTimestampForSkippedBuckets(const int64_t timeBaseNs, T* metrics) {
+ for (int i = 0; i < metrics->skipped_size(); ++i) {
+ backfillStartEndTimestampForPartialBucket(timeBaseNs, metrics->mutable_skipped(i));
+ }
+}
} // namespace statsd
} // namespace os
} // namespace android
\ No newline at end of file