diff options
Diffstat (limited to 'src/trace_processor')
223 files changed, 7529 insertions, 3603 deletions
diff --git a/src/trace_processor/BUILD.gn b/src/trace_processor/BUILD.gn index 84c09ce23..802f9bc9b 100644 --- a/src/trace_processor/BUILD.gn +++ b/src/trace_processor/BUILD.gn @@ -109,6 +109,8 @@ source_set("storage_minimal") { "trace_processor_storage.cc", "trace_processor_storage_impl.cc", "trace_processor_storage_impl.h", + "trace_reader_registry.cc", + "trace_reader_registry.h", "virtual_destructors.cc", ] deps = [ @@ -121,6 +123,7 @@ source_set("storage_minimal") { "importers/ftrace:minimal", "importers/fuchsia:fuchsia_record", "importers/memory_tracker:graph_processor", + "importers/perf:tracker", "importers/proto:gen_cc_chrome_track_event_descriptor", "importers/proto:gen_cc_track_event_descriptor", "importers/proto:minimal", @@ -132,6 +135,7 @@ source_set("storage_minimal") { "util:descriptors", "util:gzip", "util:proto_to_args_parser", + "util:trace_type", ] public_deps = [ "../../include/perfetto/trace_processor:storage" ] } @@ -173,6 +177,7 @@ if (enable_perfetto_trace_processor_sqlite) { "importers/proto:full", "importers/proto:minimal", "importers/systrace:full", + "importers/zip:full", "metrics", "perfetto_sql/engine", "perfetto_sql/intrinsics/functions", @@ -190,6 +195,7 @@ if (enable_perfetto_trace_processor_sqlite) { "util:protozero_to_text", "util:regex", "util:stdlib", + "util:trace_type", ] public_deps = [ "../../gn:sqlite", # iterator_impl.h includes sqlite3.h. @@ -247,6 +253,7 @@ perfetto_unittest_source_set("top_level_unittests") { "../../gn:default_deps", "../../gn:gtest_and_gmock", "../../include/perfetto/trace_processor", + "util:trace_type", ] if (enable_perfetto_trace_processor_json && !is_win) { diff --git a/src/trace_processor/containers/string_pool.h b/src/trace_processor/containers/string_pool.h index 75a57a24d..2e035924b 100644 --- a/src/trace_processor/containers/string_pool.h +++ b/src/trace_processor/containers/string_pool.h @@ -17,21 +17,25 @@ #ifndef SRC_TRACE_PROCESSOR_CONTAINERS_STRING_POOL_H_ #define SRC_TRACE_PROCESSOR_CONTAINERS_STRING_POOL_H_ -#include <stddef.h> -#include <stdint.h> - +#include <cstddef> +#include <cstdint> +#include <functional> #include <limits> +#include <memory> #include <optional> +#include <string> +#include <utility> #include <vector> +#include "perfetto/base/logging.h" #include "perfetto/ext/base/flat_hash_map.h" #include "perfetto/ext/base/hash.h" #include "perfetto/ext/base/paged_memory.h" +#include "perfetto/ext/base/string_view.h" #include "perfetto/protozero/proto_utils.h" #include "src/trace_processor/containers/null_term_string_view.h" -namespace perfetto { -namespace trace_processor { +namespace perfetto::trace_processor { // Interns strings in a string pool and hands out compact StringIds which can // be used to retrieve the string in O(1). @@ -40,34 +44,38 @@ class StringPool { struct Id { Id() = default; - bool operator==(const Id& other) const { return other.id == id; } - bool operator!=(const Id& other) const { return !(other == *this); } - bool operator<(const Id& other) const { return id < other.id; } + constexpr bool operator==(const Id& other) const { return other.id == id; } + constexpr bool operator!=(const Id& other) const { + return !(other == *this); + } + constexpr bool operator<(const Id& other) const { return id < other.id; } - bool is_null() const { return id == 0u; } + constexpr bool is_null() const { return id == 0u; } - bool is_large_string() const { return id & kLargeStringFlagBitMask; } + constexpr bool is_large_string() const { + return id & kLargeStringFlagBitMask; + } - uint32_t block_offset() const { return id & kBlockOffsetBitMask; } + constexpr uint32_t block_offset() const { return id & kBlockOffsetBitMask; } - uint32_t block_index() const { + constexpr uint32_t block_index() const { return (id & kBlockIndexBitMask) >> kNumBlockOffsetBits; } - uint32_t large_string_index() const { + constexpr uint32_t large_string_index() const { PERFETTO_DCHECK(is_large_string()); return id & ~kLargeStringFlagBitMask; } - uint32_t raw_id() const { return id; } + constexpr uint32_t raw_id() const { return id; } - static Id LargeString(size_t index) { + static constexpr Id LargeString(size_t index) { PERFETTO_DCHECK(index <= static_cast<uint32_t>(index)); PERFETTO_DCHECK(!(index & kLargeStringFlagBitMask)); return Id(kLargeStringFlagBitMask | static_cast<uint32_t>(index)); } - static Id BlockString(size_t index, uint32_t offset) { + static constexpr Id BlockString(size_t index, uint32_t offset) { PERFETTO_DCHECK(index < (1u << (kNumBlockIndexBits + 1))); PERFETTO_DCHECK(offset < (1u << (kNumBlockOffsetBits + 1))); return Id(~kLargeStringFlagBitMask & @@ -80,7 +88,7 @@ class StringPool { static constexpr Id Null() { return Id(0u); } private: - constexpr Id(uint32_t i) : id(i) {} + constexpr explicit Id(uint32_t i) : id(i) {} uint32_t id; }; @@ -88,7 +96,7 @@ class StringPool { // Iterator over the strings in the pool. class Iterator { public: - Iterator(const StringPool*); + explicit Iterator(const StringPool*); explicit operator bool() const; Iterator& operator++(); @@ -278,7 +286,7 @@ class StringPool { static NullTermStringView GetFromBlockPtr(const uint8_t* ptr) { uint32_t size = 0; const uint8_t* str_ptr = ReadSize(ptr, &size); - return NullTermStringView(reinterpret_cast<const char*>(str_ptr), size); + return {reinterpret_cast<const char*>(str_ptr), size}; } // Lookup a string in the |large_strings_| vector. |id| should have the MSB @@ -288,7 +296,7 @@ class StringPool { size_t index = id.large_string_index(); PERFETTO_DCHECK(index < large_strings_.size()); const std::string* str = large_strings_[index].get(); - return NullTermStringView(str->c_str(), str->size()); + return {str->c_str(), str->size()}; } // The actual memory storing the strings. @@ -308,8 +316,7 @@ class StringPool { string_index_{/*initial_capacity=*/4096u}; }; -} // namespace trace_processor -} // namespace perfetto +} // namespace perfetto::trace_processor template <> struct std::hash<::perfetto::trace_processor::StringPool::Id> { diff --git a/src/trace_processor/db/column/BUILD.gn b/src/trace_processor/db/column/BUILD.gn index d53e7ae23..f77887f13 100644 --- a/src/trace_processor/db/column/BUILD.gn +++ b/src/trace_processor/db/column/BUILD.gn @@ -43,6 +43,7 @@ source_set("column") { "utils.h", ] deps = [ + "..:compare", "../..:metatrace", "../../../../gn:default_deps", "../../../../include/perfetto/trace_processor", diff --git a/src/trace_processor/db/column/arrangement_overlay.cc b/src/trace_processor/db/column/arrangement_overlay.cc index c31d7c26b..e7bc8ff4b 100644 --- a/src/trace_processor/db/column/arrangement_overlay.cc +++ b/src/trace_processor/db/column/arrangement_overlay.cc @@ -68,12 +68,21 @@ RangeOrBitVector ArrangementOverlay::ChainImpl::SearchValidated( if (does_arrangement_order_storage_ && op != FilterOp::kGlob && op != FilterOp::kRegex) { - Range inner_res = inner_->OrderedIndexSearchValidated( - op, sql_val, - OrderedIndices{arrangement_->data() + in.start, in.size(), - arrangement_state_}); + OrderedIndices indices{arrangement_->data() + in.start, in.size(), + arrangement_state_}; + if (op == FilterOp::kNe) { + // Do an equality search and "invert" the range. + Range inner_res = + inner_->OrderedIndexSearchValidated(FilterOp::kEq, sql_val, indices); + BitVector bv(in.start); + bv.Resize(in.start + inner_res.start, true); + bv.Resize(in.start + inner_res.end, false); + bv.Resize(in.end, true); + return RangeOrBitVector(std::move(bv)); + } + Range inner_res = inner_->OrderedIndexSearchValidated(op, sql_val, indices); return RangeOrBitVector( - Range(inner_res.start + in.start, inner_res.end + in.start)); + Range(in.start + inner_res.start, in.start + inner_res.end)); } const auto& arrangement = *arrangement_; @@ -196,6 +205,11 @@ std::optional<Token> ArrangementOverlay::ChainImpl::MinElement( return inner_->MinElement(indices); } +SqlValue ArrangementOverlay::ChainImpl::Get_AvoidUsingBecauseSlow( + uint32_t index) const { + return inner_->Get_AvoidUsingBecauseSlow((*arrangement_)[index]); +} + void ArrangementOverlay::ChainImpl::Serialize(StorageProto* storage) const { auto* arrangement_overlay = storage->set_arrangement_overlay(); arrangement_overlay->set_values( diff --git a/src/trace_processor/db/column/arrangement_overlay.h b/src/trace_processor/db/column/arrangement_overlay.h index 3656b07ad..85b9ce449 100644 --- a/src/trace_processor/db/column/arrangement_overlay.h +++ b/src/trace_processor/db/column/arrangement_overlay.h @@ -19,10 +19,10 @@ #include <cstdint> #include <memory> +#include <optional> #include <string> #include <vector> -#include "perfetto/base/logging.h" #include "perfetto/trace_processor/basic_types.h" #include "src/trace_processor/db/column/data_layer.h" #include "src/trace_processor/db/column/types.h" @@ -61,13 +61,6 @@ class ArrangementOverlay final : public DataLayer { void IndexSearchValidated(FilterOp, SqlValue, Indices&) const override; - Range OrderedIndexSearchValidated(FilterOp, - SqlValue, - const OrderedIndices&) const override { - PERFETTO_FATAL( - "OrderedIndexSearch can't be called on ArrangementOverlay"); - } - void StableSort(SortToken* start, SortToken* end, SortDirection) const override; @@ -78,6 +71,8 @@ class ArrangementOverlay final : public DataLayer { std::optional<Token> MinElement(Indices&) const override; + SqlValue Get_AvoidUsingBecauseSlow(uint32_t index) const override; + void Serialize(StorageProto*) const override; uint32_t size() const override { diff --git a/src/trace_processor/db/column/arrangement_overlay_unittest.cc b/src/trace_processor/db/column/arrangement_overlay_unittest.cc index 55f310b50..882033674 100644 --- a/src/trace_processor/db/column/arrangement_overlay_unittest.cc +++ b/src/trace_processor/db/column/arrangement_overlay_unittest.cc @@ -108,15 +108,20 @@ TEST(ArrangementOverlay, IndexSearch) { } TEST(ArrangementOverlay, OrderingSearch) { - std::vector<uint32_t> arrangement{0, 2, 4, 1, 3}; - auto fake = FakeStorageChain::SearchSubset(5, BitVector({0, 1, 0, 1, 0})); + std::vector<uint32_t> numeric_data{0, 1, 2, 0, 1, 0}; + NumericStorage<uint32_t> numeric(&numeric_data, ColumnType::kUint32, false); + + std::vector<uint32_t> arrangement{0, 3, 5, 1, 4, 2}; ArrangementOverlay storage(&arrangement, Indices::State::kNonmonotonic); - auto chain = - storage.MakeChain(std::move(fake), DataLayer::ChainCreationArgs(true)); + auto chain = storage.MakeChain(numeric.MakeChain(), + DataLayer::ChainCreationArgs(true)); RangeOrBitVector res = - chain->Search(FilterOp::kGe, SqlValue::Long(0u), Range(0, 5)); + chain->Search(FilterOp::kGe, SqlValue::Long(1u), Range(0, 5)); ASSERT_THAT(utils::ToIndexVectorForTests(res), ElementsAre(3, 4)); + + res = chain->Search(FilterOp::kNe, SqlValue::Long(1u), Range(1, 6)); + ASSERT_THAT(utils::ToIndexVectorForTests(res), ElementsAre(1, 2, 5)); } TEST(ArrangementOverlay, StableSort) { diff --git a/src/trace_processor/db/column/data_layer.cc b/src/trace_processor/db/column/data_layer.cc index 634cbce29..a570415b9 100644 --- a/src/trace_processor/db/column/data_layer.cc +++ b/src/trace_processor/db/column/data_layer.cc @@ -16,12 +16,15 @@ #include "src/trace_processor/db/column/data_layer.h" +#include <algorithm> #include <cstdint> +#include <iterator> #include <memory> #include <utility> #include <vector> #include "perfetto/base/logging.h" +#include "perfetto/trace_processor/basic_types.h" #include "src/trace_processor/containers/bit_vector.h" #include "src/trace_processor/containers/string_pool.h" #include "src/trace_processor/db/column/arrangement_overlay.h" @@ -35,6 +38,7 @@ #include "src/trace_processor/db/column/set_id_storage.h" #include "src/trace_processor/db/column/string_storage.h" #include "src/trace_processor/db/column/types.h" +#include "src/trace_processor/db/compare.h" namespace perfetto::trace_processor::column { @@ -113,6 +117,53 @@ std::unique_ptr<DataLayerChain> DataLayer::MakeChain( PERFETTO_FATAL("For GCC"); } +Range DataLayerChain::OrderedIndexSearchValidated( + FilterOp op, + SqlValue value, + const OrderedIndices& indices) const { + auto lb = [&]() { + return static_cast<uint32_t>(std::distance( + indices.data, + std::lower_bound(indices.data, indices.data + indices.size, value, + [this](uint32_t idx, const SqlValue& v) { + return compare::SqlValueComparator( + Get_AvoidUsingBecauseSlow(idx), v); + }))); + }; + auto ub = [&]() { + return static_cast<uint32_t>(std::distance( + indices.data, + std::upper_bound(indices.data, indices.data + indices.size, value, + [this](const SqlValue& v, uint32_t idx) { + return compare::SqlValueComparator( + v, Get_AvoidUsingBecauseSlow(idx)); + }))); + }; + switch (op) { + case FilterOp::kEq: + return {lb(), ub()}; + case FilterOp::kLe: + return {0, ub()}; + case FilterOp::kLt: + return {0, lb()}; + case FilterOp::kGe: + return {lb(), indices.size}; + case FilterOp::kGt: + return {ub(), indices.size}; + case FilterOp::kIsNull: + PERFETTO_CHECK(value.is_null()); + return {0, ub()}; + case FilterOp::kIsNotNull: + PERFETTO_CHECK(value.is_null()); + return {ub(), indices.size}; + case FilterOp::kNe: + case FilterOp::kGlob: + case FilterOp::kRegex: + PERFETTO_FATAL("Wrong filtering operation"); + } + PERFETTO_FATAL("For GCC"); +} + ArrangementOverlay::ArrangementOverlay( const std::vector<uint32_t>* arrangement, DataLayerChain::Indices::State arrangement_state) diff --git a/src/trace_processor/db/column/data_layer.h b/src/trace_processor/db/column/data_layer.h index 5e34539df..b6f9382d8 100644 --- a/src/trace_processor/db/column/data_layer.h +++ b/src/trace_processor/db/column/data_layer.h @@ -321,9 +321,26 @@ class DataLayerChain { // Post-validated implementation of |OrderedIndexSearch|. See // |OrderedIndexSearch|'s documentation. - virtual Range OrderedIndexSearchValidated(FilterOp, - SqlValue, - const OrderedIndices&) const = 0; + Range OrderedIndexSearchValidated(FilterOp op, + SqlValue value, + const OrderedIndices& indices) const; + + // Returns the SqlValue representing the value at a given index. + // + // This function might be very tempting to use as it appears cheap on the + // surface but because of how DataLayerChains might be layered on top of each + // other, this might require *several* virtual function calls per index. + // If you're tempted to use this, please consider instead create a new + // "vectorized" function instead and only using this as a last resort. + // + // The correct "class" of algorithms to use this function are cases where you + // have a set of indices you want to lookup and based on the value returned + // you will only use a fraction of them. In this case, it might be worth + // paying the non-vectorized lookup to vastly reduce how many indices need + // to be translated. + // + // An example of such an algorithm is binary search on indices. + virtual SqlValue Get_AvoidUsingBecauseSlow(uint32_t index) const = 0; }; } // namespace perfetto::trace_processor::column diff --git a/src/trace_processor/db/column/dense_null_overlay.cc b/src/trace_processor/db/column/dense_null_overlay.cc index 79ca3c0f8..426c6d23b 100644 --- a/src/trace_processor/db/column/dense_null_overlay.cc +++ b/src/trace_processor/db/column/dense_null_overlay.cc @@ -219,50 +219,6 @@ void DenseNullOverlay::ChainImpl::IndexSearchValidated(FilterOp op, inner_->IndexSearchValidated(op, sql_val, indices); } -Range DenseNullOverlay::ChainImpl::OrderedIndexSearchValidated( - FilterOp op, - SqlValue sql_val, - const OrderedIndices& indices) const { - // For NOT EQUAL the further analysis needs to be done by the caller. - PERFETTO_CHECK(op != FilterOp::kNe); - - PERFETTO_TP_TRACE(metatrace::Category::DB, - "DenseNullOverlay::ChainImpl::OrderedIndexSearch"); - - // We assume all NULLs are ordered to be in the front. We are looking for the - // first index that points to non NULL value. - const uint32_t* first_non_null = - std::partition_point(indices.data, indices.data + indices.size, - [this](uint32_t i) { return !non_null_->IsSet(i); }); - - auto non_null_offset = - static_cast<uint32_t>(std::distance(indices.data, first_non_null)); - auto non_null_size = static_cast<uint32_t>( - std::distance(first_non_null, indices.data + indices.size)); - - if (op == FilterOp::kIsNull) { - return {0, non_null_offset}; - } - - if (op == FilterOp::kIsNotNull) { - switch (inner_->ValidateSearchConstraints(op, sql_val)) { - case SearchValidationResult::kNoData: - return {}; - case SearchValidationResult::kAllData: - return {non_null_offset, indices.size}; - case SearchValidationResult::kOk: - break; - } - } - - Range inner_range = inner_->OrderedIndexSearchValidated( - op, sql_val, - OrderedIndices{first_non_null, non_null_size, - Indices::State::kNonmonotonic}); - return {inner_range.start + non_null_offset, - inner_range.end + non_null_offset}; -} - void DenseNullOverlay::ChainImpl::StableSort(SortToken* start, SortToken* end, SortDirection direction) const { @@ -314,6 +270,12 @@ std::optional<Token> DenseNullOverlay::ChainImpl::MinElement( : *first_null_it; } +SqlValue DenseNullOverlay::ChainImpl::Get_AvoidUsingBecauseSlow( + uint32_t index) const { + return non_null_->IsSet(index) ? inner_->Get_AvoidUsingBecauseSlow(index) + : SqlValue(); +} + void DenseNullOverlay::ChainImpl::Serialize(StorageProto* storage) const { auto* null_overlay = storage->set_dense_null_overlay(); non_null_->Serialize(null_overlay->set_bit_vector()); diff --git a/src/trace_processor/db/column/dense_null_overlay.h b/src/trace_processor/db/column/dense_null_overlay.h index e25340036..d0a95d26c 100644 --- a/src/trace_processor/db/column/dense_null_overlay.h +++ b/src/trace_processor/db/column/dense_null_overlay.h @@ -19,6 +19,7 @@ #include <cstdint> #include <memory> +#include <optional> #include <string> #include "perfetto/trace_processor/basic_types.h" @@ -56,10 +57,6 @@ class DenseNullOverlay final : public DataLayer { void IndexSearchValidated(FilterOp, SqlValue, Indices&) const override; - Range OrderedIndexSearchValidated(FilterOp, - SqlValue, - const OrderedIndices&) const override; - void StableSort(SortToken* start, SortToken* end, SortDirection) const override; @@ -70,6 +67,8 @@ class DenseNullOverlay final : public DataLayer { std::optional<Token> MinElement(Indices&) const override; + SqlValue Get_AvoidUsingBecauseSlow(uint32_t index) const override; + void Serialize(StorageProto*) const override; uint32_t size() const override { return non_null_->size(); } diff --git a/src/trace_processor/db/column/dense_null_overlay_unittest.cc b/src/trace_processor/db/column/dense_null_overlay_unittest.cc index 24b1eeeff..7cd74f687 100644 --- a/src/trace_processor/db/column/dense_null_overlay_unittest.cc +++ b/src/trace_processor/db/column/dense_null_overlay_unittest.cc @@ -129,11 +129,12 @@ TEST(DenseNullOverlay, IsNullIndexSearch) { } TEST(DenseNullOverlay, OrderedIndexSearch) { - auto fake = FakeStorageChain::SearchSubset(6, BitVector({0, 1, 0, 1, 0, 1})); + std::vector<uint32_t> numeric_data{0, 1, 0, 1, 0, 1}; + NumericStorage<uint32_t> numeric(&numeric_data, ColumnType::kUint32, false); BitVector bv{0, 1, 0, 1, 0, 1}; DenseNullOverlay storage(&bv); - auto chain = storage.MakeChain(std::move(fake)); + auto chain = storage.MakeChain(numeric.MakeChain()); std::vector<uint32_t> indices_vec({0, 2, 4, 1, 3, 5}); OrderedIndices indices{indices_vec.data(), 6, Indices::State::kNonmonotonic}; @@ -146,25 +147,25 @@ TEST(DenseNullOverlay, OrderedIndexSearch) { ASSERT_EQ(res.start, 3u); ASSERT_EQ(res.end, 6u); - res = chain->OrderedIndexSearch(FilterOp::kEq, SqlValue::Long(3), indices); + res = chain->OrderedIndexSearch(FilterOp::kEq, SqlValue::Long(1), indices); ASSERT_EQ(res.start, 3u); ASSERT_EQ(res.end, 6u); - res = chain->OrderedIndexSearch(FilterOp::kGt, SqlValue::Long(3), indices); + res = chain->OrderedIndexSearch(FilterOp::kGt, SqlValue::Long(0), indices); ASSERT_EQ(res.start, 3u); ASSERT_EQ(res.end, 6u); - res = chain->OrderedIndexSearch(FilterOp::kGe, SqlValue::Long(3), indices); + res = chain->OrderedIndexSearch(FilterOp::kGe, SqlValue::Long(1), indices); ASSERT_EQ(res.start, 3u); ASSERT_EQ(res.end, 6u); - res = chain->OrderedIndexSearch(FilterOp::kLt, SqlValue::Long(3), indices); - ASSERT_EQ(res.start, 3u); - ASSERT_EQ(res.end, 6u); + res = chain->OrderedIndexSearch(FilterOp::kLt, SqlValue::Long(1), indices); + ASSERT_EQ(res.start, 0u); + ASSERT_EQ(res.end, 3u); - res = chain->OrderedIndexSearch(FilterOp::kLe, SqlValue::Long(3), indices); - ASSERT_EQ(res.start, 3u); - ASSERT_EQ(res.end, 6u); + res = chain->OrderedIndexSearch(FilterOp::kLe, SqlValue::Long(0), indices); + ASSERT_EQ(res.start, 0u); + ASSERT_EQ(res.end, 3u); } TEST(DenseNullOverlay, SingleSearch) { diff --git a/src/trace_processor/db/column/dummy_storage.cc b/src/trace_processor/db/column/dummy_storage.cc index 631cbb75f..ef9cacf27 100644 --- a/src/trace_processor/db/column/dummy_storage.cc +++ b/src/trace_processor/db/column/dummy_storage.cc @@ -17,6 +17,7 @@ #include "src/trace_processor/db/column/dummy_storage.h" #include <cstdint> +#include <optional> #include "perfetto/base/logging.h" #include "perfetto/trace_processor/basic_types.h" @@ -49,13 +50,6 @@ void DummyStorage::ChainImpl::IndexSearchValidated(FilterOp, PERFETTO_FATAL("Shouldn't be called"); } -Range DummyStorage::ChainImpl::OrderedIndexSearchValidated( - FilterOp, - SqlValue, - const OrderedIndices&) const { - PERFETTO_FATAL("Shouldn't be called"); -} - void DummyStorage::ChainImpl::StableSort(SortToken*, SortToken*, SortDirection) const { @@ -78,6 +72,10 @@ std::optional<Token> DummyStorage::ChainImpl::MinElement(Indices&) const { PERFETTO_FATAL("Shouldn't be called"); } +SqlValue DummyStorage::ChainImpl::Get_AvoidUsingBecauseSlow(uint32_t) const { + PERFETTO_FATAL("Shouldn't be called"); +} + void DummyStorage::ChainImpl::Serialize(StorageProto*) const { PERFETTO_FATAL("Shouldn't be called"); } diff --git a/src/trace_processor/db/column/dummy_storage.h b/src/trace_processor/db/column/dummy_storage.h index ffa0c7359..66f2497a5 100644 --- a/src/trace_processor/db/column/dummy_storage.h +++ b/src/trace_processor/db/column/dummy_storage.h @@ -18,6 +18,7 @@ #include <cstdint> #include <memory> +#include <optional> #include <string> #include "perfetto/trace_processor/basic_types.h" @@ -45,10 +46,6 @@ class DummyStorage final : public DataLayer { void IndexSearchValidated(FilterOp, SqlValue, Indices&) const override; - Range OrderedIndexSearchValidated(FilterOp, - SqlValue, - const OrderedIndices&) const override; - void StableSort(SortToken* start, SortToken* end, SortDirection) const override; @@ -59,6 +56,8 @@ class DummyStorage final : public DataLayer { std::optional<Token> MinElement(Indices&) const override; + SqlValue Get_AvoidUsingBecauseSlow(uint32_t index) const override; + void Serialize(StorageProto*) const override; uint32_t size() const override; diff --git a/src/trace_processor/db/column/fake_storage.cc b/src/trace_processor/db/column/fake_storage.cc index 5a269615c..5015bf3ef 100644 --- a/src/trace_processor/db/column/fake_storage.cc +++ b/src/trace_processor/db/column/fake_storage.cc @@ -19,6 +19,7 @@ #include <algorithm> #include <cstdint> #include <iterator> +#include <optional> #include <utility> #include "perfetto/base/logging.h" @@ -112,43 +113,6 @@ void FakeStorageChain::IndexSearchValidated(FilterOp, PERFETTO_FATAL("For GCC"); } -Range FakeStorageChain::OrderedIndexSearchValidated( - FilterOp, - SqlValue, - const OrderedIndices& indices) const { - switch (strategy_) { - case kAll: - return {0, indices.size}; - case kNone: - return {}; - case kRange: { - // We are looking at intersection of |range_| and |indices_|. - const uint32_t* first_in_range = std::partition_point( - indices.data, indices.data + indices.size, - [this](uint32_t i) { return !range_.Contains(i); }); - const uint32_t* first_outside_range = std::partition_point( - first_in_range, indices.data + indices.size, - [this](uint32_t i) { return range_.Contains(i); }); - return { - static_cast<uint32_t>(std::distance(indices.data, first_in_range)), - static_cast<uint32_t>( - std::distance(indices.data, first_outside_range))}; - } - case kBitVector: - // We are looking at intersection of |range_| and |bit_vector_|. - const uint32_t* first_set = std::partition_point( - indices.data, indices.data + indices.size, - [this](uint32_t i) { return !bit_vector_.IsSet(i); }); - const uint32_t* first_non_set = std::partition_point( - first_set, indices.data + indices.size, - [this](uint32_t i) { return bit_vector_.IsSet(i); }); - return { - static_cast<uint32_t>(std::distance(indices.data, first_set)), - static_cast<uint32_t>(std::distance(indices.data, first_non_set))}; - } - PERFETTO_FATAL("For GCC"); -} - void FakeStorageChain::Distinct(Indices&) const { // Fake storage shouldn't implement Distinct as it's not a binary (this index // passes or not) operation on a column. @@ -166,6 +130,10 @@ void FakeStorageChain::StableSort(SortToken*, SortToken*, SortDirection) const { PERFETTO_FATAL("Not implemented"); } +SqlValue FakeStorageChain::Get_AvoidUsingBecauseSlow(uint32_t) const { + PERFETTO_FATAL("Not implemented"); +} + void FakeStorageChain::Serialize(StorageProto*) const { // FakeStorage doesn't really make sense to serialize. PERFETTO_FATAL("Not implemented"); diff --git a/src/trace_processor/db/column/fake_storage.h b/src/trace_processor/db/column/fake_storage.h index 7adf78641..980a95143 100644 --- a/src/trace_processor/db/column/fake_storage.h +++ b/src/trace_processor/db/column/fake_storage.h @@ -19,6 +19,7 @@ #include <cstdint> #include <memory> +#include <optional> #include <string> #include <utility> #include <vector> @@ -84,10 +85,6 @@ class FakeStorageChain : public DataLayerChain { void IndexSearchValidated(FilterOp, SqlValue, Indices&) const override; - Range OrderedIndexSearchValidated(FilterOp, - SqlValue, - const OrderedIndices&) const override; - void StableSort(SortToken* start, SortToken* end, SortDirection) const override; @@ -95,8 +92,11 @@ class FakeStorageChain : public DataLayerChain { void Distinct(Indices&) const override; std::optional<Token> MaxElement(Indices&) const override; + std::optional<Token> MinElement(Indices&) const override; + SqlValue Get_AvoidUsingBecauseSlow(uint32_t index) const override; + void Serialize(StorageProto*) const override; uint32_t size() const override { return size_; } diff --git a/src/trace_processor/db/column/fake_storage_unittest.cc b/src/trace_processor/db/column/fake_storage_unittest.cc index 908b7e021..28c497f84 100644 --- a/src/trace_processor/db/column/fake_storage_unittest.cc +++ b/src/trace_processor/db/column/fake_storage_unittest.cc @@ -43,7 +43,6 @@ using testing::ElementsAre; using testing::IsEmpty; using Indices = DataLayerChain::Indices; -using OrderedIndices = DataLayerChain::OrderedIndices; TEST(FakeStorage, ValidateSearchConstraints) { { @@ -197,48 +196,6 @@ TEST(FakeStorage, IndexSearchValidated) { } } -TEST(FakeStorage, OrderedIndexSearchValidated) { - std::vector<uint32_t> table_idx{4, 3, 2, 1}; - OrderedIndices indices{table_idx.data(), uint32_t(table_idx.size()), - Indices::State::kNonmonotonic}; - { - // All passes - auto fake = FakeStorageChain::SearchAll(5); - Range ret = - fake->OrderedIndexSearch(FilterOp::kGe, SqlValue::Long(0u), indices); - EXPECT_EQ(ret, Range(0, 4)); - } - { - // None passes - auto fake = FakeStorageChain::SearchNone(5); - Range ret = - fake->OrderedIndexSearch(FilterOp::kGe, SqlValue::Long(0u), indices); - EXPECT_EQ(ret, Range(0, 0)); - } - { - // BitVector - auto fake = FakeStorageChain::SearchSubset(5, BitVector{0, 0, 1, 1, 1}); - Range ret = - fake->OrderedIndexSearch(FilterOp::kGe, SqlValue::Long(0u), indices); - EXPECT_EQ(ret, Range(0, 3)); - } - { - // Index vector - auto fake = - FakeStorageChain::SearchSubset(5, std::vector<uint32_t>{1, 2, 3}); - Range ret = - fake->OrderedIndexSearch(FilterOp::kGe, SqlValue::Long(0u), indices); - EXPECT_EQ(ret, Range(1, 4)); - } - { - // Range - auto fake = FakeStorageChain::SearchSubset(5, Range(1, 4)); - Range ret = - fake->OrderedIndexSearch(FilterOp::kGe, SqlValue::Long(0u), indices); - EXPECT_EQ(ret, Range(1, 4)); - } -} - } // namespace } // namespace column } // namespace perfetto::trace_processor diff --git a/src/trace_processor/db/column/id_storage.cc b/src/trace_processor/db/column/id_storage.cc index ea1e90991..1ebe459fe 100644 --- a/src/trace_processor/db/column/id_storage.cc +++ b/src/trace_processor/db/column/id_storage.cc @@ -246,47 +246,6 @@ void IdStorage::ChainImpl::IndexSearchValidated(FilterOp op, PERFETTO_FATAL("FilterOp not matched"); } -Range IdStorage::ChainImpl::OrderedIndexSearchValidated( - FilterOp op, - SqlValue sql_val, - const OrderedIndices& indices) const { - PERFETTO_DCHECK(op != FilterOp::kNe); - - PERFETTO_TP_TRACE( - metatrace::Category::DB, "IdStorage::ChainImpl::OrderedIndexSearch", - [indices, op](metatrace::Record* r) { - r->AddArg("Count", std::to_string(indices.size)); - r->AddArg("Op", std::to_string(static_cast<uint32_t>(op))); - }); - - // It's a valid filter operation if |sql_val| is a double, although it - // requires special logic. - if (sql_val.type == SqlValue::kDouble) { - switch (utils::CompareIntColumnWithDouble(op, &sql_val)) { - case SearchValidationResult::kOk: - break; - case SearchValidationResult::kAllData: - return {0, indices.size}; - case SearchValidationResult::kNoData: - return {}; - } - } - auto val = static_cast<uint32_t>(sql_val.AsLong()); - - // OrderedIndices are monotonic non contiguous values if OrderedIndexSearch - // was called. Look for the first and last index and find the result of - // looking for this range in IdStorage. - Range indices_range(indices.data[0], indices.data[indices.size - 1] + 1); - Range bin_search_ret = BinarySearchIntrinsic(op, val, indices_range); - - const auto* start_ptr = std::lower_bound( - indices.data, indices.data + indices.size, bin_search_ret.start); - const auto* end_ptr = std::lower_bound(start_ptr, indices.data + indices.size, - bin_search_ret.end); - return {static_cast<uint32_t>(std::distance(indices.data, start_ptr)), - static_cast<uint32_t>(std::distance(indices.data, end_ptr))}; -} - Range IdStorage::ChainImpl::BinarySearchIntrinsic(FilterOp op, Id val, Range range) { @@ -363,6 +322,10 @@ std::optional<Token> IdStorage::ChainImpl::MinElement(Indices& indices) const { return *tok; } +SqlValue IdStorage::ChainImpl::Get_AvoidUsingBecauseSlow(uint32_t index) const { + return SqlValue::Long(index); +} + void IdStorage::ChainImpl::Serialize(StorageProto* storage) const { storage->set_id_storage(); } diff --git a/src/trace_processor/db/column/id_storage.h b/src/trace_processor/db/column/id_storage.h index 420568d64..c3aeadff1 100644 --- a/src/trace_processor/db/column/id_storage.h +++ b/src/trace_processor/db/column/id_storage.h @@ -19,6 +19,7 @@ #include <cstdint> #include <limits> #include <memory> +#include <optional> #include <string> #include "perfetto/trace_processor/basic_types.h" @@ -55,10 +56,6 @@ class IdStorage final : public DataLayer { void IndexSearchValidated(FilterOp, SqlValue, Indices&) const override; - Range OrderedIndexSearchValidated(FilterOp, - SqlValue, - const OrderedIndices&) const override; - void StableSort(SortToken* start, SortToken* end, SortDirection) const override; @@ -69,6 +66,8 @@ class IdStorage final : public DataLayer { std::optional<Token> MinElement(Indices&) const override; + SqlValue Get_AvoidUsingBecauseSlow(uint32_t index) const override; + void Serialize(StorageProto*) const override; uint32_t size() const override { diff --git a/src/trace_processor/db/column/null_overlay.cc b/src/trace_processor/db/column/null_overlay.cc index 128d3ff00..372d3b02a 100644 --- a/src/trace_processor/db/column/null_overlay.cc +++ b/src/trace_processor/db/column/null_overlay.cc @@ -256,56 +256,6 @@ void NullOverlay::ChainImpl::IndexSearchValidated(FilterOp op, inner_->IndexSearchValidated(op, sql_val, indices); } -Range NullOverlay::ChainImpl::OrderedIndexSearchValidated( - FilterOp op, - SqlValue sql_val, - const OrderedIndices& indices) const { - // For NOT EQUAL the translation or results from EQUAL needs to be done by the - // caller. - PERFETTO_CHECK(op != FilterOp::kNe); - - PERFETTO_TP_TRACE(metatrace::Category::DB, - "NullOverlay::ChainImpl::OrderedIndexSearch"); - - // We assume all NULLs are ordered to be in the front. We are looking for the - // first index that points to non NULL value. - const uint32_t* first_non_null = - std::partition_point(indices.data, indices.data + indices.size, - [this](uint32_t i) { return !non_null_->IsSet(i); }); - auto non_null_offset = - static_cast<uint32_t>(std::distance(indices.data, first_non_null)); - auto non_null_size = static_cast<uint32_t>( - std::distance(first_non_null, indices.data + indices.size)); - - if (op == FilterOp::kIsNull) { - return {0, non_null_offset}; - } - - if (op == FilterOp::kIsNotNull) { - switch (inner_->ValidateSearchConstraints(op, sql_val)) { - case SearchValidationResult::kNoData: - return {}; - case SearchValidationResult::kAllData: - return {non_null_offset, indices.size}; - case SearchValidationResult::kOk: - break; - } - } - - std::vector<uint32_t> storage_iv; - storage_iv.reserve(non_null_size); - for (const uint32_t* it = first_non_null; - it != first_non_null + non_null_size; it++) { - storage_iv.push_back(non_null_->CountSetBits(*it)); - } - - Range inner_range = inner_->OrderedIndexSearchValidated( - op, sql_val, - OrderedIndices{storage_iv.data(), non_null_size, indices.state}); - return {inner_range.start + non_null_offset, - inner_range.end + non_null_offset}; -} - void NullOverlay::ChainImpl::StableSort(SortToken* start, SortToken* end, SortDirection direction) const { @@ -369,6 +319,13 @@ std::optional<Token> NullOverlay::ChainImpl::MinElement( return inner_->MinElement(indices); } +SqlValue NullOverlay::ChainImpl::Get_AvoidUsingBecauseSlow( + uint32_t index) const { + return non_null_->IsSet(index) + ? inner_->Get_AvoidUsingBecauseSlow(non_null_->CountSetBits(index)) + : SqlValue(); +} + void NullOverlay::ChainImpl::Serialize(StorageProto* storage) const { auto* null_storage = storage->set_null_overlay(); non_null_->Serialize(null_storage->set_bit_vector()); diff --git a/src/trace_processor/db/column/null_overlay.h b/src/trace_processor/db/column/null_overlay.h index 187c056c8..a74f211aa 100644 --- a/src/trace_processor/db/column/null_overlay.h +++ b/src/trace_processor/db/column/null_overlay.h @@ -19,6 +19,7 @@ #include <cstdint> #include <memory> +#include <optional> #include <string> #include "perfetto/trace_processor/basic_types.h" @@ -55,10 +56,6 @@ class NullOverlay final : public DataLayer { void IndexSearchValidated(FilterOp, SqlValue, Indices&) const override; - Range OrderedIndexSearchValidated(FilterOp, - SqlValue, - const OrderedIndices&) const override; - void StableSort(SortToken* start, SortToken* end, SortDirection) const override; @@ -69,6 +66,8 @@ class NullOverlay final : public DataLayer { std::optional<Token> MinElement(Indices&) const override; + SqlValue Get_AvoidUsingBecauseSlow(uint32_t index) const override; + void Serialize(StorageProto*) const override; uint32_t size() const override { return non_null_->size(); } diff --git a/src/trace_processor/db/column/null_overlay_unittest.cc b/src/trace_processor/db/column/null_overlay_unittest.cc index 981c7bdc4..9cff0958d 100644 --- a/src/trace_processor/db/column/null_overlay_unittest.cc +++ b/src/trace_processor/db/column/null_overlay_unittest.cc @@ -200,15 +200,15 @@ TEST(NullOverlay, IndexSearchIsNotNullOp) { } TEST(NullOverlay, OrderedIndexSearch) { + std::vector<uint32_t> numeric_data{1, 0, 1, 0}; + NumericStorage<uint32_t> numeric(&numeric_data, ColumnType::kUint32, false); + BitVector bv{0, 1, 1, 1, 0, 1}; - // Passing values in final storage (on normal operations) - // 0, 1, 0, 1, 0, 0 - auto fake = FakeStorageChain::SearchSubset(4, BitVector{1, 0, 1, 0}); NullOverlay storage(&bv); - auto chain = storage.MakeChain(std::move(fake)); + auto chain = storage.MakeChain(numeric.MakeChain()); // Passing values on final data - // NULL, NULL, 0, 1, 1 + // NULL, NULL, 0, 0, 1, 1 std::vector<uint32_t> table_idx{0, 4, 5, 1, 3}; OrderedIndices indices{table_idx.data(), uint32_t(table_idx.size()), Indices::State::kNonmonotonic}; @@ -218,28 +218,28 @@ TEST(NullOverlay, OrderedIndexSearch) { ASSERT_EQ(res.end, 2u); res = chain->OrderedIndexSearch(FilterOp::kIsNotNull, SqlValue(), indices); - ASSERT_EQ(res.start, 3u); + ASSERT_EQ(res.start, 2u); ASSERT_EQ(res.end, 5u); - res = chain->OrderedIndexSearch(FilterOp::kEq, SqlValue::Long(3), indices); + res = chain->OrderedIndexSearch(FilterOp::kEq, SqlValue::Long(1), indices); ASSERT_EQ(res.start, 3u); ASSERT_EQ(res.end, 5u); - res = chain->OrderedIndexSearch(FilterOp::kGt, SqlValue::Long(3), indices); + res = chain->OrderedIndexSearch(FilterOp::kGt, SqlValue::Long(0), indices); ASSERT_EQ(res.start, 3u); ASSERT_EQ(res.end, 5u); - res = chain->OrderedIndexSearch(FilterOp::kGe, SqlValue::Long(3), indices); + res = chain->OrderedIndexSearch(FilterOp::kGe, SqlValue::Long(1), indices); ASSERT_EQ(res.start, 3u); ASSERT_EQ(res.end, 5u); - res = chain->OrderedIndexSearch(FilterOp::kLt, SqlValue::Long(3), indices); - ASSERT_EQ(res.start, 3u); - ASSERT_EQ(res.end, 5u); + res = chain->OrderedIndexSearch(FilterOp::kLt, SqlValue::Long(1), indices); + ASSERT_EQ(res.start, 0u); + ASSERT_EQ(res.end, 3u); - res = chain->OrderedIndexSearch(FilterOp::kLe, SqlValue::Long(3), indices); - ASSERT_EQ(res.start, 3u); - ASSERT_EQ(res.end, 5u); + res = chain->OrderedIndexSearch(FilterOp::kLe, SqlValue::Long(0), indices); + ASSERT_EQ(res.start, 0u); + ASSERT_EQ(res.end, 3u); } TEST(NullOverlay, StableSort) { diff --git a/src/trace_processor/db/column/numeric_storage.cc b/src/trace_processor/db/column/numeric_storage.cc index b39b4cb91..3612356ad 100644 --- a/src/trace_processor/db/column/numeric_storage.cc +++ b/src/trace_processor/db/column/numeric_storage.cc @@ -138,60 +138,6 @@ uint32_t UpperBoundIntrinsic(const void* vector_ptr, } template <typename T> -uint32_t TypedLowerBoundExtrinsic(T val, - const T* data, - OrderedIndices indices) { - const auto* lower = std::lower_bound( - indices.data, indices.data + indices.size, val, - [data](uint32_t index, T value) { return data[index] < value; }); - return static_cast<uint32_t>(std::distance(indices.data, lower)); -} - -uint32_t LowerBoundExtrinsic(const void* vector_ptr, - NumericValue val, - OrderedIndices indices) { - if (const auto* u32 = std::get_if<uint32_t>(&val)) { - const auto* start = - static_cast<const std::vector<uint32_t>*>(vector_ptr)->data(); - return TypedLowerBoundExtrinsic(*u32, start, indices); - } - if (const auto* i64 = std::get_if<int64_t>(&val)) { - const auto* start = - static_cast<const std::vector<int64_t>*>(vector_ptr)->data(); - return TypedLowerBoundExtrinsic(*i64, start, indices); - } - if (const auto* i32 = std::get_if<int32_t>(&val)) { - const auto* start = - static_cast<const std::vector<int32_t>*>(vector_ptr)->data(); - return TypedLowerBoundExtrinsic(*i32, start, indices); - } - if (const auto* db = std::get_if<double>(&val)) { - const auto* start = - static_cast<const std::vector<double>*>(vector_ptr)->data(); - return TypedLowerBoundExtrinsic(*db, start, indices); - } - PERFETTO_FATAL("Type not handled"); -} - -uint32_t UpperBoundExtrinsic(const void* vector_ptr, - NumericValue val, - OrderedIndices indices) { - return std::visit( - [vector_ptr, indices](auto val_data) { - using T = decltype(val_data); - const T* typed_start = - static_cast<const std::vector<T>*>(vector_ptr)->data(); - const auto* upper = - std::upper_bound(indices.data, indices.data + indices.size, - val_data, [typed_start](T value, uint32_t index) { - return value < typed_start[index]; - }); - return static_cast<uint32_t>(std::distance(indices.data, upper)); - }, - val); -} - -template <typename T> void TypedLinearSearch(T typed_val, const T* start, FilterOp op, @@ -504,79 +450,6 @@ void NumericStorageBase::ChainImpl::IndexSearchValidated( val); } -Range NumericStorageBase::ChainImpl::OrderedIndexSearchValidated( - FilterOp op, - SqlValue sql_val, - const OrderedIndices& indices) const { - PERFETTO_TP_TRACE( - metatrace::Category::DB, "NumericStorage::ChainImpl::OrderedIndexSearch", - [indices, op](metatrace::Record* r) { - r->AddArg("Count", std::to_string(indices.size)); - r->AddArg("Op", std::to_string(static_cast<uint32_t>(op))); - }); - - // Mismatched types - value is double and column is int. - if (sql_val.type == SqlValue::kDouble && - storage_type_ != ColumnType::kDouble) { - if (auto ret = utils::CanReturnEarly(IntColumnWithDouble(op, &sql_val), - indices.size); - ret) { - return *ret; - } - } - - // Mismatched types - column is double and value is int. - if (sql_val.type != SqlValue::kDouble && - storage_type_ == ColumnType::kDouble) { - if (auto ret = utils::CanReturnEarly(DoubleColumnWithInt(op, &sql_val), - indices.size); - ret) { - return *ret; - } - } - - NumericValue val; - switch (storage_type_) { - case ColumnType::kDouble: - val = sql_val.AsDouble(); - break; - case ColumnType::kInt64: - val = sql_val.AsLong(); - break; - case ColumnType::kInt32: - val = static_cast<int32_t>(sql_val.AsLong()); - break; - case ColumnType::kUint32: - val = static_cast<uint32_t>(sql_val.AsLong()); - break; - case ColumnType::kString: - case ColumnType::kDummy: - case ColumnType::kId: - PERFETTO_FATAL("Invalid type"); - } - - switch (op) { - case FilterOp::kEq: - return {LowerBoundExtrinsic(vector_ptr_, val, indices), - UpperBoundExtrinsic(vector_ptr_, val, indices)}; - case FilterOp::kLe: - return {0, UpperBoundExtrinsic(vector_ptr_, val, indices)}; - case FilterOp::kLt: - return {0, LowerBoundExtrinsic(vector_ptr_, val, indices)}; - case FilterOp::kGe: - return {LowerBoundExtrinsic(vector_ptr_, val, indices), indices.size}; - case FilterOp::kGt: - return {UpperBoundExtrinsic(vector_ptr_, val, indices), indices.size}; - case FilterOp::kNe: - case FilterOp::kIsNull: - case FilterOp::kIsNotNull: - case FilterOp::kGlob: - case FilterOp::kRegex: - PERFETTO_FATAL("Wrong filtering operation"); - } - PERFETTO_FATAL("For GCC"); -} - BitVector NumericStorageBase::ChainImpl::LinearSearchInternal( FilterOp op, NumericValue val, diff --git a/src/trace_processor/db/column/numeric_storage.h b/src/trace_processor/db/column/numeric_storage.h index fd27d2693..3c4416b28 100644 --- a/src/trace_processor/db/column/numeric_storage.h +++ b/src/trace_processor/db/column/numeric_storage.h @@ -23,6 +23,7 @@ #include <optional> #include <set> #include <string> +#include <type_traits> #include <unordered_set> #include <variant> #include <vector> @@ -49,10 +50,6 @@ class NumericStorageBase : public DataLayer { void IndexSearchValidated(FilterOp, SqlValue, Indices&) const override; - Range OrderedIndexSearchValidated(FilterOp, - SqlValue, - const OrderedIndices&) const override; - void Serialize(StorageProto*) const override; std::string DebugString() const override { return "NumericStorage"; } @@ -145,6 +142,13 @@ class NumericStorage final : public NumericStorageBase { return *tok; } + SqlValue Get_AvoidUsingBecauseSlow(uint32_t index) const override { + if constexpr (std::is_same_v<T, double>) { + return SqlValue::Double((*vector_)[index]); + } + return SqlValue::Long((*vector_)[index]); + } + void StableSort(SortToken* start, SortToken* end, SortDirection direction) const override { diff --git a/src/trace_processor/db/column/range_overlay.cc b/src/trace_processor/db/column/range_overlay.cc index 81bb7ae5e..dee783888 100644 --- a/src/trace_processor/db/column/range_overlay.cc +++ b/src/trace_processor/db/column/range_overlay.cc @@ -18,6 +18,7 @@ #include <cstdint> #include <memory> +#include <optional> #include <utility> #include <vector> @@ -118,22 +119,6 @@ void RangeOverlay::ChainImpl::IndexSearchValidated(FilterOp op, inner_->IndexSearchValidated(op, sql_val, indices); } -Range RangeOverlay::ChainImpl::OrderedIndexSearchValidated( - FilterOp op, - SqlValue sql_val, - const OrderedIndices& indices) const { - PERFETTO_TP_TRACE(metatrace::Category::DB, "RangeOverlay::IndexSearch"); - - // Should be SIMD optimized. - std::vector<uint32_t> storage_iv(indices.size); - for (uint32_t i = 0; i < indices.size; ++i) { - storage_iv[i] = indices.data[i] + range_->start; - } - return inner_->OrderedIndexSearchValidated( - op, sql_val, - OrderedIndices{storage_iv.data(), indices.size, indices.state}); -} - void RangeOverlay::ChainImpl::StableSort(SortToken* start, SortToken* end, SortDirection direction) const { @@ -160,6 +145,11 @@ std::optional<Token> RangeOverlay::ChainImpl::MaxElement( return inner_->MaxElement(indices); } +SqlValue RangeOverlay::ChainImpl::Get_AvoidUsingBecauseSlow( + uint32_t index) const { + return inner_->Get_AvoidUsingBecauseSlow(index + range_->start); +} + std::optional<Token> RangeOverlay::ChainImpl::MinElement( Indices& indices) const { PERFETTO_TP_TRACE(metatrace::Category::DB, "RangeOverlay::MinElement"); diff --git a/src/trace_processor/db/column/range_overlay.h b/src/trace_processor/db/column/range_overlay.h index f15e8dce4..a58df40a3 100644 --- a/src/trace_processor/db/column/range_overlay.h +++ b/src/trace_processor/db/column/range_overlay.h @@ -19,6 +19,7 @@ #include <cstdint> #include <memory> +#include <optional> #include <string> #include "perfetto/trace_processor/basic_types.h" @@ -52,10 +53,6 @@ class RangeOverlay final : public DataLayer { void IndexSearchValidated(FilterOp p, SqlValue, Indices&) const override; - Range OrderedIndexSearchValidated(FilterOp, - SqlValue, - const OrderedIndices&) const override; - void StableSort(SortToken* start, SortToken* end, SortDirection) const override; @@ -66,6 +63,8 @@ class RangeOverlay final : public DataLayer { std::optional<Token> MinElement(Indices&) const override; + SqlValue Get_AvoidUsingBecauseSlow(uint32_t index) const override; + void Serialize(StorageProto*) const override; uint32_t size() const override { return range_->size(); } diff --git a/src/trace_processor/db/column/selector_overlay.cc b/src/trace_processor/db/column/selector_overlay.cc index b2cbc4359..9e957cba2 100644 --- a/src/trace_processor/db/column/selector_overlay.cc +++ b/src/trace_processor/db/column/selector_overlay.cc @@ -16,9 +16,9 @@ #include "src/trace_processor/db/column/selector_overlay.h" -#include <algorithm> #include <cstdint> #include <memory> +#include <optional> #include <utility> #include <vector> @@ -35,7 +35,7 @@ namespace perfetto::trace_processor::column { namespace { -static constexpr uint32_t kIndexOfNthSetRatio = 32; +constexpr uint32_t kIndexOfNthSetRatio = 32; } // namespace @@ -97,23 +97,6 @@ void SelectorOverlay::ChainImpl::IndexSearchValidated(FilterOp op, return inner_->IndexSearchValidated(op, sql_val, indices); } -Range SelectorOverlay::ChainImpl::OrderedIndexSearchValidated( - FilterOp op, - SqlValue sql_val, - const OrderedIndices& indices) const { - // To go from TableIndexVector to StorageIndexVector we need to find index in - // |selector_| by looking only into set bits. - std::vector<uint32_t> inner_indices(indices.size); - for (uint32_t i = 0; i < indices.size; ++i) { - inner_indices[i] = selector_->IndexOfNthSet(indices.data[i]); - } - return inner_->OrderedIndexSearchValidated( - op, sql_val, - OrderedIndices{inner_indices.data(), - static_cast<uint32_t>(inner_indices.size()), - indices.state}); -} - void SelectorOverlay::ChainImpl::StableSort(SortToken* start, SortToken* end, SortDirection direction) const { @@ -148,6 +131,11 @@ std::optional<Token> SelectorOverlay::ChainImpl::MinElement( return inner_->MinElement(indices); } +SqlValue SelectorOverlay::ChainImpl::Get_AvoidUsingBecauseSlow( + uint32_t index) const { + return inner_->Get_AvoidUsingBecauseSlow(selector_->IndexOfNthSet(index)); +} + void SelectorOverlay::ChainImpl::Serialize(StorageProto* storage) const { auto* selector_overlay = storage->set_selector_overlay(); inner_->Serialize(selector_overlay->set_storage()); @@ -164,15 +152,15 @@ void SelectorOverlay::ChainImpl::TranslateToInnerIndices( for (auto& token : indices.tokens) { token.index = selector_->IndexOfNthSet(token.index); } - } else { - // TODO(mayzner): once we have a reverse index for IndexOfNthSet in - // BitVector, this should no longer be necessary. - std::vector<uint32_t> lookup = selector_->GetSetBitIndices(); - for (auto& token : indices.tokens) { - token.index = lookup[token.index]; - } + return; + } + + // TODO(mayzner): once we have a reverse index for IndexOfNthSet in + // BitVector, this should no longer be necessary. + std::vector<uint32_t> lookup = selector_->GetSetBitIndices(); + for (auto& token : indices.tokens) { + token.index = lookup[token.index]; } - return; } } // namespace perfetto::trace_processor::column diff --git a/src/trace_processor/db/column/selector_overlay.h b/src/trace_processor/db/column/selector_overlay.h index d54bdf59b..3726f9f13 100644 --- a/src/trace_processor/db/column/selector_overlay.h +++ b/src/trace_processor/db/column/selector_overlay.h @@ -19,6 +19,7 @@ #include <cstdint> #include <memory> +#include <optional> #include <string> #include "perfetto/trace_processor/basic_types.h" @@ -56,10 +57,6 @@ class SelectorOverlay final : public DataLayer { void IndexSearchValidated(FilterOp p, SqlValue, Indices&) const override; - Range OrderedIndexSearchValidated(FilterOp, - SqlValue, - const OrderedIndices&) const override; - void StableSort(SortToken* start, SortToken* end, SortDirection) const override; @@ -70,6 +67,8 @@ class SelectorOverlay final : public DataLayer { std::optional<Token> MinElement(Indices&) const override; + SqlValue Get_AvoidUsingBecauseSlow(uint32_t index) const override; + void Serialize(StorageProto*) const override; uint32_t size() const override { return selector_->size(); } diff --git a/src/trace_processor/db/column/selector_overlay_unittest.cc b/src/trace_processor/db/column/selector_overlay_unittest.cc index 0abae4c62..def2a77b7 100644 --- a/src/trace_processor/db/column/selector_overlay_unittest.cc +++ b/src/trace_processor/db/column/selector_overlay_unittest.cc @@ -17,6 +17,7 @@ #include "src/trace_processor/db/column/selector_overlay.h" #include <cstdint> +#include <utility> #include <vector> #include "data_layer.h" @@ -103,33 +104,21 @@ TEST(SelectorOverlay, IndexSearch) { ASSERT_THAT(utils::ExtractPayloadForTesting(indices), ElementsAre(1u)); } -TEST(SelectorOverlay, OrderedIndexSearchTrivial) { - BitVector selector{1, 0, 1, 0, 1}; - auto fake = FakeStorageChain::SearchAll(5); - SelectorOverlay storage(&selector); - auto chain = storage.MakeChain(std::move(fake)); - - std::vector<uint32_t> table_idx{1u, 0u, 2u}; - Range res = chain->OrderedIndexSearch( - FilterOp::kGe, SqlValue::Long(0u), - OrderedIndices{table_idx.data(), static_cast<uint32_t>(table_idx.size()), - Indices::State::kNonmonotonic}); - ASSERT_EQ(res.start, 0u); - ASSERT_EQ(res.end, 3u); -} +TEST(SelectorOverlay, OrderedIndexSearch) { + std::vector<uint32_t> numeric_data{1, 0, 0, 1, 1}; + NumericStorage<uint32_t> numeric(&numeric_data, ColumnType::kUint32, false); -TEST(SelectorOverlay, OrderedIndexSearchNone) { BitVector selector{1, 0, 1, 0, 1}; - auto fake = FakeStorageChain::SearchNone(5); SelectorOverlay storage(&selector); - auto chain = storage.MakeChain(std::move(fake)); + auto chain = storage.MakeChain(numeric.MakeChain()); std::vector<uint32_t> table_idx{1u, 0u, 2u}; Range res = chain->OrderedIndexSearch( - FilterOp::kGe, SqlValue::Long(0u), + FilterOp::kGe, SqlValue::Long(1u), OrderedIndices{table_idx.data(), static_cast<uint32_t>(table_idx.size()), Indices::State::kNonmonotonic}); - ASSERT_EQ(res.size(), 0u); + ASSERT_EQ(res.start, 1u); + ASSERT_EQ(res.end, 3u); } TEST(SelectorOverlay, StableSort) { diff --git a/src/trace_processor/db/column/set_id_storage.cc b/src/trace_processor/db/column/set_id_storage.cc index 207b649b7..42ab4ea78 100644 --- a/src/trace_processor/db/column/set_id_storage.cc +++ b/src/trace_processor/db/column/set_id_storage.cc @@ -19,10 +19,7 @@ #include <algorithm> #include <cstdint> #include <functional> -#include <iterator> -#include <limits> -#include <memory> -#include <set> +#include <optional> #include <string> #include <unordered_set> #include <utility> @@ -250,38 +247,17 @@ void SetIdStorage::ChainImpl::IndexSearchValidated(FilterOp op, } } -Range SetIdStorage::ChainImpl::OrderedIndexSearchValidated( - FilterOp op, - SqlValue sql_val, - const OrderedIndices& indices) const { - PERFETTO_TP_TRACE(metatrace::Category::DB, - "SetIdStorage::ChainImpl::OrderedIndexSearch"); - // OrderedIndices are monotonic non-contiguous values. - auto res = SearchValidated( - op, sql_val, Range(indices.data[0], indices.data[indices.size - 1] + 1)); - PERFETTO_CHECK(res.IsRange()); - Range res_range = std::move(res).TakeIfRange(); - - const auto* start_ptr = std::lower_bound( - indices.data, indices.data + indices.size, res_range.start); - const auto* end_ptr = - std::lower_bound(start_ptr, indices.data + indices.size, res_range.end); - - return {static_cast<uint32_t>(std::distance(indices.data, start_ptr)), - static_cast<uint32_t>(std::distance(indices.data, end_ptr))}; -} - Range SetIdStorage::ChainImpl::BinarySearchIntrinsic(FilterOp op, SetId val, Range range) const { switch (op) { case FilterOp::kEq: { - if (values_->data()[val] != val) { - return Range(); + if ((*values_)[val] != val) { + return {}; } uint32_t start = std::max(val, range.start); uint32_t end = UpperBoundIntrinsic(values_->data(), val, range); - return Range(std::min(start, end), end); + return {std::min(start, end), end}; } case FilterOp::kLe: { return {range.start, UpperBoundIntrinsic(values_->data(), val, range)}; @@ -370,6 +346,11 @@ std::optional<Token> SetIdStorage::ChainImpl::MinElement( return *tok; } +SqlValue SetIdStorage::ChainImpl::Get_AvoidUsingBecauseSlow( + uint32_t index) const { + return SqlValue::Long((*values_)[index]); +} + void SetIdStorage::ChainImpl::Serialize(StorageProto* msg) const { auto* vec_msg = msg->set_set_id_storage(); vec_msg->set_values(reinterpret_cast<const uint8_t*>(values_->data()), diff --git a/src/trace_processor/db/column/set_id_storage.h b/src/trace_processor/db/column/set_id_storage.h index 2fa6c81bb..e4591067e 100644 --- a/src/trace_processor/db/column/set_id_storage.h +++ b/src/trace_processor/db/column/set_id_storage.h @@ -18,6 +18,7 @@ #include <cstdint> #include <memory> +#include <optional> #include <string> #include <vector> @@ -54,10 +55,6 @@ class SetIdStorage final : public DataLayer { void IndexSearchValidated(FilterOp, SqlValue, Indices&) const override; - Range OrderedIndexSearchValidated(FilterOp, - SqlValue, - const OrderedIndices&) const override; - void StableSort(SortToken* start, SortToken* end, SortDirection direction) const override; @@ -70,6 +67,8 @@ class SetIdStorage final : public DataLayer { std::optional<Token> MinElement(Indices&) const override; + SqlValue Get_AvoidUsingBecauseSlow(uint32_t index) const override; + uint32_t size() const override { return static_cast<uint32_t>(values_->size()); } diff --git a/src/trace_processor/db/column/string_storage.cc b/src/trace_processor/db/column/string_storage.cc index adf3eda06..1d25dbbcc 100644 --- a/src/trace_processor/db/column/string_storage.cc +++ b/src/trace_processor/db/column/string_storage.cc @@ -163,36 +163,6 @@ uint32_t UpperBoundIntrinsic(StringPool* pool, return static_cast<uint32_t>(std::distance(data, upper)); } -uint32_t LowerBoundExtrinsic(StringPool* pool, - const StringPool::Id* data, - NullTermStringView val, - const uint32_t* indices, - uint32_t indices_count, - uint32_t offset) { - Less comp{pool}; - const auto* lower = - std::lower_bound(indices + offset, indices + indices_count, val, - [comp, data](uint32_t index, NullTermStringView val) { - return comp(data[index], val); - }); - return static_cast<uint32_t>(std::distance(indices, lower)); -} - -uint32_t UpperBoundExtrinsic(StringPool* pool, - const StringPool::Id* data, - NullTermStringView val, - const uint32_t* indices, - uint32_t indices_count, - uint32_t offset) { - Greater comp{pool}; - const auto* upper = - std::upper_bound(indices + offset, indices + indices_count, val, - [comp, data](NullTermStringView val, uint32_t index) { - return comp(data[index], val); - }); - return static_cast<uint32_t>(std::distance(indices, upper)); -} - } // namespace StringStorage::ChainImpl::ChainImpl(StringPool* string_pool, @@ -518,64 +488,6 @@ BitVector StringStorage::ChainImpl::LinearSearch(FilterOp op, return std::move(builder).Build(); } -Range StringStorage::ChainImpl::OrderedIndexSearchValidated( - FilterOp op, - SqlValue sql_val, - const OrderedIndices& indices) const { - PERFETTO_TP_TRACE(metatrace::Category::DB, - "StringStorage::ChainImpl::OrderedIndexSearch"); - StringPool::Id val = - (op == FilterOp::kIsNull || op == FilterOp::kIsNotNull) - ? StringPool::Id::Null() - : string_pool_->InternString(base::StringView(sql_val.AsString())); - NullTermStringView val_str = string_pool_->Get(val); - - auto first_non_null = static_cast<uint32_t>(std::distance( - indices.data, - std::partition_point(indices.data, indices.data + indices.size, - [this](uint32_t i) { - return (*data_)[i] == StringPool::Id::Null(); - }))); - - switch (op) { - case FilterOp::kEq: - return {LowerBoundExtrinsic(string_pool_, data_->data(), val_str, - indices.data, indices.size, first_non_null), - UpperBoundExtrinsic(string_pool_, data_->data(), val_str, - indices.data, indices.size, first_non_null)}; - case FilterOp::kLe: - return {first_non_null, - UpperBoundExtrinsic(string_pool_, data_->data(), val_str, - indices.data, indices.size, first_non_null)}; - case FilterOp::kLt: - return {first_non_null, - LowerBoundExtrinsic(string_pool_, data_->data(), val_str, - indices.data, indices.size, first_non_null)}; - case FilterOp::kGe: - return {LowerBoundExtrinsic(string_pool_, data_->data(), val_str, - indices.data, indices.size, first_non_null), - indices.size}; - case FilterOp::kGt: - return {UpperBoundExtrinsic(string_pool_, data_->data(), val_str, - indices.data, indices.size, first_non_null), - indices.size}; - case FilterOp::kIsNull: { - // Assuming nulls are at the front. - return Range(0, first_non_null); - } - case FilterOp::kIsNotNull: { - // Assuming nulls are at the front. - return Range(first_non_null, indices.size); - } - - case FilterOp::kNe: - case FilterOp::kGlob: - case FilterOp::kRegex: - PERFETTO_FATAL("Not supported for OrderedIndexSearch"); - } - PERFETTO_FATAL("For GCC"); -} - Range StringStorage::ChainImpl::BinarySearchIntrinsic( FilterOp op, SqlValue sql_val, @@ -715,6 +627,14 @@ std::optional<Token> StringStorage::ChainImpl::MinElement( return *tok; } +SqlValue StringStorage::ChainImpl::Get_AvoidUsingBecauseSlow( + uint32_t index) const { + StringPool::Id id = (*data_)[index]; + return id == StringPool::Id::Null() + ? SqlValue() + : SqlValue::String(string_pool_->Get(id).c_str()); +} + void StringStorage::ChainImpl::Serialize(StorageProto* msg) const { auto* string_storage_msg = msg->set_string_storage(); string_storage_msg->set_is_sorted(is_sorted_); diff --git a/src/trace_processor/db/column/string_storage.h b/src/trace_processor/db/column/string_storage.h index 470ccd1d7..1c05dd75f 100644 --- a/src/trace_processor/db/column/string_storage.h +++ b/src/trace_processor/db/column/string_storage.h @@ -18,6 +18,7 @@ #include <cstdint> #include <memory> +#include <optional> #include <string> #include <vector> @@ -57,10 +58,6 @@ class StringStorage final : public DataLayer { void IndexSearchValidated(FilterOp, SqlValue, Indices&) const override; - Range OrderedIndexSearchValidated(FilterOp, - SqlValue, - const OrderedIndices&) const override; - void StableSort(SortToken* start, SortToken* end, SortDirection direction) const override; @@ -71,6 +68,8 @@ class StringStorage final : public DataLayer { std::optional<Token> MinElement(Indices&) const override; + SqlValue Get_AvoidUsingBecauseSlow(uint32_t index) const override; + void Serialize(StorageProto*) const override; uint32_t size() const override { diff --git a/src/trace_processor/db/column/string_storage_unittest.cc b/src/trace_processor/db/column/string_storage_unittest.cc index 4474ea9fd..7e7cadb7f 100644 --- a/src/trace_processor/db/column/string_storage_unittest.cc +++ b/src/trace_processor/db/column/string_storage_unittest.cc @@ -383,12 +383,12 @@ TEST(StringStorage, OrderedIndexSearch) { op = FilterOp::kLt; res = chain->OrderedIndexSearch(op, val, indices); - ASSERT_EQ(res.start, 1u); + ASSERT_EQ(res.start, 0u); ASSERT_EQ(res.end, 4u); op = FilterOp::kLe; res = chain->OrderedIndexSearch(op, val, indices); - ASSERT_EQ(res.start, 1u); + ASSERT_EQ(res.start, 0u); ASSERT_EQ(res.end, 5u); op = FilterOp::kGt; @@ -402,6 +402,7 @@ TEST(StringStorage, OrderedIndexSearch) { ASSERT_EQ(res.end, 6u); op = FilterOp::kIsNull; + val = SqlValue(); res = chain->OrderedIndexSearch(op, val, indices); ASSERT_EQ(res.start, 0u); ASSERT_EQ(res.end, 1u); diff --git a/src/trace_processor/db/query_executor_benchmark.cc b/src/trace_processor/db/query_executor_benchmark.cc index d6cc57f7e..95b64ac79 100644 --- a/src/trace_processor/db/query_executor_benchmark.cc +++ b/src/trace_processor/db/query_executor_benchmark.cc @@ -198,12 +198,12 @@ struct FtraceEventTableForBenchmark { while (cur_idx < idx) { std::vector<std::string> raw_row = SplitCSVLine(raw_rows[cur_idx + 1]); RawTable::Row r; - r.cpu = *base::StringToUInt32(raw_row[1]); + r.ucpu = tables::CpuTable::Id(*base::StringToUInt32(raw_row[1])); raw_.Insert(r); cur_idx++; } FtraceEventTable::Row row; - row.cpu = *base::StringToUInt32(row_vec[1]); + row.ucpu = tables::CpuTable::Id(*base::StringToUInt32(row_vec[1])); table_.Insert(row); } } @@ -374,7 +374,7 @@ BENCHMARK(BM_QEFilterWithSparseSelector); void BM_QEFilterWithDenseSelector(benchmark::State& state) { FtraceEventTableForBenchmark table(state); Query q; - q.constraints = {table.table_.cpu().eq(4)}; + q.constraints = {table.table_.ucpu().eq(4)}; BenchmarkFtraceEventTableQuery(state, table, q); } BENCHMARK(BM_QEFilterWithDenseSelector); @@ -513,6 +513,29 @@ void BM_QEFilterOrderedArrangement(benchmark::State& state) { } BENCHMARK(BM_QEFilterOrderedArrangement); +void BM_QEFilterNullOrderedArrangement(benchmark::State& state) { + SliceTableForBenchmark table(state); + Order order{table.table_.parent_id().index_in_table(), false}; + Table slice_sorted_with_parent_id = table.table_.Sort({order}); + + Constraint c{table.table_.parent_id().index_in_table(), FilterOp::kGt, + SqlValue::Long(26091)}; + Query q; + q.constraints = {c}; + for (auto _ : state) { + benchmark::DoNotOptimize(slice_sorted_with_parent_id.QueryToRowMap(q)); + } + state.counters["s/row"] = benchmark::Counter( + static_cast<double>(slice_sorted_with_parent_id.row_count()), + benchmark::Counter::kIsIterationInvariantRate | + benchmark::Counter::kInvert); + state.counters["s/out"] = benchmark::Counter( + static_cast<double>(table.table_.QueryToRowMap(q).size()), + benchmark::Counter::kIsIterationInvariantRate | + benchmark::Counter::kInvert); +} +BENCHMARK(BM_QEFilterNullOrderedArrangement); + void BM_QESliceFilterIndexSearchOneElement(benchmark::State& state) { SliceTableForBenchmark table(state); BenchmarkSliceTableFilter( @@ -543,14 +566,15 @@ BENCHMARK(BM_QESliceSortNullNumericAsc); void BM_QEFtraceEventSortSelectorNumericAsc(benchmark::State& state) { FtraceEventTableForBenchmark table(state); - BenchmarkFtraceEventTableSort(state, table, {table.table_.cpu().ascending()}); + BenchmarkFtraceEventTableSort(state, table, + {table.table_.ucpu().ascending()}); } BENCHMARK(BM_QEFtraceEventSortSelectorNumericAsc); void BM_QEFtraceEventSortSelectorNumericDesc(benchmark::State& state) { FtraceEventTableForBenchmark table(state); BenchmarkFtraceEventTableSort(state, table, - {table.table_.cpu().descending()}); + {table.table_.ucpu().descending()}); } BENCHMARK(BM_QEFtraceEventSortSelectorNumericDesc); @@ -567,7 +591,7 @@ void BM_QEDistinctWithDenseSelector(benchmark::State& state) { FtraceEventTableForBenchmark table(state); Query q; q.order_type = Query::OrderType::kDistinct; - q.orders = {table.table_.cpu().descending()}; + q.orders = {table.table_.ucpu().descending()}; BenchmarkFtraceEventTableQuery(state, table, q); } BENCHMARK(BM_QEDistinctWithDenseSelector); @@ -585,7 +609,7 @@ void BM_QEDistinctSortedWithDenseSelector(benchmark::State& state) { FtraceEventTableForBenchmark table(state); Query q; q.order_type = Query::OrderType::kDistinctAndSort; - q.orders = {table.table_.cpu().descending()}; + q.orders = {table.table_.ucpu().descending()}; BenchmarkFtraceEventTableQuery(state, table, q); } BENCHMARK(BM_QEDistinctSortedWithDenseSelector); diff --git a/src/trace_processor/export_json_unittest.cc b/src/trace_processor/export_json_unittest.cc index a8f75644a..90677c2f6 100644 --- a/src/trace_processor/export_json_unittest.cc +++ b/src/trace_processor/export_json_unittest.cc @@ -27,8 +27,11 @@ #include "perfetto/ext/base/string_utils.h" #include "perfetto/ext/base/temp_file.h" #include "src/trace_processor/importers/common/args_tracker.h" +#include "src/trace_processor/importers/common/cpu_tracker.h" #include "src/trace_processor/importers/common/event_tracker.h" +#include "src/trace_processor/importers/common/machine_tracker.h" #include "src/trace_processor/importers/common/metadata_tracker.h" +#include "src/trace_processor/importers/common/process_track_translation_table.h" #include "src/trace_processor/importers/common/process_tracker.h" #include "src/trace_processor/importers/common/track_tracker.h" #include "src/trace_processor/importers/proto/track_event_tracker.h" @@ -76,9 +79,13 @@ class ExportJsonTest : public ::testing::Test { context_.args_tracker.reset(new ArgsTracker(&context_)); context_.event_tracker.reset(new EventTracker(&context_)); context_.track_tracker.reset(new TrackTracker(&context_)); + context_.machine_tracker.reset(new MachineTracker(&context_, 0)); + context_.cpu_tracker.reset(new CpuTracker(&context_)); context_.metadata_tracker.reset( new MetadataTracker(context_.storage.get())); context_.process_tracker.reset(new ProcessTracker(&context_)); + context_.process_track_translation_table.reset( + new ProcessTrackTranslationTable(context_.storage.get())); } std::string ToJson(ArgumentFilterPredicate argument_filter = nullptr, @@ -391,10 +398,11 @@ TEST_F(ExportJsonTest, StorageWithChromeMetadata) { TraceStorage* storage = context_.storage.get(); - RawId id = - storage->mutable_raw_table() - ->Insert({0, storage->InternString("chrome_event.metadata"), 0, 0}) - .id; + auto ucpu = context_.cpu_tracker->GetOrCreateCpu(0); + RawId id = storage->mutable_raw_table() + ->Insert({0, storage->InternString("chrome_event.metadata"), 0, + 0, 0, ucpu}) + .id; StringId name1_id = storage->InternString(base::StringView(kName1)); StringId name2_id = storage->InternString(base::StringView(kName2)); @@ -1388,9 +1396,10 @@ TEST_F(ExportJsonTest, RawEvent) { UniquePid upid = context_.process_tracker->GetOrCreateProcess(kProcessID); context_.storage->mutable_thread_table()->mutable_upid()->Set(utid, upid); + auto ucpu = context_.cpu_tracker->GetOrCreateCpu(0); auto id_and_row = storage->mutable_raw_table()->Insert( - {kTimestamp, storage->InternString("track_event.legacy_event"), /*cpu=*/0, - utid}); + {kTimestamp, storage->InternString("track_event.legacy_event"), utid, 0, + 0, ucpu}); auto inserter = context_.args_tracker->AddArgsTo(id_and_row.id); auto add_arg = [&](const char* key, Variadic value) { diff --git a/src/trace_processor/forwarding_trace_parser.cc b/src/trace_processor/forwarding_trace_parser.cc index 466ee96dc..a3b6a7f2c 100644 --- a/src/trace_processor/forwarding_trace_parser.cc +++ b/src/trace_processor/forwarding_trace_parser.cc @@ -16,29 +16,25 @@ #include "src/trace_processor/forwarding_trace_parser.h" +#include <memory> +#include <optional> + #include "perfetto/base/logging.h" -#include "perfetto/ext/base/string_utils.h" +#include "perfetto/base/status.h" +#include "perfetto/ext/base/status_or.h" +#include "src/trace_processor/importers/common/chunked_trace_reader.h" #include "src/trace_processor/importers/common/process_tracker.h" #include "src/trace_processor/importers/proto/proto_trace_reader.h" #include "src/trace_processor/sorter/trace_sorter.h" +#include "src/trace_processor/trace_reader_registry.h" #include "src/trace_processor/types/trace_processor_context.h" +#include "src/trace_processor/util/status_macros.h" +#include "src/trace_processor/util/trace_type.h" namespace perfetto { namespace trace_processor { namespace { -const char kNoZlibErr[] = - "Cannot open compressed trace. zlib not enabled in the build config"; - -inline bool isspace(unsigned char c) { - return ::isspace(c); -} - -std::string RemoveWhitespace(std::string str) { - str.erase(std::remove_if(str.begin(), str.end(), isspace), str.end()); - return str; -} - TraceSorter::SortingMode ConvertSortingMode(SortingMode sorting_mode) { switch (sorting_mode) { case SortingMode::kDefaultHeuristics: @@ -50,10 +46,30 @@ TraceSorter::SortingMode ConvertSortingMode(SortingMode sorting_mode) { PERFETTO_FATAL("For GCC"); } -// Fuchsia traces have a magic number as documented here: -// https://fuchsia.googlesource.com/fuchsia/+/HEAD/docs/development/tracing/trace-format/README.md#magic-number-record-trace-info-type-0 -constexpr uint64_t kFuchsiaMagicNumber = 0x0016547846040010; -constexpr char kPerfMagic[] = "PERFILE2"; +std::optional<TraceSorter::SortingMode> GetMinimumSortingMode( + TraceType trace_type, + const TraceProcessorContext& context) { + switch (trace_type) { + case kNinjaLogTraceType: + case kSystraceTraceType: + case kGzipTraceType: + case kCtraceTraceType: + case kZipFile: + return std::nullopt; + + case kPerfDataTraceType: + return TraceSorter::SortingMode::kDefault; + + case kUnknownTraceType: + case kJsonTraceType: + case kFuchsiaTraceType: + return TraceSorter::SortingMode::kFullSort; + + case kProtoTraceType: + return ConvertSortingMode(context.config.sorting_mode); + } + PERFETTO_FATAL("For GCC"); +} } // namespace @@ -62,102 +78,69 @@ ForwardingTraceParser::ForwardingTraceParser(TraceProcessorContext* context) ForwardingTraceParser::~ForwardingTraceParser() {} +base::Status ForwardingTraceParser::Init(const TraceBlobView& blob) { + PERFETTO_CHECK(!reader_); + + TraceType trace_type; + { + auto scoped_trace = context_->storage->TraceExecutionTimeIntoStats( + stats::guess_trace_type_duration_ns); + trace_type = GuessTraceType(blob.data(), blob.size()); + context_->trace_type = trace_type; + } + + if (trace_type == kUnknownTraceType) { + // If renaming this error message don't remove the "(ERR:fmt)" part. + // The UI's error_dialog.ts uses it to make the dialog more graceful. + return base::ErrStatus("Unknown trace type provided (ERR:fmt)"); + } + + base::StatusOr<std::unique_ptr<ChunkedTraceReader>> reader_or = + context_->reader_registry->CreateTraceReader(trace_type); + if (!reader_or.ok()) { + return reader_or.status(); + } + reader_ = std::move(*reader_or); + + PERFETTO_DLOG("%s detected", ToString(trace_type)); + UpdateSorterForTraceType(trace_type); + + // TODO(b/334978369) Make sure kProtoTraceType and kSystraceTraceType are + // parsed first so that we do not get issues with + // SetPidZeroIsUpidZeroIdleProcess() + if (trace_type == kProtoTraceType || trace_type == kSystraceTraceType) { + context_->process_tracker->SetPidZeroIsUpidZeroIdleProcess(); + } + + return base::OkStatus(); +} + +void ForwardingTraceParser::UpdateSorterForTraceType(TraceType trace_type) { + std::optional<TraceSorter::SortingMode> minimum_sorting_mode = + GetMinimumSortingMode(trace_type, *context_); + if (!minimum_sorting_mode.has_value()) { + return; + } + + if (!context_->sorter) { + context_->sorter.reset(new TraceSorter(context_, *minimum_sorting_mode)); + } + + switch (context_->sorter->sorting_mode()) { + case TraceSorter::SortingMode::kDefault: + PERFETTO_CHECK(minimum_sorting_mode == + TraceSorter::SortingMode::kDefault); + break; + case TraceSorter::SortingMode::kFullSort: + break; + } +} + base::Status ForwardingTraceParser::Parse(TraceBlobView blob) { // If this is the first Parse() call, guess the trace type and create the // appropriate parser. if (!reader_) { - TraceType trace_type; - { - auto scoped_trace = context_->storage->TraceExecutionTimeIntoStats( - stats::guess_trace_type_duration_ns); - trace_type = GuessTraceType(blob.data(), blob.size()); - context_->trace_type = trace_type; - } - switch (trace_type) { - case kJsonTraceType: { - PERFETTO_DLOG("JSON trace detected"); - if (context_->json_trace_tokenizer && context_->json_trace_parser) { - reader_ = std::move(context_->json_trace_tokenizer); - - // JSON traces have no guarantees about the order of events in them. - context_->sorter.reset( - new TraceSorter(context_, TraceSorter::SortingMode::kFullSort)); - break; - } - return base::ErrStatus("JSON support is disabled"); - } - case kProtoTraceType: { - PERFETTO_DLOG("Proto trace detected"); - auto sorting_mode = ConvertSortingMode(context_->config.sorting_mode); - reader_.reset(new ProtoTraceReader(context_)); - context_->sorter.reset(new TraceSorter(context_, sorting_mode)); - context_->process_tracker->SetPidZeroIsUpidZeroIdleProcess(); - break; - } - case kNinjaLogTraceType: { - PERFETTO_DLOG("Ninja log detected"); - if (context_->ninja_log_parser) { - reader_ = std::move(context_->ninja_log_parser); - break; - } - return base::ErrStatus("Ninja support is disabled"); - } - case kFuchsiaTraceType: { - PERFETTO_DLOG("Fuchsia trace detected"); - if (context_->fuchsia_record_parser && - context_->fuchsia_trace_tokenizer) { - reader_ = std::move(context_->fuchsia_trace_tokenizer); - - // Fuschia traces can have massively out of order events. - context_->sorter.reset( - new TraceSorter(context_, TraceSorter::SortingMode::kFullSort)); - break; - } - return base::ErrStatus("Fuchsia support is disabled"); - } - case kSystraceTraceType: - PERFETTO_DLOG("Systrace trace detected"); - context_->process_tracker->SetPidZeroIsUpidZeroIdleProcess(); - if (context_->systrace_trace_parser) { - reader_ = std::move(context_->systrace_trace_parser); - break; - } - return base::ErrStatus("Systrace support is disabled"); - case kGzipTraceType: - case kCtraceTraceType: - if (trace_type == kGzipTraceType) { - PERFETTO_DLOG("gzip trace detected"); - } else { - PERFETTO_DLOG("ctrace trace detected"); - } - if (context_->gzip_trace_parser) { - reader_ = std::move(context_->gzip_trace_parser); - break; - } - return base::ErrStatus(kNoZlibErr); - case kAndroidBugreportTraceType: - PERFETTO_DLOG("Android Bugreport detected"); - if (context_->android_bugreport_parser) { - reader_ = std::move(context_->android_bugreport_parser); - break; - } - return base::ErrStatus("Android Bugreport support is disabled. %s", - kNoZlibErr); - case kPerfDataTraceType: - PERFETTO_DLOG("perf data detected"); - if (context_->perf_data_trace_tokenizer && - context_->perf_record_parser) { - reader_ = std::move(context_->perf_data_trace_tokenizer); - context_->sorter.reset( - new TraceSorter(context_, TraceSorter::SortingMode::kDefault)); - break; - } - return base::ErrStatus("perf.data parsing support is disabled."); - case kUnknownTraceType: - // If renaming this error message don't remove the "(ERR:fmt)" part. - // The UI's error_dialog.ts uses it to make the dialog more graceful. - return base::ErrStatus("Unknown trace type provided (ERR:fmt)"); - } + RETURN_IF_ERROR(Init(blob)); } return reader_->Parse(std::move(blob)); @@ -167,71 +150,5 @@ void ForwardingTraceParser::NotifyEndOfFile() { reader_->NotifyEndOfFile(); } -TraceType GuessTraceType(const uint8_t* data, size_t size) { - if (size == 0) - return kUnknownTraceType; - std::string start(reinterpret_cast<const char*>(data), - std::min<size_t>(size, kGuessTraceMaxLookahead)); - if (size >= 8) { - uint64_t first_word; - memcpy(&first_word, data, sizeof(first_word)); - if (first_word == kFuchsiaMagicNumber) - return kFuchsiaTraceType; - } - if (base::StartsWith(start, kPerfMagic)) { - return kPerfDataTraceType; - } - std::string start_minus_white_space = RemoveWhitespace(start); - if (base::StartsWith(start_minus_white_space, "{\"")) - return kJsonTraceType; - if (base::StartsWith(start_minus_white_space, "[{\"")) - return kJsonTraceType; - - // Systrace with header but no leading HTML. - if (base::Contains(start, "# tracer")) - return kSystraceTraceType; - - // Systrace with leading HTML. - // Both: <!DOCTYPE html> and <!DOCTYPE HTML> have been observed. - std::string lower_start = base::ToLower(start); - if (base::StartsWith(lower_start, "<!doctype html>") || - base::StartsWith(lower_start, "<html>")) - return kSystraceTraceType; - - // Traces obtained from atrace -z (compress). - // They all have the string "TRACE:" followed by 78 9C which is a zlib header - // for "deflate, default compression, window size=32K" (see b/208691037) - if (base::Contains(start, "TRACE:\n\x78\x9c")) - return kCtraceTraceType; - - // Traces obtained from atrace without -z (no compression). - if (base::Contains(start, "TRACE:\n")) - return kSystraceTraceType; - - // Ninja's build log (.ninja_log). - if (base::StartsWith(start, "# ninja log")) - return kNinjaLogTraceType; - - // Systrace with no header or leading HTML. - if (base::StartsWith(start, " ")) - return kSystraceTraceType; - - // gzip'ed trace containing one of the other formats. - if (base::StartsWith(start, "\x1f\x8b")) - return kGzipTraceType; - - if (base::StartsWith(start, "\x0a")) - return kProtoTraceType; - - // Android bugreport.zip - // TODO(primiano). For now we assume any .zip file is a bugreport. In future, - // if we want to support different trace formats based on a .zip arachive we - // will need an extra layer similar to what we did kGzipTraceType. - if (base::StartsWith(start, "PK\x03\x04")) - return kAndroidBugreportTraceType; - - return kUnknownTraceType; -} - } // namespace trace_processor } // namespace perfetto diff --git a/src/trace_processor/forwarding_trace_parser.h b/src/trace_processor/forwarding_trace_parser.h index e63a272d4..12fe447b1 100644 --- a/src/trace_processor/forwarding_trace_parser.h +++ b/src/trace_processor/forwarding_trace_parser.h @@ -17,16 +17,15 @@ #ifndef SRC_TRACE_PROCESSOR_FORWARDING_TRACE_PARSER_H_ #define SRC_TRACE_PROCESSOR_FORWARDING_TRACE_PARSER_H_ +#include "perfetto/base/status.h" +#include "perfetto/trace_processor/trace_blob_view.h" #include "src/trace_processor/importers/common/chunked_trace_reader.h" - -#include "src/trace_processor/types/trace_processor_context.h" +#include "src/trace_processor/util/trace_type.h" namespace perfetto { namespace trace_processor { -constexpr size_t kGuessTraceMaxLookahead = 64; - -TraceType GuessTraceType(const uint8_t* data, size_t size); +class TraceProcessorContext; class ForwardingTraceParser : public ChunkedTraceReader { public: @@ -38,6 +37,8 @@ class ForwardingTraceParser : public ChunkedTraceReader { void NotifyEndOfFile() override; private: + base::Status Init(const TraceBlobView&); + void UpdateSorterForTraceType(TraceType trace_type); TraceProcessorContext* const context_; std::unique_ptr<ChunkedTraceReader> reader_; }; diff --git a/src/trace_processor/forwarding_trace_parser_unittest.cc b/src/trace_processor/forwarding_trace_parser_unittest.cc index 74cb63124..42d577d78 100644 --- a/src/trace_processor/forwarding_trace_parser_unittest.cc +++ b/src/trace_processor/forwarding_trace_parser_unittest.cc @@ -16,6 +16,7 @@ #include "src/trace_processor/forwarding_trace_parser.h" +#include "src/trace_processor/util/trace_type.h" #include "test/gtest_and_gmock.h" namespace perfetto { diff --git a/src/trace_processor/importers/android_bugreport/android_bugreport_parser.cc b/src/trace_processor/importers/android_bugreport/android_bugreport_parser.cc index 87aa0f5f9..1d46efc82 100644 --- a/src/trace_processor/importers/android_bugreport/android_bugreport_parser.cc +++ b/src/trace_processor/importers/android_bugreport/android_bugreport_parser.cc @@ -17,54 +17,90 @@ #include "src/trace_processor/importers/android_bugreport/android_bugreport_parser.h" #include <algorithm> +#include <cstddef> #include <optional> +#include <string> +#include <vector> #include "perfetto/base/logging.h" +#include "perfetto/base/status.h" #include "perfetto/ext/base/string_utils.h" -#include "perfetto/trace_processor/trace_blob.h" -#include "perfetto/trace_processor/trace_blob_view.h" +#include "protos/perfetto/common/builtin_clock.pbzero.h" #include "src/trace_processor/importers/android_bugreport/android_log_parser.h" #include "src/trace_processor/importers/common/clock_tracker.h" #include "src/trace_processor/importers/common/process_tracker.h" -#include "src/trace_processor/importers/common/track_tracker.h" #include "src/trace_processor/types/trace_processor_context.h" #include "src/trace_processor/util/zip_reader.h" -#include "protos/perfetto/common/builtin_clock.pbzero.h" - namespace perfetto { namespace trace_processor { +namespace { +const util::ZipFile* FindBugReportFile( + const std::vector<util::ZipFile>& zip_file_entries) { + for (const auto& zf : zip_file_entries) { + if (base::StartsWith(zf.name(), "bugreport-") && + base::EndsWith(zf.name(), ".txt")) { + return &zf; + } + } + return nullptr; +} -AndroidBugreportParser::AndroidBugreportParser(TraceProcessorContext* ctx) - : context_(ctx), zip_reader_(new util::ZipReader()) {} +std::optional<int32_t> ExtractYearFromBugReportFilename( + const std::string& filename) { + // Typical name: "bugreport-product-TP1A.220623.001-2022-06-24-16-24-37.txt". + auto year_str = + filename.substr(filename.size() - strlen("2022-12-31-23-59-00.txt"), 4); + return base::StringToInt32(year_str); +} -AndroidBugreportParser::~AndroidBugreportParser() = default; +} // namespace -util::Status AndroidBugreportParser::Parse(TraceBlobView tbv) { - if (!first_chunk_seen_) { - first_chunk_seen_ = true; - // All logs in Android bugreports use wall time (which creates problems - // in case of early boot events before NTP kicks in, which get emitted as - // 1970), but that is the state of affairs. - context_->clock_tracker->SetTraceTimeClock( - protos::pbzero::BUILTIN_CLOCK_REALTIME); +// static +bool AndroidBugreportParser::IsAndroidBugReport( + const std::vector<util::ZipFile>& zip_file_entries) { + if (const util::ZipFile* file = FindBugReportFile(zip_file_entries); + file != nullptr) { + return ExtractYearFromBugReportFilename(file->name()).has_value(); } - return zip_reader_->Parse(tbv.data(), tbv.size()); + return false; } -void AndroidBugreportParser::NotifyEndOfFile() { +// static +util::Status AndroidBugreportParser::Parse( + TraceProcessorContext* context, + std::vector<util::ZipFile> zip_file_entries) { + return AndroidBugreportParser(context, std::move(zip_file_entries)) + .ParseImpl(); +} + +AndroidBugreportParser::AndroidBugreportParser( + TraceProcessorContext* context, + std::vector<util::ZipFile> zip_file_entries) + : context_(context), zip_file_entries_(std::move(zip_file_entries)) {} + +AndroidBugreportParser::~AndroidBugreportParser() = default; + +util::Status AndroidBugreportParser::ParseImpl() { + // All logs in Android bugreports use wall time (which creates problems + // in case of early boot events before NTP kicks in, which get emitted as + // 1970), but that is the state of affairs. + context_->clock_tracker->SetTraceTimeClock( + protos::pbzero::BUILTIN_CLOCK_REALTIME); if (!DetectYearAndBrFilename()) { context_->storage->IncrementStats(stats::android_br_parse_errors); - return; + return base::ErrStatus("Zip file does not contain bugreport file."); } ParsePersistentLogcat(); ParseDumpstateTxt(); SortAndStoreLogcat(); + return base::OkStatus(); } void AndroidBugreportParser::ParseDumpstateTxt() { + PERFETTO_CHECK(dumpstate_file_); // Dumpstate is organized in a two level hierarchy, beautifully flattened into // one text file with load bearing ----- markers: // 1. Various dumpstate sections, examples: @@ -95,80 +131,81 @@ void AndroidBugreportParser::ParseDumpstateTxt() { // Here we put each line in a dedicated table, android_dumpstate, keeping // track of the dumpstate `section` and dumpsys `service`. AndroidLogParser log_parser(br_year_, context_->storage.get()); - util::ZipFile* zf = zip_reader_->Find(dumpstate_fname_); StringId section_id = StringId::Null(); // The current dumpstate section. StringId service_id = StringId::Null(); // The current dumpsys service. static constexpr size_t npos = base::StringView::npos; enum { OTHER = 0, DUMPSYS, LOG } cur_sect = OTHER; - zf->DecompressLines([&](const std::vector<base::StringView>& lines) { - // Optimization for ParseLogLines() below. Avoids ctor/dtor-ing a new vector - // on every line. - std::vector<base::StringView> log_line(1); - for (const base::StringView& line : lines) { - if (line.StartsWith("------ ") && line.EndsWith(" ------")) { - // These lines mark the beginning and end of dumpstate sections: - // ------ DUMPSYS CRITICAL (/system/bin/dumpsys) ------ - // ------ 0.356s was the duration of 'DUMPSYS CRITICAL' ------ - base::StringView section = line.substr(7); - section = section.substr(0, section.size() - 7); - bool end_marker = section.find("was the duration of") != npos; - service_id = StringId::Null(); - if (end_marker) { - section_id = StringId::Null(); - } else { - section_id = context_->storage->InternString(section); - cur_sect = OTHER; - if (section.StartsWith("DUMPSYS")) { - cur_sect = DUMPSYS; - } else if (section.StartsWith("SYSTEM LOG") || - section.StartsWith("EVENT LOG") || - section.StartsWith("RADIO LOG")) { - // KERNEL LOG is deliberately omitted because SYSTEM LOG is a - // superset. KERNEL LOG contains all dupes. - cur_sect = LOG; - } else if (section.StartsWith("BLOCK STAT")) { - // Coalesce all the block stats into one section. Otherwise they - // pollute the table with one section per block device. - section_id = context_->storage->InternString("BLOCK STAT"); + dumpstate_file_->DecompressLines( + [&](const std::vector<base::StringView>& lines) { + // Optimization for ParseLogLines() below. Avoids ctor/dtor-ing a new + // vector on every line. + std::vector<base::StringView> log_line(1); + for (const base::StringView& line : lines) { + if (line.StartsWith("------ ") && line.EndsWith(" ------")) { + // These lines mark the beginning and end of dumpstate sections: + // ------ DUMPSYS CRITICAL (/system/bin/dumpsys) ------ + // ------ 0.356s was the duration of 'DUMPSYS CRITICAL' ------ + base::StringView section = line.substr(7); + section = section.substr(0, section.size() - 7); + bool end_marker = section.find("was the duration of") != npos; + service_id = StringId::Null(); + if (end_marker) { + section_id = StringId::Null(); + } else { + section_id = context_->storage->InternString(section); + cur_sect = OTHER; + if (section.StartsWith("DUMPSYS")) { + cur_sect = DUMPSYS; + } else if (section.StartsWith("SYSTEM LOG") || + section.StartsWith("EVENT LOG") || + section.StartsWith("RADIO LOG")) { + // KERNEL LOG is deliberately omitted because SYSTEM LOG is a + // superset. KERNEL LOG contains all dupes. + cur_sect = LOG; + } else if (section.StartsWith("BLOCK STAT")) { + // Coalesce all the block stats into one section. Otherwise they + // pollute the table with one section per block device. + section_id = context_->storage->InternString("BLOCK STAT"); + } + } + continue; + } + // Skip end marker lines for dumpsys sections. + if (cur_sect == DUMPSYS && line.StartsWith("--------- ") && + line.find("was the duration of dumpsys") != npos) { + service_id = StringId::Null(); + continue; + } + if (cur_sect == DUMPSYS && service_id.is_null() && + line.StartsWith( + "----------------------------------------------")) { + continue; + } + if (cur_sect == DUMPSYS && line.StartsWith("DUMP OF SERVICE")) { + // DUMP OF SERVICE [CRITICAL|HIGH] ServiceName: + base::StringView svc = line.substr(line.rfind(' ') + 1); + svc = svc.substr(0, svc.size() - 1); + service_id = context_->storage->InternString(svc); + } else if (cur_sect == LOG) { + // Parse the non-persistent logcat and append to `log_events_`, + // together with the persistent one previously parsed by + // ParsePersistentLogcat(). Skips entries that are already seen in + // the persistent logcat, handling us vs ms truncation. + PERFETTO_DCHECK(log_line.size() == 1); + log_line[0] = line; + log_parser.ParseLogLines(log_line, &log_events_, + log_events_last_sorted_idx_); } - } - continue; - } - // Skip end marker lines for dumpsys sections. - if (cur_sect == DUMPSYS && line.StartsWith("--------- ") && - line.find("was the duration of dumpsys") != npos) { - service_id = StringId::Null(); - continue; - } - if (cur_sect == DUMPSYS && service_id.is_null() && - line.StartsWith("----------------------------------------------")) { - continue; - } - if (cur_sect == DUMPSYS && line.StartsWith("DUMP OF SERVICE")) { - // DUMP OF SERVICE [CRITICAL|HIGH] ServiceName: - base::StringView svc = line.substr(line.rfind(' ') + 1); - svc = svc.substr(0, svc.size() - 1); - service_id = context_->storage->InternString(svc); - } else if (cur_sect == LOG) { - // Parse the non-persistent logcat and append to `log_events_`, together - // with the persistent one previously parsed by ParsePersistentLogcat(). - // Skips entries that are already seen in the persistent logcat, - // handling us vs ms truncation. - PERFETTO_DCHECK(log_line.size() == 1); - log_line[0] = line; - log_parser.ParseLogLines(log_line, &log_events_, - log_events_last_sorted_idx_); - } - if (build_fpr_.empty() && line.StartsWith("Build fingerprint:")) { - build_fpr_ = line.substr(20, line.size() - 20).ToStdString(); - } + if (build_fpr_.empty() && line.StartsWith("Build fingerprint:")) { + build_fpr_ = line.substr(20, line.size() - 20).ToStdString(); + } - // Append the line to the android_dumpstate table. - context_->storage->mutable_android_dumpstate_table()->Insert( - {section_id, service_id, context_->storage->InternString(line)}); - } - }); + // Append the line to the android_dumpstate table. + context_->storage->mutable_android_dumpstate_table()->Insert( + {section_id, service_id, context_->storage->InternString(line)}); + } + }); } void AndroidBugreportParser::ParsePersistentLogcat() { @@ -182,22 +219,23 @@ void AndroidBugreportParser::ParsePersistentLogcat() { // Sort files to ease the job of the subsequent line-based sort. Unfortunately // lines within each file are not 100% timestamp-ordered, due to things like // kernel messages where log time != event time. - std::vector<std::pair<uint64_t, std::string>> log_paths; - for (const util::ZipFile& zf : zip_reader_->files()) { + std::vector<std::pair<uint64_t, const util::ZipFile*>> log_files; + for (const util::ZipFile& zf : zip_file_entries_) { if (base::StartsWith(zf.name(), "FS/data/misc/logd/logcat") && !base::EndsWith(zf.name(), "logcat.id")) { - log_paths.emplace_back(std::make_pair(zf.GetDatetime(), zf.name())); + log_files.push_back(std::make_pair(zf.GetDatetime(), &zf)); } } - std::sort(log_paths.begin(), log_paths.end()); + + std::sort(log_files.begin(), log_files.end()); // Push all events into the AndroidLogParser. It will take care of string // interning into the pool. Appends entries into `log_events`. - for (const auto& kv : log_paths) { - util::ZipFile* zf = zip_reader_->Find(kv.second); - zf->DecompressLines([&](const std::vector<base::StringView>& lines) { - log_parser.ParseLogLines(lines, &log_events_); - }); + for (const auto& log_file : log_files) { + log_file.second->DecompressLines( + [&](const std::vector<base::StringView>& lines) { + log_parser.ParseLogLines(lines, &log_events_); + }); } // Do an initial sorting pass. This is not the final sorting because we @@ -230,30 +268,20 @@ void AndroidBugreportParser::SortAndStoreLogcat() { // This is obviously bugged for cases of bugreports collected across new year // but we'll live with that. bool AndroidBugreportParser::DetectYearAndBrFilename() { - const util::ZipFile* br_file = nullptr; - for (const auto& zf : zip_reader_->files()) { - if (base::StartsWith(zf.name(), "bugreport-") && - base::EndsWith(zf.name(), ".txt")) { - br_file = &zf; - break; - } - } - + const util::ZipFile* br_file = FindBugReportFile(zip_file_entries_); if (!br_file) { PERFETTO_ELOG("Could not find bugreport-*.txt in the zip file"); return false; } - // Typical name: "bugreport-product-TP1A.220623.001-2022-06-24-16-24-37.txt". - auto year_str = br_file->name().substr( - br_file->name().size() - strlen("2022-12-31-23-59-00.txt"), 4); - std::optional<int32_t> year = base::StringToInt32(year_str); + std::optional<int32_t> year = + ExtractYearFromBugReportFilename(br_file->name()); if (!year.has_value()) { PERFETTO_ELOG("Could not parse the year from %s", br_file->name().c_str()); return false; } br_year_ = *year; - dumpstate_fname_ = br_file->name(); + dumpstate_file_ = br_file; return true; } diff --git a/src/trace_processor/importers/android_bugreport/android_bugreport_parser.h b/src/trace_processor/importers/android_bugreport/android_bugreport_parser.h index 99691598f..a9ca3226c 100644 --- a/src/trace_processor/importers/android_bugreport/android_bugreport_parser.h +++ b/src/trace_processor/importers/android_bugreport/android_bugreport_parser.h @@ -17,8 +17,11 @@ #ifndef SRC_TRACE_PROCESSOR_IMPORTERS_ANDROID_BUGREPORT_ANDROID_BUGREPORT_PARSER_H_ #define SRC_TRACE_PROCESSOR_IMPORTERS_ANDROID_BUGREPORT_ANDROID_BUGREPORT_PARSER_H_ -#include "src/trace_processor/importers/common/chunked_trace_reader.h" -#include "src/trace_processor/storage/trace_storage.h" +#include <cstddef> +#include <vector> + +#include "perfetto/trace_processor/status.h" +#include "src/trace_processor/util/zip_reader.h" namespace perfetto { namespace trace_processor { @@ -31,16 +34,19 @@ struct AndroidLogEvent; class TraceProcessorContext; // Trace importer for Android bugreport.zip archives. -class AndroidBugreportParser : public ChunkedTraceReader { +class AndroidBugreportParser { public: - explicit AndroidBugreportParser(TraceProcessorContext*); - ~AndroidBugreportParser() override; - - // ChunkedTraceReader implementation. - util::Status Parse(TraceBlobView) override; - void NotifyEndOfFile() override; + static bool IsAndroidBugReport( + const std::vector<util::ZipFile>& zip_file_entries); + static util::Status Parse(TraceProcessorContext* context, + std::vector<util::ZipFile> zip_file_entries); private: + AndroidBugreportParser(TraceProcessorContext* context, + std::vector<util::ZipFile> zip_file_entries); + ~AndroidBugreportParser(); + util::Status ParseImpl(); + bool DetectYearAndBrFilename(); void ParsePersistentLogcat(); void ParseDumpstateTxt(); @@ -48,11 +54,11 @@ class AndroidBugreportParser : public ChunkedTraceReader { void SortLogEvents(); TraceProcessorContext* const context_; + std::vector<util::ZipFile> zip_file_entries_; int br_year_ = 0; // The year when the bugreport has been taken. - std::string dumpstate_fname_; // The name of bugreport-xxx-2022-08-04....txt + const util::ZipFile* dumpstate_file_ = + nullptr; // The bugreport-xxx-2022-08-04....txt file std::string build_fpr_; - bool first_chunk_seen_ = false; - std::unique_ptr<util::ZipReader> zip_reader_; std::vector<AndroidLogEvent> log_events_; size_t log_events_last_sorted_idx_ = 0; }; diff --git a/src/trace_processor/importers/common/BUILD.gn b/src/trace_processor/importers/common/BUILD.gn index 43dfb2345..9cde6e32d 100644 --- a/src/trace_processor/importers/common/BUILD.gn +++ b/src/trace_processor/importers/common/BUILD.gn @@ -28,6 +28,8 @@ source_set("common") { "clock_converter.h", "clock_tracker.cc", "clock_tracker.h", + "cpu_tracker.cc", + "cpu_tracker.h", "create_mapping_params.h", "deobfuscation_mapping_table.cc", "deobfuscation_mapping_table.h", @@ -45,6 +47,8 @@ source_set("common") { "mapping_tracker.h", "metadata_tracker.cc", "metadata_tracker.h", + "process_track_translation_table.cc", + "process_track_translation_table.h", "process_tracker.cc", "process_tracker.h", "sched_event_state.h", @@ -87,6 +91,7 @@ source_set("common") { "../../util:build_id", "../../util:profiler_util", "../fuchsia:fuchsia_record", + "../perf:record", "../systrace:systrace_line", ] } @@ -116,6 +121,7 @@ source_set("unittests") { "deobfuscation_mapping_table_unittest.cc", "event_tracker_unittest.cc", "flow_tracker_unittest.cc", + "process_track_translation_table_unittest.cc", "process_tracker_unittest.cc", "slice_tracker_unittest.cc", "slice_translation_table_unittest.cc", diff --git a/src/trace_processor/importers/common/args_tracker.h b/src/trace_processor/importers/common/args_tracker.h index 964a6ad0e..e72b7c914 100644 --- a/src/trace_processor/importers/common/args_tracker.h +++ b/src/trace_processor/importers/common/args_tracker.h @@ -154,12 +154,31 @@ class ArgsTracker { context_->storage->mutable_surfaceflinger_transactions_table(), id); } + BoundInserter AddArgsTo(tables::ViewCaptureTable::Id id) { + return AddArgsTo(context_->storage->mutable_viewcapture_table(), id); + } + BoundInserter AddArgsTo(tables::WindowManagerShellTransitionsTable::Id id) { return AddArgsTo( context_->storage->mutable_window_manager_shell_transitions_table(), id); } + BoundInserter AddArgsTo(tables::AndroidKeyEventsTable::Id id) { + return AddArgsTo(context_->storage->mutable_android_key_events_table(), + id); + } + + BoundInserter AddArgsTo(tables::AndroidMotionEventsTable::Id id) { + return AddArgsTo(context_->storage->mutable_android_motion_events_table(), + id); + } + + BoundInserter AddArgsTo(tables::AndroidInputEventDispatchTable::Id id) { + return AddArgsTo( + context_->storage->mutable_android_input_event_dispatch_table(), id); + } + BoundInserter AddArgsTo(MetadataId id) { auto* table = context_->storage->mutable_metadata_table(); uint32_t row = *table->id().IndexOf(id); diff --git a/src/trace_processor/importers/common/async_track_set_tracker_unittest.cc b/src/trace_processor/importers/common/async_track_set_tracker_unittest.cc index b10aec821..173fd8921 100644 --- a/src/trace_processor/importers/common/async_track_set_tracker_unittest.cc +++ b/src/trace_processor/importers/common/async_track_set_tracker_unittest.cc @@ -18,6 +18,7 @@ #include "src/trace_processor/importers/common/args_tracker.h" #include "src/trace_processor/importers/common/global_args_tracker.h" +#include "src/trace_processor/importers/common/process_track_translation_table.h" #include "src/trace_processor/importers/common/track_tracker.h" #include "src/trace_processor/types/trace_processor_context.h" #include "test/gtest_and_gmock.h" @@ -34,6 +35,8 @@ class AsyncTrackSetTrackerUnittest : public testing::Test { context_.args_tracker.reset(new ArgsTracker(&context_)); context_.track_tracker.reset(new TrackTracker(&context_)); context_.async_track_set_tracker.reset(new AsyncTrackSetTracker(&context_)); + context_.process_track_translation_table.reset( + new ProcessTrackTranslationTable(context_.storage.get())); storage_ = context_.storage.get(); tracker_ = context_.async_track_set_tracker.get(); diff --git a/src/trace_processor/importers/common/cpu_tracker.cc b/src/trace_processor/importers/common/cpu_tracker.cc new file mode 100644 index 000000000..b7039269b --- /dev/null +++ b/src/trace_processor/importers/common/cpu_tracker.cc @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/trace_processor/importers/common/cpu_tracker.h" + +#include "perfetto/base/logging.h" +#include "perfetto/public/compiler.h" + +#include "src/trace_processor/importers/common/machine_tracker.h" + +namespace perfetto::trace_processor { + +CpuTracker::CpuTracker(TraceProcessorContext* context) : context_(context) { + // Preallocate ucpu of this machine for maintaining the relative order between + // ucpu and cpu. + auto machine_id = context_->machine_tracker->machine_id(); + if (machine_id.has_value()) + ucpu_offset_ = machine_id->value * kMaxCpusPerMachine; + + for (auto id = 0u; id < kMaxCpusPerMachine; id++) { + // Only populate the |machine_id| column. The |cpu| column is update only + // when the CPU is present. + tables::CpuTable::Row cpu_row; + cpu_row.machine_id = machine_id; + context_->storage->mutable_cpu_table()->Insert(cpu_row); + } +} + +CpuTracker::~CpuTracker() = default; + +tables::CpuTable::Id CpuTracker::SetCpuInfo(uint32_t cpu, + base::StringView processor, + uint32_t cluster_id) { + auto cpu_id = GetOrCreateCpu(cpu); + + auto cpu_row = context_->storage->mutable_cpu_table()->FindById(cpu_id); + PERFETTO_DCHECK(cpu_row.has_value()); + + if (!processor.empty()) { + auto string_id = context_->storage->InternString(processor); + cpu_row->set_processor(string_id); + } + cpu_row->set_cluster_id(cluster_id); + + return cpu_id; +} + +} // namespace perfetto::trace_processor diff --git a/src/trace_processor/importers/common/cpu_tracker.h b/src/trace_processor/importers/common/cpu_tracker.h new file mode 100644 index 000000000..31c0b61e9 --- /dev/null +++ b/src/trace_processor/importers/common/cpu_tracker.h @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SRC_TRACE_PROCESSOR_IMPORTERS_COMMON_CPU_TRACKER_H_ +#define SRC_TRACE_PROCESSOR_IMPORTERS_COMMON_CPU_TRACKER_H_ + +#include <bitset> + +#include "src/trace_processor/storage/trace_storage.h" +#include "src/trace_processor/tables/metadata_tables_py.h" +#include "src/trace_processor/types/trace_processor_context.h" + +namespace perfetto::trace_processor { + +class TraceProcessorContext; + +class CpuTracker { + public: + // The CPU table Id serves as the 'ucpu' in sched_slice and related for + // joining with this table. To optimize for single-machine traces, this class + // assumes a maximum of |kMaxCpusPerMachine| CPUs per machine to maintain a + // relative order of |cpu| and |ucpu| by pre-allocating |kMaxCpusPerMachine| + // records in the CPU table. The mapping between |ucpu| and |cpu| becomes + // |cpu| = |ucpu| % |kMaxCpusPerMachine|. + static constexpr uint32_t kMaxCpusPerMachine = 4096; + + explicit CpuTracker(TraceProcessorContext*); + ~CpuTracker(); + + tables::CpuTable::Id GetOrCreateCpu(uint32_t cpu) { + // CPU core number is in the range of 0..kMaxCpusPerMachine-1. + PERFETTO_CHECK(cpu < kMaxCpusPerMachine); + auto ucpu = ucpu_offset_ + cpu; + if (PERFETTO_LIKELY(cpu_ids_[cpu])) + return tables::CpuTable::Id(ucpu); + + // Populate the optional |cpu| column. + context_->storage->mutable_cpu_table()->mutable_cpu()->Set(ucpu, cpu); + return tables::CpuTable::Id(ucpu); + } + + // Sets or updates the information for the specified CPU in the CpuTable. + tables::CpuTable::Id SetCpuInfo(uint32_t cpu, + base::StringView processor, + uint32_t cluster_id); + + private: + TraceProcessorContext* const context_; + + // Tracks the mapping of CPU number to CpuTable::Id of the current + // machine. + std::bitset<kMaxCpusPerMachine> cpu_ids_; + uint32_t ucpu_offset_ = 0; +}; + +} // namespace perfetto::trace_processor + +#endif // SRC_TRACE_PROCESSOR_IMPORTERS_COMMON_CPU_TRACKER_H_ diff --git a/src/trace_processor/importers/common/event_tracker.cc b/src/trace_processor/importers/common/event_tracker.cc index 43cf71051..a38d77624 100644 --- a/src/trace_processor/importers/common/event_tracker.cc +++ b/src/trace_processor/importers/common/event_tracker.cc @@ -16,20 +16,18 @@ #include "src/trace_processor/importers/common/event_tracker.h" -#include <math.h> +#include <cinttypes> +#include <cstdint> #include <optional> #include "perfetto/base/logging.h" -#include "perfetto/ext/base/utils.h" #include "src/trace_processor/importers/common/args_tracker.h" -#include "src/trace_processor/importers/common/process_tracker.h" #include "src/trace_processor/importers/common/track_tracker.h" #include "src/trace_processor/storage/stats.h" +#include "src/trace_processor/storage/trace_storage.h" #include "src/trace_processor/types/trace_processor_context.h" -#include "src/trace_processor/types/variadic.h" -namespace perfetto { -namespace trace_processor { +namespace perfetto::trace_processor { EventTracker::EventTracker(TraceProcessorContext* context) : context_(context) {} @@ -65,9 +63,7 @@ std::optional<CounterId> EventTracker::PushCounter(int64_t timestamp, max_timestamp_ = timestamp; auto* counter_values = context_->storage->mutable_counter_table(); - return counter_values - ->Insert({timestamp, track_id, value, {}, context_->machine_id()}) - .id; + return counter_values->Insert({timestamp, track_id, value, {}}).id; } std::optional<CounterId> EventTracker::PushCounter( @@ -106,5 +102,4 @@ void EventTracker::FlushPendingEvents() { pending_upid_resolution_counter_.clear(); } -} // namespace trace_processor -} // namespace perfetto +} // namespace perfetto::trace_processor diff --git a/src/trace_processor/importers/common/mapping_tracker.cc b/src/trace_processor/importers/common/mapping_tracker.cc index 0dec3e5ca..d0a2d8881 100644 --- a/src/trace_processor/importers/common/mapping_tracker.cc +++ b/src/trace_processor/importers/common/mapping_tracker.cc @@ -81,9 +81,6 @@ KernelMemoryMapping& MappingTracker::CreateKernelMemoryMapping( UserMemoryMapping& MappingTracker::CreateUserMemoryMapping( UniquePid upid, CreateMappingParams params) { - // TODO(carlscab): Guess build_id if not provided. Some tools like simpleperf - // add a mapping file_name ->build_id that we could use here - const AddressRange mapping_range = params.memory_range; std::unique_ptr<UserMemoryMapping> mapping( new UserMemoryMapping(context_, upid, std::move(params))); @@ -164,5 +161,15 @@ void MappingTracker::AddJitRange(UniquePid upid, }); } +VirtualMemoryMapping* MappingTracker::GetDummyMapping() { + if (!dummy_mapping_) { + CreateMappingParams params; + params.memory_range = + AddressRange::FromStartAndSize(0, std::numeric_limits<uint64_t>::max()); + dummy_mapping_ = &InternMemoryMapping(params); + } + return dummy_mapping_; +} + } // namespace trace_processor } // namespace perfetto diff --git a/src/trace_processor/importers/common/mapping_tracker.h b/src/trace_processor/importers/common/mapping_tracker.h index bc45befe8..08c767e0a 100644 --- a/src/trace_processor/importers/common/mapping_tracker.h +++ b/src/trace_processor/importers/common/mapping_tracker.h @@ -91,6 +91,10 @@ class MappingTracker { // Jitted ranges will only be applied to UserMemoryMappings void AddJitRange(UniquePid upid, AddressRange range, JitCache* jit_cache); + // Sometimes we just need a mapping and we are lacking trace data to create a + // proper one. Use this mapping in those cases. + VirtualMemoryMapping* GetDummyMapping(); + private: template <typename MappingImpl> MappingImpl& AddMapping(std::unique_ptr<MappingImpl> mapping); @@ -136,6 +140,8 @@ class MappingTracker { KernelMemoryMapping* kernel_ = nullptr; base::FlatHashMap<UniquePid, AddressRangeMap<JitCache*>> jit_caches_; + + VirtualMemoryMapping* dummy_mapping_; }; } // namespace trace_processor diff --git a/src/trace_processor/importers/common/process_track_translation_table.cc b/src/trace_processor/importers/common/process_track_translation_table.cc new file mode 100644 index 000000000..7cd542473 --- /dev/null +++ b/src/trace_processor/importers/common/process_track_translation_table.cc @@ -0,0 +1,25 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/trace_processor/importers/common/process_track_translation_table.h" + +namespace perfetto::trace_processor { + +ProcessTrackTranslationTable::ProcessTrackTranslationTable( + TraceStorage* storage) + : storage_(storage) {} + +} // namespace perfetto::trace_processor diff --git a/src/trace_processor/importers/common/process_track_translation_table.h b/src/trace_processor/importers/common/process_track_translation_table.h new file mode 100644 index 000000000..30f5f56d5 --- /dev/null +++ b/src/trace_processor/importers/common/process_track_translation_table.h @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SRC_TRACE_PROCESSOR_IMPORTERS_COMMON_PROCESS_TRACK_TRANSLATION_TABLE_H_ +#define SRC_TRACE_PROCESSOR_IMPORTERS_COMMON_PROCESS_TRACK_TRANSLATION_TABLE_H_ + +#include <cstdint> + +#include "perfetto/ext/base/flat_hash_map.h" +#include "perfetto/ext/base/string_view.h" +#include "src/trace_processor/storage/trace_storage.h" + +namespace perfetto::trace_processor { + +// Tracks and stores slice translation rules. It allows Trace Processor +// to for example deobfuscate slice names. +class ProcessTrackTranslationTable { + public: + ProcessTrackTranslationTable(TraceStorage* storage); + + // If the name is not mapped to anything, assumes that no translation is + // necessry, and returns the raw_name. + StringId TranslateName(StringId raw_name) const { + const auto* mapped_name = raw_to_deobfuscated_name_.Find(raw_name); + return mapped_name ? *mapped_name : raw_name; + } + + void AddNameTranslationRule(base::StringView raw, + base::StringView deobfuscated) { + const StringId raw_id = storage_->InternString(raw); + const StringId deobfuscated_id = storage_->InternString(deobfuscated); + raw_to_deobfuscated_name_[raw_id] = deobfuscated_id; + } + + private: + TraceStorage* storage_; + base::FlatHashMap<StringId, StringId> raw_to_deobfuscated_name_; +}; + +} // namespace perfetto::trace_processor + +#endif // SRC_TRACE_PROCESSOR_IMPORTERS_COMMON_PROCESS_TRACK_TRANSLATION_TABLE_H_ diff --git a/src/trace_processor/importers/common/process_track_translation_table_unittest.cc b/src/trace_processor/importers/common/process_track_translation_table_unittest.cc new file mode 100644 index 000000000..a947204ba --- /dev/null +++ b/src/trace_processor/importers/common/process_track_translation_table_unittest.cc @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/trace_processor/importers/common/process_track_translation_table.h" +#include "test/gtest_and_gmock.h" + +namespace perfetto::trace_processor { +namespace { + +TEST(ProcessTrackTranslationTable, UnknownName) { + TraceStorage storage; + ProcessTrackTranslationTable table(&storage); + const StringId raw_name = storage.InternString("name1"); + EXPECT_EQ(raw_name, table.TranslateName(raw_name)); +} + +TEST(ProcessTrackTranslationTable, MappedName) { + TraceStorage storage; + ProcessTrackTranslationTable table(&storage); + table.AddNameTranslationRule("raw_name1", "mapped_name1"); + const StringId raw_name = storage.InternString("raw_name1"); + const StringId mapped_name = storage.InternString("mapped_name1"); + EXPECT_EQ(mapped_name, table.TranslateName(raw_name)); +} + +} // namespace +} // namespace perfetto::trace_processor diff --git a/src/trace_processor/importers/common/process_tracker.cc b/src/trace_processor/importers/common/process_tracker.cc index 3fc95fcbd..e29280174 100644 --- a/src/trace_processor/importers/common/process_tracker.cc +++ b/src/trace_processor/importers/common/process_tracker.cc @@ -356,6 +356,17 @@ UniquePid ProcessTracker::SetProcessMetadata(uint32_t pid, UniquePid upid = GetOrCreateProcess(pid); auto* process_table = context_->storage->mutable_process_table(); + // If we both know the previous and current parent pid and the two are not + // matching, we must have died and restarted: create a new process. + if (pupid) { + std::optional<UniquePid> prev_parent_upid = + process_table->parent_upid()[upid]; + if (prev_parent_upid && prev_parent_upid != pupid) { + upid = StartNewProcess(std::nullopt, ppid, pid, kNullStringId, + ThreadNamePriority::kOther); + } + } + StringId proc_name_id = context_->storage->InternString(name); process_table->mutable_name()->Set(upid, proc_name_id); process_table->mutable_cmdline()->Set( diff --git a/src/trace_processor/importers/common/sched_event_tracker.h b/src/trace_processor/importers/common/sched_event_tracker.h index 12aa4c02d..ff4f53709 100644 --- a/src/trace_processor/importers/common/sched_event_tracker.h +++ b/src/trace_processor/importers/common/sched_event_tracker.h @@ -19,6 +19,7 @@ #include "src/trace_processor/importers/common/event_tracker.h" +#include "src/trace_processor/importers/common/cpu_tracker.h" #include "src/trace_processor/storage/trace_storage.h" #include "src/trace_processor/types/destructible.h" #include "src/trace_processor/types/trace_processor_context.h" @@ -45,9 +46,10 @@ class SchedEventTracker : public Destructible { // just switched to. Set the duration to -1, to indicate that the event is // not finished. Duration will be updated later after event finish. auto* sched = context_->storage->mutable_sched_slice_table(); - auto row_and_id = - sched->Insert({ts, /* duration */ -1, cpu, next_utid, kNullStringId, - next_prio, context_->machine_id()}); + // Get the unique CPU Id over all machines from the CPU table. + auto ucpu = context_->cpu_tracker->GetOrCreateCpu(cpu); + auto row_and_id = sched->Insert( + {ts, /* duration */ -1, next_utid, kNullStringId, next_prio, ucpu}); SchedId sched_id = row_and_id.id; return *sched->id().IndexOf(sched_id); } diff --git a/src/trace_processor/importers/common/thread_state_tracker.cc b/src/trace_processor/importers/common/thread_state_tracker.cc index 2d98ffb1a..a0d98ce74 100644 --- a/src/trace_processor/importers/common/thread_state_tracker.cc +++ b/src/trace_processor/importers/common/thread_state_tracker.cc @@ -19,6 +19,7 @@ #include <cstdint> #include <optional> +#include "src/trace_processor/importers/common/cpu_tracker.h" #include "src/trace_processor/importers/common/process_tracker.h" namespace perfetto { @@ -137,12 +138,12 @@ void ThreadStateTracker::AddOpenState(int64_t ts, // Insert row with unfinished state tables::ThreadStateTable::Row row; row.ts = ts; - row.cpu = cpu; row.waker_utid = waker_utid; row.dur = -1; row.utid = utid; row.state = state; - row.machine_id = context_->machine_id(); + if (cpu) + row.ucpu = context_->cpu_tracker->GetOrCreateCpu(*cpu); if (common_flags.has_value()) { row.irq_context = CommonFlagsToIrqContext(*common_flags); } diff --git a/src/trace_processor/importers/common/thread_state_tracker_unittest.cc b/src/trace_processor/importers/common/thread_state_tracker_unittest.cc index 153b02ba8..12381413e 100644 --- a/src/trace_processor/importers/common/thread_state_tracker_unittest.cc +++ b/src/trace_processor/importers/common/thread_state_tracker_unittest.cc @@ -19,7 +19,9 @@ #include <algorithm> #include "src/trace_processor/importers/common/args_tracker.h" +#include "src/trace_processor/importers/common/cpu_tracker.h" #include "src/trace_processor/importers/common/global_args_tracker.h" +#include "src/trace_processor/importers/common/machine_tracker.h" #include "src/trace_processor/importers/common/process_tracker.h" #include "src/trace_processor/types/trace_processor_context.h" #include "test/gtest_and_gmock.h" @@ -45,6 +47,8 @@ class ThreadStateTrackerUnittest : public testing::Test { context_.process_tracker.reset(new ProcessTracker(&context_)); context_.global_args_tracker.reset( new GlobalArgsTracker(context_.storage.get())); + context_.machine_tracker.reset(new MachineTracker(&context_, 0)); + context_.cpu_tracker.reset(new CpuTracker(&context_)); context_.args_tracker.reset(new ArgsTracker(&context_)); tracker_.reset(new ThreadStateTracker(&context_)); } @@ -72,12 +76,12 @@ class ThreadStateTrackerUnittest : public testing::Test { ASSERT_EQ(it.utid(), utid); if (state == kRunning) { if (cpu.has_value()) { - ASSERT_EQ(it.cpu(), cpu); + ASSERT_EQ(it.ucpu().value().value, cpu); } else { - ASSERT_EQ(it.cpu(), CPU_A); + ASSERT_EQ(it.ucpu().value().value, CPU_A); } } else { - ASSERT_EQ(it.cpu(), std::nullopt); + ASSERT_EQ(it.ucpu(), std::nullopt); } ASSERT_STREQ(context_.storage->GetString(it.state()).c_str(), state); ASSERT_EQ(it.io_wait(), io_wait); diff --git a/src/trace_processor/importers/common/trace_parser.h b/src/trace_processor/importers/common/trace_parser.h index bf90c95fb..8e41a7dba 100644 --- a/src/trace_processor/importers/common/trace_parser.h +++ b/src/trace_processor/importers/common/trace_parser.h @@ -20,8 +20,10 @@ #include <stdint.h> #include <string> -namespace perfetto { -namespace trace_processor { +namespace perfetto::trace_processor { +namespace perf_importer { +struct Record; +} class PacketSequenceStateGeneration; class TraceBlobView; @@ -59,10 +61,9 @@ class FuchsiaRecordParser { class PerfRecordParser { public: virtual ~PerfRecordParser(); - virtual void ParsePerfRecord(int64_t, TraceBlobView) = 0; + virtual void ParsePerfRecord(int64_t, perf_importer::Record) = 0; }; -} // namespace trace_processor -} // namespace perfetto +} // namespace perfetto::trace_processor #endif // SRC_TRACE_PROCESSOR_IMPORTERS_COMMON_TRACE_PARSER_H_ diff --git a/src/trace_processor/importers/common/track_tracker.cc b/src/trace_processor/importers/common/track_tracker.cc index 502639152..a2ba1f8d4 100644 --- a/src/trace_processor/importers/common/track_tracker.cc +++ b/src/trace_processor/importers/common/track_tracker.cc @@ -19,7 +19,10 @@ #include <optional> #include "src/trace_processor/importers/common/args_tracker.h" +#include "src/trace_processor/importers/common/process_track_translation_table.h" #include "src/trace_processor/storage/trace_storage.h" +#include "src/trace_processor/tables/profiler_tables_py.h" +#include "src/trace_processor/types/trace_processor_context.h" namespace perfetto { namespace trace_processor { @@ -140,7 +143,7 @@ TrackId TrackTracker::InternGpuWorkPeriodTrack( } TrackId TrackTracker::InternLegacyChromeAsyncTrack( - StringId name, + StringId raw_name, uint32_t upid, int64_t trace_id, bool trace_id_is_process_scoped, @@ -151,6 +154,8 @@ TrackId TrackTracker::InternLegacyChromeAsyncTrack( tuple.trace_id = trace_id; tuple.source_scope = source_scope; + const StringId name = + context_->process_track_translation_table->TranslateName(raw_name); auto it = chrome_tracks_.find(tuple); if (it != chrome_tracks_.end()) { if (name != kNullStringId) { @@ -194,9 +199,11 @@ TrackId TrackTracker::CreateGlobalAsyncTrack(StringId name, StringId source) { return id; } -TrackId TrackTracker::CreateProcessAsyncTrack(StringId name, +TrackId TrackTracker::CreateProcessAsyncTrack(StringId raw_name, UniquePid upid, StringId source) { + const StringId name = + context_->process_track_translation_table->TranslateName(raw_name); tables::ProcessTrackTable::Row row(name); row.upid = upid; row.machine_id = context_->machine_id(); @@ -318,10 +325,12 @@ TrackId TrackTracker::InternThreadCounterTrack(StringId name, UniqueTid utid) { return track; } -TrackId TrackTracker::InternProcessCounterTrack(StringId name, +TrackId TrackTracker::InternProcessCounterTrack(StringId raw_name, UniquePid upid, StringId unit, StringId description) { + const StringId name = + context_->process_track_translation_table->TranslateName(raw_name); auto it = upid_counter_tracks_.find(std::make_pair(name, upid)); if (it != upid_counter_tracks_.end()) { return it->second; @@ -447,10 +456,11 @@ TrackId TrackTracker::CreateGpuCounterTrack(StringId name, return context_->storage->mutable_gpu_counter_track_table()->Insert(row).id; } -TrackId TrackTracker::CreatePerfCounterTrack(StringId name, - uint32_t perf_session_id, - uint32_t cpu, - bool is_timebase) { +TrackId TrackTracker::CreatePerfCounterTrack( + StringId name, + tables::PerfSessionTable::Id perf_session_id, + uint32_t cpu, + bool is_timebase) { tables::PerfCounterTrackTable::Row row(name); row.perf_session_id = perf_session_id; row.cpu = cpu; diff --git a/src/trace_processor/importers/common/track_tracker.h b/src/trace_processor/importers/common/track_tracker.h index 0fec299ab..c0662049b 100644 --- a/src/trace_processor/importers/common/track_tracker.h +++ b/src/trace_processor/importers/common/track_tracker.h @@ -18,8 +18,10 @@ #define SRC_TRACE_PROCESSOR_IMPORTERS_COMMON_TRACK_TRACKER_H_ #include <optional> + #include "src/trace_processor/importers/common/args_tracker.h" #include "src/trace_processor/storage/trace_storage.h" +#include "src/trace_processor/tables/profiler_tables_py.h" #include "src/trace_processor/types/trace_processor_context.h" namespace perfetto { @@ -145,7 +147,7 @@ class TrackTracker { // Creates a counter track for values within perf samples. // The tracks themselves are managed by PerfSampleTracker. TrackId CreatePerfCounterTrack(StringId name, - uint32_t perf_session_id, + tables::PerfSessionTable::Id perf_session_id, uint32_t cpu, bool is_timebase); diff --git a/src/trace_processor/importers/ftrace/ftrace_descriptors.cc b/src/trace_processor/importers/ftrace/ftrace_descriptors.cc index 64ae4d574..91bfb5e31 100644 --- a/src/trace_processor/importers/ftrace/ftrace_descriptors.cc +++ b/src/trace_processor/importers/ftrace/ftrace_descriptors.cc @@ -24,7 +24,7 @@ namespace perfetto { namespace trace_processor { namespace { -std::array<FtraceMessageDescriptor, 508> descriptors{{ +std::array<FtraceMessageDescriptor, 510> descriptors{{ {nullptr, 0, {}}, {nullptr, 0, {}}, {nullptr, 0, {}}, @@ -5619,6 +5619,24 @@ std::array<FtraceMessageDescriptor, 508> descriptors{{ {"k_i", ProtoSchemaType::kInt32}, }, }, + { + "dcvsh_freq", + 2, + { + {}, + {"cpu", ProtoSchemaType::kUint64}, + {"freq", ProtoSchemaType::kUint64}, + }, + }, + { + "kgsl_gpu_frequency", + 2, + { + {}, + {"gpu_freq", ProtoSchemaType::kUint32}, + {"gpu_id", ProtoSchemaType::kUint32}, + }, + }, }}; } // namespace diff --git a/src/trace_processor/importers/ftrace/ftrace_parser.cc b/src/trace_processor/importers/ftrace/ftrace_parser.cc index 406f30199..75bfd00eb 100644 --- a/src/trace_processor/importers/ftrace/ftrace_parser.cc +++ b/src/trace_processor/importers/ftrace/ftrace_parser.cc @@ -24,6 +24,7 @@ #include "perfetto/trace_processor/basic_types.h" #include "src/trace_processor/importers/common/args_tracker.h" #include "src/trace_processor/importers/common/async_track_set_tracker.h" +#include "src/trace_processor/importers/common/cpu_tracker.h" #include "src/trace_processor/importers/common/metadata_tracker.h" #include "src/trace_processor/importers/common/parser_types.h" #include "src/trace_processor/importers/common/process_tracker.h" @@ -49,6 +50,7 @@ #include "protos/perfetto/trace/ftrace/cma.pbzero.h" #include "protos/perfetto/trace/ftrace/cpuhp.pbzero.h" #include "protos/perfetto/trace/ftrace/cros_ec.pbzero.h" +#include "protos/perfetto/trace/ftrace/dcvsh.pbzero.h" #include "protos/perfetto/trace/ftrace/dmabuf_heap.pbzero.h" #include "protos/perfetto/trace/ftrace/dpu.pbzero.h" #include "protos/perfetto/trace/ftrace/fastrpc.pbzero.h" @@ -63,6 +65,7 @@ #include "protos/perfetto/trace/ftrace/i2c.pbzero.h" #include "protos/perfetto/trace/ftrace/ion.pbzero.h" #include "protos/perfetto/trace/ftrace/irq.pbzero.h" +#include "protos/perfetto/trace/ftrace/kgsl.pbzero.h" #include "protos/perfetto/trace/ftrace/kmem.pbzero.h" #include "protos/perfetto/trace/ftrace/lwis.pbzero.h" #include "protos/perfetto/trace/ftrace/mali.pbzero.h" @@ -317,6 +320,8 @@ FtraceParser::FtraceParser(TraceProcessorContext* context) sched_waking_name_id_(context->storage->InternString("sched_waking")), cpu_id_(context->storage->InternString("cpu")), cpu_freq_name_id_(context->storage->InternString("cpufreq")), + cpu_freq_throttle_name_id_( + context->storage->InternString("cpufreq_throttle")), gpu_freq_name_id_(context->storage->InternString("gpufreq")), cpu_idle_name_id_(context->storage->InternString("cpuidle")), suspend_resume_name_id_( @@ -761,10 +766,18 @@ base::Status FtraceParser::ParseFtraceEvent(uint32_t cpu, ParseCpuFreq(ts, fld_bytes); break; } + case FtraceEvent::kDcvshFreqFieldNumber: { + ParseCpuFreqThrottle(ts, fld_bytes); + break; + } case FtraceEvent::kGpuFrequencyFieldNumber: { ParseGpuFreq(ts, fld_bytes); break; } + case FtraceEvent::kKgslGpuFrequencyFieldNumber: { + ParseKgslGpuFreq(ts, fld_bytes); + break; + } case FtraceEvent::kCpuIdleFieldNumber: { ParseCpuIdle(ts, fld_bytes); break; @@ -951,8 +964,7 @@ base::Status FtraceParser::ParseFtraceEvent(uint32_t cpu, break; } case FtraceEvent::kThermalExynosAcpmHighOverheadFieldNumber: { - thermal_tracker_.ParseThermalExynosAcpmHighOverhead( - ts, fld_bytes); + thermal_tracker_.ParseThermalExynosAcpmHighOverhead(ts, fld_bytes); break; } case FtraceEvent::kCdevUpdateFieldNumber: { @@ -1362,10 +1374,10 @@ void FtraceParser::ParseGenericFtrace(int64_t ts, protos::pbzero::GenericFtraceEvent::Decoder evt(blob.data, blob.size); StringId event_id = context_->storage->InternString(evt.event_name()); UniqueTid utid = context_->process_tracker->GetOrCreateThread(tid); - RawId id = - context_->storage->mutable_ftrace_event_table() - ->Insert({ts, event_id, cpu, utid, {}, {}, context_->machine_id()}) - .id; + auto ucpu = context_->cpu_tracker->GetOrCreateCpu(cpu); + RawId id = context_->storage->mutable_ftrace_event_table() + ->Insert({ts, event_id, utid, {}, {}, ucpu}) + .id; auto inserter = context_->args_tracker->AddArgsTo(id); for (auto it = evt.field(); it; ++it) { @@ -1404,15 +1416,12 @@ void FtraceParser::ParseTypedFtraceToRaw( FtraceMessageDescriptor* m = GetMessageDescriptorForId(ftrace_id); const auto& message_strings = ftrace_message_strings_[ftrace_id]; UniqueTid utid = context_->process_tracker->GetOrCreateThread(tid); - RawId id = context_->storage->mutable_ftrace_event_table() - ->Insert({timestamp, - message_strings.message_name_id, - cpu, - utid, - {}, - {}, - context_->machine_id()}) - .id; + auto ucpu = context_->cpu_tracker->GetOrCreateCpu(cpu); + RawId id = + context_->storage->mutable_ftrace_event_table() + ->Insert( + {timestamp, message_strings.message_name_id, utid, {}, {}, ucpu}) + .id; auto inserter = context_->args_tracker->AddArgsTo(id); for (auto fld = decoder.ReadField(); fld.valid(); fld = decoder.ReadField()) { @@ -1542,6 +1551,16 @@ void FtraceParser::ParseCpuFreq(int64_t timestamp, ConstBytes blob) { context_->event_tracker->PushCounter(timestamp, new_freq, track); } +void FtraceParser::ParseCpuFreqThrottle(int64_t timestamp, ConstBytes blob) { + protos::pbzero::DcvshFreqFtraceEvent::Decoder freq(blob.data, blob.size); + uint32_t cpu = static_cast<uint32_t>(freq.cpu()); + // Source data is frequency / 1000, so we correct that here: + double new_freq = static_cast<double>(freq.freq()) * 1000.0; + TrackId track = context_->track_tracker->InternCpuCounterTrack( + cpu_freq_throttle_name_id_, cpu); + context_->event_tracker->PushCounter(timestamp, new_freq, track); +} + void FtraceParser::ParseGpuFreq(int64_t timestamp, ConstBytes blob) { protos::pbzero::GpuFrequencyFtraceEvent::Decoder freq(blob.data, blob.size); uint32_t gpu = freq.gpu_id(); @@ -1551,6 +1570,17 @@ void FtraceParser::ParseGpuFreq(int64_t timestamp, ConstBytes blob) { context_->event_tracker->PushCounter(timestamp, new_freq, track); } +void FtraceParser::ParseKgslGpuFreq(int64_t timestamp, ConstBytes blob) { + protos::pbzero::KgslGpuFrequencyFtraceEvent::Decoder freq(blob.data, + blob.size); + uint32_t gpu = freq.gpu_id(); + // Source data is frequency / 1000, so we correct that here: + double new_freq = static_cast<double>(freq.gpu_freq()) * 1000.0; + TrackId track = + context_->track_tracker->InternGpuCounterTrack(gpu_freq_name_id_, gpu); + context_->event_tracker->PushCounter(timestamp, new_freq, track); +} + void FtraceParser::ParseCpuIdle(int64_t timestamp, ConstBytes blob) { protos::pbzero::CpuIdleFtraceEvent::Decoder idle(blob.data, blob.size); uint32_t cpu = idle.cpu_id(); diff --git a/src/trace_processor/importers/ftrace/ftrace_parser.h b/src/trace_processor/importers/ftrace/ftrace_parser.h index d8bafde94..2520572de 100644 --- a/src/trace_processor/importers/ftrace/ftrace_parser.h +++ b/src/trace_processor/importers/ftrace/ftrace_parser.h @@ -74,7 +74,9 @@ class FtraceParser { void ParseSchedWaking(int64_t timestamp, uint32_t pid, protozero::ConstBytes); void ParseSchedProcessFree(int64_t timestamp, protozero::ConstBytes); void ParseCpuFreq(int64_t timestamp, protozero::ConstBytes); + void ParseCpuFreqThrottle(int64_t timestamp, protozero::ConstBytes); void ParseGpuFreq(int64_t timestamp, protozero::ConstBytes); + void ParseKgslGpuFreq(int64_t timestamp, protozero::ConstBytes); void ParseCpuIdle(int64_t timestamp, protozero::ConstBytes); void ParsePrint(int64_t timestamp, uint32_t pid, protozero::ConstBytes); void ParseZero(int64_t timestamp, uint32_t pid, protozero::ConstBytes); @@ -319,6 +321,7 @@ class FtraceParser { const StringId sched_waking_name_id_; const StringId cpu_id_; const StringId cpu_freq_name_id_; + const StringId cpu_freq_throttle_name_id_; const StringId gpu_freq_name_id_; const StringId cpu_idle_name_id_; const StringId suspend_resume_name_id_; diff --git a/src/trace_processor/importers/ftrace/ftrace_sched_event_tracker.cc b/src/trace_processor/importers/ftrace/ftrace_sched_event_tracker.cc index a7b1ced40..feb616c31 100644 --- a/src/trace_processor/importers/ftrace/ftrace_sched_event_tracker.cc +++ b/src/trace_processor/importers/ftrace/ftrace_sched_event_tracker.cc @@ -238,10 +238,9 @@ void FtraceSchedEventTracker::PushSchedWakingCompact(uint32_t cpu, tables::FtraceEventTable::Row row; row.ts = ts; row.name = sched_waking_id_; - row.cpu = cpu; row.utid = curr_utid; row.common_flags = common_flags; - row.machine_id = context_->machine_id(); + row.ucpu = context_->cpu_tracker->GetOrCreateCpu(cpu); // Add an entry to the raw table. RawId id = context_->storage->mutable_ftrace_event_table()->Insert(row).id; @@ -280,14 +279,9 @@ void FtraceSchedEventTracker::AddRawSchedSwitchEvent(uint32_t cpu, if (PERFETTO_LIKELY(context_->config.ingest_ftrace_in_raw_table)) { // Push the raw event - this is done as the raw ftrace event codepath does // not insert sched_switch. + auto ucpu = context_->cpu_tracker->GetOrCreateCpu(cpu); RawId id = context_->storage->mutable_ftrace_event_table() - ->Insert({ts, - sched_switch_id_, - cpu, - prev_utid, - {}, - {}, - context_->machine_id()}) + ->Insert({ts, sched_switch_id_, prev_utid, {}, {}, ucpu}) .id; // Note: this ordering is important. The events should be pushed in the same diff --git a/src/trace_processor/importers/ftrace/ftrace_sched_event_tracker_unittest.cc b/src/trace_processor/importers/ftrace/ftrace_sched_event_tracker_unittest.cc index a085078cd..8becd2ed9 100644 --- a/src/trace_processor/importers/ftrace/ftrace_sched_event_tracker_unittest.cc +++ b/src/trace_processor/importers/ftrace/ftrace_sched_event_tracker_unittest.cc @@ -18,9 +18,11 @@ #include "perfetto/base/logging.h" #include "src/trace_processor/importers/common/args_tracker.h" +#include "src/trace_processor/importers/common/cpu_tracker.h" #include "src/trace_processor/importers/common/event_tracker.h" -#include "src/trace_processor/importers/common/sched_event_tracker.h" +#include "src/trace_processor/importers/common/machine_tracker.h" #include "src/trace_processor/importers/common/process_tracker.h" +#include "src/trace_processor/importers/common/sched_event_tracker.h" #include "test/gtest_and_gmock.h" namespace perfetto { @@ -40,6 +42,8 @@ class SchedEventTrackerTest : public ::testing::Test { context.args_tracker.reset(new ArgsTracker(&context)); context.event_tracker.reset(new EventTracker(&context)); context.process_tracker.reset(new ProcessTracker(&context)); + context.machine_tracker.reset(new MachineTracker(&context, 0)); + context.cpu_tracker.reset(new CpuTracker(&context)); context.sched_event_tracker.reset(new SchedEventTracker(&context)); sched_tracker = FtraceSchedEventTracker::GetOrCreate(&context); } diff --git a/src/trace_processor/importers/ftrace/ftrace_tokenizer.cc b/src/trace_processor/importers/ftrace/ftrace_tokenizer.cc index 02209863b..1e61a465f 100644 --- a/src/trace_processor/importers/ftrace/ftrace_tokenizer.cc +++ b/src/trace_processor/importers/ftrace/ftrace_tokenizer.cc @@ -271,7 +271,8 @@ void FtraceTokenizer::TokenizeFtraceEvent( } else if (PERFETTO_UNLIKELY(event_id == protos::pbzero::FtraceEvent:: kThermalExynosAcpmBulkFieldNumber)) { - TokenizeFtraceThermalExynosAcpmBulk(cpu, std::move(event), std::move(state)); + TokenizeFtraceThermalExynosAcpmBulk(cpu, std::move(event), + std::move(state)); return; } @@ -475,7 +476,8 @@ void FtraceTokenizer::TokenizeFtraceGpuWorkPeriod( } void FtraceTokenizer::TokenizeFtraceThermalExynosAcpmBulk( - uint32_t cpu, TraceBlobView event, + uint32_t cpu, + TraceBlobView event, RefPtr<PacketSequenceStateGeneration> state) { // Special handling of valid thermal_exynos_acpm_bulk tracepoint events which // contains the right timestamp value nested inside the event data. diff --git a/src/trace_processor/importers/ftrace/ftrace_tokenizer.h b/src/trace_processor/importers/ftrace/ftrace_tokenizer.h index e0b4f7c95..6aff47da1 100644 --- a/src/trace_processor/importers/ftrace/ftrace_tokenizer.h +++ b/src/trace_processor/importers/ftrace/ftrace_tokenizer.h @@ -64,9 +64,10 @@ class FtraceTokenizer { void TokenizeFtraceGpuWorkPeriod(uint32_t cpu, TraceBlobView event, RefPtr<PacketSequenceStateGeneration> state); - void TokenizeFtraceThermalExynosAcpmBulk(uint32_t cpu, - TraceBlobView event, - RefPtr<PacketSequenceStateGeneration> state); + void TokenizeFtraceThermalExynosAcpmBulk( + uint32_t cpu, + TraceBlobView event, + RefPtr<PacketSequenceStateGeneration> state); void DlogWithLimit(const base::Status& status) { static std::atomic<uint32_t> dlog_count(0); diff --git a/src/trace_processor/importers/ftrace/thermal_tracker.cc b/src/trace_processor/importers/ftrace/thermal_tracker.cc index 4e1e367f5..f2fe7bb16 100644 --- a/src/trace_processor/importers/ftrace/thermal_tracker.cc +++ b/src/trace_processor/importers/ftrace/thermal_tracker.cc @@ -14,12 +14,12 @@ * limitations under the License. */ +#include "src/trace_processor/importers/ftrace/thermal_tracker.h" #include "perfetto/ext/base/string_utils.h" #include "protos/perfetto/trace/ftrace/thermal.pbzero.h" #include "protos/perfetto/trace/ftrace/thermal_exynos.pbzero.h" #include "src/trace_processor/importers/common/event_tracker.h" #include "src/trace_processor/importers/common/track_tracker.h" -#include "src/trace_processor/importers/ftrace/thermal_tracker.h" namespace perfetto { namespace trace_processor { @@ -84,7 +84,8 @@ void ThermalTracker::ParseThermalExynosAcpmBulk(protozero::ConstBytes blob) { } void ThermalTracker::ParseThermalExynosAcpmHighOverhead( - int64_t timestamp, protozero::ConstBytes blob) { + int64_t timestamp, + protozero::ConstBytes blob) { protos::pbzero::ThermalExynosAcpmHighOverheadFtraceEvent::Decoder event(blob); auto tz_id = static_cast<size_t>(event.tz_id()); if (tz_id >= kAcpmThermalZones) { @@ -100,7 +101,8 @@ void ThermalTracker::ParseThermalExynosAcpmHighOverhead( static_cast<double>(event.cdev_state())); } -void ThermalTracker::PushCounter(int64_t timestamp, StringId counter_id, +void ThermalTracker::PushCounter(int64_t timestamp, + StringId counter_id, double value) { TrackId track = context_->track_tracker->InternGlobalCounterTrack( TrackTracker::Group::kThermals, counter_id); diff --git a/src/trace_processor/importers/fuchsia/fuchsia_parser_unittest.cc b/src/trace_processor/importers/fuchsia/fuchsia_parser_unittest.cc index ca02bdcb0..740640873 100644 --- a/src/trace_processor/importers/fuchsia/fuchsia_parser_unittest.cc +++ b/src/trace_processor/importers/fuchsia/fuchsia_parser_unittest.cc @@ -14,25 +14,29 @@ * limitations under the License. */ -#include "src/trace_processor/importers/common/trace_parser.h" #include "src/trace_processor/importers/fuchsia/fuchsia_trace_parser.h" -#include "src/trace_processor/importers/fuchsia/fuchsia_trace_tokenizer.h" #include "perfetto/base/logging.h" #include "perfetto/ext/base/string_view.h" #include "perfetto/protozero/scattered_heap_buffer.h" #include "perfetto/trace_processor/trace_blob.h" + #include "src/trace_processor/importers/common/args_tracker.h" #include "src/trace_processor/importers/common/args_translation_table.h" #include "src/trace_processor/importers/common/clock_tracker.h" +#include "src/trace_processor/importers/common/cpu_tracker.h" #include "src/trace_processor/importers/common/event_tracker.h" #include "src/trace_processor/importers/common/flow_tracker.h" +#include "src/trace_processor/importers/common/machine_tracker.h" #include "src/trace_processor/importers/common/metadata_tracker.h" +#include "src/trace_processor/importers/common/process_track_translation_table.h" #include "src/trace_processor/importers/common/process_tracker.h" #include "src/trace_processor/importers/common/slice_tracker.h" #include "src/trace_processor/importers/common/stack_profile_tracker.h" +#include "src/trace_processor/importers/common/trace_parser.h" #include "src/trace_processor/importers/common/track_tracker.h" #include "src/trace_processor/importers/ftrace/ftrace_sched_event_tracker.h" +#include "src/trace_processor/importers/fuchsia/fuchsia_trace_tokenizer.h" #include "src/trace_processor/importers/proto/additional_modules.h" #include "src/trace_processor/importers/proto/default_modules.h" #include "src/trace_processor/importers/proto/proto_trace_parser_impl.h" @@ -242,12 +246,16 @@ class FuchsiaTraceParserTest : public ::testing::Test { context_.args_translation_table.reset(new ArgsTranslationTable(storage_)); context_.metadata_tracker.reset( new MetadataTracker(context_.storage.get())); + context_.machine_tracker.reset(new MachineTracker(&context_, 0)); + context_.cpu_tracker.reset(new CpuTracker(&context_)); event_ = new MockEventTracker(&context_); context_.event_tracker.reset(event_); sched_ = new MockSchedEventTracker(&context_); context_.ftrace_sched_tracker.reset(sched_); process_ = new NiceMock<MockProcessTracker>(&context_); context_.process_tracker.reset(process_); + context_.process_track_translation_table.reset( + new ProcessTrackTranslationTable(storage_)); slice_ = new NiceMock<MockSliceTracker>(&context_); context_.slice_tracker.reset(slice_); context_.slice_translation_table.reset(new SliceTranslationTable(storage_)); diff --git a/src/trace_processor/importers/fuchsia/fuchsia_trace_tokenizer.cc b/src/trace_processor/importers/fuchsia/fuchsia_trace_tokenizer.cc index 0cbd4f787..a668c42d3 100644 --- a/src/trace_processor/importers/fuchsia/fuchsia_trace_tokenizer.cc +++ b/src/trace_processor/importers/fuchsia/fuchsia_trace_tokenizer.cc @@ -22,6 +22,7 @@ #include "perfetto/base/logging.h" #include "perfetto/ext/base/string_view.h" #include "perfetto/trace_processor/trace_blob.h" +#include "src/trace_processor/importers/common/cpu_tracker.h" #include "src/trace_processor/importers/common/process_tracker.h" #include "src/trace_processor/importers/common/slice_tracker.h" #include "src/trace_processor/importers/fuchsia/fuchsia_record.h" @@ -257,7 +258,7 @@ void FuchsiaTraceTokenizer::SwitchFrom(Thread* thread, // state. tables::ThreadStateTable::Row state_row; state_row.ts = ts; - state_row.cpu = cpu; + state_row.ucpu = context_->cpu_tracker->GetOrCreateCpu(cpu); state_row.dur = -1; state_row.state = state; state_row.utid = utid; @@ -287,10 +288,11 @@ void FuchsiaTraceTokenizer::SwitchTo(Thread* thread, thread->last_state_row.reset(); } + auto ucpu = context_->cpu_tracker->GetOrCreateCpu(cpu); // Open a new slice record for this thread. tables::SchedSliceTable::Row slice_row; slice_row.ts = ts; - slice_row.cpu = cpu; + slice_row.ucpu = ucpu; slice_row.dur = -1; slice_row.utid = utid; slice_row.priority = weight; @@ -301,7 +303,7 @@ void FuchsiaTraceTokenizer::SwitchTo(Thread* thread, // Open a new state record for this thread. tables::ThreadStateTable::Row state_row; state_row.ts = ts; - state_row.cpu = cpu; + state_row.ucpu = context_->cpu_tracker->GetOrCreateCpu(cpu); state_row.dur = -1; state_row.state = running_string_id_; state_row.utid = utid; @@ -331,7 +333,7 @@ void FuchsiaTraceTokenizer::Wake(Thread* thread, int64_t ts, uint32_t cpu) { // Open a new state record for this thread. tables::ThreadStateTable::Row state_row; state_row.ts = ts; - state_row.cpu = cpu; + state_row.ucpu = context_->cpu_tracker->GetOrCreateCpu(cpu); state_row.dur = -1; state_row.state = waking_string_id_; state_row.utid = utid; diff --git a/src/trace_processor/importers/perf/BUILD.gn b/src/trace_processor/importers/perf/BUILD.gn index 376f9b420..da4153a6d 100644 --- a/src/trace_processor/importers/perf/BUILD.gn +++ b/src/trace_processor/importers/perf/BUILD.gn @@ -16,6 +16,8 @@ import("../../../../gn/test.gni") source_set("record") { sources = [ + "perf_counter.cc", + "perf_counter.h", "perf_event.h", "perf_event_attr.cc", "perf_event_attr.h", @@ -26,38 +28,64 @@ source_set("record") { ] deps = [ "../../../../gn:default_deps", + "../../../../include/perfetto/ext/base:base", + "../../../../include/perfetto/trace_processor:trace_processor", "../../../../protos/perfetto/trace/profiling:zero", - "../../sorter", "../../storage", "../../tables:tables_python", "../../types", - "../common", + "../../util:build_id", "../common:parser_types", ] } + +source_set("tracker") { + sources = [ + "dso_tracker.cc", + "dso_tracker.h", + ] + deps = [ + "../../../../gn:default_deps", + "../../../../include/perfetto/ext/base:base", + "../../../../protos/third_party/simpleperf:zero", + "../..//storage:storage", + "../..//tables:tables", + "../..//types:types", + "../common:common", + ] +} + source_set("perf") { sources = [ - "perf_data_parser.cc", - "perf_data_parser.h", - "perf_data_reader.cc", - "perf_data_reader.h", + "attrs_section_reader.cc", + "attrs_section_reader.h", + "features.cc", + "features.h", + "mmap_record.cc", + "mmap_record.h", "perf_data_tokenizer.cc", "perf_data_tokenizer.h", - "perf_data_tracker.cc", - "perf_data_tracker.h", "perf_file.h", - "reader.h", + "record_parser.cc", + "record_parser.h", + "sample.cc", + "sample.h", ] public_deps = [ ":record" ] deps = [ + ":tracker", "../../../../gn:default_deps", + "../../../../protos/perfetto/trace:zero", "../../../../protos/perfetto/trace/profiling:zero", + "../../../../protos/third_party/simpleperf:zero", "../../sorter", "../../storage", "../../tables:tables_python", "../../types", - "../common", - "../common:parser_types", + "../../util:build_id", + "../../util:file_buffer", + "../../util:util", + "../common:common", "../proto:minimal", ] } @@ -65,8 +93,6 @@ source_set("perf") { perfetto_unittest_source_set("unittests") { testonly = true sources = [ - "perf_data_reader_unittest.cc", - "perf_data_tracker_unittest.cc", "perf_session_unittest.cc", "reader_unittest.cc", ] @@ -76,6 +102,8 @@ perfetto_unittest_source_set("unittests") { "../../../../gn:gtest_and_gmock", "../../../../protos/perfetto/trace/profiling:zero", "../../../base", - "../../importers/common", + "../../storage", + "../../types", + "../common", ] } diff --git a/src/trace_processor/importers/perf/attrs_section_reader.cc b/src/trace_processor/importers/perf/attrs_section_reader.cc new file mode 100644 index 000000000..f19ea3efc --- /dev/null +++ b/src/trace_processor/importers/perf/attrs_section_reader.cc @@ -0,0 +1,76 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/trace_processor/importers/perf/attrs_section_reader.h" + +#include <cinttypes> + +#include "perfetto/base/logging.h" +#include "perfetto/base/status.h" +#include "perfetto/trace_processor/trace_blob_view.h" +#include "src/trace_processor/importers/perf/perf_file.h" + +namespace perfetto::trace_processor::perf_importer { + +// static +base::StatusOr<AttrsSectionReader> AttrsSectionReader::Create( + const PerfFile::Header& header, + TraceBlobView section) { + PERFETTO_CHECK(section.size() == header.attrs.size); + + if (header.attr_size == 0) { + return base::ErrStatus("Invalid attr_size (0) in perf file header."); + } + + if (header.attrs.size % header.attr_size != 0) { + return base::ErrStatus("Invalid attrs section size %" PRIu64 + " for attr_size %" PRIu64 " in perf file header.", + header.attrs.size, header.attr_size); + } + + const size_t num_attr = header.attrs.size / header.attr_size; + + // Each entry is a perf_event_attr followed by a Section, but the size of + // the perf_event_attr struct written in the file might not be the same as + // sizeof(perf_event_attr) as this struct might grow over time (can be + // bigger or smaller). + static constexpr size_t kSectionSize = sizeof(PerfFile::Section); + if (header.attr_size < kSectionSize) { + return base::ErrStatus( + "Invalid attr_size in file header. Expected at least %zu, found " + "%" PRIu64, + kSectionSize, header.attr_size); + } + const size_t attr_size = header.attr_size - kSectionSize; + + return AttrsSectionReader(std::move(section), num_attr, attr_size); +} + +base::Status AttrsSectionReader::ReadNext(PerfFile::AttrsEntry& entry) { + PERFETTO_CHECK(reader_.ReadPerfEventAttr(entry.attr, attr_size_)); + + if (entry.attr.size != attr_size_) { + return base::ErrStatus( + "Invalid attr.size. Expected %zu, but found %" PRIu32, attr_size_, + entry.attr.size); + } + + PERFETTO_CHECK(reader_.Read(entry.ids)); + --num_attr_; + return base::OkStatus(); +} + +} // namespace perfetto::trace_processor::perf_importer diff --git a/src/trace_processor/importers/perf/attrs_section_reader.h b/src/trace_processor/importers/perf/attrs_section_reader.h new file mode 100644 index 000000000..a7eaaa3e7 --- /dev/null +++ b/src/trace_processor/importers/perf/attrs_section_reader.h @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SRC_TRACE_PROCESSOR_IMPORTERS_PERF_ATTRS_SECTION_READER_H_ +#define SRC_TRACE_PROCESSOR_IMPORTERS_PERF_ATTRS_SECTION_READER_H_ + +#include "perfetto/ext/base/status_or.h" +#include "perfetto/trace_processor/trace_blob_view.h" +#include "src/trace_processor/importers/perf/perf_file.h" +#include "src/trace_processor/importers/perf/reader.h" + +namespace perfetto::trace_processor::perf_importer { + +// Helper to read the attrs section of a perf file. Provides an iterator like +// interface over the perf_event_attr entries. +class AttrsSectionReader { + public: + // Creates a new iterator. + // `attrs_section` data contained in the attrs section of the perf file. + static base::StatusOr<AttrsSectionReader> Create( + const PerfFile::Header& header, + TraceBlobView attrs_section); + + // Returns true while there are available entries to read via `ReadNext`. + bool CanReadNext() const { return num_attr_ != 0; } + + // Reads the next entry. Can onlybe called if `HasMore` returns true. + base::Status ReadNext(PerfFile::AttrsEntry& entry); + + private: + AttrsSectionReader(TraceBlobView section, size_t num_attr, size_t attr_size) + : reader_(std::move(section)), + num_attr_(num_attr), + attr_size_(attr_size) {} + + Reader reader_; + size_t num_attr_; + const size_t attr_size_; +}; + +} // namespace perfetto::trace_processor::perf_importer + +#endif // SRC_TRACE_PROCESSOR_IMPORTERS_PERF_ATTRS_SECTION_READER_H_ diff --git a/src/trace_processor/importers/perf/dso_tracker.cc b/src/trace_processor/importers/perf/dso_tracker.cc new file mode 100644 index 000000000..3ff3cefff --- /dev/null +++ b/src/trace_processor/importers/perf/dso_tracker.cc @@ -0,0 +1,134 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/trace_processor/importers/perf/dso_tracker.h" + +#include <cstdint> + +#include "perfetto/base/status.h" +#include "perfetto/ext/base/string_view.h" +#include "protos/third_party/simpleperf/record_file.pbzero.h" +#include "src/trace_processor/storage/trace_storage.h" +#include "src/trace_processor/tables/profiler_tables_py.h" +#include "src/trace_processor/types/trace_processor_context.h" + +namespace perfetto::trace_processor::perf_importer { +namespace { + +using third_party::simpleperf::proto::pbzero::FileFeature; +using DexFile = FileFeature::DexFile; +using ElfFile = FileFeature::ElfFile; +using KernelModule = FileFeature::KernelModule; +using DsoType = FileFeature::DsoType; +using Symbol = FileFeature::Symbol; + +void InsertSymbols(const FileFeature::Decoder& file, + AddressRangeMap<std::string>& out) { + for (auto raw_symbol = file.symbol(); raw_symbol; ++raw_symbol) { + Symbol::Decoder symbol(*raw_symbol); + out.DeleteOverlapsAndEmplace( + AddressRange::FromStartAndSize(symbol.vaddr(), symbol.len()), + symbol.name().ToStdString()); + } +} +} // namespace + +DsoTracker::DsoTracker(TraceProcessorContext* context) + : context_(context), + mapping_table_(context_->storage->stack_profile_mapping_table()) {} +DsoTracker::~DsoTracker() = default; + +void DsoTracker::AddSimpleperfFile2(const FileFeature::Decoder& file) { + Dso dso; + switch (file.type()) { + case DsoType::DSO_KERNEL: + InsertSymbols(file, kernel_symbols_); + return; + + case DsoType::DSO_ELF_FILE: { + ElfFile::Decoder elf(file.elf_file()); + dso.load_bias = file.min_vaddr() - elf.file_offset_of_min_vaddr(); + break; + } + + case DsoType::DSO_KERNEL_MODULE: { + KernelModule::Decoder module(file.kernel_module()); + dso.load_bias = file.min_vaddr() - module.memory_offset_of_min_vaddr(); + break; + } + + case DsoType::DSO_DEX_FILE: + case DsoType::DSO_SYMBOL_MAP_FILE: + case DsoType::DSO_UNKNOWN_FILE: + return; + } + + InsertSymbols(file, dso.symbols); + files_.Insert(context_->storage->InternString(file.path()), std::move(dso)); +} + +void DsoTracker::SymbolizeFrames() { + const StringId kEmptyString = context_->storage->InternString(""); + for (auto frame = context_->storage->mutable_stack_profile_frame_table() + ->IterateRows(); + frame; ++frame) { + if (frame.name() != kNullStringId && frame.name() != kEmptyString) { + continue; + } + + if (!TrySymbolizeFrame(frame.row_reference())) { + SymbolizeKernelFrame(frame.row_reference()); + } + } +} + +void DsoTracker::SymbolizeKernelFrame( + tables::StackProfileFrameTable::RowReference frame) { + const auto mapping = *mapping_table_.FindById(frame.mapping()); + uint64_t address = static_cast<uint64_t>(frame.rel_pc()) + + static_cast<uint64_t>(mapping.start()); + auto symbol = kernel_symbols_.Find(address); + if (symbol == kernel_symbols_.end()) { + return; + } + frame.set_name( + context_->storage->InternString(base::StringView(symbol->second))); +} + +bool DsoTracker::TrySymbolizeFrame( + tables::StackProfileFrameTable::RowReference frame) { + const auto mapping = *mapping_table_.FindById(frame.mapping()); + auto* file = files_.Find(mapping.name()); + if (!file) { + return false; + } + + // Load bias is something we can only determine by looking at the actual elf + // file. Thus PERF_RECORD_MMAP{2} events do not record it. So we need to + // potentially do an adjustment here if the load_bias tracked in the mapping + // table and the one reported by the file are mismatched. + uint64_t adj = file->load_bias - static_cast<uint64_t>(mapping.load_bias()); + + auto symbol = file->symbols.Find(static_cast<uint64_t>(frame.rel_pc()) + adj); + if (symbol == file->symbols.end()) { + return false; + } + frame.set_name( + context_->storage->InternString(base::StringView(symbol->second))); + return true; +} + +} // namespace perfetto::trace_processor::perf_importer diff --git a/src/trace_processor/importers/perf/dso_tracker.h b/src/trace_processor/importers/perf/dso_tracker.h new file mode 100644 index 000000000..314d54935 --- /dev/null +++ b/src/trace_processor/importers/perf/dso_tracker.h @@ -0,0 +1,78 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SRC_TRACE_PROCESSOR_IMPORTERS_PERF_DSO_TRACKER_H_ +#define SRC_TRACE_PROCESSOR_IMPORTERS_PERF_DSO_TRACKER_H_ + +#include <cstdint> +#include <string> + +#include "perfetto/base/status.h" +#include "perfetto/ext/base/flat_hash_map.h" +#include "protos/third_party/simpleperf/record_file.pbzero.h" +#include "src/trace_processor/importers/common/address_range.h" +#include "src/trace_processor/storage/trace_storage.h" +#include "src/trace_processor/tables/profiler_tables_py.h" +#include "src/trace_processor/types/destructible.h" +#include "src/trace_processor/types/trace_processor_context.h" + +namespace perfetto::trace_processor::perf_importer { + +// Keeps track of DSO symbols to symbolize frames at the end of the trace +// parsing. +// TODO(b/334978369): We could potentially use this class (or a similar one) to +// process the ModuleSymbols proto packets and consolidate all symbolization in +// one place. +class DsoTracker : public Destructible { + public: + static DsoTracker& GetOrCreate(TraceProcessorContext* context) { + if (!context->perf_dso_tracker) { + context->perf_dso_tracker.reset(new DsoTracker(context)); + } + return static_cast<DsoTracker&>(*context->perf_dso_tracker); + } + ~DsoTracker() override; + + // Add symbol data contained in a `FileFeature` proto. + void AddSimpleperfFile2( + const third_party::simpleperf::proto::pbzero::FileFeature::Decoder& file); + + // Tries to symbolize any `STACK_PROFILE_FRAME` frame missing the `name` + // attribute. This should be called at the end of parsing when all packets + // have been processed and all tables updated. + void SymbolizeFrames(); + + private: + struct Dso { + uint64_t load_bias; + AddressRangeMap<std::string> symbols; + }; + + explicit DsoTracker(TraceProcessorContext* context); + + void SymbolizeKernelFrame(tables::StackProfileFrameTable::RowReference frame); + // Returns true it the frame was symbolized. + bool TrySymbolizeFrame(tables::StackProfileFrameTable::RowReference frame); + + TraceProcessorContext* const context_; + const tables::StackProfileMappingTable& mapping_table_; + base::FlatHashMap<StringId, Dso> files_; + AddressRangeMap<std::string> kernel_symbols_; +}; + +} // namespace perfetto::trace_processor::perf_importer + +#endif // SRC_TRACE_PROCESSOR_IMPORTERS_PERF_DSO_TRACKER_H_ diff --git a/src/trace_processor/importers/perf/features.cc b/src/trace_processor/importers/perf/features.cc new file mode 100644 index 000000000..877f2fd09 --- /dev/null +++ b/src/trace_processor/importers/perf/features.cc @@ -0,0 +1,296 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/trace_processor/importers/perf/features.h" + +#include <cstdint> +#include <utility> + +#include "perfetto/base/logging.h" +#include "perfetto/base/status.h" +#include "perfetto/ext/base/string_utils.h" +#include "perfetto/ext/base/string_view.h" +#include "perfetto/trace_processor/status.h" +#include "perfetto/trace_processor/trace_blob_view.h" +#include "src/trace_processor/importers/perf/perf_event.h" +#include "src/trace_processor/importers/perf/reader.h" +#include "src/trace_processor/util/status_macros.h" + +namespace perfetto::trace_processor::perf_importer::feature { +namespace { + +struct BuildIdRecord { + static constexpr uint8_t kMaxSize = 20; + char data[kMaxSize]; + uint8_t size; + uint8_t reserved[3]; +}; + +uint8_t CountTrailingZeros(const BuildIdRecord& build_id) { + for (uint8_t i = 0; i < BuildIdRecord::kMaxSize; ++i) { + if (build_id.data[BuildIdRecord::kMaxSize - i - 1] != 0) { + return i; + } + } + return sizeof(build_id.data); +} + +// BuildIds are usually SHA-1 hashes (20 bytes), sometimes MD5 (16 bytes), +// sometimes 8 bytes long. Simpleperf adds trailing zeros up to 20. Do a best +// guess based on the number of trailing zeros. +uint8_t GuessBuildIdSize(const BuildIdRecord& build_id) { + static_assert(BuildIdRecord::kMaxSize == 20); + uint8_t len = BuildIdRecord::kMaxSize - CountTrailingZeros(build_id); + if (len > 16) { + return BuildIdRecord::kMaxSize; + } + if (len > 8) { + return 16; + } + return 8; +} + +bool ParseString(Reader& reader, std::string& out) { + uint32_t len; + base::StringView str; + if (!reader.Read(len) || len == 0 || !reader.ReadStringView(str, len)) { + return false; + } + + if (str.at(len - 1) != '\0') { + return false; + } + + // Strings are padded with null values, stop at first null + out = std::string(str.data()); + return true; +} + +bool ParseBuildId(const perf_event_header& header, + TraceBlobView blob, + BuildId& out) { + Reader reader(std::move(blob)); + + BuildIdRecord build_id; + + if (!reader.Read(out.pid) || !reader.Read(build_id) || + !reader.ReadStringUntilEndOrNull(out.filename)) { + return false; + } + + if (header.misc & PERF_RECORD_MISC_EXT_RESERVED) { + if (build_id.size > BuildIdRecord::kMaxSize) { + return false; + } + } else { + // Probably a simpleperf trace. Simpleperf fills build_ids with zeros up + // to a length of 20 and leaves the rest uninitialized :( so we can not read + // build_id.size or build_id.reserved to do any checks. + // TODO(b/334978369): We should be able to tell for sure whether this is + // simpleperf or not by checking the existence of SimpleperfMetaInfo. + build_id.size = GuessBuildIdSize(build_id); + } + out.build_id = std::string(build_id.data, build_id.size); + return true; +} + +util::Status ParseEventTypeInfo(std::string value, SimpleperfMetaInfo& out) { + for (const auto& line : base::SplitString(value, "\n")) { + auto tokens = base::SplitString(line, ","); + if (tokens.size() != 3) { + return util::ErrStatus("Invalid event_type_info: '%s'", line.c_str()); + } + + auto type = base::StringToUInt32(tokens[1]); + if (!type) { + return util::ErrStatus("Could not parse type in event_type_info: '%s'", + tokens[1].c_str()); + } + auto config = base::StringToUInt64(tokens[2]); + if (!config) { + return util::ErrStatus("Could not parse config in event_type_info: '%s'", + tokens[2].c_str()); + } + + out.event_type_info.Insert({*type, *config}, std::move(tokens[0])); + } + + return util::OkStatus(); +} + +util::Status ParseSimpleperfMetaInfoEntry( + std::pair<std::string, std::string> entry, + SimpleperfMetaInfo& out) { + static constexpr char kEventTypeInfoKey[] = "event_type_info"; + if (entry.first == kEventTypeInfoKey) { + return ParseEventTypeInfo(std::move(entry.second), out); + } + + PERFETTO_CHECK( + out.entries.Insert(std::move(entry.first), std::move(entry.second)) + .second); + return util::OkStatus(); +} + +} // namespace + +// static +util::Status BuildId::Parse(TraceBlobView bytes, + std::function<util::Status(BuildId)> cb) { + Reader reader(std::move(bytes)); + while (reader.size_left() != 0) { + perf_event_header header; + TraceBlobView payload; + if (!reader.Read(header)) { + return base::ErrStatus( + "Failed to parse feature BuildId. Could not read header."); + } + if (header.size < sizeof(header)) { + return base::ErrStatus( + "Failed to parse feature BuildId. Invalid size in header."); + } + if (!reader.ReadBlob(payload, header.size - sizeof(header))) { + return base::ErrStatus( + "Failed to parse feature BuildId. Could not read payload."); + } + + BuildId build_id; + if (!ParseBuildId(header, std::move(payload), build_id)) { + return base::ErrStatus( + "Failed to parse feature BuildId. Could not read entry."); + } + + RETURN_IF_ERROR(cb(std::move(build_id))); + } + return util::OkStatus(); +} + +// static +util::Status SimpleperfMetaInfo::Parse(const TraceBlobView& bytes, + SimpleperfMetaInfo& out) { + auto* it_end = reinterpret_cast<const char*>(bytes.data() + bytes.size()); + for (auto* it = reinterpret_cast<const char*>(bytes.data()); it != it_end;) { + auto end = std::find(it, it_end, '\0'); + if (end == it_end) { + return util::ErrStatus("Failed to read key from Simpleperf MetaInfo"); + } + std::string key(it, end); + it = end; + ++it; + if (it == it_end) { + return util::ErrStatus("Missing value in Simpleperf MetaInfo"); + } + end = std::find(it, it_end, '\0'); + if (end == it_end) { + return util::ErrStatus("Failed to read value from Simpleperf MetaInfo"); + } + std::string value(it, end); + it = end; + ++it; + + RETURN_IF_ERROR(ParseSimpleperfMetaInfoEntry( + std::make_pair(std::move(key), std::move(value)), out)); + } + return util::OkStatus(); +} + +// static +util::Status EventDescription::Parse( + TraceBlobView bytes, + std::function<util::Status(EventDescription)> cb) { + Reader reader(std::move(bytes)); + uint32_t nr; + uint32_t attr_size; + if (!reader.Read(nr) || !reader.Read(attr_size)) { + return util::ErrStatus("Failed to parse header for PERF_EVENT_DESC"); + } + + for (; nr != 0; --nr) { + EventDescription desc; + uint32_t nr_ids; + if (!reader.ReadPerfEventAttr(desc.attr, attr_size) || + !reader.Read(nr_ids) || !ParseString(reader, desc.event_string)) { + return util::ErrStatus("Failed to parse record for PERF_EVENT_DESC"); + } + + desc.ids.resize(nr_ids); + for (uint64_t& id : desc.ids) { + if (!reader.Read(id)) { + return util::ErrStatus("Failed to parse ids for PERF_EVENT_DESC"); + } + } + RETURN_IF_ERROR(cb(std::move(desc))); + } + return util::OkStatus(); +} + +util::Status ParseSimpleperfFile2(TraceBlobView bytes, + std::function<void(TraceBlobView)> cb) { + Reader reader(std::move(bytes)); + while (reader.size_left() != 0) { + uint32_t len; + if (!reader.Read(len)) { + return base::ErrStatus("Failed to parse len in FEATURE_SIMPLEPERF_FILE2"); + } + TraceBlobView payload; + if (!reader.ReadBlob(payload, len)) { + return base::ErrStatus( + "Failed to parse payload in FEATURE_SIMPLEPERF_FILE2"); + } + cb(std::move(payload)); + } + return util::OkStatus(); +} + +// static +util::Status HeaderGroupDesc::Parse(TraceBlobView bytes, HeaderGroupDesc& out) { + Reader reader(std::move(bytes)); + uint32_t nr; + if (!reader.Read(nr)) { + return util::ErrStatus("Failed to parse header for HEADER_GROUP_DESC"); + } + + HeaderGroupDesc group_desc; + group_desc.entries.resize(nr); + for (auto& e : group_desc.entries) { + if (!ParseString(reader, e.string) || !reader.Read(e.leader_idx) || + !reader.Read(e.nr_members)) { + return util::ErrStatus("Failed to parse HEADER_GROUP_DESC entry"); + } + } + out = std::move(group_desc); + return base::OkStatus(); +} + +base::StatusOr<std::vector<std::string>> ParseCmdline(TraceBlobView bytes) { + Reader reader(std::move(bytes)); + uint32_t nr; + if (!reader.Read(nr)) { + return util::ErrStatus("Failed to parse nr for CMDLINE"); + } + + std::vector<std::string> args; + args.reserve(nr); + for (; nr != 0; --nr) { + args.emplace_back(); + if (!ParseString(reader, args.back())) { + return base::ErrStatus("Failed to parse string for CMDLINE"); + } + } + return std::move(args); +} + +} // namespace perfetto::trace_processor::perf_importer::feature diff --git a/src/trace_processor/importers/perf/features.h b/src/trace_processor/importers/perf/features.h new file mode 100644 index 000000000..e02895727 --- /dev/null +++ b/src/trace_processor/importers/perf/features.h @@ -0,0 +1,134 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SRC_TRACE_PROCESSOR_IMPORTERS_PERF_FEATURES_H_ +#define SRC_TRACE_PROCESSOR_IMPORTERS_PERF_FEATURES_H_ + +#include <cstdint> +#include <functional> +#include <limits> +#include <string> +#include <vector> + +#include "perfetto/ext/base/flat_hash_map.h" +#include "perfetto/ext/base/hash.h" +#include "perfetto/ext/base/status_or.h" +#include "perfetto/trace_processor/status.h" +#include "src/trace_processor/importers/perf/perf_event.h" + +namespace perfetto ::trace_processor { +class TraceBlobView; + +namespace perf_importer::feature { + +enum Id : uint8_t { + ID_RESERVED = 0, + ID_TRACING_DATA = 1, + ID_BUILD_ID = 2, + ID_HOSTNAME = 3, + ID_OS_RELEASE = 4, + ID_VERSION = 5, + ID_ARCH = 6, + ID_NR_CPUS = 7, + ID_CPU_DESC = 8, + ID_CPU_ID = 9, + ID_TOTAL_MEM = 10, + ID_CMD_LINE = 11, + ID_EVENT_DESC = 12, + ID_CPU_TOPOLOGY = 13, + ID_NUMA_TOPOLOGY = 14, + ID_BRANCH_STACK = 15, + ID_PMU_MAPPINGS = 16, + ID_GROUP_DESC = 17, + ID_AUX_TRACE = 18, + ID_STAT = 19, + ID_CACHE = 20, + ID_SAMPLE_TIME = 21, + ID_SAMPLE_TOPOLOGY = 22, + ID_CLOCK_ID = 23, + ID_DIR_FORMAT = 24, + ID_BPF_PROG_INFO = 25, + ID_BPF_BTF = 26, + ID_COMPRESSED = 27, + ID_CPU_PUM_CAPS = 28, + ID_CLOCK_DATA = 29, + ID_HYBRID_TOPOLOGY = 30, + ID_PMU_CAPS = 31, + ID_SIMPLEPERF_FILE = 128, + ID_SIMPLEPERF_META_INFO = 129, + ID_SIMPLEPERF_FILE2 = 132, + ID_MAX = std::numeric_limits<uint8_t>::max(), +}; + +struct BuildId { + static util::Status Parse(TraceBlobView, + std::function<util::Status(BuildId)> cb); + int32_t pid; + std::string build_id; + std::string filename; +}; + +struct HeaderGroupDesc { + static util::Status Parse(TraceBlobView, HeaderGroupDesc& out); + struct Entry { + std::string string; + uint32_t leader_idx; + uint32_t nr_members; + }; + std::vector<Entry> entries; +}; + +struct EventDescription { + static util::Status Parse(TraceBlobView, + std::function<util::Status(EventDescription)> cb); + perf_event_attr attr; + std::string event_string; + std::vector<uint64_t> ids; +}; + +struct SimpleperfMetaInfo { + static util::Status Parse(const TraceBlobView&, SimpleperfMetaInfo& out); + base::FlatHashMap<std::string, std::string> entries; + struct EventTypeAndConfig { + uint32_t type; + uint64_t config; + bool operator==(const EventTypeAndConfig& other) { + return type == other.type && config == other.config; + } + bool operator!=(const EventTypeAndConfig& other) { + return !(*this == other); + } + struct Hasher { + size_t operator()(const EventTypeAndConfig& o) const { + return static_cast<size_t>(base::Hasher::Combine(o.config, o.type)); + } + }; + }; + using EventName = std::string; + base::FlatHashMap<EventTypeAndConfig, EventName, EventTypeAndConfig::Hasher> + event_type_info; +}; + +util::Status ParseSimpleperfFile2(TraceBlobView, + std::function<void(TraceBlobView)> cb); + +base::StatusOr<std::vector<std::string>> ParseCmdline(TraceBlobView blob); + +} // namespace perf_importer::feature + +} // namespace perfetto::trace_processor + +#endif // SRC_TRACE_PROCESSOR_IMPORTERS_PERF_FEATURES_H_ diff --git a/src/trace_processor/importers/perf/mmap_record.cc b/src/trace_processor/importers/perf/mmap_record.cc new file mode 100644 index 000000000..d11fdd541 --- /dev/null +++ b/src/trace_processor/importers/perf/mmap_record.cc @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/trace_processor/importers/perf/mmap_record.h" + +#include <optional> + +#include "perfetto/base/status.h" +#include "src/trace_processor/importers/perf/reader.h" +#include "src/trace_processor/importers/perf/record.h" + +namespace perfetto::trace_processor::perf_importer { + +base::Status MmapRecord::Parse(const Record& record) { + Reader reader(record.payload.copy()); + if (!reader.Read(*static_cast<CommonMmapRecordFields*>(this)) || + !reader.ReadCString(filename)) { + return base::ErrStatus("Failed to parse MMAP record"); + } + cpu_mode = record.GetCpuMode(); + return base::OkStatus(); +} + +base::Status Mmap2Record::Parse(const Record& record) { + Reader reader(record.payload.copy()); + if (!reader.Read(*static_cast<BaseMmap2Record*>(this)) || + !reader.ReadCString(filename)) { + return base::ErrStatus("Failed to parse MMAP record"); + } + + has_build_id = record.mmap_has_build_id(); + + if (has_build_id && build_id.build_id_size > + BaseMmap2Record::BuildIdFields::kMaxBuildIdSize) { + return base::ErrStatus( + "Invalid build_id_size in MMAP2 record. Expected <= %zu but found " + "%" PRIu8, + BaseMmap2Record::BuildIdFields::kMaxBuildIdSize, + build_id.build_id_size); + } + + cpu_mode = record.GetCpuMode(); + + return base::OkStatus(); +} + +std::optional<BuildId> Mmap2Record::GetBuildId() const { + return has_build_id ? std::make_optional(BuildId::FromRaw(std::string( + build_id.build_id_buf, build_id.build_id_size))) + : std::nullopt; +} + +} // namespace perfetto::trace_processor::perf_importer diff --git a/src/trace_processor/importers/perf/mmap_record.h b/src/trace_processor/importers/perf/mmap_record.h new file mode 100644 index 000000000..37b393942 --- /dev/null +++ b/src/trace_processor/importers/perf/mmap_record.h @@ -0,0 +1,81 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SRC_TRACE_PROCESSOR_IMPORTERS_PERF_MMAP_RECORD_H_ +#define SRC_TRACE_PROCESSOR_IMPORTERS_PERF_MMAP_RECORD_H_ + +#include <cstdint> +#include <optional> +#include <string> +#include "perfetto/base/status.h" +#include "protos/perfetto/trace/profiling/profile_packet.pbzero.h" +#include "src/trace_processor/util/build_id.h" + +namespace perfetto::trace_processor::perf_importer { + +struct Record; + +struct CommonMmapRecordFields { + uint32_t pid; + uint32_t tid; + uint64_t addr; + uint64_t len; + uint64_t pgoff; +}; + +struct MmapRecord : public CommonMmapRecordFields { + std::string filename; + protos::pbzero::Profiling::CpuMode cpu_mode; + + base::Status Parse(const Record& record); +}; + +struct BaseMmap2Record : public CommonMmapRecordFields { + struct BuildIdFields { + static constexpr size_t kMaxBuildIdSize = 20; + uint8_t build_id_size; + uint8_t reserved_1; + uint16_t reserved_2; + char build_id_buf[kMaxBuildIdSize]; + }; + struct InodeFields { + uint32_t maj; + uint32_t min; + int64_t ino; + uint64_t ino_generation; + }; + static_assert(sizeof(BuildIdFields) == sizeof(InodeFields)); + + union { + BuildIdFields build_id; + InodeFields inode; + }; + uint32_t prot; + uint32_t flags; +}; + +struct Mmap2Record : public BaseMmap2Record { + std::string filename; + protos::pbzero::Profiling::CpuMode cpu_mode; + bool has_build_id; + + base::Status Parse(const Record& record); + std::optional<BuildId> GetBuildId() const; +}; + +} // namespace perfetto::trace_processor::perf_importer + +#endif // SRC_TRACE_PROCESSOR_IMPORTERS_PERF_MMAP_RECORD_H_ diff --git a/src/trace_processor/importers/perf/perf_counter.cc b/src/trace_processor/importers/perf/perf_counter.cc new file mode 100644 index 000000000..685e94040 --- /dev/null +++ b/src/trace_processor/importers/perf/perf_counter.cc @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/trace_processor/importers/perf/perf_counter.h" + +#include <cstdint> + +#include "perfetto/base/logging.h" +#include "src/trace_processor/tables/counter_tables_py.h" + +namespace perfetto::trace_processor::perf_importer { + +void PerfCounter::AddDelta(int64_t ts, double delta) { + last_count_ += delta; + counter_table_.Insert({ts, track_id_, last_count_}); +} + +void PerfCounter::AddCount(int64_t ts, double count) { + PERFETTO_CHECK(count >= last_count_); + last_count_ = count; + counter_table_.Insert({ts, track_id_, last_count_}); +} + +} // namespace perfetto::trace_processor::perf_importer diff --git a/src/trace_processor/importers/perf/perf_counter.h b/src/trace_processor/importers/perf/perf_counter.h new file mode 100644 index 000000000..173620ef8 --- /dev/null +++ b/src/trace_processor/importers/perf/perf_counter.h @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SRC_TRACE_PROCESSOR_IMPORTERS_PERF_PERF_COUNTER_H_ +#define SRC_TRACE_PROCESSOR_IMPORTERS_PERF_PERF_COUNTER_H_ + +#include <cstdint> + +#include "src/trace_processor/tables/counter_tables_py.h" +#include "src/trace_processor/tables/profiler_tables_py.h" + +namespace perfetto::trace_processor::perf_importer { + +// Helper class to keep track of perf counters and convert delta values found in +// perf files to absolute values needed for the perfetto counter table. +class PerfCounter { + public: + PerfCounter(tables::CounterTable* counter_table, + const tables::PerfCounterTrackTable::ConstRowReference& track) + : counter_table_(*counter_table), + track_id_(track.id()), + is_timebase_(track.is_timebase()) {} + + bool is_timebase() const { return is_timebase_; } + + void AddDelta(int64_t ts, double delta); + void AddCount(int64_t ts, double count); + + private: + tables::CounterTable& counter_table_; + tables::PerfCounterTrackTable::Id track_id_; + const bool is_timebase_; + double last_count_{0}; +}; + +} // namespace perfetto::trace_processor::perf_importer + +#endif // SRC_TRACE_PROCESSOR_IMPORTERS_PERF_PERF_COUNTER_H_ diff --git a/src/trace_processor/importers/perf/perf_data_parser.cc b/src/trace_processor/importers/perf/perf_data_parser.cc deleted file mode 100644 index e3dfef308..000000000 --- a/src/trace_processor/importers/perf/perf_data_parser.cc +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Copyright (C) 2023 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "src/trace_processor/importers/perf/perf_data_parser.h" - -#include <optional> -#include <string> -#include <vector> -#include "perfetto/base/logging.h" -#include "perfetto/ext/base/string_utils.h" -#include "perfetto/trace_processor/trace_blob_view.h" -#include "src/trace_processor/importers/common/mapping_tracker.h" -#include "src/trace_processor/importers/common/process_tracker.h" -#include "src/trace_processor/importers/perf/perf_data_reader.h" -#include "src/trace_processor/importers/perf/perf_data_tracker.h" -#include "src/trace_processor/storage/trace_storage.h" -#include "src/trace_processor/tables/profiler_tables_py.h" - -namespace perfetto { -namespace trace_processor { -namespace perf_importer { - -using FramesTable = tables::StackProfileFrameTable; -using CallsitesTable = tables::StackProfileCallsiteTable; - -PerfDataParser::PerfDataParser(TraceProcessorContext* context) - : context_(context), tracker_(PerfDataTracker::GetOrCreate(context_)) {} - -PerfDataParser::~PerfDataParser() = default; - -base::StatusOr<PerfDataTracker::PerfSample> PerfDataParser::ParseSample( - TraceBlobView tbv) { - perf_importer::PerfDataReader reader(std::move(tbv)); - return tracker_->ParseSample(reader); -} - -void PerfDataParser::ParsePerfRecord(int64_t ts, TraceBlobView tbv) { - auto sample_status = ParseSample(std::move(tbv)); - if (!sample_status.ok()) { - return; - } - PerfDataTracker::PerfSample sample = *sample_status; - - // The sample has been validated in tokenizer so callchain shouldn't be empty. - PERFETTO_CHECK(!sample.callchain.empty()); - - // First instruction pointer in the callchain should be from kernel space, so - // it shouldn't be available in mappings. - UniquePid upid = context_->process_tracker->GetOrCreateProcess(*sample.pid); - if (context_->mapping_tracker->FindUserMappingForAddress( - upid, sample.callchain.front())) { - context_->storage->IncrementStats(stats::perf_samples_skipped); - return; - } - - if (sample.callchain.size() == 1) { - context_->storage->IncrementStats(stats::perf_samples_skipped); - return; - } - - std::vector<FramesTable::Row> frame_rows; - for (uint32_t i = 1; i < sample.callchain.size(); i++) { - UserMemoryMapping* mapping = - context_->mapping_tracker->FindUserMappingForAddress( - upid, sample.callchain[i]); - if (!mapping) { - context_->storage->IncrementStats(stats::perf_samples_skipped); - return; - } - FramesTable::Row new_row; - std::string mock_name = - base::StackString<1024>( - "%" PRIu64, sample.callchain[i] - mapping->memory_range().start()) - .ToStdString(); - new_row.name = context_->storage->InternString(mock_name.c_str()); - new_row.mapping = mapping->mapping_id(); - new_row.rel_pc = - static_cast<int64_t>(mapping->ToRelativePc(sample.callchain[i])); - frame_rows.push_back(new_row); - } - - // Insert frames. We couldn't do it before as no frames should be added if the - // mapping couldn't be found for any of them. - const auto& frames = context_->storage->mutable_stack_profile_frame_table(); - std::vector<FramesTable::Id> frame_ids; - for (const auto& row : frame_rows) { - frame_ids.push_back(frames->Insert(row).id); - } - - // Insert callsites. - const auto& callsites = - context_->storage->mutable_stack_profile_callsite_table(); - - std::optional<CallsitesTable::Id> parent_callsite_id; - for (uint32_t i = 0; i < frame_ids.size(); i++) { - CallsitesTable::Row callsite_row; - callsite_row.frame_id = frame_ids[i]; - callsite_row.depth = i; - callsite_row.parent_id = parent_callsite_id; - parent_callsite_id = callsites->Insert(callsite_row).id; - } - - // Insert stack sample. - tables::PerfSampleTable::Row perf_sample_row; - perf_sample_row.callsite_id = parent_callsite_id; - perf_sample_row.ts = ts; - if (sample.cpu) { - perf_sample_row.cpu = *sample.cpu; - } - if (sample.tid) { - auto utid = context_->process_tracker->GetOrCreateThread(*sample.tid); - perf_sample_row.utid = utid; - } - context_->storage->mutable_perf_sample_table()->Insert(perf_sample_row); -} - -} // namespace perf_importer -} // namespace trace_processor -} // namespace perfetto diff --git a/src/trace_processor/importers/perf/perf_data_parser.h b/src/trace_processor/importers/perf/perf_data_parser.h deleted file mode 100644 index f2ab0a321..000000000 --- a/src/trace_processor/importers/perf/perf_data_parser.h +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (C) 2023 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef SRC_TRACE_PROCESSOR_IMPORTERS_PERF_PERF_DATA_PARSER_H_ -#define SRC_TRACE_PROCESSOR_IMPORTERS_PERF_PERF_DATA_PARSER_H_ - -#include <stdint.h> - -#include "perfetto/base/compiler.h" -#include "perfetto/trace_processor/trace_blob_view.h" -#include "src/trace_processor/importers/common/trace_parser.h" -#include "src/trace_processor/importers/perf/perf_data_tracker.h" - -namespace perfetto { -namespace trace_processor { -namespace perf_importer { - -// Parses samples from perf.data files. -class PerfDataParser : public PerfRecordParser { - public: - explicit PerfDataParser(TraceProcessorContext*); - ~PerfDataParser() override; - - // The data in TraceBlobView has to be a perf.data sample. - void ParsePerfRecord(int64_t timestamp, TraceBlobView) override; - - private: - base::StatusOr<PerfDataTracker::PerfSample> ParseSample(TraceBlobView); - - TraceProcessorContext* context_ = nullptr; - PerfDataTracker* tracker_ = nullptr; -}; - -} // namespace perf_importer -} // namespace trace_processor -} // namespace perfetto - -#endif // SRC_TRACE_PROCESSOR_IMPORTERS_PERF_PERF_DATA_PARSER_H_ diff --git a/src/trace_processor/importers/perf/perf_data_reader.cc b/src/trace_processor/importers/perf/perf_data_reader.cc deleted file mode 100644 index e0618266a..000000000 --- a/src/trace_processor/importers/perf/perf_data_reader.cc +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright (C) 2023 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "src/trace_processor/importers/perf/perf_data_reader.h" - -#include <cstddef> -#include <optional> -#include <vector> -#include "perfetto/base/logging.h" -#include "perfetto/trace_processor/trace_blob_view.h" - -namespace perfetto { -namespace trace_processor { -namespace perf_importer { -void PerfDataReader::SkipSlow(size_t bytes_to_skip) { - size_t bytes_in_buffer = BytesInBuffer(); - - // Size fits in buffer. - if (bytes_in_buffer >= bytes_to_skip) { - buffer_offset_ += bytes_to_skip; - return; - } - - // Empty the buffer and increase the |blob_offset_|. - buffer_offset_ = 0; - buffer_.clear(); - blob_offset_ += bytes_to_skip - bytes_in_buffer; -} - -void PerfDataReader::PeekSlow(uint8_t* obj_data, size_t size) const { - size_t bytes_in_buffer = BytesInBuffer(); - - // Read from buffer. - if (bytes_in_buffer >= size) { - memcpy(obj_data, buffer_.data() + buffer_offset_, size); - return; - } - - // Read from blob and buffer. - memcpy(obj_data, buffer_.data() + buffer_offset_, bytes_in_buffer); - memcpy(obj_data + bytes_in_buffer, tbv_.data() + blob_offset_, - size - bytes_in_buffer); -} - -TraceBlobView PerfDataReader::PeekTraceBlobViewSlow(size_t size) const { - auto blob = TraceBlob::Allocate(size); - size_t bytes_in_buffer = BytesInBuffer(); - - // Data is in buffer, so we need to create a new TraceBlob from it. - if (bytes_in_buffer >= size) { - memcpy(blob.data(), buffer_.data() + buffer_offset_, size); - return TraceBlobView(std::move(blob)); - } - - // Data is in between blob and buffer and we need to dump data from buffer - // and blob to a new TraceBlob. - size_t bytes_from_blob = size - bytes_in_buffer; - memcpy(blob.data(), buffer_.data() + buffer_offset_, bytes_in_buffer); - memcpy(blob.data() + bytes_in_buffer, tbv_.data() + blob_offset_, - bytes_from_blob); - return TraceBlobView(std::move(blob)); -} - -} // namespace perf_importer -} // namespace trace_processor -} // namespace perfetto diff --git a/src/trace_processor/importers/perf/perf_data_reader.h b/src/trace_processor/importers/perf/perf_data_reader.h deleted file mode 100644 index acf07d1e6..000000000 --- a/src/trace_processor/importers/perf/perf_data_reader.h +++ /dev/null @@ -1,187 +0,0 @@ -/* - * Copyright (C) 2023 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef SRC_TRACE_PROCESSOR_IMPORTERS_PERF_PERF_DATA_READER_H_ -#define SRC_TRACE_PROCESSOR_IMPORTERS_PERF_PERF_DATA_READER_H_ - -#include <stdint.h> -#include <cstddef> -#include <cstring> -#include <optional> -#include <vector> - -#include "perfetto/base/logging.h" -#include "perfetto/trace_processor/trace_blob.h" -#include "perfetto/trace_processor/trace_blob_view.h" - -namespace perfetto { -namespace trace_processor { -namespace perf_importer { - -// Reader class for tokenizing and parsing. Currently used by perf importer, but -// it's design is not related to perf. Responsible for hiding away the -// complexity of reading values from TraceBlobView and glueing the tbvs together -// in case there is data between many of them. -class PerfDataReader { - public: - PerfDataReader() = default; - explicit PerfDataReader(TraceBlobView tbv) : tbv_(std::move(tbv)) {} - - // Updates old TraceBlobView with new one. If there is data left in the old - // one, it will be saved in the buffer. - void Append(TraceBlobView tbv) { - uint64_t size_before = BytesAvailable(); - buffer_.insert(buffer_.end(), tbv_.data() + blob_offset_, - tbv_.data() + tbv_.size()); - tbv_ = std::move(tbv); - blob_offset_ = 0; - - // Post condition. Checks whether no data has been lost in the Append. - PERFETTO_DCHECK(BytesAvailable() == size_before + tbv.size()); - } - - // Reads the |obj| and updates |file_offset_| of the reader. - // NOTE: Assumes count of bytes available is higher than sizeof(T). - template <typename T> - void Read(T& obj) { - Peek(obj); - Skip<T>(); - } - - // Reads the T value for std::optional<T>. - // NOTE: Assumes count of bytes available is higher than sizeof(T). - template <typename T> - void ReadOptional(std::optional<T>& obj) { - T val; - Read(val); - obj = val; - } - - // Reads all of the data in the |vec| and updates |file_offset_| of the - // reader. - // NOTE: Assumes count of bytes available is higher than sizeof(T). - template <typename T> - void ReadVector(std::vector<T>& vec) { - PERFETTO_DCHECK(CanReadSize(sizeof(T) * vec.size())); - for (T& val : vec) { - Read(val); - } - } - - // Updates the |file_offset_| by the sizeof(T). - // NOTE: Assumes count of bytes available is higher than sizeof(T). - template <typename T> - void Skip() { - Skip(sizeof(T)); - } - - // Updates the |file_offset_| by the |bytes_to_skip|. - // NOTE: Assumes count of bytes available is higher than sizeof(T). - void Skip(uint64_t bytes_to_skip) { - uint64_t bytes_available_before = BytesAvailable(); - PERFETTO_DCHECK(CanReadSize(bytes_to_skip)); - size_t skip = static_cast<size_t>(bytes_to_skip); - - // Incrementing file offset is not related to the way data is split. - file_offset_ += skip; - size_t bytes_in_buffer = BytesInBuffer(); - - // Empty buffer. Increment |blob_offset_|. - if (PERFETTO_LIKELY(bytes_in_buffer == 0)) { - buffer_offset_ = 0; - buffer_.clear(); - blob_offset_ += skip; - } else { - SkipSlow(skip); - } - PERFETTO_DCHECK(BytesAvailable() == bytes_available_before - skip); - } - - // Peeks the |obj| without updating the |file_offset_| of the reader. - // NOTE: Assumes count of bytes available is higher than sizeof(T). - template <typename T> - void Peek(T& obj) const { - PERFETTO_DCHECK(CanReadSize(sizeof(T))); - size_t bytes_available_before = BytesAvailable(); - - // Read from blob. - if (PERFETTO_LIKELY(BytesInBuffer() == 0)) { - memcpy(&obj, tbv_.data() + blob_offset_, sizeof(T)); - } else { - PeekSlow(reinterpret_cast<uint8_t*>(&obj), sizeof(T)); - } - - PERFETTO_DCHECK(BytesAvailable() == bytes_available_before); - } - - // Creates TraceBlobView with data of |data_size| bytes from current offset. - // NOTE: Assumes count of bytes available is higher than sizeof(T). - TraceBlobView PeekTraceBlobView(uint64_t data_size) const { - PERFETTO_DCHECK(CanReadSize(data_size)); - size_t size = static_cast<size_t>(data_size); - size_t bytes_in_buffer = BytesInBuffer(); - - // Data is in blob, so it's enough to slice the existing |tbv_|. - if (PERFETTO_LIKELY(bytes_in_buffer == 0)) { - return tbv_.slice(tbv_.data() + blob_offset_, size); - } - return PeekTraceBlobViewSlow(size); - } - - // Returns if there is enough data to read offsets between |start| and |end|. - bool CanAccessFileRange(uint64_t start, uint64_t end) const { - return CanAccessFileOffset(static_cast<size_t>(start)) && - CanAccessFileOffset(static_cast<size_t>(end)); - } - - // Returns if there is enough data to read |size| bytes. - bool CanReadSize(uint64_t size) const { return size <= BytesAvailable(); } - - uint64_t current_file_offset() const { return file_offset_; } - - private: - void SkipSlow(size_t bytes_to_skip); - - void PeekSlow(uint8_t* obj_data, size_t) const; - - TraceBlobView PeekTraceBlobViewSlow(size_t) const; - - size_t BytesInBuffer() const { - PERFETTO_DCHECK(buffer_.size() >= buffer_offset_); - return buffer_.size() - buffer_offset_; - } - size_t BytesInBlob() const { return tbv_.size() - blob_offset_; } - size_t BytesAvailable() const { return BytesInBuffer() + BytesInBlob(); } - - bool CanAccessFileOffset(size_t off) const { - return off >= file_offset_ && off <= file_offset_ + BytesAvailable(); - } - - TraceBlobView tbv_; - std::vector<uint8_t> buffer_; - - // Where we are in relation to the current blob. - size_t blob_offset_ = 0; - // Where we are in relation to the file. - size_t file_offset_ = 0; - // Where we are in relation to the buffer. - size_t buffer_offset_ = 0; -}; -} // namespace perf_importer -} // namespace trace_processor -} // namespace perfetto - -#endif // SRC_TRACE_PROCESSOR_IMPORTERS_PERF_PERF_DATA_READER_H_ diff --git a/src/trace_processor/importers/perf/perf_data_reader_unittest.cc b/src/trace_processor/importers/perf/perf_data_reader_unittest.cc deleted file mode 100644 index 5bf30810d..000000000 --- a/src/trace_processor/importers/perf/perf_data_reader_unittest.cc +++ /dev/null @@ -1,234 +0,0 @@ -/* - * Copyright (C) 2023 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "src/trace_processor/importers/perf/perf_data_reader.h" - -#include <stddef.h> - -#include "perfetto/base/build_config.h" -#include "test/gtest_and_gmock.h" - -namespace perfetto { -namespace trace_processor { -namespace perf_importer { - -namespace { -template <typename T> -TraceBlobView TraceBlobViewFromVector(std::vector<T> nums) { - size_t data_size = sizeof(T) * nums.size(); - auto blob = TraceBlob::Allocate(data_size); - memcpy(blob.data(), nums.data(), data_size); - return TraceBlobView(std::move(blob)); -} -} // namespace - -TEST(PerfDataReaderUnittest, AppendToEmpty) { - TraceBlobView tbv = TraceBlobViewFromVector(std::vector<uint64_t>{1, 2, 3}); - PerfDataReader reader; - EXPECT_FALSE(reader.CanReadSize(1)); - reader.Append(std::move(tbv)); - EXPECT_TRUE(reader.CanReadSize(sizeof(uint64_t) * 2)); -} - -TEST(PerfDataReaderUnittest, Append) { - TraceBlobView tbv = TraceBlobViewFromVector(std::vector<uint64_t>{1, 2, 3}); - PerfDataReader reader(std::move(tbv)); - - EXPECT_TRUE(reader.CanReadSize(sizeof(uint64_t) * 3)); - EXPECT_FALSE(reader.CanReadSize(sizeof(uint64_t) * 3 + 1)); - - reader.Append(TraceBlobViewFromVector(std::vector<uint64_t>{1, 2})); - EXPECT_TRUE(reader.CanReadSize(sizeof(uint64_t) * 5)); -} - -TEST(PerfDataReaderUnittest, Read) { - TraceBlobView tbv = TraceBlobViewFromVector(std::vector<uint64_t>{2, 4, 8}); - PerfDataReader reader(std::move(tbv)); - uint64_t val; - reader.Read(val); - EXPECT_EQ(val, 2u); -} - -TEST(PerfDataReaderUnittest, ReadFromBuffer) { - TraceBlobView tbv = TraceBlobViewFromVector(std::vector<uint64_t>{2, 4, 6}); - PerfDataReader reader(std::move(tbv)); - reader.Append(TraceBlobViewFromVector(std::vector<uint64_t>{1, 3})); - - // Now the first vector should be in the buffer. - uint64_t val; - reader.Read(val); - EXPECT_EQ(val, 2u); -} - -TEST(PerfDataReaderUnittest, ReadBetweenBufferAndBlob) { - TraceBlobView tbv = TraceBlobViewFromVector(std::vector<uint64_t>{2, 4}); - PerfDataReader reader(std::move(tbv)); - reader.Append(TraceBlobViewFromVector(std::vector<uint64_t>{1, 3, 5})); - - struct Nums { - uint64_t x; - uint64_t y; - uint64_t z; - }; - - Nums nums; - reader.Read(nums); - - EXPECT_EQ(nums.x, 2u); - EXPECT_EQ(nums.y, 4u); - EXPECT_EQ(nums.z, 1u); -} - -TEST(PerfDataReaderUnittest, ReadOptional) { - TraceBlobView tbv = TraceBlobViewFromVector(std::vector<uint64_t>{2, 4, 8}); - PerfDataReader reader(std::move(tbv)); - std::optional<uint64_t> val; - reader.ReadOptional(val); - EXPECT_EQ(val, 2u); -} - -TEST(PerfDataReaderUnittest, ReadVector) { - TraceBlobView tbv = - TraceBlobViewFromVector(std::vector<uint64_t>{2, 4, 8, 16, 32}); - PerfDataReader reader(std::move(tbv)); - - std::vector<uint64_t> res(3); - reader.ReadVector(res); - - std::vector<uint64_t> valid{2, 4, 8}; - EXPECT_EQ(res, valid); -} - -TEST(PerfDataReaderUnittest, Skip) { - TraceBlobView tbv = TraceBlobViewFromVector(std::vector<uint64_t>{2, 4, 8}); - PerfDataReader reader(std::move(tbv)); - - reader.Skip<uint64_t>(); - - uint64_t val; - reader.Read(val); - EXPECT_EQ(val, 4u); -} - -TEST(PerfDataReaderUnittest, SkipInBuffer) { - TraceBlobView tbv = TraceBlobViewFromVector(std::vector<uint64_t>{2, 4}); - PerfDataReader reader(std::move(tbv)); - reader.Append(TraceBlobViewFromVector(std::vector<uint64_t>{1, 3, 5})); - - reader.Skip<uint64_t>(); - EXPECT_EQ(reader.current_file_offset(), sizeof(uint64_t)); -} - -TEST(PerfDataReaderUnittest, SkipBetweenBufferAndBlob) { - TraceBlobView tbv = TraceBlobViewFromVector(std::vector<uint64_t>{2, 4}); - PerfDataReader reader(std::move(tbv)); - reader.Append(TraceBlobViewFromVector(std::vector<uint64_t>{1, 3, 5})); - - struct Nums { - uint64_t x; - uint64_t y; - uint64_t z; - }; - - reader.Skip<Nums>(); - EXPECT_EQ(reader.current_file_offset(), sizeof(Nums)); -} - -TEST(PerfDataReaderUnittest, Peek) { - TraceBlobView tbv = TraceBlobViewFromVector(std::vector<uint64_t>{2, 4, 8}); - PerfDataReader reader(std::move(tbv)); - - uint64_t peek_val; - reader.Peek(peek_val); - - uint64_t val; - reader.Read(val); - EXPECT_EQ(val, 2u); -} - -TEST(PerfDataReaderUnittest, PeekFromBuffer) { - TraceBlobView tbv = TraceBlobViewFromVector(std::vector<uint64_t>{2, 4, 6}); - PerfDataReader reader(std::move(tbv)); - reader.Append(TraceBlobViewFromVector(std::vector<uint64_t>{1, 3})); - - uint64_t val; - reader.Peek(val); - EXPECT_EQ(val, 2u); -} - -TEST(PerfDataReaderUnittest, PeekBetweenBufferAndBlob) { - TraceBlobView tbv = TraceBlobViewFromVector(std::vector<uint64_t>{2, 4}); - PerfDataReader reader(std::move(tbv)); - reader.Append(TraceBlobViewFromVector(std::vector<uint64_t>{1, 3, 5})); - - struct Nums { - uint64_t x; - uint64_t y; - uint64_t z; - }; - - Nums nums; - reader.Peek(nums); - - EXPECT_EQ(nums.x, 2u); - EXPECT_EQ(nums.y, 4u); - EXPECT_EQ(nums.z, 1u); -} - -TEST(PerfDataReaderUnittest, GetTraceBlobView) { - TraceBlobView tbv = TraceBlobViewFromVector(std::vector<uint64_t>{2, 4, 8}); - PerfDataReader reader(std::move(tbv)); - EXPECT_TRUE(reader.CanReadSize(sizeof(uint64_t) * 3)); - - TraceBlobView new_tbv = reader.PeekTraceBlobView(sizeof(uint64_t) * 2); - PerfDataReader new_reader(std::move(new_tbv)); - EXPECT_TRUE(new_reader.CanReadSize(sizeof(uint64_t) * 2)); - EXPECT_FALSE(new_reader.CanReadSize(sizeof(uint64_t) * 3)); -} - -TEST(PerfDataReaderUnittest, GetTraceBlobViewFromBuffer) { - TraceBlobView tbv = TraceBlobViewFromVector(std::vector<uint64_t>{2, 4}); - PerfDataReader reader(std::move(tbv)); - reader.Append(TraceBlobViewFromVector(std::vector<uint64_t>{1, 3, 5})); - - TraceBlobView new_tbv = reader.PeekTraceBlobView(sizeof(uint64_t) * 2); - PerfDataReader new_reader(std::move(new_tbv)); - EXPECT_TRUE(new_reader.CanReadSize(sizeof(uint64_t) * 2)); - EXPECT_FALSE(new_reader.CanReadSize(sizeof(uint64_t) * 3)); -} - -TEST(PerfDataReaderUnittest, GetTraceBlobViewFromBetweenBufferAndBlob) { - TraceBlobView tbv = TraceBlobViewFromVector(std::vector<uint64_t>{2, 4}); - PerfDataReader reader(std::move(tbv)); - reader.Append(TraceBlobViewFromVector(std::vector<uint64_t>{1, 3, 5})); - - TraceBlobView new_tbv = reader.PeekTraceBlobView(sizeof(uint64_t) * 3); - PerfDataReader new_reader(std::move(new_tbv)); - EXPECT_TRUE(new_reader.CanReadSize(sizeof(uint64_t) * 3)); - EXPECT_FALSE(new_reader.CanReadSize(sizeof(uint64_t) * 4)); -} - -TEST(PerfDataReaderUnittest, CanAccessFileRange) { - TraceBlobView tbv = TraceBlobViewFromVector(std::vector<uint64_t>{2, 4, 8}); - PerfDataReader reader(std::move(tbv)); - EXPECT_TRUE(reader.CanAccessFileRange(2, sizeof(uint64_t) * 3)); - EXPECT_FALSE(reader.CanAccessFileRange(2, sizeof(uint64_t) * 3 + 10)); -} - -} // namespace perf_importer - -} // namespace trace_processor -} // namespace perfetto diff --git a/src/trace_processor/importers/perf/perf_data_tokenizer.cc b/src/trace_processor/importers/perf/perf_data_tokenizer.cc index 25b54b983..4fd64b5f2 100644 --- a/src/trace_processor/importers/perf/perf_data_tokenizer.cc +++ b/src/trace_processor/importers/perf/perf_data_tokenizer.cc @@ -16,56 +16,101 @@ #include "src/trace_processor/importers/perf/perf_data_tokenizer.h" +#include <cstddef> #include <cstdint> #include <cstring> +#include <optional> +#include <string> +#include <utility> #include <vector> +#include "perfetto/base/flat_set.h" #include "perfetto/base/logging.h" #include "perfetto/base/status.h" #include "perfetto/ext/base/status_or.h" +#include "perfetto/public/compiler.h" #include "perfetto/trace_processor/trace_blob_view.h" +#include "protos/perfetto/trace/clock_snapshot.pbzero.h" +#include "protos/third_party/simpleperf/record_file.pbzero.h" +#include "src/trace_processor/importers/common/clock_tracker.h" #include "src/trace_processor/importers/common/slice_tracker.h" -#include "src/trace_processor/importers/perf/perf_data_reader.h" -#include "src/trace_processor/importers/perf/perf_data_tracker.h" +#include "src/trace_processor/importers/perf/attrs_section_reader.h" +#include "src/trace_processor/importers/perf/dso_tracker.h" +#include "src/trace_processor/importers/perf/features.h" #include "src/trace_processor/importers/perf/perf_event.h" +#include "src/trace_processor/importers/perf/perf_file.h" +#include "src/trace_processor/importers/perf/perf_session.h" +#include "src/trace_processor/importers/perf/reader.h" +#include "src/trace_processor/importers/perf/record.h" +#include "src/trace_processor/importers/proto/perf_sample_tracker.h" #include "src/trace_processor/sorter/trace_sorter.h" #include "src/trace_processor/storage/stats.h" +#include "src/trace_processor/util/build_id.h" #include "src/trace_processor/util/status_macros.h" -#include "protos/perfetto/trace/profiling/profile_packet.pbzero.h" - namespace perfetto { namespace trace_processor { namespace perf_importer { namespace { -protos::pbzero::Profiling::CpuMode GetCpuMode(const perf_event_header& header) { - switch (header.misc & kPerfRecordMiscCpumodeMask) { - case PERF_RECORD_MISC_KERNEL: - return protos::pbzero::Profiling::MODE_KERNEL; - case PERF_RECORD_MISC_USER: - return protos::pbzero::Profiling::MODE_USER; - case PERF_RECORD_MISC_HYPERVISOR: - return protos::pbzero::Profiling::MODE_HYPERVISOR; - case PERF_RECORD_MISC_GUEST_KERNEL: - return protos::pbzero::Profiling::MODE_GUEST_KERNEL; - case PERF_RECORD_MISC_GUEST_USER: - return protos::pbzero::Profiling::MODE_GUEST_USER; - default: - return protos::pbzero::Profiling::MODE_UNKNOWN; + +void AddIds(uint8_t id_offset, + uint64_t flags, + base::FlatSet<uint8_t>& feature_ids) { + for (size_t i = 0; i < sizeof(flags) * 8; ++i) { + if (flags & 1) { + feature_ids.insert(id_offset); + } + flags >>= 1; + ++id_offset; } } + +base::FlatSet<uint8_t> ExtractFeatureIds(const uint64_t& flags, + const uint64_t (&flags1)[3]) { + base::FlatSet<uint8_t> feature_ids; + AddIds(0, flags, feature_ids); + AddIds(64, flags1[0], feature_ids); + AddIds(128, flags1[1], feature_ids); + AddIds(192, flags1[2], feature_ids); + return feature_ids; +} + +bool ReadTime(const Record& record, std::optional<uint64_t>& time) { + if (!record.attr) { + time = std::nullopt; + return true; + } + Reader reader(record.payload.copy()); + if (record.header.type != PERF_RECORD_SAMPLE) { + std::optional<size_t> offset = record.attr->time_offset_from_end(); + if (!offset.has_value()) { + time = std::nullopt; + return true; + } + if (*offset > reader.size_left()) { + return false; + } + return reader.Skip(reader.size_left() - *offset) && + reader.ReadOptional(time); + } + + std::optional<size_t> offset = record.attr->time_offset_from_start(); + if (!offset.has_value()) { + time = std::nullopt; + return true; + } + return reader.Skip(*offset) && reader.ReadOptional(time); +} + } // namespace PerfDataTokenizer::PerfDataTokenizer(TraceProcessorContext* ctx) - : context_(ctx), - tracker_(PerfDataTracker::GetOrCreate(context_)), - reader_() {} + : context_(ctx) {} PerfDataTokenizer::~PerfDataTokenizer() = default; // A normal perf.data consts of: // [ header ] -// [ event ids (one array per attr) ] // [ attr section ] // [ data section ] // [ optional feature sections ] @@ -75,221 +120,323 @@ PerfDataTokenizer::~PerfDataTokenizer() = default; // Most file format documentation is outdated or misleading, instead see // perf_session__do_write_header() in linux/tools/perf/util/header.c. base::Status PerfDataTokenizer::Parse(TraceBlobView blob) { - reader_.Append(std::move(blob)); + buffer_.PushBack(std::move(blob)); - while (parsing_state_ != ParsingState::Records) { - base::StatusOr<ParsingResult> parsed = ParsingResult::Success; + base::StatusOr<ParsingResult> result = ParsingResult::kSuccess; + while (result.ok() && result.value() == ParsingResult::kSuccess && + !buffer_.empty()) { switch (parsing_state_) { - case ParsingState::Records: + case ParsingState::kParseHeader: + result = ParseHeader(); break; - case ParsingState::Header: - parsed = ParseHeader(); + + case ParsingState::kParseAttrs: + result = ParseAttrs(); break; - case ParsingState::AfterHeaderBuffer: - parsed = ParseAfterHeaderBuffer(); + + case ParsingState::kSeekRecords: + result = SeekRecords(); break; - case ParsingState::Attrs: - parsed = ParseAttrs(); + + case ParsingState::kParseRecords: + result = ParseRecords(); break; - case ParsingState::AttrIdsFromBuffer: - parsed = ParseAttrIdsFromBuffer(); + + case ParsingState::kParseFeatures: + result = ParseFeatures(); break; - case ParsingState::AttrIds: - parsed = ParseAttrIds(); + + case ParsingState::kParseFeatureSections: + result = ParseFeatureSections(); break; + + case ParsingState::kDone: + result = base::ErrStatus("Unexpected data"); } + } + return result.status(); +} - // There has been an error while parsing. - RETURN_IF_ERROR(parsed.status()); +base::StatusOr<PerfDataTokenizer::ParsingResult> +PerfDataTokenizer::ParseHeader() { + auto tbv = buffer_.SliceOff(0, sizeof(header_)); + if (!tbv) { + return ParsingResult::kMoreDataNeeded; + } + PERFETTO_CHECK(Reader(std::move(*tbv)).Read(header_)); - // There is not enough data to parse so we need to load another blob. - if (*parsed == ParsingResult::NoSpace) - return base::OkStatus(); + // TODO: Check for endianess (big endian will have letters reversed); + if (memcmp(header_.magic, PerfFile::kPerfMagic, + sizeof(PerfFile::kPerfMagic)) != 0) { + return util::ErrStatus("Invalid magic string"); } - while (reader_.current_file_offset() < header_.data.end()) { - // Make sure |perf_event_header| of the sample is available. - if (!reader_.CanReadSize(sizeof(perf_event_header))) { - return base::OkStatus(); - } + if (header_.size != sizeof(PerfFile::Header)) { + return util::ErrStatus("Failed to perf file header size. Expected %" PRIu64 + ", found %zu", + sizeof(PerfFile::Header)); + } - perf_event_header ev_header; - reader_.Peek(ev_header); - PERFETTO_CHECK(ev_header.size >= sizeof(perf_event_header)); + feature_ids_ = ExtractFeatureIds(header_.flags, header_.flags1); + feature_headers_section_ = {header_.data.end(), + feature_ids_.size() * sizeof(PerfFile::Section)}; + context_->clock_tracker->SetTraceTimeClock( + protos::pbzero::ClockSnapshot::Clock::MONOTONIC); - if (!reader_.CanReadSize(ev_header.size)) { - return base::OkStatus(); + PERFETTO_CHECK(buffer_.PopFrontUntil(sizeof(PerfFile::Header))); + parsing_state_ = ParsingState::kParseAttrs; + return ParsingResult::kSuccess; +} + +base::StatusOr<PerfDataTokenizer::ParsingResult> +PerfDataTokenizer::ParseAttrs() { + std::optional<TraceBlobView> tbv = + buffer_.SliceOff(header_.attrs.offset, header_.attrs.size); + if (!tbv) { + return ParsingResult::kMoreDataNeeded; + } + + ASSIGN_OR_RETURN(AttrsSectionReader attr_reader, + AttrsSectionReader::Create(header_, std::move(*tbv))); + + PerfSession::Builder builder(context_); + while (attr_reader.CanReadNext()) { + PerfFile::AttrsEntry entry; + RETURN_IF_ERROR(attr_reader.ReadNext(entry)); + + if (entry.ids.size % sizeof(uint64_t) != 0) { + return base::ErrStatus("Invalid id section size: %" PRIu64, + entry.ids.size); } - reader_.Skip<perf_event_header>(); - uint64_t record_offset = reader_.current_file_offset(); - uint64_t record_size = ev_header.size - sizeof(perf_event_header); - - switch (ev_header.type) { - case PERF_RECORD_SAMPLE: { - TraceBlobView tbv = reader_.PeekTraceBlobView(record_size); - auto sample_status = tracker_->ParseSample(reader_); - if (!sample_status.ok()) { - continue; - } - PerfDataTracker::PerfSample sample = *sample_status; - if (!ValidateSample(*sample_status)) { - continue; - } - context_->sorter->PushPerfRecord( - static_cast<int64_t>(*sample_status->ts), std::move(tbv)); - break; - } - case PERF_RECORD_MMAP2: { - PERFETTO_CHECK(ev_header.size >= - sizeof(PerfDataTracker::Mmap2Record::Numeric)); - auto record = ParseMmap2Record(record_size); - RETURN_IF_ERROR(record.status()); - record->cpu_mode = GetCpuMode(ev_header); - tracker_->PushMmap2Record(*record); - break; - } - default: - break; + tbv = buffer_.SliceOff(entry.ids.offset, entry.ids.size); + if (!tbv) { + return ParsingResult::kMoreDataNeeded; } - reader_.Skip((record_offset + record_size) - reader_.current_file_offset()); + std::vector<uint64_t> ids; + ids.resize(entry.ids.size / sizeof(uint64_t)); + PERFETTO_CHECK(Reader(std::move(*tbv)).ReadVector(ids)); + + builder.AddAttrAndIds(entry.attr, std::move(ids)); } - return base::OkStatus(); + ASSIGN_OR_RETURN(perf_session_, builder.Build()); + parsing_state_ = ParsingState::kSeekRecords; + return ParsingResult::kSuccess; } base::StatusOr<PerfDataTokenizer::ParsingResult> -PerfDataTokenizer::ParseHeader() { - if (!reader_.CanReadSize(sizeof(PerfHeader))) { - return ParsingResult::NoSpace; +PerfDataTokenizer::SeekRecords() { + if (!buffer_.PopFrontUntil(header_.data.offset)) { + return ParsingResult::kMoreDataNeeded; } - reader_.Read(header_); - PERFETTO_CHECK(header_.size == sizeof(PerfHeader)); - if (header_.attr_size != - sizeof(perf_event_attr) + sizeof(PerfDataTracker::PerfFileSection)) { - return base::ErrStatus( - "Unsupported: perf.data collected with a different ABI version of " - "perf_event_attr."); + parsing_state_ = ParsingState::kParseRecords; + return ParsingResult::kSuccess; +} + +base::StatusOr<PerfDataTokenizer::ParsingResult> +PerfDataTokenizer::ParseRecords() { + while (buffer_.file_offset() < header_.data.end()) { + Record record; + + if (auto res = ParseRecord(record); + !res.ok() || *res != ParsingResult::kSuccess) { + return res; + } + + if (!PushRecord(std::move(record))) { + context_->storage->IncrementStats(stats::perf_record_skipped); + } } - if (header_.attrs.offset > header_.data.offset) { - return base::ErrStatus( - "Can only import files where samples are located after the metadata."); + parsing_state_ = ParsingState::kParseFeatureSections; + return ParsingResult::kSuccess; +} + +base::StatusOr<PerfDataTokenizer::ParsingResult> PerfDataTokenizer::ParseRecord( + Record& record) { + record.session = perf_session_; + std::optional<TraceBlobView> tbv = + buffer_.SliceOff(buffer_.file_offset(), sizeof(record.header)); + if (!tbv) { + return ParsingResult::kMoreDataNeeded; } + PERFETTO_CHECK(Reader(std::move(*tbv)).Read(record.header)); - if (header_.size == header_.attrs.offset) { - parsing_state_ = ParsingState::Attrs; - } else { - parsing_state_ = ParsingState::AfterHeaderBuffer; + if (record.header.size < sizeof(record.header)) { + return base::ErrStatus("Invalid record size: %" PRIu16, record.header.size); } - return ParsingResult::Success; + + tbv = buffer_.SliceOff(buffer_.file_offset() + sizeof(record.header), + record.header.size - sizeof(record.header)); + if (!tbv) { + return ParsingResult::kMoreDataNeeded; + } + + record.payload = std::move(*tbv); + + base::StatusOr<RefPtr<const PerfEventAttr>> attr = + perf_session_->FindAttrForRecord(record.header, record.payload); + if (!attr.ok()) { + return base::ErrStatus("Unable to determine perf_event_attr for record. %s", + attr.status().c_message()); + } + record.attr = *attr; + + buffer_.PopFrontBytes(record.header.size); + return ParsingResult::kSuccess; } -base::StatusOr<PerfDataTokenizer::ParsingResult> -PerfDataTokenizer::ParseAfterHeaderBuffer() { - if (!reader_.CanAccessFileRange(header_.size, header_.attrs.offset)) { - return ParsingResult::NoSpace; +base::StatusOr<int64_t> PerfDataTokenizer::ToTraceTimestamp( + std::optional<uint64_t> time) { + base::StatusOr<int64_t> trace_ts = + time.has_value() + ? context_->clock_tracker->ToTraceTime( + protos::pbzero::ClockSnapshot::Clock::MONOTONIC, + static_cast<int64_t>(*time)) + : std::max(latest_timestamp_, context_->sorter->max_timestamp()); + + if (PERFETTO_LIKELY(trace_ts.ok())) { + latest_timestamp_ = std::max(latest_timestamp_, *trace_ts); } - after_header_buffer_.resize( - static_cast<size_t>(header_.attrs.offset - header_.size)); - reader_.ReadVector(after_header_buffer_); - parsing_state_ = ParsingState::Attrs; - return ParsingResult::Success; + + return trace_ts; } -base::StatusOr<PerfDataTokenizer::ParsingResult> -PerfDataTokenizer::ParseAttrs() { - if (!reader_.CanAccessFileRange(header_.attrs.offset, header_.attrs.end())) { - return ParsingResult::NoSpace; +bool PerfDataTokenizer::PushRecord(Record record) { + std::optional<uint64_t> time; + if (!ReadTime(record, time)) { + return false; } - reader_.Skip(header_.attrs.offset - reader_.current_file_offset()); - PerfDataTracker::PerfFileAttr attr; - for (uint64_t i = header_.attrs.offset; i < header_.attrs.end(); - i += header_.attr_size) { - reader_.Read(attr); - PERFETTO_CHECK(attr.ids.size % sizeof(uint64_t) == 0); - ids_start_ = std::min(ids_start_, attr.ids.offset); - ids_end_ = std::max(ids_end_, attr.ids.end()); - attrs_.push_back(attr); + + base::StatusOr<int64_t> trace_ts = ToTraceTimestamp(time); + if (!trace_ts.ok()) { + return false; } - if (ids_start_ == header_.size && ids_end_ <= header_.attrs.offset) { - parsing_state_ = ParsingState::AttrIdsFromBuffer; - } else { - parsing_state_ = ParsingState::AttrIds; + switch (record.header.type) { + case PERF_RECORD_AUXTRACE_INFO: + case PERF_RECORD_AUXTRACE: + case PERF_RECORD_AUX: + break; + default: + context_->sorter->PushPerfRecord(*trace_ts, std::move(record)); + break; } - return ParsingResult::Success; + + return true; } base::StatusOr<PerfDataTokenizer::ParsingResult> -PerfDataTokenizer::ParseAttrIds() { - if (!reader_.CanAccessFileRange(ids_start_, ids_end_)) { - return ParsingResult::NoSpace; +PerfDataTokenizer::ParseFeatureSections() { + PERFETTO_CHECK(buffer_.file_offset() == header_.data.end()); + auto tbv = buffer_.SliceOff(feature_headers_section_.offset, + feature_headers_section_.size); + if (!tbv) { + return ParsingResult::kMoreDataNeeded; } - for (const auto& attr_file : attrs_) { - reader_.Skip(attr_file.ids.offset - reader_.current_file_offset()); - std::vector<uint64_t> ids(static_cast<size_t>(attr_file.ids.size) / - sizeof(uint64_t)); - reader_.ReadVector(ids); - tracker_->PushAttrAndIds({attr_file.attr, std::move(ids)}); + + Reader reader(std::move(*tbv)); + for (auto feature_id : feature_ids_) { + feature_sections_.emplace_back(std::piecewise_construct, + std::forward_as_tuple(feature_id), + std::forward_as_tuple()); + PERFETTO_CHECK(reader.Read(feature_sections_.back().second)); } - tracker_->ComputeCommonSampleType(); - reader_.Skip(header_.data.offset - reader_.current_file_offset()); - parsing_state_ = ParsingState::Records; - return ParsingResult::Success; + std::sort(feature_sections_.begin(), feature_sections_.end(), + [](const std::pair<uint8_t, PerfFile::Section>& lhs, + const std::pair<uint8_t, PerfFile::Section>& rhs) { + return lhs.second.offset > rhs.second.offset; + }); + + buffer_.PopFrontUntil(feature_headers_section_.end()); + parsing_state_ = feature_sections_.empty() ? ParsingState::kDone + : ParsingState::kParseFeatures; + return ParsingResult::kSuccess; } base::StatusOr<PerfDataTokenizer::ParsingResult> -PerfDataTokenizer::ParseAttrIdsFromBuffer() { - // Each attribute points at an array of event ids. In this case, the ids are - // in |after_header_buffer_|, i.e. the file contents between the header and - // the start of the attr section. - for (const auto& attr_file : attrs_) { - size_t num_ids = static_cast<size_t>(attr_file.ids.size / sizeof(uint64_t)); - std::vector<uint64_t> ids(num_ids); - size_t rd_offset = static_cast<size_t>(attr_file.ids.offset - ids_start_); - size_t rd_size = static_cast<size_t>(attr_file.ids.size); - PERFETTO_CHECK(rd_offset + rd_size <= after_header_buffer_.size()); - memcpy(ids.data(), after_header_buffer_.data() + rd_offset, rd_size); - - tracker_->PushAttrAndIds({attr_file.attr, std::move(ids)}); +PerfDataTokenizer::ParseFeatures() { + while (!feature_sections_.empty()) { + const auto feature_id = feature_sections_.back().first; + const auto& section = feature_sections_.back().second; + auto tbv = buffer_.SliceOff(section.offset, section.size); + if (!tbv) { + return ParsingResult::kMoreDataNeeded; + } + + RETURN_IF_ERROR(ParseFeature(feature_id, std::move(*tbv))); + buffer_.PopFrontUntil(section.end()); + feature_sections_.pop_back(); } - after_header_buffer_.clear(); - tracker_->ComputeCommonSampleType(); - reader_.Skip(header_.data.offset - reader_.current_file_offset()); - parsing_state_ = ParsingState::Records; - return ParsingResult::Success; + parsing_state_ = ParsingState::kDone; + return ParsingResult::kSuccess; } -base::StatusOr<PerfDataTracker::Mmap2Record> -PerfDataTokenizer::ParseMmap2Record(uint64_t record_size) { - uint64_t start_offset = reader_.current_file_offset(); - PerfDataTracker::Mmap2Record record; - reader_.Read(record.num); - std::vector<char> filename_buffer( - static_cast<size_t>(record_size) - - sizeof(PerfDataTracker::Mmap2Record::Numeric)); - reader_.ReadVector(filename_buffer); - if (filename_buffer.back() != '\0') { - return base::ErrStatus( - "Invalid MMAP2 record: filename is not null terminated."); - } - record.filename = std::string(filename_buffer.begin(), filename_buffer.end()); - PERFETTO_CHECK(reader_.current_file_offset() == start_offset + record_size); - return record; -} +base::Status PerfDataTokenizer::ParseFeature(uint8_t feature_id, + TraceBlobView data) { + switch (feature_id) { + case feature::ID_CMD_LINE: { + ASSIGN_OR_RETURN(std::vector<std::string> args, + feature::ParseCmdline(std::move(data))); + perf_session_->SetCmdline(args); + return base::OkStatus(); + } -bool PerfDataTokenizer::ValidateSample( - const PerfDataTracker::PerfSample& sample) { - if (!sample.cpu.has_value() || !sample.ts.has_value() || - sample.callchain.empty() || !sample.pid.has_value()) { - context_->storage->IncrementStats(stats::perf_samples_skipped); - return false; + case feature::ID_EVENT_DESC: + return feature::EventDescription::Parse( + std::move(data), [&](feature::EventDescription desc) { + for (auto id : desc.ids) { + perf_session_->SetEventName(id, std::move(desc.event_string)); + } + return base::OkStatus(); + }); + + case feature::ID_BUILD_ID: + return feature::BuildId::Parse( + std::move(data), [&](feature::BuildId build_id) { + perf_session_->AddBuildId( + build_id.pid, std::move(build_id.filename), + BuildId::FromRaw(std::move(build_id.build_id))); + return base::OkStatus(); + }); + + case feature::ID_GROUP_DESC: { + feature::HeaderGroupDesc group_desc; + RETURN_IF_ERROR( + feature::HeaderGroupDesc::Parse(std::move(data), group_desc)); + // TODO(carlscab): Do someting + break; + } + + case feature::ID_SIMPLEPERF_META_INFO: { + feature::SimpleperfMetaInfo meta_info; + RETURN_IF_ERROR( + feature::SimpleperfMetaInfo::Parse(std::move(data), meta_info)); + for (auto it = meta_info.event_type_info.GetIterator(); it; ++it) { + perf_session_->SetEventName(it.key().type, it.key().config, it.value()); + } + break; + } + case feature::ID_SIMPLEPERF_FILE2: { + RETURN_IF_ERROR(feature::ParseSimpleperfFile2( + std::move(data), [&](TraceBlobView blob) { + third_party::simpleperf::proto::pbzero::FileFeature::Decoder file( + blob.data(), blob.length()); + DsoTracker::GetOrCreate(context_).AddSimpleperfFile2(file); + })); + + break; + } + default: + context_->storage->IncrementIndexedStats(stats::perf_features_skipped, + feature_id); } - return true; + + return base::OkStatus(); } void PerfDataTokenizer::NotifyEndOfFile() {} diff --git a/src/trace_processor/importers/perf/perf_data_tokenizer.h b/src/trace_processor/importers/perf/perf_data_tokenizer.h index 7a54088c8..61c13089c 100644 --- a/src/trace_processor/importers/perf/perf_data_tokenizer.h +++ b/src/trace_processor/importers/perf/perf_data_tokenizer.h @@ -18,45 +18,30 @@ #define SRC_TRACE_PROCESSOR_IMPORTERS_PERF_PERF_DATA_TOKENIZER_H_ #include <stdint.h> +#include <cstdint> +#include <optional> +#include <vector> + +#include "perfetto/base/flat_set.h" #include "perfetto/base/status.h" #include "perfetto/ext/base/status_or.h" -#include "perfetto/ext/base/string_utils.h" +#include "perfetto/trace_processor/ref_counted.h" #include "perfetto/trace_processor/trace_blob_view.h" -#include "src/trace_processor/importers/perf/perf_data_reader.h" -#include "src/trace_processor/importers/perf/perf_data_tracker.h" -#include "src/trace_processor/importers/perf/perf_event.h" - -#include <limits> -#include <map> -#include <string> -#include <vector> - #include "src/trace_processor/importers/common/chunked_trace_reader.h" +#include "src/trace_processor/importers/perf/perf_file.h" +#include "src/trace_processor/importers/perf/perf_session.h" +#include "src/trace_processor/util/file_buffer.h" namespace perfetto { namespace trace_processor { +class TraceProcessorContext; + namespace perf_importer { -using Section = PerfDataTracker::PerfFileSection; +struct Record; class PerfDataTokenizer : public ChunkedTraceReader { public: - struct PerfHeader { - static constexpr char PERF_MAGIC[] = "PERFILE2"; - - char magic[8]; - uint64_t size; - // Size of PerfFileAttr struct and section pointing to ids. - uint64_t attr_size; - Section attrs; - Section data; - Section event_types; - uint64_t flags; - uint64_t flags1[3]; - - uint64_t num_attrs() const { return attrs.size / attr_size; } - }; - explicit PerfDataTokenizer(TraceProcessorContext*); ~PerfDataTokenizer() override; PerfDataTokenizer(const PerfDataTokenizer&) = delete; @@ -68,39 +53,46 @@ class PerfDataTokenizer : public ChunkedTraceReader { private: enum class ParsingState { - Header = 0, - AfterHeaderBuffer = 1, - Attrs = 2, - AttrIds = 3, - AttrIdsFromBuffer = 4, - Records = 5 + kParseHeader, + kParseAttrs, + kSeekRecords, + kParseRecords, + kParseFeatureSections, + kParseFeatures, + kDone, }; - enum class ParsingResult { NoSpace = 0, Success = 1 }; + enum class ParsingResult { kMoreDataNeeded = 0, kSuccess = 1 }; base::StatusOr<ParsingResult> ParseHeader(); - base::StatusOr<ParsingResult> ParseAfterHeaderBuffer(); base::StatusOr<ParsingResult> ParseAttrs(); - base::StatusOr<ParsingResult> ParseAttrIds(); - base::StatusOr<ParsingResult> ParseAttrIdsFromBuffer(); + base::StatusOr<ParsingResult> SeekRecords(); + base::StatusOr<ParsingResult> ParseRecords(); + base::StatusOr<ParsingResult> ParseFeatureSections(); + base::StatusOr<ParsingResult> ParseFeatures(); - base::StatusOr<PerfDataTracker::Mmap2Record> ParseMmap2Record( - uint64_t record_size); + base::StatusOr<PerfDataTokenizer::ParsingResult> ParseRecord(Record& record); + bool PushRecord(Record record); + base::Status ParseFeature(uint8_t feature_id, TraceBlobView payload); - bool ValidateSample(const PerfDataTracker::PerfSample&); + base::StatusOr<int64_t> ToTraceTimestamp(std::optional<uint64_t> time); TraceProcessorContext* context_; - PerfDataTracker* tracker_; - ParsingState parsing_state_ = ParsingState::Header; + ParsingState parsing_state_ = ParsingState::kParseHeader; + + PerfFile::Header header_; + base::FlatSet<uint8_t> feature_ids_; + PerfFile::Section feature_headers_section_; + // Sections for the features present in the perf file sorted by descending + // section offset. This is done so that we can pop from the back as we process + // the sections. + std::vector<std::pair<uint8_t, PerfFile::Section>> feature_sections_; - PerfHeader header_; + RefPtr<PerfSession> perf_session_; - std::vector<PerfDataTracker::PerfFileAttr> attrs_; - uint64_t ids_start_ = std::numeric_limits<uint64_t>::max(); - uint64_t ids_end_ = 0; - std::vector<uint8_t> after_header_buffer_; + util::FileBuffer buffer_; - perf_importer::PerfDataReader reader_; + int64_t latest_timestamp_ = 0; }; } // namespace perf_importer diff --git a/src/trace_processor/importers/perf/perf_data_tracker.cc b/src/trace_processor/importers/perf/perf_data_tracker.cc deleted file mode 100644 index 8b3bc470a..000000000 --- a/src/trace_processor/importers/perf/perf_data_tracker.cc +++ /dev/null @@ -1,182 +0,0 @@ -/* - * Copyright (C) 2023 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "src/trace_processor/importers/perf/perf_data_tracker.h" - -#include <optional> - -#include "perfetto/base/status.h" -#include "src/trace_processor/importers/common/address_range.h" -#include "src/trace_processor/importers/common/mapping_tracker.h" -#include "src/trace_processor/importers/common/process_tracker.h" -#include "src/trace_processor/storage/stats.h" -#include "src/trace_processor/storage/trace_storage.h" - -#include "protos/perfetto/trace/profiling/profile_packet.pbzero.h" - -namespace perfetto { -namespace trace_processor { -namespace perf_importer { -namespace { - -bool IsInKernel(protos::pbzero::Profiling::CpuMode cpu_mode) { - switch (cpu_mode) { - case protos::pbzero::Profiling::MODE_UNKNOWN: - PERFETTO_CHECK(false); - case protos::pbzero::Profiling::MODE_GUEST_KERNEL: - case protos::pbzero::Profiling::MODE_KERNEL: - return true; - case protos::pbzero::Profiling::MODE_USER: - case protos::pbzero::Profiling::MODE_HYPERVISOR: - case protos::pbzero::Profiling::MODE_GUEST_USER: - return false; - } - PERFETTO_CHECK(false); -} - -CreateMappingParams BuildCreateMappingParams( - PerfDataTracker::Mmap2Record record) { - return {AddressRange::FromStartAndSize(record.num.addr, record.num.len), - record.num.pgoff, - // start_offset: This is the offset into the file where the ELF header - // starts. We assume all file mappings are ELF files an thus this - // offset is 0. - 0, - // load_bias: This can only be read out of the actual ELF file, which - // we do not have here, so we set it to 0. When symbolizing we will - // hopefully have the real load bias and we can compensate there for a - // possible mismatch. - 0, record.filename, std::nullopt}; -} -} // namespace - -PerfDataTracker::~PerfDataTracker() = default; - -uint64_t PerfDataTracker::ComputeCommonSampleType() { - if (attrs_.empty()) { - return 0; - } - common_sample_type_ = std::numeric_limits<uint64_t>::max(); - for (const auto& a : attrs_) { - common_sample_type_ &= a.attr.sample_type; - } - return common_sample_type_; -} - -const perf_event_attr* PerfDataTracker::FindAttrWithId(uint64_t id) const { - for (const auto& attr_and_ids : attrs_) { - if (auto x = - std::find(attr_and_ids.ids.begin(), attr_and_ids.ids.end(), id); - x == attr_and_ids.ids.end()) { - continue; - } - return &attr_and_ids.attr; - } - return nullptr; -} - -void PerfDataTracker::PushMmap2Record(Mmap2Record record) { - if (IsInKernel(record.cpu_mode)) { - context_->mapping_tracker->CreateKernelMemoryMapping( - BuildCreateMappingParams(std::move(record))); - } else { - UniquePid upid = - context_->process_tracker->GetOrCreateProcess(record.num.pid); - context_->mapping_tracker->CreateUserMemoryMapping( - upid, BuildCreateMappingParams(std::move(record))); - } -} - -base::StatusOr<PerfDataTracker::PerfSample> PerfDataTracker::ParseSample( - perfetto::trace_processor::perf_importer::PerfDataReader& reader) { - uint64_t sample_type = common_sample_type(); - PerfDataTracker::PerfSample sample; - - if (sample_type & PERF_SAMPLE_IDENTIFIER) { - reader.ReadOptional(sample.id); - if (auto attr = FindAttrWithId(*sample.id); attr) { - sample_type = attr->sample_type; - } else { - return base::ErrStatus("No attr for sample_id"); - } - } - - if (sample_type & PERF_SAMPLE_IP) { - reader.Skip<uint64_t>(); - } - - if (sample_type & PERF_SAMPLE_TID) { - reader.ReadOptional(sample.pid); - reader.ReadOptional(sample.tid); - } - - if (sample_type & PERF_SAMPLE_TIME) { - reader.ReadOptional(sample.ts); - } - - // Ignored. Checked because we need to access later parts of sample. - if (sample_type & PERF_SAMPLE_ADDR) { - reader.Skip<uint64_t>(); - } - - // The same value as PERF_SAMPLE_IDENTIFIER, so should be ignored. - if (sample_type & PERF_SAMPLE_ID) { - reader.Skip<uint64_t>(); - } - - // Ignored. Checked because we need to access later parts of sample. - if (sample_type & PERF_SAMPLE_STREAM_ID) { - reader.Skip<uint64_t>(); - } - - if (sample_type & PERF_SAMPLE_CPU) { - reader.ReadOptional(sample.cpu); - // Ignore next uint32_t res. - reader.Skip<uint32_t>(); - } - - // Ignored. Checked because we need to access later parts of sample. - if (sample_type & PERF_SAMPLE_PERIOD) { - reader.Skip<uint64_t>(); - } - - // Ignored. - // TODO(mayzner): Implement. - if (sample_type & PERF_SAMPLE_READ) { - context_->storage->IncrementStats(stats::perf_samples_skipped); - return base::ErrStatus("PERF_SAMPLE_READ is not supported"); - } - - if (sample_type & PERF_SAMPLE_CALLCHAIN) { - uint64_t vec_size; - reader.Read(vec_size); - - sample.callchain.resize(static_cast<size_t>(vec_size)); - reader.ReadVector(sample.callchain); - } - - return sample; -} - -PerfDataTracker* PerfDataTracker::GetOrCreate(TraceProcessorContext* context) { - if (!context->perf_data_tracker) { - context->perf_data_tracker.reset(new PerfDataTracker(context)); - } - return static_cast<PerfDataTracker*>(context->perf_data_tracker.get()); -} -} // namespace perf_importer -} // namespace trace_processor -} // namespace perfetto diff --git a/src/trace_processor/importers/perf/perf_data_tracker.h b/src/trace_processor/importers/perf/perf_data_tracker.h deleted file mode 100644 index c96b08d39..000000000 --- a/src/trace_processor/importers/perf/perf_data_tracker.h +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Copyright (C) 2022 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef SRC_TRACE_PROCESSOR_IMPORTERS_PERF_PERF_DATA_TRACKER_H_ -#define SRC_TRACE_PROCESSOR_IMPORTERS_PERF_PERF_DATA_TRACKER_H_ - -#include <cstdint> -#include <string> -#include <vector> -#include "perfetto/base/logging.h" -#include "perfetto/base/status.h" -#include "perfetto/ext/base/flat_hash_map.h" -#include "perfetto/ext/base/status_or.h" -#include "perfetto/ext/base/string_utils.h" -#include "protos/perfetto/trace/profiling/profile_packet.pbzero.h" -#include "src/trace_processor/importers/perf/perf_data_reader.h" -#include "src/trace_processor/importers/perf/perf_event.h" -#include "src/trace_processor/storage/trace_storage.h" -#include "src/trace_processor/tables/profiler_tables_py.h" -#include "src/trace_processor/types/destructible.h" -#include "src/trace_processor/types/trace_processor_context.h" - -namespace perfetto { -namespace trace_processor { -namespace perf_importer { - -using MappingTable = tables::StackProfileMappingTable; - -class PerfDataTracker : public Destructible { - public: - struct PerfFileSection { - uint64_t offset; - uint64_t size; - - uint64_t end() const { return offset + size; } - }; - struct PerfFileAttr { - perf_event_attr attr; - PerfFileSection ids; - }; - struct AttrAndIds { - perf_event_attr attr; - std::vector<uint64_t> ids; - }; - struct PerfSample { - std::optional<uint64_t> id = 0; - std::optional<uint32_t> pid = 0; - std::optional<uint32_t> tid = 0; - std::optional<uint64_t> ts = 0; - std::optional<uint32_t> cpu = 0; - std::vector<uint64_t> callchain; - }; - struct Mmap2Record { - struct Numeric { - uint32_t pid; - uint32_t tid; - uint64_t addr; - uint64_t len; - uint64_t pgoff; - uint32_t maj; - uint32_t min; - uint64_t ino; - uint64_t ino_generation; - uint32_t prot; - uint32_t flags; - }; - protos::pbzero::Profiling::CpuMode cpu_mode; - Numeric num; - std::string filename; - }; - - PerfDataTracker(const PerfDataTracker&) = delete; - PerfDataTracker& operator=(const PerfDataTracker&) = delete; - explicit PerfDataTracker(TraceProcessorContext* context) - : context_(context) {} - ~PerfDataTracker() override; - static PerfDataTracker* GetOrCreate(TraceProcessorContext* context); - - uint64_t ComputeCommonSampleType(); - - void PushAttrAndIds(AttrAndIds data) { attrs_.push_back(std::move(data)); } - - void PushMmap2Record(Mmap2Record record); - - uint64_t common_sample_type() { return common_sample_type_; } - - base::StatusOr<PerfSample> ParseSample( - perfetto::trace_processor::perf_importer::PerfDataReader&); - - private: - const perf_event_attr* FindAttrWithId(uint64_t id) const; - TraceProcessorContext* context_; - std::vector<AttrAndIds> attrs_; - - uint64_t common_sample_type_; -}; -} // namespace perf_importer -} // namespace trace_processor -} // namespace perfetto - -#endif // SRC_TRACE_PROCESSOR_IMPORTERS_PERF_PERF_DATA_TRACKER_H_ diff --git a/src/trace_processor/importers/perf/perf_data_tracker_unittest.cc b/src/trace_processor/importers/perf/perf_data_tracker_unittest.cc deleted file mode 100644 index 923473f3a..000000000 --- a/src/trace_processor/importers/perf/perf_data_tracker_unittest.cc +++ /dev/null @@ -1,243 +0,0 @@ -/* - * Copyright (C) 2023 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "src/trace_processor/importers/perf/perf_data_tracker.h" - -#include <stddef.h> -#include <cstring> -#include <memory> -#include <vector> - -#include "perfetto/base/build_config.h" -#include "protos/perfetto/trace/profiling/profile_packet.pbzero.h" -#include "src/trace_processor/importers/common/address_range.h" -#include "src/trace_processor/importers/common/mapping_tracker.h" -#include "src/trace_processor/importers/common/process_tracker.h" -#include "src/trace_processor/importers/common/stack_profile_tracker.h" -#include "src/trace_processor/importers/perf/perf_event.h" -#include "test/gtest_and_gmock.h" - -namespace perfetto { -namespace trace_processor { -namespace perf_importer { -namespace { - -class PerfDataTrackerUnittest : public testing::Test { - public: - PerfDataTrackerUnittest() { - context_.storage = std::make_unique<TraceStorage>(); - context_.process_tracker = std::make_unique<ProcessTracker>(&context_); - context_.stack_profile_tracker = - std::make_unique<StackProfileTracker>(&context_); - context_.mapping_tracker = std::make_unique<MappingTracker>(&context_); - } - - protected: - TraceProcessorContext context_; -}; - -TEST_F(PerfDataTrackerUnittest, ComputeCommonSampleType) { - PerfDataTracker* tracker = PerfDataTracker::GetOrCreate(&context_); - - PerfDataTracker::AttrAndIds attr_and_ids; - attr_and_ids.attr.sample_type = - PERF_SAMPLE_CALLCHAIN | PERF_SAMPLE_CPU | PERF_SAMPLE_TIME; - tracker->PushAttrAndIds(attr_and_ids); - - attr_and_ids.attr.sample_type = PERF_SAMPLE_ADDR | PERF_SAMPLE_CPU; - tracker->PushAttrAndIds(attr_and_ids); - - tracker->ComputeCommonSampleType(); - EXPECT_TRUE(tracker->common_sample_type() & PERF_SAMPLE_CPU); - EXPECT_FALSE(tracker->common_sample_type() & PERF_SAMPLE_CALLCHAIN); -} - -TEST_F(PerfDataTrackerUnittest, FindMapping) { - PerfDataTracker* tracker = PerfDataTracker::GetOrCreate(&context_); - - PerfDataTracker::Mmap2Record rec; - rec.cpu_mode = protos::pbzero::Profiling::MODE_USER; - rec.filename = "file1"; - rec.num.addr = 1000; - rec.num.len = 100; - rec.num.pid = 1; - rec.cpu_mode = protos::pbzero::Profiling::MODE_USER; - tracker->PushMmap2Record(rec); - - rec.num.addr = 2000; - tracker->PushMmap2Record(rec); - - rec.num.addr = 3000; - tracker->PushMmap2Record(rec); - - UserMemoryMapping* mapping = - context_.mapping_tracker->FindUserMappingForAddress( - context_.process_tracker->GetOrCreateProcess(1), 2050); - ASSERT_NE(mapping, nullptr); - EXPECT_EQ(mapping->memory_range().start(), 2000u); - EXPECT_EQ(mapping->memory_range().end(), 2100u); -} - -TEST_F(PerfDataTrackerUnittest, FindMappingFalse) { - PerfDataTracker* tracker = PerfDataTracker::GetOrCreate(&context_); - - PerfDataTracker::Mmap2Record rec; - rec.cpu_mode = protos::pbzero::Profiling::MODE_USER; - rec.filename = "file1"; - rec.num.addr = 1000; - rec.num.len = 100; - rec.num.pid = 1; - rec.cpu_mode = protos::pbzero::Profiling::MODE_USER; - tracker->PushMmap2Record(rec); - - UserMemoryMapping* mapping = - context_.mapping_tracker->FindUserMappingForAddress( - context_.process_tracker->GetOrCreateProcess(2), 2050); - EXPECT_EQ(mapping, nullptr); -} - -TEST_F(PerfDataTrackerUnittest, ParseSampleTrivial) { - PerfDataTracker* tracker = PerfDataTracker::GetOrCreate(&context_); - - PerfDataTracker::AttrAndIds attr_and_ids; - attr_and_ids.attr.sample_type = PERF_SAMPLE_TIME; - tracker->PushAttrAndIds(attr_and_ids); - tracker->ComputeCommonSampleType(); - - uint64_t ts = 100; - - TraceBlob blob = - TraceBlob::CopyFrom(static_cast<const void*>(&ts), sizeof(uint64_t)); - PerfDataReader reader(TraceBlobView(std::move(blob))); - - auto parsed_sample = tracker->ParseSample(reader); - EXPECT_TRUE(parsed_sample.ok()); - EXPECT_EQ(parsed_sample->ts, 100u); -} - -TEST_F(PerfDataTrackerUnittest, ParseSampleCallchain) { - PerfDataTracker* tracker = PerfDataTracker::GetOrCreate(&context_); - - PerfDataTracker::AttrAndIds attr_and_ids; - attr_and_ids.attr.sample_type = PERF_SAMPLE_CALLCHAIN; - tracker->PushAttrAndIds(attr_and_ids); - tracker->ComputeCommonSampleType(); - - struct Sample { - uint64_t callchain_size; /* if PERF_SAMPLE_CALLCHAIN */ - std::vector<uint64_t> callchain; /* if PERF_SAMPLE_CALLCHAIN */ - }; - - Sample sample; - sample.callchain_size = 3; - sample.callchain = std::vector<uint64_t>{1, 2, 3}; - - TraceBlob blob = TraceBlob::Allocate(4 * sizeof(uint64_t)); - memcpy(blob.data(), &sample.callchain_size, sizeof(uint64_t)); - memcpy(blob.data() + sizeof(uint64_t), sample.callchain.data(), - sizeof(uint64_t) * 3); - PerfDataReader reader(TraceBlobView(std::move(blob))); - - auto parsed_sample = tracker->ParseSample(reader); - EXPECT_TRUE(parsed_sample.ok()); - EXPECT_EQ(parsed_sample->callchain.size(), 3u); -} - -TEST_F(PerfDataTrackerUnittest, ParseSampleWithoutId) { - PerfDataTracker* tracker = PerfDataTracker::GetOrCreate(&context_); - - PerfDataTracker::AttrAndIds attr_and_ids; - attr_and_ids.attr.sample_type = PERF_SAMPLE_TID | PERF_SAMPLE_TIME | - PERF_SAMPLE_CPU | PERF_SAMPLE_CALLCHAIN; - tracker->PushAttrAndIds(attr_and_ids); - tracker->ComputeCommonSampleType(); - - struct Sample { - uint32_t pid; /* if PERF_SAMPLE_TID */ - uint32_t tid; /* if PERF_SAMPLE_TID */ - uint64_t ts; /* if PERF_SAMPLE_TIME */ - uint32_t cpu; /* if PERF_SAMPLE_CPU */ - uint32_t res_ignore; /* if PERF_SAMPLE_CPU */ - uint64_t callchain_size; /* if PERF_SAMPLE_CALLCHAIN */ - }; - - Sample sample; - sample.pid = 2; - sample.ts = 100; - sample.cpu = 1; - sample.callchain_size = 3; - std::vector<uint64_t> callchain{1, 2, 3}; - - TraceBlob blob = TraceBlob::Allocate(sizeof(Sample) + sizeof(uint64_t) * 3); - memcpy(blob.data(), &sample, sizeof(Sample)); - memcpy(blob.data() + sizeof(Sample), callchain.data(), sizeof(uint64_t) * 3); - - PerfDataReader reader(TraceBlobView(std::move(blob))); - EXPECT_TRUE(reader.CanReadSize(sizeof(Sample))); - - auto parsed_sample = tracker->ParseSample(reader); - EXPECT_TRUE(parsed_sample.ok()); - EXPECT_EQ(parsed_sample->callchain.size(), 3u); - EXPECT_EQ(sample.ts, parsed_sample->ts); -} - -TEST_F(PerfDataTrackerUnittest, ParseSampleWithId) { - PerfDataTracker* tracker = PerfDataTracker::GetOrCreate(&context_); - - PerfDataTracker::AttrAndIds attr_and_ids; - attr_and_ids.attr.sample_type = PERF_SAMPLE_CPU | PERF_SAMPLE_TID | - PERF_SAMPLE_IDENTIFIER | PERF_SAMPLE_ID | - PERF_SAMPLE_CALLCHAIN | PERF_SAMPLE_TIME; - attr_and_ids.ids.push_back(10); - tracker->PushAttrAndIds(attr_and_ids); - tracker->ComputeCommonSampleType(); - - struct Sample { - uint64_t identifier; /* if PERF_SAMPLE_IDENTIFIER */ - uint32_t pid; /* if PERF_SAMPLE_TID */ - uint32_t tid; /* if PERF_SAMPLE_TID */ - uint64_t ts; /* if PERF_SAMPLE_TIME */ - uint64_t id; /* if PERF_SAMPLE_ID */ - uint32_t cpu; /* if PERF_SAMPLE_CPU */ - uint32_t res_ignore; /* if PERF_SAMPLE_CPU */ - uint64_t callchain_size; /* if PERF_SAMPLE_CALLCHAIN */ - }; - - Sample sample; - sample.id = 10; - sample.identifier = 10; - sample.cpu = 1; - sample.pid = 2; - sample.ts = 100; - sample.callchain_size = 3; - std::vector<uint64_t> callchain{1, 2, 3}; - - TraceBlob blob = TraceBlob::Allocate(sizeof(Sample) + sizeof(uint64_t) * 3); - memcpy(blob.data(), &sample, sizeof(Sample)); - memcpy(blob.data() + sizeof(Sample), callchain.data(), sizeof(uint64_t) * 3); - - PerfDataReader reader(TraceBlobView(std::move(blob))); - - auto parsed_sample = tracker->ParseSample(reader); - EXPECT_TRUE(parsed_sample.ok()); - EXPECT_EQ(parsed_sample->callchain.size(), 3u); - EXPECT_EQ(100u, parsed_sample->ts); -} - -} // namespace -} // namespace perf_importer -} // namespace trace_processor -} // namespace perfetto diff --git a/src/trace_processor/importers/perf/perf_event.h b/src/trace_processor/importers/perf/perf_event.h index 4763e23f7..58077be2e 100644 --- a/src/trace_processor/importers/perf/perf_event.h +++ b/src/trace_processor/importers/perf/perf_event.h @@ -238,6 +238,7 @@ enum perf_record_misc { PERF_RECORD_MISC_GUEST_USER = 5, PERF_RECORD_MISC_MMAP_BUILD_ID = 1U << 14, + PERF_RECORD_MISC_EXT_RESERVED = 1U << 15, }; enum perf_event_read_format { @@ -250,7 +251,7 @@ enum perf_event_read_format { PERF_FORMAT_MAX = 1U << 5, /* non-ABI */ }; -enum perf_callchain_context { +enum perf_callchain_context : uint64_t { PERF_CONTEXT_HV = static_cast<uint64_t>(-32), PERF_CONTEXT_KERNEL = static_cast<uint64_t>(-128), PERF_CONTEXT_USER = static_cast<uint64_t>(-512), diff --git a/src/trace_processor/importers/perf/perf_event_attr.cc b/src/trace_processor/importers/perf/perf_event_attr.cc index 1abf8d0aa..b6c3edce7 100644 --- a/src/trace_processor/importers/perf/perf_event_attr.cc +++ b/src/trace_processor/importers/perf/perf_event_attr.cc @@ -21,7 +21,12 @@ #include <cstring> #include <optional> +#include "perfetto/ext/base/string_view.h" +#include "src/trace_processor/importers/perf/perf_counter.h" #include "src/trace_processor/importers/perf/perf_event.h" +#include "src/trace_processor/storage/trace_storage.h" +#include "src/trace_processor/tables/profiler_tables_py.h" +#include "src/trace_processor/types/trace_processor_context.h" namespace perfetto::trace_processor::perf_importer { @@ -90,11 +95,41 @@ std::optional<size_t> IdOffsetFromEndOfNonSampleRecord( } } // namespace -PerfEventAttr::PerfEventAttr(perf_event_attr attr) - : attr_(std::move(attr)), +PerfEventAttr::PerfEventAttr(TraceProcessorContext* context, + tables::PerfSessionTable::Id perf_session_id, + perf_event_attr attr) + : context_(context), + perf_session_id_(perf_session_id), + attr_(std::move(attr)), time_offset_from_start_(TimeOffsetFromStartOfSampleRecord(attr_)), time_offset_from_end_(TimeOffsetFromEndOfNonSampleRecord(attr_)), id_offset_from_start_(IdOffsetFromStartOfSampleRecord(attr_)), id_offset_from_end_(IdOffsetFromEndOfNonSampleRecord(attr_)) {} +PerfEventAttr::~PerfEventAttr() = default; + +PerfCounter& PerfEventAttr::GetOrCreateCounter(uint32_t cpu) const { + auto it = counters_.find(cpu); + if (it == counters_.end()) { + it = counters_.emplace(cpu, CreateCounter(cpu)).first; + } + return it->second; +} + +PerfCounter PerfEventAttr::CreateCounter(uint32_t cpu) const { + tables::PerfCounterTrackTable::Row row; + row.name = context_->storage->InternString(base::StringView(event_name_)); + row.unit = context_->storage->InternString(base::StringView("")); + row.description = context_->storage->InternString(base::StringView("")); + row.perf_session_id = perf_session_id_; + row.cpu = cpu; + row.is_timebase = is_timebase(); + const auto counter_track_ref = + context_->storage->mutable_perf_counter_track_table() + ->Insert(std::move(row)) + .row_reference; + return PerfCounter(context_->storage->mutable_counter_table(), + std::move(counter_track_ref)); +} + } // namespace perfetto::trace_processor::perf_importer diff --git a/src/trace_processor/importers/perf/perf_event_attr.h b/src/trace_processor/importers/perf/perf_event_attr.h index 4f219aadf..57a0c769b 100644 --- a/src/trace_processor/importers/perf/perf_event_attr.h +++ b/src/trace_processor/importers/perf/perf_event_attr.h @@ -21,17 +21,28 @@ #include <cstddef> #include <cstdint> #include <optional> +#include <unordered_map> -#include "perfetto/ext/base/string_view.h" #include "perfetto/trace_processor/ref_counted.h" +#include "src/trace_processor/importers/perf/perf_counter.h" #include "src/trace_processor/importers/perf/perf_event.h" +#include "src/trace_processor/tables/profiler_tables_py.h" -namespace perfetto::trace_processor::perf_importer { +namespace perfetto::trace_processor { + +class TraceProcessorContext; + +namespace perf_importer { // Wrapper around a `perf_event_attr` object that add some helper methods. class PerfEventAttr : public RefCounted { public: - explicit PerfEventAttr(perf_event_attr attr); + PerfEventAttr(TraceProcessorContext* context, + tables::PerfSessionTable::Id perf_session_id_, + perf_event_attr attr); + ~PerfEventAttr(); + uint32_t type() const { return attr_.type; } + uint64_t config() const { return attr_.config; } uint64_t sample_type() const { return attr_.sample_type; } uint64_t read_format() const { return attr_.read_format; } bool sample_id_all() const { return !!attr_.sample_id_all; } @@ -48,12 +59,6 @@ class PerfEventAttr : public RefCounted { return attr_.freq ? std::make_optional(attr_.sample_freq) : std::nullopt; } - bool is_timebase() const { - // This is what simpleperf uses for events that are not supposed to sample - // TODO(b/334978369): Determine if there is a better way to figure this out. - return attr_.sample_period < (1ull << 62); - } - // Offset from the end of a record's payload to the time filed (if present). // To be used with non `PERF_RECORD_SAMPLE` records std::optional<size_t> time_offset_from_end() const { @@ -81,14 +86,33 @@ class PerfEventAttr : public RefCounted { return id_offset_from_end_; } + void set_event_name(std::string event_name) { + event_name_ = std::move(event_name); + } + + PerfCounter& GetOrCreateCounter(uint32_t cpu) const; + private: + bool is_timebase() const { + // This is what simpleperf uses for events that are not supposed to sample + // TODO(b/334978369): Determine if there is a better way to figure this out. + return attr_.sample_period < (1ull << 62); + } + + PerfCounter CreateCounter(uint32_t cpu) const; + + TraceProcessorContext* const context_; + tables::PerfSessionTable::Id perf_session_id_; perf_event_attr attr_; std::optional<size_t> time_offset_from_start_; std::optional<size_t> time_offset_from_end_; std::optional<size_t> id_offset_from_start_; std::optional<size_t> id_offset_from_end_; + mutable std::unordered_map<uint32_t, PerfCounter> counters_; + std::string event_name_; }; -} // namespace perfetto::trace_processor::perf_importer +} // namespace perf_importer +} // namespace perfetto::trace_processor #endif // SRC_TRACE_PROCESSOR_IMPORTERS_PERF_PERF_EVENT_ATTR_H_ diff --git a/src/trace_processor/importers/perf/perf_session.cc b/src/trace_processor/importers/perf/perf_session.cc index 86d983d0b..9923fcbad 100644 --- a/src/trace_processor/importers/perf/perf_session.cc +++ b/src/trace_processor/importers/perf/perf_session.cc @@ -20,17 +20,24 @@ #include <cstddef> #include <cstdint> #include <cstring> +#include <limits> #include <optional> +#include <string> #include <vector> #include "perfetto/base/logging.h" #include "perfetto/base/status.h" #include "perfetto/ext/base/flat_hash_map.h" #include "perfetto/ext/base/status_or.h" +#include "perfetto/ext/base/string_utils.h" +#include "perfetto/ext/base/string_view.h" #include "perfetto/trace_processor/ref_counted.h" #include "src/trace_processor/importers/perf/perf_event.h" #include "src/trace_processor/importers/perf/perf_event_attr.h" #include "src/trace_processor/importers/perf/reader.h" +#include "src/trace_processor/storage/trace_storage.h" +#include "src/trace_processor/types/trace_processor_context.h" +#include "src/trace_processor/util/build_id.h" namespace perfetto::trace_processor::perf_importer { namespace { @@ -46,11 +53,16 @@ base::StatusOr<RefPtr<PerfSession>> PerfSession::Builder::Build() { return base::ErrStatus("No perf_event_attr"); } - const PerfEventAttr base_attr(attr_with_ids_[0].attr); + auto perf_session_id = + context_->storage->mutable_perf_session_table()->Insert({}).id; + + const PerfEventAttr base_attr(context_, perf_session_id, + attr_with_ids_[0].attr); base::FlatHashMap<uint64_t, RefPtr<PerfEventAttr>> attrs_by_id; for (const auto& entry : attr_with_ids_) { - RefPtr<PerfEventAttr> attr(new PerfEventAttr(entry.attr)); + RefPtr<PerfEventAttr> attr( + new PerfEventAttr(context_, perf_session_id, entry.attr)); if (base_attr.sample_id_all() != attr->sample_id_all()) { return base::ErrStatus( "perf_event_attr with different sample_id_all values"); @@ -75,8 +87,9 @@ base::StatusOr<RefPtr<PerfSession>> PerfSession::Builder::Build() { return base::ErrStatus("No id offsets for multiple perf_event_attr"); } - return RefPtr<PerfSession>(new PerfSession( - perf_session_id_, std::move(attrs_by_id), attr_with_ids_.size() == 1)); + return RefPtr<PerfSession>(new PerfSession(context_, perf_session_id, + std::move(attrs_by_id), + attr_with_ids_.size() == 1)); } base::StatusOr<RefPtr<const PerfEventAttr>> PerfSession::FindAttrForRecord( @@ -133,4 +146,47 @@ RefPtr<const PerfEventAttr> PerfSession::FindAttrForEventId(uint64_t id) const { return RefPtr<const PerfEventAttr>(it->get()); } +void PerfSession::SetEventName(uint64_t event_id, std::string name) { + auto it = attrs_by_id_.Find(event_id); + if (!it) { + return; + } + (*it)->set_event_name(std::move(name)); +} + +void PerfSession::SetEventName(uint32_t type, + uint64_t config, + const std::string& name) { + for (auto it = attrs_by_id_.GetIterator(); it; ++it) { + if (it.value()->type() == type && it.value()->config() == config) { + it.value()->set_event_name(name); + } + } +} + +void PerfSession::AddBuildId(int32_t pid, + std::string filename, + BuildId build_id) { + build_ids_.Insert({pid, std::move(filename)}, std::move(build_id)); +} + +std::optional<BuildId> PerfSession::LookupBuildId( + uint32_t pid, + const std::string& filename) const { + // -1 is used in BUILD_ID feature to match any pid. + static constexpr int32_t kAnyPid = -1; + auto it = build_ids_.Find({static_cast<int32_t>(pid), filename}); + if (!it) { + it = build_ids_.Find({kAnyPid, filename}); + } + return it ? std::make_optional(*it) : std::nullopt; +} + +void PerfSession::SetCmdline(const std::vector<std::string>& args) { + context_->storage->mutable_perf_session_table() + ->FindById(perf_session_id_) + ->set_cmdline(context_->storage->InternString( + base::StringView(base::Join(args, " ")))); +} + } // namespace perfetto::trace_processor::perf_importer diff --git a/src/trace_processor/importers/perf/perf_session.h b/src/trace_processor/importers/perf/perf_session.h index abea41bec..796969c82 100644 --- a/src/trace_processor/importers/perf/perf_session.h +++ b/src/trace_processor/importers/perf/perf_session.h @@ -21,23 +21,31 @@ #include <cstddef> #include <cstdint> #include <optional> +#include <string> +#include <vector> #include "perfetto/ext/base/flat_hash_map.h" +#include "perfetto/ext/base/hash.h" #include "perfetto/ext/base/status_or.h" #include "perfetto/trace_processor/ref_counted.h" #include "perfetto/trace_processor/trace_blob_view.h" #include "src/trace_processor/importers/perf/perf_event.h" #include "src/trace_processor/importers/perf/perf_event_attr.h" +#include "src/trace_processor/tables/profiler_tables_py.h" +#include "src/trace_processor/util/build_id.h" -namespace perfetto::trace_processor::perf_importer { +namespace perfetto::trace_processor { + +class TraceProcessorContext; + +namespace perf_importer { // Helper to deal with perf_event_attr instances in a perf file. class PerfSession : public RefCounted { public: class Builder { public: - explicit Builder(uint32_t perf_session_id) - : perf_session_id_(perf_session_id) {} + explicit Builder(TraceProcessorContext* context) : context_(context) {} base::StatusOr<RefPtr<PerfSession>> Build(); Builder& AddAttrAndIds(perf_event_attr attr, std::vector<uint64_t> ids) { attr_with_ids_.push_back({std::move(attr), std::move(ids)}); @@ -50,11 +58,13 @@ class PerfSession : public RefCounted { std::vector<uint64_t> ids; }; - uint32_t perf_session_id_; + TraceProcessorContext* const context_; std::vector<PerfEventAttrWithIds> attr_with_ids_; }; - uint32_t perf_session_id() const { return perf_session_id_; } + tables::PerfSessionTable::Id perf_session_id() const { + return perf_session_id_; + } RefPtr<const PerfEventAttr> FindAttrForEventId(uint64_t id) const; @@ -62,11 +72,36 @@ class PerfSession : public RefCounted { const perf_event_header& header, const TraceBlobView& payload) const; + void SetCmdline(const std::vector<std::string>& args); + void SetEventName(uint64_t event_id, std::string name); + void SetEventName(uint32_t type, uint64_t config, const std::string& name); + + void AddBuildId(int32_t pid, std::string filename, BuildId build_id); + std::optional<BuildId> LookupBuildId(uint32_t pid, + const std::string& filename) const; + private: - PerfSession(uint32_t perf_session_id, + struct BuildIdMapKey { + int32_t pid; + std::string filename; + + struct Hasher { + size_t operator()(const BuildIdMapKey& k) const { + return static_cast<size_t>(base::Hasher::Combine(k.pid, k.filename)); + } + }; + + bool operator==(const BuildIdMapKey& o) const { + return pid == o.pid && filename == o.filename; + } + }; + + PerfSession(TraceProcessorContext* context, + tables::PerfSessionTable::Id perf_session_id, base::FlatHashMap<uint64_t, RefPtr<PerfEventAttr>> attrs_by_id, bool has_single_perf_event_attr) - : perf_session_id_(perf_session_id), + : context_(context), + perf_session_id_(perf_session_id), attrs_by_id_(std::move(attrs_by_id)), has_single_perf_event_attr_(has_single_perf_event_attr) {} @@ -74,15 +109,19 @@ class PerfSession : public RefCounted { const TraceBlobView& payload, uint64_t& id) const; - uint32_t perf_session_id_; + TraceProcessorContext* const context_; + tables::PerfSessionTable::Id perf_session_id_; base::FlatHashMap<uint64_t, RefPtr<PerfEventAttr>> attrs_by_id_; // Multiple ids can map to the same perf_event_attr. This member tells us // whether there was only one perf_event_attr (with potentially different ids // associated). This makes the attr lookup given a record trivial and not // dependant no having any id field in the records. bool has_single_perf_event_attr_; + + base::FlatHashMap<BuildIdMapKey, BuildId, BuildIdMapKey::Hasher> build_ids_; }; -} // namespace perfetto::trace_processor::perf_importer +} // namespace perf_importer +} // namespace perfetto::trace_processor #endif // SRC_TRACE_PROCESSOR_IMPORTERS_PERF_PERF_SESSION_H_ diff --git a/src/trace_processor/importers/perf/perf_session_unittest.cc b/src/trace_processor/importers/perf/perf_session_unittest.cc index 0ab85f23a..2b15af862 100644 --- a/src/trace_processor/importers/perf/perf_session_unittest.cc +++ b/src/trace_processor/importers/perf/perf_session_unittest.cc @@ -23,6 +23,8 @@ #include "perfetto/trace_processor/trace_blob.h" #include "perfetto/trace_processor/trace_blob_view.h" #include "src/trace_processor/importers/perf/perf_event.h" +#include "src/trace_processor/storage/trace_storage.h" +#include "src/trace_processor/types/trace_processor_context.h" #include "test/gtest_and_gmock.h" namespace perfetto::trace_processor::perf_importer { @@ -41,12 +43,16 @@ MATCHER_P(IsOkAndHolds, matcher, "") { } TEST(PerfSessionTest, NoAttrBuildFails) { - PerfSession::Builder builder(0); + TraceProcessorContext context; + context.storage.reset(new TraceStorage()); + PerfSession::Builder builder(&context); EXPECT_FALSE(builder.Build().ok()); } TEST(PerfSessionTest, OneAttrAndNoIdBuildSucceeds) { - PerfSession::Builder builder(0); + TraceProcessorContext context; + context.storage.reset(new TraceStorage()); + PerfSession::Builder builder(&context); perf_event_attr attr; attr.sample_id_all = false; attr.sample_type = PERF_SAMPLE_CALLCHAIN | PERF_SAMPLE_CPU | PERF_SAMPLE_TIME; @@ -61,7 +67,9 @@ TEST(PerfSessionTest, OneAttrAndNoIdBuildSucceeds) { } TEST(PerfSessionTest, MultipleAttrsAndNoIdBuildFails) { - PerfSession::Builder builder(0); + TraceProcessorContext context; + context.storage.reset(new TraceStorage()); + PerfSession::Builder builder(&context); perf_event_attr attr; attr.sample_id_all = true; attr.sample_type = PERF_SAMPLE_CALLCHAIN | PERF_SAMPLE_CPU | PERF_SAMPLE_TIME; @@ -71,7 +79,9 @@ TEST(PerfSessionTest, MultipleAttrsAndNoIdBuildFails) { } TEST(PerfSessionTest, MultipleIdsSameAttrAndNoIdCanExtractAttrFromRecord) { - PerfSession::Builder builder(0); + TraceProcessorContext context; + context.storage.reset(new TraceStorage()); + PerfSession::Builder builder(&context); perf_event_attr attr; attr.sample_id_all = true; attr.sample_type = PERF_SAMPLE_IP | PERF_SAMPLE_CPU | PERF_SAMPLE_TIME; @@ -95,7 +105,9 @@ TEST(PerfSessionTest, MultipleIdsSameAttrAndNoIdCanExtractAttrFromRecord) { } TEST(PerfSessionTest, NoCommonSampleIdAllBuildFails) { - PerfSession::Builder builder(0); + TraceProcessorContext context; + context.storage.reset(new TraceStorage()); + PerfSession::Builder builder(&context); perf_event_attr attr; attr.sample_id_all = true; attr.sample_type = PERF_SAMPLE_IDENTIFIER; @@ -111,7 +123,9 @@ TEST(PerfSessionTest, NoCommonSampleIdAllBuildFails) { } TEST(PerfSessionTest, NoCommonOffsetForSampleBuildFails) { - PerfSession::Builder builder(0); + TraceProcessorContext context; + context.storage.reset(new TraceStorage()); + PerfSession::Builder builder(&context); perf_event_attr attr; attr.sample_id_all = true; attr.sample_type = PERF_SAMPLE_IP | PERF_SAMPLE_ID; @@ -122,7 +136,9 @@ TEST(PerfSessionTest, NoCommonOffsetForSampleBuildFails) { } TEST(PerfSessionTest, NoCommonOffsetForNonSampleBuildFails) { - PerfSession::Builder builder(0); + TraceProcessorContext context; + context.storage.reset(new TraceStorage()); + PerfSession::Builder builder(&context); perf_event_attr attr; attr.sample_id_all = true; attr.sample_type = PERF_SAMPLE_ID | PERF_SAMPLE_TID; @@ -138,7 +154,9 @@ TEST(PerfSessionTest, NoCommonOffsetForNonSampleBuildFails) { } TEST(PerfSessionTest, NoCommonOffsetForNonSampleAndNoSampleIdAllBuildSucceeds) { - PerfSession::Builder builder(0); + TraceProcessorContext context; + context.storage.reset(new TraceStorage()); + PerfSession::Builder builder(&context); perf_event_attr attr; attr.sample_id_all = false; attr.sample_type = PERF_SAMPLE_IDENTIFIER | PERF_SAMPLE_TID; @@ -149,7 +167,9 @@ TEST(PerfSessionTest, NoCommonOffsetForNonSampleAndNoSampleIdAllBuildSucceeds) { } TEST(PerfSessionTest, MultiplesessionBuildSucceeds) { - PerfSession::Builder builder(0); + TraceProcessorContext context; + context.storage.reset(new TraceStorage()); + PerfSession::Builder builder(&context); perf_event_attr attr; attr.sample_id_all = true; attr.sample_type = PERF_SAMPLE_IP | PERF_SAMPLE_ID; @@ -159,7 +179,9 @@ TEST(PerfSessionTest, MultiplesessionBuildSucceeds) { } TEST(PerfSessionTest, FindAttrInRecordWithId) { - PerfSession::Builder builder(0); + TraceProcessorContext context; + context.storage.reset(new TraceStorage()); + PerfSession::Builder builder(&context); perf_event_attr attr; attr.sample_id_all = true; attr.sample_type = PERF_SAMPLE_IP | PERF_SAMPLE_ID; @@ -194,7 +216,9 @@ TEST(PerfSessionTest, FindAttrInRecordWithId) { } TEST(PerfSessionTest, FindAttrInRecordWithIdentifier) { - PerfSession::Builder builder(0); + TraceProcessorContext context; + context.storage.reset(new TraceStorage()); + PerfSession::Builder builder(&context); perf_event_attr attr; attr.sample_id_all = true; attr.sample_type = PERF_SAMPLE_IDENTIFIER | PERF_SAMPLE_IP; diff --git a/src/trace_processor/importers/perf/reader.h b/src/trace_processor/importers/perf/reader.h index faf31a215..25d4f4c3e 100644 --- a/src/trace_processor/importers/perf/reader.h +++ b/src/trace_processor/importers/perf/reader.h @@ -26,7 +26,9 @@ #include <type_traits> #include <vector> +#include "perfetto/ext/base/string_view.h" #include "perfetto/trace_processor/trace_blob_view.h" +#include "src/trace_processor/importers/perf/perf_event.h" namespace perfetto::trace_processor::perf_importer { @@ -45,6 +47,50 @@ class Reader { // methods are called. size_t size_left() const { return static_cast<size_t>(end_ - current_); } + bool ReadStringView(base::StringView& str, size_t size) { + if (size_left() < size) { + return false; + } + str = base::StringView(reinterpret_cast<const char*>(current_), size); + current_ += size; + return true; + } + + bool ReadPerfEventAttr(perf_event_attr& attr, size_t attr_size) { + const size_t bytes_to_read = std::min(attr_size, sizeof(attr)); + const size_t bytes_to_skip = attr_size - bytes_to_read; + static_assert(std::has_unique_object_representations_v<perf_event_attr>); + + if (size_left() < bytes_to_read + bytes_to_skip) { + return false; + } + + memset(&attr, 0, sizeof(attr)); + + return Read(&attr, bytes_to_read) && Skip(bytes_to_skip); + } + + bool ReadBlob(TraceBlobView& blob, uint32_t size) { + if (size_left() < size) { + return false; + } + blob = TraceBlobView(buffer_, + static_cast<size_t>(current_ - buffer_->data()), size); + current_ += size; + return true; + } + + bool ReadStringUntilEndOrNull(std::string& out) { + const uint8_t* ptr = current_; + while (ptr != end_ && *ptr != 0) { + ++ptr; + } + out = std::string(reinterpret_cast<const char*>(current_), + static_cast<size_t>(ptr - current_)); + current_ = ptr; + return true; + } + template <typename T> bool Read(T& obj) { static_assert(std::has_unique_object_representations_v<T>); diff --git a/src/trace_processor/importers/perf/record_parser.cc b/src/trace_processor/importers/perf/record_parser.cc new file mode 100644 index 000000000..c36d3dbe6 --- /dev/null +++ b/src/trace_processor/importers/perf/record_parser.cc @@ -0,0 +1,319 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/trace_processor/importers/perf/record_parser.h" + +#include <cstdint> +#include <optional> +#include <string> +#include <vector> + +#include "perfetto/base/logging.h" +#include "perfetto/base/status.h" +#include "perfetto/ext/base/string_view.h" +#include "perfetto/public/compiler.h" +#include "perfetto/trace_processor/ref_counted.h" +#include "src/trace_processor/importers/common/mapping_tracker.h" +#include "src/trace_processor/importers/common/process_tracker.h" +#include "src/trace_processor/importers/common/stack_profile_tracker.h" +#include "src/trace_processor/importers/perf/perf_counter.h" +#include "src/trace_processor/importers/perf/perf_event.h" +#include "src/trace_processor/importers/perf/perf_event_attr.h" +#include "src/trace_processor/importers/perf/reader.h" +#include "src/trace_processor/importers/perf/record.h" +#include "src/trace_processor/importers/perf/sample.h" +#include "src/trace_processor/importers/proto/perf_sample_tracker.h" +#include "src/trace_processor/importers/proto/profile_packet_utils.h" +#include "src/trace_processor/storage/stats.h" +#include "src/trace_processor/storage/trace_storage.h" +#include "src/trace_processor/tables/profiler_tables_py.h" +#include "src/trace_processor/util/build_id.h" +#include "src/trace_processor/util/status_macros.h" + +namespace perfetto { +namespace trace_processor { +namespace perf_importer { +namespace { + +CreateMappingParams BuildCreateMappingParams( + const CommonMmapRecordFields& fields, + std::string filename, + std::optional<BuildId> build_id) { + return {AddressRange::FromStartAndSize(fields.addr, fields.len), fields.pgoff, + // start_offset: This is the offset into the file where the ELF header + // starts. We assume all file mappings are ELF files an thus this + // offset is 0. + 0, + // load_bias: This can only be read out of the actual ELF file, which + // we do not have here, so we set it to 0. When symbolizing we will + // hopefully have the real load bias and we can compensate there for a + // possible mismatch. + 0, std::move(filename), std::move(build_id)}; +} + +bool IsInKernel(protos::pbzero::Profiling::CpuMode cpu_mode) { + switch (cpu_mode) { + case protos::pbzero::Profiling::MODE_UNKNOWN: + PERFETTO_FATAL("Unknown CPU mode"); + case protos::pbzero::Profiling::MODE_GUEST_KERNEL: + case protos::pbzero::Profiling::MODE_KERNEL: + return true; + case protos::pbzero::Profiling::MODE_USER: + case protos::pbzero::Profiling::MODE_HYPERVISOR: + case protos::pbzero::Profiling::MODE_GUEST_USER: + return false; + } + PERFETTO_FATAL("For GCC."); +} + +} // namespace + +using FramesTable = tables::StackProfileFrameTable; +using CallsitesTable = tables::StackProfileCallsiteTable; + +RecordParser::RecordParser(TraceProcessorContext* context) + : context_(context) {} + +RecordParser::~RecordParser() = default; + +void RecordParser::ParsePerfRecord(int64_t ts, Record record) { + if (base::Status status = ParseRecord(ts, std::move(record)); !status.ok()) { + context_->storage->IncrementStats(record.header.type == PERF_RECORD_SAMPLE + ? stats::perf_samples_skipped + : stats::perf_record_skipped); + } +} + +base::Status RecordParser::ParseRecord(int64_t ts, Record record) { + switch (record.header.type) { + case PERF_RECORD_COMM: + return ParseComm(std::move(record)); + + case PERF_RECORD_SAMPLE: + return ParseSample(ts, std::move(record)); + + case PERF_RECORD_MMAP: + return ParseMmap(std::move(record)); + + case PERF_RECORD_MMAP2: + return ParseMmap2(std::move(record)); + + case PERF_RECORD_AUX: + case PERF_RECORD_AUXTRACE: + case PERF_RECORD_AUXTRACE_INFO: + // These should be dealt with at tokenization time + PERFETTO_FATAL("Unexpected record type at parsing time: %" PRIu32, + record.header.type); + + default: + context_->storage->IncrementIndexedStats( + stats::perf_unknown_record_type, + static_cast<int>(record.header.type)); + return base::ErrStatus("Unknown PERF_RECORD with type %" PRIu32, + record.header.type); + } +} + +base::Status RecordParser::ParseSample(int64_t ts, Record record) { + Sample sample; + RETURN_IF_ERROR(sample.Parse(ts, record)); + + if (!sample.period.has_value() && record.attr != nullptr) { + sample.period = record.attr->sample_period(); + } + + return InternSample(std::move(sample)); +} + +base::Status RecordParser::InternSample(Sample sample) { + if (!sample.time.has_value()) { + // We do not really use this TS as this is using the perf clock, but we need + // it to be present so that we can compute the trace_ts done during + // tokenization. (Actually at tokenization time we do estimate a trace_ts if + // no perf ts is present, but for samples we want this to be as accurate as + // possible) + base::ErrStatus("Can not parse samples with no PERF_SAMPLE_TIME field"); + } + + if (!sample.pid_tid.has_value()) { + base::ErrStatus("Can not parse samples with no PERF_SAMPLE_TID field"); + } + + if (!sample.cpu.has_value()) { + base::ErrStatus("Can not parse samples with no PERF_SAMPLE_CPU field"); + } + + UniqueTid utid = context_->process_tracker->UpdateThread(sample.pid_tid->tid, + sample.pid_tid->pid); + const auto upid = *context_->storage->thread_table() + .FindById(tables::ThreadTable::Id(utid)) + ->upid(); + + if (sample.callchain.empty() && sample.ip.has_value()) { + sample.callchain.push_back(Sample::Frame{sample.cpu_mode, *sample.ip}); + } + std::optional<CallsiteId> callsite_id = + InternCallchain(upid, sample.callchain); + + context_->storage->mutable_perf_sample_table()->Insert( + {sample.trace_ts, utid, *sample.cpu, + context_->storage->InternString( + ProfilePacketUtils::StringifyCpuMode(sample.cpu_mode)), + callsite_id, std::nullopt, sample.perf_session->perf_session_id()}); + + return UpdateCounters(sample); +} + +std::optional<CallsiteId> RecordParser::InternCallchain( + UniquePid upid, + const std::vector<Sample::Frame>& callchain) { + if (callchain.empty()) { + return std::nullopt; + } + + auto& stack_profile_tracker = *context_->stack_profile_tracker; + auto& mapping_tracker = *context_->mapping_tracker; + + std::optional<CallsiteId> parent; + uint32_t depth = 0; + for (auto it = callchain.rbegin(); it != callchain.rend(); ++it) { + VirtualMemoryMapping* mapping; + if (IsInKernel(it->cpu_mode)) { + mapping = mapping_tracker.FindKernelMappingForAddress(it->ip); + } else { + mapping = mapping_tracker.FindUserMappingForAddress(upid, it->ip); + } + + if (!mapping) { + context_->storage->IncrementStats(stats::perf_dummy_mapping_used); + // Simpleperf will not create mappings for anonymous executable mappings + // which are used by JITted code (e.g. V8 JavaScript). + mapping = mapping_tracker.GetDummyMapping(); + } + + const FrameId frame_id = + mapping->InternFrame(mapping->ToRelativePc(it->ip), ""); + + parent = stack_profile_tracker.InternCallsite(parent, frame_id, depth); + depth++; + } + return parent; +} + +base::Status RecordParser::ParseComm(Record record) { + Reader reader(record.payload.copy()); + uint32_t pid; + uint32_t tid; + std::string comm; + if (!reader.Read(pid) || !reader.Read(tid) || !reader.ReadCString(comm)) { + return base::ErrStatus("Failed to parse PERF_RECORD_COMM"); + } + + context_->process_tracker->UpdateThread(tid, pid); + context_->process_tracker->UpdateThreadName( + tid, context_->storage->InternString(base::StringView(comm)), + ThreadNamePriority::kFtrace); + + return base::OkStatus(); +} + +base::Status RecordParser::ParseMmap(Record record) { + MmapRecord mmap; + RETURN_IF_ERROR(mmap.Parse(record)); + std::optional<BuildId> build_id = + record.session->LookupBuildId(mmap.pid, mmap.filename); + if (IsInKernel(record.GetCpuMode())) { + context_->mapping_tracker->CreateKernelMemoryMapping( + BuildCreateMappingParams(mmap, std::move(mmap.filename), + std::move(build_id))); + return base::OkStatus(); + } + + context_->mapping_tracker->CreateUserMemoryMapping( + GetUpid(mmap), BuildCreateMappingParams(mmap, std::move(mmap.filename), + std::move(build_id))); + + return base::OkStatus(); +} + +util::Status RecordParser::ParseMmap2(Record record) { + Mmap2Record mmap2; + RETURN_IF_ERROR(mmap2.Parse(record)); + std::optional<BuildId> build_id = mmap2.GetBuildId(); + if (!build_id.has_value()) { + build_id = record.session->LookupBuildId(mmap2.pid, mmap2.filename); + } + if (IsInKernel(record.GetCpuMode())) { + context_->mapping_tracker->CreateKernelMemoryMapping( + BuildCreateMappingParams(mmap2, std::move(mmap2.filename), + std::move(build_id))); + return base::OkStatus(); + } + + context_->mapping_tracker->CreateUserMemoryMapping( + GetUpid(mmap2), BuildCreateMappingParams(mmap2, std::move(mmap2.filename), + std::move(build_id))); + + return base::OkStatus(); +} + +UniquePid RecordParser::GetUpid(const CommonMmapRecordFields& fields) const { + UniqueTid utid = + context_->process_tracker->UpdateThread(fields.tid, fields.pid); + auto upid = context_->storage->thread_table() + .FindById(tables::ThreadTable::Id(utid)) + ->upid(); + PERFETTO_CHECK(upid.has_value()); + return *upid; +} + +base::Status RecordParser::UpdateCounters(const Sample& sample) { + if (!sample.read_groups.empty()) { + return UpdateCountersInReadGroups(sample); + } + + if (!sample.period.has_value() && !sample.attr->sample_period().has_value()) { + return base::ErrStatus("No period for sample"); + } + + uint64_t period = sample.period.has_value() ? *sample.period + : *sample.attr->sample_period(); + sample.attr->GetOrCreateCounter(*sample.cpu) + .AddDelta(sample.trace_ts, static_cast<double>(period)); + return base::OkStatus(); +} + +base::Status RecordParser::UpdateCountersInReadGroups(const Sample& sample) { + if (!sample.cpu.has_value()) { + return base::ErrStatus("No cpu for sample"); + } + + for (const auto& entry : sample.read_groups) { + RefPtr<const PerfEventAttr> attr = + sample.perf_session->FindAttrForEventId(*entry.event_id); + if (PERFETTO_UNLIKELY(!attr)) { + return base::ErrStatus("No perf_event_attr for id %" PRIu64, + *entry.event_id); + } + attr->GetOrCreateCounter(*sample.cpu) + .AddCount(sample.trace_ts, static_cast<double>(entry.value)); + } + return base::OkStatus(); +} + +} // namespace perf_importer +} // namespace trace_processor +} // namespace perfetto diff --git a/src/trace_processor/importers/perf/record_parser.h b/src/trace_processor/importers/perf/record_parser.h new file mode 100644 index 000000000..0a71b49a0 --- /dev/null +++ b/src/trace_processor/importers/perf/record_parser.h @@ -0,0 +1,74 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SRC_TRACE_PROCESSOR_IMPORTERS_PERF_RECORD_PARSER_H_ +#define SRC_TRACE_PROCESSOR_IMPORTERS_PERF_RECORD_PARSER_H_ + +#include <stdint.h> +#include <cstdint> +#include <vector> + +#include "perfetto/base/status.h" +#include "src/trace_processor/importers/common/trace_parser.h" +#include "src/trace_processor/importers/perf/mmap_record.h" +#include "src/trace_processor/importers/perf/record.h" +#include "src/trace_processor/importers/perf/sample.h" +#include "src/trace_processor/storage/trace_storage.h" + +namespace perfetto { +namespace trace_processor { + +class TraceProcessorContext; + +namespace perf_importer { + +class PerfDataTracker; +class Reader; + +// Parses samples from perf.data files. +class RecordParser : public PerfRecordParser { + public: + explicit RecordParser(TraceProcessorContext*); + ~RecordParser() override; + + void ParsePerfRecord(int64_t timestamp, Record record) override; + + private: + base::Status ParseRecord(int64_t timestamp, Record record); + base::Status ParseSample(int64_t ts, Record record); + base::Status ParseComm(Record record); + base::Status ParseMmap(Record record); + base::Status ParseMmap2(Record record); + + base::Status InternSample(Sample sample); + + base::Status UpdateCounters(const Sample& sample); + base::Status UpdateCountersInReadGroups(const Sample& sample); + + std::optional<CallsiteId> InternCallchain( + UniquePid upid, + const std::vector<Sample::Frame>& callchain); + + UniquePid GetUpid(const CommonMmapRecordFields& fields) const; + + TraceProcessorContext* context_ = nullptr; +}; + +} // namespace perf_importer +} // namespace trace_processor +} // namespace perfetto + +#endif // SRC_TRACE_PROCESSOR_IMPORTERS_PERF_RECORD_PARSER_H_ diff --git a/src/trace_processor/importers/perf/sample.cc b/src/trace_processor/importers/perf/sample.cc new file mode 100644 index 000000000..63ded53c9 --- /dev/null +++ b/src/trace_processor/importers/perf/sample.cc @@ -0,0 +1,252 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/trace_processor/importers/perf/sample.h" + +#include <cstdint> + +#include "perfetto/base/logging.h" +#include "perfetto/base/status.h" +#include "perfetto/public/compiler.h" +#include "src/trace_processor/importers/perf/reader.h" +#include "src/trace_processor/importers/perf/record.h" + +namespace perfetto::trace_processor::perf_importer { +namespace { + +bool ParseSampleReadGroup(Reader& reader, + uint64_t read_format, + uint64_t num_records, + std::vector<Sample::ReadGroup>& out) { + out.resize(num_records); + for (auto& read : out) { + if (PERFETTO_UNLIKELY(!reader.Read(read.value))) { + return false; + } + + if (read_format & PERF_FORMAT_ID) { + if (PERFETTO_UNLIKELY(!reader.ReadOptional(read.event_id))) { + return false; + } + } + + if (read_format & PERF_FORMAT_LOST) { + uint64_t lost; + if (PERFETTO_UNLIKELY(!reader.Read(lost))) { + return false; + } + } + } + + return true; +} + +bool ParseSampleRead(Reader& reader, + uint64_t read_format, + std::vector<Sample::ReadGroup>& out) { + uint64_t value_or_nr; + + if (PERFETTO_UNLIKELY(!reader.Read(value_or_nr))) { + return false; + } + + if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) { + uint64_t total_time_enabled; + if (PERFETTO_UNLIKELY(!reader.Read(total_time_enabled))) { + return false; + } + } + + if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) { + uint64_t total_time_running; + if (PERFETTO_UNLIKELY(!reader.Read(total_time_running))) { + return false; + } + } + + if (read_format & PERF_FORMAT_GROUP) { + return ParseSampleReadGroup(reader, read_format, value_or_nr, out); + } + + std::optional<uint64_t> event_id; + if (read_format & PERF_FORMAT_ID) { + event_id.emplace(0); + if (PERFETTO_UNLIKELY(!reader.ReadOptional(event_id))) { + return false; + } + } + + if (read_format & PERF_FORMAT_LOST) { + uint64_t lost; + if (PERFETTO_UNLIKELY(!reader.Read(lost))) { + return false; + } + } + + out.push_back({event_id, value_or_nr}); + + return true; +} + +protos::pbzero::Profiling::CpuMode PerfCallchainContextToCpuMode(uint64_t ip) { + switch (ip) { + case PERF_CONTEXT_HV: + return protos::pbzero::Profiling::MODE_HYPERVISOR; + case PERF_CONTEXT_KERNEL: + return protos::pbzero::Profiling::MODE_KERNEL; + case PERF_CONTEXT_USER: + return protos::pbzero::Profiling::MODE_USER; + case PERF_CONTEXT_GUEST_KERNEL: + return protos::pbzero::Profiling::MODE_GUEST_KERNEL; + case PERF_CONTEXT_GUEST_USER: + return protos::pbzero::Profiling::MODE_GUEST_USER; + case PERF_CONTEXT_GUEST: + default: + return protos::pbzero::Profiling::MODE_UNKNOWN; + } + PERFETTO_FATAL("For GCC"); +} + +bool IsPerfContextMark(uint64_t ip) { + return ip >= PERF_CONTEXT_MAX; +} + +bool ParseSampleCallchain(Reader& reader, + protos::pbzero::Profiling::CpuMode cpu_mode, + std::vector<Sample::Frame>& out) { + uint64_t nr; + if (PERFETTO_UNLIKELY(!reader.Read(nr))) { + return false; + } + + std::vector<Sample::Frame> frames; + frames.reserve(nr); + for (; nr != 0; --nr) { + uint64_t ip; + if (PERFETTO_UNLIKELY(!reader.Read(ip))) { + return false; + } + if (PERFETTO_UNLIKELY(IsPerfContextMark(ip))) { + cpu_mode = PerfCallchainContextToCpuMode(ip); + continue; + } + frames.push_back({cpu_mode, ip}); + } + + out = std::move(frames); + return true; +} +} // namespace + +base::Status Sample::Parse(int64_t in_trace_ts, const Record& record) { + PERFETTO_CHECK(record.attr); + const uint64_t sample_type = record.attr->sample_type(); + + trace_ts = in_trace_ts; + cpu_mode = record.GetCpuMode(); + perf_session = record.session; + attr = record.attr; + + Reader reader(record.payload.copy()); + + std::optional<uint64_t> identifier; + if (sample_type & PERF_SAMPLE_IDENTIFIER) { + if (PERFETTO_UNLIKELY(!reader.ReadOptional(identifier))) { + return base ::ErrStatus("Not enough data to read PERF_SAMPLE_IDENTIFIER"); + } + } + + if (sample_type & PERF_SAMPLE_IP) { + if (PERFETTO_UNLIKELY(!reader.ReadOptional(ip))) { + return base ::ErrStatus("Not enough data to read PERF_SAMPLE_IP"); + } + } + + if (sample_type & PERF_SAMPLE_TID) { + if (PERFETTO_UNLIKELY(!reader.ReadOptional(pid_tid))) { + return base ::ErrStatus("Not enough data to read PERF_SAMPLE_TID"); + } + } + + if (sample_type & PERF_SAMPLE_TIME) { + if (PERFETTO_UNLIKELY(!reader.ReadOptional(time))) { + return base ::ErrStatus("Not enough data to read PERF_SAMPLE_TIME"); + } + } + + if (sample_type & PERF_SAMPLE_ADDR) { + if (PERFETTO_UNLIKELY(!reader.ReadOptional(addr))) { + return base ::ErrStatus("Not enough data to read PERF_SAMPLE_ADDR"); + } + } + + if (sample_type & PERF_SAMPLE_ID) { + if (PERFETTO_UNLIKELY(!reader.ReadOptional(id))) { + return base ::ErrStatus("Not enough data to read PERF_SAMPLE_ID"); + } + } + + if (identifier.has_value()) { + if (!id.has_value()) { + id = identifier; + } else if (PERFETTO_UNLIKELY(*identifier != *id)) { + return base::ErrStatus("ID and IDENTIFIER mismatch"); + } + } + + if (sample_type & PERF_SAMPLE_STREAM_ID) { + if (PERFETTO_UNLIKELY(!reader.ReadOptional(stream_id))) { + return base ::ErrStatus("Not enough data to read PERF_SAMPLE_STREAM_ID"); + } + } + + if (sample_type & PERF_SAMPLE_CPU) { + struct { + int32_t cpu; + int32_t unused; + } tmp; + if (PERFETTO_UNLIKELY(!reader.Read(tmp))) { + return base ::ErrStatus("Not enough data to read PERF_SAMPLE_CPU"); + } + cpu = tmp.cpu; + } + + if (sample_type & PERF_SAMPLE_PERIOD) { + if (PERFETTO_UNLIKELY(!reader.ReadOptional(period))) { + return base ::ErrStatus("Not enough data to read PERF_SAMPLE_PERIOD"); + } + } + + if (sample_type & PERF_SAMPLE_READ) { + if (PERFETTO_UNLIKELY( + !ParseSampleRead(reader, attr->read_format(), read_groups))) { + return base::ErrStatus("Failed to read PERF_SAMPLE_READ field"); + } + if (read_groups.empty()) { + return base::ErrStatus("No data in PERF_SAMPLE_READ field"); + } + } + + if (sample_type & PERF_SAMPLE_CALLCHAIN) { + if (PERFETTO_UNLIKELY(!ParseSampleCallchain(reader, cpu_mode, callchain))) { + return base::ErrStatus("Failed to read PERF_SAMPLE_CALLCHAIN field"); + } + } + + return base::OkStatus(); +} + +} // namespace perfetto::trace_processor::perf_importer diff --git a/src/trace_processor/importers/perf/sample.h b/src/trace_processor/importers/perf/sample.h new file mode 100644 index 000000000..532b6131f --- /dev/null +++ b/src/trace_processor/importers/perf/sample.h @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SRC_TRACE_PROCESSOR_IMPORTERS_PERF_SAMPLE_H_ +#define SRC_TRACE_PROCESSOR_IMPORTERS_PERF_SAMPLE_H_ + +#include <cstdint> +#include <optional> +#include <vector> + +#include "perfetto/base/status.h" +#include "perfetto/trace_processor/ref_counted.h" +#include "protos/perfetto/trace/profiling/profile_packet.pbzero.h" +#include "src/trace_processor/importers/perf/perf_event_attr.h" +#include "src/trace_processor/importers/perf/perf_session.h" + +namespace perfetto::trace_processor::perf_importer { + +struct Record; + +struct Sample { + struct Frame { + protos::pbzero::Profiling::CpuMode cpu_mode; + uint64_t ip; + }; + + struct PidTid { + uint32_t pid; + uint32_t tid; + }; + + struct ReadGroup { + std::optional<uint64_t> event_id; + uint64_t value; + }; + + int64_t trace_ts; + protos::pbzero::Profiling::CpuMode cpu_mode; + RefPtr<PerfSession> perf_session; + RefPtr<const PerfEventAttr> attr; + + std::optional<uint64_t> ip; + std::optional<PidTid> pid_tid; + std::optional<uint64_t> time; + std::optional<uint64_t> addr; + std::optional<uint64_t> id; + std::optional<uint64_t> stream_id; + std::optional<uint32_t> cpu; + std::optional<uint64_t> period; + std::vector<ReadGroup> read_groups; + std::vector<Frame> callchain; + + base::Status Parse(int64_t trace_ts, const Record& record); +}; + +} // namespace perfetto::trace_processor::perf_importer + +#endif // SRC_TRACE_PROCESSOR_IMPORTERS_PERF_SAMPLE_H_ diff --git a/src/trace_processor/importers/proto/BUILD.gn b/src/trace_processor/importers/proto/BUILD.gn index dde995cd9..effda8244 100644 --- a/src/trace_processor/importers/proto/BUILD.gn +++ b/src/trace_processor/importers/proto/BUILD.gn @@ -18,6 +18,8 @@ source_set("minimal") { sources = [ "active_chrome_processes_tracker.cc", "active_chrome_processes_tracker.h", + "args_parser.cc", + "args_parser.h", "chrome_string_lookup.cc", "chrome_string_lookup.h", "chrome_system_probes_module.cc", @@ -113,6 +115,8 @@ source_set("full") { "additional_modules.h", "android_camera_event_module.cc", "android_camera_event_module.h", + "android_input_event_module.cc", + "android_input_event_module.h", "android_probes_module.cc", "android_probes_module.h", "android_probes_parser.cc", diff --git a/src/trace_processor/importers/proto/additional_modules.cc b/src/trace_processor/importers/proto/additional_modules.cc index 91645c393..cf2ee84ed 100644 --- a/src/trace_processor/importers/proto/additional_modules.cc +++ b/src/trace_processor/importers/proto/additional_modules.cc @@ -18,6 +18,7 @@ #include "src/trace_processor/importers/etw/etw_module_impl.h" #include "src/trace_processor/importers/ftrace/ftrace_module_impl.h" #include "src/trace_processor/importers/proto/android_camera_event_module.h" +#include "src/trace_processor/importers/proto/android_input_event_module.h" #include "src/trace_processor/importers/proto/android_probes_module.h" #include "src/trace_processor/importers/proto/graphics_event_module.h" #include "src/trace_processor/importers/proto/heap_graph_module.h" @@ -45,6 +46,7 @@ void RegisterAdditionalModules(TraceProcessorContext* context) { context->modules.emplace_back(new MetadataModule(context)); context->modules.emplace_back(new V8Module(context)); context->modules.emplace_back(new WinscopeModule(context)); + context->modules.emplace_back(new AndroidInputEventModule(context)); // Ftrace/Etw modules are special, because it has one extra method for parsing // ftrace/etw packets. So we need to store a pointer to it separately. diff --git a/src/trace_processor/importers/proto/android_input_event_module.cc b/src/trace_processor/importers/proto/android_input_event_module.cc new file mode 100644 index 000000000..26671a77f --- /dev/null +++ b/src/trace_processor/importers/proto/android_input_event_module.cc @@ -0,0 +1,147 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/trace_processor/importers/proto/android_input_event_module.h" + +#include "protos/perfetto/trace/android/android_input_event.pbzero.h" +#include "src/trace_processor/importers/common/args_tracker.h" +#include "src/trace_processor/importers/proto/args_parser.h" +#include "src/trace_processor/importers/proto/trace.descriptor.h" +#include "src/trace_processor/storage/trace_storage.h" +#include "src/trace_processor/tables/android_tables_py.h" +#include "src/trace_processor/types/trace_processor_context.h" + +namespace perfetto::trace_processor { + +using perfetto::protos::pbzero::AndroidInputEvent; +using perfetto::protos::pbzero::AndroidKeyEvent; +using perfetto::protos::pbzero::AndroidMotionEvent; +using perfetto::protos::pbzero::AndroidWindowInputDispatchEvent; +using perfetto::protos::pbzero::TracePacket; + +AndroidInputEventModule::AndroidInputEventModule(TraceProcessorContext* context) + : context_(*context), args_parser_(pool_) { + pool_.AddFromFileDescriptorSet(kTraceDescriptor.data(), + kTraceDescriptor.size()); + RegisterForField(TracePacket::kAndroidInputEventFieldNumber, context); +} + +void AndroidInputEventModule::ParseTracePacketData( + const TracePacket::Decoder& decoder, + int64_t packet_ts, + const TracePacketData&, + uint32_t field_id) { + if (field_id != TracePacket::kAndroidInputEventFieldNumber) + return; + + auto input_event = AndroidInputEvent::Decoder(decoder.android_input_event()); + + constexpr static auto supported_fields = std::array{ + AndroidInputEvent::kDispatcherMotionEventFieldNumber, + AndroidInputEvent::kDispatcherMotionEventRedactedFieldNumber, + AndroidInputEvent::kDispatcherKeyEventFieldNumber, + AndroidInputEvent::kDispatcherKeyEventRedactedFieldNumber, + AndroidInputEvent::kDispatcherWindowDispatchEventFieldNumber, + AndroidInputEvent::kDispatcherWindowDispatchEventRedactedFieldNumber}; + + for (auto sub_field_id : supported_fields) { + auto sub_field = input_event.Get(static_cast<uint32_t>(sub_field_id)); + if (!sub_field.valid()) + continue; + + switch (sub_field_id) { + case AndroidInputEvent::kDispatcherMotionEventFieldNumber: + case AndroidInputEvent::kDispatcherMotionEventRedactedFieldNumber: + ParseMotionEvent(packet_ts, sub_field.as_bytes()); + return; + case AndroidInputEvent::kDispatcherKeyEventFieldNumber: + case AndroidInputEvent::kDispatcherKeyEventRedactedFieldNumber: + ParseKeyEvent(packet_ts, sub_field.as_bytes()); + return; + case AndroidInputEvent::kDispatcherWindowDispatchEventFieldNumber: + case AndroidInputEvent::kDispatcherWindowDispatchEventRedactedFieldNumber: + ParseWindowDispatchEvent(packet_ts, sub_field.as_bytes()); + return; + } + } +} + +void AndroidInputEventModule::ParseMotionEvent( + int64_t packet_ts, + const protozero::ConstBytes& bytes) { + AndroidMotionEvent::Decoder event_proto(bytes); + tables::AndroidMotionEventsTable::Row event_row; + event_row.event_id = event_proto.event_id(); + event_row.ts = event_proto.event_time_nanos(); + + auto event_row_id = context_.storage->mutable_android_motion_events_table() + ->Insert(event_row) + .id; + auto inserter = context_.args_tracker->AddArgsTo(event_row_id); + ArgsParser writer{packet_ts, inserter, *context_.storage}; + + base::Status status = + args_parser_.ParseMessage(bytes, ".perfetto.protos.AndroidMotionEvent", + nullptr /*parse all fields*/, writer); + if (!status.ok()) + context_.storage->IncrementStats(stats::android_input_event_parse_errors); +} + +void AndroidInputEventModule::ParseKeyEvent( + int64_t packet_ts, + const protozero::ConstBytes& bytes) { + AndroidKeyEvent::Decoder event_proto(bytes); + tables::AndroidKeyEventsTable::Row event_row; + event_row.event_id = event_proto.event_id(); + event_row.ts = event_proto.event_time_nanos(); + + auto event_row_id = context_.storage->mutable_android_key_events_table() + ->Insert(event_row) + .id; + auto inserter = context_.args_tracker->AddArgsTo(event_row_id); + ArgsParser writer{packet_ts, inserter, *context_.storage}; + + base::Status status = + args_parser_.ParseMessage(bytes, ".perfetto.protos.AndroidKeyEvent", + nullptr /*parse all fields*/, writer); + if (!status.ok()) + context_.storage->IncrementStats(stats::android_input_event_parse_errors); +} + +void AndroidInputEventModule::ParseWindowDispatchEvent( + int64_t packet_ts, + const protozero::ConstBytes& bytes) { + AndroidWindowInputDispatchEvent::Decoder event_proto(bytes); + tables::AndroidInputEventDispatchTable::Row event_row; + event_row.event_id = event_proto.event_id(); + event_row.vsync_id = event_proto.vsync_id(); + event_row.window_id = event_proto.window_id(); + + auto event_row_id = + context_.storage->mutable_android_input_event_dispatch_table() + ->Insert(event_row) + .id; + auto inserter = context_.args_tracker->AddArgsTo(event_row_id); + ArgsParser writer{packet_ts, inserter, *context_.storage}; + + base::Status status = args_parser_.ParseMessage( + bytes, ".perfetto.protos.AndroidWindowInputDispatchEvent", + nullptr /*parse all fields*/, writer); + if (!status.ok()) + context_.storage->IncrementStats(stats::android_input_event_parse_errors); +} + +} // namespace perfetto::trace_processor diff --git a/src/trace_processor/importers/proto/android_input_event_module.h b/src/trace_processor/importers/proto/android_input_event_module.h new file mode 100644 index 000000000..c31090909 --- /dev/null +++ b/src/trace_processor/importers/proto/android_input_event_module.h @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SRC_TRACE_PROCESSOR_IMPORTERS_PROTO_ANDROID_INPUT_EVENT_MODULE_H_ +#define SRC_TRACE_PROCESSOR_IMPORTERS_PROTO_ANDROID_INPUT_EVENT_MODULE_H_ + +#include <cstdint> +#include "perfetto/base/build_config.h" +#include "protos/perfetto/trace/trace_packet.pbzero.h" +#include "src/trace_processor/importers/common/parser_types.h" +#include "src/trace_processor/importers/proto/proto_importer_module.h" +#include "src/trace_processor/util/descriptors.h" +#include "src/trace_processor/util/proto_to_args_parser.h" + +namespace perfetto::trace_processor { + +class AndroidInputEventModule : public ProtoImporterModule { + public: + explicit AndroidInputEventModule(TraceProcessorContext* context); + + void ParseTracePacketData(const protos::pbzero::TracePacket::Decoder&, + int64_t packet_ts, + const TracePacketData&, + uint32_t field_id) override; + + private: + TraceProcessorContext& context_; + DescriptorPool pool_; + util::ProtoToArgsParser args_parser_; + + void ParseMotionEvent(int64_t packet_ts, const protozero::ConstBytes& bytes); + void ParseKeyEvent(int64_t packet_ts, const protozero::ConstBytes& bytes); + void ParseWindowDispatchEvent(int64_t packet_ts, const protozero::ConstBytes& bytes); +}; + +} // namespace perfetto::trace_processor + +#endif // SRC_TRACE_PROCESSOR_IMPORTERS_PROTO_ANDROID_INPUT_EVENT_MODULE_H_ diff --git a/src/trace_processor/importers/proto/args_parser.cc b/src/trace_processor/importers/proto/args_parser.cc new file mode 100644 index 000000000..2aeb785c4 --- /dev/null +++ b/src/trace_processor/importers/proto/args_parser.cc @@ -0,0 +1,130 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/trace_processor/importers/proto/args_parser.h" + +#include "perfetto/ext/base/base64.h" +#include "src/trace_processor/importers/json/json_utils.h" + +namespace perfetto::trace_processor { + +using BoundInserter = ArgsTracker::BoundInserter; + +ArgsParser::ArgsParser(int64_t packet_timestamp, + BoundInserter& inserter, + TraceStorage& storage, + PacketSequenceStateGeneration* sequence_state, + bool support_json) + : support_json_(support_json), + packet_timestamp_(packet_timestamp), + sequence_state_(sequence_state), + inserter_(inserter), + storage_(storage) {} + +ArgsParser::~ArgsParser() = default; + +void ArgsParser::AddInteger(const Key& key, int64_t value) { + inserter_.AddArg(storage_.InternString(base::StringView(key.flat_key)), + storage_.InternString(base::StringView(key.key)), + Variadic::Integer(value)); +} + +void ArgsParser::AddUnsignedInteger(const Key& key, uint64_t value) { + inserter_.AddArg(storage_.InternString(base::StringView(key.flat_key)), + storage_.InternString(base::StringView(key.key)), + Variadic::UnsignedInteger(value)); +} + +void ArgsParser::AddString(const Key& key, const protozero::ConstChars& value) { + inserter_.AddArg(storage_.InternString(base::StringView(key.flat_key)), + storage_.InternString(base::StringView(key.key)), + Variadic::String(storage_.InternString(value))); +} + +void ArgsParser::AddString(const Key& key, const std::string& value) { + inserter_.AddArg( + storage_.InternString(base::StringView(key.flat_key)), + storage_.InternString(base::StringView(key.key)), + Variadic::String(storage_.InternString(base::StringView(value)))); +} + +void ArgsParser::AddDouble(const Key& key, double value) { + inserter_.AddArg(storage_.InternString(base::StringView(key.flat_key)), + storage_.InternString(base::StringView(key.key)), + Variadic::Real(value)); +} + +void ArgsParser::AddPointer(const Key& key, const void* value) { + inserter_.AddArg(storage_.InternString(base::StringView(key.flat_key)), + storage_.InternString(base::StringView(key.key)), + Variadic::Pointer(reinterpret_cast<uintptr_t>(value))); +} + +void ArgsParser::AddBoolean(const Key& key, bool value) { + inserter_.AddArg(storage_.InternString(base::StringView(key.flat_key)), + storage_.InternString(base::StringView(key.key)), + Variadic::Boolean(value)); +} + +void ArgsParser::AddBytes(const Key& key, const protozero::ConstBytes& value) { + std::string b64_data = base::Base64Encode(value.data, value.size); + AddString(key, b64_data); +} + +bool ArgsParser::AddJson(const Key& key, const protozero::ConstChars& value) { + if (!support_json_) + PERFETTO_FATAL("Unexpected JSON value when parsing data"); + + auto json_value = json::ParseJsonString(value); + if (!json_value) + return false; + return json::AddJsonValueToArgs(*json_value, base::StringView(key.flat_key), + base::StringView(key.key), &storage_, + &inserter_); +} + +void ArgsParser::AddNull(const Key& key) { + inserter_.AddArg(storage_.InternString(base::StringView(key.flat_key)), + storage_.InternString(base::StringView(key.key)), + Variadic::Null()); +} + +size_t ArgsParser::GetArrayEntryIndex(const std::string& array_key) { + return inserter_.GetNextArrayEntryIndex( + storage_.InternString(base::StringView(array_key))); +} + +size_t ArgsParser::IncrementArrayEntryIndex(const std::string& array_key) { + return inserter_.IncrementArrayEntryIndex( + storage_.InternString(base::StringView(array_key))); +} + +int64_t ArgsParser::packet_timestamp() { + return packet_timestamp_; +} + +PacketSequenceStateGeneration* ArgsParser::seq_state() { + return sequence_state_; +} + +InternedMessageView* ArgsParser::GetInternedMessageView(uint32_t field_id, + uint64_t iid) { + if (!sequence_state_) + return nullptr; + return sequence_state_->GetInternedMessageView(field_id, iid); +} + +} // namespace perfetto::trace_processor diff --git a/src/trace_processor/importers/proto/winscope/winscope_args_parser.h b/src/trace_processor/importers/proto/args_parser.h index a0dfa4632..7cd157848 100644 --- a/src/trace_processor/importers/proto/winscope/winscope_args_parser.h +++ b/src/trace_processor/importers/proto/args_parser.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2023 The Android Open Source Project + * Copyright (C) 2024 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,22 +14,28 @@ * limitations under the License. */ -#ifndef SRC_TRACE_PROCESSOR_IMPORTERS_PROTO_WINSCOPE_WINSCOPE_ARGS_PARSER_H_ -#define SRC_TRACE_PROCESSOR_IMPORTERS_PROTO_WINSCOPE_WINSCOPE_ARGS_PARSER_H_ +#ifndef SRC_TRACE_PROCESSOR_IMPORTERS_PROTO_ARGS_PARSER_H_ +#define SRC_TRACE_PROCESSOR_IMPORTERS_PROTO_ARGS_PARSER_H_ #include "src/trace_processor/importers/common/args_tracker.h" +#include "src/trace_processor/importers/proto/packet_sequence_state_generation.h" #include "src/trace_processor/types/trace_processor_context.h" #include "src/trace_processor/util/proto_to_args_parser.h" -namespace perfetto { -namespace trace_processor { +namespace perfetto::trace_processor { -class WinscopeArgsParser : public util::ProtoToArgsParser::Delegate { +// A ProtoToArgsParser::Delegate that writes the parsed proto data into +// TraceStorage after interning key strings. +class ArgsParser : public util::ProtoToArgsParser::Delegate { public: using Key = util::ProtoToArgsParser::Key; - WinscopeArgsParser(ArgsTracker::BoundInserter& inserter, - TraceStorage& storage); + ArgsParser(int64_t packet_timestamp, + ArgsTracker::BoundInserter& inserter, + TraceStorage& storage, + PacketSequenceStateGeneration* sequence_state = nullptr, + bool support_json = false); + ~ArgsParser() override; void AddInteger(const Key&, int64_t) override; void AddUnsignedInteger(const Key&, uint64_t) override; void AddString(const Key&, const protozero::ConstChars&) override; @@ -37,10 +43,12 @@ class WinscopeArgsParser : public util::ProtoToArgsParser::Delegate { void AddDouble(const Key&, double) override; void AddPointer(const Key&, const void*) override; void AddBoolean(const Key&, bool) override; + void AddBytes(const Key&, const protozero::ConstBytes&) override; bool AddJson(const Key&, const protozero::ConstChars&) override; void AddNull(const Key&) override; size_t GetArrayEntryIndex(const std::string& array_key) override; size_t IncrementArrayEntryIndex(const std::string& array_key) override; + int64_t packet_timestamp() override; PacketSequenceStateGeneration* seq_state() override; protected: @@ -48,11 +56,13 @@ class WinscopeArgsParser : public util::ProtoToArgsParser::Delegate { uint64_t iid) override; private: + const bool support_json_; + const int64_t packet_timestamp_; + PacketSequenceStateGeneration* const sequence_state_; ArgsTracker::BoundInserter& inserter_; TraceStorage& storage_; }; -} // namespace trace_processor -} // namespace perfetto +} // namespace perfetto::trace_processor -#endif // SRC_TRACE_PROCESSOR_IMPORTERS_PROTO_WINSCOPE_WINSCOPE_ARGS_PARSER_H_ +#endif // SRC_TRACE_PROCESSOR_IMPORTERS_PROTO_ARGS_PARSER_H_ diff --git a/src/trace_processor/importers/proto/chrome_system_probes_parser.h b/src/trace_processor/importers/proto/chrome_system_probes_parser.h index ca688146b..9c9397b7f 100644 --- a/src/trace_processor/importers/proto/chrome_system_probes_parser.h +++ b/src/trace_processor/importers/proto/chrome_system_probes_parser.h @@ -43,7 +43,7 @@ class ChromeSystemProbesParser { // Maps a proto field number for memcounters in ProcessStats::Process to // their StringId. Keep kProcStatsProcessSize equal to 1 + max proto field // id of ProcessStats::Process. Also update SystemProbesParser. - static constexpr size_t kProcStatsProcessSize = 23; + static constexpr size_t kProcStatsProcessSize = 24; std::array<StringId, kProcStatsProcessSize> proc_stats_process_names_{}; }; diff --git a/src/trace_processor/importers/proto/gpu_event_parser.cc b/src/trace_processor/importers/proto/gpu_event_parser.cc index 9022cbb02..5664fba8c 100644 --- a/src/trace_processor/importers/proto/gpu_event_parser.cc +++ b/src/trace_processor/importers/proto/gpu_event_parser.cc @@ -330,6 +330,7 @@ void GpuEventParser::ParseGpuRenderStageEvent( ConstBytes blob) { protos::pbzero::GpuRenderStageEvent::Decoder event(blob.data, blob.size); + int32_t pid = 0; if (event.has_specifications()) { protos::pbzero::GpuRenderStageEvent_Specifications::Decoder spec( event.specifications().data, event.specifications().size); @@ -349,6 +350,23 @@ void GpuEventParser::ParseGpuRenderStageEvent( context_->storage->InternString(stage.description()))); } } + if (spec.has_context_spec()) { + protos::pbzero::GpuRenderStageEvent_Specifications_ContextSpec::Decoder + context_spec(spec.context_spec()); + if (context_spec.has_pid()) { + pid = context_spec.pid(); + } + } + } + + if (event.has_context()) { + uint64_t context_id = event.context(); + auto* decoder = sequence_state->LookupInternedMessage< + protos::pbzero::InternedData::kGraphicsContextsFieldNumber, + protos::pbzero::InternedGraphicsContext>(context_id); + if (decoder) { + pid = decoder->pid(); + } } auto args_callback = [this, &event, @@ -468,7 +486,8 @@ void GpuEventParser::ParseGpuRenderStageEvent( row.command_buffer_name = command_buffer_name_id; row.submission_id = event.submission_id(); row.hw_queue_id = static_cast<int64_t>(hw_queue_id); - + row.upid = context_->process_tracker->GetOrCreateProcess( + static_cast<uint32_t>(pid)); context_->slice_tracker->ScopedTyped( context_->storage->mutable_gpu_slice_table(), row, args_callback); } diff --git a/src/trace_processor/importers/proto/network_trace_module_unittest.cc b/src/trace_processor/importers/proto/network_trace_module_unittest.cc index f174b352a..017b7f752 100644 --- a/src/trace_processor/importers/proto/network_trace_module_unittest.cc +++ b/src/trace_processor/importers/proto/network_trace_module_unittest.cc @@ -20,6 +20,7 @@ #include "src/trace_processor/importers/common/args_translation_table.h" #include "src/trace_processor/importers/common/async_track_set_tracker.h" #include "src/trace_processor/importers/common/global_args_tracker.h" +#include "src/trace_processor/importers/common/process_track_translation_table.h" #include "src/trace_processor/importers/common/slice_tracker.h" #include "src/trace_processor/importers/common/slice_translation_table.h" #include "src/trace_processor/importers/common/track_tracker.h" @@ -45,6 +46,8 @@ class NetworkTraceModuleTest : public testing::Test { context_.args_tracker.reset(new ArgsTracker(&context_)); context_.global_args_tracker.reset(new GlobalArgsTracker(storage_)); context_.slice_translation_table.reset(new SliceTranslationTable(storage_)); + context_.process_track_translation_table.reset( + new ProcessTrackTranslationTable(storage_)); context_.args_translation_table.reset(new ArgsTranslationTable(storage_)); context_.async_track_set_tracker.reset(new AsyncTrackSetTracker(&context_)); context_.proto_trace_parser.reset(new ProtoTraceParserImpl(&context_)); diff --git a/src/trace_processor/importers/proto/perf_sample_tracker.cc b/src/trace_processor/importers/proto/perf_sample_tracker.cc index e47239e1c..52ff1152b 100644 --- a/src/trace_processor/importers/proto/perf_sample_tracker.cc +++ b/src/trace_processor/importers/proto/perf_sample_tracker.cc @@ -129,7 +129,7 @@ PerfSampleTracker::SamplingStreamInfo PerfSampleTracker::GetSamplingStreamInfo( seq_it = seq_state_.emplace(seq_id, CreatePerfSession()).first; } SequenceState* seq_state = &seq_it->second; - uint32_t session_id = seq_state->perf_session_id; + tables::PerfSessionTable::Id session_id = seq_state->perf_session_id; auto cpu_it = seq_state->per_cpu.find(cpu); if (cpu_it != seq_state->per_cpu.end()) @@ -162,15 +162,19 @@ PerfSampleTracker::SamplingStreamInfo PerfSampleTracker::GetSamplingStreamInfo( // tracing session). if (perf_defaults.has_value() && perf_defaults->process_shard_count() > 0) { context_->storage->SetIndexedStats( - stats::perf_process_shard_count, static_cast<int>(session_id), + stats::perf_process_shard_count, static_cast<int>(session_id.value), static_cast<int64_t>(perf_defaults->process_shard_count())); context_->storage->SetIndexedStats( - stats::perf_chosen_process_shard, static_cast<int>(session_id), + stats::perf_chosen_process_shard, static_cast<int>(session_id.value), static_cast<int64_t>(perf_defaults->chosen_process_shard())); } return {session_id, timebase_track_id}; } +tables::PerfSessionTable::Id PerfSampleTracker::CreatePerfSession() { + return context_->storage->mutable_perf_session_table()->Insert({}).id; +} + } // namespace trace_processor } // namespace perfetto diff --git a/src/trace_processor/importers/proto/perf_sample_tracker.h b/src/trace_processor/importers/proto/perf_sample_tracker.h index c1e62e25a..178683db3 100644 --- a/src/trace_processor/importers/proto/perf_sample_tracker.h +++ b/src/trace_processor/importers/proto/perf_sample_tracker.h @@ -23,6 +23,7 @@ #include <unordered_map> #include "src/trace_processor/storage/trace_storage.h" +#include "src/trace_processor/tables/profiler_tables_py.h" namespace perfetto { namespace protos { @@ -36,10 +37,11 @@ class TraceProcessorContext; class PerfSampleTracker { public: struct SamplingStreamInfo { - uint32_t perf_session_id = 0; + tables::PerfSessionTable::Id perf_session_id; TrackId timebase_track_id = kInvalidTrackId; - SamplingStreamInfo(uint32_t _perf_session_id, TrackId _timebase_track_id) + SamplingStreamInfo(tables::PerfSessionTable::Id _perf_session_id, + TrackId _timebase_track_id) : perf_session_id(_perf_session_id), timebase_track_id(_timebase_track_id) {} }; @@ -52,8 +54,6 @@ class PerfSampleTracker { uint32_t cpu, protos::pbzero::TracePacketDefaults_Decoder* nullable_defaults); - uint32_t CreatePerfSession() { return next_perf_session_id_++; } - private: struct CpuSequenceState { TrackId timebase_track_id = kInvalidTrackId; @@ -63,15 +63,16 @@ class PerfSampleTracker { }; struct SequenceState { - uint32_t perf_session_id = 0; + tables::PerfSessionTable::Id perf_session_id; std::unordered_map<uint32_t, CpuSequenceState> per_cpu; - SequenceState(uint32_t _perf_session_id) + explicit SequenceState(tables::PerfSessionTable::Id _perf_session_id) : perf_session_id(_perf_session_id) {} }; + tables::PerfSessionTable::Id CreatePerfSession(); + std::unordered_map<uint32_t, SequenceState> seq_state_; - uint32_t next_perf_session_id_ = 0; TraceProcessorContext* const context_; }; diff --git a/src/trace_processor/importers/proto/perf_sample_tracker_unittest.cc b/src/trace_processor/importers/proto/perf_sample_tracker_unittest.cc index 7515a04a5..df1fba0cc 100644 --- a/src/trace_processor/importers/proto/perf_sample_tracker_unittest.cc +++ b/src/trace_processor/importers/proto/perf_sample_tracker_unittest.cc @@ -215,10 +215,10 @@ TEST_F(PerfSampleTrackerTest, ProcessShardingStatsEntries) { std::optional<int64_t> shard_count = context.storage->GetIndexedStats( stats::perf_process_shard_count, - static_cast<int>(stream.perf_session_id)); + static_cast<int>(stream.perf_session_id.value)); std::optional<int64_t> chosen_shard = context.storage->GetIndexedStats( stats::perf_chosen_process_shard, - static_cast<int>(stream.perf_session_id)); + static_cast<int>(stream.perf_session_id.value)); ASSERT_TRUE(shard_count.has_value()); EXPECT_EQ(shard_count.value(), 8); @@ -227,10 +227,10 @@ TEST_F(PerfSampleTrackerTest, ProcessShardingStatsEntries) { std::optional<int64_t> shard_count2 = context.storage->GetIndexedStats( stats::perf_process_shard_count, - static_cast<int>(stream.perf_session_id)); + static_cast<int>(stream.perf_session_id.value)); std::optional<int64_t> chosen_shard2 = context.storage->GetIndexedStats( stats::perf_chosen_process_shard, - static_cast<int>(stream.perf_session_id)); + static_cast<int>(stream.perf_session_id.value)); ASSERT_TRUE(shard_count2.has_value()); EXPECT_EQ(shard_count2.value(), 8); diff --git a/src/trace_processor/importers/proto/profile_module.cc b/src/trace_processor/importers/proto/profile_module.cc index 55e3547bd..ab44d2a3e 100644 --- a/src/trace_processor/importers/proto/profile_module.cc +++ b/src/trace_processor/importers/proto/profile_module.cc @@ -238,7 +238,7 @@ void ProfileModule::ParsePerfSample( PerfSample::ProducerEvent::PROFILER_STOP_GUARDRAIL) { context_->storage->SetIndexedStats( stats::perf_guardrail_stop_ts, - static_cast<int>(sampling_stream.perf_session_id), ts); + static_cast<int>(sampling_stream.perf_session_id.value), ts); } return; } diff --git a/src/trace_processor/importers/proto/proto_trace_parser_impl.cc b/src/trace_processor/importers/proto/proto_trace_parser_impl.cc index 9d318a1c8..8a11d5d8a 100644 --- a/src/trace_processor/importers/proto/proto_trace_parser_impl.cc +++ b/src/trace_processor/importers/proto/proto_trace_parser_impl.cc @@ -29,8 +29,8 @@ #include "perfetto/ext/base/uuid.h" #include "src/trace_processor/importers/common/args_tracker.h" +#include "src/trace_processor/importers/common/cpu_tracker.h" #include "src/trace_processor/importers/common/event_tracker.h" -#include "src/trace_processor/importers/common/machine_tracker.h" #include "src/trace_processor/importers/common/metadata_tracker.h" #include "src/trace_processor/importers/common/parser_types.h" #include "src/trace_processor/importers/common/process_tracker.h" @@ -44,7 +44,6 @@ #include "src/trace_processor/types/trace_processor_context.h" #include "src/trace_processor/types/variadic.h" -#include "protos/perfetto/common/trace_stats.pbzero.h" #include "protos/perfetto/config/trace_config.pbzero.h" #include "protos/perfetto/trace/chrome/chrome_trace_event.pbzero.h" #include "protos/perfetto/trace/perfetto/perfetto_metatrace.pbzero.h" @@ -85,9 +84,6 @@ void ProtoTraceParserImpl::ParseTracePacket(int64_t ts, TracePacketData data) { } } - if (packet.has_trace_stats()) - ParseTraceStats(packet.trace_stats()); - if (packet.has_chrome_events()) { ParseChromeEvents(ts, packet.chrome_events()); } @@ -160,116 +156,14 @@ void ProtoTraceParserImpl::ParseInlineSchedWaking(uint32_t cpu, context_->args_tracker->Flush(); } -void ProtoTraceParserImpl::ParseTraceStats(ConstBytes blob) { - protos::pbzero::TraceStats::Decoder evt(blob.data, blob.size); - auto* storage = context_->storage.get(); - storage->SetStats(stats::traced_producers_connected, - static_cast<int64_t>(evt.producers_connected())); - storage->SetStats(stats::traced_producers_seen, - static_cast<int64_t>(evt.producers_seen())); - storage->SetStats(stats::traced_data_sources_registered, - static_cast<int64_t>(evt.data_sources_registered())); - storage->SetStats(stats::traced_data_sources_seen, - static_cast<int64_t>(evt.data_sources_seen())); - storage->SetStats(stats::traced_tracing_sessions, - static_cast<int64_t>(evt.tracing_sessions())); - storage->SetStats(stats::traced_total_buffers, - static_cast<int64_t>(evt.total_buffers())); - storage->SetStats(stats::traced_chunks_discarded, - static_cast<int64_t>(evt.chunks_discarded())); - storage->SetStats(stats::traced_patches_discarded, - static_cast<int64_t>(evt.patches_discarded())); - storage->SetStats(stats::traced_flushes_requested, - static_cast<int64_t>(evt.flushes_requested())); - storage->SetStats(stats::traced_flushes_succeeded, - static_cast<int64_t>(evt.flushes_succeeded())); - storage->SetStats(stats::traced_flushes_failed, - static_cast<int64_t>(evt.flushes_failed())); - - if (evt.has_filter_stats()) { - protos::pbzero::TraceStats::FilterStats::Decoder fstat(evt.filter_stats()); - storage->SetStats(stats::filter_errors, - static_cast<int64_t>(fstat.errors())); - storage->SetStats(stats::filter_input_bytes, - static_cast<int64_t>(fstat.input_bytes())); - storage->SetStats(stats::filter_input_packets, - static_cast<int64_t>(fstat.input_packets())); - storage->SetStats(stats::filter_output_bytes, - static_cast<int64_t>(fstat.output_bytes())); - storage->SetStats(stats::filter_time_taken_ns, - static_cast<int64_t>(fstat.time_taken_ns())); - for (auto [i, it] = std::tuple{0, fstat.bytes_discarded_per_buffer()}; it; - ++it, ++i) { - storage->SetIndexedStats(stats::traced_buf_bytes_filtered_out, i, - static_cast<int64_t>(*it)); - } - } - - switch (evt.final_flush_outcome()) { - case protos::pbzero::TraceStats::FINAL_FLUSH_SUCCEEDED: - storage->IncrementStats(stats::traced_final_flush_succeeded, 1); - break; - case protos::pbzero::TraceStats::FINAL_FLUSH_FAILED: - storage->IncrementStats(stats::traced_final_flush_failed, 1); - break; - case protos::pbzero::TraceStats::FINAL_FLUSH_UNSPECIFIED: - break; - } - - int buf_num = 0; - for (auto it = evt.buffer_stats(); it; ++it, ++buf_num) { - protos::pbzero::TraceStats::BufferStats::Decoder buf(*it); - storage->SetIndexedStats(stats::traced_buf_buffer_size, buf_num, - static_cast<int64_t>(buf.buffer_size())); - storage->SetIndexedStats(stats::traced_buf_bytes_written, buf_num, - static_cast<int64_t>(buf.bytes_written())); - storage->SetIndexedStats(stats::traced_buf_bytes_overwritten, buf_num, - static_cast<int64_t>(buf.bytes_overwritten())); - storage->SetIndexedStats(stats::traced_buf_bytes_read, buf_num, - static_cast<int64_t>(buf.bytes_read())); - storage->SetIndexedStats(stats::traced_buf_padding_bytes_written, buf_num, - static_cast<int64_t>(buf.padding_bytes_written())); - storage->SetIndexedStats(stats::traced_buf_padding_bytes_cleared, buf_num, - static_cast<int64_t>(buf.padding_bytes_cleared())); - storage->SetIndexedStats(stats::traced_buf_chunks_written, buf_num, - static_cast<int64_t>(buf.chunks_written())); - storage->SetIndexedStats(stats::traced_buf_chunks_rewritten, buf_num, - static_cast<int64_t>(buf.chunks_rewritten())); - storage->SetIndexedStats(stats::traced_buf_chunks_overwritten, buf_num, - static_cast<int64_t>(buf.chunks_overwritten())); - storage->SetIndexedStats(stats::traced_buf_chunks_discarded, buf_num, - static_cast<int64_t>(buf.chunks_discarded())); - storage->SetIndexedStats(stats::traced_buf_chunks_read, buf_num, - static_cast<int64_t>(buf.chunks_read())); - storage->SetIndexedStats( - stats::traced_buf_chunks_committed_out_of_order, buf_num, - static_cast<int64_t>(buf.chunks_committed_out_of_order())); - storage->SetIndexedStats(stats::traced_buf_write_wrap_count, buf_num, - static_cast<int64_t>(buf.write_wrap_count())); - storage->SetIndexedStats(stats::traced_buf_patches_succeeded, buf_num, - static_cast<int64_t>(buf.patches_succeeded())); - storage->SetIndexedStats(stats::traced_buf_patches_failed, buf_num, - static_cast<int64_t>(buf.patches_failed())); - storage->SetIndexedStats(stats::traced_buf_readaheads_succeeded, buf_num, - static_cast<int64_t>(buf.readaheads_succeeded())); - storage->SetIndexedStats(stats::traced_buf_readaheads_failed, buf_num, - static_cast<int64_t>(buf.readaheads_failed())); - storage->SetIndexedStats(stats::traced_buf_abi_violations, buf_num, - static_cast<int64_t>(buf.abi_violations())); - storage->SetIndexedStats( - stats::traced_buf_trace_writer_packet_loss, buf_num, - static_cast<int64_t>(buf.trace_writer_packet_loss())); - } -} - void ProtoTraceParserImpl::ParseChromeEvents(int64_t ts, ConstBytes blob) { TraceStorage* storage = context_->storage.get(); protos::pbzero::ChromeEventBundle::Decoder bundle(blob.data, blob.size); ArgsTracker args(context_); if (bundle.has_metadata()) { + auto ucpu = context_->cpu_tracker->GetOrCreateCpu(0); RawId id = storage->mutable_raw_table() - ->Insert({ts, raw_chrome_metadata_event_id_, 0, 0, 0, 0, - context_->machine_id()}) + ->Insert({ts, raw_chrome_metadata_event_id_, 0, 0, 0, ucpu}) .id; auto inserter = args.AddArgsTo(id); @@ -315,9 +209,10 @@ void ProtoTraceParserImpl::ParseChromeEvents(int64_t ts, ConstBytes blob) { } if (bundle.has_legacy_ftrace_output()) { + auto ucpu = context_->cpu_tracker->GetOrCreateCpu(0); RawId id = storage->mutable_raw_table() ->Insert({ts, raw_chrome_legacy_system_trace_event_id_, 0, 0, - 0, 0, context_->machine_id()}) + 0, ucpu}) .id; std::string data; @@ -336,9 +231,10 @@ void ProtoTraceParserImpl::ParseChromeEvents(int64_t ts, ConstBytes blob) { protos::pbzero::ChromeLegacyJsonTrace::USER_TRACE) { continue; } + auto ucpu = context_->cpu_tracker->GetOrCreateCpu(0); RawId id = storage->mutable_raw_table() ->Insert({ts, raw_chrome_legacy_user_trace_event_id_, 0, 0, - 0, 0, context_->machine_id()}) + 0, ucpu}) .id; Variadic value = Variadic::String(storage->InternString(legacy_trace.data())); diff --git a/src/trace_processor/importers/proto/proto_trace_parser_impl.h b/src/trace_processor/importers/proto/proto_trace_parser_impl.h index e9bce35b3..2c4dc077f 100644 --- a/src/trace_processor/importers/proto/proto_trace_parser_impl.h +++ b/src/trace_processor/importers/proto/proto_trace_parser_impl.h @@ -65,7 +65,6 @@ class ProtoTraceParserImpl : public ProtoTraceParser { int64_t /*ts*/, InlineSchedWaking data) override; - void ParseTraceStats(ConstBytes); void ParseChromeEvents(int64_t ts, ConstBytes); void ParseMetatraceEvent(int64_t ts, ConstBytes); diff --git a/src/trace_processor/importers/proto/proto_trace_parser_impl_unittest.cc b/src/trace_processor/importers/proto/proto_trace_parser_impl_unittest.cc index 95b16d130..e959f59e4 100644 --- a/src/trace_processor/importers/proto/proto_trace_parser_impl_unittest.cc +++ b/src/trace_processor/importers/proto/proto_trace_parser_impl_unittest.cc @@ -23,10 +23,13 @@ #include "src/trace_processor/importers/common/args_tracker.h" #include "src/trace_processor/importers/common/args_translation_table.h" #include "src/trace_processor/importers/common/clock_tracker.h" +#include "src/trace_processor/importers/common/cpu_tracker.h" #include "src/trace_processor/importers/common/event_tracker.h" #include "src/trace_processor/importers/common/flow_tracker.h" +#include "src/trace_processor/importers/common/machine_tracker.h" #include "src/trace_processor/importers/common/mapping_tracker.h" #include "src/trace_processor/importers/common/metadata_tracker.h" +#include "src/trace_processor/importers/common/process_track_translation_table.h" #include "src/trace_processor/importers/common/process_tracker.h" #include "src/trace_processor/importers/common/slice_tracker.h" #include "src/trace_processor/importers/common/stack_profile_tracker.h" @@ -259,12 +262,16 @@ class ProtoTraceParserTest : public ::testing::Test { context_.args_translation_table.reset(new ArgsTranslationTable(storage_)); context_.metadata_tracker.reset( new MetadataTracker(context_.storage.get())); + context_.machine_tracker.reset(new MachineTracker(&context_, 0)); + context_.cpu_tracker.reset(new CpuTracker(&context_)); event_ = new MockEventTracker(&context_); context_.event_tracker.reset(event_); sched_ = new MockSchedEventTracker(&context_); context_.ftrace_sched_tracker.reset(sched_); process_ = new NiceMock<MockProcessTracker>(&context_); context_.process_tracker.reset(process_); + context_.process_track_translation_table.reset( + new ProcessTrackTranslationTable(storage_)); slice_ = new NiceMock<MockSliceTracker>(&context_); context_.slice_tracker.reset(slice_); context_.slice_translation_table.reset(new SliceTranslationTable(storage_)); @@ -2368,7 +2375,9 @@ TEST_F(ProtoTraceParserTest, TrackEventParseLegacyEventIntoRawTable) { EXPECT_EQ(raw_table.ts()[0], 1010000); EXPECT_EQ(raw_table.name()[0], storage_->InternString("track_event.legacy_event")); - EXPECT_EQ(raw_table.cpu()[0], 0u); + auto ucpu = raw_table.ucpu()[0]; + const auto& cpu_table = storage_->cpu_table(); + EXPECT_EQ(cpu_table.cpu()[ucpu.value], 0u); EXPECT_EQ(raw_table.utid()[0], 1u); EXPECT_EQ(raw_table.arg_set_id()[0], 1u); diff --git a/src/trace_processor/importers/proto/proto_trace_reader.cc b/src/trace_processor/importers/proto/proto_trace_reader.cc index 2e50f0cc3..d958c0b6a 100644 --- a/src/trace_processor/importers/proto/proto_trace_reader.cc +++ b/src/trace_processor/importers/proto/proto_trace_reader.cc @@ -21,6 +21,7 @@ #include "perfetto/base/build_config.h" #include "perfetto/base/logging.h" +#include "perfetto/ext/base/flat_hash_map.h" #include "perfetto/ext/base/string_view.h" #include "perfetto/ext/base/utils.h" #include "perfetto/protozero/proto_decoder.h" @@ -41,6 +42,7 @@ #include "src/trace_processor/util/gzip_utils.h" #include "protos/perfetto/common/builtin_clock.pbzero.h" +#include "protos/perfetto/common/trace_stats.pbzero.h" #include "protos/perfetto/config/trace_config.pbzero.h" #include "protos/perfetto/trace/clock_snapshot.pbzero.h" #include "protos/perfetto/trace/extension_descriptor.pbzero.h" @@ -118,6 +120,16 @@ util::Status ProtoTraceReader::ParsePacket(TraceBlobView packet) { HandlePreviousPacketDropped(decoder); } + uint32_t sequence_id = decoder.trusted_packet_sequence_id(); + if (sequence_id) { + auto [data_loss, inserted] = + packet_sequence_data_loss_.Insert(sequence_id, 0); + + if (!inserted && decoder.previous_packet_dropped()) { + *data_loss += 1; + } + } + // It is important that we parse defaults before parsing other fields such as // the timestamp, since the defaults could affect them. if (decoder.has_trace_packet_defaults()) { @@ -131,8 +143,11 @@ util::Status ProtoTraceReader::ParsePacket(TraceBlobView packet) { } if (decoder.has_clock_snapshot()) { - return ParseClockSnapshot(decoder.clock_snapshot(), - decoder.trusted_packet_sequence_id()); + return ParseClockSnapshot(decoder.clock_snapshot(), sequence_id); + } + + if (decoder.has_trace_stats()) { + ParseTraceStats(decoder.trace_stats()); } if (decoder.has_service_event()) { @@ -474,6 +489,125 @@ util::Status ProtoTraceReader::ParseServiceEvent(int64_t ts, ConstBytes blob) { return util::OkStatus(); } +void ProtoTraceReader::ParseTraceStats(ConstBytes blob) { + protos::pbzero::TraceStats::Decoder evt(blob.data, blob.size); + auto* storage = context_->storage.get(); + storage->SetStats(stats::traced_producers_connected, + static_cast<int64_t>(evt.producers_connected())); + storage->SetStats(stats::traced_producers_seen, + static_cast<int64_t>(evt.producers_seen())); + storage->SetStats(stats::traced_data_sources_registered, + static_cast<int64_t>(evt.data_sources_registered())); + storage->SetStats(stats::traced_data_sources_seen, + static_cast<int64_t>(evt.data_sources_seen())); + storage->SetStats(stats::traced_tracing_sessions, + static_cast<int64_t>(evt.tracing_sessions())); + storage->SetStats(stats::traced_total_buffers, + static_cast<int64_t>(evt.total_buffers())); + storage->SetStats(stats::traced_chunks_discarded, + static_cast<int64_t>(evt.chunks_discarded())); + storage->SetStats(stats::traced_patches_discarded, + static_cast<int64_t>(evt.patches_discarded())); + storage->SetStats(stats::traced_flushes_requested, + static_cast<int64_t>(evt.flushes_requested())); + storage->SetStats(stats::traced_flushes_succeeded, + static_cast<int64_t>(evt.flushes_succeeded())); + storage->SetStats(stats::traced_flushes_failed, + static_cast<int64_t>(evt.flushes_failed())); + + if (evt.has_filter_stats()) { + protos::pbzero::TraceStats::FilterStats::Decoder fstat(evt.filter_stats()); + storage->SetStats(stats::filter_errors, + static_cast<int64_t>(fstat.errors())); + storage->SetStats(stats::filter_input_bytes, + static_cast<int64_t>(fstat.input_bytes())); + storage->SetStats(stats::filter_input_packets, + static_cast<int64_t>(fstat.input_packets())); + storage->SetStats(stats::filter_output_bytes, + static_cast<int64_t>(fstat.output_bytes())); + storage->SetStats(stats::filter_time_taken_ns, + static_cast<int64_t>(fstat.time_taken_ns())); + for (auto [i, it] = std::tuple{0, fstat.bytes_discarded_per_buffer()}; it; + ++it, ++i) { + storage->SetIndexedStats(stats::traced_buf_bytes_filtered_out, i, + static_cast<int64_t>(*it)); + } + } + + switch (evt.final_flush_outcome()) { + case protos::pbzero::TraceStats::FINAL_FLUSH_SUCCEEDED: + storage->IncrementStats(stats::traced_final_flush_succeeded, 1); + break; + case protos::pbzero::TraceStats::FINAL_FLUSH_FAILED: + storage->IncrementStats(stats::traced_final_flush_failed, 1); + break; + case protos::pbzero::TraceStats::FINAL_FLUSH_UNSPECIFIED: + break; + } + + int buf_num = 0; + for (auto it = evt.buffer_stats(); it; ++it, ++buf_num) { + protos::pbzero::TraceStats::BufferStats::Decoder buf(*it); + storage->SetIndexedStats(stats::traced_buf_buffer_size, buf_num, + static_cast<int64_t>(buf.buffer_size())); + storage->SetIndexedStats(stats::traced_buf_bytes_written, buf_num, + static_cast<int64_t>(buf.bytes_written())); + storage->SetIndexedStats(stats::traced_buf_bytes_overwritten, buf_num, + static_cast<int64_t>(buf.bytes_overwritten())); + storage->SetIndexedStats(stats::traced_buf_bytes_read, buf_num, + static_cast<int64_t>(buf.bytes_read())); + storage->SetIndexedStats(stats::traced_buf_padding_bytes_written, buf_num, + static_cast<int64_t>(buf.padding_bytes_written())); + storage->SetIndexedStats(stats::traced_buf_padding_bytes_cleared, buf_num, + static_cast<int64_t>(buf.padding_bytes_cleared())); + storage->SetIndexedStats(stats::traced_buf_chunks_written, buf_num, + static_cast<int64_t>(buf.chunks_written())); + storage->SetIndexedStats(stats::traced_buf_chunks_rewritten, buf_num, + static_cast<int64_t>(buf.chunks_rewritten())); + storage->SetIndexedStats(stats::traced_buf_chunks_overwritten, buf_num, + static_cast<int64_t>(buf.chunks_overwritten())); + storage->SetIndexedStats(stats::traced_buf_chunks_discarded, buf_num, + static_cast<int64_t>(buf.chunks_discarded())); + storage->SetIndexedStats(stats::traced_buf_chunks_read, buf_num, + static_cast<int64_t>(buf.chunks_read())); + storage->SetIndexedStats( + stats::traced_buf_chunks_committed_out_of_order, buf_num, + static_cast<int64_t>(buf.chunks_committed_out_of_order())); + storage->SetIndexedStats(stats::traced_buf_write_wrap_count, buf_num, + static_cast<int64_t>(buf.write_wrap_count())); + storage->SetIndexedStats(stats::traced_buf_patches_succeeded, buf_num, + static_cast<int64_t>(buf.patches_succeeded())); + storage->SetIndexedStats(stats::traced_buf_patches_failed, buf_num, + static_cast<int64_t>(buf.patches_failed())); + storage->SetIndexedStats(stats::traced_buf_readaheads_succeeded, buf_num, + static_cast<int64_t>(buf.readaheads_succeeded())); + storage->SetIndexedStats(stats::traced_buf_readaheads_failed, buf_num, + static_cast<int64_t>(buf.readaheads_failed())); + storage->SetIndexedStats(stats::traced_buf_abi_violations, buf_num, + static_cast<int64_t>(buf.abi_violations())); + storage->SetIndexedStats( + stats::traced_buf_trace_writer_packet_loss, buf_num, + static_cast<int64_t>(buf.trace_writer_packet_loss())); + } + + base::FlatHashMap<int32_t, int64_t> data_loss_per_buffer; + + for (auto it = evt.writer_stats(); it; ++it) { + protos::pbzero::TraceStats::WriterStats::Decoder writer(*it); + auto* data_loss = packet_sequence_data_loss_.Find( + static_cast<uint32_t>(writer.sequence_id())); + if (data_loss) { + data_loss_per_buffer[static_cast<int32_t>(writer.buffer())] += + static_cast<int64_t>(*data_loss); + } + } + + for (auto it = data_loss_per_buffer.GetIterator(); it; ++it) { + storage->SetIndexedStats(stats::traced_buf_sequence_packet_loss, it.key(), + it.value()); + } +} + void ProtoTraceReader::NotifyEndOfFile() {} } // namespace trace_processor diff --git a/src/trace_processor/importers/proto/proto_trace_reader.h b/src/trace_processor/importers/proto/proto_trace_reader.h index a93ad9881..4c4e2ca2d 100644 --- a/src/trace_processor/importers/proto/proto_trace_reader.h +++ b/src/trace_processor/importers/proto/proto_trace_reader.h @@ -78,6 +78,7 @@ class ProtoTraceReader : public ChunkedTraceReader { void ParseInternedData(const protos::pbzero::TracePacket_Decoder&, TraceBlobView interned_data); void ParseTraceConfig(ConstBytes); + void ParseTraceStats(ConstBytes); std::optional<StringId> GetBuiltinClockNameOrNull(int64_t clock_id); @@ -104,6 +105,8 @@ class ProtoTraceReader : public ChunkedTraceReader { base::FlatHashMap<uint32_t, PacketSequenceStateBuilder> packet_sequence_state_builders_; + base::FlatHashMap<uint32_t, size_t> packet_sequence_data_loss_; + StringId skipped_packet_key_id_; StringId invalid_incremental_state_key_id_; }; diff --git a/src/trace_processor/importers/proto/statsd_module.cc b/src/trace_processor/importers/proto/statsd_module.cc index e925b5a63..62faf0102 100644 --- a/src/trace_processor/importers/proto/statsd_module.cc +++ b/src/trace_processor/importers/proto/statsd_module.cc @@ -23,6 +23,7 @@ #include "src/trace_processor/importers/common/machine_tracker.h" #include "src/trace_processor/importers/common/slice_tracker.h" #include "src/trace_processor/importers/common/track_tracker.h" +#include "src/trace_processor/importers/proto/args_parser.h" #include "src/trace_processor/importers/proto/packet_sequence_state_generation.h" #include "src/trace_processor/sorter/trace_sorter.h" #include "src/trace_processor/storage/stats.h" @@ -37,111 +38,6 @@ namespace { constexpr const char* kAtomProtoName = ".android.os.statsd.Atom"; -using BoundInserter = ArgsTracker::BoundInserter; - -class InserterDelegate : public util::ProtoToArgsParser::Delegate { - public: - InserterDelegate(BoundInserter& inserter, TraceStorage& storage) - : inserter_(inserter), storage_(storage) {} - ~InserterDelegate() override = default; - - using Key = util::ProtoToArgsParser::Key; - - void AddInteger(const Key& key, int64_t value) override { - StringId flat_key_id = - storage_.InternString(base::StringView(key.flat_key)); - StringId key_id = storage_.InternString(base::StringView(key.key)); - Variadic variadic_val = Variadic::Integer(value); - inserter_.AddArg(flat_key_id, key_id, variadic_val); - } - - void AddUnsignedInteger(const Key& key, uint64_t value) override { - StringId flat_key_id = - storage_.InternString(base::StringView(key.flat_key)); - StringId key_id = storage_.InternString(base::StringView(key.key)); - Variadic variadic_val = Variadic::UnsignedInteger(value); - inserter_.AddArg(flat_key_id, key_id, variadic_val); - } - - void AddString(const Key& key, const protozero::ConstChars& value) override { - StringId flat_key_id = - storage_.InternString(base::StringView(key.flat_key)); - StringId key_id = storage_.InternString(base::StringView(key.key)); - Variadic variadic_val = Variadic::String(storage_.InternString(value)); - inserter_.AddArg(flat_key_id, key_id, variadic_val); - } - - void AddString(const Key& key, const std::string& value) override { - StringId flat_key_id = - storage_.InternString(base::StringView(key.flat_key)); - StringId key_id = storage_.InternString(base::StringView(key.key)); - Variadic variadic_val = - Variadic::String(storage_.InternString(base::StringView(value))); - inserter_.AddArg(flat_key_id, key_id, variadic_val); - } - - void AddDouble(const Key& key, double value) override { - StringId flat_key_id = - storage_.InternString(base::StringView(key.flat_key)); - StringId key_id = storage_.InternString(base::StringView(key.key)); - Variadic variadic_val = Variadic::Real(value); - inserter_.AddArg(flat_key_id, key_id, variadic_val); - } - - void AddPointer(const Key& key, const void* value) override { - StringId flat_key_id = - storage_.InternString(base::StringView(key.flat_key)); - StringId key_id = storage_.InternString(base::StringView(key.key)); - Variadic variadic_val = - Variadic::Pointer(reinterpret_cast<uintptr_t>(value)); - inserter_.AddArg(flat_key_id, key_id, variadic_val); - } - - void AddBoolean(const Key& key, bool value) override { - StringId flat_key_id = - storage_.InternString(base::StringView(key.flat_key)); - StringId key_id = storage_.InternString(base::StringView(key.key)); - Variadic variadic_val = Variadic::Boolean(value); - inserter_.AddArg(flat_key_id, key_id, variadic_val); - } - - bool AddJson(const Key&, const protozero::ConstChars&) override { - PERFETTO_FATAL("Unexpected JSON value when parsing statsd data"); - } - - void AddNull(const Key& key) override { - StringId flat_key_id = - storage_.InternString(base::StringView(key.flat_key)); - StringId key_id = storage_.InternString(base::StringView(key.key)); - Variadic variadic_val = Variadic::Null(); - inserter_.AddArg(flat_key_id, key_id, variadic_val); - } - - size_t GetArrayEntryIndex(const std::string& array_key) override { - base::ignore_result(array_key); - return 0; - } - - size_t IncrementArrayEntryIndex(const std::string& array_key) override { - base::ignore_result(array_key); - return 0; - } - - PacketSequenceStateGeneration* seq_state() override { return nullptr; } - - protected: - InternedMessageView* GetInternedMessageView(uint32_t field_id, - uint64_t iid) override { - base::ignore_result(field_id); - base::ignore_result(iid); - return nullptr; - } - - private: - BoundInserter& inserter_; - TraceStorage& storage_; -}; - // If we don't know about the atom format put whatever details we // can. This has the following restrictions: // - We can't tell the difference between double, fixed64, sfixed64 @@ -296,7 +192,7 @@ void StatsdModule::ParseAtom(int64_t ts, protozero::ConstBytes nested_bytes) { } SliceId slice = opt_slice.value(); auto inserter = context_->args_tracker->AddArgsTo(slice); - InserterDelegate delegate(inserter, *context_->storage.get()); + ArgsParser delegate(ts, inserter, *context_->storage.get()); const auto& fields = pool_.descriptor()->fields(); const auto& field_it = fields.find(nested_field_id); diff --git a/src/trace_processor/importers/proto/system_probes_parser.cc b/src/trace_processor/importers/proto/system_probes_parser.cc index 5b878eddd..1ebc13be7 100644 --- a/src/trace_processor/importers/proto/system_probes_parser.cc +++ b/src/trace_processor/importers/proto/system_probes_parser.cc @@ -22,6 +22,7 @@ #include "perfetto/ext/traced/sys_stats_counters.h" #include "perfetto/protozero/proto_decoder.h" #include "src/trace_processor/importers/common/clock_tracker.h" +#include "src/trace_processor/importers/common/cpu_tracker.h" #include "src/trace_processor/importers/common/event_tracker.h" #include "src/trace_processor/importers/common/metadata_tracker.h" #include "src/trace_processor/importers/common/process_tracker.h" @@ -168,6 +169,8 @@ SystemProbesParser::SystemProbesParser(TraceProcessorContext* context) context->storage->InternString("mem.smaps.pss.file"); proc_stats_process_names_[ProcessStats::Process::kSmrPssShmemKbFieldNumber] = context->storage->InternString("mem.smaps.pss.shmem"); + proc_stats_process_names_[ProcessStats::Process::kSmrSwapPssKbFieldNumber] = + context->storage->InternString("mem.smaps.swap.pss"); proc_stats_process_names_ [ProcessStats::Process::kRuntimeUserModeFieldNumber] = context->storage->InternString("runtime.user_ns"); @@ -744,13 +747,10 @@ void SystemProbesParser::ParseSystemInfo(ConstBytes blob) { void SystemProbesParser::ParseCpuInfo(ConstBytes blob) { protos::pbzero::CpuInfo::Decoder packet(blob.data, blob.size); uint32_t cluster_id = 0; + uint32_t cpu_id = 0; std::vector<uint32_t> last_cpu_freqs; - for (auto it = packet.cpus(); it; it++) { + for (auto it = packet.cpus(); it; it++, cpu_id++) { protos::pbzero::CpuInfo::Cpu::Decoder cpu(*it); - tables::CpuTable::Row cpu_row; - if (cpu.has_processor()) { - cpu_row.processor = context_->storage->InternString(cpu.processor()); - } std::vector<uint32_t> freqs; for (auto freq_it = cpu.frequencies(); freq_it; freq_it++) { freqs.push_back(*freq_it); @@ -760,19 +760,17 @@ void SystemProbesParser::ParseCpuInfo(ConstBytes blob) { if (freqs != last_cpu_freqs && !last_cpu_freqs.empty()) { cluster_id++; } - cpu_row.cluster_id = cluster_id; - cpu_row.machine_id = context_->machine_id(); last_cpu_freqs = freqs; - tables::CpuTable::Id cpu_row_id = - context_->storage->mutable_cpu_table()->Insert(cpu_row).id; + + tables::CpuTable::Id ucpu = + context_->cpu_tracker->SetCpuInfo(cpu_id, cpu.processor(), cluster_id); for (auto freq_it = cpu.frequencies(); freq_it; freq_it++) { uint32_t freq = *freq_it; tables::CpuFreqTable::Row cpu_freq_row; - cpu_freq_row.cpu_id = cpu_row_id; + cpu_freq_row.ucpu = ucpu; cpu_freq_row.freq = freq; - cpu_freq_row.machine_id = context_->machine_id(); context_->storage->mutable_cpu_freq_table()->Insert(cpu_freq_row); } } diff --git a/src/trace_processor/importers/proto/system_probes_parser.h b/src/trace_processor/importers/proto/system_probes_parser.h index a54beb7b3..472e334ee 100644 --- a/src/trace_processor/importers/proto/system_probes_parser.h +++ b/src/trace_processor/importers/proto/system_probes_parser.h @@ -75,7 +75,7 @@ class SystemProbesParser { // their StringId. Keep kProcStatsProcessSize equal to 1 + max proto field // id of ProcessStats::Process. Also update the value in // ChromeSystemProbesParser. - static constexpr size_t kProcStatsProcessSize = 23; + static constexpr size_t kProcStatsProcessSize = 24; std::array<StringId, kProcStatsProcessSize> proc_stats_process_names_{}; // Maps a SysStats::PsiSample::PsiResource type to its StringId. diff --git a/src/trace_processor/importers/proto/track_event_parser.cc b/src/trace_processor/importers/proto/track_event_parser.cc index 585463049..e252be5e1 100644 --- a/src/trace_processor/importers/proto/track_event_parser.cc +++ b/src/trace_processor/importers/proto/track_event_parser.cc @@ -26,13 +26,15 @@ #include "perfetto/trace_processor/status.h" #include "src/trace_processor/importers/common/args_tracker.h" #include "src/trace_processor/importers/common/args_translation_table.h" +#include "src/trace_processor/importers/common/cpu_tracker.h" #include "src/trace_processor/importers/common/event_tracker.h" #include "src/trace_processor/importers/common/flow_tracker.h" -#include "src/trace_processor/importers/common/machine_tracker.h" +#include "src/trace_processor/importers/common/process_track_translation_table.h" #include "src/trace_processor/importers/common/process_tracker.h" #include "src/trace_processor/importers/common/track_tracker.h" #include "src/trace_processor/importers/common/virtual_memory_mapping.h" #include "src/trace_processor/importers/json/json_utils.h" +#include "src/trace_processor/importers/proto/args_parser.h" #include "src/trace_processor/importers/proto/packet_analyzer.h" #include "src/trace_processor/importers/proto/profile_packet_utils.h" #include "src/trace_processor/importers/proto/stack_profile_sequence_state.h" @@ -74,103 +76,6 @@ using protozero::ConstBytes; constexpr int64_t kPendingThreadDuration = -1; constexpr int64_t kPendingThreadInstructionDelta = -1; -class TrackEventArgsParser : public util::ProtoToArgsParser::Delegate { - public: - TrackEventArgsParser(int64_t packet_timestamp, - BoundInserter& inserter, - TraceStorage& storage, - PacketSequenceStateGeneration& sequence_state) - : packet_timestamp_(packet_timestamp), - inserter_(inserter), - storage_(storage), - sequence_state_(sequence_state) {} - - ~TrackEventArgsParser() override; - - using Key = util::ProtoToArgsParser::Key; - - void AddInteger(const Key& key, int64_t value) final { - inserter_.AddArg(storage_.InternString(base::StringView(key.flat_key)), - storage_.InternString(base::StringView(key.key)), - Variadic::Integer(value)); - } - void AddUnsignedInteger(const Key& key, uint64_t value) final { - inserter_.AddArg(storage_.InternString(base::StringView(key.flat_key)), - storage_.InternString(base::StringView(key.key)), - Variadic::UnsignedInteger(value)); - } - void AddString(const Key& key, const protozero::ConstChars& value) final { - inserter_.AddArg(storage_.InternString(base::StringView(key.flat_key)), - storage_.InternString(base::StringView(key.key)), - Variadic::String(storage_.InternString(value))); - } - void AddString(const Key& key, const std::string& value) final { - inserter_.AddArg( - storage_.InternString(base::StringView(key.flat_key)), - storage_.InternString(base::StringView(key.key)), - Variadic::String(storage_.InternString(base::StringView(value)))); - } - void AddDouble(const Key& key, double value) final { - inserter_.AddArg(storage_.InternString(base::StringView(key.flat_key)), - storage_.InternString(base::StringView(key.key)), - Variadic::Real(value)); - } - void AddPointer(const Key& key, const void* value) final { - inserter_.AddArg(storage_.InternString(base::StringView(key.flat_key)), - storage_.InternString(base::StringView(key.key)), - Variadic::Pointer(reinterpret_cast<uintptr_t>(value))); - } - void AddBoolean(const Key& key, bool value) final { - inserter_.AddArg(storage_.InternString(base::StringView(key.flat_key)), - storage_.InternString(base::StringView(key.key)), - Variadic::Boolean(value)); - } - void AddBytes(const Key& key, const protozero::ConstBytes& value) final { - std::string b64_data = base::Base64Encode(value.data, value.size); - AddString(key, b64_data); - } - bool AddJson(const Key& key, const protozero::ConstChars& value) final { - auto json_value = json::ParseJsonString(value); - if (!json_value) - return false; - return json::AddJsonValueToArgs(*json_value, base::StringView(key.flat_key), - base::StringView(key.key), &storage_, - &inserter_); - } - void AddNull(const Key& key) final { - inserter_.AddArg(storage_.InternString(base::StringView(key.flat_key)), - storage_.InternString(base::StringView(key.key)), - Variadic::Null()); - } - - size_t GetArrayEntryIndex(const std::string& array_key) final { - return inserter_.GetNextArrayEntryIndex( - storage_.InternString(base::StringView(array_key))); - } - - size_t IncrementArrayEntryIndex(const std::string& array_key) final { - return inserter_.IncrementArrayEntryIndex( - storage_.InternString(base::StringView(array_key))); - } - - InternedMessageView* GetInternedMessageView(uint32_t field_id, - uint64_t iid) final { - return sequence_state_.GetInternedMessageView(field_id, iid); - } - - int64_t packet_timestamp() final { return packet_timestamp_; } - - PacketSequenceStateGeneration* seq_state() final { return &sequence_state_; } - - private: - int64_t packet_timestamp_; - BoundInserter& inserter_; - TraceStorage& storage_; - PacketSequenceStateGeneration& sequence_state_; -}; - -TrackEventArgsParser::~TrackEventArgsParser() = default; - // Paths on Windows use backslash rather than slash as a separator. // Normalise the paths by replacing backslashes with slashes to make it // easier to write cross-platform scripts. @@ -1118,10 +1023,11 @@ class TrackEventParser::EventImporter { if (!utid_) return util::ErrStatus("raw legacy event without thread association"); - RawId id = storage_->mutable_raw_table() - ->Insert({ts_, parser_->raw_legacy_event_id_, 0, *utid_, 0, - 0, context_->machine_id()}) - .id; + auto ucpu = context_->cpu_tracker->GetOrCreateCpu(0); + RawId id = + storage_->mutable_raw_table() + ->Insert({ts_, parser_->raw_legacy_event_id_, *utid_, 0, 0, ucpu}) + .id; auto inserter = context_->args_tracker->AddArgsTo(id); inserter @@ -1226,8 +1132,8 @@ class TrackEventParser::EventImporter { } } - TrackEventArgsParser args_writer(ts_, *inserter, *storage_, - *sequence_state_); + ArgsParser args_writer(ts_, *inserter, *storage_, sequence_state_, + /*support_json=*/true); int unknown_extensions = 0; log_errors(parser_->args_parser_.ParseMessage( blob_, ".perfetto.protos.TrackEvent", &parser_->reflect_fields_, @@ -1624,8 +1530,10 @@ void TrackEventParser::ParseTrackDescriptor( // Override the name with the most recent name seen (after sorting by ts). if (decoder.has_name() || decoder.has_static_name()) { auto* tracks = context_->storage->mutable_track_table(); - StringId name_id = context_->storage->InternString( + const StringId raw_name_id = context_->storage->InternString( decoder.has_name() ? decoder.name() : decoder.static_name()); + const StringId name_id = + context_->process_track_translation_table->TranslateName(raw_name_id); tracks->mutable_name()->Set(*tracks->id().IndexOf(track_id), name_id); } } diff --git a/src/trace_processor/importers/proto/track_event_tracker.cc b/src/trace_processor/importers/proto/track_event_tracker.cc index 56e081dd6..e758b9d05 100644 --- a/src/trace_processor/importers/proto/track_event_tracker.cc +++ b/src/trace_processor/importers/proto/track_event_tracker.cc @@ -18,6 +18,7 @@ #include "src/trace_processor/importers/common/args_tracker.h" #include "src/trace_processor/importers/common/args_translation_table.h" +#include "src/trace_processor/importers/common/process_track_translation_table.h" #include "src/trace_processor/importers/common/process_tracker.h" #include "src/trace_processor/importers/common/track_tracker.h" #include "src/trace_processor/tables/track_tables_py.h" @@ -201,7 +202,9 @@ std::optional<TrackId> TrackEventTracker::GetDescriptorTrack( reservation_it->second.is_counter) { return track_id; } - tracks->mutable_name()->Set(row, event_name); + const StringId track_name = + context_->process_track_translation_table->TranslateName(event_name); + tracks->mutable_name()->Set(row, track_name); return track_id; } diff --git a/src/trace_processor/importers/proto/translation_table_module.cc b/src/trace_processor/importers/proto/translation_table_module.cc index 8684df031..f45e35326 100644 --- a/src/trace_processor/importers/proto/translation_table_module.cc +++ b/src/trace_processor/importers/proto/translation_table_module.cc @@ -16,6 +16,7 @@ #include "src/trace_processor/importers/proto/translation_table_module.h" #include "src/trace_processor/importers/common/args_translation_table.h" +#include "src/trace_processor/importers/common/process_track_translation_table.h" #include "src/trace_processor/importers/common/slice_translation_table.h" #include "src/trace_processor/importers/proto/packet_sequence_state_generation.h" @@ -54,6 +55,8 @@ ModuleResult TranslationTableModule::TokenizePacket( translation_table.chrome_performance_mark()); } else if (translation_table.has_slice_name()) { ParseSliceNameRules(translation_table.slice_name()); + } else if (translation_table.has_process_track_name()) { + ParseProcessTrackNameRules(translation_table.process_track_name()); } return ModuleResult::Handled(); } @@ -113,5 +116,17 @@ void TranslationTableModule::ParseSliceNameRules(protozero::ConstBytes bytes) { } } +void TranslationTableModule::ParseProcessTrackNameRules( + protozero::ConstBytes bytes) { + const auto process_track_name = + protos::pbzero::ProcessTrackNameTranslationTable::Decoder(bytes); + for (auto it = process_track_name.raw_to_deobfuscated_name(); it; ++it) { + protos::pbzero::ProcessTrackNameTranslationTable:: + RawToDeobfuscatedNameEntry::Decoder entry(*it); + context_->process_track_translation_table->AddNameTranslationRule( + entry.key(), entry.value()); + } +} + } // namespace trace_processor } // namespace perfetto diff --git a/src/trace_processor/importers/proto/translation_table_module.h b/src/trace_processor/importers/proto/translation_table_module.h index 2060f4139..62d4c725b 100644 --- a/src/trace_processor/importers/proto/translation_table_module.h +++ b/src/trace_processor/importers/proto/translation_table_module.h @@ -46,6 +46,7 @@ class TranslationTableModule : public ProtoImporterModule { void ParseChromeUserEventRules(protozero::ConstBytes bytes); void ParseChromePerformanceMarkRules(protozero::ConstBytes bytes); void ParseSliceNameRules(protozero::ConstBytes bytes); + void ParseProcessTrackNameRules(protozero::ConstBytes bytes); TraceProcessorContext* context_; }; diff --git a/src/trace_processor/importers/proto/winscope/BUILD.gn b/src/trace_processor/importers/proto/winscope/BUILD.gn index 09b7b77b3..f518f424a 100644 --- a/src/trace_processor/importers/proto/winscope/BUILD.gn +++ b/src/trace_processor/importers/proto/winscope/BUILD.gn @@ -28,8 +28,8 @@ source_set("full") { "surfaceflinger_layers_parser.h", "surfaceflinger_transactions_parser.cc", "surfaceflinger_transactions_parser.h", - "winscope_args_parser.cc", - "winscope_args_parser.h", + "viewcapture_args_parser.cc", + "viewcapture_args_parser.h", "winscope_module.cc", "winscope_module.h", ] @@ -38,10 +38,10 @@ source_set("full") { "../:proto_importer_module", "../../../../../gn:default_deps", "../../../../../protos/perfetto/trace:zero", - "../../../../../protos/perfetto/trace/android:zero", "../../../../../protos/perfetto/trace/android:winscope_common_zero", "../../../../../protos/perfetto/trace/android:winscope_extensions_zero", "../../../../../protos/perfetto/trace/android:winscope_regular_zero", + "../../../../../protos/perfetto/trace/android:zero", "../../../../../protos/perfetto/trace/interned_data:zero", "../../../../../protos/perfetto/trace/profiling:zero", "../../../../protozero", diff --git a/src/trace_processor/importers/proto/winscope/shell_transitions_parser.cc b/src/trace_processor/importers/proto/winscope/shell_transitions_parser.cc index 68d2a82c6..12618acdc 100644 --- a/src/trace_processor/importers/proto/winscope/shell_transitions_parser.cc +++ b/src/trace_processor/importers/proto/winscope/shell_transitions_parser.cc @@ -19,8 +19,8 @@ #include "protos/perfetto/trace/android/shell_transition.pbzero.h" #include "src/trace_processor/importers/common/args_tracker.h" +#include "src/trace_processor/importers/proto/args_parser.h" #include "src/trace_processor/importers/proto/winscope/winscope.descriptor.h" -#include "src/trace_processor/importers/proto/winscope/winscope_args_parser.h" #include "src/trace_processor/storage/trace_storage.h" #include "src/trace_processor/types/trace_processor_context.h" @@ -49,7 +49,7 @@ void ShellTransitionsParser::ParseTransition(protozero::ConstBytes blob) { } auto inserter = context_->args_tracker->AddArgsTo(row_id); - WinscopeArgsParser writer(inserter, *context_->storage.get()); + ArgsParser writer(/*timestamp=*/0, inserter, *context_->storage.get()); base::Status status = args_parser_.ParseMessage( blob, kShellTransitionsProtoName, nullptr /* parse all fields */, writer); diff --git a/src/trace_processor/importers/proto/winscope/surfaceflinger_layers_parser.cc b/src/trace_processor/importers/proto/winscope/surfaceflinger_layers_parser.cc index ebe370c6d..20687e4e1 100644 --- a/src/trace_processor/importers/proto/winscope/surfaceflinger_layers_parser.cc +++ b/src/trace_processor/importers/proto/winscope/surfaceflinger_layers_parser.cc @@ -18,8 +18,8 @@ #include "protos/perfetto/trace/android/surfaceflinger_layers.pbzero.h" #include "src/trace_processor/importers/common/args_tracker.h" +#include "src/trace_processor/importers/proto/args_parser.h" #include "src/trace_processor/importers/proto/winscope/winscope.descriptor.h" -#include "src/trace_processor/importers/proto/winscope/winscope_args_parser.h" #include "src/trace_processor/types/trace_processor_context.h" namespace perfetto { @@ -44,7 +44,7 @@ void SurfaceFlingerLayersParser::Parse(int64_t timestamp, .id; auto inserter = context_->args_tracker->AddArgsTo(snapshot_id); - WinscopeArgsParser writer(inserter, *context_->storage.get()); + ArgsParser writer(timestamp, inserter, *context_->storage); base::Status status = args_parser_.ParseMessage(blob, kLayersSnapshotProtoName, &kLayersSnapshotFieldsToArgsParse, writer); @@ -55,11 +55,12 @@ void SurfaceFlingerLayersParser::Parse(int64_t timestamp, protos::pbzero::LayersProto::Decoder layers_decoder( snapshot_decoder.layers().data, snapshot_decoder.layers().size); for (auto it = layers_decoder.layers(); it; ++it) { - ParseLayer(*it, snapshot_id); + ParseLayer(timestamp, *it, snapshot_id); } } void SurfaceFlingerLayersParser::ParseLayer( + int64_t timestamp, protozero::ConstBytes blob, tables::SurfaceFlingerLayersSnapshotTable::Id snapshot_id) { tables::SurfaceFlingerLayerTable::Row layer; @@ -69,7 +70,7 @@ void SurfaceFlingerLayersParser::ParseLayer( ArgsTracker tracker(context_); auto inserter = tracker.AddArgsTo(layerId); - WinscopeArgsParser writer(inserter, *context_->storage.get()); + ArgsParser writer(timestamp, inserter, *context_->storage); base::Status status = args_parser_.ParseMessage( blob, kLayerProtoName, nullptr /* parse all fields */, writer); if (!status.ok()) { diff --git a/src/trace_processor/importers/proto/winscope/surfaceflinger_layers_parser.h b/src/trace_processor/importers/proto/winscope/surfaceflinger_layers_parser.h index 866cbae68..f615a8e42 100644 --- a/src/trace_processor/importers/proto/winscope/surfaceflinger_layers_parser.h +++ b/src/trace_processor/importers/proto/winscope/surfaceflinger_layers_parser.h @@ -39,7 +39,8 @@ class SurfaceFlingerLayersParser { ".perfetto.protos.LayersSnapshotProto"; static constexpr auto* kLayerProtoName = ".perfetto.protos.LayerProto"; - void ParseLayer(protozero::ConstBytes blob, + void ParseLayer(int64_t timestamp, + protozero::ConstBytes blob, tables::SurfaceFlingerLayersSnapshotTable::Id); TraceProcessorContext* const context_; diff --git a/src/trace_processor/importers/proto/winscope/surfaceflinger_transactions_parser.cc b/src/trace_processor/importers/proto/winscope/surfaceflinger_transactions_parser.cc index ae3a3e912..c2c563813 100644 --- a/src/trace_processor/importers/proto/winscope/surfaceflinger_transactions_parser.cc +++ b/src/trace_processor/importers/proto/winscope/surfaceflinger_transactions_parser.cc @@ -18,8 +18,8 @@ #include "protos/perfetto/trace/android/surfaceflinger_transactions.pbzero.h" #include "src/trace_processor/importers/common/args_tracker.h" +#include "src/trace_processor/importers/proto/args_parser.h" #include "src/trace_processor/importers/proto/winscope/winscope.descriptor.h" -#include "src/trace_processor/importers/proto/winscope/winscope_args_parser.h" #include "src/trace_processor/storage/trace_storage.h" #include "src/trace_processor/types/trace_processor_context.h" @@ -43,7 +43,7 @@ void SurfaceFlingerTransactionsParser::Parse(int64_t timestamp, ArgsTracker tracker(context_); auto inserter = tracker.AddArgsTo(rowId); - WinscopeArgsParser writer(inserter, *context_->storage.get()); + ArgsParser writer(timestamp, inserter, *context_->storage.get()); base::Status status = args_parser_.ParseMessage(blob, kTransactionTraceEntryProtoName, nullptr /* parse all fields */, writer); diff --git a/src/trace_processor/importers/proto/winscope/viewcapture_args_parser.cc b/src/trace_processor/importers/proto/winscope/viewcapture_args_parser.cc new file mode 100644 index 000000000..ed881632e --- /dev/null +++ b/src/trace_processor/importers/proto/winscope/viewcapture_args_parser.cc @@ -0,0 +1,128 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/trace_processor/importers/proto/winscope/viewcapture_args_parser.h" +#include "perfetto/ext/base/string_utils.h" +#include "perfetto/ext/base/string_view.h" +#include "protos/perfetto/trace/interned_data/interned_data.pbzero.h" +#include "protos/perfetto/trace/profiling/profile_common.pbzero.h" +#include "src/trace_processor/importers/proto/packet_sequence_state_generation.h" + +namespace perfetto { +namespace trace_processor { + +ViewCaptureArgsParser::ViewCaptureArgsParser( + int64_t packet_timestamp, + ArgsTracker::BoundInserter& inserter, + TraceStorage& storage, + PacketSequenceStateGeneration* sequence_state) + : ArgsParser(packet_timestamp, inserter, storage, sequence_state), + storage_{storage} {} + +void ViewCaptureArgsParser::AddInteger(const Key& key, int64_t value) { + if (TryAddDeinternedString(key, static_cast<uint64_t>(value))) { + return; + } + ArgsParser::AddInteger(key, value); +} + +void ViewCaptureArgsParser::AddUnsignedInteger(const Key& key, uint64_t value) { + if (TryAddDeinternedString(key, value)) { + return; + } + ArgsParser::AddUnsignedInteger(key, value); +} + +bool ViewCaptureArgsParser::TryAddDeinternedString(const Key& key, + uint64_t iid) { + bool is_interned_field = base::EndsWith(key.key, "_iid"); + if (!is_interned_field) { + return false; + } + + const auto deintern_key = key.key.substr(0, key.key.size() - 4); + const auto deintern_flat_key = + key.flat_key.substr(0, key.flat_key.size() - 4); + const auto deintern_key_combined = Key{deintern_flat_key, deintern_key}; + const auto deintern_val = TryDeinternString(key, iid); + + if (!deintern_val) { + ArgsParser::AddString( + deintern_key_combined, + protozero::ConstChars{ERROR_MSG.data(), ERROR_MSG.size()}); + storage_.IncrementStats( + stats::winscope_viewcapture_missing_interned_string_parse_errors); + return false; + } + + ArgsParser::AddString(deintern_key_combined, *deintern_val); + return true; +} + +std::optional<protozero::ConstChars> ViewCaptureArgsParser::TryDeinternString( + const Key& key, + uint64_t iid) { + if (base::EndsWith(key.key, "class_name_iid")) { + auto* decoder = + seq_state() + ->LookupInternedMessage< + protos::pbzero::InternedData::kViewcaptureClassNameFieldNumber, + protos::pbzero::InternedString>(iid); + if (decoder) { + return protozero::ConstChars{ + reinterpret_cast<const char*>(decoder->str().data), + decoder->str().size}; + } + } else if (base::EndsWith(key.key, "package_name_iid")) { + auto* decoder = + seq_state() + ->LookupInternedMessage<protos::pbzero::InternedData:: + kViewcapturePackageNameFieldNumber, + protos::pbzero::InternedString>(iid); + if (decoder) { + return protozero::ConstChars{ + reinterpret_cast<const char*>(decoder->str().data), + decoder->str().size}; + } + } else if (base::EndsWith(key.key, "view_id_iid")) { + auto* decoder = + seq_state() + ->LookupInternedMessage< + protos::pbzero::InternedData::kViewcaptureViewIdFieldNumber, + protos::pbzero::InternedString>(iid); + if (decoder) { + return protozero::ConstChars{ + reinterpret_cast<const char*>(decoder->str().data), + decoder->str().size}; + } + } else if (base::EndsWith(key.key, "window_name_iid")) { + auto* decoder = + seq_state() + ->LookupInternedMessage< + protos::pbzero::InternedData::kViewcaptureWindowNameFieldNumber, + protos::pbzero::InternedString>(iid); + if (decoder) { + return protozero::ConstChars{ + reinterpret_cast<const char*>(decoder->str().data), + decoder->str().size}; + } + } + + return std::nullopt; +} + +} // namespace trace_processor +} // namespace perfetto diff --git a/src/trace_processor/importers/proto/winscope/viewcapture_args_parser.h b/src/trace_processor/importers/proto/winscope/viewcapture_args_parser.h new file mode 100644 index 000000000..32b76b6e6 --- /dev/null +++ b/src/trace_processor/importers/proto/winscope/viewcapture_args_parser.h @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SRC_TRACE_PROCESSOR_IMPORTERS_PROTO_WINSCOPE_VIEWCAPTURE_ARGS_PARSER_H_ +#define SRC_TRACE_PROCESSOR_IMPORTERS_PROTO_WINSCOPE_VIEWCAPTURE_ARGS_PARSER_H_ + +#include <optional> + +#include "src/trace_processor/importers/proto/args_parser.h" + +namespace perfetto { +namespace trace_processor { + +// Specialized args parser to de-intern ViewCapture strings +class ViewCaptureArgsParser : public ArgsParser { + public: + using Key = ArgsParser::Key; + + ViewCaptureArgsParser(int64_t packet_timestamp, + ArgsTracker::BoundInserter& inserter, + TraceStorage& storage, + PacketSequenceStateGeneration* sequence_state); + void AddInteger(const Key&, int64_t) override; + void AddUnsignedInteger(const Key&, uint64_t) override; + + private: + bool TryAddDeinternedString(const Key&, uint64_t); + std::optional<protozero::ConstChars> TryDeinternString(const Key&, uint64_t); + + const base::StringView ERROR_MSG{"STRING DE-INTERNING ERROR"}; + TraceStorage& storage_; +}; + +} // namespace trace_processor +} // namespace perfetto + +#endif // SRC_TRACE_PROCESSOR_IMPORTERS_PROTO_WINSCOPE_VIEWCAPTURE_ARGS_PARSER_H_ diff --git a/src/trace_processor/importers/proto/winscope/winscope_args_parser.cc b/src/trace_processor/importers/proto/winscope/winscope_args_parser.cc deleted file mode 100644 index aa4ff3e69..000000000 --- a/src/trace_processor/importers/proto/winscope/winscope_args_parser.cc +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Copyright (C) 2023 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "src/trace_processor/importers/proto/winscope/winscope_args_parser.h" - -namespace perfetto { -namespace trace_processor { - -WinscopeArgsParser::WinscopeArgsParser(ArgsTracker::BoundInserter& inserter, - TraceStorage& storage) - : inserter_{inserter}, storage_{storage} {} - -void WinscopeArgsParser::AddInteger(const Key& key, int64_t value) { - const auto flat_key_id = - storage_.InternString(base::StringView(key.flat_key)); - const auto key_id = storage_.InternString(base::StringView(key.key)); - const auto variadic_val = Variadic::Integer(value); - inserter_.AddArg(flat_key_id, key_id, variadic_val); -} - -void WinscopeArgsParser::AddUnsignedInteger(const Key& key, uint64_t value) { - const auto flat_key_id = - storage_.InternString(base::StringView(key.flat_key)); - const auto key_id = storage_.InternString(base::StringView(key.key)); - const auto variadic_val = Variadic::UnsignedInteger(value); - inserter_.AddArg(flat_key_id, key_id, variadic_val); -} - -void WinscopeArgsParser::AddString(const Key& key, - const protozero::ConstChars& value) { - const auto flat_key_id = - storage_.InternString(base::StringView(key.flat_key)); - const auto key_id = storage_.InternString(base::StringView(key.key)); - const auto variadic_val = Variadic::String(storage_.InternString(value)); - inserter_.AddArg(flat_key_id, key_id, variadic_val); -} - -void WinscopeArgsParser::AddString(const Key& key, const std::string& value) { - const auto flat_key_id = - storage_.InternString(base::StringView(key.flat_key)); - const auto key_id = storage_.InternString(base::StringView(key.key)); - const auto variadic_val = - Variadic::String(storage_.InternString(base::StringView(value))); - inserter_.AddArg(flat_key_id, key_id, variadic_val); -} - -void WinscopeArgsParser::AddDouble(const Key& key, double value) { - const auto flat_key_id = - storage_.InternString(base::StringView(key.flat_key)); - const auto key_id = storage_.InternString(base::StringView(key.key)); - const auto variadic_val = Variadic::Real(value); - inserter_.AddArg(flat_key_id, key_id, variadic_val); -} - -void WinscopeArgsParser::AddPointer(const Key& key, const void* value) { - const auto flat_key_id = - storage_.InternString(base::StringView(key.flat_key)); - const auto key_id = storage_.InternString(base::StringView(key.key)); - const auto variadic_val = - Variadic::Pointer(reinterpret_cast<uintptr_t>(value)); - inserter_.AddArg(flat_key_id, key_id, variadic_val); -} - -void WinscopeArgsParser::AddBoolean(const Key& key, bool value) { - const auto flat_key_id = - storage_.InternString(base::StringView(key.flat_key)); - const auto key_id = storage_.InternString(base::StringView(key.key)); - const auto variadic_val = Variadic::Boolean(value); - inserter_.AddArg(flat_key_id, key_id, variadic_val); -} - -bool WinscopeArgsParser::AddJson(const Key&, const protozero::ConstChars&) { - PERFETTO_FATAL("Unexpected JSON value when parsing SurfaceFlinger data"); -} - -void WinscopeArgsParser::AddNull(const Key& key) { - const auto flat_key_id = - storage_.InternString(base::StringView(key.flat_key)); - const auto key_id = storage_.InternString(base::StringView(key.key)); - const auto variadic_val = Variadic::Null(); - inserter_.AddArg(flat_key_id, key_id, variadic_val); -} - -size_t WinscopeArgsParser::GetArrayEntryIndex(const std::string& array_key) { - return inserter_.GetNextArrayEntryIndex( - storage_.InternString(base::StringView(array_key))); -} - -size_t WinscopeArgsParser::IncrementArrayEntryIndex( - const std::string& array_key) { - return inserter_.IncrementArrayEntryIndex( - storage_.InternString(base::StringView(array_key))); -} - -PacketSequenceStateGeneration* WinscopeArgsParser::seq_state() { - return nullptr; -} - -InternedMessageView* WinscopeArgsParser::GetInternedMessageView( - uint32_t field_id, - uint64_t iid) { - base::ignore_result(field_id); - base::ignore_result(iid); - return nullptr; -} - -} // namespace trace_processor -} // namespace perfetto diff --git a/src/trace_processor/importers/proto/winscope/winscope_module.cc b/src/trace_processor/importers/proto/winscope/winscope_module.cc index 8d7e03753..81c7a5953 100644 --- a/src/trace_processor/importers/proto/winscope/winscope_module.cc +++ b/src/trace_processor/importers/proto/winscope/winscope_module.cc @@ -17,8 +17,9 @@ #include "src/trace_processor/importers/proto/winscope/winscope_module.h" #include "protos/perfetto/trace/android/winscope_extensions.pbzero.h" #include "protos/perfetto/trace/android/winscope_extensions_impl.pbzero.h" +#include "src/trace_processor/importers/proto/args_parser.h" +#include "src/trace_processor/importers/proto/winscope/viewcapture_args_parser.h" #include "src/trace_processor/importers/proto/winscope/winscope.descriptor.h" -#include "src/trace_processor/importers/proto/winscope/winscope_args_parser.h" namespace perfetto { namespace trace_processor { @@ -76,13 +77,15 @@ void WinscopeModule::ParseTracePacketData(const TracePacket::Decoder& decoder, decoder.protolog_viewer_config()); return; case TracePacket::kWinscopeExtensionsFieldNumber: - ParseWinscopeExtensionsData(decoder.winscope_extensions(), timestamp); + ParseWinscopeExtensionsData(decoder.winscope_extensions(), timestamp, + data); return; } } void WinscopeModule::ParseWinscopeExtensionsData(protozero::ConstBytes blob, - int64_t timestamp) { + int64_t timestamp, + const TracePacketData& data) { WinscopeExtensionsImpl::Decoder decoder(blob.data, blob.size); if (auto field = @@ -97,6 +100,11 @@ void WinscopeModule::ParseWinscopeExtensionsData(protozero::ConstBytes blob, WinscopeExtensionsImpl::kInputmethodServiceFieldNumber); field.valid()) { ParseInputMethodServiceData(timestamp, field.as_bytes()); + } else if (field = + decoder.Get(WinscopeExtensionsImpl::kViewcaptureFieldNumber); + field.valid()) { + ParseViewCaptureData(timestamp, field.as_bytes(), + data.sequence_state.get()); } } @@ -109,7 +117,7 @@ void WinscopeModule::ParseInputMethodClientsData(int64_t timestamp, ArgsTracker tracker(context_); auto inserter = tracker.AddArgsTo(rowId); - WinscopeArgsParser writer(inserter, *context_->storage.get()); + ArgsParser writer(timestamp, inserter, *context_->storage.get()); base::Status status = args_parser_.ParseMessage(blob, kInputMethodClientsProtoName, nullptr /* parse all fields */, writer); @@ -130,7 +138,7 @@ void WinscopeModule::ParseInputMethodManagerServiceData( ArgsTracker tracker(context_); auto inserter = tracker.AddArgsTo(rowId); - WinscopeArgsParser writer(inserter, *context_->storage.get()); + ArgsParser writer(timestamp, inserter, *context_->storage.get()); base::Status status = args_parser_.ParseMessage(blob, kInputMethodManagerServiceProtoName, nullptr /* parse all fields */, writer); @@ -149,7 +157,7 @@ void WinscopeModule::ParseInputMethodServiceData(int64_t timestamp, ArgsTracker tracker(context_); auto inserter = tracker.AddArgsTo(rowId); - WinscopeArgsParser writer(inserter, *context_->storage.get()); + ArgsParser writer(timestamp, inserter, *context_->storage.get()); base::Status status = args_parser_.ParseMessage(blob, kInputMethodServiceProtoName, nullptr /* parse all fields */, writer); @@ -159,5 +167,24 @@ void WinscopeModule::ParseInputMethodServiceData(int64_t timestamp, } } +void WinscopeModule::ParseViewCaptureData( + int64_t timestamp, + protozero::ConstBytes blob, + PacketSequenceStateGeneration* sequence_state) { + tables::ViewCaptureTable::Row row; + row.ts = timestamp; + auto rowId = context_->storage->mutable_viewcapture_table()->Insert(row).id; + + ArgsTracker tracker(context_); + auto inserter = tracker.AddArgsTo(rowId); + ViewCaptureArgsParser writer(timestamp, inserter, *context_->storage.get(), + sequence_state); + base::Status status = args_parser_.ParseMessage( + blob, kViewCaptureProtoName, nullptr /* parse all fields */, writer); + if (!status.ok()) { + context_->storage->IncrementStats(stats::winscope_viewcapture_parse_errors); + } +} + } // namespace trace_processor } // namespace perfetto diff --git a/src/trace_processor/importers/proto/winscope/winscope_module.h b/src/trace_processor/importers/proto/winscope/winscope_module.h index c77fcb49c..7f6ae083c 100644 --- a/src/trace_processor/importers/proto/winscope/winscope_module.h +++ b/src/trace_processor/importers/proto/winscope/winscope_module.h @@ -42,21 +42,25 @@ class WinscopeModule : public ProtoImporterModule { private: void ParseWinscopeExtensionsData(protozero::ConstBytes blob, - int64_t timestamp); + int64_t timestamp, + const TracePacketData&); void ParseInputMethodClientsData(int64_t timestamp, protozero::ConstBytes blob); void ParseInputMethodManagerServiceData(int64_t timestamp, protozero::ConstBytes blob); void ParseInputMethodServiceData(int64_t timestamp, protozero::ConstBytes blob); + void ParseViewCaptureData(int64_t timestamp, + protozero::ConstBytes blob, + PacketSequenceStateGeneration* sequence_state); static constexpr auto* kInputMethodClientsProtoName = ".perfetto.protos.InputMethodClientsTraceProto"; static constexpr auto* kInputMethodManagerServiceProtoName = ".perfetto.protos.InputMethodManagerServiceTraceProto"; - static constexpr auto* kInputMethodServiceProtoName = ".perfetto.protos.InputMethodServiceTraceProto"; + static constexpr auto* kViewCaptureProtoName = ".perfetto.protos.ViewCapture"; TraceProcessorContext* const context_; DescriptorPool pool_; diff --git a/src/trace_processor/importers/zip/BUILD.gn b/src/trace_processor/importers/zip/BUILD.gn new file mode 100644 index 000000000..0262e02cd --- /dev/null +++ b/src/trace_processor/importers/zip/BUILD.gn @@ -0,0 +1,32 @@ +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source_set("full") { + sources = [ + "zip_trace_reader.cc", + "zip_trace_reader.h", + ] + deps = [ + "../../../../gn:default_deps", + "../../../../include/perfetto/ext/base:base", + "../../../trace_processor:storage_minimal", + "../../types", + "../../util:trace_type", + "../../util:util", + "../../util:zip_reader", + "../android_bugreport", + "../common", + "../proto:minimal", + ] +} diff --git a/src/trace_processor/importers/zip/zip_trace_reader.cc b/src/trace_processor/importers/zip/zip_trace_reader.cc new file mode 100644 index 000000000..20f019f55 --- /dev/null +++ b/src/trace_processor/importers/zip/zip_trace_reader.cc @@ -0,0 +1,158 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/trace_processor/importers/zip/zip_trace_reader.h" + +#include <algorithm> +#include <cinttypes> +#include <cstdint> +#include <cstring> +#include <memory> +#include <string> +#include <tuple> +#include <utility> +#include <vector> + +#include "perfetto/base/logging.h" +#include "perfetto/base/status.h" +#include "perfetto/ext/base/status_or.h" +#include "perfetto/trace_processor/trace_blob.h" +#include "perfetto/trace_processor/trace_blob_view.h" +#include "src/trace_processor/forwarding_trace_parser.h" +#include "src/trace_processor/importers/android_bugreport/android_bugreport_parser.h" +#include "src/trace_processor/importers/proto/proto_trace_tokenizer.h" +#include "src/trace_processor/types/trace_processor_context.h" +#include "src/trace_processor/util/status_macros.h" +#include "src/trace_processor/util/trace_type.h" + +namespace perfetto { +namespace trace_processor { +namespace { + +// Proto traces should always parsed first as they might contains clock sync +// data needed to correctly parse other traces. +// The rest of the types are sorted by position in the enum but this is not +// something users should rely on. +// TODO(carlscab): Proto traces with just ModuleSymbols packets should be an +// exception. We actually need those are the very end (once whe have all the +// Frames). Alternatively we could build a map address -> symbol during +// tokenization and use this during parsing to resolve symbols. +bool CompareTraceType(TraceType lhs, TraceType rhs) { + if (rhs == TraceType::kProtoTraceType) { + return false; + } + if (lhs == TraceType::kProtoTraceType) { + return true; + } + return lhs < rhs; +} + +bool HasSymbols(const TraceBlobView& blob) { + bool has_symbols = false; + ProtoTraceTokenizer().Tokenize(blob.copy(), [&](TraceBlobView raw) { + protos::pbzero::TracePacket::Decoder packet(raw.data(), raw.size()); + has_symbols = packet.has_module_symbols(); + return base::ErrStatus("break"); + }); + return has_symbols; +} + +} // namespace + +ZipTraceReader::ZipTraceReader(TraceProcessorContext* context) + : context_(context) {} +ZipTraceReader::~ZipTraceReader() = default; + +bool ZipTraceReader::Entry::operator<(const Entry& rhs) const { + // Traces with symbols should be the last ones to be read. + if (has_symbols) { + return false; + } + if (rhs.has_symbols) { + return true; + } + if (CompareTraceType(trace_type, rhs.trace_type)) { + return true; + } + if (CompareTraceType(rhs.trace_type, trace_type)) { + return false; + } + return std::tie(name, index) < std::tie(rhs.name, rhs.index); +} + +util::Status ZipTraceReader::Parse(TraceBlobView blob) { + zip_reader_.Parse(blob.data(), blob.size()); + return base::OkStatus(); +} + +void ZipTraceReader::NotifyEndOfFile() { + base::Status status = NotifyEndOfFileImpl(); + if (!status.ok()) { + PERFETTO_ELOG("ZipTraceReader failed: %s", status.c_message()); + } +} + +base::Status ZipTraceReader::NotifyEndOfFileImpl() { + std::vector<util::ZipFile> files = zip_reader_.TakeFiles(); + + // Android bug reports are ZIP files and its files do not get handled + // separately. + if (AndroidBugreportParser::IsAndroidBugReport(files)) { + return AndroidBugreportParser::Parse(context_, std::move(files)); + } + + base::StatusOr<std::vector<Entry>> entries = ExtractEntries(std::move(files)); + if (!entries.ok()) { + return entries.status(); + } + std::sort(entries->begin(), entries->end()); + + for (Entry& e : *entries) { + parsers_.push_back(std::make_unique<ForwardingTraceParser>(context_)); + auto& parser = *parsers_.back(); + RETURN_IF_ERROR(parser.Parse(std::move(e.uncompressed_data))); + parser.NotifyEndOfFile(); + } + return base::OkStatus(); +} + +base::StatusOr<std::vector<ZipTraceReader::Entry>> +ZipTraceReader::ExtractEntries(std::vector<util::ZipFile> files) const { + // TODO(carlsacab): There is a lot of unnecessary copying going on here. + // ZipTraceReader can directly parse the ZIP file and given that we know the + // decompressed size we could directly decompress into TraceBlob chunks and + // send them to the tokenizer. + std::vector<Entry> entries; + std::vector<uint8_t> buffer; + for (size_t i = 0; i < files.size(); ++i) { + const util::ZipFile& zip_file = files[i]; + Entry entry; + entry.name = zip_file.name(); + entry.index = i; + RETURN_IF_ERROR(files[i].Decompress(&buffer)); + entry.uncompressed_data = + TraceBlobView(TraceBlob::CopyFrom(buffer.data(), buffer.size())); + entry.trace_type = GuessTraceType(entry.uncompressed_data.data(), + entry.uncompressed_data.size()); + entry.has_symbols = entry.trace_type == TraceType::kProtoTraceType && + HasSymbols(entry.uncompressed_data); + entries.push_back(std::move(entry)); + } + return std::move(entries); +} + +} // namespace trace_processor +} // namespace perfetto diff --git a/src/trace_processor/importers/zip/zip_trace_reader.h b/src/trace_processor/importers/zip/zip_trace_reader.h new file mode 100644 index 000000000..d4f3d8818 --- /dev/null +++ b/src/trace_processor/importers/zip/zip_trace_reader.h @@ -0,0 +1,84 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SRC_TRACE_PROCESSOR_IMPORTERS_ZIP_ZIP_TRACE_READER_H_ +#define SRC_TRACE_PROCESSOR_IMPORTERS_ZIP_ZIP_TRACE_READER_H_ + +#include <cstddef> +#include <memory> +#include <string> +#include <vector> + +#include "perfetto/base/status.h" +#include "perfetto/ext/base/status_or.h" +#include "perfetto/trace_processor/trace_blob_view.h" +#include "src/trace_processor/importers/common/chunked_trace_reader.h" +#include "src/trace_processor/util/trace_type.h" +#include "src/trace_processor/util/zip_reader.h" + +namespace perfetto::trace_processor { + +class ForwardingTraceParser; +class TraceProcessorContext; + +// Forwards files contained in a ZIP to the appropiate ChunkedTraceReader. It is +// guaranteed that proto traces will be parsed first. +class ZipTraceReader : public ChunkedTraceReader { + public: + explicit ZipTraceReader(TraceProcessorContext* context); + ~ZipTraceReader() override; + + // ChunkedTraceReader implementation + util::Status Parse(TraceBlobView) override; + void NotifyEndOfFile() override; + + private: + // Represents a file in the ZIP file. Used to sort them before sending the + // files one by one to a `ForwardingTraceParser` instance. + struct Entry { + // File name. Used to break ties. + std::string name; + // Position in the zip file. Used to break ties. + size_t index; + // Trace type. This is the main attribute traces are ordered by. Proto + // traces are always parsed first as they might contains clock sync + // data needed to correctly parse other traces. + TraceType trace_type; + TraceBlobView uncompressed_data; + // True for proto trace_types whose fist message is a ModuleSymbols packet + bool has_symbols = false; + // Comparator used to determine the order in which files in the ZIP will be + // read. + bool operator<(const Entry& rhs) const; + }; + + base::Status NotifyEndOfFileImpl(); + base::StatusOr<std::vector<Entry>> ExtractEntries( + std::vector<util::ZipFile> files) const; + base::Status ParseEntry(Entry entry); + + TraceProcessorContext* const context_; + util::ZipReader zip_reader_; + // For every file in the ZIP we will create a `ForwardingTraceParser`instance + // and send that file to it for tokenization. The instances are kept around + // here as some tokenizers might keep state that is later needed after + // sorting. + std::vector<std::unique_ptr<ForwardingTraceParser>> parsers_; +}; + +} // namespace perfetto::trace_processor + +#endif // SRC_TRACE_PROCESSOR_IMPORTERS_ZIP_ZIP_TRACE_READER_H_ diff --git a/src/trace_processor/metrics/sql/android/android_boot.sql b/src/trace_processor/metrics/sql/android/android_boot.sql index 91403878f..5738bffc5 100644 --- a/src/trace_processor/metrics/sql/android/android_boot.sql +++ b/src/trace_processor/metrics/sql/android/android_boot.sql @@ -19,6 +19,24 @@ INCLUDE PERFETTO MODULE android.app_process_starts; INCLUDE PERFETTO MODULE android.garbage_collection; INCLUDE PERFETTO MODULE android.oom_adjuster; +DROP VIEW IF EXISTS android_oom_adj_intervals_with_detailed_bucket_name; +CREATE PERFETTO VIEW android_oom_adj_intervals_with_detailed_bucket_name AS +SELECT + ts, + dur, + score, + android_oom_adj_score_to_detailed_bucket_name(score, android_appid) AS bucket, + upid, + process_name, + oom_adj_id, + oom_adj_ts, + oom_adj_dur, + oom_adj_track_id, + oom_adj_thread_name, + oom_adj_reason, + oom_adj_trigger +FROM _oom_adjuster_intervals; + CREATE OR REPLACE PERFETTO FUNCTION get_durations(process_name STRING) RETURNS TABLE(uint_sleep_dur LONG, total_dur LONG) AS SELECT @@ -41,7 +59,7 @@ SELECT bucket, process_name, oom_adj_reason -FROM android_oom_adj_intervals; +FROM android_oom_adj_intervals_with_detailed_bucket_name; DROP VIEW IF EXISTS oom_adj_events_by_process_name; CREATE PERFETTO VIEW oom_adj_events_by_process_name AS @@ -280,7 +298,7 @@ SELECT AndroidBootMetric( NULL as name, bucket, SUM(dur) as total_dur - FROM android_oom_adj_intervals + FROM android_oom_adj_intervals_with_detailed_bucket_name WHERE ts > first_user_unlocked() GROUP BY bucket) ), @@ -296,7 +314,7 @@ SELECT AndroidBootMetric( process_name as name, bucket, SUM(dur) as total_dur - FROM android_oom_adj_intervals + FROM android_oom_adj_intervals_with_detailed_bucket_name WHERE ts > first_user_unlocked() AND process_name IS NOT NULL GROUP BY process_name, bucket) @@ -318,7 +336,7 @@ SELECT AndroidBootMetric( AVG(oom_adj_dur) as avg_oom_adj_dur, COUNT(DISTINCT(oom_adj_id)) oom_adj_event_count, oom_adj_reason - FROM android_oom_adj_intervals + FROM android_oom_adj_intervals_with_detailed_bucket_name WHERE ts > first_user_unlocked() GROUP BY oom_adj_reason ) diff --git a/src/trace_processor/metrics/sql/android/android_lmk_reason.sql b/src/trace_processor/metrics/sql/android/android_lmk_reason.sql index 0ca22583a..b7c7ca524 100644 --- a/src/trace_processor/metrics/sql/android/android_lmk_reason.sql +++ b/src/trace_processor/metrics/sql/android/android_lmk_reason.sql @@ -64,8 +64,8 @@ lmk_process_sizes AS ( FROM lmk_events JOIN rss_and_swap_span WHERE lmk_events.ts - BETWEEN rss_and_swap_span.ts - AND rss_and_swap_span.ts + MAX(rss_and_swap_span.dur - 1, 0) + BETWEEN rss_and_swap_span.ts + AND rss_and_swap_span.ts + MAX(rss_and_swap_span.dur - 1, 0) ), lmk_process_sizes_output AS ( SELECT ts, RepeatedField(AndroidLmkReasonMetric_Process( diff --git a/src/trace_processor/metrics/sql/android/android_oom_adjuster.sql b/src/trace_processor/metrics/sql/android/android_oom_adjuster.sql index 7b752a89d..156458608 100644 --- a/src/trace_processor/metrics/sql/android/android_oom_adjuster.sql +++ b/src/trace_processor/metrics/sql/android/android_oom_adjuster.sql @@ -15,6 +15,51 @@ -- INCLUDE PERFETTO MODULE android.oom_adjuster; +DROP VIEW IF EXISTS android_oom_adj_intervals_with_detailed_bucket_name; +CREATE PERFETTO VIEW android_oom_adj_intervals_with_detailed_bucket_name ( + -- Timestamp the oom_adj score of the process changed + ts INT, + -- Duration until the next oom_adj score change of the process. + dur INT, + -- oom_adj score of the process. + score INT, + -- oom_adj bucket of the process. + bucket STRING, + -- Upid of the process having an oom_adj update. + upid INT, + -- Name of the process having an oom_adj update. + process_name STRING, + -- Slice id of the latest oom_adj update in the system_server. + oom_adj_id INT, + -- Timestamp of the latest oom_adj update in the system_server. + oom_adj_ts INT, + -- Duration of the latest oom_adj update in the system_server. + oom_adj_dur INT, + -- Track id of the latest oom_adj update in the system_server + oom_adj_track_id INT, + -- Thread name of the latest oom_adj update in the system_server. + oom_adj_thread_name STRING, + -- Reason for the latest oom_adj update in the system_server. + oom_adj_reason STRING, + -- Trigger for the latest oom_adj update in the system_server. + oom_adj_trigger STRING + ) AS +SELECT + ts, + dur, + score, + android_oom_adj_score_to_detailed_bucket_name(score, android_appid) AS bucket, + upid, + process_name, + oom_adj_id, + oom_adj_ts, + oom_adj_dur, + oom_adj_track_id, + oom_adj_thread_name, + oom_adj_reason, + oom_adj_trigger +FROM _oom_adjuster_intervals; + DROP TABLE IF EXISTS _oom_adj_events_with_src_bucket; CREATE PERFETTO TABLE _oom_adj_events_with_src_bucket AS @@ -24,7 +69,7 @@ SELECT bucket, process_name, oom_adj_reason -FROM android_oom_adj_intervals; +FROM android_oom_adj_intervals_with_detailed_bucket_name; DROP VIEW IF EXISTS oom_adj_events_by_process_name; CREATE PERFETTO VIEW oom_adj_events_by_process_name AS @@ -100,7 +145,7 @@ SELECT AndroidOomAdjusterMetric( ) FROM ( SELECT NULL as name, bucket, SUM(dur) as total_dur - FROM android_oom_adj_intervals GROUP BY bucket + FROM android_oom_adj_intervals_with_detailed_bucket_name GROUP BY bucket ) ), 'oom_adj_bucket_duration_agg_by_process',(SELECT RepeatedField( @@ -112,7 +157,7 @@ SELECT AndroidOomAdjusterMetric( ) FROM ( SELECT process_name as name, bucket, SUM(dur) as total_dur - FROM android_oom_adj_intervals + FROM android_oom_adj_intervals_with_detailed_bucket_name WHERE process_name IS NOT NULL GROUP BY process_name, bucket ) @@ -133,7 +178,7 @@ SELECT AndroidOomAdjusterMetric( AVG(oom_adj_dur) as avg_oom_adj_dur, COUNT(DISTINCT(oom_adj_id)) oom_adj_event_count, oom_adj_reason - FROM android_oom_adj_intervals GROUP BY oom_adj_reason + FROM android_oom_adj_intervals_with_detailed_bucket_name GROUP BY oom_adj_reason ) ) ); diff --git a/src/trace_processor/metrics/sql/android/android_startup.sql b/src/trace_processor/metrics/sql/android/android_startup.sql index 95d050bbe..2e2e48e71 100644 --- a/src/trace_processor/metrics/sql/android/android_startup.sql +++ b/src/trace_processor/metrics/sql/android/android_startup.sql @@ -514,7 +514,7 @@ SELECT ) ), - 'slow_start_reason_detailed', get_slow_start_reason_detailed(launches.startup_id) + 'slow_start_reason_with_details', get_slow_start_reason_with_details(launches.startup_id) ) AS startup FROM android_startups launches; diff --git a/src/trace_processor/metrics/sql/android/process_mem.sql b/src/trace_processor/metrics/sql/android/process_mem.sql index f52f63e1f..f3b1c88f2 100644 --- a/src/trace_processor/metrics/sql/android/process_mem.sql +++ b/src/trace_processor/metrics/sql/android/process_mem.sql @@ -14,75 +14,51 @@ -- limitations under the License. -- --- Create all the views used to generate the Android Memory metrics proto. --- Anon RSS -SELECT RUN_METRIC('android/process_counter_span_view.sql', - 'table_name', 'anon_rss', - 'counter_name', 'mem.rss.anon'); +INCLUDE PERFETTO MODULE memory.linux.process; --- File RSS -SELECT RUN_METRIC('android/process_counter_span_view.sql', - 'table_name', 'file_rss', - 'counter_name', 'mem.rss.file'); +SELECT RUN_METRIC('android/process_oom_score.sql'); -SELECT RUN_METRIC('android/process_counter_span_view.sql', - 'table_name', 'shmem_rss', - 'counter_name', 'mem.rss.shmem'); +DROP VIEW IF EXISTS anon_rss_span; +CREATE PERFETTO VIEW anon_rss_span AS +SELECT * FROM _anon_rss; --- Swap -SELECT RUN_METRIC('android/process_counter_span_view.sql', - 'table_name', 'swap', - 'counter_name', 'mem.swap'); +DROP VIEW IF EXISTS file_rss_span; +CREATE PERFETTO VIEW file_rss_span AS +SELECT * FROM _file_rss; --- OOM score -SELECT RUN_METRIC('android/process_oom_score.sql'); +DROP VIEW IF EXISTS shmem_rss_span; +CREATE PERFETTO VIEW shmem_rss_span AS +SELECT * FROM _shmem_rss; --- Anon RSS + Swap -DROP TABLE IF EXISTS anon_and_swap_join; -CREATE VIRTUAL TABLE anon_and_swap_join -USING SPAN_OUTER_JOIN(anon_rss_span PARTITIONED upid, swap_span PARTITIONED upid); +DROP VIEW IF EXISTS swap_span; +CREATE PERFETTO VIEW swap_span AS +SELECT * FROM _swap; DROP VIEW IF EXISTS anon_and_swap_span; CREATE PERFETTO VIEW anon_and_swap_span AS SELECT ts, dur, upid, IFNULL(anon_rss_val, 0) + IFNULL(swap_val, 0) AS anon_and_swap_val -FROM anon_and_swap_join; - --- Anon RSS + file RSS + Swap -DROP TABLE IF EXISTS anon_and_file_and_swap_join; -CREATE VIRTUAL TABLE anon_and_file_and_swap_join -USING SPAN_OUTER_JOIN( - anon_and_swap_join PARTITIONED upid, - file_rss_span PARTITIONED upid -); - --- RSS + Swap -DROP TABLE IF EXISTS rss_and_swap_join; -CREATE VIRTUAL TABLE rss_and_swap_join -USING SPAN_OUTER_JOIN( - anon_and_file_and_swap_join PARTITIONED upid, - shmem_rss_span PARTITIONED upid -); +FROM _anon_swap_sj; DROP VIEW IF EXISTS rss_and_swap_span; CREATE PERFETTO VIEW rss_and_swap_span AS SELECT - ts, dur, upid, - CAST(IFNULL(file_rss_val, 0) AS INT) AS file_rss_val, - CAST(IFNULL(anon_rss_val, 0) AS INT) AS anon_rss_val, - CAST(IFNULL(shmem_rss_val, 0) AS INT) AS shmem_rss_val, - CAST(IFNULL(swap_val, 0) AS INT) AS swap_val, - CAST( - IFNULL(anon_rss_val, 0) - + IFNULL(file_rss_val, 0) - + IFNULL(shmem_rss_val, 0) AS int) AS rss_val, - CAST( - IFNULL(anon_rss_val, 0) - + IFNULL(swap_val, 0) - + IFNULL(file_rss_val, 0) - + IFNULL(shmem_rss_val, 0) AS int) AS rss_and_swap_val -FROM rss_and_swap_join; + ts, + dur, + upid, + file_rss AS file_rss_val, + anon_rss AS anon_rss_val, + shmem_rss AS shmem_rss_val, + swap AS swap_val, + COALESCE(file_rss, 0) + + COALESCE(anon_rss, 0) + + COALESCE(shmem_rss, 0) AS rss_val, + COALESCE(file_rss, 0) + + COALESCE(anon_rss, 0) + + COALESCE(shmem_rss, 0) + + COALESCE(swap, 0) AS rss_and_swap_val +FROM _memory_rss_and_swap_per_process_table; -- If we have dalvik events enabled (for ART trace points) we can construct the java heap timeline. SELECT RUN_METRIC('android/process_counter_span_view.sql', @@ -94,22 +70,22 @@ CREATE PERFETTO VIEW java_heap_span AS SELECT ts, dur, upid, java_heap_kb_val * 1024 AS java_heap_val FROM java_heap_kb_span; +DROP TABLE IF EXISTS java_heap_by_oom_span; +CREATE VIRTUAL TABLE java_heap_by_oom_span +USING SPAN_JOIN(java_heap_span PARTITIONED upid, oom_score_span PARTITIONED upid); + DROP TABLE IF EXISTS anon_rss_by_oom_span; CREATE VIRTUAL TABLE anon_rss_by_oom_span -USING SPAN_JOIN(anon_rss_span PARTITIONED upid, oom_score_span PARTITIONED upid); +USING SPAN_JOIN(_anon_rss PARTITIONED upid, oom_score_span PARTITIONED upid); DROP TABLE IF EXISTS file_rss_by_oom_span; CREATE VIRTUAL TABLE file_rss_by_oom_span -USING SPAN_JOIN(file_rss_span PARTITIONED upid, oom_score_span PARTITIONED upid); +USING SPAN_JOIN(_file_rss PARTITIONED upid, oom_score_span PARTITIONED upid); DROP TABLE IF EXISTS swap_by_oom_span; CREATE VIRTUAL TABLE swap_by_oom_span -USING SPAN_JOIN(swap_span PARTITIONED upid, oom_score_span PARTITIONED upid); +USING SPAN_JOIN(_swap PARTITIONED upid, oom_score_span PARTITIONED upid); DROP TABLE IF EXISTS anon_and_swap_by_oom_span; CREATE VIRTUAL TABLE anon_and_swap_by_oom_span USING SPAN_JOIN(anon_and_swap_span PARTITIONED upid, oom_score_span PARTITIONED upid); - -DROP TABLE IF EXISTS java_heap_by_oom_span; -CREATE VIRTUAL TABLE java_heap_by_oom_span -USING SPAN_JOIN(java_heap_span PARTITIONED upid, oom_score_span PARTITIONED upid); diff --git a/src/trace_processor/metrics/sql/android/startup/slow_start_reasons.sql b/src/trace_processor/metrics/sql/android/startup/slow_start_reasons.sql index 2a673c003..9165f80b8 100644 --- a/src/trace_processor/metrics/sql/android/startup/slow_start_reasons.sql +++ b/src/trace_processor/metrics/sql/android/startup/slow_start_reasons.sql @@ -201,110 +201,227 @@ RETURNS STRING AS ORDER BY slice_dur DESC LIMIT 1; - -CREATE OR REPLACE PERFETTO FUNCTION get_slow_start_reason_detailed(startup_id LONG) +CREATE OR REPLACE PERFETTO FUNCTION get_slow_start_reason_with_details(startup_id LONG) RETURNS PROTO AS - SELECT RepeatedField(AndroidStartupMetric_SlowStartReasonDetailed( + SELECT RepeatedField(AndroidStartupMetric_SlowStartReason( + 'reason_id', reason_id, 'reason', slow_cause, - 'details', details)) + 'launch_dur', launch_dur, + 'expected_value', expected_val, + 'actual_value', actual_val)) FROM ( - SELECT 'No baseline or cloud profiles' AS slow_cause, - get_missing_baseline_profile_for_launch(launch.startup_id, launch.package) as details + SELECT 'No baseline or cloud profiles' as slow_cause, + launch.dur as launch_dur, + 'NO_BASELINE_OR_CLOUD_PROFILES' as reason_id, + AndroidStartupMetric_ThresholdValue( + 'value', FALSE, + 'unit', 'TRUE_OR_FALSE') as expected_val, + AndroidStartupMetric_ActualValue( + 'value', TRUE) as actual_val FROM android_startups launch WHERE launch.startup_id = $startup_id AND missing_baseline_profile_for_launch(launch.startup_id, launch.package) UNION ALL - SELECT 'Optimized artifacts missing, run from apk' as slow_cause, NULL as details + SELECT 'Optimized artifacts missing, run from apk' as slow_cause, + launch.dur as launch_dur, + 'RUN_FROM_APK' as reason_id, + AndroidStartupMetric_ThresholdValue( + 'value', FALSE, + 'unit', 'TRUE_OR_FALSE') as expected_val, + AndroidStartupMetric_ActualValue( + 'value', TRUE) as actual_val FROM android_startups launch WHERE launch.startup_id = $startup_id - AND run_from_apk_for_launch(launch.startup_id) + AND run_from_apk_for_launch(launch.startup_id) UNION ALL - SELECT 'Unlock running during launch' as slow_cause, NULL as details + SELECT 'Unlock running during launch' as slow_cause, + launch.dur as launch_dur, + 'UNLOCK_RUNNING' as reason_id, + AndroidStartupMetric_ThresholdValue( + 'value', FALSE, + 'unit', 'TRUE_OR_FALSE') as expected_val, + AndroidStartupMetric_ActualValue( + 'value', TRUE) as actual_val FROM android_startups launch WHERE launch.startup_id = $startup_id AND is_unlock_running_during_launch(launch.startup_id) UNION ALL - SELECT 'App in debuggable mode' as slow_cause, NULL as details - FROM android_startups launch + SELECT 'App in debuggable mode' as slow_cause, + launch.dur as launch_dur, + 'APP_IN_DEBUGGABLE_MODE' as reason_id, + AndroidStartupMetric_ThresholdValue( + 'value', FALSE, + 'unit', 'TRUE_OR_FALSE') as expected_val, + AndroidStartupMetric_ActualValue( + 'value', TRUE) as actual_val + FROM android_startups launch WHERE launch.startup_id = $startup_id AND is_process_debuggable(launch.package) UNION ALL - SELECT 'GC Activity' as slow_cause, NULL as details + SELECT 'GC Activity' as slow_cause, + launch.dur as launch_dur, + 'GC_ACTIVITY' as reason_id, + AndroidStartupMetric_ThresholdValue( + 'value', FALSE, + 'unit', 'TRUE_OR_FALSE') as expected_val, + AndroidStartupMetric_ActualValue( + 'value', TRUE) as actual_val FROM android_startups launch WHERE launch.startup_id = $startup_id AND total_gc_time_by_launch(launch.startup_id) > 0 UNION ALL - SELECT 'dex2oat running during launch' AS slow_cause, NULL as details + SELECT 'dex2oat running during launch' AS slow_cause, + launch.dur as launch_dur, + 'DEX2OAT_RUNNING' as reason_id, + AndroidStartupMetric_ThresholdValue( + 'value', FALSE, + 'unit', 'TRUE_OR_FALSE') as expected_val, + AndroidStartupMetric_ActualValue( + 'value', TRUE) as actual_val FROM android_startups launch WHERE launch.startup_id = $startup_id AND dur_of_process_running_concurrent_to_launch(launch.startup_id, '*dex2oat64') > 0 UNION ALL - SELECT 'installd running during launch' AS slow_cause, NULL as details + SELECT 'installd running during launch' AS slow_cause, + launch.dur as launch_dur, + 'INSTALLD_RUNNING' as reason_id, + AndroidStartupMetric_ThresholdValue( + 'value', FALSE, + 'unit', 'TRUE_OR_FALSE') as expected_val, + AndroidStartupMetric_ActualValue( + 'value', TRUE) as actual_val FROM android_startups launch WHERE launch.startup_id = $startup_id AND dur_of_process_running_concurrent_to_launch(launch.startup_id, '*installd') > 0 UNION ALL SELECT 'Main Thread - Time spent in Runnable state' as slow_cause, - get_main_thread_time_for_launch_in_runnable_state( - launch.startup_id, launch.dur) as details + launch.dur as launch_dur, + 'MAIN_THREAD_TIME_SPENT_IN_RUNNABLE' as reason_id, + AndroidStartupMetric_ThresholdValue( + 'value', 15, + 'unit', 'PERCENTAGE', + 'higher_expected', FALSE) as expected_val, + AndroidStartupMetric_ActualValue( + 'value', + main_thread_time_for_launch_in_runnable_state(launch.startup_id) * 100 / launch.dur, + 'dur', main_thread_time_for_launch_in_runnable_state(launch.startup_id)) as actual_val FROM android_startups launch WHERE launch.startup_id = $startup_id AND main_thread_time_for_launch_in_runnable_state(launch.startup_id) > launch.dur * 0.15 UNION ALL - SELECT 'Main Thread - Time spent in interruptible sleep state' - AS slow_cause, NULL as details + SELECT 'Main Thread - Time spent in interruptible sleep state' as slow_cause, + launch.dur as launch_dur, + 'MAIN_THREAD_TIME_SPENT_IN_INTERRUPTIBLE_SLEEP' as reason_id, + AndroidStartupMetric_ThresholdValue( + 'value', 2900000000, + 'unit', 'NS', + 'higher_expected', FALSE) as expected_val, + AndroidStartupMetric_ActualValue( + 'value', main_thread_time_for_launch_and_state(launch.startup_id, 'S')) as actual_val FROM android_startups launch WHERE launch.startup_id = $startup_id AND main_thread_time_for_launch_and_state(launch.startup_id, 'S') > 2900e6 UNION ALL - SELECT 'Main Thread - Time spent in Blocking I/O' as slow_cause, NULL as details + SELECT 'Main Thread - Time spent in Blocking I/O' as slow_cause, + launch.dur as launch_dur, + 'MAIN_THREAD_TIME_SPENT_IN_BLOCKING_IO' as reason_id, + AndroidStartupMetric_ThresholdValue( + 'value', 450000000, + 'unit', 'NS', + 'higher_expected', FALSE) as expected_val, + AndroidStartupMetric_ActualValue( + 'value', main_thread_time_for_launch_state_and_io_wait( + launch.startup_id, 'D*', TRUE)) as actual_val FROM android_startups launch WHERE launch.startup_id = $startup_id AND main_thread_time_for_launch_state_and_io_wait(launch.startup_id, 'D*', TRUE) > 450e6 UNION ALL SELECT 'Main Thread - Time spent in OpenDexFilesFromOat*' as slow_cause, - get_android_sum_dur_on_main_thread_for_startup_and_slice( - launch.startup_id, 'OpenDexFilesFromOat*', launch.dur) as details + launch.dur as launch_dur, + 'MAIN_THREAD_TIME_SPENT_IN_OPEN_DEX_FILES_FROM_OAT' as reason_id, + AndroidStartupMetric_ThresholdValue( + 'value', 20, + 'unit', 'PERCENTAGE', + 'higher_expected', FALSE) as expected_val, + AndroidStartupMetric_ActualValue( + 'value', android_sum_dur_on_main_thread_for_startup_and_slice( + launch.startup_id, 'OpenDexFilesFromOat*') * 100 / launch.dur, + 'dur', android_sum_dur_on_main_thread_for_startup_and_slice( + launch.startup_id, 'OpenDexFilesFromOat*')) as actual_val FROM android_startups launch WHERE launch.startup_id = $startup_id AND android_sum_dur_on_main_thread_for_startup_and_slice( launch.startup_id, 'OpenDexFilesFromOat*') > launch.dur * 0.2 UNION ALL - SELECT 'Time spent in bindApplication' - AS slow_cause, NULL as details + SELECT 'Time spent in bindApplication' as slow_cause, + launch.dur as launch_dur, + 'TIME_SPENT_IN_BIND_APPLICATION' as reason_id, + AndroidStartupMetric_ThresholdValue( + 'value', 1250000000, + 'unit', 'NS', + 'higher_expected', FALSE) as expected_val, + AndroidStartupMetric_ActualValue( + 'value', android_sum_dur_for_startup_and_slice( + launch.startup_id, 'bindApplication')) as actual_val FROM android_startups launch WHERE launch.startup_id = $startup_id AND android_sum_dur_for_startup_and_slice(launch.startup_id, 'bindApplication') > 1250e6 UNION ALL - SELECT 'Time spent in view inflation' as slow_cause, NULL as details + SELECT 'Time spent in view inflation' as slow_cause, + launch.dur as launch_dur, + 'TIME_SPENT_IN_VIEW_INFLATION' as reason_id, + AndroidStartupMetric_ThresholdValue( + 'value', 450000000, + 'unit', 'NS', + 'higher_expected', FALSE) as expected_val, + AndroidStartupMetric_ActualValue( + 'value', android_sum_dur_for_startup_and_slice( + launch.startup_id, 'inflate')) as actual_val FROM android_startups launch WHERE launch.startup_id = $startup_id AND android_sum_dur_for_startup_and_slice(launch.startup_id, 'inflate') > 450e6 UNION ALL SELECT 'Time spent in ResourcesManager#getResources' as slow_cause, - get_android_sum_dur_for_startup_and_slice( - launch.startup_id, 'ResourcesManager#getResources', 130) as details + launch.dur as launch_dur, + 'TIME_SPENT_IN_RESOURCES_MANAGER_GET_RESOURCES' as reason_id, + AndroidStartupMetric_ThresholdValue( + 'value', 130000000, + 'unit', 'NS', + 'higher_expected', FALSE) as expected_val, + AndroidStartupMetric_ActualValue( + 'value', android_sum_dur_for_startup_and_slice( + launch.startup_id, 'ResourcesManager#getResources')) as actual_val FROM android_startups launch WHERE launch.startup_id = $startup_id AND android_sum_dur_for_startup_and_slice( launch.startup_id, 'ResourcesManager#getResources') > 130e6 UNION ALL - SELECT 'Time spent verifying classes' - AS slow_cause, NULL as details + SELECT 'Time spent verifying classes' as slow_cause, + launch.dur as launch_dur, + 'TIME_SPENT_VERIFYING_CLASSES' as reason_id, + AndroidStartupMetric_ThresholdValue( + 'value', 15, + 'unit', 'PERCENTAGE', + 'higher_expected', FALSE) as expected_val, + AndroidStartupMetric_ActualValue( + 'value', android_sum_dur_for_startup_and_slice( + launch.startup_id, 'VerifyClass*') * 100 / launch.dur, + 'dur', android_sum_dur_for_startup_and_slice( + launch.startup_id, 'VerifyClass*')) as actual_val FROM android_startups launch WHERE launch.startup_id = $startup_id AND android_sum_dur_for_startup_and_slice(launch.startup_id, 'VerifyClass*') @@ -312,7 +429,15 @@ RETURNS PROTO AS UNION ALL SELECT 'Potential CPU contention with another process' AS slow_cause, - get_potential_cpu_contention_with_another_process(launch.startup_id) as details + launch.dur as launch_dur, + 'POTENTIAL_CPU_CONTENTION_WITH_ANOTHER_PROCESS' as reason_id, + AndroidStartupMetric_ThresholdValue( + 'value', 100000000, + 'unit', 'NS', + 'higher_expected', FALSE) as expected_val, + AndroidStartupMetric_ActualValue( + 'value', + main_thread_time_for_launch_in_runnable_state(launch.startup_id)) as actual_val FROM android_startups launch WHERE launch.startup_id = $startup_id AND main_thread_time_for_launch_in_runnable_state(launch.startup_id) > 100e6 AND @@ -320,7 +445,15 @@ RETURNS PROTO AS UNION ALL SELECT 'JIT Activity' as slow_cause, - get_jit_activity(launch.startup_id) as details + launch.dur as launch_dur, + 'JIT_ACTIVITY' as reason_id, + AndroidStartupMetric_ThresholdValue( + 'value', 100000000, + 'unit', 'NS', + 'higher_expected', FALSE) as expected_val, + AndroidStartupMetric_ActualValue( + 'value', thread_time_for_launch_state_and_thread( + launch.startup_id, 'Running', 'Jit thread pool')) as actual_val FROM android_startups launch WHERE launch.startup_id = $startup_id AND thread_time_for_launch_state_and_thread( @@ -330,8 +463,18 @@ RETURNS PROTO AS ) > 100e6 UNION ALL - SELECT 'Main Thread - Lock contention' - AS slow_cause, NULL as details + SELECT 'Main Thread - Lock contention' as slow_cause, + launch.dur as launch_dur, + 'MAIN_THREAD_LOCK_CONTENTION' as reason_id, + AndroidStartupMetric_ThresholdValue( + 'value', 20, + 'unit', 'PERCENTAGE', + 'higher_expected', FALSE) as expected_val, + AndroidStartupMetric_ActualValue( + 'value', android_sum_dur_on_main_thread_for_startup_and_slice( + launch.startup_id, 'Lock contention on*') * 100 / launch.dur, + 'dur', android_sum_dur_on_main_thread_for_startup_and_slice( + launch.startup_id, 'Lock contention on*')) as actual_val FROM android_startups launch WHERE launch.startup_id = $startup_id AND android_sum_dur_on_main_thread_for_startup_and_slice( @@ -340,61 +483,107 @@ RETURNS PROTO AS ) > launch.dur * 0.2 UNION ALL - SELECT 'Main Thread - Monitor contention' - AS slow_cause, NULL as details + SELECT 'Main Thread - Monitor contention' as slow_cause, + launch.dur as launch_dur, + 'MAIN_THREAD_MONITOR_CONTENTION' as reason_id, + AndroidStartupMetric_ThresholdValue( + 'value', 15, + 'unit', 'PERCENTAGE', + 'higher_expected', FALSE) as expected_val, + AndroidStartupMetric_ActualValue( + 'value', android_sum_dur_on_main_thread_for_startup_and_slice( + launch.startup_id, 'Lock contention on a monitor*') * 100 / launch.dur, + 'dur', android_sum_dur_on_main_thread_for_startup_and_slice( + launch.startup_id, 'Lock contention on a monitor*')) as actual_val FROM android_startups launch WHERE launch.startup_id = $startup_id AND android_sum_dur_on_main_thread_for_startup_and_slice( - launch.startup_id, - 'Lock contention on a monitor*' - ) > launch.dur * 0.15 + launch.startup_id, + 'Lock contention on a monitor*' + ) > launch.dur * 0.15 UNION ALL - SELECT 'JIT compiled methods' as slow_cause, NULL as details + SELECT 'JIT compiled methods' as slow_cause, + launch.dur as launch_dur, + 'JIT_COMPILED_METHODS' as reason_id, + AndroidStartupMetric_ThresholdValue( + 'value', 65, + 'unit', 'COUNT', + 'higher_expected', FALSE) as expected_val, + AndroidStartupMetric_ActualValue( + 'value', (SELECT COUNT(1) + FROM ANDROID_SLICES_FOR_STARTUP_AND_SLICE_NAME(launch.startup_id, 'JIT compiling*') + WHERE thread_name = 'Jit thread pool')) as actual_val FROM android_startups launch WHERE launch.startup_id = $startup_id AND ( SELECT COUNT(1) FROM ANDROID_SLICES_FOR_STARTUP_AND_SLICE_NAME(launch.startup_id, 'JIT compiling*') - WHERE thread_name = 'Jit thread pool' - ) > 65 + WHERE thread_name = 'Jit thread pool') > 65 UNION ALL - SELECT 'Broadcast dispatched count' as slow_cause, NULL as details + SELECT 'Broadcast dispatched count' as slow_cause, + launch.dur as launch_dur, + 'BROADCAST_DISPATCHED_COUNT' as reason_id, + AndroidStartupMetric_ThresholdValue( + 'value', 15, + 'unit', 'COUNT', + 'higher_expected', FALSE) as expected_val, + AndroidStartupMetric_ActualValue( + 'value', count_slices_concurrent_to_launch(launch.startup_id, + 'Broadcast dispatched*')) as actual_val FROM android_startups launch WHERE launch.startup_id = $startup_id AND count_slices_concurrent_to_launch( launch.startup_id, - 'Broadcast dispatched*' - ) > 15 + 'Broadcast dispatched*') > 15 UNION ALL - SELECT 'Broadcast received count' as slow_cause, NULL as details + SELECT 'Broadcast received count' as slow_cause, + launch.dur as launch_dur, + 'BROADCAST_RECEIVED_COUNT' as reason_id, + AndroidStartupMetric_ThresholdValue( + 'value', 50, + 'unit', 'COUNT', + 'higher_expected', FALSE) as expected_val, + AndroidStartupMetric_ActualValue( + 'value', count_slices_concurrent_to_launch(launch.startup_id, + 'broadcastReceiveReg*')) as actual_val FROM android_startups launch WHERE launch.startup_id = $startup_id AND count_slices_concurrent_to_launch( - launch.startup_id, - 'broadcastReceiveReg*' - ) > 50 + launch.startup_id, + 'broadcastReceiveReg*') > 50 UNION ALL - SELECT 'Startup running concurrent to launch' as slow_cause, NULL as details + SELECT 'Startup running concurrent to launch' as slow_cause, + launch.dur as launch_dur, + 'STARTUP_RUNNING_CONCURRENT' as reason_id, + AndroidStartupMetric_ThresholdValue( + 'value', FALSE, + 'unit', 'TRUE_OR_FALSE') as expected_val, + AndroidStartupMetric_ActualValue( + 'value', TRUE) as actual_val FROM android_startups launch WHERE launch.startup_id = $startup_id AND EXISTS( SELECT package FROM android_startups l WHERE l.startup_id != launch.startup_id - AND _is_spans_overlapping(l.ts, l.ts_end, launch.ts, launch.ts_end) - ) + AND _is_spans_overlapping(l.ts, l.ts_end, launch.ts, launch.ts_end)) UNION ALL SELECT 'Main Thread - Binder transactions blocked' as slow_cause, - get_main_thread_binder_transactions_blocked(launch.startup_id, 2e7) as details + launch.dur as launch_dur, + 'MAIN_THREAD_BINDER_TRANSCATIONS_BLOCKED' as reason_id, + AndroidStartupMetric_ThresholdValue( + 'value', FALSE, + 'unit', 'TRUE_OR_FALSE') as expected_val, + AndroidStartupMetric_ActualValue( + 'value', TRUE) as actual_val FROM android_startups launch WHERE launch.startup_id = $startup_id AND ( SELECT COUNT(1) - FROM BINDER_TRANSACTION_REPLY_SLICES_FOR_LAUNCH(launch.startup_id, 2e7) - ) > 0 - ); + FROM BINDER_TRANSACTION_REPLY_SLICES_FOR_LAUNCH(launch.startup_id, 2e7)) > 0 + ); diff --git a/src/trace_processor/perfetto_sql/engine/perfetto_sql_engine.cc b/src/trace_processor/perfetto_sql/engine/perfetto_sql_engine.cc index 8f163a006..3048e68ee 100644 --- a/src/trace_processor/perfetto_sql/engine/perfetto_sql_engine.cc +++ b/src/trace_processor/perfetto_sql/engine/perfetto_sql_engine.cc @@ -32,6 +32,7 @@ #include "perfetto/base/logging.h" #include "perfetto/base/status.h" +#include "perfetto/ext/base/flat_hash_map.h" #include "perfetto/ext/base/status_or.h" #include "perfetto/ext/base/string_utils.h" #include "perfetto/ext/base/string_view.h" @@ -848,7 +849,18 @@ base::Status PerfettoSqlEngine::ValidateColumnNames( const std::vector<std::string>& column_names, const std::vector<sql_argument::ArgumentDefinition>& schema, const char* tag) { - // If the user has not provided a schema, we have nothing to validate. + std::vector<std::string> duplicate_columns; + for (auto it = column_names.begin(); it != column_names.end(); ++it) { + if (std::count(it + 1, column_names.end(), *it) > 0) { + duplicate_columns.push_back(*it); + } + } + if (!duplicate_columns.empty()) { + return base::ErrStatus("%s: multiple columns are named: %s", tag, + base::Join(duplicate_columns, ", ").c_str()); + } + + // If the user has not provided a schema, we have nothing further to validate. if (schema.empty()) { return base::OkStatus(); } diff --git a/src/trace_processor/perfetto_sql/intrinsics/functions/clock_functions.h b/src/trace_processor/perfetto_sql/intrinsics/functions/clock_functions.h index a30993d8e..93df66d8e 100644 --- a/src/trace_processor/perfetto_sql/intrinsics/functions/clock_functions.h +++ b/src/trace_processor/perfetto_sql/intrinsics/functions/clock_functions.h @@ -188,7 +188,7 @@ base::Status ToTimecode::Run(void*, int64_t mm = ss / 60; ss = ss % 60; - int64_t hh = mm % 60; + int64_t hh = mm / 60; mm = mm % 60; base::StackString<64> buf("%02" PRId64 ":%02" PRId64 ":%02" PRId64 diff --git a/src/trace_processor/perfetto_sql/intrinsics/functions/to_ftrace.cc b/src/trace_processor/perfetto_sql/intrinsics/functions/to_ftrace.cc index 387664e6a..efa7659d6 100644 --- a/src/trace_processor/perfetto_sql/intrinsics/functions/to_ftrace.cc +++ b/src/trace_processor/perfetto_sql/intrinsics/functions/to_ftrace.cc @@ -630,9 +630,11 @@ SystraceSerializer::ScopedCString SystraceSerializer::SerializeToString( void SystraceSerializer::SerializePrefix(uint32_t raw_row, base::StringWriter* writer) { const auto& raw = storage_->raw_table(); + const auto& cpu_table = storage_->cpu_table(); int64_t ts = raw.ts()[raw_row]; - uint32_t cpu = raw.cpu()[raw_row]; + auto ucpu = raw.ucpu()[raw_row]; + auto cpu = cpu_table.cpu()[ucpu.value]; UniqueTid utid = raw.utid()[raw_row]; uint32_t tid = storage_->thread_table().tid()[utid]; @@ -675,7 +677,7 @@ void SystraceSerializer::SerializePrefix(uint32_t raw_row, writer->AppendPaddedInt<' ', 5>(tgid); } writer->AppendLiteral(") ["); - writer->AppendPaddedInt<'0', 3>(cpu); + writer->AppendPaddedInt<'0', 3>(cpu ? *cpu : 0); writer->AppendLiteral("] .... "); writer->AppendInt(ftrace_time.secs); diff --git a/src/trace_processor/perfetto_sql/intrinsics/table_functions/tables.py b/src/trace_processor/perfetto_sql/intrinsics/table_functions/tables.py index bfc121763..b501e2d78 100644 --- a/src/trace_processor/perfetto_sql/intrinsics/table_functions/tables.py +++ b/src/trace_processor/perfetto_sql/intrinsics/table_functions/tables.py @@ -123,7 +123,7 @@ EXPERIMENTAL_COUNTER_DUR_TABLE = Table( EXPERIMENTAL_SCHED_UPID_TABLE = Table( python_module=__file__, class_name="ExperimentalSchedUpidTable", - sql_name="experimental_sched_upid", + sql_name="__intrinsic_sched_upid", columns=[ C("upid", CppOptional(CppTableId(PROCESS_TABLE))), ], diff --git a/src/trace_processor/perfetto_sql/prelude/views.sql b/src/trace_processor/perfetto_sql/prelude/views.sql index edcefe8fe..6dcbae25c 100644 --- a/src/trace_processor/perfetto_sql/prelude/views.sql +++ b/src/trace_processor/perfetto_sql/prelude/views.sql @@ -53,3 +53,8 @@ SELECT WHEN 'json' THEN string_value ELSE NULL END AS display_value FROM internal_args; + +CREATE VIEW perf_session +AS +SELECT *, id AS perf_session_id +FROM __intrinsic_perf_session;
\ No newline at end of file diff --git a/src/trace_processor/perfetto_sql/stdlib/BUILD.gn b/src/trace_processor/perfetto_sql/stdlib/BUILD.gn index 349e2f695..28d51e9c1 100644 --- a/src/trace_processor/perfetto_sql/stdlib/BUILD.gn +++ b/src/trace_processor/perfetto_sql/stdlib/BUILD.gn @@ -25,6 +25,8 @@ perfetto_amalgamated_sql_header("stdlib") { "counters", "cpu", "deprecated/v42/common", + "export", + "gpu", "graphs", "intervals", "linux", diff --git a/src/trace_processor/perfetto_sql/stdlib/android/auto/BUILD.gn b/src/trace_processor/perfetto_sql/stdlib/android/auto/BUILD.gn index bf52d69d5..7c89f2575 100644 --- a/src/trace_processor/perfetto_sql/stdlib/android/auto/BUILD.gn +++ b/src/trace_processor/perfetto_sql/stdlib/android/auto/BUILD.gn @@ -15,7 +15,5 @@ import("../../../../../../gn/perfetto_sql.gni") perfetto_sql_source_set("auto") { - sources = [ - "multiuser.sql", - ] + sources = [ "multiuser.sql" ] } diff --git a/src/trace_processor/perfetto_sql/stdlib/android/auto/multiuser.sql b/src/trace_processor/perfetto_sql/stdlib/android/auto/multiuser.sql index ea95dcc4e..2b408ef9c 100644 --- a/src/trace_processor/perfetto_sql/stdlib/android/auto/multiuser.sql +++ b/src/trace_processor/perfetto_sql/stdlib/android/auto/multiuser.sql @@ -64,7 +64,7 @@ FROM ( FROM android_startups UNION SELECT - slice.ts as event_end_time, + slice.ts + slice.dur as event_end_time, slice.name as event_end_name FROM slice WHERE slice.name GLOB "finishUserStopped-10*" diff --git a/src/trace_processor/perfetto_sql/stdlib/android/binder.sql b/src/trace_processor/perfetto_sql/stdlib/android/binder.sql index e16897087..5ea176b39 100644 --- a/src/trace_processor/perfetto_sql/stdlib/android/binder.sql +++ b/src/trace_processor/perfetto_sql/stdlib/android/binder.sql @@ -477,7 +477,7 @@ FROM all_binder LEFT JOIN android_process_metadata client_process_metadata ON all_binder.client_upid = client_process_metadata.upid LEFT JOIN android_process_metadata server_process_metadata - ON all_binder.server_upid = client_process_metadata.upid; + ON all_binder.server_upid = server_process_metadata.upid; -- Returns a DAG of all outgoing binder txns from a process. -- The roots of the graph are the threads making the txns and the graph flows from: diff --git a/src/trace_processor/perfetto_sql/stdlib/android/broadcasts.sql b/src/trace_processor/perfetto_sql/stdlib/android/broadcasts.sql index 6467ea7ae..7779ec93f 100644 --- a/src/trace_processor/perfetto_sql/stdlib/android/broadcasts.sql +++ b/src/trace_processor/perfetto_sql/stdlib/android/broadcasts.sql @@ -64,8 +64,10 @@ WITH ), broadcast_process_running AS ( SELECT + slice.id AS id, slice.ts, slice.dur, + broadcast_queues.queue_id, _extract_broadcast_process_name(slice.name) AS process_name, CAST(str_split(str_split(str_split(slice.name, '/', 0), ' ', 1), ':', 0) AS INT) AS pid, queue_id @@ -73,23 +75,28 @@ WITH JOIN broadcast_queues ON broadcast_queues.id = slice.track_id WHERE slice.name GLOB '* running' + ), + broadcast_intent_action AS ( + SELECT + str_split(str_split(slice.name, '/', 0), ' ', 1) AS intent_action, + slice.parent_id, + slice.id AS intent_id, + slice.ts AS intent_ts, + slice.track_id AS track_id, + slice.dur AS intent_dur + FROM slice + WHERE slice.name GLOB '* scheduled' ) -SELECT - str_split(str_split(slice.name, '/', 0), ' ', 1) AS intent_action, - process_name, - pid, - _pid_to_upid(pid, slice.ts) AS upid, - queue_id, - slice.id, - slice.ts, - slice.dur, - slice.track_id -FROM broadcast_process_running -JOIN broadcast_queues - USING (queue_id) -JOIN slice - ON ( - broadcast_process_running.ts < slice.ts - AND slice.ts < broadcast_process_running.ts + broadcast_process_running.dur - AND slice.track_id = broadcast_queues.id) -WHERE slice.name GLOB '* scheduled'; + SELECT + broadcast_intent_action.intent_action, + broadcast_process_running.process_name, + broadcast_process_running.pid, + _pid_to_upid(broadcast_process_running.pid, broadcast_intent_action.intent_ts) AS upid, + broadcast_process_running.queue_id, + broadcast_intent_action.intent_id AS id, + broadcast_intent_action.intent_ts AS ts, + broadcast_intent_action.intent_dur AS dur, + broadcast_intent_action.track_id + FROM broadcast_intent_action + JOIN broadcast_process_running + ON parent_id = id; diff --git a/src/trace_processor/perfetto_sql/stdlib/android/input.sql b/src/trace_processor/perfetto_sql/stdlib/android/input.sql index a1d7b114d..5ac6c7484 100644 --- a/src/trace_processor/perfetto_sql/stdlib/android/input.sql +++ b/src/trace_processor/perfetto_sql/stdlib/android/input.sql @@ -130,3 +130,62 @@ JOIN (SELECT * FROM _input_message_sent WHERE thread_name != 'InputDispatcher') AND dispatch.event_seq = finish.event_seq JOIN (SELECT * FROM _input_message_received WHERE event_type = '0x2') finish_ack ON finish_ack.event_channel = dispatch.event_channel AND dispatch.event_seq = finish_ack.event_seq; + +-- Key events processed by the Android framework (from android.input.inputevent data source). +CREATE PERFETTO VIEW android_key_events( + -- ID of the trace entry + id INT, + -- The randomly-generated ID associated with each input event processed + -- by Android Framework, used to track the event through the input pipeline + event_id INT, + -- The timestamp associated with the input event + ts INT, + -- Details of the input event parsed from the proto message + arg_set_id INT +) AS +SELECT + id, + event_id, + ts, + arg_set_id +FROM __intrinsic_android_key_events; + +-- Motion events processed by the Android framework (from android.input.inputevent data source). +CREATE PERFETTO VIEW android_motion_events( + -- ID of the trace entry + id INT, + -- The randomly-generated ID associated with each input event processed + -- by Android Framework, used to track the event through the input pipeline + event_id INT, + -- The timestamp associated with the input event + ts INT, + -- Details of the input event parsed from the proto message + arg_set_id INT +) AS +SELECT + id, + event_id, + ts, + arg_set_id +FROM __intrinsic_android_motion_events; + +-- Input event dispatching information in Android (from android.input.inputevent data source). +CREATE PERFETTO VIEW android_input_event_dispatch( + -- ID of the trace entry + id INT, + -- Event ID of the input event that was dispatched + event_id INT, + -- Extra args parsed from the proto message + arg_set_id INT, + -- Vsync ID that identifies the state of the windows during which the dispatch decision was made + vsync_id INT, + -- Window ID of the window receiving the event + window_id INT +) AS +SELECT + id, + event_id, + arg_set_id, + vsync_id, + window_id +FROM __intrinsic_android_input_event_dispatch; diff --git a/src/trace_processor/perfetto_sql/stdlib/android/oom_adjuster.sql b/src/trace_processor/perfetto_sql/stdlib/android/oom_adjuster.sql index 3ce017e42..c09414e23 100644 --- a/src/trace_processor/perfetto_sql/stdlib/android/oom_adjuster.sql +++ b/src/trace_processor/perfetto_sql/stdlib/android/oom_adjuster.sql @@ -17,8 +17,30 @@ INCLUDE PERFETTO MODULE slices.with_context; INCLUDE PERFETTO MODULE counters.intervals; --- Converts an oom_adj score Integer to String bucket name. +-- Converts an oom_adj score Integer to String sample name. +-- One of: cached, background, job, foreground_service, bfgs, foreground and +-- system. CREATE PERFETTO FUNCTION android_oom_adj_score_to_bucket_name( + -- `oom_score` value + oom_score INT +) +-- Returns the sample bucket based on the oom score. +RETURNS STRING +AS +SELECT + CASE + WHEN $oom_score >= 900 THEN 'cached' + WHEN $oom_score BETWEEN 250 AND 900 THEN 'background' + WHEN $oom_score BETWEEN 201 AND 250 THEN 'job' + WHEN $oom_score = 200 THEN 'foreground_service' + WHEN $oom_score BETWEEN 100 AND 200 THEN 'bfgs' + WHEN $oom_score BETWEEN 0 AND 100 THEN 'foreground' + WHEN $oom_score < 0 THEN 'system' +END; + +-- Converts an oom_adj score Integer to String bucket name. +-- Deprecated: use `android_oom_adj_score_to_bucket_name` instead. +CREATE PERFETTO FUNCTION android_oom_adj_score_to_detailed_bucket_name( -- oom_adj score. value INT, -- android_app id of the process. @@ -52,8 +74,58 @@ SELECT ELSE 'unknown_app' END; +CREATE PERFETTO TABLE _oom_adjuster_intervals AS +WITH reason AS ( + SELECT + thread_slice.id AS oom_adj_id, + thread_slice.ts AS oom_adj_ts, + thread_slice.dur AS oom_adj_dur, + thread_slice.track_id AS oom_adj_track_id, + utid AS oom_adj_utid, + thread_name AS oom_adj_thread_name, + str_split(thread_slice.name, '_', 1) AS oom_adj_reason, + slice.name AS oom_adj_trigger, + LEAD(thread_slice.ts) OVER (ORDER BY thread_slice.ts) AS oom_adj_next_ts + FROM thread_slice + LEFT JOIN slice ON slice.id = thread_slice.parent_id AND slice.dur != -1 + WHERE thread_slice.name GLOB 'updateOomAdj_*' AND process_name = 'system_server' +) +SELECT + ts, + dur, + cast_int!(value) AS score, + process.upid, + process.name AS process_name, + reason.oom_adj_id, + reason.oom_adj_ts, + reason.oom_adj_dur, + reason.oom_adj_track_id, + reason.oom_adj_thread_name, + reason.oom_adj_utid, + reason.oom_adj_reason, + reason.oom_adj_trigger, + android_appid +FROM + counter_leading_intervals + !( + ( + SELECT counter.* + FROM counter + JOIN counter_track track + ON track.id = counter.track_id AND track.name = 'oom_score_adj' + )) + counter +JOIN process_counter_track track + ON counter.track_id = track.id +JOIN process + USING (upid) +LEFT JOIN reason + ON counter.ts BETWEEN oom_adj_ts AND COALESCE(oom_adj_next_ts, trace_end()) +WHERE track.name = 'oom_score_adj'; + + -- All oom adj state intervals across all processes along with the reason for the state update. -CREATE PERFETTO TABLE android_oom_adj_intervals ( +CREATE PERFETTO VIEW android_oom_adj_intervals ( -- Timestamp the oom_adj score of the process changed ts INT, -- Duration until the next oom_adj score change of the process. @@ -81,49 +153,18 @@ CREATE PERFETTO TABLE android_oom_adj_intervals ( -- Trigger for the latest oom_adj update in the system_server. oom_adj_trigger STRING ) AS -WITH - reason AS ( - SELECT - thread_slice.id AS oom_adj_id, - thread_slice.ts AS oom_adj_ts, - thread_slice.dur AS oom_adj_dur, - thread_slice.track_id AS oom_adj_track_id, - thread_name AS oom_adj_thread_name, - str_split(thread_slice.name, '_', 1) AS oom_adj_reason, - slice.name AS oom_adj_trigger, - LEAD(thread_slice.ts) OVER (ORDER BY thread_slice.ts) AS oom_adj_next_ts - FROM thread_slice - LEFT JOIN slice ON slice.id = thread_slice.parent_id AND slice.dur != -1 - WHERE thread_slice.name GLOB 'updateOomAdj_*' AND process_name = 'system_server' - ) SELECT ts, dur, - value AS score, - android_oom_adj_score_to_bucket_name(value, android_appid) AS bucket, - process.upid, - process.name AS process_name, - reason.oom_adj_id, - reason.oom_adj_ts, - reason.oom_adj_dur, - reason.oom_adj_track_id, - reason.oom_adj_thread_name, - reason.oom_adj_reason, - reason.oom_adj_trigger -FROM - counter_leading_intervals - !( - ( - SELECT counter.* - FROM counter - JOIN counter_track track - ON track.id = counter.track_id AND track.name = 'oom_score_adj' - )) - counter -JOIN process_counter_track track - ON counter.track_id = track.id -JOIN process - USING (upid) -LEFT JOIN reason - ON counter.ts BETWEEN oom_adj_ts AND COALESCE(oom_adj_next_ts, trace_end()) -WHERE track.name = 'oom_score_adj'; + score, + android_oom_adj_score_to_bucket_name(score) AS bucket, + upid, + process_name, + oom_adj_id, + oom_adj_ts, + oom_adj_dur, + oom_adj_track_id, + oom_adj_thread_name, + oom_adj_reason, + oom_adj_trigger +FROM _oom_adjuster_intervals; diff --git a/src/trace_processor/perfetto_sql/stdlib/android/process_metadata.sql b/src/trace_processor/perfetto_sql/stdlib/android/process_metadata.sql index a50b03306..a3d47f276 100644 --- a/src/trace_processor/perfetto_sql/stdlib/android/process_metadata.sql +++ b/src/trace_processor/perfetto_sql/stdlib/android/process_metadata.sql @@ -20,6 +20,47 @@ SELECT uid, COUNT(1) AS cnt FROM package_list GROUP BY 1; +CREATE PERFETTO FUNCTION _android_package_for_process( + uid INT, + uid_count INT, + process_name STRING +) +RETURNS TABLE( + package_name STRING, + version_code INT, + debuggable BOOL +) +AS +WITH min_distance AS ( + SELECT + -- SQLite allows omitting the group-by for the MIN: the other columns + -- will match the row with the minimum value. + MIN(LENGTH($process_name) - LENGTH(package_name)), + package_name, + version_code, + debuggable + FROM package_list + WHERE ( + ( + $uid = uid + AND ( + -- unique match + $uid_count = 1 + -- or process name is a prefix the package name + OR $process_name GLOB package_name || '*' + ) + ) + OR + ( + -- isolated processes can only be matched based on the name + $uid >= 90000 AND $uid < 100000 + AND STR_SPLIT($process_name, ':', 0) GLOB package_name || '*' + ) + ) +) +SELECT package_name, version_code, debuggable +FROM min_distance; + -- Data about packages running on the process. CREATE PERFETTO TABLE android_process_metadata( -- Process upid. @@ -57,21 +98,6 @@ SELECT plist.debuggable FROM process LEFT JOIN _uid_package_count ON process.android_appid = _uid_package_count.uid -LEFT JOIN package_list plist - ON ( - ( - process.android_appid = plist.uid - AND _uid_package_count.uid = plist.uid - AND ( - -- unique match - _uid_package_count.cnt = 1 - -- or process name starts with the package name - OR process.name GLOB plist.package_name || '*') - ) - OR - ( - -- isolated processes can only be matched based on the name - process.android_appid >= 90000 AND process.android_appid < 100000 - AND STR_SPLIT(process.name, ':', 0) = plist.package_name - ) - ); +LEFT JOIN _android_package_for_process( + process.android_appid, _uid_package_count.cnt, process.name +) AS plist; diff --git a/src/trace_processor/perfetto_sql/stdlib/android/suspend.sql b/src/trace_processor/perfetto_sql/stdlib/android/suspend.sql index 0d31b0dc7..1551ea679 100644 --- a/src/trace_processor/perfetto_sql/stdlib/android/suspend.sql +++ b/src/trace_processor/perfetto_sql/stdlib/android/suspend.sql @@ -73,7 +73,9 @@ SELECT ts, dur, 'awake' AS power_state FROM awake_slice UNION ALL SELECT ts, dur, 'suspended' AS power_state -FROM suspend_slice; +FROM suspend_slice +ORDER BY ts; -- Order by will cause Perfetto table to index by ts. + -- Extracts the duration without counting CPU suspended time from an event. -- This is the same as converting an event duration from wall clock to monotonic clock. diff --git a/src/trace_processor/perfetto_sql/stdlib/android/winscope/BUILD.gn b/src/trace_processor/perfetto_sql/stdlib/android/winscope/BUILD.gn index 93abe356e..ff7e3efe9 100644 --- a/src/trace_processor/perfetto_sql/stdlib/android/winscope/BUILD.gn +++ b/src/trace_processor/perfetto_sql/stdlib/android/winscope/BUILD.gn @@ -17,5 +17,6 @@ import("../../../../../../gn/perfetto_sql.gni") perfetto_sql_source_set("winscope") { sources = [ "inputmethod.sql", + "viewcapture.sql", ] } diff --git a/src/trace_processor/perfetto_sql/stdlib/android/winscope/viewcapture.sql b/src/trace_processor/perfetto_sql/stdlib/android/winscope/viewcapture.sql new file mode 100644 index 000000000..77e68b5f1 --- /dev/null +++ b/src/trace_processor/perfetto_sql/stdlib/android/winscope/viewcapture.sql @@ -0,0 +1,29 @@ +-- +-- Copyright 2024 The Android Open Source Project +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- https://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +-- Android viewcapture (from android.viewcapture data source). +CREATE PERFETTO VIEW android_viewcapture( + -- Snapshot id + id INT, + -- Timestamp when the snapshot was triggered + ts INT, + -- Extra args parsed from the proto message + arg_set_id INT +) AS +SELECT + id, + ts, + arg_set_id +FROM __intrinsic_viewcapture; diff --git a/src/trace_processor/perfetto_sql/stdlib/counters/intervals.sql b/src/trace_processor/perfetto_sql/stdlib/counters/intervals.sql index 4b11c7c24..7bb0c96db 100644 --- a/src/trace_processor/perfetto_sql/stdlib/counters/intervals.sql +++ b/src/trace_processor/perfetto_sql/stdlib/counters/intervals.sql @@ -49,7 +49,12 @@ CREATE PERFETTO MACRO counter_leading_intervals( RETURNS TableOrSubquery AS ( WITH base AS ( - SELECT id, ts, track_id, value, LAG(value) OVER (PARTITION BY track_id ORDER BY ts) AS lag_value + SELECT + id, + ts, + track_id, + value, + LAG(value) OVER (PARTITION BY track_id ORDER BY ts) AS lag_value FROM $counter_table ) SELECT @@ -57,7 +62,7 @@ RETURNS TableOrSubquery AS ts, track_id, LEAD(ts, 1, trace_end()) OVER(PARTITION BY track_id ORDER BY ts) - ts AS dur, - CAST(value AS INT) AS value + value FROM base WHERE value != lag_value OR lag_value IS NULL ); diff --git a/src/trace_processor/perfetto_sql/stdlib/cpu/BUILD.gn b/src/trace_processor/perfetto_sql/stdlib/cpu/BUILD.gn index 28fc5a00d..13b108e40 100644 --- a/src/trace_processor/perfetto_sql/stdlib/cpu/BUILD.gn +++ b/src/trace_processor/perfetto_sql/stdlib/cpu/BUILD.gn @@ -15,6 +15,7 @@ import("../../../../../gn/perfetto_sql.gni") perfetto_sql_source_set("cpu") { + deps = [ "utilization" ] sources = [ "cpus.sql", "freq.sql", diff --git a/src/trace_processor/perfetto_sql/stdlib/cpu/freq.sql b/src/trace_processor/perfetto_sql/stdlib/cpu/freq.sql index c0226bfd5..5dd2a483d 100644 --- a/src/trace_processor/perfetto_sql/stdlib/cpu/freq.sql +++ b/src/trace_processor/perfetto_sql/stdlib/cpu/freq.sql @@ -37,7 +37,7 @@ SELECT count_w_dur.track_id, count_w_dur.ts, count_w_dur.dur, - count_w_dur.value as freq, + cast_int!(count_w_dur.value) as freq, cct.cpu FROM counter_leading_intervals!(( diff --git a/src/trace_processor/perfetto_sql/stdlib/cpu/idle.sql b/src/trace_processor/perfetto_sql/stdlib/cpu/idle.sql index 7f837a2ac..703a913c3 100644 --- a/src/trace_processor/perfetto_sql/stdlib/cpu/idle.sql +++ b/src/trace_processor/perfetto_sql/stdlib/cpu/idle.sql @@ -39,7 +39,7 @@ SELECT count_w_dur.track_id, count_w_dur.ts, count_w_dur.dur, - IIF(count_w_dur.value = 4294967295, -1, count_w_dur.value) AS idle, + cast_int!(IIF(count_w_dur.value = 4294967295, -1, count_w_dur.value)) AS idle, cct.cpu FROM counter_leading_intervals!(( diff --git a/src/trace_processor/perfetto_sql/stdlib/sched/utilization/BUILD.gn b/src/trace_processor/perfetto_sql/stdlib/cpu/utilization/BUILD.gn index d4be2cc87..d4be2cc87 100644 --- a/src/trace_processor/perfetto_sql/stdlib/sched/utilization/BUILD.gn +++ b/src/trace_processor/perfetto_sql/stdlib/cpu/utilization/BUILD.gn diff --git a/src/trace_processor/perfetto_sql/stdlib/sched/utilization/general.sql b/src/trace_processor/perfetto_sql/stdlib/cpu/utilization/general.sql index 629e5a306..de0e45bb4 100644 --- a/src/trace_processor/perfetto_sql/stdlib/sched/utilization/general.sql +++ b/src/trace_processor/perfetto_sql/stdlib/cpu/utilization/general.sql @@ -13,6 +13,8 @@ -- See the License for the specific language governing permissions and -- limitations under the License. +INCLUDE PERFETTO MODULE cpu.freq; + -- Returns the timestamp of the start of the partition that contains the |ts|. CREATE PERFETTO FUNCTION _partition_start(ts INT, size INT) RETURNS INT AS -- Division of two ints would result in floor(ts/size). @@ -58,7 +60,7 @@ ON (p.ts <= i.ts AND i.ts < p.ts_end)); -- Utilization is calculated as sum of average utilization of each CPU in each -- period, which is defined as a multiply of |interval|. For this reason -- first and last period might have lower then real utilization. -CREATE PERFETTO MACRO _sched_avg_utilization_per_period( +CREATE PERFETTO MACRO _cpu_avg_utilization_per_period( -- Length of the period on which utilization should be averaged. interval Expr, -- Either sched table or its filtered down version. @@ -73,3 +75,26 @@ SELECT SUM(ts_end - ts)/cast_double!($interval) AS unnormalized_utilization FROM _interval_partitions!(_partitions($interval), $sched_table) GROUP BY 1); + +CREATE PERFETTO VIEW _cpu_freq_for_metrics AS +SELECT + id, + ts, + dur, + cpu, + freq +FROM cpu_freq_counters; + +CREATE PERFETTO VIEW _sched_without_id AS +SELECT ts, dur, utid, cpu +FROM sched +WHERE utid != 0 AND dur != -1; + +CREATE VIRTUAL TABLE _cpu_freq_per_thread_span_join +USING SPAN_LEFT_JOIN( + _sched_without_id PARTITIONED cpu, + _cpu_freq_for_metrics PARTITIONED cpu); + +CREATE PERFETTO TABLE _cpu_freq_per_thread +AS SELECT * FROM _cpu_freq_per_thread_span_join; + diff --git a/src/trace_processor/perfetto_sql/stdlib/sched/utilization/process.sql b/src/trace_processor/perfetto_sql/stdlib/cpu/utilization/process.sql index 8437cb49c..e40b62484 100644 --- a/src/trace_processor/perfetto_sql/stdlib/sched/utilization/process.sql +++ b/src/trace_processor/perfetto_sql/stdlib/cpu/utilization/process.sql @@ -13,14 +13,14 @@ -- See the License for the specific language governing permissions and -- limitations under the License. -INCLUDE PERFETTO MODULE sched.utilization.general; +INCLUDE PERFETTO MODULE cpu.utilization.general; INCLUDE PERFETTO MODULE time.conversion; -- Returns a table of process utilization per given period. -- Utilization is calculated as sum of average utilization of each CPU in each -- period, which is defined as a multiply of |interval|. For this reason -- first and last period might have lower then real utilization. -CREATE PERFETTO FUNCTION sched_process_utilization_per_period( +CREATE PERFETTO FUNCTION cpu_process_utilization_per_period( -- Length of the period on which utilization should be averaged. interval INT, -- Upid of the process. @@ -47,13 +47,13 @@ WITH sched_for_upid AS ( JOIN thread USING (utid) JOIN process USING (upid) WHERE upid = $upid AND utid != 0) -SELECT * FROM _sched_avg_utilization_per_period!($interval, sched_for_upid); +SELECT * FROM _cpu_avg_utilization_per_period!($interval, sched_for_upid); -- Returns a table of process utilization per second. -- Utilization is calculated as sum of average utilization of each CPU in each -- period, which is defined as a multiply of |interval|. For this reason -- first and last period might have lower then real utilization. -CREATE PERFETTO FUNCTION sched_process_utilization_per_second( +CREATE PERFETTO FUNCTION cpu_process_utilization_per_second( -- Upid of the process. upid INT ) @@ -69,4 +69,36 @@ RETURNS TABLE ( -- [0, cpu_count] range. unnormalized_utilization DOUBLE ) AS -SELECT * FROM sched_process_utilization_per_period(time_from_s(1), $upid);
\ No newline at end of file +SELECT * FROM cpu_process_utilization_per_period(time_from_s(1), $upid); + +-- Aggregated CPU statistics for each process. +CREATE PERFETTO TABLE cpu_cycles_per_process( + -- Unique process id + upid INT, + -- Sum of CPU millicycles + millicycles INT, + -- Sum of CPU megacycles + megacycles INT, + -- Total runtime duration + runtime INT, + -- Minimum CPU frequency in kHz + min_freq INT, + -- Maximum CPU frequency in kHz + max_freq INT, + -- Average CPU frequency in kHz + avg_freq INT +) AS +WITH threads AS ( + SELECT upid, utid FROM thread +) +SELECT + upid, + cast_int!(SUM(dur * freq) / 1000) AS millicycles, + cast_int!(SUM(dur * freq) / 1000 / 1e9) AS megacycles, + SUM(dur) AS runtime, + MIN(freq) AS min_freq, + MAX(freq) AS max_freq, + cast_int!(SUM((dur * freq) / 1000) / SUM(dur / 1000)) AS avg_freq +FROM _cpu_freq_per_thread +JOIN threads USING (utid) +GROUP BY upid;
\ No newline at end of file diff --git a/src/trace_processor/perfetto_sql/stdlib/cpu/utilization/system.sql b/src/trace_processor/perfetto_sql/stdlib/cpu/utilization/system.sql new file mode 100644 index 000000000..2846f609c --- /dev/null +++ b/src/trace_processor/perfetto_sql/stdlib/cpu/utilization/system.sql @@ -0,0 +1,140 @@ +-- +-- Copyright 2024 The Android Open Source Project +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- https://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +INCLUDE PERFETTO MODULE cpu.utilization.general; +INCLUDE PERFETTO MODULE cpu.size; + +INCLUDE PERFETTO MODULE time.conversion; + +-- The purpose of this module is to provide high level aggregates of system +-- utilization, akin to /proc/stat results. + +-- Returns a table of system utilization per given period. +-- Utilization is calculated as sum of average utilization of each CPU in each +-- period, which is defined as a multiply of |interval|. For this reason +-- first and last period might have lower then real utilization. +CREATE PERFETTO FUNCTION cpu_utilization_per_period( + -- Length of the period on which utilization should be averaged. + interval INT) +RETURNS TABLE ( + -- Timestamp of start of a second. + ts INT, + -- Sum of average utilization over period. + -- Note: as the data is normalized, the values will be in the + -- [0, 1] range. + utilization DOUBLE, + -- Sum of average utilization over all CPUs over period. + -- Note: as the data is unnormalized, the values will be in the + -- [0, cpu_count] range. + unnormalized_utilization DOUBLE +) AS +SELECT * +FROM _cpu_avg_utilization_per_period!( + $interval, + (SELECT * FROM sched WHERE utid != 0) +); + +-- Table with system utilization per second. +-- Utilization is calculated by sum of average utilization of each CPU every +-- second. For this reason first and last second might have lower then real +-- utilization. +CREATE PERFETTO TABLE cpu_utilization_per_second( + -- Timestamp of start of a second. + ts INT, + -- Sum of average utilization over period. + -- Note: as the data is normalized, the values will be in the + -- [0, 1] range. + utilization DOUBLE, + -- Sum of average utilization over all CPUs over period. + -- Note: as the data is unnormalized, the values will be in the + -- [0, cpu_count] range. + unnormalized_utilization DOUBLE +) AS +SELECT + ts, + utilization, + unnormalized_utilization +FROM cpu_utilization_per_period(time_from_s(1)); + +-- Aggregated CPU statistics for runtime of each thread on a CPU. +CREATE PERFETTO TABLE _cpu_cycles_raw( + -- The id of CPU + cpu INT, + -- Unique thread id + utid INT, + -- Sum of CPU millicycles + millicycles INT, + -- Sum of CPU megacycles + megacycles INT, + -- Total runtime duration + runtime INT, + -- Minimum CPU frequency in kHz + min_freq INT, + -- Maximum CPU frequency in kHz + max_freq INT, + -- Average CPU frequency in kHz + avg_freq INT +) AS +SELECT + cpu, + utid, + -- We divide by 1e3 here as dur is in ns and freq in khz. In total + -- this means we need to divide the duration by 1e9 and multiply the + -- frequency by 1e3 then multiply again by 1e3 to get millicycles + -- i.e. divide by 1e3 in total. + -- We use millicycles as we want to preserve this level of precision + -- for future calculations. + cast_int!(SUM(dur * freq) / 1000) AS millicycles, + cast_int!(SUM(dur * freq) / 1000 / 1e9) AS megacycles, + SUM(dur) AS runtime, + MIN(freq) AS min_freq, + MAX(freq) AS max_freq, + -- We choose to work in micros space in both the numerator and + -- denominator as this gives us good enough precision without risking + -- overflows. + cast_int!(SUM((dur * freq) / 1000) / SUM(dur / 1000)) AS avg_freq +FROM _cpu_freq_per_thread +GROUP BY utid, cpu; + +-- Aggregated CPU statistics for each CPU. +CREATE PERFETTO TABLE cpu_cycles_per_cpu( + -- The id of CPU + cpu INT, + -- CPU type + cpu_type STRING, + -- Sum of CPU millicycles + millicycles INT, + -- Sum of CPU megacycles + megacycles INT, + -- Total runtime of all threads running on CPU + runtime INT, + -- Minimum CPU frequency in kHz + min_freq INT, + -- Maximum CPU frequency in kHz + max_freq INT, + -- Average CPU frequency in kHz + avg_freq INT +) AS +SELECT + cpu, + cpu_guess_core_type(cpu) AS cpu_type, + cast_int!(SUM(dur * freq) / 1000) AS millicycles, + cast_int!(SUM(dur * freq) / 1000 / 1e9) AS megacycles, + SUM(dur) AS runtime, + MIN(freq) AS min_freq, + MAX(freq) AS max_freq, + cast_int!(SUM((dur * freq) / 1000) / SUM(dur / 1000)) AS avg_freq +FROM _cpu_freq_per_thread +GROUP BY cpu;
\ No newline at end of file diff --git a/src/trace_processor/perfetto_sql/stdlib/sched/utilization/thread.sql b/src/trace_processor/perfetto_sql/stdlib/cpu/utilization/thread.sql index ae1d3f673..1f4b50eab 100644 --- a/src/trace_processor/perfetto_sql/stdlib/sched/utilization/thread.sql +++ b/src/trace_processor/perfetto_sql/stdlib/cpu/utilization/thread.sql @@ -13,14 +13,14 @@ -- See the License for the specific language governing permissions and -- limitations under the License. -INCLUDE PERFETTO MODULE sched.utilization.general; +INCLUDE PERFETTO MODULE cpu.utilization.general; INCLUDE PERFETTO MODULE time.conversion; -- Returns a table of thread utilization per given period. -- Utilization is calculated as sum of average utilization of each CPU in each -- period, which is defined as a multiply of |interval|. For this reason -- first and last period might have lower then real utilization. -CREATE PERFETTO FUNCTION sched_thread_utilization_per_period( +CREATE PERFETTO FUNCTION cpu_thread_utilization_per_period( -- Length of the period on which utilization should be averaged. interval INT, -- Utid of the thread. @@ -45,13 +45,13 @@ WITH sched_for_utid AS ( utid FROM sched WHERE utid = $utid -) SELECT * FROM _sched_avg_utilization_per_period!($interval, sched_for_utid); +) SELECT * FROM _cpu_avg_utilization_per_period!($interval, sched_for_utid); -- Returns a table of thread utilization per second. -- Utilization is calculated as sum of average utilization of each CPU in each -- period, which is defined as a multiply of |interval|. For this reason -- first and last period might have lower then real utilization. -CREATE PERFETTO FUNCTION sched_thread_utilization_per_second( +CREATE PERFETTO FUNCTION cpu_thread_utilization_per_second( -- Utid of the thread. utid INT ) @@ -67,4 +67,32 @@ RETURNS TABLE ( -- [0, cpu_count] range. unnormalized_utilization DOUBLE ) AS -SELECT * FROM sched_thread_utilization_per_period(time_from_s(1), $utid);
\ No newline at end of file +SELECT * FROM cpu_thread_utilization_per_period(time_from_s(1), $utid); + +-- Aggregated CPU statistics for each thread. +CREATE PERFETTO TABLE cpu_cycles_per_thread( + -- Unique thread id + utid INT, + -- Sum of CPU millicycles + millicycles INT, + -- Sum of CPU megacycles + megacycles INT, + -- Total runtime duration + runtime INT, + -- Minimum CPU frequency in kHz + min_freq INT, + -- Maximum CPU frequency in kHz + max_freq INT, + -- Average CPU frequency in kHz + avg_freq INT +) AS +SELECT + utid, + cast_int!(SUM(dur * freq) / 1000) AS millicycles, + cast_int!(SUM(dur * freq) / 1000 / 1e9) AS megacycles, + SUM(dur) AS runtime, + MIN(freq) AS min_freq, + MAX(freq) AS max_freq, + cast_int!(SUM((dur * freq) / 1000) / SUM(dur / 1000)) AS avg_freq +FROM _cpu_freq_per_thread +GROUP BY utid;
\ No newline at end of file diff --git a/src/trace_processor/perfetto_sql/stdlib/export/BUILD.gn b/src/trace_processor/perfetto_sql/stdlib/export/BUILD.gn new file mode 100644 index 000000000..7cfcfc76a --- /dev/null +++ b/src/trace_processor/perfetto_sql/stdlib/export/BUILD.gn @@ -0,0 +1,24 @@ +# Copyright (C) 2024 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import("../../../../../gn/perfetto_sql.gni") + +perfetto_sql_source_set("export") { + sources = [] + + # Chrmouim builds set DSQLITE_OMIT_JSON so SQL json functions will not exist. + if (!build_with_chromium) { + sources += [ "to_firefox_profile.sql" ] + } +} diff --git a/src/trace_processor/perfetto_sql/stdlib/export/to_firefox_profile.sql b/src/trace_processor/perfetto_sql/stdlib/export/to_firefox_profile.sql new file mode 100644 index 000000000..b84208789 --- /dev/null +++ b/src/trace_processor/perfetto_sql/stdlib/export/to_firefox_profile.sql @@ -0,0 +1,434 @@ +-- +-- Copyright 2024 The Android Open Source Project +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- https://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + + +-- Returns an instance of `RawMarkerTable` as defined in +-- https://github.com/firefox-devtools/profiler/blob/main/src/types/profile.js +CREATE PERFETTO FUNCTION _export_firefox_thread_markers() +RETURNS STRING +AS +SELECT json_object( + 'data', json_array(), + 'name', json_array(), + 'startTime', json_array(), + 'endTime', json_array(), + 'phase', json_array(), + 'category', json_array(), + -- threadId?: Tid[] + 'length', 0); + +-- Returns an instance of `SamplesTable` as defined in +-- https://github.com/firefox-devtools/profiler/blob/main/src/types/profile.js +CREATE PERFETTO MACRO _export_firefox_samples_table(samples_table TableOrSubquery) +RETURNS Expr +AS ( + SELECT + json_object( + -- responsiveness?: Array<?Milliseconds> + -- eventDelay?: Array<?Milliseconds> + 'stack', json_group_array(stack ORDER BY idx), + 'time', json_group_array(time ORDER BY idx), + 'weight', NULL, + 'weightType', 'samples', + -- threadCPUDelta?: Array<number | null> + -- threadId?: Tid[] + 'length', COUNT(*)) + FROM samples_table +); + +-- Returns an instance of `StackTable` as defined in +-- https://github.com/firefox-devtools/profiler/blob/main/src/types/profile.js +CREATE PERFETTO MACRO _export_firefox_stack_table(stack_table TableOrSubquery) +RETURNS Expr +AS ( + SELECT + json_object( + 'frame', json_group_array(frame ORDER BY idx), + 'category', json_group_array(category ORDER BY idx), + 'subcategory', json_group_array(subcategory ORDER BY idx), + 'prefix', json_group_array(prefix ORDER BY idx), + 'length', COUNT(*)) + FROM stack_table +); + +-- Returns an instance of `FrameTable` as defined in +-- https://github.com/firefox-devtools/profiler/blob/main/src/types/profile.js +CREATE PERFETTO MACRO _export_firefox_frame_table(frame_table TableOrSubquery) +RETURNS Expr +AS ( + SELECT + json_object( + 'address', json_group_array(address ORDER BY idx), + 'inlineDepth', json_group_array(inline_depth ORDER BY idx), + 'category', json_group_array(category ORDER BY idx), + 'subcategory', json_group_array(subcategory ORDER BY idx), + 'func', json_group_array(func ORDER BY idx), + 'nativeSymbol', json_group_array(native_symbol ORDER BY idx), + 'innerWindowID', json_group_array(inner_window_id ORDER BY idx), + 'implementation', json_group_array(implementation ORDER BY idx), + 'line', json_group_array(line ORDER BY idx), + 'column', json_group_array(column ORDER BY idx), + 'length', COUNT(*)) + FROM frame_table +); + +-- Returns an array of strings. +CREATE PERFETTO MACRO _export_firefox_string_array(string_table TableOrSubquery) +RETURNS Expr +AS ( + SELECT json_group_array(str ORDER BY idx) + FROM string_table +); + +-- Returns an instance of `FuncTable` as defined in +-- https://github.com/firefox-devtools/profiler/blob/main/src/types/profile.js +CREATE PERFETTO MACRO _export_firefox_func_table(func_table TableOrSubquery) +RETURNS Expr +AS ( + SELECT + json_object( + 'name', json_group_array(name ORDER BY idx), + 'isJS', json_group_array(is_js ORDER BY idx), + 'relevantForJS', json_group_array(relevant_for_js ORDER BY idx), + 'resource', json_group_array(resource ORDER BY idx), + 'fileName', json_group_array(file_name ORDER BY idx), + 'lineNumber', json_group_array(line_number ORDER BY idx), + 'columnNumber', json_group_array(column_number ORDER BY idx), + 'length', COUNT(*)) + FROM func_table +); + +-- Returns an empty instance of `NativeSymbolTable` as defined in +-- https://github.com/firefox-devtools/profiler/blob/main/src/types/profile.js +CREATE PERFETTO FUNCTION _export_firefox_native_symbol_table() +RETURNS STRING +AS +SELECT + json_object( + 'libIndex', json_array(), + 'address', json_array(), + 'name', json_array(), + 'functionSize', json_array(), + 'length', 0); + + +-- Returns an empty instance of `ResourceTable` as defined in +-- https://github.com/firefox-devtools/profiler/blob/main/src/types/profile.js +CREATE PERFETTO FUNCTION _export_firefox_resource_table() +RETURNS STRING +AS +SELECT + json_object( + 'length', 0, + 'lib', json_array(), + 'name', json_array(), + 'host', json_array(), + 'type', json_array()); + +-- Returns an instance of `Thread` as defined in +-- https://github.com/firefox-devtools/profiler/blob/main/src/types/profile.js +-- for the given `utid`. +CREATE PERFETTO FUNCTION _export_firefox_thread(utid INT) +RETURNS STRING +AS +WITH + symbol AS ( + SELECT + symbol_set_id, + RANK() + OVER ( + PARTITION BY symbol_set_id + ORDER BY id DESC + RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING + ) + - 1 AS inline_depth, + COUNT() OVER (PARTITION BY symbol_set_id) - 1 AS max_inline_depth, + name + FROM stack_profile_symbol + ), + callsite_base AS ( + SELECT + id, + parent_id, + name, + symbol_set_id, + IIF(inline_count = 0, 0, inline_count - 1) AS max_inline_depth + FROM + ( + SELECT + spc.id, + spc.parent_id, + COALESCE(s.name, spf.name, '') AS name, + sfp.symbol_set_id, + ( + SELECT COUNT(*) + FROM stack_profile_symbol s + WHERE s.symbol_set_id = sfp.symbol_set_id + ) AS inline_count + FROM stack_profile_callsite sfc, stack_profile_frame AS spf + ON (c.frame_id = spf.id) + ) + ), + callsite_recursive AS ( + SELECT + spc.id, + spc.parent_id, + spc.frame_id, + TRUE AS is_leaf + FROM + (SELECT DISTINCT callsite_id FROM perf_sample WHERE utid = $utid) s, + stack_profile_callsite spc + ON (spc.id = s.callsite_id) + UNION ALL + SELECT + parent.id, + parent.parent_id, + parent.frame_id, + FALSE AS is_leaf + FROM callsite_recursive AS child, stack_profile_callsite AS parent + ON (child.parent_id = parent.id) + ), + unique_callsite AS ( + SELECT DISTINCT * FROM callsite_recursive + ), + expanded_callsite_base AS ( + SELECT + c.id, + c.parent_id, + c.is_leaf, + c.frame_id, + COALESCE(s.name, spf.name, '') AS name, + COALESCE(s.inline_depth, 0) AS inline_depth, + COALESCE(s.max_inline_depth, 0) AS max_inline_depth + FROM unique_callsite c, stack_profile_frame AS spf + ON (c.frame_id = spf.id) + LEFT JOIN symbol s + USING (symbol_set_id) + ), + expanded_callsite AS ( + SELECT + *, + DENSE_RANK() OVER (ORDER BY id, inline_depth) - 1 AS stack_table_index, + DENSE_RANK() + OVER (ORDER BY frame_id, inline_depth) - 1 AS frame_table_index, + DENSE_RANK() OVER (ORDER BY name) - 1 AS func_table_index, + DENSE_RANK() OVER (ORDER BY name) - 1 AS string_table_index + FROM expanded_callsite_base + ), + string_table AS ( + SELECT DISTINCT + string_table_index AS idx, + name AS str + FROM + expanded_callsite + ), + func_table AS ( + SELECT DISTINCT + func_table_index AS idx, + string_table_index AS name, + FALSE AS is_js, + FALSE AS relevant_for_js, + - 1 AS resource, + NULL AS file_name, + NULL AS line_number, + NULL AS column_number + FROM expanded_callsite + ), + frame_table AS ( + SELECT DISTINCT + frame_table_index AS idx, + -1 AS address, + inline_depth, + 0 AS category, + 0 AS subcategory, + func_table_index AS func, + NULL AS native_symbol, + NULL AS inner_window_id, + NULL AS implementation, + NULL AS line, + NULL AS column + FROM expanded_callsite + ), + stack_table AS ( + SELECT + stack_table_index AS idx, + frame_table_index AS frame, + 0 AS category, + 0 AS subcategory, + ( + SELECT stack_table_index + FROM expanded_callsite AS parent + WHERE + ( + child.inline_depth = 0 + AND child.parent_id = parent.id + AND parent.inline_depth = parent.max_inline_depth) + OR ( + child.inline_depth - 1 = parent.inline_depth + AND child.id = parent.id) + ) AS prefix + FROM expanded_callsite AS child + ), + samples_table AS ( + SELECT + ROW_NUMBER() OVER(ORDER BY s.id) - 1 AS idx, + s.ts AS time, + ( + SELECT stack_table_index + FROM expanded_callsite c + WHERE s.callsite_id = c.id AND c.inline_depth = c.max_inline_depth + ) AS stack + FROM perf_sample AS s + WHERE utid = $utid + ) +SELECT + json_object( + 'processType', 'default', + -- processStartupTime: Milliseconds + -- processShutdownTime: Milliseconds | null + -- registerTime: Milliseconds + -- unregisterTime: Milliseconds | null + -- pausedRanges: PausedRange[] + -- showMarkersInTimeline?: boolean + 'name', COALESCE(thread.name, ''), + 'isMainThread', FALSE, + -- 'eTLD+1'?: string + -- processName?: string + -- isJsTracer?: boolean + 'pid', COALESCE(process.pid, 0), + 'tid', COALESCE(thread.tid, 0), + 'samples', _export_firefox_samples_table!(samples_table), + -- jsAllocations?: JsAllocationsTable + -- nativeAllocations?: NativeAllocationsTable + 'markers', json(_export_firefox_thread_markers()), + 'stackTable', _export_firefox_stack_table!(stack_table), + 'frameTable', _export_firefox_frame_table!(frame_table), + 'stringArray', _export_firefox_string_array!(string_table), + 'funcTable', _export_firefox_func_table!(func_table), + 'resourceTable', json(_export_firefox_resource_table()), + 'nativeSymbols', json(_export_firefox_native_symbol_table()) + -- jsTracer?: JsTracerTable + -- isPrivateBrowsing?: boolean + -- userContextId?: number + ) +FROM thread, process +USING (upid) +WHERE utid = $utid; + +-- Returns an array of `Thread` instances as defined in +-- https://github.com/firefox-devtools/profiler/blob/main/src/types/profile.js +-- for each thread present in the trace. +CREATE PERFETTO FUNCTION _export_firefox_threads() +RETURNS STRING +AS +SELECT json_group_array(json(_export_firefox_thread(utid))) FROM thread; + +-- Returns an instance of `ProfileMeta` as defined in +-- https://github.com/firefox-devtools/profiler/blob/main/src/types/profile.js +CREATE + PERFETTO FUNCTION _export_firefox_meta() +RETURNS STRING +AS +SELECT + json_object( + 'interval', 1, + 'startTime', 0, + -- startTime: Milliseconds + -- endTime?: Milliseconds + -- profilingStartTime?: Milliseconds + -- profilingEndTime?: Milliseconds + -- processType: number + 'processType', + 0, -- default + -- extensions?: ExtensionTable + -- categories?: CategoryList + 'categories', + json_array( + json_object( + 'name', + 'Other', + 'color', + 'grey', + 'subcategories', + json_array('Other'))), + -- product: 'Firefox' | string + 'product', + 'Perfetto', + -- stackwalk: 0 | 1 + 'stackwalk', + 1, + -- debug?: boolean + -- version: number + 'version', + 29, -- Taken from a generated profile + -- preprocessedProfileVersion: number + 'preprocessedProfileVersion', + 48, -- Taken from a generated profile + -- abi?: string + -- misc?: string + -- oscpu?: string + -- mainMemory?: Bytes + -- platform?: 'Android' | 'Windows' | 'Macintosh' | 'X11' | string + -- toolkit?: 'gtk' | 'gtk3' | 'windows' | 'cocoa' | 'android' | string + -- appBuildID?: string + -- arguments?: string + -- sourceURL?: string + -- physicalCPUs?: number + -- logicalCPUs?: number + -- CPUName?: string + -- symbolicated?: boolean + -- symbolicationNotSupported?: boolean + -- updateChannel?: 'default' | 'nightly' | 'nightly-try' | 'aurora' | 'beta' | 'release' | 'esr' | string + -- visualMetrics?: VisualMetrics + -- configuration?: ProfilerConfiguration + 'markerSchema', json_array(), + 'sampleUnits', json_object('time', 'ms', 'eventDelay', 'ms', 'threadCPUDelta', 'µs') + -- device?: string + -- importedFrom?: string + -- usesOnlyOneStackType?: boolean + -- doesNotUseFrameImplementation?: boolean + -- sourceCodeIsNotOnSearchfox?: boolean + -- extra?: ExtraProfileInfoSection[] + -- initialVisibleThreads?: ThreadIndex[] + -- initialSelectedThreads?: ThreadIndex[] + -- keepProfileThreadOrder?: boolean + -- gramsOfCO2ePerKWh?: number + ); + +-- Dumps all trace data as a Firefox profile json string +-- See `Profile` in +-- https://github.com/firefox-devtools/profiler/blob/main/src/types/profile.js +-- Also +-- https://firefox-source-docs.mozilla.org/tools/profiler/code-overview.html +-- +-- You would probably want to download the generated json and then open at +-- https://https://profiler.firefox.com +-- You can easily do this from the UI via the following SQL +-- `SELECT CAST(export_to_firefox_profile() AS BLOB) AS profile;` +-- The result will have a link for you to download this json as a file. +CREATE PERFETTO FUNCTION export_to_firefox_profile() +-- Json profile +RETURNS STRING +AS +SELECT + json_object( + 'meta', json(_export_firefox_meta()), + 'libs', json_array(), + 'pages', NULL, + 'counters', NULL, + 'profilerOverhead', NULL, + 'threads', json(_export_firefox_threads()), + 'profilingLog', NULL, + 'profileGatheringLog', NULL); diff --git a/src/trace_processor/perfetto_sql/stdlib/gpu/BUILD.gn b/src/trace_processor/perfetto_sql/stdlib/gpu/BUILD.gn new file mode 100644 index 000000000..a5f431dea --- /dev/null +++ b/src/trace_processor/perfetto_sql/stdlib/gpu/BUILD.gn @@ -0,0 +1,19 @@ +# Copyright (C) 2024 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import("../../../../../gn/perfetto_sql.gni") + +perfetto_sql_source_set("gpu") { + sources = [ "frequency.sql" ] +} diff --git a/src/trace_processor/perfetto_sql/stdlib/gpu/frequency.sql b/src/trace_processor/perfetto_sql/stdlib/gpu/frequency.sql new file mode 100644 index 000000000..2bd87b289 --- /dev/null +++ b/src/trace_processor/perfetto_sql/stdlib/gpu/frequency.sql @@ -0,0 +1,42 @@ +-- +-- Copyright 2024 The Android Open Source Project +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- https://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +INCLUDE PERFETTO MODULE counters.intervals; + +-- GPU frequency counter per GPU. +CREATE PERFETTO TABLE gpu_frequency( + -- Timestamp + ts INT, + -- Duration + dur INT, + -- GPU id. Joinable with `gpu_counter_track.gpu_id`. + gpu_id INT, + -- GPU frequency + gpu_freq INT +) AS +SELECT + ts, + dur, + gpu_id, + cast_int!(value) AS gpu_freq +FROM counter_leading_intervals!(( + SELECT c.* + FROM counter c + JOIN gpu_counter_track t + ON t.id = c.track_id AND t.name = 'gpufreq' + WHERE gpu_id IS NOT NULL +)) +JOIN gpu_counter_track t ON t.id = track_id;
\ No newline at end of file diff --git a/src/trace_processor/perfetto_sql/stdlib/memory/BUILD.gn b/src/trace_processor/perfetto_sql/stdlib/memory/BUILD.gn index ae3ad179c..ddc641950 100644 --- a/src/trace_processor/perfetto_sql/stdlib/memory/BUILD.gn +++ b/src/trace_processor/perfetto_sql/stdlib/memory/BUILD.gn @@ -15,5 +15,9 @@ import("../../../../../gn/perfetto_sql.gni") perfetto_sql_source_set("memory") { + deps = [ + "android", + "linux", + ] sources = [ "heap_graph_dominator_tree.sql" ] } diff --git a/src/trace_processor/perfetto_sql/stdlib/memory/android/BUILD.gn b/src/trace_processor/perfetto_sql/stdlib/memory/android/BUILD.gn new file mode 100644 index 000000000..55123b62c --- /dev/null +++ b/src/trace_processor/perfetto_sql/stdlib/memory/android/BUILD.gn @@ -0,0 +1,19 @@ +# Copyright (C) 2024 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import("../../../../../../gn/perfetto_sql.gni") + +perfetto_sql_source_set("android") { + sources = [ "gpu.sql" ] +} diff --git a/src/trace_processor/perfetto_sql/stdlib/memory/android/gpu.sql b/src/trace_processor/perfetto_sql/stdlib/memory/android/gpu.sql new file mode 100644 index 000000000..39abaece4 --- /dev/null +++ b/src/trace_processor/perfetto_sql/stdlib/memory/android/gpu.sql @@ -0,0 +1,35 @@ +-- +-- Copyright 2024 The Android Open Source Project +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- https://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +INCLUDE PERFETTO MODULE memory.linux.general; + +-- Counter for GPU memory per process with duration. +CREATE PERFETTO TABLE memory_gpu_per_process( + -- Timestamp + ts INT, + -- Duration + dur INT, + -- Upid of the process + upid INT, + -- GPU memory + gpu_memory INT +) AS +SELECT + ts, + dur, + upid, + cast_int!(value) AS gpu_memory +FROM _all_counters_per_process +WHERE name = 'GPU Memory';
\ No newline at end of file diff --git a/src/trace_processor/perfetto_sql/stdlib/memory/linux/BUILD.gn b/src/trace_processor/perfetto_sql/stdlib/memory/linux/BUILD.gn new file mode 100644 index 000000000..07111973f --- /dev/null +++ b/src/trace_processor/perfetto_sql/stdlib/memory/linux/BUILD.gn @@ -0,0 +1,23 @@ +# Copyright (C) 2024 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import("../../../../../../gn/perfetto_sql.gni") + +perfetto_sql_source_set("linux") { + sources = [ + "general.sql", + "high_watermark.sql", + "process.sql", + ] +} diff --git a/src/trace_processor/perfetto_sql/stdlib/memory/linux/general.sql b/src/trace_processor/perfetto_sql/stdlib/memory/linux/general.sql new file mode 100644 index 000000000..372cbf9b4 --- /dev/null +++ b/src/trace_processor/perfetto_sql/stdlib/memory/linux/general.sql @@ -0,0 +1,30 @@ +-- +-- Copyright 2024 The Android Open Source Project +-- +-- Licensed under the Apache License, Version 2.0 (the 'License'); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- https://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an 'AS IS' BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +CREATE PERFETTO VIEW _all_counters_per_process AS +SELECT + ts, + LEAD( + ts, 1, + (SELECT COALESCE(end_ts, trace_end()) + FROM process p WHERE p.upid = t.upid) + 1) + OVER (PARTITION BY track_id ORDER BY ts) - ts AS dur, + upid, + value, + track_id, + name +FROM counter c JOIN process_counter_track t +ON t.id = c.track_id +WHERE upid IS NOT NULL;
\ No newline at end of file diff --git a/src/trace_processor/perfetto_sql/stdlib/memory/linux/high_watermark.sql b/src/trace_processor/perfetto_sql/stdlib/memory/linux/high_watermark.sql new file mode 100644 index 000000000..b01f2df24 --- /dev/null +++ b/src/trace_processor/perfetto_sql/stdlib/memory/linux/high_watermark.sql @@ -0,0 +1,67 @@ +-- +-- Copyright 2024 The Android Open Source Project +-- +-- Licensed under the Apache License, Version 2.0 (the 'License'); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- https://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an 'AS IS' BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +INCLUDE PERFETTO MODULE memory.linux.process; +INCLUDE PERFETTO MODULE counters.intervals; + +CREATE PERFETTO TABLE _memory_rss_high_watermark_per_process_table AS +WITH with_rss AS ( + SELECT + ts, + dur, + upid, + COALESCE(file_rss, 0) + COALESCE(anon_rss, 0) + COALESCE(shmem_rss, 0) AS rss + FROM _memory_rss_and_swap_per_process_table +), +high_watermark_as_counter AS ( +SELECT + ts, + MAX(rss) OVER (PARTITION BY upid ORDER BY ts) AS value, + -- `id` and `track_id` are hacks to use this table in + -- `counter_leading_intervals` macro. As `track_id` is using for looking + -- for duplicates, we are aliasing `upid` with it. `Id` is ignored by the macro. + upid AS track_id, + 0 AS id +FROM with_rss +) +SELECT ts, dur, track_id AS upid, cast_int!(value) AS rss_high_watermark +FROM counter_leading_intervals!(high_watermark_as_counter); + +-- For each process fetches the memory high watermark until or during +-- timestamp. +CREATE PERFETTO VIEW memory_rss_high_watermark_per_process +( + -- Timestamp + ts INT, + -- Duration + dur INT, + -- Upid of the process + upid INT, + -- Pid of the process + pid INT, + -- Name of the process + process_name STRING, + -- Maximum `rss` value until now + rss_high_watermark INT +) AS +SELECT + ts, + dur, + upid, + pid, + name AS process_name, + rss_high_watermark +FROM _memory_rss_high_watermark_per_process_table +JOIN process USING (upid);
\ No newline at end of file diff --git a/src/trace_processor/perfetto_sql/stdlib/memory/linux/process.sql b/src/trace_processor/perfetto_sql/stdlib/memory/linux/process.sql new file mode 100644 index 000000000..4601c21c2 --- /dev/null +++ b/src/trace_processor/perfetto_sql/stdlib/memory/linux/process.sql @@ -0,0 +1,217 @@ +-- +-- Copyright 2024 The Android Open Source Project +-- +-- Licensed under the Apache License, Version 2.0 (the 'License'); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- https://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an 'AS IS' BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +INCLUDE PERFETTO MODULE android.oom_adjuster; +INCLUDE PERFETTO MODULE memory.linux.general; + +-- All memory counters tables. + +CREATE PERFETTO VIEW _anon_rss AS +SELECT + ts, + dur, + upid, + value AS anon_rss_val +FROM _all_counters_per_process +WHERE name = 'mem.rss.anon'; + +CREATE PERFETTO VIEW _file_rss AS +SELECT + ts, + dur, + upid, + value AS file_rss_val +FROM _all_counters_per_process +WHERE name = 'mem.rss.file'; + +CREATE PERFETTO VIEW _shmem_rss AS +SELECT + ts, + dur, + upid, + value AS shmem_rss_val +FROM _all_counters_per_process +WHERE name = 'mem.rss.shmem'; + +CREATE PERFETTO VIEW _swap AS +SELECT + ts, + dur, + upid, + value AS swap_val +FROM _all_counters_per_process +WHERE name = 'mem.swap'; + +-- Span joins + +CREATE VIRTUAL TABLE _anon_swap_sj +USING SPAN_OUTER_JOIN( + _anon_rss PARTITIONED upid, + _swap PARTITIONED upid); + +CREATE VIRTUAL TABLE _anon_swap_file_sj +USING SPAN_OUTER_JOIN( + _anon_swap_sj PARTITIONED upid, + _file_rss PARTITIONED upid +); + +CREATE VIRTUAL TABLE _rss_swap_sj +USING SPAN_OUTER_JOIN( + _anon_swap_file_sj PARTITIONED upid, + _shmem_rss PARTITIONED upid +); + +CREATE PERFETTO TABLE _memory_rss_and_swap_per_process_table AS +SELECT + ts, dur, upid, + cast_int!(anon_rss_val) AS anon_rss, + cast_int!(file_rss_val) AS file_rss, + cast_int!(shmem_rss_val) AS shmem_rss, + cast_int!(swap_val) AS swap +FROM _rss_swap_sj; + + +-- Memory metrics timeline for each process. +CREATE PERFETTO VIEW memory_rss_and_swap_per_process( + -- Timestamp + ts INT, + -- Duration + dur INT, + -- Upid of the process + upid INT, + -- Pid of the process + pid INT, + -- Name of the process + process_name STRING, + -- Anon RSS counter value + anon_rss INT, + -- File RSS counter value + file_rss INT, + -- Shared memory RSS counter value + shmem_rss INT, + -- Total RSS value. Sum of `anon_rss`, `file_rss` and `shmem_rss`. Returns + -- value even if one of the values is NULL. + rss INT, + -- Swap counter value + swap INT, + -- Sum or `anon_rss` and `swap`. Returns value even if one of the values is + -- NULL. + anon_rss_and_swap INT, + -- Sum or `rss` and `swap`. Returns value even if one of the values is NULL. + rss_and_swap INT +) AS +SELECT + ts, + dur, + upid, + pid, + name AS process_name, + anon_rss, + file_rss, + shmem_rss, + -- We do COALESCE only on `shmem_rss` and `swap`, as it can be expected all + -- process start to emit anon rss and file rss events (you'll need to at + -- least read code and have some memory to work with) - so the NULLs are real + -- values. But it is possible that you will never swap or never use shmem, + -- so those values are expected to often be NULLs, which shouldn't propagate + -- into the values like `anon_and_swap` or `rss`. + file_rss + anon_rss + COALESCE(shmem_rss, 0) AS rss, + swap, + anon_rss + COALESCE(swap, 0) AS anon_rss_and_swap, + anon_rss + file_rss + COALESCE(shmem_rss, 0) + COALESCE(swap, 0) AS rss_and_swap +FROM _memory_rss_and_swap_per_process_table +JOIN process USING (upid); + +-- OOM score tables + +CREATE VIRTUAL TABLE _mem_ooms_sj +USING SPAN_OUTER_JOIN( + android_oom_adj_intervals PARTITIONED upid, + _memory_rss_and_swap_per_process_table PARTITIONED upid); + +-- Process memory and it's OOM adjuster scores. Detects transitions, each new +-- interval means that either the memory or OOM adjuster score of the process changed. +CREATE PERFETTO TABLE memory_oom_score_with_rss_and_swap_per_process( + -- Timestamp the oom_adj score or memory of the process changed + ts INT, + -- Duration until the next oom_adj score or memory change of the process. + dur INT, + -- oom adjuster score of the process. + score INT, + -- oom adjuster bucket of the process. + bucket STRING, + -- Upid of the process having an oom_adj update. + upid INT, + -- Name of the process having an oom_adj update. + process_name STRING, + -- Pid of the process having an oom_adj update. + pid INT, + -- Slice of the latest oom_adj update in the system_server. Alias of + -- `slice.id`. + oom_adj_id INT, + -- Timestamp of the latest oom_adj update in the system_server. + oom_adj_ts INT, + -- Duration of the latest oom_adj update in the system_server. + oom_adj_dur INT, + -- Track of the latest oom_adj update in the system_server. Alias of + -- `track.id`. + oom_adj_track_id INT, + -- Thread name of the latest oom_adj update in the system_server. + oom_adj_thread_name STRING, + -- Reason for the latest oom_adj update in the system_server. + oom_adj_reason STRING, + -- Trigger for the latest oom_adj update in the system_server. + oom_adj_trigger STRING, + -- Anon RSS counter value + anon_rss INT, + -- File RSS counter value + file_rss INT, + -- Shared memory RSS counter value + shmem_rss INT, + -- Total RSS value. Sum of `anon_rss`, `file_rss` and `shmem_rss`. Returns + -- value even if one of the values is NULL. + rss INT, + -- Swap counter value + swap INT, + -- Sum or `anon_rss` and `swap`. Returns value even if one of the values is + -- NULL. + anon_rss_and_swap INT, + -- Sum or `rss` and `swap`. Returns value even if one of the values is NULL. + rss_and_swap INT +) AS +SELECT + ts, + dur, + score, + bucket, + upid, + process_name, + pid, + oom_adj_id, + oom_adj_ts, + oom_adj_dur, + oom_adj_track_id, + oom_adj_thread_name, + oom_adj_reason, + oom_adj_trigger, + anon_rss, + file_rss, + shmem_rss, + file_rss + anon_rss + COALESCE(shmem_rss, 0) AS rss, + swap, + anon_rss + COALESCE(swap, 0) AS anon_rss_and_swap, + anon_rss + file_rss + COALESCE(shmem_rss, 0) + COALESCE(swap, 0) AS rss_and_swap +FROM _mem_ooms_sj +JOIN process USING (upid); diff --git a/src/trace_processor/perfetto_sql/stdlib/prelude/BUILD.gn b/src/trace_processor/perfetto_sql/stdlib/prelude/BUILD.gn index 499c2e6db..596d327e5 100644 --- a/src/trace_processor/perfetto_sql/stdlib/prelude/BUILD.gn +++ b/src/trace_processor/perfetto_sql/stdlib/prelude/BUILD.gn @@ -18,6 +18,7 @@ perfetto_sql_source_set("prelude") { sources = [ "casts.sql", "slices.sql", + "tables_views.sql", "trace_bounds.sql", ] } diff --git a/src/trace_processor/perfetto_sql/stdlib/prelude/tables_views.sql b/src/trace_processor/perfetto_sql/stdlib/prelude/tables_views.sql new file mode 100644 index 000000000..402ac521f --- /dev/null +++ b/src/trace_processor/perfetto_sql/stdlib/prelude/tables_views.sql @@ -0,0 +1,288 @@ +-- +-- Copyright 2024 The Android Open Source Project +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- https://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- Contains information of CPUs seen during the trace. +CREATE PERFETTO VIEW cpu ( + -- Unique identifier for this CPU. Identical to |ucpu|, prefer using |ucpu| + -- instead. + id UINT, + -- Unique identifier for this CPU. Isn't equal to |cpu| for remote machines + -- and is equal to |cpu| for the host machine. + ucpu UINT, + -- The 0-based CPU core identifier. + cpu UINT, + -- The name of the "most-specific" child table containing this row. + type STRING, + -- The cluster id is shared by CPUs in the same cluster. + cluster_id UINT, + -- A string describing this core. + processor STRING, + -- Machine identifier, non-null for CPUs on a remote machine. + machine_id UINT +) AS +SELECT + id, + id AS ucpu, + cpu, + type AS type, + cluster_id, + processor, + machine_id +FROM + __intrinsic_cpu +WHERE + cpu IS NOT NULL; + +-- Contains information of available frequencies of CPUs. +CREATE PERFETTO VIEW cpu_frequencies ( + -- Unique identifier for this cpu frequency. + id UINT, + -- The CPU for this frequency, meaningful only in single machine traces. + -- For multi-machine, join with the `cpu` table on `ucpu` to get the CPU + -- identifier of each machine. + cpu UINT, + -- CPU frequency in KHz. + freq UINT, + -- The CPU that the slice executed on (meaningful only in single machine + -- traces). For multi-machine, join with the `cpu` table on `ucpu` to get the + -- CPU identifier of each machine. + ucpu UINT +) AS +SELECT id, ucpu AS cpu, freq, ucpu +FROM __intrinsic_cpu_freq; + +-- This table holds slices with kernel thread scheduling information. These +-- slices are collected when the Linux "ftrace" data source is used with the +-- "sched/switch" and "sched/wakeup*" events enabled. +-- +-- The rows in this table will always have a matching row in the |thread_state| +-- table with |thread_state.state| = 'Running' +CREATE PERFETTO VIEW sched_slice ( + -- Unique identifier for this scheduling slice. + id UINT, + -- The name of the "most-specific" child table containing this row. + type STRING, + -- The timestamp at the start of the slice (in nanoseconds). + ts LONG, + -- The duration of the slice (in nanoseconds). + dur LONG, + -- The CPU that the slice executed on (meaningful only in single machine + -- traces). For multi-machine, join with the `cpu` table on `ucpu` to get the + -- CPU identifier of each machine. + cpu UINT, + -- The thread's unique id in the trace. + utid UINT, + -- A string representing the scheduling state of the kernel + -- thread at the end of the slice. The individual characters in + -- the string mean the following: R (runnable), S (awaiting a + -- wakeup), D (in an uninterruptible sleep), T (suspended), + -- t (being traced), X (exiting), P (parked), W (waking), + -- I (idle), N (not contributing to the load average), + -- K (wakeable on fatal signals) and Z (zombie, awaiting + -- cleanup). + end_state STRING, + -- The kernel priority that the thread ran at. + priority INT, + -- The unique CPU identifier that the slice executed on. + ucpu UINT +) AS +SELECT + id, + type, + ts, + dur, + ucpu AS cpu, + utid, + end_state, + priority, + ucpu +FROM + __intrinsic_sched_slice; + +-- This table contains the scheduling state of every thread on the system during +-- the trace. +-- +-- The rows in this table which have |state| = 'Running', will have a +-- corresponding row in the |sched_slice| table. +CREATE PERFETTO VIEW thread_state ( + -- Unique identifier for this thread state. + id UINT, + -- The name of the "most-specific" child table containing this row. + type STRING, + -- The timestamp at the start of the slice (in nanoseconds). + ts LONG, + -- The duration of the slice (in nanoseconds). + dur LONG, + -- The CPU that the thread executed on (meaningful only in single machine + -- traces). For multi-machine, join with the `cpu` table on `ucpu` to get the + -- CPU identifier of each machine. + cpu UINT, + -- The thread's unique id in the trace. + utid UINT, + -- The scheduling state of the thread. Can be "Running" or any of the states + -- described in |sched_slice.end_state|. + state STRING, + -- Indicates whether this thread was blocked on IO. + io_wait UINT, + -- The function in the kernel this thread was blocked on. + blocked_function STRING, + -- The unique thread id of the thread which caused a wakeup of this thread. + waker_utid UINT, + -- The unique thread state id which caused a wakeup of this thread. + waker_id UINT, + -- Whether the wakeup was from interrupt context or process context. + irq_context UINT, + -- The unique CPU identifier that the thread executed on. + ucpu UINT +) AS +SELECT + id, + type, + ts, + dur, + ucpu AS cpu, + utid, + state, + io_wait, + blocked_function, + waker_utid, + waker_id, + irq_context, + ucpu +FROM + __intrinsic_thread_state; + +-- Contains 'raw' events from the trace for some types of events. This table +-- only exists for debugging purposes and should not be relied on in production +-- usecases (i.e. metrics, standard library etc.) +CREATE PERFETTO VIEW raw ( + -- Unique identifier for this raw event. + id UINT, + -- The name of the "most-specific" child table containing this row. + type STRING, + -- The timestamp of this event. + ts LONG, + -- The name of the event. For ftrace events, this will be the ftrace event + -- name. + name STRING, + -- The CPU this event was emitted on (meaningful only in single machine + -- traces). For multi-machine, join with the `cpu` table on `ucpu` to get the + -- CPU identifier of each machine. + cpu UINT, + -- The thread this event was emitted on. + utid UINT, + -- The set of key/value pairs associated with this event. + arg_set_id UINT, + -- Ftrace event flags for this event. Currently only emitted for sched_waking + -- events. + common_flags UINT, + -- The unique CPU identifier that this event was emitted on. + ucpu UINT +) AS +SELECT + id, + type, + ts, + name, + ucpu AS cpu, + utid, + arg_set_id, + common_flags, + ucpu +FROM + __intrinsic_raw; + +-- Contains all the ftrace events in the trace. This table exists only for +-- debugging purposes and should not be relied on in production usecases (i.e. +-- metrics, standard library etc). Note also that this table might be empty if +-- raw ftrace parsing has been disabled. +CREATE PERFETTO VIEW ftrace_event ( + -- Unique identifier for this ftrace event. + id UINT, + -- The name of the "most-specific" child table containing this row. + type STRING, + -- The timestamp of this event. + ts LONG, + -- The ftrace event name. + name STRING, + -- The CPU this event was emitted on (meaningful only in single machine + -- traces). For multi-machine, join with the `cpu` table on `ucpu` to get the + -- CPU identifier of each machine. + cpu UINT, + -- The thread this event was emitted on. + utid UINT, + -- The set of key/value pairs associated with this event. + arg_set_id UINT, + -- Ftrace event flags for this event. Currently only emitted for + -- sched_waking events. + common_flags UINT, + -- The unique CPU identifier that this event was emitted on. + ucpu UINT +) AS +SELECT + id, + type, + ts, + name, + ucpu AS cpu, + utid, + arg_set_id, + common_flags, + ucpu +FROM + __intrinsic_ftrace_event; + +-- The sched_slice table with the upid column. +CREATE PERFETTO VIEW experimental_sched_upid ( + -- Unique identifier for this scheduling slice. + id UINT, + -- The name of the "most-specific" child table containing this row. + type STRING, + -- The timestamp at the start of the slice (in nanoseconds). + ts LONG, + -- The duration of the slice (in nanoseconds). + dur LONG, + -- The CPU that the slice executed on (meaningful only in single machine + -- traces). For multi-machine, join with the `cpu` table on `ucpu` to get the + -- CPU identifier of each machine. + cpu UINT, + -- The thread's unique id in the trace. + utid UINT, + -- A string representing the scheduling state of the kernel thread at the end + -- of the slice. The individual characters in the string mean the following: R + -- (runnable), S (awaiting a wakeup), D (in an uninterruptible sleep), T + -- (suspended), t (being traced), X (exiting), P (parked), W (waking), I + -- (idle), N (not contributing to the load average), K (wakeable on fatal + -- signals) and Z (zombie, awaiting cleanup). + end_state STRING, + -- The kernel priority that the thread ran at. + priority INT, + -- The unique CPU identifier that the slice executed on. + ucpu UINT, + -- The process's unique id in the trace. + upid UINT +) AS +SELECT + id, + type, + ts, + dur, + ucpu AS cpu, + utid, + end_state, + priority, + ucpu, + upid +FROM + __intrinsic_sched_upid;
\ No newline at end of file diff --git a/src/trace_processor/perfetto_sql/stdlib/sched/BUILD.gn b/src/trace_processor/perfetto_sql/stdlib/sched/BUILD.gn index 916892692..d7e96087b 100644 --- a/src/trace_processor/perfetto_sql/stdlib/sched/BUILD.gn +++ b/src/trace_processor/perfetto_sql/stdlib/sched/BUILD.gn @@ -15,7 +15,6 @@ import("../../../../../gn/perfetto_sql.gni") perfetto_sql_source_set("sched") { - deps = [ "utilization" ] sources = [ "runnable.sql", "states.sql", diff --git a/src/trace_processor/perfetto_sql/stdlib/sched/thread_executing_span_with_slice.sql b/src/trace_processor/perfetto_sql/stdlib/sched/thread_executing_span_with_slice.sql index 89085f374..74f73a5c5 100644 --- a/src/trace_processor/perfetto_sql/stdlib/sched/thread_executing_span_with_slice.sql +++ b/src/trace_processor/perfetto_sql/stdlib/sched/thread_executing_span_with_slice.sql @@ -213,7 +213,7 @@ RETURNS self_thread_state_id AS id, ts, dur, - utid, + critical_path_utid AS utid, 0 AS stack_depth, 'thread_state: ' || self_state AS name, 'thread_state' AS table_name, @@ -225,7 +225,7 @@ RETURNS self_thread_state_id AS id, ts, dur, - utid, + critical_path_utid AS utid, 1 AS stack_depth, IIF(self_state GLOB 'R*', NULL, 'kernel function: ' || self_function) AS name, 'thread_state' AS table_name, @@ -237,7 +237,7 @@ RETURNS self_thread_state_id AS id, ts, dur, - utid, + critical_path_utid AS utid, 2 AS stack_depth, IIF(self_state GLOB 'R*', NULL, 'io_wait: ' || self_io_wait) AS name, 'thread_state' AS table_name, @@ -281,7 +281,7 @@ RETURNS anc.id, slice.ts, slice.dur, - slice.utid, + critical_path_utid AS utid, anc.depth + 5 AS stack_depth, IIF($enable_self_slice, anc.name, NULL) AS name, 'slice' AS table_name, @@ -294,7 +294,7 @@ RETURNS self_slice_id AS id, ts, dur, - utid, + critical_path_utid AS utid, self_slice_depth + 5 AS stack_depth, IIF($enable_self_slice, self_slice_name, NULL) AS name, 'slice' AS table_name, diff --git a/src/trace_processor/perfetto_sql/stdlib/sched/utilization/system.sql b/src/trace_processor/perfetto_sql/stdlib/sched/utilization/system.sql deleted file mode 100644 index 7dd671d6a..000000000 --- a/src/trace_processor/perfetto_sql/stdlib/sched/utilization/system.sql +++ /dev/null @@ -1,67 +0,0 @@ --- --- Copyright 2024 The Android Open Source Project --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- https://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. - -INCLUDE PERFETTO MODULE sched.utilization.general; -INCLUDE PERFETTO MODULE time.conversion; - --- The purpose of this module is to provide high level aggregates of system --- utilization, akin to /proc/stat results. - --- Returns a table of system utilization per given period. --- Utilization is calculated as sum of average utilization of each CPU in each --- period, which is defined as a multiply of |interval|. For this reason --- first and last period might have lower then real utilization. -CREATE PERFETTO FUNCTION sched_utilization_per_period( - -- Length of the period on which utilization should be averaged. - interval INT) -RETURNS TABLE ( - -- Timestamp of start of a second. - ts INT, - -- Sum of average utilization over period. - -- Note: as the data is normalized, the values will be in the - -- [0, 1] range. - utilization DOUBLE, - -- Sum of average utilization over all CPUs over period. - -- Note: as the data is unnormalized, the values will be in the - -- [0, cpu_count] range. - unnormalized_utilization DOUBLE -) AS -SELECT * -FROM _sched_avg_utilization_per_period!( - $interval, - (SELECT * FROM sched WHERE utid != 0) -); - --- Table with system utilization per second. --- Utilization is calculated by sum of average utilization of each CPU every --- second. For this reason first and last second might have lower then real --- utilization. -CREATE PERFETTO TABLE sched_utilization_per_second( - -- Timestamp of start of a second. - ts INT, - -- Sum of average utilization over period. - -- Note: as the data is normalized, the values will be in the - -- [0, 1] range. - utilization DOUBLE, - -- Sum of average utilization over all CPUs over period. - -- Note: as the data is unnormalized, the values will be in the - -- [0, cpu_count] range. - unnormalized_utilization DOUBLE -) AS -SELECT - ts, - utilization, - unnormalized_utilization -FROM sched_utilization_per_period(time_from_s(1));
\ No newline at end of file diff --git a/src/trace_processor/perfetto_sql/stdlib/viz/summary/BUILD.gn b/src/trace_processor/perfetto_sql/stdlib/viz/summary/BUILD.gn index 3f664bd72..b02d5a7d2 100644 --- a/src/trace_processor/perfetto_sql/stdlib/viz/summary/BUILD.gn +++ b/src/trace_processor/perfetto_sql/stdlib/viz/summary/BUILD.gn @@ -16,6 +16,7 @@ import("../../../../../../gn/perfetto_sql.gni") perfetto_sql_source_set("summary") { sources = [ + "counters.sql", "processes.sql", "slices.sql", "threads.sql", diff --git a/src/trace_processor/perfetto_sql/stdlib/viz/summary/counters.sql b/src/trace_processor/perfetto_sql/stdlib/viz/summary/counters.sql new file mode 100644 index 000000000..7ae6d1309 --- /dev/null +++ b/src/trace_processor/perfetto_sql/stdlib/viz/summary/counters.sql @@ -0,0 +1,18 @@ +-- +-- Copyright 2024 The Android Open Source Project +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- https://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +CREATE PERFETTO TABLE _counter_track_summary AS +SELECT DISTINCT track_id as id +FROM counter; diff --git a/src/trace_processor/perfetto_sql/stdlib/wattson/BUILD.gn b/src/trace_processor/perfetto_sql/stdlib/wattson/BUILD.gn index 4167d8c32..8a489123b 100644 --- a/src/trace_processor/perfetto_sql/stdlib/wattson/BUILD.gn +++ b/src/trace_processor/perfetto_sql/stdlib/wattson/BUILD.gn @@ -19,6 +19,7 @@ perfetto_sql_source_set("wattson") { "arm_dsu.sql", "cpu_freq.sql", "cpu_idle.sql", + "cpu_idle_offsets.sql", "system_state.sql", ] } diff --git a/src/trace_processor/perfetto_sql/stdlib/wattson/cpu_idle.sql b/src/trace_processor/perfetto_sql/stdlib/wattson/cpu_idle.sql index 006c87677..388f54788 100644 --- a/src/trace_processor/perfetto_sql/stdlib/wattson/cpu_idle.sql +++ b/src/trace_processor/perfetto_sql/stdlib/wattson/cpu_idle.sql @@ -15,22 +15,7 @@ INCLUDE PERFETTO MODULE android.device; INCLUDE PERFETTO MODULE cpu.idle; - --- Device specific info for deep idle time offsets -CREATE PERFETTO TABLE _device_cpu_deep_idle_offsets -AS -WITH data(device, cpu, offset_ns) AS ( - VALUES - ("oriole", 6, 200000), - ("oriole", 7, 200000), - ("raven", 6, 200000), - ("raven", 7, 200000), - ("eos", 0, 450000), - ("eos", 1, 450000), - ("eos", 2, 450000), - ("eos", 3, 450000) -) -select * from data; +INCLUDE PERFETTO MODULE wattson.cpu_idle_offsets; -- Get the corresponding deep idle time offset based on device and CPU. CREATE PERFETTO FUNCTION _get_deep_idle_offset(cpu INT) diff --git a/src/trace_processor/perfetto_sql/stdlib/wattson/cpu_idle_offsets.sql b/src/trace_processor/perfetto_sql/stdlib/wattson/cpu_idle_offsets.sql new file mode 100644 index 000000000..1cecaa4c2 --- /dev/null +++ b/src/trace_processor/perfetto_sql/stdlib/wattson/cpu_idle_offsets.sql @@ -0,0 +1,31 @@ +-- +-- Copyright 2024 The Android Open Source Project +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- https://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +-- Device specific info for deep idle time offsets +CREATE PERFETTO TABLE _device_cpu_deep_idle_offsets +AS +WITH data(device, cpu, offset_ns) AS ( + VALUES + ("oriole", 6, 200000), + ("oriole", 7, 200000), + ("raven", 6, 200000), + ("raven", 7, 200000), + ("eos", 0, 450000), + ("eos", 1, 450000), + ("eos", 2, 450000), + ("eos", 3, 450000) +) +select * from data; + diff --git a/src/trace_processor/read_trace.cc b/src/trace_processor/read_trace.cc index 06bd39d44..9d5a1f7ea 100644 --- a/src/trace_processor/read_trace.cc +++ b/src/trace_processor/read_trace.cc @@ -25,12 +25,12 @@ #include "perfetto/trace_processor/trace_blob.h" #include "perfetto/trace_processor/trace_blob_view.h" -#include "src/trace_processor/forwarding_trace_parser.h" #include "src/trace_processor/importers/gzip/gzip_trace_parser.h" #include "src/trace_processor/importers/proto/proto_trace_tokenizer.h" #include "src/trace_processor/read_trace_internal.h" #include "src/trace_processor/util/gzip_utils.h" #include "src/trace_processor/util/status_macros.h" +#include "src/trace_processor/util/trace_type.h" #include "protos/perfetto/trace/trace.pbzero.h" #include "protos/perfetto/trace/trace_packet.pbzero.h" diff --git a/src/trace_processor/read_trace_internal.cc b/src/trace_processor/read_trace_internal.cc index a632134da..3b537cdf4 100644 --- a/src/trace_processor/read_trace_internal.cc +++ b/src/trace_processor/read_trace_internal.cc @@ -95,8 +95,8 @@ util::Status ReadTraceUnfinalized( RETURN_IF_ERROR(tp->Parse(std::move(slice))); bytes_read += slice_size; } // while (slices) - } // if (mapped.IsValid()) - } // if (use_mmap) + } // if (mapped.IsValid()) + } // if (use_mmap) if (bytes_read == 0) PERFETTO_LOG("Cannot use mmap on this system. Falling back on read()"); #endif // PERFETTO_HAS_MMAP() diff --git a/src/trace_processor/rpc/wasm_bridge.cc b/src/trace_processor/rpc/wasm_bridge.cc index ca27abc0b..2473e4c18 100644 --- a/src/trace_processor/rpc/wasm_bridge.cc +++ b/src/trace_processor/rpc/wasm_bridge.cc @@ -15,8 +15,13 @@ */ #include <emscripten/emscripten.h> + #include <cstdint> +#include <cstdio> +#include <cstdlib> +#include <new> +#include "perfetto/base/compiler.h" #include "src/trace_processor/rpc/rpc.h" namespace perfetto::trace_processor { @@ -29,6 +34,12 @@ Rpc* g_trace_processor_rpc; // The buffer used to pass the request arguments. The caller (JS) decides how // big this buffer should be in the Initialize() call. uint8_t* g_req_buf; + +PERFETTO_NO_INLINE void OutOfMemoryHandler() { + fprintf(stderr, "\nCannot enlarge memory\n"); + abort(); +} + } // namespace // +---------------------------------------------------------------------------+ @@ -41,6 +52,13 @@ uint8_t* EMSCRIPTEN_KEEPALIVE trace_processor_rpc_init(RpcResponseFn* RpcResponseFn, uint32_t); uint8_t* trace_processor_rpc_init(RpcResponseFn* resp_function, uint32_t req_buffer_size) { + // Usually OOMs manifest as a failure in dlmalloc() -> sbrk() -> + //_emscripten_resize_heap() which aborts itself. However in some rare cases + // sbrk() can fail outside of _emscripten_resize_heap and just return null. + // When that happens, just abort with the same message that + // _emscripten_resize_heap uses, so error_dialog.ts shows a OOM message. + std::set_new_handler(&OutOfMemoryHandler); + g_trace_processor_rpc = new Rpc(); // |resp_function| is a JS-bound function passed by wasm_bridge.ts. It will diff --git a/src/trace_processor/sorter/BUILD.gn b/src/trace_processor/sorter/BUILD.gn index 8d4c9f97b..12ccf3985 100644 --- a/src/trace_processor/sorter/BUILD.gn +++ b/src/trace_processor/sorter/BUILD.gn @@ -32,6 +32,7 @@ source_set("sorter") { "../importers/common:parser_types", "../importers/common:trace_parser_hdr", "../importers/fuchsia:fuchsia_record", + "../importers/perf:record", "../importers/systrace:systrace_line", "../storage", "../types", diff --git a/src/trace_processor/sorter/trace_sorter.cc b/src/trace_processor/sorter/trace_sorter.cc index a2d9861d9..9ccea77f9 100644 --- a/src/trace_processor/sorter/trace_sorter.cc +++ b/src/trace_processor/sorter/trace_sorter.cc @@ -15,20 +15,28 @@ */ #include <algorithm> +#include <cstddef> +#include <cstdint> +#include <cstdlib> +#include <cstring> +#include <limits> #include <memory> #include <utility> #include "perfetto/base/compiler.h" +#include "perfetto/base/logging.h" +#include "perfetto/public/compiler.h" #include "src/trace_processor/importers/common/parser_types.h" #include "src/trace_processor/importers/common/trace_parser.h" #include "src/trace_processor/importers/fuchsia/fuchsia_record.h" +#include "src/trace_processor/importers/perf/record.h" #include "src/trace_processor/sorter/trace_sorter.h" -#include "src/trace_processor/storage/trace_storage.h" +#include "src/trace_processor/sorter/trace_token_buffer.h" +#include "src/trace_processor/storage/stats.h" #include "src/trace_processor/types/trace_processor_context.h" #include "src/trace_processor/util/bump_allocator.h" -namespace perfetto { -namespace trace_processor { +namespace perfetto::trace_processor { TraceSorter::TraceSorter(TraceProcessorContext* context, SortingMode sorting_mode) @@ -116,10 +124,12 @@ void TraceSorter::SortAndExtractEventsUntilAllocId( auto& queue = sorter_data.queues[i]; if (queue.events_.empty()) continue; - all_queues_empty = false; - PERFETTO_DCHECK(queue.max_ts_ <= append_max_ts_); - if (queue.min_ts_ < min_queue_ts[0]) { + + // Checking for |all_queues_empty| is necessary here as in fuzzer cases + // we can end up with |int64::max()| as the value here. + // See https://crbug.com/oss-fuzz/69164 for an example. + if (all_queues_empty || queue.min_ts_ < min_queue_ts[0]) { min_queue_ts[1] = min_queue_ts[0]; min_queue_ts[0] = queue.min_ts_; min_queue_idx = i; @@ -127,6 +137,7 @@ void TraceSorter::SortAndExtractEventsUntilAllocId( } else if (queue.min_ts_ < min_queue_ts[1]) { min_queue_ts[1] = queue.min_ts_; } + all_queues_empty = false; } } if (all_queues_empty) @@ -188,7 +199,7 @@ void TraceSorter::ParseTracePacket(TraceProcessorContext& context, switch (static_cast<TimestampedEvent::Type>(event.event_type)) { case TimestampedEvent::Type::kPerfRecord: context.perf_record_parser->ParsePerfRecord( - event.ts, token_buffer_.Extract<TraceBlobView>(id)); + event.ts, token_buffer_.Extract<perf_importer::Record>(id)); return; case TimestampedEvent::Type::kTracePacket: context.proto_trace_parser->ParseTracePacket( @@ -275,10 +286,9 @@ void TraceSorter::ExtractAndDiscardTokenizedObject( const TimestampedEvent& event) { TraceTokenBuffer::Id id = GetTokenBufferId(event); switch (static_cast<TimestampedEvent::Type>(event.event_type)) { - case TimestampedEvent::Type::kPerfRecord: - base::ignore_result(token_buffer_.Extract<TraceBlobView>(id)); - return; case TimestampedEvent::Type::kTracePacket: + case TimestampedEvent::Type::kFtraceEvent: + case TimestampedEvent::Type::kEtwEvent: base::ignore_result(token_buffer_.Extract<TracePacketData>(id)); return; case TimestampedEvent::Type::kTrackEvent: @@ -299,11 +309,8 @@ void TraceSorter::ExtractAndDiscardTokenizedObject( case TimestampedEvent::Type::kInlineSchedWaking: base::ignore_result(token_buffer_.Extract<InlineSchedWaking>(id)); return; - case TimestampedEvent::Type::kFtraceEvent: - base::ignore_result(token_buffer_.Extract<TracePacketData>(id)); - return; - case TimestampedEvent::Type::kEtwEvent: - base::ignore_result(token_buffer_.Extract<TracePacketData>(id)); + case TimestampedEvent::Type::kPerfRecord: + base::ignore_result(token_buffer_.Extract<perf_importer::Record>(id)); return; } PERFETTO_FATAL("For GCC"); @@ -342,5 +349,4 @@ void TraceSorter::MaybeExtractEvent(size_t min_machine_idx, } } -} // namespace trace_processor -} // namespace perfetto +} // namespace perfetto::trace_processor diff --git a/src/trace_processor/sorter/trace_sorter.h b/src/trace_processor/sorter/trace_sorter.h index 5ea0e0daa..dd819204b 100644 --- a/src/trace_processor/sorter/trace_sorter.h +++ b/src/trace_processor/sorter/trace_sorter.h @@ -18,26 +18,33 @@ #define SRC_TRACE_PROCESSOR_SORTER_TRACE_SORTER_H_ #include <algorithm> +#include <cstddef> +#include <cstdint> +#include <limits> #include <memory> #include <optional> +#include <string> +#include <tuple> +#include <type_traits> #include <utility> #include <vector> +#include "perfetto/base/logging.h" #include "perfetto/ext/base/circular_queue.h" -#include "perfetto/ext/base/utils.h" #include "perfetto/public/compiler.h" -#include "perfetto/trace_processor/basic_types.h" +#include "perfetto/trace_processor/ref_counted.h" #include "perfetto/trace_processor/trace_blob_view.h" +#include "src/trace_processor/importers/common/parser_types.h" #include "src/trace_processor/importers/common/trace_parser.h" #include "src/trace_processor/importers/fuchsia/fuchsia_record.h" +#include "src/trace_processor/importers/perf/record.h" #include "src/trace_processor/importers/systrace/systrace_line.h" #include "src/trace_processor/sorter/trace_token_buffer.h" #include "src/trace_processor/storage/trace_storage.h" #include "src/trace_processor/types/trace_processor_context.h" #include "src/trace_processor/util/bump_allocator.h" -namespace perfetto { -namespace trace_processor { +namespace perfetto::trace_processor { // This class takes care of sorting events parsed from the trace stream in // arbitrary order and pushing them to the next pipeline stages (parsing) in @@ -104,7 +111,7 @@ class TraceSorter { inline void PushPerfRecord( int64_t timestamp, - TraceBlobView record, + perf_importer::Record record, std::optional<MachineId> machine_id = std::nullopt) { TraceTokenBuffer::Id id = token_buffer_.Append(std::move(record)); AppendNonFtraceEvent(timestamp, TimestampedEvent::Type::kPerfRecord, id, @@ -218,7 +225,7 @@ class TraceSorter { SortAndExtractEventsUntilAllocId(end_id); for (auto& sorter_data : sorter_data_by_machine_) { for (const auto& queue : sorter_data.queues) { - PERFETTO_DCHECK(queue.events_.empty()); + PERFETTO_CHECK(queue.events_.empty()); } sorter_data.queues.clear(); } @@ -294,13 +301,13 @@ class TraceSorter { static_assert(sizeof(TimestampedEvent) == 16, "TimestampedEvent must be equal to 16 bytes"); - static_assert(std::is_trivially_copyable<TimestampedEvent>::value, + static_assert(std::is_trivially_copyable_v<TimestampedEvent>, "TimestampedEvent must be trivially copyable"); - static_assert(std::is_trivially_move_assignable<TimestampedEvent>::value, + static_assert(std::is_trivially_move_assignable_v<TimestampedEvent>, "TimestampedEvent must be trivially move assignable"); - static_assert(std::is_trivially_move_constructible<TimestampedEvent>::value, + static_assert(std::is_trivially_move_constructible_v<TimestampedEvent>, "TimestampedEvent must be trivially move constructible"); - static_assert(std::is_nothrow_swappable<TimestampedEvent>::value, + static_assert(std::is_nothrow_swappable_v<TimestampedEvent>, "TimestampedEvent must be trivially swappable"); struct Queue { @@ -357,7 +364,7 @@ class TraceSorter { auto* queues = &sorter_data_by_machine_[0].queues; // Find the TraceSorterData instance when |machine_id| is not nullopt. - if (PERFETTO_UNLIKELY(!!machine_id)) { + if (PERFETTO_UNLIKELY(machine_id.has_value())) { auto it = std::find_if(sorter_data_by_machine_.begin() + 1, sorter_data_by_machine_.end(), [machine_id](const TraceSorterData& item) { @@ -400,7 +407,7 @@ class TraceSorter { const TimestampedEvent&); void ExtractAndDiscardTokenizedObject(const TimestampedEvent& event); - TraceTokenBuffer::Id GetTokenBufferId(const TimestampedEvent& event) { + static TraceTokenBuffer::Id GetTokenBufferId(const TimestampedEvent& event) { return TraceTokenBuffer::Id{event.alloc_id()}; } @@ -447,7 +454,6 @@ class TraceSorter { int64_t latest_pushed_event_ts_ = std::numeric_limits<int64_t>::min(); }; -} // namespace trace_processor -} // namespace perfetto +} // namespace perfetto::trace_processor #endif // SRC_TRACE_PROCESSOR_SORTER_TRACE_SORTER_H_ diff --git a/src/trace_processor/sqlite/db_sqlite_table.cc b/src/trace_processor/sqlite/db_sqlite_table.cc index 60a812206..cc4f545da 100644 --- a/src/trace_processor/sqlite/db_sqlite_table.cc +++ b/src/trace_processor/sqlite/db_sqlite_table.cc @@ -571,7 +571,7 @@ int DbSqliteModule::BestIndex(sqlite3_vtab* vtab, sqlite3_index_info* info) { // Distinct: idx_str += "D"; - if (ob_idxes.size() == 1) { + if (ob_idxes.size() == 1 && PERFETTO_POPCOUNT(info->colUsed) == 1) { switch (sqlite3_vtab_distinct(info)) { case 0: case 1: diff --git a/src/trace_processor/sqlite/module_lifecycle_manager.h b/src/trace_processor/sqlite/module_lifecycle_manager.h index 66054d599..f1f93e82b 100644 --- a/src/trace_processor/sqlite/module_lifecycle_manager.h +++ b/src/trace_processor/sqlite/module_lifecycle_manager.h @@ -67,6 +67,11 @@ class ModuleStateManager { public: // Per-vtab state. The pointer to this class should be stored in the Vtab. struct PerVtabState { + private: + // The below fields should only be accessed by the manager, use GetState to + // access the state from outside this class. + friend class ModuleStateManager<Module>; + ModuleStateManager* manager; bool disconnected = false; std::string table_name; diff --git a/src/trace_processor/storage/stats.h b/src/trace_processor/storage/stats.h index 9067b0977..057c2fa95 100644 --- a/src/trace_processor/storage/stats.h +++ b/src/trace_processor/storage/stats.h @@ -155,7 +155,13 @@ namespace stats { F(traced_buf_patches_succeeded, kIndexed, kInfo, kTrace, ""), \ F(traced_buf_readaheads_failed, kIndexed, kInfo, kTrace, ""), \ F(traced_buf_readaheads_succeeded, kIndexed, kInfo, kTrace, ""), \ - F(traced_buf_trace_writer_packet_loss, kIndexed, kDataLoss, kTrace, ""), \ + F(traced_buf_trace_writer_packet_loss, kIndexed, kDataLoss, kTrace, \ + "The tracing service observed packet loss for this buffer during this " \ + "tracing session. This also counts packet loss that happened before " \ + "the RING_BUFFER start or after the DISCARD buffer end."), \ + F(traced_buf_sequence_packet_loss, kIndexed, kDataLoss, kAnalysis, \ + "The number of groups of consecutive packets lost in each sequence for " \ + "this buffer"), \ F(traced_buf_write_wrap_count, kIndexed, kInfo, kTrace, ""), \ F(traced_chunks_discarded, kSingle, kInfo, kTrace, ""), \ F(traced_data_sources_registered, kSingle, kInfo, kTrace, ""), \ @@ -262,8 +268,13 @@ namespace stats { F(perf_process_shard_count, kIndexed, kInfo, kTrace, ""), \ F(perf_chosen_process_shard, kIndexed, kInfo, kTrace, ""), \ F(perf_guardrail_stop_ts, kIndexed, kDataLoss, kTrace, ""), \ - F(perf_samples_skipped, kSingle, kInfo, kTrace, ""), \ + F(perf_unknown_record_type, kIndexed, kInfo, kAnalysis, ""), \ + F(perf_record_skipped, kSingle, kError, kAnalysis, ""), \ + F(perf_samples_skipped, kSingle, kError, kAnalysis, ""), \ + F(perf_features_skipped, kIndexed, kInfo, kAnalysis, ""), \ F(perf_samples_skipped_dataloss, kSingle, kDataLoss, kTrace, ""), \ + F(perf_dummy_mapping_used, kSingle, kInfo, kAnalysis, ""), \ + F(perf_invalid_event_id, kSingle, kError, kTrace, ""), \ F(memory_snapshot_parser_failure, kSingle, kError, kAnalysis, ""), \ F(thread_time_in_state_out_of_order, kSingle, kError, kAnalysis, ""), \ F(thread_time_in_state_unknown_cpu_freq, \ @@ -338,13 +349,25 @@ namespace stats { F(winscope_protolog_missing_interned_stacktrace_parse_errors, \ kSingle, kInfo, kAnalysis, \ "Failed to find interned ProtoLog stacktrace."), \ + F(winscope_viewcapture_parse_errors, \ + kSingle, kInfo, kAnalysis, \ + "ViewCapture packet has unknown fields, which results in some " \ + "arguments missing. You may need a newer version of trace processor " \ + "to parse them."), \ + F(winscope_viewcapture_missing_interned_string_parse_errors, \ + kSingle, kInfo, kAnalysis, \ + "Failed to find interned ViewCapture string."), \ F(jit_unknown_frame, kSingle, kDataLoss, kTrace, \ "Indicates that we were unable to determine the function for a frame in "\ "a jitted memory region"), \ F(ftrace_missing_event_id, kSingle, kInfo, kAnalysis, \ "Indicates that the ftrace event was dropped because the event id was " \ "missing. This is an 'info' stat rather than an error stat because " \ - "this can be legitimately missing due to proto filtering.") + "this can be legitimately missing due to proto filtering."), \ + F(android_input_event_parse_errors, kSingle, kInfo, kAnalysis, \ + "Android input event packet has unknown fields, which results " \ + "in some arguments missing. You may need a newer version of trace " \ + "processor to parse them.") // clang-format on enum Type { diff --git a/src/trace_processor/storage/trace_storage.h b/src/trace_processor/storage/trace_storage.h index ff280e16c..61da6d9e5 100644 --- a/src/trace_processor/storage/trace_storage.h +++ b/src/trace_processor/storage/trace_storage.h @@ -514,6 +514,29 @@ class TraceStorage { return &android_dumpstate_table_; } + const tables::AndroidKeyEventsTable& android_key_events_table() const { + return android_key_events_table_; + } + tables::AndroidKeyEventsTable* mutable_android_key_events_table() { + return &android_key_events_table_; + } + + const tables::AndroidMotionEventsTable& android_motion_events_table() const { + return android_motion_events_table_; + } + tables::AndroidMotionEventsTable* mutable_android_motion_events_table() { + return &android_motion_events_table_; + } + + const tables::AndroidInputEventDispatchTable& + android_input_event_dispatch_table() const { + return android_input_event_dispatch_table_; + } + tables::AndroidInputEventDispatchTable* + mutable_android_input_event_dispatch_table() { + return &android_input_event_dispatch_table_; + } + const StatsMap& stats() const { return stats_; } const tables::MetadataTable& metadata_table() const { @@ -618,6 +641,13 @@ class TraceStorage { return &cpu_profile_stack_sample_table_; } + const tables::PerfSessionTable& perf_session_table() const { + return perf_session_table_; + } + tables::PerfSessionTable* mutable_perf_session_table() { + return &perf_session_table_; + } + const tables::PerfSampleTable& perf_sample_table() const { return perf_sample_table_; } @@ -845,6 +875,13 @@ class TraceStorage { return &surfaceflinger_transactions_table_; } + const tables::ViewCaptureTable& viewcapture_table() const { + return viewcapture_table_; + } + tables::ViewCaptureTable* mutable_viewcapture_table() { + return &viewcapture_table_; + } + const tables::WindowManagerShellTransitionsTable& window_manager_shell_transitions_table() const { return window_manager_shell_transitions_table_; @@ -1083,6 +1120,11 @@ class TraceStorage { tables::AndroidDumpstateTable android_dumpstate_table_{&string_pool_}; + tables::AndroidKeyEventsTable android_key_events_table_{&string_pool_}; + tables::AndroidMotionEventsTable android_motion_events_table_{&string_pool_}; + tables::AndroidInputEventDispatchTable + android_input_event_dispatch_table_{&string_pool_}; + tables::StackProfileMappingTable stack_profile_mapping_table_{&string_pool_}; tables::StackProfileFrameTable stack_profile_frame_table_{&string_pool_}; tables::StackProfileCallsiteTable stack_profile_callsite_table_{ @@ -1092,6 +1134,7 @@ class TraceStorage { &string_pool_}; tables::CpuProfileStackSampleTable cpu_profile_stack_sample_table_{ &string_pool_, &stack_sample_table_}; + tables::PerfSessionTable perf_session_table_{&string_pool_}; tables::PerfSampleTable perf_sample_table_{&string_pool_}; tables::PackageListTable package_list_table_{&string_pool_}; tables::AndroidGameInterventionListTable @@ -1147,6 +1190,7 @@ class TraceStorage { tables::SurfaceFlingerLayerTable surfaceflinger_layer_table_{&string_pool_}; tables::SurfaceFlingerTransactionsTable surfaceflinger_transactions_table_{ &string_pool_}; + tables::ViewCaptureTable viewcapture_table_{&string_pool_}; tables::WindowManagerShellTransitionsTable window_manager_shell_transitions_table_{&string_pool_}; tables::WindowManagerShellTransitionHandlersTable diff --git a/src/trace_processor/tables/android_tables.py b/src/trace_processor/tables/android_tables.py index 88f25921b..5954d5341 100644 --- a/src/trace_processor/tables/android_tables.py +++ b/src/trace_processor/tables/android_tables.py @@ -14,11 +14,11 @@ """Contains tables for relevant for Android.""" from python.generators.trace_processor_table.public import Column as C +from python.generators.trace_processor_table.public import ColumnDoc from python.generators.trace_processor_table.public import CppDouble from python.generators.trace_processor_table.public import CppInt32 from python.generators.trace_processor_table.public import CppInt64 from python.generators.trace_processor_table.public import CppOptional -from python.generators.trace_processor_table.public import CppSelfTableId from python.generators.trace_processor_table.public import CppString from python.generators.trace_processor_table.public import Table from python.generators.trace_processor_table.public import TableDoc @@ -156,9 +156,100 @@ ANDROID_DUMPSTATE_TABLE = Table( ''' })) +ANDROID_MOTION_EVENTS_TABLE = Table( + python_module=__file__, + class_name='AndroidMotionEventsTable', + sql_name='__intrinsic_android_motion_events', + columns=[ + C('event_id', CppUint32()), + C('ts', CppInt64()), + C('arg_set_id', CppUint32()), + ], + tabledoc=TableDoc( + doc='Contains Android MotionEvents processed by the system', + group='Android', + columns={ + 'event_id': + ''' + The randomly-generated ID associated with each input event processed + by Android Framework, used to track the event through the input pipeline. + ''', + 'ts': + '''The timestamp associated with the input event.''', + 'arg_set_id': + ColumnDoc( + doc='Details of the motion event parsed from the proto message.', + joinable='args.arg_set_id'), + })) + +ANDROID_KEY_EVENTS_TABLE = Table( + python_module=__file__, + class_name='AndroidKeyEventsTable', + sql_name='__intrinsic_android_key_events', + columns=[ + C('event_id', CppUint32()), + C('ts', CppInt64()), + C('arg_set_id', CppUint32()), + ], + tabledoc=TableDoc( + doc='Contains Android KeyEvents processed by the system', + group='Android', + columns={ + 'event_id': + ''' + The randomly-generated ID associated with each input event processed + by Android Framework, used to track the event through the input pipeline. + ''', + 'ts': + '''The timestamp associated with the input event.''', + 'arg_set_id': + ColumnDoc( + doc='Details of the key event parsed from the proto message.', + joinable='args.arg_set_id'), + })) + +ANDROID_INPUT_EVENT_DISPATCH_TABLE = Table( + python_module=__file__, + class_name='AndroidInputEventDispatchTable', + sql_name='__intrinsic_android_input_event_dispatch', + columns=[ + C('event_id', CppUint32()), + C('arg_set_id', CppUint32()), + C('vsync_id', CppInt64()), + C('window_id', CppInt32()), + ], + tabledoc=TableDoc( + doc= + ''' + Contains records of Android input events being dispatched to input windows + by the Android Framework. + ''', + group='Android', + columns={ + 'event_id': + ColumnDoc( + doc='The id of the input event that was dispatched.', + joinable='android_input_event.event_id'), + 'arg_set_id': + ColumnDoc( + doc='Details of the dispatched event parsed from the proto message.', + joinable='args.arg_set_id'), + 'vsync_id': + ''' + The id of the vsync during which the Framework made the decision to + dispatch this input event, used to identify the state of the input windows + when the dispatching decision was made. + ''', + 'window_id': + 'The id of the window to which the event was dispatched.', + })) + # Keep this list sorted. ALL_TABLES = [ ANDROID_LOG_TABLE, ANDROID_DUMPSTATE_TABLE, ANDROID_GAME_INTERVENTION_LIST_TABLE, + ANDROID_KEY_EVENTS_TABLE, + ANDROID_MOTION_EVENTS_TABLE, + ANDROID_INPUT_EVENT_DISPATCH_TABLE, ] diff --git a/src/trace_processor/tables/counter_tables.py b/src/trace_processor/tables/counter_tables.py index 3a9e06251..6892aec00 100644 --- a/src/trace_processor/tables/counter_tables.py +++ b/src/trace_processor/tables/counter_tables.py @@ -35,25 +35,15 @@ COUNTER_TABLE = Table( C('track_id', CppTableId(COUNTER_TRACK_TABLE)), C('value', CppDouble()), C('arg_set_id', CppOptional(CppUint32())), - C('machine_id', CppOptional(CppTableId(MACHINE_TABLE))), ], tabledoc=TableDoc( doc='''''', group='Events', columns={ - 'ts': - '''''', - 'track_id': - '''''', - 'value': - '''''', - 'arg_set_id': - '''''', - 'machine_id': - ''' - Machine identifier, non-null for counters from a remote - machine. - ''', + 'ts': '''''', + 'track_id': '''''', + 'value': '''''', + 'arg_set_id': '''''', })) # Keep this list sorted. diff --git a/src/trace_processor/tables/metadata_tables.py b/src/trace_processor/tables/metadata_tables.py index 0d91d8d7f..35e36e2d0 100644 --- a/src/trace_processor/tables/metadata_tables.py +++ b/src/trace_processor/tables/metadata_tables.py @@ -191,18 +191,46 @@ THREAD_TABLE = Table( ''', })) +CPU_TABLE = Table( + python_module=__file__, + class_name='CpuTable', + sql_name='__intrinsic_cpu', + columns=[ + C('cpu', CppOptional(CppUint32())), + C('cluster_id', CppUint32()), + C('processor', CppString()), + C('machine_id', CppOptional(CppTableId(MACHINE_TABLE))), + ], + tabledoc=TableDoc( + doc=''' + Contains information of processes seen during the trace + ''', + group='Misc', + columns={ + 'cpu': + '''the index (0-based) of the CPU core on the device''', + 'cluster_id': + '''the cluster id is shared by CPUs in +the same cluster''', + 'processor': + '''a string describing this core''', + 'machine_id': + ''' + Machine identifier, non-null for CPUs on a remote machine. + ''', + })) + RAW_TABLE = Table( python_module=__file__, class_name='RawTable', - sql_name='raw', + sql_name='__intrinsic_raw', columns=[ C('ts', CppInt64(), flags=ColumnFlag.SORTED), C('name', CppString()), - C('cpu', CppUint32()), C('utid', CppTableId(THREAD_TABLE)), C('arg_set_id', CppUint32()), C('common_flags', CppUint32()), - C('machine_id', CppOptional(CppTableId(MACHINE_TABLE))), + C('ucpu', CppTableId(CPU_TABLE)) ], tabledoc=TableDoc( doc=''' @@ -223,8 +251,6 @@ RAW_TABLE = Table( The name of the event. For ftrace events, this will be the ftrace event name. ''', - 'cpu': - 'The CPU this event was emitted on.', 'utid': 'The thread this event was emitted on.', 'common_flags': @@ -232,17 +258,16 @@ RAW_TABLE = Table( Ftrace event flags for this event. Currently only emitted for sched_waking events. ''', - 'machine_id': + 'ucpu': ''' - Machine identifier, non-null for raw events on a remote - machine. + The unique CPU indentifier. ''', })) FTRACE_EVENT_TABLE = Table( python_module=__file__, class_name='FtraceEventTable', - sql_name='ftrace_event', + sql_name='__intrinsic_ftrace_event', parent=RAW_TABLE, columns=[], tabledoc=TableDoc( @@ -357,51 +382,18 @@ EXP_MISSING_CHROME_PROC_TABLE = Table( 'reliable_from': '''''' })) -CPU_TABLE = Table( - python_module=__file__, - class_name='CpuTable', - sql_name='cpu', - columns=[ - C('cluster_id', CppUint32()), - C('processor', CppString()), - C('machine_id', CppOptional(CppTableId(MACHINE_TABLE))), - ], - tabledoc=TableDoc( - doc=''' - Contains information of processes seen during the trace - ''', - group='Misc', - columns={ - 'cluster_id': - '''the cluster id is shared by CPUs in -the same cluster''', - 'processor': - '''a string describing this core''', - 'machine_id': - ''' - Machine identifier, non-null for CPUs on a remote machine. - ''', - })) - CPU_FREQ_TABLE = Table( python_module=__file__, class_name='CpuFreqTable', - sql_name='cpu_freq', + sql_name='__intrinsic_cpu_freq', columns=[ - C('cpu_id', CppTableId(CPU_TABLE)), + C('ucpu', CppTableId(CPU_TABLE)), C('freq', CppUint32()), - C('machine_id', CppOptional(CppTableId(MACHINE_TABLE))), ], tabledoc=TableDoc( - doc='''''', - group='Misc', - columns={ - 'cpu_id': '''''', + doc='''''', group='Misc', columns={ + 'ucpu': '''''', 'freq': '''''', - 'machine_id': - ''' - Machine identifier, non-null for CPUs on a remote machine. - ''', })) CLOCK_SNAPSHOT_TABLE = Table( diff --git a/src/trace_processor/tables/profiler_tables.py b/src/trace_processor/tables/profiler_tables.py index 8e9cbd790..0f90d6553 100644 --- a/src/trace_processor/tables/profiler_tables.py +++ b/src/trace_processor/tables/profiler_tables.py @@ -25,7 +25,7 @@ from python.generators.trace_processor_table.public import TableDoc from python.generators.trace_processor_table.public import CppTableId from python.generators.trace_processor_table.public import CppUint32 -from src.trace_processor.tables.track_tables import TRACK_TABLE +from src.trace_processor.tables.track_tables import TRACK_TABLE, COUNTER_TRACK_TABLE PROFILER_SMAPS_TABLE = Table( python_module=__file__, @@ -246,6 +246,22 @@ CPU_PROFILE_STACK_SAMPLE_TABLE = Table( 'process_priority': '''''' })) +PERF_SESSION_TABLE = Table( + python_module=__file__, + class_name='PerfSessionTable', + sql_name='__intrinsic_perf_session', + columns=[ + C('cmdline', CppOptional(CppString())), + ], + tabledoc=TableDoc( + doc=''' + Perf sessions. + ''', + group='Callstack profilers', + columns={ + 'cmdline': '''Command line used to collect the data.''', + })) + PERF_SAMPLE_TABLE = Table( python_module=__file__, class_name='PerfSampleTable', @@ -257,7 +273,7 @@ PERF_SAMPLE_TABLE = Table( C('cpu_mode', CppString()), C('callsite_id', CppOptional(CppTableId(STACK_PROFILE_CALLSITE_TABLE))), C('unwind_error', CppOptional(CppString())), - C('perf_session_id', CppUint32()), + C('perf_session_id', CppTableId(PERF_SESSION_TABLE)), ], tabledoc=TableDoc( doc=''' @@ -610,6 +626,31 @@ GPU_COUNTER_GROUP_TABLE = Table( 'track_id': '''''' })) +PERF_COUNTER_TRACK_TABLE = Table( + python_module=__file__, + class_name='PerfCounterTrackTable', + sql_name='perf_counter_track', + columns=[ + C('perf_session_id', CppTableId(PERF_SESSION_TABLE)), + C('cpu', CppUint32()), + C('is_timebase', CppUint32()), + ], + parent=COUNTER_TRACK_TABLE, + tabledoc=TableDoc( + doc='Sampled counters\' values for samples in the perf_sample table.', + group='Counter Tracks', + columns={ + 'perf_session_id': + 'id of a distict profiling stream', + 'cpu': + 'the core the sample was taken on', + 'is_timebase': + ''' + If true, indicates this counter was the sampling timebase for + this perf_session_id + ''' + })) + # Keep this list sorted. ALL_TABLES = [ CPU_PROFILE_STACK_SAMPLE_TABLE, @@ -621,6 +662,7 @@ ALL_TABLES = [ HEAP_PROFILE_ALLOCATION_TABLE, PACKAGE_LIST_TABLE, PERF_SAMPLE_TABLE, + PERF_SESSION_TABLE, PROFILER_SMAPS_TABLE, STACK_PROFILE_CALLSITE_TABLE, STACK_PROFILE_FRAME_TABLE, @@ -628,4 +670,5 @@ ALL_TABLES = [ STACK_SAMPLE_TABLE, SYMBOL_TABLE, VULKAN_MEMORY_ALLOCATIONS_TABLE, + PERF_COUNTER_TRACK_TABLE, ] diff --git a/src/trace_processor/tables/sched_tables.py b/src/trace_processor/tables/sched_tables.py index 4ef4de825..35395437b 100644 --- a/src/trace_processor/tables/sched_tables.py +++ b/src/trace_processor/tables/sched_tables.py @@ -27,20 +27,19 @@ from python.generators.trace_processor_table.public import Table from python.generators.trace_processor_table.public import TableDoc from python.generators.trace_processor_table.public import WrappingSqlView -from src.trace_processor.tables.metadata_tables import MACHINE_TABLE +from src.trace_processor.tables.metadata_tables import MACHINE_TABLE, CPU_TABLE SCHED_SLICE_TABLE = Table( python_module=__file__, class_name='SchedSliceTable', - sql_name='sched_slice', + sql_name='__intrinsic_sched_slice', columns=[ C('ts', CppInt64(), flags=ColumnFlag.SORTED), C('dur', CppInt64()), - C('cpu', CppUint32()), C('utid', CppUint32()), C('end_state', CppString()), C('priority', CppInt32()), - C('machine_id', CppOptional(CppTableId(MACHINE_TABLE))), + C('ucpu', CppTableId(CPU_TABLE)), ], tabledoc=TableDoc( doc=''' @@ -58,9 +57,7 @@ SCHED_SLICE_TABLE = Table( 'dur': '''The duration of the slice (in nanoseconds).''', 'utid': - '''The thread's unique id in the trace..''', - 'cpu': - '''The CPU that the slice executed on.''', + '''The thread's unique id in the trace.''', 'end_state': ''' A string representing the scheduling state of the kernel @@ -74,10 +71,9 @@ SCHED_SLICE_TABLE = Table( ''', 'priority': '''The kernel priority that the thread ran at.''', - 'machine_id': + 'ucpu': ''' - Machine identifier, non-null for scheduling slices on a remote - machine. + The unique CPU identifier that the slice executed on. ''', })) @@ -94,18 +90,24 @@ SPURIOUS_SCHED_WAKEUP_TABLE = Table( ], tabledoc=TableDoc( doc=''' - This table contains the scheduling wakeups that occurred while a thread was - not blocked, i.e. running or runnable. Such wakeups are not tracked in the - |thread_state_table|. + This table contains the scheduling wakeups that occurred while a + thread was not blocked, i.e. running or runnable. Such wakeups are not + tracked in the |thread_state_table|. ''', group='Events', columns={ 'ts': 'The timestamp at the start of the slice (in nanoseconds).', 'thread_state_id': - 'The id of the row in the thread_state table that this row is associated with.', + ''' + The id of the row in the thread_state table that this row is + associated with. + ''', 'irq_context': - '''Whether the wakeup was from interrupt context or process context.''', + ''' + Whether the wakeup was from interrupt context or process + context. + ''', 'utid': '''The thread's unique id in the trace..''', 'waker_utid': @@ -118,11 +120,10 @@ SPURIOUS_SCHED_WAKEUP_TABLE = Table( THREAD_STATE_TABLE = Table( python_module=__file__, class_name='ThreadStateTable', - sql_name='thread_state', + sql_name='__intrinsic_thread_state', columns=[ C('ts', CppInt64(), flags=ColumnFlag.SORTED), C('dur', CppInt64()), - C('cpu', CppOptional(CppUint32())), C('utid', CppUint32()), C('state', CppString()), C('io_wait', CppOptional(CppUint32())), @@ -130,7 +131,7 @@ THREAD_STATE_TABLE = Table( C('waker_utid', CppOptional(CppUint32())), C('waker_id', CppOptional(CppSelfTableId())), C('irq_context', CppOptional(CppUint32())), - C('machine_id', CppOptional(CppTableId(MACHINE_TABLE))), + C('ucpu', CppOptional(CppTableId(CPU_TABLE))), ], tabledoc=TableDoc( doc=''' @@ -146,12 +147,8 @@ THREAD_STATE_TABLE = Table( 'The timestamp at the start of the slice (in nanoseconds).', 'dur': 'The duration of the slice (in nanoseconds).', - 'cpu': - '''The CPU that the slice executed on.''', - 'irq_context': - '''Whether the wakeup was from interrupt context or process context.''', 'utid': - '''The thread's unique id in the trace..''', + '''The thread's unique id in the trace.''', 'state': ''' The scheduling state of the thread. Can be "Running" or any @@ -168,11 +165,17 @@ THREAD_STATE_TABLE = Table( ''', 'waker_id': ''' - The unique thread state id which caused a wakeup of this thread. + The unique thread state id which caused a wakeup of this + thread. + ''', + 'irq_context': + ''' + Whether the wakeup was from interrupt context or process + context. ''', - 'machine_id': + 'ucpu': ''' - Machine identifier, non-null for threads on a remote machine. + The unique CPU identifier that the thread executed on. ''', })) diff --git a/src/trace_processor/tables/slice_tables.py b/src/trace_processor/tables/slice_tables.py index 90c81765f..1deca2e5f 100644 --- a/src/trace_processor/tables/slice_tables.py +++ b/src/trace_processor/tables/slice_tables.py @@ -135,6 +135,7 @@ GPU_SLICE_TABLE = Table( C('frame_id', CppOptional(CppUint32())), C('submission_id', CppOptional(CppUint32())), C('hw_queue_id', CppOptional(CppInt64())), + C('upid', CppOptional(CppUint32())), C('render_subpasses', CppString()), ], parent=SLICE_TABLE, @@ -142,17 +143,33 @@ GPU_SLICE_TABLE = Table( doc='''''', group='Slice', columns={ - 'context_id': '''''', - 'render_target': '''''', - 'render_target_name': '''''', - 'render_pass': '''''', - 'render_pass_name': '''''', - 'command_buffer': '''''', - 'command_buffer_name': '''''', - 'frame_id': '''''', - 'submission_id': '''''', - 'hw_queue_id': '''''', - 'render_subpasses': '''''' + 'context_id': + '''''', + 'render_target': + '''''', + 'render_target_name': + '''''', + 'render_pass': + '''''', + 'render_pass_name': + '''''', + 'command_buffer': + '''''', + 'command_buffer_name': + '''''', + 'frame_id': + '''''', + 'submission_id': + '''''', + 'hw_queue_id': + '''''', + 'upid': + ''' + Unique process id of the app that generates this gpu render + stage event. + ''', + 'render_subpasses': + '''''' })) GRAPHICS_FRAME_SLICE_TABLE = Table( diff --git a/src/trace_processor/tables/table_destructors.cc b/src/trace_processor/tables/table_destructors.cc index 6093c839d..067cefa37 100644 --- a/src/trace_processor/tables/table_destructors.cc +++ b/src/trace_processor/tables/table_destructors.cc @@ -40,6 +40,9 @@ namespace tables { AndroidDumpstateTable::~AndroidDumpstateTable() = default; AndroidGameInterventionListTable::~AndroidGameInterventionListTable() = default; AndroidLogTable::~AndroidLogTable() = default; +AndroidKeyEventsTable::~AndroidKeyEventsTable() = default; +AndroidMotionEventsTable::~AndroidMotionEventsTable() = default; +AndroidInputEventDispatchTable::~AndroidInputEventDispatchTable() = default; // counter_tables_py.h CounterTable::~CounterTable() = default; @@ -68,6 +71,7 @@ StackProfileFrameTable::~StackProfileFrameTable() = default; StackProfileCallsiteTable::~StackProfileCallsiteTable() = default; StackSampleTable::~StackSampleTable() = default; CpuProfileStackSampleTable::~CpuProfileStackSampleTable() = default; +PerfSessionTable::~PerfSessionTable() = default; PerfSampleTable::~PerfSampleTable() = default; SymbolTable::~SymbolTable() = default; HeapProfileAllocationTable::~HeapProfileAllocationTable() = default; @@ -139,15 +143,16 @@ V8RegexpCodeTable::~V8RegexpCodeTable() = default; InputMethodClientsTable::~InputMethodClientsTable() = default; InputMethodManagerServiceTable::~InputMethodManagerServiceTable() = default; InputMethodServiceTable::~InputMethodServiceTable() = default; +ProtoLogTable::~ProtoLogTable() = default; SurfaceFlingerLayersSnapshotTable::~SurfaceFlingerLayersSnapshotTable() = default; SurfaceFlingerLayerTable::~SurfaceFlingerLayerTable() = default; SurfaceFlingerTransactionsTable::~SurfaceFlingerTransactionsTable() = default; +ViewCaptureTable::~ViewCaptureTable() = default; WindowManagerShellTransitionsTable::~WindowManagerShellTransitionsTable() = default; WindowManagerShellTransitionHandlersTable:: ~WindowManagerShellTransitionHandlersTable() = default; -ProtoLogTable::~ProtoLogTable() = default; } // namespace tables diff --git a/src/trace_processor/tables/track_tables.py b/src/trace_processor/tables/track_tables.py index 4d7bc93f6..2ea7ec1d5 100644 --- a/src/trace_processor/tables/track_tables.py +++ b/src/trace_processor/tables/track_tables.py @@ -290,30 +290,6 @@ GPU_COUNTER_TRACK_TABLE = Table( group='Counter Tracks', columns={'gpu_id': 'The identifier for the GPU.'})) -PERF_COUNTER_TRACK_TABLE = Table( - python_module=__file__, - class_name='PerfCounterTrackTable', - sql_name='perf_counter_track', - columns=[ - C('perf_session_id', CppUint32()), - C('cpu', CppUint32()), - C('is_timebase', CppUint32()), - ], - parent=COUNTER_TRACK_TABLE, - tabledoc=TableDoc( - doc='Sampled counters\' values for samples in the perf_sample table.', - group='Counter Tracks', - columns={ - 'perf_session_id': - 'id of a distict profiling stream', - 'cpu': - 'the core the sample was taken on', - 'is_timebase': - ''' - If true, indicates this counter was the sampling timebase for - this perf_session_id - ''' - })) ENERGY_COUNTER_TRACK_TABLE = Table( python_module=__file__, @@ -393,7 +369,6 @@ ALL_TABLES = [ GPU_WORK_PERIOD_TRACK_TABLE, IRQ_COUNTER_TRACK_TABLE, LINUX_DEVICE_TRACK_TABLE, - PERF_COUNTER_TRACK_TABLE, PROCESS_COUNTER_TRACK_TABLE, PROCESS_TRACK_TABLE, SOFTIRQ_COUNTER_TRACK_TABLE, diff --git a/src/trace_processor/tables/winscope_tables.py b/src/trace_processor/tables/winscope_tables.py index 8c058aa18..15b93bb1f 100644 --- a/src/trace_processor/tables/winscope_tables.py +++ b/src/trace_processor/tables/winscope_tables.py @@ -117,6 +117,22 @@ SURFACE_FLINGER_TRANSACTIONS_TABLE = Table( 'arg_set_id': 'Extra args parsed from the proto message', })) +VIEWCAPTURE_TABLE = Table( + python_module=__file__, + class_name='ViewCaptureTable', + sql_name='__intrinsic_viewcapture', + columns=[ + C('ts', CppInt64()), + C('arg_set_id', CppUint32()), + ], + tabledoc=TableDoc( + doc='ViewCapture', + group='Winscope', + columns={ + 'ts': 'The timestamp the views were captured', + 'arg_set_id': 'Extra args parsed from the proto message', + })) + WINDOW_MANAGER_SHELL_TRANSITIONS_TABLE = Table( python_module=__file__, class_name='WindowManagerShellTransitionsTable', @@ -182,6 +198,7 @@ ALL_TABLES = [ SURFACE_FLINGER_LAYERS_SNAPSHOT_TABLE, SURFACE_FLINGER_LAYER_TABLE, SURFACE_FLINGER_TRANSACTIONS_TABLE, + VIEWCAPTURE_TABLE, WINDOW_MANAGER_SHELL_TRANSITIONS_TABLE, WINDOW_MANAGER_SHELL_TRANSITION_HANDLERS_TABLE, ] diff --git a/src/trace_processor/trace_database_integrationtest.cc b/src/trace_processor/trace_database_integrationtest.cc index d09c8a7b4..731f4e375 100644 --- a/src/trace_processor/trace_database_integrationtest.cc +++ b/src/trace_processor/trace_database_integrationtest.cc @@ -14,17 +14,23 @@ * limitations under the License. */ -#include <algorithm> +#include <cstdint> #include <cstdio> -#include <map> -#include <optional> +#include <cstring> +#include <memory> #include <random> #include <string> +#include <utility> +#include <vector> +#include "perfetto/base/build_config.h" #include "perfetto/base/logging.h" +#include "perfetto/base/status.h" #include "perfetto/ext/base/scoped_file.h" #include "perfetto/ext/base/string_utils.h" #include "perfetto/trace_processor/basic_types.h" +#include "perfetto/trace_processor/iterator.h" +#include "perfetto/trace_processor/status.h" #include "perfetto/trace_processor/trace_processor.h" #include "protos/perfetto/common/descriptor.pbzero.h" #include "protos/perfetto/trace_processor/trace_processor.pbzero.h" @@ -32,11 +38,12 @@ #include "src/base/test/utils.h" #include "test/gtest_and_gmock.h" -namespace perfetto { -namespace trace_processor { +namespace perfetto::trace_processor { namespace { -constexpr size_t kMaxChunkSize = 4 * 1024 * 1024; +using testing::HasSubstr; + +constexpr size_t kMaxChunkSize = 4ul * 1024 * 1024; TEST(TraceProcessorCustomConfigTest, SkipInternalMetricsMatchingMountPath) { auto config = Config(); @@ -97,12 +104,13 @@ class TraceProcessorIntegrationTest : public ::testing::Test { : processor_(TraceProcessor::CreateInstance(Config())) {} protected: - util::Status LoadTrace(const char* name, + base::Status LoadTrace(const char* name, size_t min_chunk_size = 512, size_t max_chunk_size = kMaxChunkSize) { EXPECT_LE(min_chunk_size, max_chunk_size); - base::ScopedFstream f(fopen( - base::GetTestDataPath(std::string("test/data/") + name).c_str(), "rb")); + base::ScopedFstream f( + fopen(base::GetTestDataPath(std::string("test/data/") + name).c_str(), + "rbe")); std::minstd_rand0 rnd_engine(0); std::uniform_int_distribution<size_t> dist(min_chunk_size, max_chunk_size); while (!feof(*f)) { @@ -118,7 +126,7 @@ class TraceProcessorIntegrationTest : public ::testing::Test { } Iterator Query(const std::string& query) { - return processor_->ExecuteQuery(query.c_str()); + return processor_->ExecuteQuery(query); } TraceProcessor* Processor() { return processor_.get(); } @@ -280,7 +288,7 @@ TEST_F(TraceProcessorIntegrationTest, SerializeMetricDescriptors) { TEST_F(TraceProcessorIntegrationTest, ComputeMetricsFormattedExtension) { std::string metric_output; - util::Status status = Processor()->ComputeMetricText( + base::Status status = Processor()->ComputeMetricText( std::vector<std::string>{"test_chrome_metric"}, TraceProcessor::MetricResultFormat::kProtoText, &metric_output); ASSERT_TRUE(status.ok()); @@ -293,7 +301,7 @@ TEST_F(TraceProcessorIntegrationTest, ComputeMetricsFormattedExtension) { TEST_F(TraceProcessorIntegrationTest, ComputeMetricsFormattedNoExtension) { std::string metric_output; - util::Status status = Processor()->ComputeMetricText( + base::Status status = Processor()->ComputeMetricText( std::vector<std::string>{"trace_metadata"}, TraceProcessor::MetricResultFormat::kProtoText, &metric_output); ASSERT_TRUE(status.ok()); @@ -321,21 +329,21 @@ TEST_F(TraceProcessorIntegrationTest, Clusterfuzz14753) { } TEST_F(TraceProcessorIntegrationTest, Clusterfuzz14762) { - ASSERT_TRUE(LoadTrace("clusterfuzz_14762", 4096 * 1024).ok()); + ASSERT_TRUE(LoadTrace("clusterfuzz_14762", 4096ul * 1024).ok()); auto it = Query("select sum(value) from stats where severity = 'error';"); ASSERT_TRUE(it.Next()); ASSERT_GT(it.Get(0).long_value, 0); } TEST_F(TraceProcessorIntegrationTest, Clusterfuzz14767) { - ASSERT_TRUE(LoadTrace("clusterfuzz_14767", 4096 * 1024).ok()); + ASSERT_TRUE(LoadTrace("clusterfuzz_14767", 4096ul * 1024).ok()); auto it = Query("select sum(value) from stats where severity = 'error';"); ASSERT_TRUE(it.Next()); ASSERT_GT(it.Get(0).long_value, 0); } TEST_F(TraceProcessorIntegrationTest, Clusterfuzz14799) { - ASSERT_TRUE(LoadTrace("clusterfuzz_14799", 4096 * 1024).ok()); + ASSERT_TRUE(LoadTrace("clusterfuzz_14799", 4096ul * 1024).ok()); auto it = Query("select sum(value) from stats where severity = 'error';"); ASSERT_TRUE(it.Next()); ASSERT_GT(it.Get(0).long_value, 0); @@ -793,6 +801,15 @@ TEST_F(TraceProcessorIntegrationTest, FunctionRegistrationError) { ASSERT_TRUE(it.Status().ok()); } +TEST_F(TraceProcessorIntegrationTest, CreateTableDuplicateNames) { + auto it = Query( + "create perfetto table foo select 1 as duplicate_a, 2 as duplicate_a, 3 " + "as duplicate_b, 4 as duplicate_b"); + ASSERT_FALSE(it.Next()); + ASSERT_FALSE(it.Status().ok()); + ASSERT_THAT(it.Status().message(), HasSubstr("duplicate_a")); + ASSERT_THAT(it.Status().message(), HasSubstr("duplicate_b")); +} + } // namespace -} // namespace trace_processor -} // namespace perfetto +} // namespace perfetto::trace_processor diff --git a/src/trace_processor/trace_processor_context.cc b/src/trace_processor/trace_processor_context.cc index 2299a1b79..cdf250c30 100644 --- a/src/trace_processor/trace_processor_context.cc +++ b/src/trace_processor/trace_processor_context.cc @@ -24,6 +24,7 @@ #include "src/trace_processor/importers/common/chunked_trace_reader.h" #include "src/trace_processor/importers/common/clock_converter.h" #include "src/trace_processor/importers/common/clock_tracker.h" +#include "src/trace_processor/importers/common/cpu_tracker.h" #include "src/trace_processor/importers/common/deobfuscation_mapping_table.h" #include "src/trace_processor/importers/common/event_tracker.h" #include "src/trace_processor/importers/common/flow_tracker.h" @@ -31,6 +32,7 @@ #include "src/trace_processor/importers/common/machine_tracker.h" #include "src/trace_processor/importers/common/mapping_tracker.h" #include "src/trace_processor/importers/common/metadata_tracker.h" +#include "src/trace_processor/importers/common/process_track_translation_table.h" #include "src/trace_processor/importers/common/process_tracker.h" #include "src/trace_processor/importers/common/sched_event_tracker.h" #include "src/trace_processor/importers/common/slice_tracker.h" @@ -45,6 +47,7 @@ #include "src/trace_processor/importers/proto/track_event.descriptor.h" #include "src/trace_processor/importers/proto/track_event_module.h" #include "src/trace_processor/sorter/trace_sorter.h" +#include "src/trace_processor/trace_reader_registry.h" #include "src/trace_processor/types/destructible.h" namespace perfetto { @@ -52,6 +55,7 @@ namespace trace_processor { TraceProcessorContext::TraceProcessorContext(const InitArgs& args) : config(args.config), storage(args.storage) { + reader_registry = std::make_unique<TraceReaderRegistry>(this); // Init the trackers. machine_tracker.reset(new MachineTracker(this, args.raw_machine_id)); if (!machine_id()) { @@ -67,12 +71,15 @@ TraceProcessorContext::TraceProcessorContext(const InitArgs& args) event_tracker.reset(new EventTracker(this)); sched_event_tracker.reset(new SchedEventTracker(this)); process_tracker.reset(new ProcessTracker(this)); + process_track_translation_table.reset( + new ProcessTrackTranslationTable(storage.get())); clock_tracker.reset(new ClockTracker(this)); clock_converter.reset(new ClockConverter(this)); mapping_tracker.reset(new MappingTracker(this)); perf_sample_tracker.reset(new PerfSampleTracker(this)); stack_profile_tracker.reset(new StackProfileTracker(this)); metadata_tracker.reset(new MetadataTracker(storage.get())); + cpu_tracker.reset(new CpuTracker(this)); global_args_tracker.reset(new GlobalArgsTracker(storage.get())); { descriptor_pool_.reset(new DescriptorPool()); diff --git a/src/trace_processor/trace_processor_impl.cc b/src/trace_processor/trace_processor_impl.cc index 7bba0e9fe..00f87bf31 100644 --- a/src/trace_processor/trace_processor_impl.cc +++ b/src/trace_processor/trace_processor_impl.cc @@ -46,6 +46,7 @@ #include "src/trace_processor/importers/android_bugreport/android_bugreport_parser.h" #include "src/trace_processor/importers/common/clock_tracker.h" #include "src/trace_processor/importers/common/metadata_tracker.h" +#include "src/trace_processor/importers/common/trace_parser.h" #include "src/trace_processor/importers/fuchsia/fuchsia_trace_parser.h" #include "src/trace_processor/importers/fuchsia/fuchsia_trace_tokenizer.h" #include "src/trace_processor/importers/gzip/gzip_trace_parser.h" @@ -53,11 +54,12 @@ #include "src/trace_processor/importers/json/json_trace_tokenizer.h" #include "src/trace_processor/importers/json/json_utils.h" #include "src/trace_processor/importers/ninja/ninja_log_parser.h" -#include "src/trace_processor/importers/perf/perf_data_parser.h" #include "src/trace_processor/importers/perf/perf_data_tokenizer.h" +#include "src/trace_processor/importers/perf/record_parser.h" #include "src/trace_processor/importers/proto/additional_modules.h" #include "src/trace_processor/importers/proto/content_analyzer.h" #include "src/trace_processor/importers/systrace/systrace_trace_parser.h" +#include "src/trace_processor/importers/zip/zip_trace_reader.h" #include "src/trace_processor/iterator_impl.h" #include "src/trace_processor/metrics/all_chrome_metrics.descriptor.h" #include "src/trace_processor/metrics/all_webview_metrics.descriptor.h" @@ -110,6 +112,7 @@ #include "src/trace_processor/storage/trace_storage.h" #include "src/trace_processor/tp_metatrace.h" #include "src/trace_processor/trace_processor_storage_impl.h" +#include "src/trace_processor/trace_reader_registry.h" #include "src/trace_processor/types/trace_processor_context.h" #include "src/trace_processor/types/variadic.h" #include "src/trace_processor/util/descriptors.h" @@ -119,6 +122,7 @@ #include "src/trace_processor/util/regex.h" #include "src/trace_processor/util/sql_modules.h" #include "src/trace_processor/util/status_macros.h" +#include "src/trace_processor/util/trace_type.h" #include "protos/perfetto/common/builtin_clock.pbzero.h" #include "protos/perfetto/trace/clock_snapshot.pbzero.h" @@ -306,8 +310,8 @@ const char* TraceTypeToString(TraceType trace_type) { return "ctrace"; case kNinjaLogTraceType: return "ninja_log"; - case kAndroidBugreportTraceType: - return "android_bugreport"; + case kZipFile: + return "zip"; case kPerfDataTraceType: return "perf_data"; } @@ -339,27 +343,33 @@ void InitializePreludeTablesViews(sqlite3* db) { TraceProcessorImpl::TraceProcessorImpl(const Config& cfg) : TraceProcessorStorageImpl(cfg), config_(cfg) { - context_.fuchsia_trace_tokenizer = - std::make_unique<FuchsiaTraceTokenizer>(&context_); + context_.reader_registry->RegisterTraceReader<FuchsiaTraceTokenizer>( + kFuchsiaTraceType); context_.fuchsia_record_parser = std::make_unique<FuchsiaTraceParser>(&context_); - context_.ninja_log_parser = std::make_unique<NinjaLogParser>(&context_); - context_.systrace_trace_parser = - std::make_unique<SystraceTraceParser>(&context_); - context_.perf_data_trace_tokenizer = - std::make_unique<perf_importer::PerfDataTokenizer>(&context_); + + context_.reader_registry->RegisterTraceReader<SystraceTraceParser>( + kSystraceTraceType); + context_.reader_registry->RegisterTraceReader<NinjaLogParser>( + kNinjaLogTraceType); + + context_.reader_registry + ->RegisterTraceReader<perf_importer::PerfDataTokenizer>( + kPerfDataTraceType); context_.perf_record_parser = - std::make_unique<perf_importer::PerfDataParser>(&context_); + std::make_unique<perf_importer::RecordParser>(&context_); if (util::IsGzipSupported()) { - context_.gzip_trace_parser = std::make_unique<GzipTraceParser>(&context_); - context_.android_bugreport_parser = - std::make_unique<AndroidBugreportParser>(&context_); + context_.reader_registry->RegisterTraceReader<GzipTraceParser>( + kGzipTraceType); + context_.reader_registry->RegisterTraceReader<GzipTraceParser>( + kCtraceTraceType); + context_.reader_registry->RegisterTraceReader<ZipTraceReader>(kZipFile); } if (json::IsJsonSupported()) { - context_.json_trace_tokenizer = - std::make_unique<JsonTraceTokenizer>(&context_); + context_.reader_registry->RegisterTraceReader<JsonTraceTokenizer>( + kJsonTraceType); context_.json_trace_parser = std::make_unique<JsonTraceParserImpl>(&context_); } @@ -798,6 +808,7 @@ void TraceProcessorImpl::InitPerfettoSqlEngine() { // Note: if adding a table here which might potentially contain many rows // (O(rows in sched/slice/counter)), then consider calling ShrinkToFit on // that table in TraceStorage::ShrinkToFitTables. + RegisterStaticTable(storage->machine_table()); RegisterStaticTable(storage->arg_table()); RegisterStaticTable(storage->raw_table()); RegisterStaticTable(storage->ftrace_event_table()); @@ -843,6 +854,7 @@ void TraceProcessorImpl::InitPerfettoSqlEngine() { RegisterStaticTable(storage->symbol_table()); RegisterStaticTable(storage->heap_profile_allocation_table()); RegisterStaticTable(storage->cpu_profile_stack_sample_table()); + RegisterStaticTable(storage->perf_session_table()); RegisterStaticTable(storage->perf_sample_table()); RegisterStaticTable(storage->stack_profile_callsite_table()); RegisterStaticTable(storage->stack_profile_mapping_table()); @@ -853,6 +865,9 @@ void TraceProcessorImpl::InitPerfettoSqlEngine() { RegisterStaticTable(storage->android_log_table()); RegisterStaticTable(storage->android_dumpstate_table()); RegisterStaticTable(storage->android_game_intervention_list_table()); + RegisterStaticTable(storage->android_key_events_table()); + RegisterStaticTable(storage->android_motion_events_table()); + RegisterStaticTable(storage->android_input_event_dispatch_table()); RegisterStaticTable(storage->vulkan_memory_allocations_table()); @@ -881,6 +896,8 @@ void TraceProcessorImpl::InitPerfettoSqlEngine() { RegisterStaticTable(storage->surfaceflinger_layer_table()); RegisterStaticTable(storage->surfaceflinger_transactions_table()); + RegisterStaticTable(storage->viewcapture_table()); + RegisterStaticTable(storage->window_manager_shell_transitions_table()); RegisterStaticTable( storage->window_manager_shell_transition_handlers_table()); diff --git a/src/trace_processor/trace_processor_storage_impl.cc b/src/trace_processor/trace_processor_storage_impl.cc index 3a686e889..09d53869c 100644 --- a/src/trace_processor/trace_processor_storage_impl.cc +++ b/src/trace_processor/trace_processor_storage_impl.cc @@ -30,12 +30,14 @@ #include "src/trace_processor/importers/common/machine_tracker.h" #include "src/trace_processor/importers/common/mapping_tracker.h" #include "src/trace_processor/importers/common/metadata_tracker.h" +#include "src/trace_processor/importers/common/process_track_translation_table.h" #include "src/trace_processor/importers/common/process_tracker.h" #include "src/trace_processor/importers/common/sched_event_tracker.h" #include "src/trace_processor/importers/common/slice_tracker.h" #include "src/trace_processor/importers/common/slice_translation_table.h" #include "src/trace_processor/importers/common/stack_profile_tracker.h" #include "src/trace_processor/importers/common/track_tracker.h" +#include "src/trace_processor/importers/perf/dso_tracker.h" #include "src/trace_processor/importers/proto/chrome_track_event.descriptor.h" #include "src/trace_processor/importers/proto/default_modules.h" #include "src/trace_processor/importers/proto/packet_analyzer.h" @@ -45,6 +47,7 @@ #include "src/trace_processor/importers/proto/proto_trace_reader.h" #include "src/trace_processor/importers/proto/track_event.descriptor.h" #include "src/trace_processor/sorter/trace_sorter.h" +#include "src/trace_processor/trace_reader_registry.h" #include "src/trace_processor/util/descriptors.h" namespace perfetto { @@ -52,6 +55,8 @@ namespace trace_processor { TraceProcessorStorageImpl::TraceProcessorStorageImpl(const Config& cfg) : context_({cfg, std::make_shared<TraceStorage>(cfg)}) { + context_.reader_registry->RegisterTraceReader<ProtoTraceReader>( + kProtoTraceType); context_.proto_trace_parser = std::make_unique<ProtoTraceParserImpl>(&context_); RegisterDefaultModules(&context_); @@ -102,6 +107,8 @@ void TraceProcessorStorageImpl::NotifyEndOfFile() { return; Flush(); context_.chunk_reader->NotifyEndOfFile(); + // NotifyEndOfFile might have pushed packets to the sorter. + Flush(); for (std::unique_ptr<ProtoImporterModule>& module : context_.modules) { module->NotifyEndOfFile(); } @@ -112,6 +119,9 @@ void TraceProcessorStorageImpl::NotifyEndOfFile() { context_.slice_tracker->FlushPendingSlices(); context_.args_tracker->Flush(); context_.process_tracker->NotifyEndOfFile(); + if (context_.perf_dso_tracker) { + perf_importer::DsoTracker::GetOrCreate(&context_).SymbolizeFrames(); + } } void TraceProcessorStorageImpl::DestroyContext() { diff --git a/src/trace_processor/trace_reader_registry.cc b/src/trace_processor/trace_reader_registry.cc new file mode 100644 index 000000000..065f6f651 --- /dev/null +++ b/src/trace_processor/trace_reader_registry.cc @@ -0,0 +1,68 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/trace_processor/trace_reader_registry.h" +#include "perfetto/base/logging.h" +#include "src/trace_processor/importers/common/chunked_trace_reader.h" +#include "src/trace_processor/types/trace_processor_context.h" +#include "src/trace_processor/util/gzip_utils.h" +#include "src/trace_processor/util/trace_type.h" + +namespace perfetto::trace_processor { +namespace { +const char kNoZlibErr[] = + "Cannot open compressed trace. zlib not enabled in the build config"; + +bool RequiresZlibSupport(TraceType type) { + switch (type) { + case kGzipTraceType: + case kCtraceTraceType: + case kZipFile: + return true; + + case kNinjaLogTraceType: + case kSystraceTraceType: + case kPerfDataTraceType: + case kUnknownTraceType: + case kJsonTraceType: + case kFuchsiaTraceType: + case kProtoTraceType: + return false; + } + PERFETTO_FATAL("For GCC"); +} +} // namespace + +void TraceReaderRegistry::RegisterFactory(TraceType trace_type, + Factory factory) { + PERFETTO_CHECK(factories_.Insert(trace_type, std::move(factory)).second); +} + +base::StatusOr<std::unique_ptr<ChunkedTraceReader>> +TraceReaderRegistry::CreateTraceReader(TraceType type) { + if (auto it = factories_.Find(type); it) { + return (*it)(context_); + } + + if (RequiresZlibSupport(type) && !util::IsGzipSupported()) { + return base::ErrStatus("%s support is disabled. %s", ToString(type), + kNoZlibErr); + } + + return base::ErrStatus("%s support is disabled", ToString(type)); +} + +} // namespace perfetto::trace_processor diff --git a/src/trace_processor/trace_reader_registry.h b/src/trace_processor/trace_reader_registry.h new file mode 100644 index 000000000..8e9b0397a --- /dev/null +++ b/src/trace_processor/trace_reader_registry.h @@ -0,0 +1,74 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SRC_TRACE_PROCESSOR_TRACE_READER_REGISTRY_H_ +#define SRC_TRACE_PROCESSOR_TRACE_READER_REGISTRY_H_ + +#include <functional> +#include <memory> +#include <optional> + +#include "perfetto/ext/base/flat_hash_map.h" + +#include "perfetto/ext/base/status_or.h" +#include "src/trace_processor/importers/common/trace_parser.h" +#include "src/trace_processor/sorter/trace_sorter.h" +#include "src/trace_processor/util/trace_type.h" + +namespace perfetto { +namespace trace_processor { + +class ChunkedTraceReader; +class TraceProcessorContext; + +// Maps `TraceType` values to `ChunkedTraceReader` subclasses. +// This class is used to create `ChunkedTraceReader` instances for a given +// `TraceType`. +class TraceReaderRegistry { + public: + explicit TraceReaderRegistry(TraceProcessorContext* context) + : context_(context) {} + + // Registers a mapping from `TraceType` value to `ChunkedTraceReader` + // subclass. Only one such mapping can be registered per `TraceType` value. + template <typename Reader> + void RegisterTraceReader(TraceType trace_type) { + RegisterFactory(trace_type, [](TraceProcessorContext* ctxt) { + return std::make_unique<Reader>(ctxt); + }); + } + + // Creates a new `ChunkedTraceReader` instance for the given `type`. Returns + // an error if no mapping has been previously registered. + base::StatusOr<std::unique_ptr<ChunkedTraceReader>> CreateTraceReader( + TraceType type); + + private: + using Factory = std::function<std::unique_ptr<ChunkedTraceReader>( + TraceProcessorContext*)>; + void RegisterFactory(TraceType trace_type, Factory factory); + + TraceProcessorContext* const context_; + base::FlatHashMap<TraceType, + std::function<std::unique_ptr<ChunkedTraceReader>( + TraceProcessorContext*)>> + factories_; +}; + +} // namespace trace_processor +} // namespace perfetto + +#endif // SRC_TRACE_PROCESSOR_TRACE_READER_REGISTRY_H_ diff --git a/src/trace_processor/types/BUILD.gn b/src/trace_processor/types/BUILD.gn index 6b66c5386..9fbff7a77 100644 --- a/src/trace_processor/types/BUILD.gn +++ b/src/trace_processor/types/BUILD.gn @@ -32,6 +32,7 @@ source_set("types") { "../../../include/perfetto/trace_processor", "../containers", "../tables:tables_python", + "../util:trace_type", ] } diff --git a/src/trace_processor/types/trace_processor_context.h b/src/trace_processor/types/trace_processor_context.h index 5a91d78b4..3ec964f9b 100644 --- a/src/trace_processor/types/trace_processor_context.h +++ b/src/trace_processor/types/trace_processor_context.h @@ -23,30 +23,18 @@ #include "perfetto/trace_processor/basic_types.h" #include "src/trace_processor/tables/metadata_tables_py.h" #include "src/trace_processor/types/destructible.h" +#include "src/trace_processor/util/trace_type.h" namespace perfetto { namespace trace_processor { -enum TraceType { - kUnknownTraceType, - kProtoTraceType, - kJsonTraceType, - kFuchsiaTraceType, - kSystraceTraceType, - kGzipTraceType, - kCtraceTraceType, - kNinjaLogTraceType, - kAndroidBugreportTraceType, - kPerfDataTraceType, -}; - -class AndroidProbesTracker; class ArgsTracker; class ArgsTranslationTable; class AsyncTrackSetTracker; class ChunkedTraceReader; class ClockConverter; class ClockTracker; +class CpuTracker; class DeobfuscationMappingTable; class DescriptorPool; class EtwModule; @@ -66,12 +54,14 @@ class PacketAnalyzer; class PerfRecordParser; class PerfSampleTracker; class ProcessTracker; +class ProcessTrackTranslationTable; class ProtoImporterModule; class ProtoTraceParser; class SchedEventTracker; class SliceTracker; class SliceTranslationTable; class StackProfileTracker; +class TraceReaderRegistry; class TraceSorter; class TraceStorage; class TrackEventModule; @@ -99,6 +89,8 @@ class TraceProcessorContext { // |storage| is shared among multiple contexts in multi-machine tracing. std::shared_ptr<TraceStorage> storage; + std::unique_ptr<TraceReaderRegistry> reader_registry; + std::unique_ptr<ChunkedTraceReader> chunk_reader; // The sorter is used to sort trace data by timestamp and is shared among @@ -118,6 +110,7 @@ class TraceProcessorContext { std::unique_ptr<SliceTranslationTable> slice_translation_table; std::unique_ptr<FlowTracker> flow_tracker; std::unique_ptr<ProcessTracker> process_tracker; + std::unique_ptr<ProcessTrackTranslationTable> process_track_translation_table; std::unique_ptr<EventTracker> event_tracker; std::unique_ptr<SchedEventTracker> sched_event_tracker; std::unique_ptr<ClockTracker> clock_tracker; @@ -127,6 +120,7 @@ class TraceProcessorContext { std::unique_ptr<PerfSampleTracker> perf_sample_tracker; std::unique_ptr<StackProfileTracker> stack_profile_tracker; std::unique_ptr<MetadataTracker> metadata_tracker; + std::unique_ptr<CpuTracker> cpu_tracker; // These fields are stored as pointers to Destructible objects rather than // their actual type (a subclass of Destructible), as the concrete subclass @@ -151,19 +145,9 @@ class TraceProcessorContext { std::unique_ptr<Destructible> ftrace_sched_tracker; // FtraceSchedEventTracker std::unique_ptr<Destructible> v8_tracker; // V8Tracker std::unique_ptr<Destructible> jit_tracker; // JitTracker + std::unique_ptr<Destructible> perf_dso_tracker; // DsoTracker // clang-format on - // These fields are trace readers which will be called by |forwarding_parser| - // once the format of the trace is discovered. They are placed here as they - // are only available in the lib target. - std::unique_ptr<ChunkedTraceReader> json_trace_tokenizer; - std::unique_ptr<ChunkedTraceReader> fuchsia_trace_tokenizer; - std::unique_ptr<ChunkedTraceReader> ninja_log_parser; - std::unique_ptr<ChunkedTraceReader> android_bugreport_parser; - std::unique_ptr<ChunkedTraceReader> systrace_trace_parser; - std::unique_ptr<ChunkedTraceReader> gzip_trace_parser; - std::unique_ptr<ChunkedTraceReader> perf_data_trace_tokenizer; - std::unique_ptr<ProtoTraceParser> proto_trace_parser; // These fields are trace parsers which will be called by |forwarding_parser| diff --git a/src/trace_processor/util/BUILD.gn b/src/trace_processor/util/BUILD.gn index 5ca39a1d0..6c40069c7 100644 --- a/src/trace_processor/util/BUILD.gn +++ b/src/trace_processor/util/BUILD.gn @@ -266,6 +266,17 @@ source_set("file_buffer") { ] } +source_set("trace_type") { + sources = [ + "trace_type.cc", + "trace_type.h", + ] + deps = [ + "../../../gn:default_deps", + "../../../include/perfetto/ext/base", + ] +} + source_set("unittests") { sources = [ "bump_allocator_unittest.cc", diff --git a/src/trace_processor/util/file_buffer.cc b/src/trace_processor/util/file_buffer.cc index aa9234814..1dbd10b7d 100644 --- a/src/trace_processor/util/file_buffer.cc +++ b/src/trace_processor/util/file_buffer.cc @@ -39,10 +39,11 @@ void FileBuffer::PushBack(TraceBlobView data) { end_offset_ += size; } -bool FileBuffer::PopFrontBytesUntil(const size_t target_offset) { +bool FileBuffer::PopFrontUntil(const size_t target_offset) { + PERFETTO_CHECK(file_offset() <= target_offset); while (!data_.empty()) { Entry& entry = data_.front(); - if (target_offset <= entry.file_offset) { + if (target_offset == entry.file_offset) { return true; } const size_t bytes_to_pop = target_offset - entry.file_offset; @@ -110,9 +111,8 @@ FileBuffer::Iterator FileBuffer::FindEntryWithOffset(size_t offset) const { auto it = std::upper_bound( data_.begin(), data_.end(), offset, [](size_t offset, const Entry& rhs) { return offset < rhs.file_offset; }); - if (it == data_.begin()) { - return end(); - } + // This can only happen if too much data was popped. + PERFETTO_CHECK(it != data_.begin()); return std::prev(it); } diff --git a/src/trace_processor/util/file_buffer.h b/src/trace_processor/util/file_buffer.h index 07de20fba..35e73844d 100644 --- a/src/trace_processor/util/file_buffer.h +++ b/src/trace_processor/util/file_buffer.h @@ -39,6 +39,8 @@ class FileBuffer { // Trivial empty ctor. FileBuffer() = default; + bool empty() const { return data_.empty(); } + // Returns the offset to the start of the buffered window of data. size_t file_offset() const { return data_.empty() ? end_offset_ : data_.front().file_offset; @@ -47,10 +49,19 @@ class FileBuffer { // Adds a `TraceBlobView` at the back. void PushBack(TraceBlobView view); - // Shrinks the buffer by dropping bytes from the front of the buffer until the + // Shrinks the buffer by dropping data from the front of the buffer until the // given offset is reached. If not enough data is present as much data as // possible will be dropped and `false` will be returned. - bool PopFrontBytesUntil(size_t offset); + // ATTENTION: If `offset` < 'file_offset()' (i.e. you try to access data + // previously popped) this method will CHECK fail. + bool PopFrontUntil(size_t offset); + + // Shrinks the buffer by dropping `bytes` from the front of the buffer. If not + // enough data is present as much data as possible will be dropped and `false` + // will be returned. + bool PopFrontBytes(size_t bytes) { + return PopFrontUntil(file_offset() + bytes); + } // Similar to `TraceBlobView::slice_off`, creates a slice with data starting // at `offset` and of the given `length`. This method might need to allocate a @@ -58,8 +69,8 @@ class FileBuffer { // TraceBlobView instances). If not enough data is present `std::nullopt` is // returned. // - // ATTENTION: If `offset` < 'file_offset()' this method will never return a - // value. + // ATTENTION: If `offset` < 'file_offset()' (i.e. you try to access data + // previously popped) this method will CHECK fail. std::optional<TraceBlobView> SliceOff(size_t offset, size_t length) const; private: diff --git a/src/trace_processor/util/file_buffer_unittest.cc b/src/trace_processor/util/file_buffer_unittest.cc index 418cf76f8..1518fe481 100644 --- a/src/trace_processor/util/file_buffer_unittest.cc +++ b/src/trace_processor/util/file_buffer_unittest.cc @@ -16,6 +16,7 @@ #include "src/trace_processor/util/file_buffer.h" +#include <algorithm> #include <cstddef> #include <cstdint> #include <cstring> @@ -43,11 +44,9 @@ class SameDataAsMatcher { : expected_data_(expected_data) {} bool MatchAndExplain(const ArgType& arg, ::testing ::MatchResultListener*) const override { - if (expected_data_.size() != arg.size()) { - return false; - } - return memcmp(expected_data_.data(), arg.data(), expected_data_.size()) == - 0; + return std::equal(expected_data_.data(), + expected_data_.data() + expected_data_.size(), + arg.data(), arg.data() + arg.size()); } void DescribeTo(::std ::ostream*) const override {} void DescribeNegationTo(::std ::ostream*) const override {} @@ -108,7 +107,7 @@ TEST(FileBuffer, ContiguousAccessAtOffset) { FileBuffer buffer = CreateFileBuffer(Slice(expected_data, kChunkSize)); for (size_t file_offset = 0; file_offset <= kExpectedSize; ++file_offset) { - EXPECT_TRUE(buffer.PopFrontBytesUntil(file_offset)); + EXPECT_TRUE(buffer.PopFrontUntil(file_offset)); for (size_t off = file_offset; off <= kExpectedSize; ++off) { auto expected = expected_data.slice_off(off, kExpectedSize - off); std::optional<TraceBlobView> tbv = buffer.SliceOff(off, expected.size()); @@ -143,18 +142,16 @@ TEST(FileBuffer, PopRemovesData) { --expected_size; ++expected_file_offset; - buffer.PopFrontBytesUntil(expected_file_offset); + buffer.PopFrontUntil(expected_file_offset); EXPECT_THAT(buffer.file_offset(), Eq(expected_file_offset)); - EXPECT_THAT(buffer.SliceOff(expected_file_offset - 1, 1), Eq(std::nullopt)); EXPECT_THAT(buffer.SliceOff(expected_file_offset, expected_size), Optional(SameDataAs(expected_data.slice_off( expected_data.size() - expected_size, expected_size)))); expected_size -= kChunkSize; expected_file_offset += kChunkSize; - buffer.PopFrontBytesUntil(expected_file_offset); + buffer.PopFrontUntil(expected_file_offset); EXPECT_THAT(buffer.file_offset(), Eq(expected_file_offset)); - EXPECT_THAT(buffer.SliceOff(expected_file_offset - 1, 1), Eq(std::nullopt)); EXPECT_THAT(buffer.SliceOff(expected_file_offset, expected_size), Optional(SameDataAs(expected_data.slice_off( expected_data.size() - expected_size, expected_size)))); diff --git a/src/trace_processor/util/trace_type.cc b/src/trace_processor/util/trace_type.cc new file mode 100644 index 000000000..ee5a56a05 --- /dev/null +++ b/src/trace_processor/util/trace_type.cc @@ -0,0 +1,133 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/trace_processor/util/trace_type.h" + +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <string> + +#include "perfetto/base/logging.h" +#include "perfetto/ext/base/string_utils.h" + +namespace perfetto::trace_processor { +namespace { +// Fuchsia traces have a magic number as documented here: +// https://fuchsia.googlesource.com/fuchsia/+/HEAD/docs/development/tracing/trace-format/README.md#magic-number-record-trace-info-type-0 +constexpr uint64_t kFuchsiaMagicNumber = 0x0016547846040010; +constexpr char kPerfMagic[] = "PERFILE2"; + +inline bool isspace(unsigned char c) { + return ::isspace(c); +} + +std::string RemoveWhitespace(std::string str) { + str.erase(std::remove_if(str.begin(), str.end(), isspace), str.end()); + return str; +} + +} // namespace + +const char* ToString(TraceType trace_type) { + switch (trace_type) { + case kJsonTraceType: + return "JSON trace"; + case kProtoTraceType: + return "proto trace"; + case kNinjaLogTraceType: + return "ninja log"; + case kFuchsiaTraceType: + return "fuchsia trace"; + case kSystraceTraceType: + return "systrace trace"; + case kGzipTraceType: + return "gzip trace"; + case kCtraceTraceType: + return "ctrace trace"; + case kZipFile: + return "ZIP file"; + case kPerfDataTraceType: + return "perf data"; + case kUnknownTraceType: + return "unknown trace"; + } + PERFETTO_FATAL("For GCC"); +} + +TraceType GuessTraceType(const uint8_t* data, size_t size) { + if (size == 0) + return kUnknownTraceType; + std::string start(reinterpret_cast<const char*>(data), + std::min<size_t>(size, kGuessTraceMaxLookahead)); + if (size >= 8) { + uint64_t first_word; + memcpy(&first_word, data, sizeof(first_word)); + if (first_word == kFuchsiaMagicNumber) + return kFuchsiaTraceType; + } + if (base::StartsWith(start, kPerfMagic)) { + return kPerfDataTraceType; + } + std::string start_minus_white_space = RemoveWhitespace(start); + if (base::StartsWith(start_minus_white_space, "{\"")) + return kJsonTraceType; + if (base::StartsWith(start_minus_white_space, "[{\"")) + return kJsonTraceType; + + // Systrace with header but no leading HTML. + if (base::Contains(start, "# tracer")) + return kSystraceTraceType; + + // Systrace with leading HTML. + // Both: <!DOCTYPE html> and <!DOCTYPE HTML> have been observed. + std::string lower_start = base::ToLower(start); + if (base::StartsWith(lower_start, "<!doctype html>") || + base::StartsWith(lower_start, "<html>")) + return kSystraceTraceType; + + // Traces obtained from atrace -z (compress). + // They all have the string "TRACE:" followed by 78 9C which is a zlib header + // for "deflate, default compression, window size=32K" (see b/208691037) + if (base::Contains(start, "TRACE:\n\x78\x9c")) + return kCtraceTraceType; + + // Traces obtained from atrace without -z (no compression). + if (base::Contains(start, "TRACE:\n")) + return kSystraceTraceType; + + // Ninja's build log (.ninja_log). + if (base::StartsWith(start, "# ninja log")) + return kNinjaLogTraceType; + + // Systrace with no header or leading HTML. + if (base::StartsWith(start, " ")) + return kSystraceTraceType; + + // gzip'ed trace containing one of the other formats. + if (base::StartsWith(start, "\x1f\x8b")) + return kGzipTraceType; + + if (base::StartsWith(start, "\x0a")) + return kProtoTraceType; + + if (base::StartsWith(start, "PK\x03\x04")) + return kZipFile; + + return kUnknownTraceType; +} + +} // namespace perfetto::trace_processor diff --git a/src/trace_processor/util/trace_type.h b/src/trace_processor/util/trace_type.h new file mode 100644 index 000000000..2d40d83bf --- /dev/null +++ b/src/trace_processor/util/trace_type.h @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SRC_TRACE_PROCESSOR_UTIL_TRACE_TYPE_H_ +#define SRC_TRACE_PROCESSOR_UTIL_TRACE_TYPE_H_ + +#include <cstddef> +#include <cstdint> + +namespace perfetto::trace_processor { + +enum TraceType { + kUnknownTraceType, + kProtoTraceType, + kJsonTraceType, + kFuchsiaTraceType, + kSystraceTraceType, + kGzipTraceType, + kCtraceTraceType, + kNinjaLogTraceType, + kZipFile, + kPerfDataTraceType, +}; + +constexpr size_t kGuessTraceMaxLookahead = 64; +TraceType GuessTraceType(const uint8_t* data, size_t size); +const char* ToString(TraceType type); + +} // namespace perfetto::trace_processor + +#endif // SRC_TRACE_PROCESSOR_UTIL_TRACE_TYPE_H_ |