1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
|
// Copyright 2020 The Pigweed Authors
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not
// use this file except in compliance with the License. You may obtain a copy of
// the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations under
// the License.
// This file defines the functions that encode tokenized logs at runtime. These
// are the only pw_tokenizer functions present in a binary that tokenizes
// strings. All other tokenizing code is resolved at compile time.
#include "pw_tokenizer/tokenize.h"
#include <cstring>
#include "pw_span/span.h"
#include "pw_tokenizer/encode_args.h"
namespace pw {
namespace tokenizer {
namespace {
// Store metadata about this compilation's string tokenization in the ELF.
//
// The tokenizer metadata will not go into the on-device executable binary code.
// This metadata will be present in the ELF file's .pw_tokenizer.info section,
// from which the host-side tooling (Python, Java, etc.) can understand how to
// decode tokenized strings for the given binary. Only attributes that affect
// the decoding process are recorded.
//
// Tokenizer metadata is stored in an array of key-value pairs. Each Metadata
// object is 32 bytes: a 24-byte string and an 8-byte value. Metadata structs
// may be parsed in Python with the struct format '24s<Q'.
PW_PACKED(struct) Metadata {
char name[24]; // name of the metadata field
uint64_t value; // value of the field
};
static_assert(sizeof(Metadata) == 32, "Metadata should be exactly 32 bytes");
// Store tokenization metadata in its own section. Mach-O files are not
// supported by pw_tokenizer, but a short, Mach-O compatible section name is
// used on macOS so that this file can at least compile.
#ifdef __APPLE__
#define PW_TOKENIZER_INFO_SECTION PW_KEEP_IN_SECTION(".pw_tokenizer")
#else
#define PW_TOKENIZER_INFO_SECTION PW_KEEP_IN_SECTION(".pw_tokenizer.info")
#endif // __APPLE__
constexpr Metadata metadata[] PW_TOKENIZER_INFO_SECTION = {
{"c_hash_length_bytes", PW_TOKENIZER_CFG_C_HASH_LENGTH},
{"sizeof_long", sizeof(long)}, // %l conversion specifier
{"sizeof_intmax_t", sizeof(intmax_t)}, // %j conversion specifier
{"sizeof_size_t", sizeof(size_t)}, // %z conversion specifier
{"sizeof_ptrdiff_t", sizeof(ptrdiff_t)}, // %t conversion specifier
};
} // namespace
extern "C" void _pw_tokenizer_ToBuffer(void* buffer,
size_t* buffer_size_bytes,
Token token,
pw_tokenizer_ArgTypes types,
...) {
if (*buffer_size_bytes < sizeof(token)) {
*buffer_size_bytes = 0;
return;
}
std::memcpy(buffer, &token, sizeof(token));
va_list args;
va_start(args, types);
#if PW_CXX_STANDARD_IS_SUPPORTED(17)
const size_t encoded_bytes = EncodeArgs(
types,
args,
span<std::byte>(static_cast<std::byte*>(buffer) + sizeof(token),
*buffer_size_bytes - sizeof(token)));
#else
const size_t encoded_bytes =
pw_tokenizer_EncodeArgs(types,
args,
static_cast<std::byte*>(buffer) + sizeof(token),
*buffer_size_bytes - sizeof(token));
#endif // PW_CXX_STANDARD_IS_SUPPORTED(17)
va_end(args);
*buffer_size_bytes = sizeof(token) + encoded_bytes;
}
} // namespace tokenizer
} // namespace pw
|
