1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
|
// Copyright 2020 The Pigweed Authors
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not
// use this file except in compliance with the License. You may obtain a copy of
// the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations under
// the License.
#include "pw_tokenizer/detokenize.h"
#include <algorithm>
#include <cstring>
#include "pw_bytes/bit.h"
#include "pw_bytes/endian.h"
#include "pw_tokenizer/base64.h"
#include "pw_tokenizer/internal/decode.h"
#include "pw_tokenizer/nested_tokenization.h"
namespace pw::tokenizer {
namespace {
class NestedMessageDetokenizer {
public:
NestedMessageDetokenizer(const Detokenizer& detokenizer)
: detokenizer_(detokenizer) {}
void Detokenize(std::string_view chunk) {
for (char next_char : chunk) {
Detokenize(next_char);
}
}
void Detokenize(char next_char) {
switch (state_) {
case kNonMessage:
if (next_char == PW_TOKENIZER_NESTED_PREFIX) {
message_buffer_.push_back(next_char);
state_ = kMessage;
} else {
output_.push_back(next_char);
}
break;
case kMessage:
if (base64::IsValidChar(next_char)) {
message_buffer_.push_back(next_char);
} else {
HandleEndOfMessage();
if (next_char == PW_TOKENIZER_NESTED_PREFIX) {
message_buffer_.push_back(next_char);
} else {
output_.push_back(next_char);
state_ = kNonMessage;
}
}
break;
}
}
std::string Flush() {
if (state_ == kMessage) {
HandleEndOfMessage();
state_ = kNonMessage;
}
return std::move(output_);
}
private:
void HandleEndOfMessage() {
if (auto result = detokenizer_.DetokenizeBase64Message(message_buffer_);
result.ok()) {
output_ += result.BestString();
} else {
output_ += message_buffer_; // Keep the original if it doesn't decode.
}
message_buffer_.clear();
}
const Detokenizer& detokenizer_;
std::string output_;
std::string message_buffer_;
enum { kNonMessage, kMessage } state_ = kNonMessage;
};
std::string UnknownTokenMessage(uint32_t value) {
std::string output(PW_TOKENIZER_ARG_DECODING_ERROR_PREFIX "unknown token ");
// Output a hexadecimal version of the token.
for (int shift = 28; shift >= 0; shift -= 4) {
output.push_back("0123456789abcdef"[(value >> shift) & 0xF]);
}
output.append(PW_TOKENIZER_ARG_DECODING_ERROR_SUFFIX);
return output;
}
// Decoding result with the date removed, for sorting.
using DecodingResult = std::pair<DecodedFormatString, uint32_t>;
// Determines if one result is better than the other if collisions occurred.
// Returns true if lhs is preferred over rhs. This logic should match the
// collision resolution logic in detokenize.py.
bool IsBetterResult(const DecodingResult& lhs, const DecodingResult& rhs) {
// Favor the result for which decoding succeeded.
if (lhs.first.ok() != rhs.first.ok()) {
return lhs.first.ok();
}
// Favor the result for which all bytes were decoded.
if ((lhs.first.remaining_bytes() == 0u) !=
(rhs.first.remaining_bytes() == 0u)) {
return lhs.first.remaining_bytes() == 0u;
}
// Favor the result with fewer decoding errors.
if (lhs.first.decoding_errors() != rhs.first.decoding_errors()) {
return lhs.first.decoding_errors() < rhs.first.decoding_errors();
}
// Favor the result that successfully decoded the most arguments.
if (lhs.first.argument_count() != rhs.first.argument_count()) {
return lhs.first.argument_count() > rhs.first.argument_count();
}
// Favor the result that was removed from the database most recently.
return lhs.second > rhs.second;
}
} // namespace
DetokenizedString::DetokenizedString(
uint32_t token,
const span<const TokenizedStringEntry>& entries,
const span<const uint8_t>& arguments)
: token_(token), has_token_(true) {
std::vector<DecodingResult> results;
for (const auto& [format, date_removed] : entries) {
results.push_back(DecodingResult{format.Format(arguments), date_removed});
}
std::sort(results.begin(), results.end(), IsBetterResult);
for (auto& result : results) {
matches_.push_back(std::move(result.first));
}
}
std::string DetokenizedString::BestString() const {
return matches_.empty() ? std::string() : matches_[0].value();
}
std::string DetokenizedString::BestStringWithErrors() const {
if (matches_.empty()) {
return has_token_ ? UnknownTokenMessage(token_)
: PW_TOKENIZER_ARG_DECODING_ERROR("missing token");
}
return matches_[0].value_with_errors();
}
Detokenizer::Detokenizer(const TokenDatabase& database) {
for (const auto& entry : database) {
database_[entry.token].emplace_back(entry.string, entry.date_removed);
}
}
DetokenizedString Detokenizer::Detokenize(
const span<const uint8_t>& encoded) const {
// The token is missing from the encoded data; there is nothing to do.
if (encoded.empty()) {
return DetokenizedString();
}
uint32_t token = bytes::ReadInOrder<uint32_t>(
endian::little, encoded.data(), encoded.size());
const auto result = database_.find(token);
return DetokenizedString(
token,
result == database_.end() ? span<TokenizedStringEntry>()
: span(result->second),
encoded.size() < sizeof(token) ? span<const uint8_t>()
: encoded.subspan(sizeof(token)));
}
DetokenizedString Detokenizer::DetokenizeBase64Message(
std::string_view text) const {
std::string buffer(text);
buffer.resize(PrefixedBase64DecodeInPlace(buffer));
return Detokenize(buffer);
}
std::string Detokenizer::DetokenizeBase64(std::string_view text) const {
NestedMessageDetokenizer nested_detokenizer(*this);
nested_detokenizer.Detokenize(text);
return nested_detokenizer.Flush();
}
} // namespace pw::tokenizer
|
