First Commit

This commit is contained in:
Hamza committed 2026-02-26 13:00:32 +01:00
commit 8a6c7106e3
1312 files changed
+425396

No files matched your search

+51
View File
@@ -0,0 +1,51 @@
#-----------------------------------------------------------------------------
#
# CMake config
#
# protozero tools
#
#-----------------------------------------------------------------------------
# Needs getopt, which is not available on Windows
if(NOT MSVC)
add_executable(pbf-decoder pbf-decoder.cpp)
add_test(NAME pbf-decoder-no-args
COMMAND pbf-decoder)
set_tests_properties(pbf-decoder-no-args PROPERTIES
PASS_REGULAR_EXPRESSION "^Usage:")
add_test(NAME pbf-decoder-help
COMMAND pbf-decoder --help)
set_tests_properties(pbf-decoder-help PROPERTIES
PASS_REGULAR_EXPRESSION "^Usage:")
add_test(NAME pbf-decoder-empty
COMMAND pbf-decoder "${CMAKE_SOURCE_DIR}/test/t/message/data-opt-empty.pbf")
set_tests_properties(pbf-decoder-empty PROPERTIES
PASS_REGULAR_EXPRESSION "^$")
add_test(NAME pbf-decoder-data
COMMAND pbf-decoder "${CMAKE_SOURCE_DIR}/test/t/message/data-message.pbf")
set_tests_properties(pbf-decoder-data PROPERTIES
PASS_REGULAR_EXPRESSION "^1:")
add_test(NAME pbf-decoder-vt
COMMAND pbf-decoder -l 999999 -o 0 "${CMAKE_SOURCE_DIR}/test/t/vector_tile/data.vector.pbf")
set_tests_properties(pbf-decoder-vt PROPERTIES
PASS_REGULAR_EXPRESSION "^3:")
add_test(NAME pbf-decoder-fail
COMMAND pbf-decoder -l 1 "${CMAKE_SOURCE_DIR}/test/t/vector_tile/data.vector.pbf")
set_tests_properties(pbf-decoder-fail PROPERTIES
WILL_FAIL true)
add_test(NAME pbf-decoder-fail-msg
COMMAND pbf-decoder -l 1 "${CMAKE_SOURCE_DIR}/test/t/vector_tile/data.vector.pbf")
set_tests_properties(pbf-decoder-fail-msg PROPERTIES
PASS_REGULAR_EXPRESSION "^end of buffer exception")
endif()
#-----------------------------------------------------------------------------
+278
View File
@@ -0,0 +1,278 @@
/*****************************************************************************
Protobuf decoder tool
Tool to decode unknown protocol buffer encoded messages. The protocol buffer
format doesn't contain enough information about the contents of a file to make
it decodable without the format description usually found in a `.proto` file,
so this tool does some informed guessing.
Usage:
pbf-decoder [OPTIONS] [FILENAME]
Use "-" as a file name to read from STDIN.
The output always goes to STDOUT.
Call with --help/-h to see more options.
*****************************************************************************/
#include <protozero/pbf_reader.hpp>
#include <algorithm>
#include <cctype>
#include <cstddef>
#include <exception>
#include <fstream>
#include <getopt.h>
#include <iostream>
#include <limits>
#include <sstream>
#include <stdexcept>
#include <string>
#include <vector>
namespace {
std::string decode(const char* data, std::size_t len, const std::string& indent);
// Try decoding as a nested message
bool decode_message(std::stringstream& out, const std::string& indent, const protozero::data_view view) {
try {
const auto nested = decode(view.data(), view.size(), indent + " ");
out << '\n' << nested;
return true;
} catch (const protozero::exception&) { // NOLINT(bugprone-empty-catch)
}
return false;
}
// Try decoding as a string (only printable characters allowed).
bool decode_printable_string(std::stringstream& out, const protozero::data_view view) {
static constexpr const std::size_t max_string_length = 60;
const std::string str{view.data(), view.size()};
if (str.find_first_not_of("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_:-") != std::string::npos) {
return false;
}
if (str.size() > max_string_length) {
out << '"' << str.substr(0, max_string_length) << "\"...\n";
} else {
out << '"' << str << '"' << '\n';
}
return true;
}
// Try decoding as a string.
bool decode_string(std::stringstream& out, const protozero::data_view view) {
static constexpr const std::size_t max_string_length = 60;
const std::string str{view.data(), std::min(view.size(), max_string_length)};
out << '"';
for (const auto c : str) {
if (std::isprint(c) != 0) {
out << c;
} else {
out << '.';
}
}
out << '"' << '\n';
return true;
}
// Print a list of numbers from a range
template <typename TRange>
void print_number_range(std::stringstream& out, const TRange& range) {
bool first = true;
for (auto val : range) {
if (first) {
first = false;
} else {
out << ',';
}
out << val;
}
out << '\n';
}
// Try decoding as packed repeated double
bool decode_packed_double(std::stringstream& out, std::size_t size, protozero::pbf_reader& message) {
if (size % 8 != 0) {
return false;
}
try {
print_number_range(out, message.get_packed_double());
return true;
} catch (const protozero::exception&) { // NOLINT(bugprone-empty-catch)
}
return false;
}
// Try decoding as packed repeated float
bool decode_packed_float(std::stringstream& out, std::size_t size, protozero::pbf_reader& message) {
if (size % 4 != 0) {
return false;
}
try {
print_number_range(out, message.get_packed_float());
return true;
} catch (const protozero::exception&) { // NOLINT(bugprone-empty-catch)
}
return false;
}
// Try decoding as packed repeated varint
bool decode_packed_varint(std::stringstream& out, protozero::pbf_reader& message) {
try {
print_number_range(out, message.get_packed_int64());
return true;
} catch (const protozero::exception&) { // NOLINT(bugprone-empty-catch)
}
return false;
}
std::string decode(const char* data, std::size_t len, const std::string& indent) {
std::stringstream stream;
protozero::pbf_reader message{data, len};
while (message.next()) {
stream << indent << message.tag() << ": ";
switch (message.wire_type()) {
case protozero::pbf_wire_type::varint: {
// This is int32, int64, uint32, uint64, sint32, sint64, bool, or enum.
// Try decoding as int64.
stream << message.get_int64() << '\n';
break;
}
case protozero::pbf_wire_type::fixed64:
// This is fixed64, sfixed64, or double.
// Try decoding as a double, because int64_t or uint64_t
// would probably be encoded as varint.
stream << message.get_double() << '\n';
break;
case protozero::pbf_wire_type::length_delimited: {
// This is string, bytes, embedded messages, or packed repeated fields.
protozero::pbf_reader message_copy{message};
const auto view = message.get_view();
decode_message(stream, indent, view) ||
decode_printable_string(stream, view) ||
decode_packed_double(stream, view.size(), message_copy) ||
decode_packed_float(stream, view.size(), message_copy) ||
decode_packed_varint(stream, message_copy) ||
decode_string(stream, view);
break;
}
case protozero::pbf_wire_type::fixed32:
// This is fixed32, sfixed32, or float.
// Try decoding as a float, because int32_t or uint32_t
// would probably be encoded as varint.
stream << message.get_float() << '\n';
break;
default:
throw protozero::unknown_pbf_wire_type_exception{};
}
}
return stream.str();
}
void print_help() {
std::cout << "Usage: pbf-decoder [OPTIONS] [INPUT_FILE]\n\n"
<< "Dump raw contents of protobuf encoded file.\n"
<< "To read from STDIN use '-' as INPUT_FILE.\n"
<< "\nOptions:\n"
<< " -h, --help This help message\n"
<< " -l, --length=LENGTH Read only LENGTH bytes\n"
<< " -o, --offset=OFFSET Start reading from OFFSET bytes\n";
}
std::vector<char> read_from_file(const char* filename) {
const std::ifstream file{filename, std::ios::binary};
return std::vector<char>{std::istreambuf_iterator<char>(file.rdbuf()),
std::istreambuf_iterator<char>()};
}
std::vector<char> read_from_stdin() {
return std::vector<char>{std::istreambuf_iterator<char>(std::cin.rdbuf()),
std::istreambuf_iterator<char>()};
}
} // anonymous namespace
int main(int argc, char* argv[]) {
static struct option long_options[] = {
{"help", no_argument, nullptr, 'h'},
{"length", required_argument, nullptr, 'l'},
{"offset", required_argument, nullptr, 'o'},
{nullptr, 0, nullptr, 0}
};
std::size_t offset = 0;
std::size_t length = std::numeric_limits<std::size_t>::max();
while (true) {
const int c = getopt_long(argc, argv, "hl:o:", long_options, nullptr); // NOLINT(concurrency-mt-unsafe) no threads here
if (c == -1) {
break;
}
switch (c) {
case 'h':
print_help();
return 0;
case 'l':
length = std::atoll(optarg); // NOLINT(cert-err34-c)
// good enough for a limited-use tool
break;
case 'o':
offset = std::atoll(optarg); // NOLINT(cert-err34-c)
// good enough for a limited-use tool
break;
default:
return 1;
}
}
const int remaining_args = argc - optind;
if (remaining_args != 1) {
std::cerr << "Usage: " << argv[0] << " [OPTIONS] [INPUT_FILE]\n\n"
<< "Call with --help/-h to see options.\n";
return 1;
}
const std::string filename{argv[optind]};
try {
std::vector<char> buffer{filename == "-" ? read_from_stdin() :
read_from_file(argv[optind])};
if (offset > buffer.size()) {
throw std::runtime_error{"offset is larger than file size"};
}
if (offset > 0) {
buffer.erase(buffer.begin(), buffer.begin() + offset);
}
if (length < buffer.size()) {
buffer.resize(length);
}
std::cout << decode(buffer.data(), buffer.size(), "");
} catch (const std::exception& ex) {
std::cerr << ex.what() << '\n';
return 1;
}
return 0;
}
+85
View File
@@ -0,0 +1,85 @@
#include <protozero/pbf_reader.hpp>
// From Google Benchmark library.
// See https://github.com/google/benchmark/blob/main/LICENSE
template <class Tp>
inline __attribute__((always_inline)) void DoNotOptimize(Tp const& value) {
asm volatile("" : : "r,m"(value) : "memory");
}
template <typename T>
int read_packed(protozero::iterator_range<T> range) {
try {
for (const auto& item : range) {
DoNotOptimize(item);
}
} catch (const protozero::exception&) {
// no-op. This is probably not a packed field of that type.
}
return 0;
}
template <typename Fn>
void try_field(const protozero::pbf_reader& reader, Fn&& fn) {
try {
DoNotOptimize(fn(reader));
} catch (const protozero::exception&) {
// no-op. This is probably not a field of that type.
}
}
int try_message(protozero::pbf_reader reader) {
try {
while (reader.next()) {
if (reader.has_wire_type(protozero::pbf_wire_type::varint)) {
// Try to decode this field as any of the types that can be encoded as varint.
try_field(reader, [](protozero::pbf_reader r) { return r.get_bool(); });
try_field(reader, [](protozero::pbf_reader r) { return r.get_enum(); });
try_field(reader, [](protozero::pbf_reader r) { return r.get_int32(); });
try_field(reader, [](protozero::pbf_reader r) { return r.get_sint32(); });
try_field(reader, [](protozero::pbf_reader r) { return r.get_uint32(); });
try_field(reader, [](protozero::pbf_reader r) { return r.get_int64(); });
try_field(reader, [](protozero::pbf_reader r) { return r.get_sint64(); });
try_field(reader, [](protozero::pbf_reader r) { return r.get_uint64(); });
} else if (reader.has_wire_type(protozero::pbf_wire_type::length_delimited)) {
// Try to decode this field as any of the types that can be encoded as length-delimited.
try_field(reader, [](protozero::pbf_reader r) { return try_message(r.get_message()); });
try_field(reader, [](protozero::pbf_reader r) { return read_packed(r.get_packed_bool()); });
try_field(reader, [](protozero::pbf_reader r) { return read_packed(r.get_packed_double()); });
try_field(reader, [](protozero::pbf_reader r) { return read_packed(r.get_packed_enum()); });
try_field(reader, [](protozero::pbf_reader r) { return read_packed(r.get_packed_fixed32()); });
try_field(reader, [](protozero::pbf_reader r) { return read_packed(r.get_packed_fixed64()); });
try_field(reader, [](protozero::pbf_reader r) { return read_packed(r.get_packed_float()); });
try_field(reader, [](protozero::pbf_reader r) { return read_packed(r.get_packed_int32()); });
try_field(reader, [](protozero::pbf_reader r) { return read_packed(r.get_packed_int64()); });
try_field(reader, [](protozero::pbf_reader r) { return read_packed(r.get_packed_sfixed32()); });
try_field(reader, [](protozero::pbf_reader r) { return read_packed(r.get_packed_sfixed64()); });
try_field(reader, [](protozero::pbf_reader r) { return read_packed(r.get_packed_sint32()); });
try_field(reader, [](protozero::pbf_reader r) { return read_packed(r.get_packed_sint64()); });
try_field(reader, [](protozero::pbf_reader r) { return read_packed(r.get_packed_uint32()); });
try_field(reader, [](protozero::pbf_reader r) { return read_packed(r.get_packed_uint64()); });
} else if (reader.has_wire_type(protozero::pbf_wire_type::fixed64)) {
// Try to decode this field as any of the types that can be encoded as fixed64.
try_field(reader, [](protozero::pbf_reader r) { return r.get_double(); });
try_field(reader, [](protozero::pbf_reader r) { return r.get_fixed64(); });
try_field(reader, [](protozero::pbf_reader r) { return r.get_sfixed64(); });
} else if (reader.has_wire_type(protozero::pbf_wire_type::fixed32)) {
// Try to decode this field as any of the types that can be encoded as fixed32.
try_field(reader, [](protozero::pbf_reader r) { return r.get_float(); });
try_field(reader, [](protozero::pbf_reader r) { return r.get_fixed32(); });
try_field(reader, [](protozero::pbf_reader r) { return r.get_sfixed32(); });
}
reader.skip();
}
} catch (const protozero::exception&) {
// no-op. This is probably not a valid message.
}
return 0;
}
extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
try_message(protozero::pbf_reader{reinterpret_cast<const char*>(data), size});
return 0;
}