First Commit

This commit is contained in:
Hamza committed 2026-02-26 13:00:32 +01:00
commit 8a6c7106e3
1312 files changed
+425396

No files matched your search

+86
View File
@@ -0,0 +1,86 @@
---
Checks: '*,-altera-*,-boost-use-ranges,-bugprone-chained-comparison,-bugprone-easily-swappable-parameters,-bugprone-signed-char-misuse,-cert-dcl21-cpp,-cert-err58-cpp,-cert-err60-cpp,-cppcoreguidelines-avoid-c-arrays,-cppcoreguidelines-avoid-non-const-global-variables,-cppcoreguidelines-avoid-magic-numbers,-cppcoreguidelines-macro-usage,-cppcoreguidelines-pro-bounds-pointer-arithmetic,-cppcoreguidelines-pro-bounds-array-to-pointer-decay,-cppcoreguidelines-pro-type-reinterpret-cast,-fuchsia-*,-google-runtime-references,-hicpp-avoid-c-arrays,-hicpp-no-array-decay,-hicpp-vararg,-llvmlibc-*,-misc-no-recursion,-modernize-avoid-c-arrays,-modernize-use-trailing-return-type,-readability-function-cognitive-complexity,-readability-identifier-length,-readability-implicit-bool-conversion,-readability-magic-numbers'
#
# Disabled checks:
#
# altera-*
# Doesn't apply.
#
# boost-use-ranges
# Would introduce extra dependency on boost.
#
# bugprone-chained-comparison
# These are generated by our test framework.
#
# bugprone-easily-swappable-parameters
# Can't change this any more, because these functions are part of our public
# interface.
#
# bugprone-signed-char-misuse
# Lots of warnings in varint.hpp otherwise.
#
# cert-dcl21-cpp
# It is unclear whether this is still a good recommendation in modern C++.
#
# cert-err58-cpp
# Due to the Catch2 test framework.
#
# cert-err60-cpp
# Reports std::runtime_error as broken which we can't do anything about.
#
# cppcoreguidelines-avoid-c-arrays
# hicpp-avoid-c-arrays
# modernize-avoid-c-arrays
# Makes sense for some array, but especially for char arrays using
# std::array isn't a good solution.
#
# cppcoreguidelines-avoid-magic-numbers
# readability-magic-numbers
# Good idea, but it goes too far to force this everywhere.
#
# cppcoreguidelines-avoid-non-const-global-variables
# Getting these from Catch2 test framework, not from the code itself.
#
# cppcoreguidelines-macro-usage
# There are cases where macros are simply needed.
#
# cppcoreguidelines-pro-bounds-array-to-pointer-decay
# Limited use and many false positives including for all asserts.
#
# cppcoreguidelines-pro-bounds-pointer-arithmetic
# This is a low-level library, it needs to do pointer arithmetic.
#
# cppcoreguidelines-pro-type-reinterpret-cast
# This is a low-level library, it needs to do reinterpret-casts.
#
# fuchsia-*
# Much too strict.
#
# google-runtime-references
# This is just a matter of preference, and we can't change the interfaces
# now anyways.
#
# hicpp-no-array-decay
# Limited use and many false positives including for all asserts.
#
# llvmlibc-*
# Doesn't apply.
#
# misc-no-recursion
# Nothing wrong with recursion.
#
# modernize-use-trailing-return-type
# We are not quite that modern.
#
# readability-function-cognitive-complexity
# Getting these mostly from Catch2 test framework.
#
# readability-identifier-length
# Short identifiers do make sense sometimes.
#
# readability-implicit-bool-conversion
# Not necessarily more readable.
#
#WarningsAsErrors: '*'
HeaderFilterRegex: '\/include\/'
...
+1
View File
@@ -0,0 +1 @@
*.pbf -text
@@ -0,0 +1,10 @@
name: Build on Windows
runs:
using: composite
steps:
- name: Build
run: cmake --build . --config Release --verbose
shell: bash
working-directory: build
+10
View File
@@ -0,0 +1,10 @@
name: Build
runs:
using: composite
steps:
- name: Build
run: make VERBOSE=1
shell: bash
working-directory: build
@@ -0,0 +1,12 @@
name: CMake on Windows
runs:
using: composite
steps:
- name: Create build directory
run: mkdir build
shell: bash
- name: Configure
run: cmake -LA .. -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake
shell: bash
working-directory: build
+16
View File
@@ -0,0 +1,16 @@
name: CMake
runs:
using: composite
steps:
- name: Create build directory
run: mkdir build
shell: bash
- name: Configure
run: |
cmake -LA .. \
-DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
-DPROTOZERO_DATA_VIEW=${PROTOZERO_DATA_VIEW} \
-DCMAKE_CXX_STANDARD=${CPP_VERSION}
shell: bash
working-directory: build
@@ -0,0 +1,10 @@
name: Test on Windows
runs:
using: composite
steps:
- name: Test
run: ctest --output-on-failure -C Release
shell: bash
working-directory: build
+10
View File
@@ -0,0 +1,10 @@
name: Test
runs:
using: composite
steps:
- name: Test
run: ctest --output-on-failure
shell: bash
working-directory: build
@@ -0,0 +1,14 @@
name: Install apt packages on Ubuntu/Debian
runs:
using: composite
steps:
- name: Install packages
run: |
sudo apt-get update -qq
sudo apt-get install -yq \
doxygen \
graphviz \
libprotobuf-dev \
protobuf-compiler
shell: bash
+164
View File
@@ -0,0 +1,164 @@
name: CI
on: [ push, pull_request ]
jobs:
linux:
runs-on: ubuntu-latest
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
image:
- "ubuntu:20.04" # gcc 9.3.0, clang 10.0.0, cmake 3.16.3
- "ubuntu:22.04" # gcc 12.2.0, clang 15.0.7, cmake 3.24.2
- "ubuntu:24.04" # gcc 14.2.0, clang 18.1.3, cmake 3.28.3
- "debian:bullseye" # gcc 10.2.1, clang 11.0.1, cmake 3.18.4
- "debian:bookworm" # gcc 12.2.0, clang 15.0.6, cmake 3.25.1
- "debian:testing"
- "debian:experimental"
- "fedora:39"
- "fedora:40"
- "fedora:41"
build_type: [Debug]
cpp_compiler: [g++]
cpp_version: [14]
include:
- image: "debian:bookworm"
c_compiler: clang
cpp_compiler: clang++
- image: "debian:bookworm"
cpp_version: 17
- image: "debian:bookworm"
cpp_version: 20
- image: "debian:bookworm"
c_compiler: clang
cpp_compiler: clang++
cpp_version: 17
- image: "debian:bookworm"
c_compiler: clang
cpp_compiler: clang++
cpp_version: 20
# Disabled because it fails due to a bug in the protobuf library
# See https://github.com/protocolbuffers/protobuf/issues/7224
# - image: "debian:bullseye"
# c_compiler: clang
# cpp_compiler: clang++
# CXXFLAGS: "-fsanitize=address,undefined,integer -fno-sanitize-recover=all -fno-omit-frame-pointer"
# LDFLAGS: "-fsanitize=address,undefined,integer"
- image: "debian:bookworm"
build_type: Release
- image: "debian:bookworm"
c_compiler: clang
cpp_compiler: clang++
cpp_version: 17
data_view: std::string_view
- image: "debian:testing"
c_compiler: clang
cpp_compiler: clang++
CXXFLAGS: -Wno-c++20-extensions
- image: "debian:experimental"
c_compiler: clang
cpp_compiler: clang++
CXXFLAGS: -Wno-c++20-extensions
container:
image: ${{ matrix.image }}
env:
BUILD_TYPE: ${{ matrix.build_type }}
CC: ${{ matrix.c_compiler }}
CXX: ${{ matrix.cpp_compiler }}
CXXFLAGS: ${{ matrix.CXXFLAGS }}
LDFLAGS: ${{ matrix.LDFLAGS }}
CPP_VERSION: ${{ matrix.cpp_version }}
PROTOZERO_DATA_VIEW: ${{ matrix.data_view }}
APT_LISTCHANGES_FRONTEND: none
DEBIAN_FRONTEND: noninteractive
steps:
- name: Prepare container (apt)
shell: bash
if: startsWith(matrix.image, 'debian:') || startsWith(matrix.image, 'ubuntu:')
run: |
apt-get update -qq
apt-get install -y \
clang \
cmake \
doxygen \
g++ \
graphviz \
libprotobuf-dev \
make \
protobuf-compiler
- name: Prepare container (dnf)
shell: bash
if: startsWith(matrix.image, 'fedora:')
run: |
dnf install --quiet --assumeyes \
cmake \
doxygen \
gcc-c++ \
graphviz \
make \
protobuf-lite-devel \
protobuf-lite
- uses: actions/checkout@v4
- uses: ./.github/actions/cmake
- uses: ./.github/actions/build
- uses: ./.github/actions/ctest
ubuntu-latest:
runs-on: ubuntu-24.04
timeout-minutes: 30
env:
CC: clang-18
CXX: clang++-18
BUILD_TYPE: Debug
steps:
- uses: actions/checkout@v4
- uses: ./.github/actions/install-ubuntu
- uses: ./.github/actions/cmake
- uses: ./.github/actions/build
- uses: ./.github/actions/ctest
macos:
runs-on: ${{ matrix.os }}
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
os:
- macos-14
- macos-15
build_type: [Debug]
include:
- os: macos-15
build_type: Release
env:
CC: clang
CXX: clang++
BUILD_TYPE: ${{ matrix.build_type }}
steps:
- run: brew install doxygen graphviz protobuf
- uses: actions/checkout@v4
- uses: ./.github/actions/cmake
- uses: ./.github/actions/build
- uses: ./.github/actions/ctest
windows:
runs-on: ${{ matrix.os }}
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
os:
- windows-2022
- windows-2025
steps:
- run: |
vcpkg install \
protobuf:x64-windows \
protobuf-c:x64-windows
shell: bash
- uses: actions/checkout@v4
- uses: ./.github/actions/cmake-windows
- uses: ./.github/actions/build-windows
- uses: ./.github/actions/ctest-windows
+52
View File
@@ -0,0 +1,52 @@
name: clang-tidy
on: [ push, pull_request ]
jobs:
clang-tidy:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
image: ["debian:bookworm", "debian:testing", "debian:experimental"]
include:
- image: "debian:bookworm"
clang: 15
- image: "debian:testing"
clang: 19
- image: "debian:experimental"
clang: 20
container:
image: ${{ matrix.image }}
env:
BUILD_TYPE: Debug
CC: clang-${{ matrix.clang }}
CXX: clang++-${{ matrix.clang }}
CXXFLAGS: -Wno-c++20-extensions
CPP_VERSION: 14
APT_LISTCHANGES_FRONTEND: none
DEBIAN_FRONTEND: noninteractive
steps:
- name: Prepare container (apt)
run: |
apt-get update -qq
apt-get install -yq \
clang-${{ matrix.clang }} \
clang-tidy-${{ matrix.clang }} \
cmake \
libprotobuf-dev \
make \
protobuf-compiler
shell: bash
- uses: actions/checkout@v4
- uses: ./.github/actions/cmake
- name: Run clang-tidy
run: make clang-tidy | tee protozero-${{ github.sha }}-clang-tidy-${{ matrix.clang }}.log
shell: bash
working-directory: build
- name: Upload clang-tidy log
uses: actions/upload-artifact@v4
if: always()
with:
name: protozero-${{ github.sha }}-clang-tidy-${{ matrix.clang }}-log
path: build/protozero-${{ github.sha }}-clang-tidy-${{ matrix.clang }}.log
+457
View File
@@ -0,0 +1,457 @@
# Changelog
All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/)
This project adheres to [Semantic Versioning](https://semver.org/).
## [unreleased] -
### Added
### Changed
### Fixed
## [1.8.1] - 2025-07-15
### Fixed
- Fix buffer overrun in `get_bool()`
- Fix test that checks that protozero also works with `std::string_view`
## [1.8.0] - 2025-01-13
### Changed
- Switched to C++14 as minimum requirement. Modernised the code accordingly.
- Only the tests need the Protobuf library when building.
- Improve compatibility with various STL flavors.
- Change license formatting to get recognized by license scanning tools.
- Add lots of `const` in places where we can. Also fix some other issues
reported by clang-tidy.
- Various other code cleanups.
- Modernize documentation config, CMake config and CI workflows.
### Fixed
- `basic_pbf_builder` constructor can not be noexcept.
## [1.7.1] - 2022-01-10
### Changed
- Don't build tests if the standard CMake `BUILD_TESTING` variable is set to
off.
- Now needs CMake 3.5.0 or greater.
- Update included catch2 framework to current version v2.13.8.
- Only enable clang-tidy make target if protobuf was found.
- Allow setting C++ version to compile with in CMake config.
### Fixed
- Fixes undefined behaviour in `float` and `double` byteswap.
- Add missing includes of "config.hpp".
- Avoid narrowing conversion by doing an explicit `static_cast`.
## [1.7.0] - 2020-06-08
### Added
- Support for buffer types other that `std::string`. `pbf_writer` is now
just a typedef for `basic_pbf_writer<std::string>`. Other buffer types
can be used with `basic_pbf_writer`. See `doc/advanced.md` for details.
### Changed
- Switched to *catch2* for testing.
- Some minor tweaks.
### Fixed
- Removed some undefined behaviour.
## [1.6.8] - 2019-08-15
### Changed
- Various code cleanups due to clang-tidy warnings.
### Fixed
- Made `data_view::compare` noexcept.
## [1.6.7] - 2018-02-21
### Fixed
- Signed-unsigned comparison on 32 bit systems.
## [1.6.6] - 2018-02-20
### Fixed
- Fixed several place with possible undefined behaviour.
## [1.6.5] - 2018-02-05
### Fixed
- Avoid UB: Do not calculate pointer outside array bounds.
- Specify proto2 syntax in .proto files to appease protoc.
## [1.6.4] - 2018-11-08
### Added
- Add function `data()` to get the not yet read data from a `pbf_reader`.
- New `add_packed_fixed()` template function for `pbf_writer`.
- New `length_of_varint()` helper function calculates how long a varint
would be for a specified value.
### Changed
- More consistent implementation of operators as free friend functions.
### Fixed
- Fixed some zigzag encoding tests on MSVC.
- Add extra cast so we do an xor with unsigned ints.
- No more bitwise operations on signed integers in varint decoder.
- No more bitwise operations on signed integers in zigzag encoder/decoder.
## [1.6.3] - 2018-07-17
### Changed
- Moved `byteswap_inplace` functions from detail into protozero namespace.
They can be useful outsize protozero.
- More asserts and unit tests and small cleanups.
## [1.6.2] - 2018-03-09
### Changed
- Update included catch.hpp to v1.12.0.
- Move basic unit tests into their own directory (`test/unit`).
- Improved clang-tidy config and fixed some code producing warnings.
### Fixed
- Buffer overflow in pbf-decoder tool.
## [1.6.1] - 2017-11-16
### Added
- Document internal handling of varints.
- Add aliases for fixed iterators, too.
### Changed
- The `const_fixed_iterator` is now a random access iterator making code
using it potentially more performant (for instance when using
`std::distance`)
- Overloads `std::distance` for the varint and svarint iterators. This is
better than the workaround with the `rage_size` function used before.
### Fixed
- Rename `.proto` files in some tests to be unique. This solves a problem
when building with newer versions of the Google Protobuf library.
- Floating point comparisons in tests are now always correctly done using
`Approx()`.
## [1.6.0] - 2017-10-24
### Added
- Comparison functions (<, <=, >, >=) for `data_view`. Allows use in `std::map`
for instance.
- Tool `pbf-decoder` for decoding raw messages. This has limited use for
normal users, but it can be used for fuzzing.
### Changed
- Protozero now uses CMake to build the tests etc. This does not affect
simple users of the library, but if you are using CMake yourself you might
want to use the `cmake/FindProtozero.cmake` module provided. The README
contains more information about build options.
- Moved `data_view` class from `types.hpp` into its own header file
`data_view.hpp`.
- Implementation of the `const_fixed_iterator` to use only a single pointer
instead of two.
- Made `operator==` and `operator!=` on `data_view` constexpr.
- The `pbf_reader` constructor taking a `std::pair` is deprecated. Use one
of the other constructors instead.
### Fixed
- Varints where the last byte was larger than what would fit in 64bit were
triggering undefined behaviour. This can only happen when the message
being decoded was corrupt in some way.
- Do not assert when reading too long varints for bools any more. A valid
encoder should never generate varints with more than one byte for bools,
but if they are longer that's not really a problem, so just handle it.
- Throw exception if the length of a packed repeated field of a fixed-length
type is invalid. The length must always be a multiple of the size of the
underlying type. This can only happen if the data is corrupted in some way,
a valid encoder would never generate data like this.
- Throw an exception when reading invalid tags. This can only happen if the
data is corrupted in some way, a valid encoder would never generate invalid
tags.
## [1.5.3] - 2017-09-22
### Added
- More documentation.
- New `size()` method on iterator range used for packed repeated fields to
find out how many elements there are in the range. This is much faster
compared to the `std::difference()` call you had to do before, because the
varints don't have to be fully decoded. See [Advanced
Topics](doc/advanced.md) for details.
### Changed
- Updated clang-tidy settings in Makefiles and fixed a lot of minor issues
reported by clang-tidy.
- Update included catch.hpp to version 1.10.0.
- Miscellaneous code cleanups.
- Support for invalid state in `pbf_writer` and `packed_repeated_fields`.
This fixes move construction and move assignement in `pbf_writer` and
disables the copy construction and copy assignement which don't have
clear semantics. It introduces an invalid or empty state in the
`pbf_writer`, `pbf_builder`, and `packed_repeated_fields` classes used for
default-constructed, moved from, or committed objects. There is a new
`commit()` function for `pbf_writer` and the `packed_repeated_fields` which
basically does the same as the destructor but can be called explicitly.
### Fixed
- The `empty()` method of the iterator range now returns a `bool` instead of
a `size_t`.
## [1.5.2] - 2017-06-30
### Added
- Add missing two-parameter version of `pbf_message::next()` function.
- Add `data_view::empty()` function.
- Add missing versions of `add_bytes()`, `add_string()`, and `add_message()`
to `pbf_builder`.
### Changed
- Clarify include file usage in tutorial.
- Updated included Catch unit test framework to version 1.9.6 and updated
tests to work with the current version.
- Make some constructors explicit (best practice to avoid silent conversions).
### Fixed
- Important bugfix in `data_view` equality operator. The equality operator is
actually never used in the protozero code itself, but users of protozero
might use it. This is a serious bug that could lead to buffer overrun type
problems.
## [1.5.1] - 2017-01-14
### Added
- Better documentation for `tag_and_type()` in doc/advanced.md.
### Fixed
- Fixed broken "make doc" build.
## [1.5.0] - 2017-01-12
### Added
- Add `add_bytes_vectored()` methods to `pbf_writer` and `pbf_builder`. This
allows single-copy scatter-gather type adding of data that has been prepared
in pieces to a protobuf message.
- New functions to check the tag and wire type at the same time: Two parameter
version of `pbf_reader::next()` and `pbf_reader::tag_and_type()` can be used
together with the free function `tag_and_type()` to easily and quickly check
that not only the tag but also the wire type is correct for a field.
### Changed
- `packed_field_*` classes now work with `pbf_builder`.
- Reorganized documentation. Advanced docs are now under doc/advanced.md.
### Fixed
- `packed_field` class is now non-copyable because data can get corrupted if
you copy it around.
- Comparison operators of `data_view` now have const& parameters.
- Make zigzag encoding/decoding functions constexpr.
## [1.4.5] - 2016-11-18
### Fixed
- Undefined behaviour in packed fixed iterator. As a result, the macro
`PROTOZERO_DO_NOT_USE_BARE_POINTER` is not used any more.
## [1.4.4] - 2016-11-15
### Fixed
- Byteswap implementation.
## [1.4.3] - 2016-11-15
### Fixed
- Undefined behaviour in byte swapping code.
- Rename some parameters to avoid "shadow" warning from some compilers.
## [1.4.2] - 2016-08-27
### Fixed
- Compile fix: Variable shadowing.
## [1.4.1] - 2016-08-21
### Fixed
- GCC 4.8 compile fixed
### Added
- New ability to dynamically require the module as a node module to ease
building against from other node C++ modules.
## [1.4.0] - 2016-07-22
### Changed
- Use more efficient new `skip_varint()` function when iterating over
packed varints.
- Split `decode_varint()` function into two functions speeding up the
common case where a varint is only one byte long.
- Introduce new class `iterator_range` used instead of `std::pair` of
iterators. This way the objects can be used in range-based for loops.
Read UPGRADING.md for details.
- Introduce new class `data_view` and functions using and returning it.
Read UPGRADING.md for details.
## [1.3.0] - 2016-02-18
### Added
- Added `config.hpp` header which now includes all the macro magic to
configure the library for different architectures etc.
- New way to create repeated packed fields without using an iterator.
- Add `rollback()` function to `pbf_writer` for "manual" rollback.
### Changed
- Various test and documentation cleanups.
- Rename `pbf_types.hpp` to `types.hpp`.
## [1.2.3] - 2015-11-30
### Added
- Added `config.hpp` header which now includes all the macro magic to
configure the library for different architectures etc.
### Fixed
- Unaligned access to floats/doubles on some ARM architectures.
## [1.2.2] - 2015-10-13
### Fixed
- Fix the recently broken writing of bools on big-endian architectures.
## [1.2.1] - 2015-10-12
### Fixed
- Removed unneeded code (1-byte "swap") which lead to test failures.
## [1.2.0] - 2015-10-08
### Added
- `pbf_message` and `pbf_builder` template classes wrapping `pbf_reader`
and `pbf_writer`, respectively. The new classes are the preferred
interface now.
### Changed
- Improved byte swapping operation.
- Detect some types of data corruption earlier and throw.
## [1.1.0] - 2015-08-22
### Changed
- Make pbf reader and writer code endianess-aware.
[unreleased]: https://github.com/osmcode/libosmium/compare/v1.8.0...HEAD
[1.8.0]: https://github.com/osmcode/libosmium/compare/v1.7.1...v1.8.0
[1.7.1]: https://github.com/osmcode/libosmium/compare/v1.7.0...v1.7.1
[1.7.0]: https://github.com/osmcode/libosmium/compare/v1.6.8...v1.7.0
[1.6.8]: https://github.com/osmcode/libosmium/compare/v1.6.7...v1.6.8
[1.6.7]: https://github.com/osmcode/libosmium/compare/v1.6.6...v1.6.7
[1.6.6]: https://github.com/osmcode/libosmium/compare/v1.6.5...v1.6.6
[1.6.5]: https://github.com/osmcode/libosmium/compare/v1.6.4...v1.6.5
[1.6.4]: https://github.com/osmcode/libosmium/compare/v1.6.3...v1.6.4
[1.6.3]: https://github.com/osmcode/libosmium/compare/v1.6.2...v1.6.3
[1.6.2]: https://github.com/osmcode/libosmium/compare/v1.6.1...v1.6.2
[1.6.1]: https://github.com/osmcode/libosmium/compare/v1.6.0...v1.6.1
[1.6.0]: https://github.com/osmcode/libosmium/compare/v1.5.3...v1.6.0
[1.5.3]: https://github.com/osmcode/libosmium/compare/v1.5.2...v1.5.3
[1.5.2]: https://github.com/osmcode/libosmium/compare/v1.5.1...v1.5.2
[1.5.1]: https://github.com/osmcode/libosmium/compare/v1.5.0...v1.5.1
[1.5.0]: https://github.com/osmcode/libosmium/compare/v1.4.5...v1.5.0
[1.4.5]: https://github.com/osmcode/libosmium/compare/v1.4.4...v1.4.5
[1.4.4]: https://github.com/osmcode/libosmium/compare/v1.4.3...v1.4.4
[1.4.3]: https://github.com/osmcode/libosmium/compare/v1.4.2...v1.4.3
[1.4.2]: https://github.com/osmcode/libosmium/compare/v1.4.1...v1.4.2
[1.4.1]: https://github.com/osmcode/libosmium/compare/v1.4.0...v1.4.1
[1.4.0]: https://github.com/osmcode/libosmium/compare/v1.3.0...v1.4.0
[1.3.0]: https://github.com/osmcode/libosmium/compare/v1.2.3...v1.3.0
[1.2.3]: https://github.com/osmcode/libosmium/compare/v1.2.2...v1.2.3
[1.2.2]: https://github.com/osmcode/libosmium/compare/v1.2.1...v1.2.2
[1.2.1]: https://github.com/osmcode/libosmium/compare/v1.2.0...v1.2.1
[1.2.0]: https://github.com/osmcode/libosmium/compare/v1.1.0...v1.2.0
[1.1.0]: https://github.com/osmcode/libosmium/compare/v1.0.0...v1.1.0
+164
View File
@@ -0,0 +1,164 @@
#-----------------------------------------------------------------------------
#
# CMake config
#
# protozero
#
#-----------------------------------------------------------------------------
cmake_minimum_required(VERSION 3.10.0 FATAL_ERROR)
#-----------------------------------------------------------------------------
project(protozero VERSION 1.8.1 LANGUAGES CXX C)
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
#-----------------------------------------------------------------------------
if (NOT "${CMAKE_CXX_STANDARD}")
set(CMAKE_CXX_STANDARD 14)
endif()
message(STATUS "Building in C++${CMAKE_CXX_STANDARD} mode")
set(CMAKE_CXX_EXTENSIONS OFF)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
#-----------------------------------------------------------------------------
option(BUILD_TESTING "Build tests" ON)
option(WERROR "Add -Werror flag to build (turns warnings into errors)" ON)
if(MSVC)
add_compile_options(/W3)
add_definitions(-D_CRT_SECURE_NO_WARNINGS -D_SCL_SECURE_NO_WARNINGS)
else()
add_compile_options(-Wall -Wextra -pedantic -Wsign-compare -Wunused-parameter -Wno-float-equal -Wno-covered-switch-default)
if(WERROR)
add_compile_options(-Werror)
endif()
endif()
include_directories("${CMAKE_SOURCE_DIR}/include")
# Used for testing
set(PROTOZERO_DATA_VIEW "" CACHE STRING "Type used for protozero::data_view")
if(NOT PROTOZERO_DATA_VIEW STREQUAL "")
message(STATUS "Using ${PROTOZERO_DATA_VIEW} as data_view")
add_definitions(-DPROTOZERO_USE_VIEW=${PROTOZERO_DATA_VIEW})
else()
message(STATUS "Using built-in data_view")
endif()
#-----------------------------------------------------------------------------
#
# Find dependencies
#
#-----------------------------------------------------------------------------
if(BUILD_TESTING)
# This is needed for the protobuf_generate_cpp() function to work on newer
# versions of the Protobuf CMake config.
set(protobuf_MODULE_COMPATIBLE ON CACHE BOOL "")
# This is needed so that we pick up the (more modern) CONFIG mode cmake file
# before the (older) MODULE mode cmake config file. Seems to be needed on macOS.
set(CMAKE_FIND_PACKAGE_PREFER_CONFIG TRUE)
find_package(Protobuf)
endif()
#-----------------------------------------------------------------------------
#
# Optional "clang-tidy" target
#
#-----------------------------------------------------------------------------
message(STATUS "Looking for clang-tidy")
find_program(CLANG_TIDY NAMES clang-tidy-20 clang-tidy-19 clang-tidy-18 clang-tidy-17 clang-tidy-16 clang-tidy-15)
if(CLANG_TIDY AND PROTOBUF_FOUND)
message(STATUS "Looking for clang-tidy - found ${CLANG_TIDY}")
add_custom_target(clang-tidy
${CLANG_TIDY}
-p ${CMAKE_BINARY_DIR}
${CMAKE_SOURCE_DIR}/test/*.cpp
${CMAKE_SOURCE_DIR}/test/t/*/reader_test_cases.cpp
${CMAKE_SOURCE_DIR}/test/t/*/writer_test_cases.cpp
${CMAKE_SOURCE_DIR}/test/unit/*.cpp
${CMAKE_SOURCE_DIR}/tools/*.cpp
)
if(BUILD_TESTING)
add_dependencies(clang-tidy writer_tests)
endif()
else()
message(STATUS "Looking for clang-tidy - not found")
message(STATUS " Build target 'clang-tidy' will not be available.")
endif()
#-----------------------------------------------------------------------------
#
# Optional "cppcheck" target
#
#-----------------------------------------------------------------------------
message(STATUS "Looking for cppcheck")
find_program(CPPCHECK NAMES cppcheck)
if(CPPCHECK)
message(STATUS "Looking for cppcheck - found")
add_custom_target(cppcheck
${CPPCHECK}
-Uassert --std=c++14 --enable=all
${CMAKE_SOURCE_DIR}/include/protozero/*.hpp
${CMAKE_SOURCE_DIR}/test/*.cpp
${CMAKE_SOURCE_DIR}/test/include/*.hpp
${CMAKE_SOURCE_DIR}/test/t/*/*.cpp
${CMAKE_SOURCE_DIR}/test/unit/*.cpp
${CMAKE_SOURCE_DIR}/tools/*.cpp
)
else()
message(STATUS "Looking for cppcheck - not found")
message(STATUS " Build target 'cppcheck' will not be available.")
endif()
#-----------------------------------------------------------------------------
#
# Include what you use
#
#-----------------------------------------------------------------------------
message(STATUS "Looking for iwyu")
find_program(IWYU_TOOL NAMES iwyu_tool)
if(IWYU_TOOL)
message(STATUS "Looking for iwyu - found")
add_custom_target(iwyu
${IWYU_TOOL} -p ${CMAKE_BINARY_DIR}
)
else()
message(STATUS "Looking for iwyu - not found")
message(STATUS " Build target 'iwyu' will not be available.")
endif()
#-----------------------------------------------------------------------------
#
# Installation
#
#-----------------------------------------------------------------------------
install(DIRECTORY include/protozero DESTINATION include)
#-----------------------------------------------------------------------------
add_subdirectory(doc)
if(BUILD_TESTING)
enable_testing()
add_subdirectory(test)
endif()
add_subdirectory(tools)
#-----------------------------------------------------------------------------
+21
View File
@@ -0,0 +1,21 @@
# Contributing to protozero
## Releasing
To release a new protozero version:
- Make sure all tests are passing locally, on Github and on Appveyor
- Make sure "make doc" builds
- Update version number in
- include/protozero/version.hpp (two places)
- CMakeLists.txt (one place)
- Update CHANGELOG.md
(don't forget links at the bottom of the file)
- Update UPGRADING.md if necessary
- `git commit -m "Release X.Y.Z" include/protozero/version.hpp CMakeLists.txt CHANGELOG.md UPGRADING.md`
- `git tag vX.Y.Z`
- `git push`
- `git push --tags`
- Go to https://github.com/mapbox/protozero/releases
and edit the new release. Put "Version x.y.z" in title and
cut-and-paste entry from CHANGELOG.md.
+33
View File
@@ -0,0 +1,33 @@
To do fuzz testing using [AFL](https://lcamtuf.coredump.cx/afl/) compile with
the AFL compiler wrappers:
mkdir build
cd build
CC=afl-clang CXX=afl-clang++ cmake ..
mkdir testcase_dir
You need some data to start the fuzzing. In this case I am using all the test
messages from the unit tests:
find ../test/t/ -name data-\*.pbf -a -not -empty -exec cp {} testcase_dir/ \;
Then do the actual fuzzing:
afl-fuzz -i testcase_dir -o findings_dir -- tools/pbf-decoder -
See the AFL documentation for more information.
For increased speed, you can also use the dedicated pbf-fuzzer tool, which skips reading
data from files or stdin:
clang++ -O2 -std=c++17 -g -DNDEBUG -Iinclude -fsanitize=address,fuzzer tools/pbf-fuzzer.cpp -o tools/pbf-fuzzer
./tools/pbf-fuzzer
or using AFL++
afl-clang-fast++ -O2 -std=c++17 -g -DNDEBUG -Iinclude -fsanitize=address,fuzzer tools/pbf-fuzzer.cpp -o tools/pbf-fuzzer
afl-fuzz -i testcase_dir -o findings_dir -- tools/pbf-fuzzer
This only checkes the reading side of Protozero!
+177
View File
@@ -0,0 +1,177 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
+26
View File
@@ -0,0 +1,26 @@
Copyright (C) 2022, Mapbox.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+155
View File
@@ -0,0 +1,155 @@
# protozero
Minimalistic protocol buffer decoder and encoder in C++.
Designed for high performance. Suitable for writing zero copy parsers and
encoders with minimal need for run-time allocation of memory.
Low-level: this is designed to be a building block for writing a very
customized decoder for a stable protobuf schema. If your protobuf schema is
changing frequently or lazy decoding is not critical for your application then
this approach offers no value: just use the C++ API that can be generated with
the Google Protobufs `protoc` program.
[![Github Build Status](https://github.com/mapbox/protozero/actions/workflows/ci.yml/badge.svg)](https://github.com/mapbox/protozero/actions/workflows/ci.yml)
[![Appveyor Build Status](https://ci.appveyor.com/api/projects/status/github/mapbox/protozero?svg=true)](https://ci.appveyor.com/project/Mapbox/protozero)
[![Packaging status](https://repology.org/badge/tiny-repos/protozero.svg)](https://repology.org/metapackage/protozero)
## Depends
* C++14 compiler
* CMake
* Some tests depend on the Google Protobuf library, but use of Protozero
doesn't need it
## How it works
The protozero code does **not** read `.proto` files used by the usual Protobuf
implementations. The developer using protozero has to manually "translate" the
`.proto` description into code. This means there is no way to access any of the
information from the `.proto` description. This results in a few restrictions:
* The names of the fields are not available.
* Enum names are not available, you'll have to use the values they are defined
with.
* Default values are not available.
* Field types have to be hardcoded. The library does not know which types to
expect, so the user of the library has to supply the right types. Some checks
are made using `assert()`, but mostly the user has to take care of that.
The library will make sure not to overrun the buffer it was given, but
basically all other checks have to be made in user code!
## Documentation
You have to have a working knowledge of how
[protocol buffer encoding works](https://developers.google.com/protocol-buffers/docs/encoding).
* Read the [tutorial](doc/tutorial.md) for an introduction on how to use
Protozero.
* Some advanced topics are described in an [extra document](doc/advanced.md).
* There is a table of all types and functions in the
[cheat sheet](doc/cheatsheet.md).
* Read the [upgrading instructions](UPGRADING.md) if you are upgrading from
an older version of Protozero.
The build process will also build the Doxygen-based reference documentation if
you have Doxygen installed. Then open `doc/html/index.html` in your browser to
read it.
## Endianness
Protozero uses a very simplistic test to check the byte order of the system it
compiles on. If this check is wrong, you'll get test failures. If this is the
case, please [open an issue](https://github.com/mapbox/protozero/issues) and
tell us about your system.
## Building tests
Extensive tests are included. Build them using CMake:
mkdir build
cd build
cmake ..
make
Call `ctest` to run the tests.
The unit and reader tests are always build, the writer tests are only build if
the Google Protobuf library is found when running CMake.
See `test/README.md` for more details about the test.
## Coverage report
To get a coverage report set `CXXFLAGS` and `LDFLAGS` before calling CMake:
CXXFLAGS="--coverage" LDFLAGS="--coverage" cmake ..
Then call `make` as usual and run the tests using `ctest`.
If you are using `g++` use `gcov` to generate a report (results are in `*.gcov`
files):
gcov -lp $(find test/ -name '*.o')
If you are using `clang++` use `llvm-cov` instead:
llvm-cov gcov -lp $(find test/ -name '*.o')
If you are using `g++` you can use `gcovr` to generate nice HTML output:
mkdir -p coverage
gcovr . -r SRCDIR --html --html-details -o coverage/index.html
Open `coverage/index.html` in your browser to see the report.
## Clang-tidy
After the CMake step, run
make clang-tidy
to check the code with [clang-tidy](https://clang.llvm.org/extra/clang-tidy/).
You might have to set `CLANG_TIDY` in CMake config.
## Cppcheck
For extra checks with [Cppcheck](https://cppcheck.sourceforge.io/) you can,
after the CMake step, call
make cppcheck
## Installation
After the CMake step, call `make install` to install the include files in
`/usr/local/include/protozero`.
If you are using CMake to build your own software, you can copy the file
`cmake/FindProtozero.cmake` and use it in your build. See the file for
details.
## Who is using Protozero?
* [Carmen](https://github.com/mapbox/carmen-cache)
* [Libosmium](https://github.com/osmcode/libosmium)
* [Mapbox GL Native](https://github.com/mapbox/mapbox-gl-native)
* [Mapbox Vector Tile library](https://github.com/mapbox/vector-tile)
* [Mapnik](https://github.com/mapbox/mapnik-vector-tile)
* [OSRM](https://github.com/Project-OSRM/osrm-backend)
* [Tippecanoe](https://github.com/mapbox/tippecanoe)
* [Vtzero](https://github.com/mapbox/vtzero)
Are you using Protozero? Tell us! Send a pull request with changes to this
README.
+101
View File
@@ -0,0 +1,101 @@
# Upgrading
This file contains instructions for users of Protozero who are upgrading from
one version to another.
You do not need to change anything if only the minor version changes, but it
is better to keep up with changes if you can. The switch to the next major
version will be easier then. And you might get some more convenient usage.
To help you with upgrading to new versions, you can define the C++ preprocessor
macro `PROTOZERO_STRICT_API` in which case Protozero will compile without the
code used for backwards compatibilty. You will then get compile errors for
older API usages.
## Upgrading from *v1.6* to *v1.7*
* The `pbf_writer` class is now a typedef for `basic_pbf_writer<std::string>`
If you have forward declared it in your code, it might have to change.
## Upgrading from *v1.5* to *v1.6.0*
* The `data_view` class moved from `types.hpp` into its own header file
`data_view.hpp`. Most people should not include those headers directly,
but if you do, you might have to change your includes.
* There are two new exceptions `invalid_tag_exception` and
`invalid_length_exception` which cover cases that were only checked by
`assert` before this version. If you catch specific exceptions in your code
you might have to amend it. But just catching `protozero::exception` is
usually fine for most code (if you catch exceptions at all).
* The `pbf_reader` constructor taking a `std::pair` is now deprecated. If you
are compiling with `PROTOZERO_STRICT_API` it is not available any more. Use
one of the other constructors instead.
## Upgrading from *v1.4.5* to *v1.5.0*
* New functions for checking tag and type at the same time to make your
program more robust. Read the section "Repeated fields in messages" in
the new [Advanced Topics documentation](doc/advanced.md).
## Upgrading from *v1.4.4* to *v1.4.5*
* The macro `PROTOZERO_DO_NOT_USE_BARE_POINTER` is not used any more. If you
have been setting this, remove it.
## Upgrading from *v1.4.0* to *v1.4.1*
* You can now do `require('protozero')` in nodejs to print the path
to the include paths for the protozero headers.
## Upgrading from *v1.3.0* to *v1.4.0*
* Functions in `pbf_reader` (and the derived `pbf_message`) called
`get_packed_*()` now return an `iterator_range` instead of a `std::pair`.
The new class is derived from `std::pair`, so changes are usually not
strictly necessary. For future compatibility, you should change all
attribute accesses on the returned objects from `first` and `second` to
`begin()` and `end()`, respectively. So change something like this:
auto x = message.get_packed_int32();
for (auto it = x.first; it != x.second; ++it) {
....
}
to:
auto x = message.get_packed_int32();
for (auto it = x.begin(); it != x.end(); ++it) {
....
}
or even better use the range-based for loop:
auto x = message.get_packed_int32();
for (auto val : x) {
....
}
Ranges can also be used in this way. This will change the range in-place:
auto range = message.get_packed_int32();
while (!range.empty()) {
auto value = range.front();
range.drop_front();
....
}
* The class `pbf_reader` has a new method `get_view()` returning an object
of the new `protozero::data_view` class. The `data_view` only has minimal
functionality, but what it has is compatible to the `std::string_view` class
which will be coming in C++17. The view autoconverts to a `std::string` if
needed. Use `get_view()` instead of `get_data()` giving you a more intuitive
interface (call `data()` and `size()` on the view instead of using `first`
and `second` on the `std::pair` returned by `get_data()`).
You can set the macro `PROTOZERO_USE_VIEW` (before including `types.hpp`) to
the name of any class that behaves like `protozero::data_view` and
`data_view` will be an alias to that class instead of the implementation
from protozero. This way you can use the C++17 `string_view` or a similar
class if it is already available on your system.
+71
View File
@@ -0,0 +1,71 @@
#-----------------------------------------------------------------------------
#
# Configuration for continuous integration service at appveyor.com
#
#-----------------------------------------------------------------------------
platform: x64
image: Visual Studio 2017
clone_depth: 1
#-----------------------------------------------------------------------------
environment:
matrix:
- config: Debug
autocrlf: true
- config: RelWithDebInfo
autocrlf: true
- config: Debug
autocrlf: false
- config: RelWithDebInfo
autocrlf: false
- config: Debug
autocrlf: false
platform: x86
- config: MSYS2
autocrlf: true
APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019
#-----------------------------------------------------------------------------
init:
- git config --global core.autocrlf %autocrlf%
- git config --get core.autocrlf
# The option --ask=20 is a workaround for a problem with the MSYS2 update
# process. Without it the following error is printed and the appveyor script
# halts: "msys2-runtime and catgets are in conflict. Remove catgets?"
# See also: https://github.com/Alexpux/MSYS2-packages/issues/1141
install:
- if "%config%"=="MSYS2" (
set "PATH=C:\msys64\mingw64\bin;C:\msys64\usr\bin;%PATH%" &&
pacman --noconfirm --sync --refresh --refresh --sysupgrade --sysupgrade --ask=20 &&
pacman -Rc --noconfirm mingw-w64-x86_64-gcc-libs &&
pacman -S --needed --noconfirm mingw-w64-x86_64-gcc mingw-w64-x86_64-cmake mingw-w64-x86_64-doxygen mingw-w64-x86_64-protobuf
)
build_script:
- cd c:\projects\protozero
- mkdir build
- cd build
- if "%platform%"=="x64" (
set vcvarsall_arg=amd64
) else (
set vcvarsall_arg=x86
)
- if "%config%"=="MSYS2" (
cmake .. -LA -G "MSYS Makefiles" &&
make VERBOSE=1
) else (
"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall" %vcvarsall_arg% &&
cmake .. -LA -G "NMake Makefiles" -DCMAKE_BUILD_TYPE=%config% &&
nmake VERBOSE=1
)
test_script:
- ctest --output-on-failure
#-----------------------------------------------------------------------------
+13
View File
@@ -0,0 +1,13 @@
mapbox-streets-v6/14/8714/8017.vector.pbf
- https://c.tile.openstreetmap.org/14/8714/8017.png
- https://a.tiles.mapbox.com/v4/mapbox.mapbox-streets-v6/14/8714/8017.vector.pbf
- https://www.mapbox.com/developers/vector-tiles/mapbox-streets/
enf-14-4824-6157.vector.pbf
- enf.8k273nmi
- https://b.tiles.mapbox.com/v4/enf.c3a2de35/14/4824/6157@2x.png
- https://www.mapbox.com/blog/twitter-map-every-tweet/
Binary file not shown.
+63
View File
@@ -0,0 +1,63 @@
#----------------------------------------------------------------------
#
# FindProtozero.cmake
#
# Find the protozero headers.
#
#----------------------------------------------------------------------
#
# Usage:
#
# Copy this file somewhere into your project directory, where cmake can
# find it. Usually this will be a directory called "cmake" which you can
# add to the CMake module search path with the following line in your
# CMakeLists.txt:
#
# list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake")
#
# Then add the following in your CMakeLists.txt:
#
# find_package(Protozero [version] [REQUIRED])
# include_directories(SYSTEM ${PROTOZERO_INCLUDE_DIR})
#
# The version number is optional. If it is not set, any version of
# protozero will do.
#
# if(NOT PROTOZERO_FOUND)
# message(WARNING "Protozero not found!\n")
# endif()
#
#----------------------------------------------------------------------
#
# Variables:
#
# PROTOZERO_FOUND - True if Protozero was found.
# PROTOZERO_INCLUDE_DIR - Where to find include files.
#
#----------------------------------------------------------------------
# find include path
find_path(PROTOZERO_INCLUDE_DIR protozero/version.hpp
PATH_SUFFIXES include
PATHS ${CMAKE_SOURCE_DIR}/../protozero
)
# Check version number
if(Protozero_FIND_VERSION)
file(STRINGS "${PROTOZERO_INCLUDE_DIR}/protozero/version.hpp" _version_define REGEX "#define PROTOZERO_VERSION_STRING")
if("${_version_define}" MATCHES "#define PROTOZERO_VERSION_STRING \"([0-9.]+)\"")
set(_version "${CMAKE_MATCH_1}")
else()
set(_version "unknown")
endif()
endif()
#set(PROTOZERO_INCLUDE_DIRS "${PROTOZERO_INCLUDE_DIR}")
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(Protozero
REQUIRED_VARS PROTOZERO_INCLUDE_DIR
VERSION_VAR _version)
#----------------------------------------------------------------------
+37
View File
@@ -0,0 +1,37 @@
#-----------------------------------------------------------------------------
#
# CMake Config
#
# protozero documentation
#
#-----------------------------------------------------------------------------
message(STATUS "Configuring documentation")
message(STATUS "Looking for doxygen")
find_package(Doxygen)
if(DOXYGEN_FOUND AND DOXYGEN_DOT_FOUND)
message(STATUS "Looking for doxygen - found")
configure_file(Doxyfile.in ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile @ONLY)
file(GLOB HEADER_FILES "${CMAKE_SOURCE_DIR}/include/protozero/*.hpp")
add_custom_command(OUTPUT html/index.html
COMMAND ${DOXYGEN_EXECUTABLE}
ARGS ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile
DEPENDS Doxyfile.in advanced.md cheatsheet.md tutorial.md
${HEADER_FILES}
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
COMMENT "Generating API documentation with Doxygen" VERBATIM)
add_custom_target(doc ALL
DEPENDS html/index.html)
else()
message(STATUS "Looking for doxygen - not found")
message(STATUS " Disabled making of documentation.")
endif()
#-----------------------------------------------------------------------------
message(STATUS "Configuring documentation - done")
#-----------------------------------------------------------------------------
File diff suppressed because it is too large. Load diff
+326
View File
@@ -0,0 +1,326 @@
# Protozero Advanced Topics
This documentation contains some mixed advanced topics for Protozero users.
Read the [tutorial](tutorial.md) first if you are new to Protozero.
## Limitations of Protozero
* A protobuf message has to fit into memory completely, otherwise it can not
be parsed with this library. There is no streaming support.
* The length of a string, bytes, or submessage can't be more than 2^31-1.
* There is no specific support for maps but they can be used as described in
the "Backwards compatibility" section of
https://developers.google.com/protocol-buffers/docs/proto3#maps.
## Checking the Protozero version number
If `protozero/version.hpp` is included, the following macros are set:
| Macro | Example | Description |
| -------------------------- | ------- | ---------------------------------------------- |
| `PROTOZERO_VERSION_MAJOR` | 1 | Major version number |
| `PROTOZERO_VERSION_MINOR` | 3 | Minor version number |
| `PROTOZERO_VERSION_PATCH` | 2 | Patch number |
| `PROTOZERO_VERSION_CODE` | 10302 | Version (major * 10,000 + minor * 100 + patch) |
| `PROTOZERO_VERSION_STRING` | "1.3.2" | Version string |
## Changing Protozero behaviour with macros
The behaviour of Protozero can be changed by defining the following macros.
They have to be set before including any of the Protozero headers.
### `PROTOZERO_STRICT_API`
If this is set, you will get some extra warnings or errors during compilation
if you are using an old (deprecated) interface to Protozero. Enable this if
you want to make sure your code will work with future versions of Protozero.
### `PROTOZERO_USE_VIEW`
Protozero uses the class `protozero::data_view` as the return type of the
`pbf_reader::get_view()` method and a few other functions take a
`protozero::data_view` as parameter.
If `PROTOZERO_USE_VIEW` is unset, `protozero::data_view` is Protozero's own
implementation of a *string view* class.
Set this macro if you want to use a different implementation such as the C++17
`std::string_view` class. In this case `protozero::data_view` will simply be
an alias to the class you specify.
#define PROTOZERO_USE_VIEW std::string_view
## Repeated fields in messages
The Google Protobuf spec documents that a non-repeated field can actually
appear several times in a message and the implementation is required to return
the value of the last version of that field in this case. `pbf_reader.hpp` does
not enforce this. If this feature is needed in your case, you have to do this
yourself.
The [spec also
says](https://developers.google.com/protocol-buffers/docs/encoding#packed)
that you must be able to read a packed repeated field where a not-packed
repeated field is expected and vice versa. Also there can be several (packed or
not-packed) repeated fields with the same tag and their contents must be
concatenated. It is your responsibility to do this, Protozero doesn't do that
for you.
### Using `tag_and_type()`
The `tag_and_type()` free function and the method of the same name on the
`pbf_reader` and `pbf_message` classes can be used to access both packed and
unpacked repeated fields. (It can also be used to check that you have the
right type of encoding for other fields.)
Here is the outline:
```cpp
enum class ExampleMsg : protozero::pbf_tag_type {
repeated_uint32_x = 1
};
std::string data = ...
pbf_message<ExampleMsg> message{data};
while (message.next()) {
switch (message.tag_and_type()) {
case tag_and_type(ExampleMsg::repeated_uint32_x, pbf_wire_type::length_delimited): {
auto xit = message.get_packed_uint32();
... // handle the repeated field when it is packed
}
break;
case tag_and_type(ExampleMsg::repeated_uint32_x, pbf_wire_type::varint): {
auto x = message.get_uint32();
... // handle the repeated field when it is not packed
}
break;
default:
message.skip();
}
}
```
All this works on `pbf_reader` in the same way as with `pbf_message` with the
usual difference that `pbf_reader` takes a numeric field tag and `pbf_message`
an enum field.
If you only want to check for one specific tag and type you can use the
two-argument version of `pbf_reader::next()`. In this case `17` is the field
tag we are looking for:
```cpp
std::string data = ...
pbf_reader message{data};
while (message.next(17, pbf_wire_type::varint)) {
auto foo = message.get_int32();
...
}
```
See the test under `test/t/tag_and_type/` for a complete example.
## Reserving memory when writing messages
If you know beforehand how large a message will become or can take an educated
guess, you can call the usual `std::string::reserve()` on the underlying string
before you give it to an `pbf_writer` or `pbf_builder` object.
Or you can (at any time) call `reserve()` on the `pbf_writer` or `pbf_builder`.
This will reserve the given amount of bytes *in addition to whatever is already
in that message*. (Note that this behaviour is different then what `reserve()`
does on `std::string` or `std::vector`.)
In the general case it is not easy to figure out how much memory you will need
because of the varint packing of integers. But sometimes you can make at least
a rough estimate. Still, you should probably only use this facility if you have
benchmarks proving that it actually makes your program faster.
## Using the low-level varint and zigzag encoding and decoding functions
Protozero gives you access to the low-level functions for encoding and
decoding varint and zigzag integer encodings, because these functions can
sometimes be useful outside the Protocol Buffer context.
### Using low-level functions
To use the low-level functions, add this include to your C++ program:
```cpp
#include <protozero/varint.hpp>
```
### Functions
The following functions are then available:
```cpp
decode_varint()
write_varint()
encode_zigzag32()
encode_zigzag64()
decode_zigzag32()
decode_zigzag64()
```
See the reference documentation created by `make doc` for details.
## Vectored input for length-delimited fields
Length-delimited fields (like string fields, byte fields and messages) are
usually set by calling `add_string()`, `add_message()`, etc. These functions
have several forms, but they basically all take a *tag*, a *size*, and a
*pointer to the data*. They write the length of the data into the message
and then copy the data over.
Sometimes you have the data not in one place, but spread over several
buffers. In this case you have to consolidate those buffers first, which needs
an extra copy. Say you have two very long strings that should be concatenated
into a message:
```cpp
std::string a{"very long string..."};
std::string b{"another very long string..."};
std::string data;
protozero::pbf_writer writer{data};
a.append(b); // expensive extra copy
writer.add_string(1, a);
```
To avoid this, the function `add_bytes_vectored()` can be used which allows
vectored (or scatter/gather) input like this:
```cpp
std::string a{"very long string..."};
std::string b{"another very long string..."};
std::string data;
protozero::pbf_writer writer{data};
writer.add_bytes_vectored(1, a, b);
```
`add_bytes_vectored()` will add up the sizes of all its arguments and copy over
all the data only once.
The function takes any number of arguments. The arguments must be of a type
supporting the `data()` and `size()` methods like `protozero::data_view()`,
`std::string` or the C++17 `std::string_view`.
Note that there is only one version of the function which can be used for any
length-delimited field including strings, bytes, messages and repeated packed
fields.
The function is also available in the `pbf_builder` class.
## Internal handling of varints
When varints are decoded they are always decoded as 64bit unsigned integers and
after that casted to the type you are requesting (using `static_cast`). This
means that if the protocol buffer message was created with a different integer
type than what you are reading it with, you might get wrong results without any
warning or error. This is the same behaviour as the Google Protocol Buffers
library has.
In normal use, this should never matter, because presumably you are using the
same types to write that data as you are using to read it later. It can happen
if the data is corrupted intentionally or unintentionally in some way. But
this can't be used to feed you any data that it wasn't possible to feed you
without this behaviour, so it doesn't open up any potential problems. You
always have to check anyway that the integers are in the range you expected
them to be in if the expected range is different than the range of the integer
type. This is especially true for enums which protozero will return as
`int32_t`.
## How many items are there in a repeated packed field?
Sometimes it is useful to know how many values there are in a repeated packed
field. For instance when you want to reserve space in a `std::vector`.
```cpp
protozero::pbf_reader message{...};
message.next(...);
const auto range = message.get_packed_sint32();
std::vector<int> myvalues;
myvalues.reserve(range.size());
for (auto value : range) {
myvalues.push_back(value);
}
```
It depends on the type of range how expensive the `size()` call is. For ranges
derived from packed repeated fixed sized values the effort will be constant,
for ranges derived from packed repeated varints, the effort will be linear, but
still considerably cheaper than decoding the varints. You have to benchmark
your use case to see whether the `reserve()` (or whatever you are using the
`size()` for) is worth it.
## Using a different buffer class than std::string
Normally you are using the `pbf_writer` or `pbf_builder` classes which use a
`std::string` that you supply as their buffer for building the actual protocol
buffers message into. But you can use a different buffer implementation
instead. This might be useful if you want to use a fixed-size buffer for
instance.
The `pbf_writer` and `pbf_builder` classes are actually only aliases for the
`basic_pbf_writer` and `basic_pbf_builder` template classes:
```cpp
using pbf_writer = basic_pbf_writer<std::string>;
template <typename T>
using pbf_builder = basic_pbf_builder<std::string, T>;
```
If you want to use a different buffer type, use the `basic_*` form of the
class and use the buffer class as template parameter. When instantiating the
`basic_pbf_writer` or `basic_pbf_builder`, the only parameter to the
constructor must always be a reference to an object of the buffer class.
```cpp
some_buffer_class buffer;
basic_pbf_writer<some_buffer_class> writer{buffer};
```
For this to work you must supply template specializations for some static
functions in the `protozero::buffer_customization` struct, see
`buffer_tmpl.hpp` for details.
Protozero already supports two buffer types:
* `std::string` (to use include `protozero/buffer_string.hpp`)
* `std::vector<char>` (to use include `protozero/buffer_vector.hpp`)
There is a class `protozero::fixed_size_buffer_adaptor` you can use as adaptor
for any fixed-sized buffer you might have. Include `protozero/buffer_fixed.hpp`
to use it:
```cpp
#include <protozero/buffer_fixed.hpp>
your_buffer_class some_buffer;
protozero::fixed_size_buffer_adaptor buffer_adaptor{some_buffer.data(), some_buffer.size()};
basic_pbf_writer<protozero::fixed_size_buffer_adaptor> writer{buffer_adaptor};
```
The buffer adaptor can be initialized with any container if it supports the
`data()` and `size()` member functions:
```cpp
protozero::fixed_size_buffer_adaptor buffer_adaptor{some_buffer};
```
+67
View File
@@ -0,0 +1,67 @@
# Protozero Cheat Sheet
See also this
[handy table](https://developers.google.com/protocol-buffers/docs/proto#scalar)
from the Google Protocol Buffers documentation.
## Scalar types
| PBF Type | Underlying Storage | C++ Type | Getter | Notes |
| -------- | ------------------ | ------------- | ---------------- | ----- |
| int32 | varint | `int32_t` | `get_int32()` | |
| sint32 | varint (zigzag) | `int32_t` | `get_sint32()` | |
| uint32 | varint | `uint32_t` | `get_uint32()` | |
| int64 | varint | `int64_t` | `get_int64()` | |
| sint64 | varint (zigzag) | `int64_t` | `get_sint64()` | |
| uint64 | varint | `uint64_t` | `get_uint64()` | |
| bool | varint | `bool` | `get_bool()` | |
| enum | varint | `int32_t` | `get_enum()` | |
| fixed32 | 32bit fixed | `uint32_t` | `get_fixed32()` | |
| sfixed32 | 32bit fixed | `int32_t` | `get_sfixed32()` | |
| fixed64 | 64bit fixed | `uint64_t` | `get_fixed64()` | |
| sfixed64 | 64bit fixed | `int64_t` | `get_sfixed64()` | |
| float | 32bit fixed | `float` | `get_float()` | |
| double | 64bit fixed | `double` | `get_double()` | |
| string | length-delimited | `data_view` | `get_view()` | (1) |
| string | length-delimited | pair | `get_data()` | (2) |
| string | length-delimited | `std::string` | `get_string()` | |
| bytes | length-delimited | `data_view` | `get_view()` | (1) |
| bytes | length-delimited | pair | `get_data()` | (2) |
| bytes | length-delimited | `std::string` | `get_bytes()` | |
| message | length-delimited | `data_view` | `get_view()` | (1) |
| message | length-delimited | pair | `get_data()` | (2) |
| message | length-delimited | `pbf_reader` | `get_message()` | |
### Notes:
* (1) preferred form, returns `protozero::data_view` which is convertible to
`std::string` if needed.
* (2) deprecated form, returns `std::pair<const char*, pbf_length_type>`,
use `get_view()` instead. This form is only available if
`PROTOZERO_STRICT_API` is not defined.
* The setter function of `pbf_writer` is always `add_` + the PBF type. Several
overloads are available.
## Packed repeated fields
| PBF Type | Getter |
| -------- | ----------------------- |
| int32 | `get_packed_int32()` |
| sint32 | `get_packed_sint32()` |
| uint32 | `get_packed_uint32()` |
| int64 | `get_packed_int64()` |
| sint64 | `get_packed_sint64()` |
| uint64 | `get_packed_uint64()` |
| bool | `get_packed_bool()` |
| enum | `get_packed_enum()` |
| fixed32 | `get_packed_fixed32()` |
| sfixed32 | `get_packed_sfixed32()` |
| fixed64 | `get_packed_fixed64()` |
| sfixed64 | `get_packed_sfixed64()` |
| float | `get_packed_float()` |
| double | `get_packed_double()` |
Packed repeated fields for string, bytes, and message types are not possible.
+626
View File
@@ -0,0 +1,626 @@
# Protozero Tutorial
## Getting to know Protocol Buffers
Protozero is a very low level library. You really have to know some of the
insides of Protocol Buffers to work with it!
So before reading any further in this document, read the following from the
Protocol Buffer documentation:
* [Developer Guide - Overview](https://developers.google.com/protocol-buffers/docs/overview)
* [Language Guide](https://developers.google.com/protocol-buffers/docs/proto)
* [Encoding](https://developers.google.com/protocol-buffers/docs/encoding)
Make sure you understand the basic types of values supported by Protocol
Buffers. Refer to this
[handy table](https://developers.google.com/protocol-buffers/docs/proto#scalar)
and [the cheat sheet](cheatsheet.md) if you are getting lost.
## Prerequisites
You need a C++11-capable compiler for Protozero to work. Copy the files in the
`include/protozero` directory somewhere where your build system can find them.
Keep the `protozero` directory and include the files in the form
```cpp
#include <protozero/FILENAME.hpp>
```
## Parsing protobuf-encoded messages
### Using `pbf_reader`
To use the `pbf_reader` class, add this include to your C++ program:
```cpp
#include <protozero/pbf_reader.hpp>
```
The `pbf_reader` class contains asserts that will detect some programming
errors. We encourage you to compile with asserts enabled in your debug builds.
### An introductory example
Lets say you have a protocol description in a `.proto` file like this:
```cpp
message Example1 {
required uint32 x = 1;
optional string s = 2;
repeated fixed64 r = 17;
}
```
To read messages created according to that description, you will have code that
looks somewhat like this:
```cpp
#include <protozero/pbf_reader.hpp>
// get data from somewhere into the input string
std::string input = get_input_data();
// initialize pbf message with this data
protozero::pbf_reader message{input};
// iterate over fields in the message
while (message.next()) {
// switch depending on the field tag (the field name is not available)
switch (message.tag()) {
case 1:
// get data for tag 1 (in this case an uint32)
auto x = message.get_uint32();
break;
case 2:
// get data for tag 2 (in this case a string)
std::string s = message.get_string();
break;
case 17:
// ignore data for tag 17
message.skip();
break;
default:
// ignore data for unknown tags to allow for future extensions
message.skip();
}
}
```
You always have to call `next()` and then either one of the accessor functions
(like `get_uint32()` or `get_string()`) to get the field value or `skip()` to
ignore this field. Then call `next()` again, and so forth. Never call `next()`
twice in a row or any if the accessor or skip functions twice in a row.
Because the `pbf_reader` class doesn't know the `.proto` file it doesn't know
which field names or tags there are and it doesn't known the types of the
fields. You have to make sure to call the right `get_...()` function for each
tag. Some `assert()s` are done to check you are calling the right functions,
but not all errors can be detected.
Note that it doesn't matter whether a field is defined as `required`,
`optional`, or `repeated`. You always have to be prepared to get zero, one, or
more instances of a field and you always have to be prepared to get other
fields, too, unless you want your program to break if somebody adds a new
field.
### If you only need a single field
If, out of a protocol buffer message, you only need the value of a single
field, you can use the version of the `next()` function with a parameter:
```cpp
// same .proto file and initialization as above
// get all fields with tag 17, skip all others
while (message.next(17)) {
auto r = message.get_fixed64();
std::cout << r << "\n";
}
```
### Handling scalar fields
As you saw in the example, handling scalar field types is reasonably easy. You
just check the `.proto` file for the type of a field and call the corresponding
function called `get_` + _field type_.
For `string` and `bytes` types the internal handling is exactly the same, but
both `get_string()` and `get_bytes()` are provided to make the code
self-documenting. Both theses calls allocate and return a `std::string` which
can add some overhead. You can call the `get_view()` function instead which
returns a `data_view` containing a pointer into the data (access with `data()`)
and the length of the data (access with `size()`).
### Handling repeated packed fields
Fields that are marked as `[packed=true]` in the `.proto` file are handled
somewhat differently. `get_packed_...()` functions returning an iterator range
are used to access the data.
So, for example, if you have a protocol description in a `.proto` file like
this:
```cpp
message Example2 {
repeated sint32 i = 1 [packed=true];
}
```
You can get to the data like this:
```cpp
protozero::pbf_reader message{input.data(), input.size()};
// set current field
message.next(1);
// get an iterator range
auto pi = message.get_packed_sint32();
// iterate to get to all values
for (auto it = pi.begin(); it != pi.end(); ++it) {
std::cout << *it << '\n';
}
```
Or, with a range-based for-loop:
```cpp
for (auto value : pi) {
std::cout << v << '\n';
}
```
So you are getting a pair of normal forward iterators wrapped in an iterator
range object. The iterators can be used with any STL algorithms etc.
Note that the previous only applies to repeated **packed** fields, normal
repeated fields are handled in the usual way for scalar fields.
### Handling embedded messages
Protocol Buffers can embed any message inside another message. To access an
embedded message use the `get_message()` function. So for this description:
```cpp
message Point {
required double x = 1;
required double y = 2;
}
message Example3 {
repeated Point point = 10;
}
```
you can parse with this code:
```cpp
protozero::pbf_reader message{input};
while (message.next(10)) {
protozero::pbf_reader point = message.get_message();
double x, y;
while (point.next()) {
switch (point.tag()) {
case 1:
x = point.get_double();
break;
case 2:
y = point.get_double();
break;
default:
point.skip();
}
}
std::cout << "x=" << x << " y=" << y << "\n";
}
```
### Handling enums
Enums are stored as varints and they can't be differentiated from them. Use
the `get_enum()` function to get the value of the enum, you have to translate
this into the symbolic name yourself. See the `enum` test case for an example.
### Asserts and exceptions in the Protozero library
Protozero uses `assert()` liberally to help you find bugs in your own code when
compiled in debug mode (ie with `NDEBUG` not set). If such an assert "fires",
this is a very strong indication that there is a bug in your code somewhere.
(Protozero will disable those asserts and "convert" them into exception in its
own test code. This is done to make sure the asserts actually work as intended.
Your test code will not need this!)
Exceptions, on the other hand, are thrown by Protozero if some kind of data
corruption was detected while it is trying to parse the data. This could also
be an indicator for a bug in the user code, but because it can happen if the
data was (intentionally or not intentionally) been messed with, it is reported
to the user code using exceptions.
Most of the functions on the writer side can throw a `std::bad_alloc`
exception if there is no space to grow a buffer. Other than that no exceptions
can occur on the writer side.
All exceptions thrown by the reader side derive from `protozero::exception`.
Note that all exceptions can also happen if you are expecting a data field of
a certain type in your code but the field actually has a different type. In
that case the `pbf_reader` class might interpret the bytes in the buffer in
the wrong way and anything can happen.
#### `end_of_buffer_exception`
This will be thrown whenever any of the functions "runs out of input data".
It means you either have an incomplete message in your input or some other
data corruption has taken place.
#### `unknown_pbf_wire_type_exception`
This will be thrown if an unsupported wire type is encountered. Either your
input data is corrupted or it was written with an unsupported version of a
Protocol Buffers implementation.
#### `varint_too_long_exception`
This exception indicates an illegal encoding of a varint. It means your input
data is corrupted in some way.
#### `invalid_tag_exception`
This exception is thrown when a tag has an invalid value. Tags must be
unsigned integers between 1 and 2^29-1. Tags between 19000 and 19999 are not
allowed. See
https://developers.google.com/protocol-buffers/docs/proto#assigning-tags
#### `invalid_length_exception`
This exception is thrown when a length field of a packed repeated field is
invalid. For fixed size types the length must be a multiple of the size of
the type.
### The `pbf_reader` class
The `pbf_reader` class behaves like a value type. Objects are reasonably small
(two pointers and two `uint32_t`, so 24 bytes on a 64bit system) and they can
be copied and moved around trivially.
`pbf_reader` objects can be constructed from a `std::string` or a `const char*`
and a length field (either supplied as separate arguments or as a `std::pair`).
In all cases objects of the `pbf_reader` class store a pointer into the input
data that was given to the constructor. You have to make sure this pointer
stays valid for the duration of the objects lifetime.
## Parsing protobuf-encoded messages using `pbf_message`
One problem in the code above are the "magic numbers" used as tags for the
different fields that you got from the `.proto` file. Instead of spreading
these magic numbers around your code you can define them once in an `enum
class` and then use the `pbf_message` template class instead of the
`pbf_reader` class.
Here is the first example again, this time using this new technique. So you
have the following in a `.proto` file:
```cpp
message Example1 {
required uint32 x = 1;
optional string s = 2;
repeated fixed64 r = 17;
}
```
Add the following declaration in one of your header files:
```cpp
enum class Example1 : protozero::pbf_tag_type {
required_uint32_x = 1,
optional_string_s = 2,
repeated_fixed64_r = 17
};
```
The message name becomes the name of the `enum class` which is always built
on top of the `protozero::pbf_tag_type` type. Each field in the message
becomes one value of the enum. In this case the name is created from the
type (including the modifiers like `required` or `optional`) and the name of
the field. You can use any name you want, but this convention makes it easier
later, to get everything right.
To read messages created according to that description, you will have code that
looks somewhat like this, this time using `pbf_message` instead of
`pbf_reader`:
```cpp
#include <protozero/pbf_message.hpp>
// get data from somewhere into the input string
std::string input = get_input_data();
// initialize pbf message with this data
protozero::pbf_message<Example1> message{input};
// iterate over fields in the message
while (message.next()) {
// switch depending on the field tag (the field name is not available)
switch (message.tag()) {
case Example1::required_uint32_x:
auto x = message.get_uint32();
break;
case Example1::optional_string_s:
std::string s = message.get_string();
break;
case Example1::repeated_fixed64_r:
message.skip();
break;
default:
// ignore data for unknown tags to allow for future extensions
message.skip();
}
}
```
Note the correspondance between the enum value (for instance
`required_uint32_x`) and the name of the getter function (for instance
`get_uint32()`). This makes it easier to get the correct types. Also the
naming makes it easier to keep different message types apart if you have
multiple (or embedded) messages.
See the `test/t/complex` test case for a complete example using this interface.
Using `pbf_message` in favour of `pbf_reader` is recommended for all code.
Note that `pbf_message` derives from `pbf_reader`, so you can always fall
back to the more generic interface if necessary.
One problem you might run into is the following: The enum class lists all
possible values you know about and you'll have lots of `switch` statements
checking those values. Some compilers will know that your `switch` covers
all possible cases and warn you if you have a `default` case that looks
unneccessary to the compiler. But you still want that `default` case to allow
for future extension of those messages (and maybe also to detect corrupted
data). You can switch of this warning with `-Wno-covered-switch-default`).
## Writing protobuf-encoded messages
### Using `pbf_writer`
To use the `pbf_writer` class, add this include to your C++ program:
```cpp
#include <protozero/pbf_writer.hpp>
```
The `pbf_writer` class contains asserts that will detect some programming
errors. We encourage you to compile with asserts enabled in your debug builds.
### An introductory example
Lets say you have a protocol description in a `.proto` file like this:
```cpp
message Example {
required uint32 x = 1;
optional string s = 2;
repeated fixed64 r = 17;
}
```
To write messages created according to that description, you will have code
that looks somewhat like this:
```cpp
#include <protozero/pbf_writer.hpp>
std::string data;
protozero::pbf_writer pbf_example{data};
pbf_example.add_uint32(1, 27); // uint32_t x
pbf_example.add_fixed64(17, 1); // fixed64 r
pbf_example.add_fixed64(17, 2);
pbf_example.add_fixed64(17, 3);
pbf_example.add_string(2, "foobar"); // string s
```
First you need a string which will be used as buffer to assemble the
protobuf-formatted message. The `pbf_writer` object contains a reference to
this string buffer and through it you add data to that buffer piece by piece.
The buffer doesn't have to be empty, the `pbf_writer` will simply append its
data to whatever is there already.
### Handling scalar fields
As you could see in the introductory example handling any kind of scalar field
is easy. The type of field doesn't matter and it doesn't matter whether it is
optional, required or repeated. You always call one of the `add_TYPE()` method
on the pbf writer object.
The first parameter of these methods is always the *tag* of the field (the
field number) from the `.proto` file. The second parameter is the value you
want to set. For the `bytes` and `string` types several versions of the add
method are available taking a `const std::string&` or a `const char*` and a
length.
For `enum` types you have to use the numeric value as the symbolic names from
the `.proto` file are not available.
### Handling repeated packed fields
Repeated packed fields can easily be set from a pair of iterators:
```cpp
std::string data;
protozero::pbf_writer pw{data};
std::vector<int> v = { 1, 4, 9, 16, 25, 36 };
pw.add_packed_int32(1, std::begin(v), std::end(v));
```
If you don't have an iterator you can use the alternative form:
```cpp
std::string data;
protozero::pbf_writer pw{data};
{
protozero::packed_field_int32 field{pw, 1};
field.add_element(1);
field.add_element(10);
field.add_element(100);
}
```
Of course you can add as many elements as you want. If you add no elements
at all, this code will still work, Protozero detects this special case and
pretends you never even initialized this field.
The nested scope is important in this case, because the destructor of the
`field` object will make sure the length stored inside the field is set to
the right value. You must close that scope before adding other fields to the
`pw` pbf writer.
If you know how many elements you will add to the field and your field contains
fixed length elements, you can tell Protozero and it can optimize this case:
```cpp
std::string data;
protozero::pbf_writer pw{data};
{
protozero::packed_field_fixed32 field{pw, 1, 2}; // exactly two elements
field.add_element(42);
field.add_element(13);
}
```
In this case you have to supply exactly as many elements as you promised,
otherwise you will get a broken protobuf message.
This works for `packed_field_fixed32`, `packed_field_sfixed32`,
`packed_field_fixed64`, `packed_field_sfixed64`, `packed_field_float`, and
`packed_field_double`.
You can abandon writing of the packed field if this becomes necessary by
calling `rollback()`:
```cpp
std::string data;
protozero::pbf_writer pw{data};
{
protozero::packed_field_int32 field{pw, 1};
field.add_element(42);
// some error occurs, you don't want to have this field at all
field.rollback();
}
```
The result is the same as if the lines inside the nested brackets had never
been called. Do not try to call `add_element()` after a rollback.
### Handling sub-messages
Nested sub-messages can be handled by first creating the submessage and then
adding to the parent message:
```cpp
std::string buffer_sub;
protozero::pbf_writer pbf_sub{buffer_sub};
// add fields to sub-message
pbf_sub.add_...(...);
// ...
// sub-message is finished here
std::string buffer_parent;
protozero::pbf_writer pbf_parent{buffer_parent};
pbf_parent.add_message(1, buffer_sub);
```
This is easy to do but it has the drawback of needing a separate `std::string`
buffer. If this concerns you (and why would you use protozero and not the
Google protobuf library if it doesn't?) there is another way:
```cpp
std::string data;
protozero::pbf_writer pbf_parent{data};
// optionally add fields to parent here
pbf_parent.add_...(...);
// open a new scope
{
// create new pbf_writer with parent and the tag (field number)
// as parameters
protozero::pbf_writer pbf_sub{pbf_parent, 1};
// add fields to sub here...
pbf_sub.add_...(...);
} // closing the scope will close the sub-message
// optionally add more fields to parent here
pbf_parent.add_...(...);
```
This can be nested arbitrarily deep.
Internally the sub-message writer re-uses the buffer from the parent. It
reserves enough space in the buffer to later write the length of the submessage
into it. It then adds the contents of the submessage to the buffer. When the
`pbf_sub` writer is destructed the length of the submessage is calculated and
written in the reserved space. If less space was needed for the length field
than was available, the rest of the buffer is moved over a few bytes.
You can abandon writing of submessage if this becomes necessary by
calling `rollback()`:
```cpp
std::string data;
protozero::pbf_writer pbf_parent{data};
// open a new scope
{
// create new pbf_writer with parent and the tag (field number)
// as parameters
protozero::pbf_writer pbf_sub{pbf_parent, 1};
// add fields to sub here...
pbf_sub.add_...(...);
// some problem occurs and you want to abandon the submessage:
pbf_sub.rollback();
}
// optionally add more fields to parent here
pbf_parent.add_...(...);
```
The result is the same as if the lines inside the nested brackets had never
been called. Do not try to call any of the `add_*` functions on the submessage
after a rollback.
## Writing protobuf-encoded messages using `pbf_builder`
Just like the `pbf_message` template class wraps the `pbf_reader` class, there
is a `pbf_builder` template class wrapping the `pbf_writer` class. It is
instantiated using the same `enum class` described above and used exactly
like the `pbf_writer` class but using the values of the enum instead of bare
integers.
See the `test/t/complex` test case for a complete example using this interface.
@@ -0,0 +1,266 @@
#ifndef PROTOZERO_BASIC_PBF_BUILDER_HPP
#define PROTOZERO_BASIC_PBF_BUILDER_HPP
/*****************************************************************************
protozero - Minimalistic protocol buffer decoder and encoder in C++.
This file is from https://github.com/mapbox/protozero where you can find more
documentation.
*****************************************************************************/
/**
* @file basic_pbf_builder.hpp
*
* @brief Contains the basic_pbf_builder template class.
*/
#include "basic_pbf_writer.hpp"
#include "types.hpp"
#include <type_traits>
namespace protozero {
/**
* The basic_pbf_builder is used to write PBF formatted messages into a buffer.
* It is based on the basic_pbf_writer class and has all the same methods. The
* difference is that while the pbf_writer class takes an integer tag,
* this template class takes a tag of the template type T. The idea is that
* T will be an enumeration value and this helps reduce the possibility of
* programming errors.
*
* Almost all methods in this class can throw an std::bad_alloc exception if
* the underlying buffer class wants to resize.
*
* Read the tutorial to understand how this class is used. In most cases you
* want to use the pbf_builder class which uses a std::string as buffer type.
*/
template <typename TBuffer, typename T>
class basic_pbf_builder : public basic_pbf_writer<TBuffer> {
static_assert(std::is_same<pbf_tag_type, std::underlying_type_t<T>>::value,
"T must be enum with underlying type protozero::pbf_tag_type");
public:
/// The type of messages this class will build.
using enum_type = T;
basic_pbf_builder() = default;
/**
* Create a builder using the given string as a data store. The object
* stores a reference to that string and adds all data to it. The string
* doesn't have to be empty. The pbf_message object will just append data.
*/
explicit basic_pbf_builder(TBuffer& data) noexcept :
basic_pbf_writer<TBuffer>{data} {
}
/**
* Construct a pbf_builder for a submessage from the pbf_message or
* pbf_writer of the parent message.
*
* @param parent_writer The parent pbf_message or pbf_writer
* @param tag Tag of the field that will be written
*/
template <typename P>
basic_pbf_builder(basic_pbf_writer<TBuffer>& parent_writer, P tag) :
basic_pbf_writer<TBuffer>{parent_writer, pbf_tag_type(tag)} {
}
/// @cond INTERNAL
#define PROTOZERO_WRITER_WRAP_ADD_SCALAR(name, type) \
void add_##name(T tag, type value) { \
basic_pbf_writer<TBuffer>::add_##name(pbf_tag_type(tag), value); \
}
PROTOZERO_WRITER_WRAP_ADD_SCALAR(bool, bool)
PROTOZERO_WRITER_WRAP_ADD_SCALAR(enum, int32_t)
PROTOZERO_WRITER_WRAP_ADD_SCALAR(int32, int32_t)
PROTOZERO_WRITER_WRAP_ADD_SCALAR(sint32, int32_t)
PROTOZERO_WRITER_WRAP_ADD_SCALAR(uint32, uint32_t)
PROTOZERO_WRITER_WRAP_ADD_SCALAR(int64, int64_t)
PROTOZERO_WRITER_WRAP_ADD_SCALAR(sint64, int64_t)
PROTOZERO_WRITER_WRAP_ADD_SCALAR(uint64, uint64_t)
PROTOZERO_WRITER_WRAP_ADD_SCALAR(fixed32, uint32_t)
PROTOZERO_WRITER_WRAP_ADD_SCALAR(sfixed32, int32_t)
PROTOZERO_WRITER_WRAP_ADD_SCALAR(fixed64, uint64_t)
PROTOZERO_WRITER_WRAP_ADD_SCALAR(sfixed64, int64_t)
PROTOZERO_WRITER_WRAP_ADD_SCALAR(float, float)
PROTOZERO_WRITER_WRAP_ADD_SCALAR(double, double)
#undef PROTOZERO_WRITER_WRAP_ADD_SCALAR
/// @endcond
/**
* Add "bytes" field to data.
*
* @param tag Tag of the field
* @param value Pointer to value to be written
* @param size Number of bytes to be written
*/
void add_bytes(T tag, const char* value, std::size_t size) {
basic_pbf_writer<TBuffer>::add_bytes(pbf_tag_type(tag), value, size);
}
/**
* Add "bytes" field to data.
*
* @param tag Tag of the field
* @param value Value to be written
*/
void add_bytes(T tag, const data_view& value) {
basic_pbf_writer<TBuffer>::add_bytes(pbf_tag_type(tag), value);
}
/**
* Add "bytes" field to data.
*
* @param tag Tag of the field
* @param value Value to be written
*/
void add_bytes(T tag, const std::string& value) {
basic_pbf_writer<TBuffer>::add_bytes(pbf_tag_type(tag), value);
}
/**
* Add "bytes" field to data. Bytes from the value are written until
* a null byte is encountered. The null byte is not added.
*
* @param tag Tag of the field
* @param value Pointer to zero-delimited value to be written
*/
void add_bytes(T tag, const char* value) {
basic_pbf_writer<TBuffer>::add_bytes(pbf_tag_type(tag), value);
}
/**
* Add "bytes" field to data using vectored input. All the data in the
* 2nd and further arguments is "concatenated" with only a single copy
* into the final buffer.
*
* This will work with objects of any type supporting the data() and
* size() methods like std::string or protozero::data_view.
*
* Example:
* @code
* std::string data1 = "abc";
* std::string data2 = "xyz";
* builder.add_bytes_vectored(1, data1, data2);
* @endcode
*
* @tparam Ts List of types supporting data() and size() methods.
* @param tag Tag of the field
* @param values List of objects of types Ts with data to be appended.
*/
template <typename... Ts>
void add_bytes_vectored(T tag, Ts&&... values) {
basic_pbf_writer<TBuffer>::add_bytes_vectored(pbf_tag_type(tag), std::forward<Ts>(values)...);
}
/**
* Add "string" field to data.
*
* @param tag Tag of the field
* @param value Pointer to value to be written
* @param size Number of bytes to be written
*/
void add_string(T tag, const char* value, std::size_t size) {
basic_pbf_writer<TBuffer>::add_string(pbf_tag_type(tag), value, size);
}
/**
* Add "string" field to data.
*
* @param tag Tag of the field
* @param value Value to be written
*/
void add_string(T tag, const data_view& value) {
basic_pbf_writer<TBuffer>::add_string(pbf_tag_type(tag), value);
}
/**
* Add "string" field to data.
*
* @param tag Tag of the field
* @param value Value to be written
*/
void add_string(T tag, const std::string& value) {
basic_pbf_writer<TBuffer>::add_string(pbf_tag_type(tag), value);
}
/**
* Add "string" field to data. Bytes from the value are written until
* a null byte is encountered. The null byte is not added.
*
* @param tag Tag of the field
* @param value Pointer to value to be written
*/
void add_string(T tag, const char* value) {
basic_pbf_writer<TBuffer>::add_string(pbf_tag_type(tag), value);
}
/**
* Add "message" field to data.
*
* @param tag Tag of the field
* @param value Pointer to message to be written
* @param size Length of the message
*/
void add_message(T tag, const char* value, std::size_t size) {
basic_pbf_writer<TBuffer>::add_message(pbf_tag_type(tag), value, size);
}
/**
* Add "message" field to data.
*
* @param tag Tag of the field
* @param value Value to be written. The value must be a complete message.
*/
void add_message(T tag, const data_view& value) {
basic_pbf_writer<TBuffer>::add_message(pbf_tag_type(tag), value);
}
/**
* Add "message" field to data.
*
* @param tag Tag of the field
* @param value Value to be written. The value must be a complete message.
*/
void add_message(T tag, const std::string& value) {
basic_pbf_writer<TBuffer>::add_message(pbf_tag_type(tag), value);
}
/// @cond INTERNAL
#define PROTOZERO_WRITER_WRAP_ADD_PACKED(name) \
template <typename InputIterator> \
void add_packed_##name(T tag, InputIterator first, InputIterator last) { \
basic_pbf_writer<TBuffer>::add_packed_##name(pbf_tag_type(tag), first, last); \
}
PROTOZERO_WRITER_WRAP_ADD_PACKED(bool)
PROTOZERO_WRITER_WRAP_ADD_PACKED(enum)
PROTOZERO_WRITER_WRAP_ADD_PACKED(int32)
PROTOZERO_WRITER_WRAP_ADD_PACKED(sint32)
PROTOZERO_WRITER_WRAP_ADD_PACKED(uint32)
PROTOZERO_WRITER_WRAP_ADD_PACKED(int64)
PROTOZERO_WRITER_WRAP_ADD_PACKED(sint64)
PROTOZERO_WRITER_WRAP_ADD_PACKED(uint64)
PROTOZERO_WRITER_WRAP_ADD_PACKED(fixed32)
PROTOZERO_WRITER_WRAP_ADD_PACKED(sfixed32)
PROTOZERO_WRITER_WRAP_ADD_PACKED(fixed64)
PROTOZERO_WRITER_WRAP_ADD_PACKED(sfixed64)
PROTOZERO_WRITER_WRAP_ADD_PACKED(float)
PROTOZERO_WRITER_WRAP_ADD_PACKED(double)
#undef PROTOZERO_WRITER_WRAP_ADD_PACKED
/// @endcond
}; // class basic_pbf_builder
} // end namespace protozero
#endif // PROTOZERO_BASIC_PBF_BUILDER_HPP
File diff suppressed because it is too large. Load diff
+222
View File
@@ -0,0 +1,222 @@
#ifndef PROTOZERO_BUFFER_FIXED_HPP
#define PROTOZERO_BUFFER_FIXED_HPP
/*****************************************************************************
protozero - Minimalistic protocol buffer decoder and encoder in C++.
This file is from https://github.com/mapbox/protozero where you can find more
documentation.
*****************************************************************************/
/**
* @file buffer_fixed.hpp
*
* @brief Contains the fixed_size_buffer_adaptor class.
*/
#include "buffer_tmpl.hpp"
#include "config.hpp"
#include <algorithm>
#include <cstddef>
#include <iterator>
#include <stdexcept>
namespace protozero {
/**
* This class can be used instead of std::string if you want to create a
* vector tile in a fixed-size buffer. Any operation that needs more space
* than is available will fail with a std::length_error exception.
*/
class fixed_size_buffer_adaptor {
char* m_data;
std::size_t m_capacity;
std::size_t m_size = 0;
public:
/// @cond usual container typedefs not documented
using size_type = std::size_t;
using value_type = char;
using reference = value_type&;
using const_reference = const value_type&;
using pointer = value_type*;
using const_pointer = const value_type*;
using iterator = pointer;
using const_iterator = const_pointer;
/// @endcond
/**
* Constructor.
*
* @param data Pointer to some memory allocated for the buffer.
* @param capacity Number of bytes available.
*/
fixed_size_buffer_adaptor(char* data, std::size_t capacity) noexcept :
m_data(data),
m_capacity(capacity) {
}
/**
* Constructor.
*
* @param container Some container class supporting the member functions
* data() and size().
*/
template <typename T>
explicit fixed_size_buffer_adaptor(T& container) :
m_data(container.data()),
m_capacity(container.size()) {
}
/// Returns a pointer to the data in the buffer.
const char* data() const noexcept {
return m_data;
}
/// Returns a pointer to the data in the buffer.
char* data() noexcept {
return m_data;
}
/// The capacity this buffer was created with.
std::size_t capacity() const noexcept {
return m_capacity;
}
/// The number of bytes used in the buffer. Always <= capacity().
std::size_t size() const noexcept {
return m_size;
}
/// Return iterator to beginning of data.
char* begin() noexcept {
return m_data;
}
/// Return iterator to beginning of data.
const char* begin() const noexcept {
return m_data;
}
/// Return iterator to beginning of data.
const char* cbegin() const noexcept {
return m_data;
}
/// Return iterator to end of data.
char* end() noexcept {
return m_data + m_size;
}
/// Return iterator to end of data.
const char* end() const noexcept {
return m_data + m_size;
}
/// Return iterator to end of data.
const char* cend() const noexcept {
return m_data + m_size;
}
/// @cond INTERNAL
// Do not rely on anything beyond this point
void append(const char* data, std::size_t count) {
if (m_size + count > m_capacity) {
throw std::length_error{"fixed size data store exhausted"};
}
std::copy_n(data, count, m_data + m_size);
m_size += count;
}
void append_zeros(std::size_t count) {
if (m_size + count > m_capacity) {
throw std::length_error{"fixed size data store exhausted"};
}
std::fill_n(m_data + m_size, count, '\0');
m_size += count;
}
void resize(std::size_t size) {
protozero_assert(size < m_size);
if (size > m_capacity) {
throw std::length_error{"fixed size data store exhausted"};
}
m_size = size;
}
void erase_range(std::size_t from, std::size_t to) {
protozero_assert(from <= m_size);
protozero_assert(to <= m_size);
protozero_assert(from < to);
std::copy(m_data + to, m_data + m_size, m_data + from);
m_size -= (to - from);
}
char* at_pos(std::size_t pos) {
protozero_assert(pos <= m_size);
return m_data + pos;
}
void push_back(char ch) {
if (m_size >= m_capacity) {
throw std::length_error{"fixed size data store exhausted"};
}
m_data[m_size++] = ch;
}
/// @endcond
}; // class fixed_size_buffer_adaptor
/// @cond INTERNAL
template <>
struct buffer_customization<fixed_size_buffer_adaptor> {
static std::size_t size(const fixed_size_buffer_adaptor* buffer) noexcept {
return buffer->size();
}
static void append(fixed_size_buffer_adaptor* buffer, const char* data, std::size_t count) {
buffer->append(data, count);
}
static void append_zeros(fixed_size_buffer_adaptor* buffer, std::size_t count) {
buffer->append_zeros(count);
}
static void resize(fixed_size_buffer_adaptor* buffer, std::size_t size) {
buffer->resize(size);
}
static void reserve_additional(fixed_size_buffer_adaptor* /*buffer*/, std::size_t /*size*/) {
/* nothing to be done for fixed-size buffers */
}
static void erase_range(fixed_size_buffer_adaptor* buffer, std::size_t from, std::size_t to) {
buffer->erase_range(from, to);
}
static char* at_pos(fixed_size_buffer_adaptor* buffer, std::size_t pos) {
return buffer->at_pos(pos);
}
static void push_back(fixed_size_buffer_adaptor* buffer, char ch) {
buffer->push_back(ch);
}
};
/// @endcond
} // namespace protozero
#endif // PROTOZERO_BUFFER_FIXED_HPP
@@ -0,0 +1,78 @@
#ifndef PROTOZERO_BUFFER_STRING_HPP
#define PROTOZERO_BUFFER_STRING_HPP
/*****************************************************************************
protozero - Minimalistic protocol buffer decoder and encoder in C++.
This file is from https://github.com/mapbox/protozero where you can find more
documentation.
*****************************************************************************/
/**
* @file buffer_string.hpp
*
* @brief Contains the customization points for buffer implementation based
* on std::string
*/
#include "buffer_tmpl.hpp"
#include "config.hpp"
#include <cstddef>
#include <iterator>
#include <string>
namespace protozero {
// Implementation of buffer customizations points for std::string
/// @cond INTERNAL
template <>
struct buffer_customization<std::string> {
static std::size_t size(const std::string* buffer) noexcept {
return buffer->size();
}
static void append(std::string* buffer, const char* data, std::size_t count) {
buffer->append(data, count);
}
static void append_zeros(std::string* buffer, std::size_t count) {
buffer->append(count, '\0');
}
static void resize(std::string* buffer, std::size_t size) {
protozero_assert(size < buffer->size());
buffer->resize(size);
}
static void reserve_additional(std::string* buffer, std::size_t size) {
buffer->reserve(buffer->size() + size);
}
static void erase_range(std::string* buffer, std::size_t from, std::size_t to) {
protozero_assert(from <= buffer->size());
protozero_assert(to <= buffer->size());
protozero_assert(from <= to);
buffer->erase(buffer->begin() + static_cast<std::string::difference_type>(from),
buffer->begin() + static_cast<std::string::difference_type>(to));
}
static char* at_pos(std::string* buffer, std::size_t pos) {
protozero_assert(pos <= buffer->size());
return (&*buffer->begin()) + pos;
}
static void push_back(std::string* buffer, char ch) {
buffer->push_back(ch);
}
};
/// @endcond
} // namespace protozero
#endif // PROTOZERO_BUFFER_STRING_HPP
+113
View File
@@ -0,0 +1,113 @@
#ifndef PROTOZERO_BUFFER_TMPL_HPP
#define PROTOZERO_BUFFER_TMPL_HPP
/*****************************************************************************
protozero - Minimalistic protocol buffer decoder and encoder in C++.
This file is from https://github.com/mapbox/protozero where you can find more
documentation.
*****************************************************************************/
/**
* @file buffer_tmpl.hpp
*
* @brief Contains the customization points for buffer implementations.
*/
#include <cstddef>
#include <iterator>
#include <string>
namespace protozero {
// Implementation of buffer customizations points for std::string
/// @cond INTERNAL
template <typename T>
struct buffer_customization {
/**
* Get the number of bytes currently used in the buffer.
*
* @param buffer Pointer to the buffer.
* @returns number of bytes used in the buffer.
*/
static std::size_t size(const std::string* buffer);
/**
* Append count bytes from data to the buffer.
*
* @param buffer Pointer to the buffer.
* @param data Pointer to the data.
* @param count Number of bytes to be added to the buffer.
*/
static void append(std::string* buffer, const char* data, std::size_t count);
/**
* Append count zero bytes to the buffer.
*
* @param buffer Pointer to the buffer.
* @param count Number of bytes to be added to the buffer.
*/
static void append_zeros(std::string* buffer, std::size_t count);
/**
* Shrink the buffer to the specified size. The new size will always be
* smaller than the current size.
*
* @param buffer Pointer to the buffer.
* @param size New size of the buffer.
*
* @pre size < current size of buffer
*/
static void resize(std::string* buffer, std::size_t size);
/**
* Reserve an additional size bytes for use in the buffer. This is used for
* variable-sized buffers to tell the buffer implementation that soon more
* memory will be used. The implementation can ignore this.
*
* @param buffer Pointer to the buffer.
* @param size Number of bytes to reserve.
*/
static void reserve_additional(std::string* buffer, std::size_t size);
/**
* Delete data from the buffer. This must move back the data after the
* part being deleted and resize the buffer accordingly.
*
* @param buffer Pointer to the buffer.
* @param from Offset into the buffer where we want to erase from.
* @param to Offset into the buffer one past the last byte we want to erase.
*
* @pre from, to <= size of the buffer, from < to
*/
static void erase_range(std::string* buffer, std::size_t from, std::size_t to);
/**
* Return a pointer to the memory at the specified position in the buffer.
*
* @param buffer Pointer to the buffer.
* @param pos The position in the buffer.
* @returns pointer to the memory in the buffer at the specified position.
*
* @pre pos <= size of the buffer
*/
static char* at_pos(std::string* buffer, std::size_t pos);
/**
* Add a char to the buffer incrementing the number of chars in the buffer.
*
* @param buffer Pointer to the buffer.
* @param ch The character to add.
*/
static void push_back(std::string* buffer, char ch);
};
/// @endcond
} // namespace protozero
#endif // PROTOZERO_BUFFER_TMPL_HPP
@@ -0,0 +1,78 @@
#ifndef PROTOZERO_BUFFER_VECTOR_HPP
#define PROTOZERO_BUFFER_VECTOR_HPP
/*****************************************************************************
protozero - Minimalistic protocol buffer decoder and encoder in C++.
This file is from https://github.com/mapbox/protozero where you can find more
documentation.
*****************************************************************************/
/**
* @file buffer_vector.hpp
*
* @brief Contains the customization points for buffer implementation based
* on std::vector<char>
*/
#include "buffer_tmpl.hpp"
#include "config.hpp"
#include <cstddef>
#include <iterator>
#include <vector>
namespace protozero {
// Implementation of buffer customizations points for std::vector<char>
/// @cond INTERNAL
template <>
struct buffer_customization<std::vector<char>> {
static std::size_t size(const std::vector<char>* buffer) noexcept {
return buffer->size();
}
static void append(std::vector<char>* buffer, const char* data, std::size_t count) {
buffer->insert(buffer->end(), data, data + count);
}
static void append_zeros(std::vector<char>* buffer, std::size_t count) {
buffer->insert(buffer->end(), count, '\0');
}
static void resize(std::vector<char>* buffer, std::size_t size) {
protozero_assert(size < buffer->size());
buffer->resize(size);
}
static void reserve_additional(std::vector<char>* buffer, std::size_t size) {
buffer->reserve(buffer->size() + size);
}
static void erase_range(std::vector<char>* buffer, std::size_t from, std::size_t to) {
protozero_assert(from <= buffer->size());
protozero_assert(to <= buffer->size());
protozero_assert(from <= to);
buffer->erase(std::next(buffer->begin(), static_cast<std::string::iterator::difference_type>(from)),
std::next(buffer->begin(), static_cast<std::string::iterator::difference_type>(to)));
}
static char* at_pos(std::vector<char>* buffer, std::size_t pos) {
protozero_assert(pos <= buffer->size());
return (&*buffer->begin()) + pos;
}
static void push_back(std::vector<char>* buffer, char ch) {
buffer->push_back(ch);
}
};
/// @endcond
} // namespace protozero
#endif // PROTOZERO_BUFFER_VECTOR_HPP
+108
View File
@@ -0,0 +1,108 @@
#ifndef PROTOZERO_BYTESWAP_HPP
#define PROTOZERO_BYTESWAP_HPP
/*****************************************************************************
protozero - Minimalistic protocol buffer decoder and encoder in C++.
This file is from https://github.com/mapbox/protozero where you can find more
documentation.
*****************************************************************************/
/**
* @file byteswap.hpp
*
* @brief Contains functions to swap bytes in values (for different endianness).
*/
#include "config.hpp"
#include <cstdint>
#include <cstring>
namespace protozero {
namespace detail {
inline uint32_t byteswap_impl(uint32_t value) noexcept {
#ifdef PROTOZERO_USE_BUILTIN_BSWAP
return __builtin_bswap32(value);
#else
return ((value & 0xff000000U) >> 24U) |
((value & 0x00ff0000U) >> 8U) |
((value & 0x0000ff00U) << 8U) |
((value & 0x000000ffU) << 24U);
#endif
}
inline uint64_t byteswap_impl(uint64_t value) noexcept {
#ifdef PROTOZERO_USE_BUILTIN_BSWAP
return __builtin_bswap64(value);
#else
return ((value & 0xff00000000000000ULL) >> 56U) |
((value & 0x00ff000000000000ULL) >> 40U) |
((value & 0x0000ff0000000000ULL) >> 24U) |
((value & 0x000000ff00000000ULL) >> 8U) |
((value & 0x00000000ff000000ULL) << 8U) |
((value & 0x0000000000ff0000ULL) << 24U) |
((value & 0x000000000000ff00ULL) << 40U) |
((value & 0x00000000000000ffULL) << 56U);
#endif
}
} // end namespace detail
/// byteswap the data pointed to by ptr in-place.
inline void byteswap_inplace(uint32_t* ptr) noexcept {
*ptr = detail::byteswap_impl(*ptr);
}
/// byteswap the data pointed to by ptr in-place.
inline void byteswap_inplace(uint64_t* ptr) noexcept {
*ptr = detail::byteswap_impl(*ptr);
}
/// byteswap the data pointed to by ptr in-place.
inline void byteswap_inplace(int32_t* ptr) noexcept {
auto* bptr = reinterpret_cast<uint32_t*>(ptr);
*bptr = detail::byteswap_impl(*bptr);
}
/// byteswap the data pointed to by ptr in-place.
inline void byteswap_inplace(int64_t* ptr) noexcept {
auto* bptr = reinterpret_cast<uint64_t*>(ptr);
*bptr = detail::byteswap_impl(*bptr);
}
/// byteswap the data pointed to by ptr in-place.
inline void byteswap_inplace(float* ptr) noexcept {
static_assert(sizeof(float) == 4, "Expecting four byte float");
uint32_t tmp = 0;
std::memcpy(&tmp, ptr, 4);
tmp = detail::byteswap_impl(tmp); // uint32 overload
std::memcpy(ptr, &tmp, 4);
}
/// byteswap the data pointed to by ptr in-place.
inline void byteswap_inplace(double* ptr) noexcept {
static_assert(sizeof(double) == 8, "Expecting eight byte double");
uint64_t tmp = 0;
std::memcpy(&tmp, ptr, 8);
tmp = detail::byteswap_impl(tmp); // uint64 overload
std::memcpy(ptr, &tmp, 8);
}
namespace detail {
// Added for backwards compatibility with any code that might use this
// function (even if it shouldn't have). Will be removed in a later
// version of protozero.
using ::protozero::byteswap_inplace;
} // end namespace detail
} // end namespace protozero
#endif // PROTOZERO_BYTESWAP_HPP
+48
View File
@@ -0,0 +1,48 @@
#ifndef PROTOZERO_CONFIG_HPP
#define PROTOZERO_CONFIG_HPP
/*****************************************************************************
protozero - Minimalistic protocol buffer decoder and encoder in C++.
This file is from https://github.com/mapbox/protozero where you can find more
documentation.
*****************************************************************************/
#include <cassert>
/**
* @file config.hpp
*
* @brief Contains macro checks for different configurations.
*/
#define PROTOZERO_LITTLE_ENDIAN 1234
#define PROTOZERO_BIG_ENDIAN 4321
// Find out which byte order the machine has.
#if defined(__BYTE_ORDER)
# if (__BYTE_ORDER == __LITTLE_ENDIAN)
# define PROTOZERO_BYTE_ORDER PROTOZERO_LITTLE_ENDIAN
# endif
# if (__BYTE_ORDER == __BIG_ENDIAN)
# define PROTOZERO_BYTE_ORDER PROTOZERO_BIG_ENDIAN
# endif
#else
// This probably isn't a very good default, but might do until we figure
// out something better.
# define PROTOZERO_BYTE_ORDER PROTOZERO_LITTLE_ENDIAN
#endif
// Check whether __builtin_bswap is available
#if defined(__GNUC__) || defined(__clang__)
# define PROTOZERO_USE_BUILTIN_BSWAP
#endif
// Wrapper for assert() used for testing
#ifndef protozero_assert
# define protozero_assert(x) assert(x)
#endif
#endif // PROTOZERO_CONFIG_HPP
+236
View File
@@ -0,0 +1,236 @@
#ifndef PROTOZERO_DATA_VIEW_HPP
#define PROTOZERO_DATA_VIEW_HPP
/*****************************************************************************
protozero - Minimalistic protocol buffer decoder and encoder in C++.
This file is from https://github.com/mapbox/protozero where you can find more
documentation.
*****************************************************************************/
/**
* @file data_view.hpp
*
* @brief Contains the implementation of the data_view class.
*/
#include "config.hpp"
#include <algorithm>
#include <cstddef>
#include <cstring>
#include <string>
#include <utility>
namespace protozero {
#ifdef PROTOZERO_USE_VIEW
using data_view = PROTOZERO_USE_VIEW;
#else
/**
* Holds a pointer to some data and a length.
*
* This class is supposed to be compatible with the std::string_view
* that will be available in C++17.
*/
class data_view {
const char* m_data = nullptr;
std::size_t m_size = 0;
public:
/**
* Default constructor. Construct an empty data_view.
*/
constexpr data_view() noexcept = default;
/**
* Create data_view from pointer and size.
*
* @param ptr Pointer to the data.
* @param length Length of the data.
*/
constexpr data_view(const char* ptr, std::size_t length) noexcept
: m_data{ptr},
m_size{length} {
}
/**
* Create data_view from string.
*
* @param str String with the data.
*/
data_view(const std::string& str) noexcept // NOLINT(google-explicit-constructor, hicpp-explicit-conversions)
: m_data{str.data()},
m_size{str.size()} {
}
/**
* Create data_view from zero-terminated string.
*
* @param ptr Pointer to the data.
*/
data_view(const char* ptr) noexcept // NOLINT(google-explicit-constructor, hicpp-explicit-conversions)
: m_data{ptr},
m_size{std::strlen(ptr)} {
}
/**
* Swap the contents of this object with the other.
*
* @param other Other object to swap data with.
*/
void swap(data_view& other) noexcept {
using std::swap;
swap(m_data, other.m_data);
swap(m_size, other.m_size);
}
/// Return pointer to data.
constexpr const char* data() const noexcept {
return m_data;
}
/// Return length of data in bytes.
constexpr std::size_t size() const noexcept {
return m_size;
}
/// Returns true if size is 0.
constexpr bool empty() const noexcept {
return m_size == 0;
}
#ifndef PROTOZERO_STRICT_API
/**
* Convert data view to string.
*
* @pre Must not be default constructed data_view.
*
* @deprecated to_string() is not available in C++17 string_view so it
* should not be used to make conversion to that class easier
* in the future.
*/
std::string to_string() const {
protozero_assert(m_data);
return {m_data, m_size};
}
#endif
/**
* Convert data view to string.
*
* @pre Must not be default constructed data_view.
*/
explicit operator std::string() const {
protozero_assert(m_data);
return {m_data, m_size};
}
/**
* Compares the contents of this object with the given other object.
*
* @returns 0 if they are the same, <0 if this object is smaller than
* the other or >0 if it is larger. If both objects have the
* same size returns <0 if this object is lexicographically
* before the other, >0 otherwise.
*
* @pre Must not be default constructed data_view.
*/
int compare(data_view other) const noexcept {
assert(m_data && other.m_data);
const int cmp = std::memcmp(data(), other.data(),
std::min(size(), other.size()));
if (cmp == 0) {
if (size() == other.size()) {
return 0;
}
return size() < other.size() ? -1 : 1;
}
return cmp;
}
}; // class data_view
/**
* Swap two data_view objects.
*
* @param lhs First object.
* @param rhs Second object.
*/
inline void swap(data_view& lhs, data_view& rhs) noexcept {
lhs.swap(rhs);
}
/**
* Two data_view instances are equal if they have the same size and the
* same content.
*
* @param lhs First object.
* @param rhs Second object.
*/
constexpr bool operator==(const data_view lhs, const data_view rhs) noexcept {
return lhs.size() == rhs.size() &&
std::equal(lhs.data(), lhs.data() + lhs.size(), rhs.data());
}
/**
* Two data_view instances are not equal if they have different sizes or the
* content differs.
*
* @param lhs First object.
* @param rhs Second object.
*/
constexpr bool operator!=(const data_view lhs, const data_view rhs) noexcept {
return !(lhs == rhs);
}
/**
* Returns true if lhs.compare(rhs) < 0.
*
* @param lhs First object.
* @param rhs Second object.
*/
inline bool operator<(const data_view lhs, const data_view rhs) noexcept {
return lhs.compare(rhs) < 0;
}
/**
* Returns true if lhs.compare(rhs) <= 0.
*
* @param lhs First object.
* @param rhs Second object.
*/
inline bool operator<=(const data_view lhs, const data_view rhs) noexcept {
return lhs.compare(rhs) <= 0;
}
/**
* Returns true if lhs.compare(rhs) > 0.
*
* @param lhs First object.
* @param rhs Second object.
*/
inline bool operator>(const data_view lhs, const data_view rhs) noexcept {
return lhs.compare(rhs) > 0;
}
/**
* Returns true if lhs.compare(rhs) >= 0.
*
* @param lhs First object.
* @param rhs Second object.
*/
inline bool operator>=(const data_view lhs, const data_view rhs) noexcept {
return lhs.compare(rhs) >= 0;
}
#endif
} // end namespace protozero
#endif // PROTOZERO_DATA_VIEW_HPP
+101
View File
@@ -0,0 +1,101 @@
#ifndef PROTOZERO_EXCEPTION_HPP
#define PROTOZERO_EXCEPTION_HPP
/*****************************************************************************
protozero - Minimalistic protocol buffer decoder and encoder in C++.
This file is from https://github.com/mapbox/protozero where you can find more
documentation.
*****************************************************************************/
/**
* @file exception.hpp
*
* @brief Contains the exceptions used in the protozero library.
*/
#include <exception>
/**
* @brief All parts of the protozero header-only library are in this namespace.
*/
namespace protozero {
/**
* All exceptions explicitly thrown by the functions of the protozero library
* derive from this exception.
*/
struct exception : std::exception {
/// Returns the explanatory string.
const char* what() const noexcept override {
return "pbf exception";
}
};
/**
* This exception is thrown when parsing a varint thats larger than allowed.
* This should never happen unless the data is corrupted.
*/
struct varint_too_long_exception : exception {
/// Returns the explanatory string.
const char* what() const noexcept override {
return "varint too long exception";
}
};
/**
* This exception is thrown when the wire type of a pdf field is unknown.
* This should never happen unless the data is corrupted.
*/
struct unknown_pbf_wire_type_exception : exception {
/// Returns the explanatory string.
const char* what() const noexcept override {
return "unknown pbf field type exception";
}
};
/**
* This exception is thrown when we are trying to read a field and there
* are not enough bytes left in the buffer to read it. Almost all functions
* of the pbf_reader class can throw this exception.
*
* This should never happen unless the data is corrupted or you have
* initialized the pbf_reader object with incomplete data.
*/
struct end_of_buffer_exception : exception {
/// Returns the explanatory string.
const char* what() const noexcept override {
return "end of buffer exception";
}
};
/**
* This exception is thrown when a tag has an invalid value. Tags must be
* unsigned integers between 1 and 2^29-1. Tags between 19000 and 19999 are
* not allowed. See
* https://developers.google.com/protocol-buffers/docs/proto#assigning-tags
*/
struct invalid_tag_exception : exception {
/// Returns the explanatory string.
const char* what() const noexcept override {
return "invalid tag exception";
}
};
/**
* This exception is thrown when a length field of a packed repeated field is
* invalid. For fixed size types the length must be a multiple of the size of
* the type.
*/
struct invalid_length_exception : exception {
/// Returns the explanatory string.
const char* what() const noexcept override {
return "invalid length exception";
}
};
} // end namespace protozero
#endif // PROTOZERO_EXCEPTION_HPP
+481
View File
@@ -0,0 +1,481 @@
#ifndef PROTOZERO_ITERATORS_HPP
#define PROTOZERO_ITERATORS_HPP
/*****************************************************************************
protozero - Minimalistic protocol buffer decoder and encoder in C++.
This file is from https://github.com/mapbox/protozero where you can find more
documentation.
*****************************************************************************/
/**
* @file iterators.hpp
*
* @brief Contains the iterators for access to packed repeated fields.
*/
#include "config.hpp"
#include "varint.hpp"
#if PROTOZERO_BYTE_ORDER != PROTOZERO_LITTLE_ENDIAN
# include <protozero/byteswap.hpp>
#endif
#include <algorithm>
#include <cstring>
#include <iterator>
#include <utility>
namespace protozero {
/**
* A range of iterators based on std::pair. Created from beginning and
* end iterators. Used as a return type from some pbf_reader methods
* that is easy to use with range-based for loops.
*/
template <typename T, typename P = std::pair<T, T>>
class iterator_range :
#ifdef PROTOZERO_STRICT_API
protected
#else
public
#endif
P {
public:
/// The type of the iterators in this range.
using iterator = T;
/// The value type of the underlying iterator.
using value_type = typename std::iterator_traits<T>::value_type;
/**
* Default constructor. Create empty iterator_range.
*/
constexpr iterator_range() :
P{iterator{}, iterator{}} {
}
/**
* Create iterator range from two iterators.
*
* @param first_iterator Iterator to beginning of range.
* @param last_iterator Iterator to end of range.
*/
constexpr iterator_range(iterator&& first_iterator, iterator&& last_iterator) :
P{std::forward<iterator>(first_iterator),
std::forward<iterator>(last_iterator)} {
}
/// Return iterator to beginning of range.
constexpr iterator begin() const noexcept {
return this->first;
}
/// Return iterator to end of range.
constexpr iterator end() const noexcept {
return this->second;
}
/// Return iterator to beginning of range.
constexpr iterator cbegin() const noexcept {
return this->first;
}
/// Return iterator to end of range.
constexpr iterator cend() const noexcept {
return this->second;
}
/**
* Return true if this range is empty.
*
* Complexity: Constant.
*/
constexpr bool empty() const noexcept {
return begin() == end();
}
/**
* Get the size of the range, ie the number of elements it contains.
*
* Complexity: Constant or linear depending on the underlaying iterator.
*/
std::size_t size() const noexcept {
return static_cast<size_t>(std::distance(begin(), end()));
}
/**
* Get element at the beginning of the range.
*
* @pre Range must not be empty.
*/
value_type front() const {
protozero_assert(!empty());
return *(this->first);
}
/**
* Advance beginning of range by one.
*
* @pre Range must not be empty.
*/
void drop_front() {
protozero_assert(!empty());
++this->first;
}
/**
* Swap the contents of this range with the other.
*
* @param other Other range to swap data with.
*/
void swap(iterator_range& other) noexcept {
using std::swap;
swap(this->first, other.first);
swap(this->second, other.second);
}
}; // struct iterator_range
/**
* Swap two iterator_ranges.
*
* @param lhs First range.
* @param rhs Second range.
*/
template <typename T>
inline void swap(iterator_range<T>& lhs, iterator_range<T>& rhs) noexcept {
lhs.swap(rhs);
}
/**
* A forward iterator used for accessing packed repeated fields of fixed
* length (fixed32, sfixed32, float, double).
*/
template <typename T>
class const_fixed_iterator {
/// Pointer to current iterator position
const char* m_data = nullptr;
public:
/// @cond usual iterator functions not documented
using iterator_category = std::random_access_iterator_tag;
using value_type = T;
using difference_type = std::ptrdiff_t;
using pointer = value_type*;
using reference = value_type&;
const_fixed_iterator() noexcept = default;
explicit const_fixed_iterator(const char* data) noexcept :
m_data{data} {
}
const_fixed_iterator(const const_fixed_iterator&) noexcept = default;
const_fixed_iterator(const_fixed_iterator&&) noexcept = default;
const_fixed_iterator& operator=(const const_fixed_iterator&) noexcept = default;
const_fixed_iterator& operator=(const_fixed_iterator&&) noexcept = default;
~const_fixed_iterator() noexcept = default;
value_type operator*() const noexcept {
value_type result;
std::memcpy(&result, m_data, sizeof(value_type));
#if PROTOZERO_BYTE_ORDER != PROTOZERO_LITTLE_ENDIAN
byteswap_inplace(&result);
#endif
return result;
}
const_fixed_iterator& operator++() noexcept {
m_data += sizeof(value_type);
return *this;
}
const_fixed_iterator operator++(int) noexcept {
const const_fixed_iterator tmp{*this};
++(*this);
return tmp;
}
const_fixed_iterator& operator--() noexcept {
m_data -= sizeof(value_type);
return *this;
}
const_fixed_iterator operator--(int) noexcept {
const const_fixed_iterator tmp{*this};
--(*this);
return tmp;
}
friend bool operator==(const_fixed_iterator lhs, const_fixed_iterator rhs) noexcept {
return lhs.m_data == rhs.m_data;
}
friend bool operator!=(const_fixed_iterator lhs, const_fixed_iterator rhs) noexcept {
return !(lhs == rhs);
}
friend bool operator<(const_fixed_iterator lhs, const_fixed_iterator rhs) noexcept {
return lhs.m_data < rhs.m_data;
}
friend bool operator>(const_fixed_iterator lhs, const_fixed_iterator rhs) noexcept {
return rhs < lhs;
}
friend bool operator<=(const_fixed_iterator lhs, const_fixed_iterator rhs) noexcept {
return !(lhs > rhs);
}
friend bool operator>=(const_fixed_iterator lhs, const_fixed_iterator rhs) noexcept {
return !(lhs < rhs);
}
const_fixed_iterator& operator+=(difference_type val) noexcept {
m_data += (sizeof(value_type) * val);
return *this;
}
friend const_fixed_iterator operator+(const_fixed_iterator lhs, difference_type rhs) noexcept {
const_fixed_iterator tmp{lhs};
tmp.m_data += (sizeof(value_type) * rhs);
return tmp;
}
friend const_fixed_iterator operator+(difference_type lhs, const_fixed_iterator rhs) noexcept {
const_fixed_iterator tmp{rhs};
tmp.m_data += (sizeof(value_type) * lhs);
return tmp;
}
const_fixed_iterator& operator-=(difference_type val) noexcept {
m_data -= (sizeof(value_type) * val);
return *this;
}
friend const_fixed_iterator operator-(const_fixed_iterator lhs, difference_type rhs) noexcept {
const_fixed_iterator tmp{lhs};
tmp.m_data -= (sizeof(value_type) * rhs);
return tmp;
}
friend difference_type operator-(const_fixed_iterator lhs, const_fixed_iterator rhs) noexcept {
return static_cast<difference_type>(lhs.m_data - rhs.m_data) / static_cast<difference_type>(sizeof(T));
}
value_type operator[](difference_type n) const noexcept {
return *(*this + n);
}
/// @endcond
}; // class const_fixed_iterator
/**
* A forward iterator used for accessing packed repeated varint fields
* (int32, uint32, int64, uint64, bool, enum).
*/
template <typename T>
class const_varint_iterator {
protected:
/// Pointer to current iterator position
const char* m_data = nullptr; // NOLINT(misc-non-private-member-variables-in-classes, cppcoreguidelines-non-private-member-variables-in-classes,-warnings-as-errors)
/// Pointer to end iterator position
const char* m_end = nullptr; // NOLINT(misc-non-private-member-variables-in-classes, cppcoreguidelines-non-private-member-variables-in-classes,-warnings-as-errors)
public:
/// @cond usual iterator functions not documented
using iterator_category = std::forward_iterator_tag;
using value_type = T;
using difference_type = std::ptrdiff_t;
using pointer = value_type*;
using reference = value_type&;
static difference_type distance(const_varint_iterator begin, const_varint_iterator end) noexcept {
// The "distance" between default initialized const_varint_iterator's
// is always 0.
if (!begin.m_data) {
return 0;
}
// We know that each varint contains exactly one byte with the most
// significant bit not set. We can use this to quickly figure out
// how many varints there are without actually decoding the varints.
return std::count_if(begin.m_data, end.m_data, [](char c) noexcept {
return (static_cast<unsigned char>(c) & 0x80U) == 0;
});
}
const_varint_iterator() noexcept = default;
const_varint_iterator(const char* data, const char* end) noexcept :
m_data{data},
m_end{end} {
}
const_varint_iterator(const const_varint_iterator&) noexcept = default;
const_varint_iterator(const_varint_iterator&&) noexcept = default;
const_varint_iterator& operator=(const const_varint_iterator&) noexcept = default;
const_varint_iterator& operator=(const_varint_iterator&&) noexcept = default;
~const_varint_iterator() noexcept = default;
value_type operator*() const {
protozero_assert(m_data);
const char* d = m_data; // will be thrown away
return static_cast<value_type>(decode_varint(&d, m_end));
}
const_varint_iterator& operator++() {
protozero_assert(m_data);
skip_varint(&m_data, m_end);
return *this;
}
const_varint_iterator operator++(int) {
protozero_assert(m_data);
const const_varint_iterator tmp{*this};
++(*this);
return tmp;
}
bool operator==(const const_varint_iterator& rhs) const noexcept {
return m_data == rhs.m_data && m_end == rhs.m_end;
}
bool operator!=(const const_varint_iterator& rhs) const noexcept {
return !(*this == rhs);
}
/// @endcond
}; // class const_varint_iterator
/**
* A forward iterator used for accessing packed repeated svarint fields
* (sint32, sint64).
*/
template <typename T>
class const_svarint_iterator : public const_varint_iterator<T> {
public:
/// @cond usual iterator functions not documented
using iterator_category = std::forward_iterator_tag;
using value_type = T;
using difference_type = std::ptrdiff_t;
using pointer = value_type*;
using reference = value_type&;
const_svarint_iterator() noexcept :
const_varint_iterator<T>{} {
}
const_svarint_iterator(const char* data, const char* end) noexcept :
const_varint_iterator<T>{data, end} {
}
const_svarint_iterator(const const_svarint_iterator&) = default;
const_svarint_iterator(const_svarint_iterator&&) noexcept = default;
const_svarint_iterator& operator=(const const_svarint_iterator&) = default;
const_svarint_iterator& operator=(const_svarint_iterator&&) noexcept = default;
~const_svarint_iterator() = default;
value_type operator*() const {
protozero_assert(this->m_data);
const char* d = this->m_data; // will be thrown away
return static_cast<value_type>(decode_zigzag64(decode_varint(&d, this->m_end)));
}
const_svarint_iterator& operator++() {
protozero_assert(this->m_data);
skip_varint(&this->m_data, this->m_end);
return *this;
}
const_svarint_iterator operator++(int) {
protozero_assert(this->m_data);
const const_svarint_iterator tmp{*this};
++(*this);
return tmp;
}
/// @endcond
}; // class const_svarint_iterator
} // end namespace protozero
namespace std {
// Specialize std::distance for all the protozero iterators. Because
// functions can't be partially specialized, we have to do this for
// every value_type we are using.
/// @cond individual overloads do not need to be documented
template <>
inline typename protozero::const_varint_iterator<int32_t>::difference_type
distance<protozero::const_varint_iterator<int32_t>>(protozero::const_varint_iterator<int32_t> first, // NOLINT(readability-inconsistent-declaration-parameter-name)
protozero::const_varint_iterator<int32_t> last) {
return protozero::const_varint_iterator<int32_t>::distance(first, last);
}
template <>
inline typename protozero::const_varint_iterator<int64_t>::difference_type
distance<protozero::const_varint_iterator<int64_t>>(protozero::const_varint_iterator<int64_t> first, // NOLINT(readability-inconsistent-declaration-parameter-name)
protozero::const_varint_iterator<int64_t> last) {
return protozero::const_varint_iterator<int64_t>::distance(first, last);
}
template <>
inline typename protozero::const_varint_iterator<uint32_t>::difference_type
distance<protozero::const_varint_iterator<uint32_t>>(protozero::const_varint_iterator<uint32_t> first, // NOLINT(readability-inconsistent-declaration-parameter-name)
protozero::const_varint_iterator<uint32_t> last) {
return protozero::const_varint_iterator<uint32_t>::distance(first, last);
}
template <>
inline typename protozero::const_varint_iterator<uint64_t>::difference_type
distance<protozero::const_varint_iterator<uint64_t>>(protozero::const_varint_iterator<uint64_t> first, // NOLINT(readability-inconsistent-declaration-parameter-name)
protozero::const_varint_iterator<uint64_t> last) {
return protozero::const_varint_iterator<uint64_t>::distance(first, last);
}
template <>
inline typename protozero::const_svarint_iterator<int32_t>::difference_type
distance<protozero::const_svarint_iterator<int32_t>>(protozero::const_svarint_iterator<int32_t> first, // NOLINT(readability-inconsistent-declaration-parameter-name)
protozero::const_svarint_iterator<int32_t> last) {
return protozero::const_svarint_iterator<int32_t>::distance(first, last);
}
template <>
inline typename protozero::const_svarint_iterator<int64_t>::difference_type
distance<protozero::const_svarint_iterator<int64_t>>(protozero::const_svarint_iterator<int64_t> first, // NOLINT(readability-inconsistent-declaration-parameter-name)
protozero::const_svarint_iterator<int64_t> last) {
return protozero::const_svarint_iterator<int64_t>::distance(first, last);
}
/// @endcond
} // end namespace std
#endif // PROTOZERO_ITERATORS_HPP
+32
View File
@@ -0,0 +1,32 @@
#ifndef PROTOZERO_PBF_BUILDER_HPP
#define PROTOZERO_PBF_BUILDER_HPP
/*****************************************************************************
protozero - Minimalistic protocol buffer decoder and encoder in C++.
This file is from https://github.com/mapbox/protozero where you can find more
documentation.
*****************************************************************************/
/**
* @file pbf_builder.hpp
*
* @brief Contains the pbf_builder template class.
*/
#include "basic_pbf_builder.hpp"
#include "pbf_writer.hpp"
#include <string>
namespace protozero {
/// Specialization of basic_pbf_builder using std::string as buffer type.
template <typename T>
using pbf_builder = basic_pbf_builder<std::string, T>;
} // end namespace protozero
#endif // PROTOZERO_PBF_BUILDER_HPP
+184
View File
@@ -0,0 +1,184 @@
#ifndef PROTOZERO_PBF_MESSAGE_HPP
#define PROTOZERO_PBF_MESSAGE_HPP
/*****************************************************************************
protozero - Minimalistic protocol buffer decoder and encoder in C++.
This file is from https://github.com/mapbox/protozero where you can find more
documentation.
*****************************************************************************/
/**
* @file pbf_message.hpp
*
* @brief Contains the pbf_message template class.
*/
#include "pbf_reader.hpp"
#include "types.hpp"
#include <type_traits>
namespace protozero {
/**
* This class represents a protobuf message. Either a top-level message or
* a nested sub-message. Top-level messages can be created from any buffer
* with a pointer and length:
*
* @code
* enum class Message : protozero::pbf_tag_type {
* ...
* };
*
* std::string buffer;
* // fill buffer...
* pbf_message<Message> message{buffer.data(), buffer.size()};
* @endcode
*
* Sub-messages are created using get_message():
*
* @code
* enum class SubMessage : protozero::pbf_tag_type {
* ...
* };
*
* pbf_message<Message> message{...};
* message.next();
* pbf_message<SubMessage> submessage = message.get_message();
* @endcode
*
* All methods of the pbf_message class except get_bytes() and get_string()
* provide the strong exception guarantee, ie they either succeed or do not
* change the pbf_message object they are called on. Use the get_data() method
* instead of get_bytes() or get_string(), if you need this guarantee.
*
* This template class is based on the pbf_reader class and has all the same
* methods. The difference is that whereever the pbf_reader class takes an
* integer tag, this template class takes a tag of the template type T.
*
* Read the tutorial to understand how this class is used.
*/
template <typename T>
class pbf_message : public pbf_reader {
static_assert(std::is_same<pbf_tag_type, std::underlying_type_t<T>>::value,
"T must be enum with underlying type protozero::pbf_tag_type");
public:
/// The type of messages this class will read.
using enum_type = T;
/**
* Construct a pbf_message. All arguments are forwarded to the pbf_reader
* parent class.
*/
template <typename... Args>
pbf_message(Args&&... args) noexcept : // NOLINT(google-explicit-constructor, hicpp-explicit-conversions)
pbf_reader{std::forward<Args>(args)...} {
}
/**
* Set next field in the message as the current field. This is usually
* called in a while loop:
*
* @code
* pbf_message<...> message(...);
* while (message.next()) {
* // handle field
* }
* @endcode
*
* @returns `true` if there is a next field, `false` if not.
* @pre There must be no current field.
* @post If it returns `true` there is a current field now.
*/
bool next() {
return pbf_reader::next();
}
/**
* Set next field with given tag in the message as the current field.
* Fields with other tags are skipped. This is usually called in a while
* loop for repeated fields:
*
* @code
* pbf_message<Example1> message{...};
* while (message.next(Example1::repeated_fixed64_r)) {
* // handle field
* }
* @endcode
*
* or you can call it just once to get the one field with this tag:
*
* @code
* pbf_message<Example1> message{...};
* if (message.next(Example1::required_uint32_x)) {
* // handle field
* }
* @endcode
*
* Note that this will not check the wire type. The two-argument version
* of this function will also check the wire type.
*
* @returns `true` if there is a next field with this tag.
* @pre There must be no current field.
* @post If it returns `true` there is a current field now with the given tag.
*/
bool next(T next_tag) {
return pbf_reader::next(pbf_tag_type(next_tag));
}
/**
* Set next field with given tag and wire type in the message as the
* current field. Fields with other tags are skipped. This is usually
* called in a while loop for repeated fields:
*
* @code
* pbf_message<Example1> message{...};
* while (message.next(Example1::repeated_fixed64_r, pbf_wire_type::varint)) {
* // handle field
* }
* @endcode
*
* or you can call it just once to get the one field with this tag:
*
* @code
* pbf_message<Example1> message{...};
* if (message.next(Example1::required_uint32_x, pbf_wire_type::varint)) {
* // handle field
* }
* @endcode
*
* Note that this will also check the wire type. The one-argument version
* of this function will not check the wire type.
*
* @returns `true` if there is a next field with this tag.
* @pre There must be no current field.
* @post If it returns `true` there is a current field now with the given tag.
*/
bool next(T next_tag, pbf_wire_type type) {
return pbf_reader::next(pbf_tag_type(next_tag), type);
}
/**
* The tag of the current field. The tag is the enum value for the field
* number from the description in the .proto file.
*
* Call next() before calling this function to set the current field.
*
* @returns tag of the current field.
* @pre There must be a current field (ie. next() must have returned `true`).
*/
T tag() const noexcept {
return T(pbf_reader::tag());
}
}; // class pbf_message
} // end namespace protozero
#endif // PROTOZERO_PBF_MESSAGE_HPP
+977
View File
@@ -0,0 +1,977 @@
#ifndef PROTOZERO_PBF_READER_HPP
#define PROTOZERO_PBF_READER_HPP
/*****************************************************************************
protozero - Minimalistic protocol buffer decoder and encoder in C++.
This file is from https://github.com/mapbox/protozero where you can find more
documentation.
*****************************************************************************/
/**
* @file pbf_reader.hpp
*
* @brief Contains the pbf_reader class.
*/
#include "config.hpp"
#include "data_view.hpp"
#include "exception.hpp"
#include "iterators.hpp"
#include "types.hpp"
#include "varint.hpp"
#if PROTOZERO_BYTE_ORDER != PROTOZERO_LITTLE_ENDIAN
# include <protozero/byteswap.hpp>
#endif
#include <cstddef>
#include <cstdint>
#include <cstring>
#include <string>
#include <utility>
namespace protozero {
/**
* This class represents a protobuf message. Either a top-level message or
* a nested sub-message. Top-level messages can be created from any buffer
* with a pointer and length:
*
* @code
* std::string buffer;
* // fill buffer...
* pbf_reader message{buffer.data(), buffer.size()};
* @endcode
*
* Sub-messages are created using get_message():
*
* @code
* pbf_reader message{...};
* message.next();
* pbf_reader submessage = message.get_message();
* @endcode
*
* All methods of the pbf_reader class except get_bytes() and get_string()
* provide the strong exception guarantee, ie they either succeed or do not
* change the pbf_reader object they are called on. Use the get_view() method
* instead of get_bytes() or get_string(), if you need this guarantee.
*/
class pbf_reader {
// A pointer to the next unread data.
const char* m_data = nullptr;
// A pointer to one past the end of data.
const char* m_end = nullptr;
// The wire type of the current field.
pbf_wire_type m_wire_type = pbf_wire_type::unknown;
// The tag of the current field.
pbf_tag_type m_tag = 0;
template <typename T>
T get_fixed() {
T result;
const char* tmp_data = m_data;
skip_bytes(sizeof(T));
std::memcpy(&result, tmp_data, sizeof(T));
#if PROTOZERO_BYTE_ORDER != PROTOZERO_LITTLE_ENDIAN
byteswap_inplace(&result);
#endif
return result;
}
template <typename T>
iterator_range<const_fixed_iterator<T>> packed_fixed() {
protozero_assert(tag() != 0 && "call next() before accessing field value");
const auto len = get_len_and_skip();
if (len % sizeof(T) != 0) {
throw invalid_length_exception{};
}
return {const_fixed_iterator<T>(m_data - len),
const_fixed_iterator<T>(m_data)};
}
template <typename T>
T get_varint() {
const auto val = static_cast<T>(decode_varint(&m_data, m_end));
return val;
}
template <typename T>
T get_svarint() {
protozero_assert((has_wire_type(pbf_wire_type::varint) || has_wire_type(pbf_wire_type::length_delimited)) && "not a varint");
return static_cast<T>(decode_zigzag64(decode_varint(&m_data, m_end)));
}
pbf_length_type get_length() {
return get_varint<pbf_length_type>();
}
void skip_bytes(pbf_length_type len) {
if (m_end - m_data < static_cast<ptrdiff_t>(len)) {
throw end_of_buffer_exception{};
}
m_data += len;
#ifndef NDEBUG
// In debug builds reset the tag to zero so that we can detect (some)
// wrong code.
m_tag = 0;
#endif
}
pbf_length_type get_len_and_skip() {
const auto len = get_length();
skip_bytes(len);
return len;
}
template <typename T>
iterator_range<T> get_packed() {
protozero_assert(tag() != 0 && "call next() before accessing field value");
const auto len = get_len_and_skip();
return {T{m_data - len, m_data},
T{m_data, m_data}};
}
public:
/**
* Construct a pbf_reader message from a data_view. The pointer from the
* data_view will be stored inside the pbf_reader object, no data is
* copied. So you must make sure the view stays valid as long as the
* pbf_reader object is used.
*
* The buffer must contain a complete protobuf message.
*
* @post There is no current field.
*/
explicit pbf_reader(const data_view& view) noexcept
: m_data{view.data()},
m_end{view.data() + view.size()} {
}
/**
* Construct a pbf_reader message from a data pointer and a length. The
* pointer will be stored inside the pbf_reader object, no data is copied.
* So you must make sure the buffer stays valid as long as the pbf_reader
* object is used.
*
* The buffer must contain a complete protobuf message.
*
* @post There is no current field.
*/
pbf_reader(const char* data, std::size_t size) noexcept
: m_data{data},
m_end{data + size} {
}
#ifndef PROTOZERO_STRICT_API
/**
* Construct a pbf_reader message from a data pointer and a length. The
* pointer will be stored inside the pbf_reader object, no data is copied.
* So you must make sure the buffer stays valid as long as the pbf_reader
* object is used.
*
* The buffer must contain a complete protobuf message.
*
* @post There is no current field.
* @deprecated Use one of the other constructors.
*/
explicit pbf_reader(const std::pair<const char*, std::size_t>& data) noexcept
: m_data{data.first},
m_end{data.first + data.second} {
}
#endif
/**
* Construct a pbf_reader message from a std::string. A pointer to the
* string internals will be stored inside the pbf_reader object, no data
* is copied. So you must make sure the string is unchanged as long as the
* pbf_reader object is used.
*
* The string must contain a complete protobuf message.
*
* @post There is no current field.
*/
explicit pbf_reader(const std::string& data) noexcept
: m_data{data.data()},
m_end{data.data() + data.size()} {
}
/**
* pbf_reader can be default constructed and behaves like it has an empty
* buffer.
*/
pbf_reader() noexcept = default;
/// pbf_reader messages can be copied trivially.
pbf_reader(const pbf_reader&) noexcept = default;
/// pbf_reader messages can be moved trivially.
pbf_reader(pbf_reader&&) noexcept = default;
/// pbf_reader messages can be copied trivially.
pbf_reader& operator=(const pbf_reader& other) noexcept = default;
/// pbf_reader messages can be moved trivially.
pbf_reader& operator=(pbf_reader&& other) noexcept = default;
~pbf_reader() = default;
/**
* Swap the contents of this object with the other.
*
* @param other Other object to swap data with.
*/
void swap(pbf_reader& other) noexcept {
using std::swap;
swap(m_data, other.m_data);
swap(m_end, other.m_end);
swap(m_wire_type, other.m_wire_type);
swap(m_tag, other.m_tag);
}
/**
* In a boolean context the pbf_reader class evaluates to `true` if there
* are still fields available and to `false` if the last field has been
* read.
*/
operator bool() const noexcept { // NOLINT(google-explicit-constructor, hicpp-explicit-conversions)
return m_data != m_end;
}
/**
* Get a view of the not yet read data.
*/
data_view data() const noexcept {
return {m_data, static_cast<std::size_t>(m_end - m_data)};
}
/**
* Return the length in bytes of the current message. If you have
* already called next() and/or any of the get_*() functions, this will
* return the remaining length.
*
* This can, for instance, be used to estimate the space needed for a
* buffer. Of course you have to know reasonably well what data to expect
* and how it is encoded for this number to have any meaning.
*/
std::size_t length() const noexcept {
return static_cast<std::size_t>(m_end - m_data);
}
/**
* Set next field in the message as the current field. This is usually
* called in a while loop:
*
* @code
* pbf_reader message(...);
* while (message.next()) {
* // handle field
* }
* @endcode
*
* @returns `true` if there is a next field, `false` if not.
* @pre There must be no current field.
* @post If it returns `true` there is a current field now.
*/
bool next() {
if (m_data == m_end) {
return false;
}
const auto value = get_varint<uint32_t>();
m_tag = static_cast<pbf_tag_type>(value >> 3U);
// tags 0 and 19000 to 19999 are not allowed as per
// https://developers.google.com/protocol-buffers/docs/proto#assigning-tags
if (m_tag == 0 || (m_tag >= 19000 && m_tag <= 19999)) {
throw invalid_tag_exception{};
}
m_wire_type = pbf_wire_type(value & 0x07U);
switch (m_wire_type) {
case pbf_wire_type::varint:
case pbf_wire_type::fixed64:
case pbf_wire_type::length_delimited:
case pbf_wire_type::fixed32:
break;
default:
throw unknown_pbf_wire_type_exception{};
}
return true;
}
/**
* Set next field with given tag in the message as the current field.
* Fields with other tags are skipped. This is usually called in a while
* loop for repeated fields:
*
* @code
* pbf_reader message{...};
* while (message.next(17)) {
* // handle field
* }
* @endcode
*
* or you can call it just once to get the one field with this tag:
*
* @code
* pbf_reader message{...};
* if (message.next(17)) {
* // handle field
* }
* @endcode
*
* Note that this will not check the wire type. The two-argument version
* of this function will also check the wire type.
*
* @returns `true` if there is a next field with this tag.
* @pre There must be no current field.
* @post If it returns `true` there is a current field now with the given tag.
*/
bool next(pbf_tag_type next_tag) {
while (next()) {
if (m_tag == next_tag) {
return true;
}
skip();
}
return false;
}
/**
* Set next field with given tag and wire type in the message as the
* current field. Fields with other tags are skipped. This is usually
* called in a while loop for repeated fields:
*
* @code
* pbf_reader message{...};
* while (message.next(17, pbf_wire_type::varint)) {
* // handle field
* }
* @endcode
*
* or you can call it just once to get the one field with this tag:
*
* @code
* pbf_reader message{...};
* if (message.next(17, pbf_wire_type::varint)) {
* // handle field
* }
* @endcode
*
* Note that this will also check the wire type. The one-argument version
* of this function will not check the wire type.
*
* @returns `true` if there is a next field with this tag.
* @pre There must be no current field.
* @post If it returns `true` there is a current field now with the given tag.
*/
bool next(pbf_tag_type next_tag, pbf_wire_type type) {
while (next()) {
if (m_tag == next_tag && m_wire_type == type) {
return true;
}
skip();
}
return false;
}
/**
* The tag of the current field. The tag is the field number from the
* description in the .proto file.
*
* Call next() before calling this function to set the current field.
*
* @returns tag of the current field.
* @pre There must be a current field (ie. next() must have returned `true`).
*/
pbf_tag_type tag() const noexcept {
return m_tag;
}
/**
* Get the wire type of the current field. The wire types are:
*
* * 0 - varint
* * 1 - 64 bit
* * 2 - length-delimited
* * 5 - 32 bit
*
* All other types are illegal.
*
* Call next() before calling this function to set the current field.
*
* @returns wire type of the current field.
* @pre There must be a current field (ie. next() must have returned `true`).
*/
pbf_wire_type wire_type() const noexcept {
return m_wire_type;
}
/**
* Get the tag and wire type of the current field in one integer suitable
* for comparison with a switch statement.
*
* Use it like this:
*
* @code
* pbf_reader message{...};
* while (message.next()) {
* switch (message.tag_and_type()) {
* case tag_and_type(17, pbf_wire_type::length_delimited):
* ....
* break;
* case tag_and_type(21, pbf_wire_type::varint):
* ....
* break;
* default:
* message.skip();
* }
* }
* @endcode
*/
uint32_t tag_and_type() const noexcept {
return protozero::tag_and_type(tag(), wire_type());
}
/**
* Check the wire type of the current field.
*
* @returns `true` if the current field has the given wire type.
* @pre There must be a current field (ie. next() must have returned `true`).
*/
bool has_wire_type(pbf_wire_type type) const noexcept {
return wire_type() == type;
}
/**
* Consume the current field.
*
* @pre There must be a current field (ie. next() must have returned `true`).
* @post The current field was consumed and there is no current field now.
*/
void skip() {
protozero_assert(tag() != 0 && "call next() before calling skip()");
switch (wire_type()) {
case pbf_wire_type::varint:
skip_varint(&m_data, m_end);
break;
case pbf_wire_type::fixed64:
skip_bytes(8);
break;
case pbf_wire_type::length_delimited:
skip_bytes(get_length());
break;
case pbf_wire_type::fixed32:
skip_bytes(4);
break;
default:
break;
}
}
///@{
/**
* @name Scalar field accessor functions
*/
/**
* Consume and return value of current "bool" field.
*
* @pre There must be a current field (ie. next() must have returned `true`).
* @pre The current field must be of type "bool".
* @post The current field was consumed and there is no current field now.
*/
bool get_bool() {
protozero_assert(tag() != 0 && "call next() before accessing field value");
protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint");
const char* value = m_data;
skip_varint(&m_data, m_end);
return *value != 0;
}
/**
* Consume and return value of current "enum" field.
*
* @pre There must be a current field (ie. next() must have returned `true`).
* @pre The current field must be of type "enum".
* @post The current field was consumed and there is no current field now.
*/
int32_t get_enum() {
protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint");
return get_varint<int32_t>();
}
/**
* Consume and return value of current "int32" varint field.
*
* @pre There must be a current field (ie. next() must have returned `true`).
* @pre The current field must be of type "int32".
* @post The current field was consumed and there is no current field now.
*/
int32_t get_int32() {
protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint");
return get_varint<int32_t>();
}
/**
* Consume and return value of current "sint32" varint field.
*
* @pre There must be a current field (ie. next() must have returned `true`).
* @pre The current field must be of type "sint32".
* @post The current field was consumed and there is no current field now.
*/
int32_t get_sint32() {
protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint");
return get_svarint<int32_t>();
}
/**
* Consume and return value of current "uint32" varint field.
*
* @pre There must be a current field (ie. next() must have returned `true`).
* @pre The current field must be of type "uint32".
* @post The current field was consumed and there is no current field now.
*/
uint32_t get_uint32() {
protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint");
return get_varint<uint32_t>();
}
/**
* Consume and return value of current "int64" varint field.
*
* @pre There must be a current field (ie. next() must have returned `true`).
* @pre The current field must be of type "int64".
* @post The current field was consumed and there is no current field now.
*/
int64_t get_int64() {
protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint");
return get_varint<int64_t>();
}
/**
* Consume and return value of current "sint64" varint field.
*
* @pre There must be a current field (ie. next() must have returned `true`).
* @pre The current field must be of type "sint64".
* @post The current field was consumed and there is no current field now.
*/
int64_t get_sint64() {
protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint");
return get_svarint<int64_t>();
}
/**
* Consume and return value of current "uint64" varint field.
*
* @pre There must be a current field (ie. next() must have returned `true`).
* @pre The current field must be of type "uint64".
* @post The current field was consumed and there is no current field now.
*/
uint64_t get_uint64() {
protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint");
return get_varint<uint64_t>();
}
/**
* Consume and return value of current "fixed32" field.
*
* @pre There must be a current field (ie. next() must have returned `true`).
* @pre The current field must be of type "fixed32".
* @post The current field was consumed and there is no current field now.
*/
uint32_t get_fixed32() {
protozero_assert(tag() != 0 && "call next() before accessing field value");
protozero_assert(has_wire_type(pbf_wire_type::fixed32) && "not a 32-bit fixed");
return get_fixed<uint32_t>();
}
/**
* Consume and return value of current "sfixed32" field.
*
* @pre There must be a current field (ie. next() must have returned `true`).
* @pre The current field must be of type "sfixed32".
* @post The current field was consumed and there is no current field now.
*/
int32_t get_sfixed32() {
protozero_assert(tag() != 0 && "call next() before accessing field value");
protozero_assert(has_wire_type(pbf_wire_type::fixed32) && "not a 32-bit fixed");
return get_fixed<int32_t>();
}
/**
* Consume and return value of current "fixed64" field.
*
* @pre There must be a current field (ie. next() must have returned `true`).
* @pre The current field must be of type "fixed64".
* @post The current field was consumed and there is no current field now.
*/
uint64_t get_fixed64() {
protozero_assert(tag() != 0 && "call next() before accessing field value");
protozero_assert(has_wire_type(pbf_wire_type::fixed64) && "not a 64-bit fixed");
return get_fixed<uint64_t>();
}
/**
* Consume and return value of current "sfixed64" field.
*
* @pre There must be a current field (ie. next() must have returned `true`).
* @pre The current field must be of type "sfixed64".
* @post The current field was consumed and there is no current field now.
*/
int64_t get_sfixed64() {
protozero_assert(tag() != 0 && "call next() before accessing field value");
protozero_assert(has_wire_type(pbf_wire_type::fixed64) && "not a 64-bit fixed");
return get_fixed<int64_t>();
}
/**
* Consume and return value of current "float" field.
*
* @pre There must be a current field (ie. next() must have returned `true`).
* @pre The current field must be of type "float".
* @post The current field was consumed and there is no current field now.
*/
float get_float() {
protozero_assert(tag() != 0 && "call next() before accessing field value");
protozero_assert(has_wire_type(pbf_wire_type::fixed32) && "not a 32-bit fixed");
return get_fixed<float>();
}
/**
* Consume and return value of current "double" field.
*
* @pre There must be a current field (ie. next() must have returned `true`).
* @pre The current field must be of type "double".
* @post The current field was consumed and there is no current field now.
*/
double get_double() {
protozero_assert(tag() != 0 && "call next() before accessing field value");
protozero_assert(has_wire_type(pbf_wire_type::fixed64) && "not a 64-bit fixed");
return get_fixed<double>();
}
/**
* Consume and return value of current "bytes", "string", or "message"
* field.
*
* @returns A data_view object.
* @pre There must be a current field (ie. next() must have returned `true`).
* @pre The current field must be of type "bytes", "string", or "message".
* @post The current field was consumed and there is no current field now.
*/
data_view get_view() {
protozero_assert(tag() != 0 && "call next() before accessing field value");
protozero_assert(has_wire_type(pbf_wire_type::length_delimited) && "not of type string, bytes or message");
const auto len = get_len_and_skip();
return {m_data - len, len};
}
#ifndef PROTOZERO_STRICT_API
/**
* Consume and return value of current "bytes" or "string" field.
*
* @returns A pair with a pointer to the data and the length of the data.
* @pre There must be a current field (ie. next() must have returned `true`).
* @pre The current field must be of type "bytes" or "string".
* @post The current field was consumed and there is no current field now.
*/
std::pair<const char*, pbf_length_type> get_data() {
protozero_assert(tag() != 0 && "call next() before accessing field value");
protozero_assert(has_wire_type(pbf_wire_type::length_delimited) && "not of type string, bytes or message");
const auto len = get_len_and_skip();
return {m_data - len, len};
}
#endif
/**
* Consume and return value of current "bytes" field.
*
* @pre There must be a current field (ie. next() must have returned `true`).
* @pre The current field must be of type "bytes".
* @post The current field was consumed and there is no current field now.
*/
std::string get_bytes() {
return std::string(get_view());
}
/**
* Consume and return value of current "string" field.
*
* @pre There must be a current field (ie. next() must have returned `true`).
* @pre The current field must be of type "string".
* @post The current field was consumed and there is no current field now.
*/
std::string get_string() {
return std::string(get_view());
}
/**
* Consume and return value of current "message" field.
*
* @pre There must be a current field (ie. next() must have returned `true`).
* @pre The current field must be of type "message".
* @post The current field was consumed and there is no current field now.
*/
pbf_reader get_message() {
return pbf_reader{get_view()};
}
///@}
/// Forward iterator for iterating over bool (int32 varint) values.
using const_bool_iterator = const_varint_iterator< int32_t>;
/// Forward iterator for iterating over enum (int32 varint) values.
using const_enum_iterator = const_varint_iterator< int32_t>;
/// Forward iterator for iterating over int32 (varint) values.
using const_int32_iterator = const_varint_iterator< int32_t>;
/// Forward iterator for iterating over sint32 (varint) values.
using const_sint32_iterator = const_svarint_iterator<int32_t>;
/// Forward iterator for iterating over uint32 (varint) values.
using const_uint32_iterator = const_varint_iterator<uint32_t>;
/// Forward iterator for iterating over int64 (varint) values.
using const_int64_iterator = const_varint_iterator< int64_t>;
/// Forward iterator for iterating over sint64 (varint) values.
using const_sint64_iterator = const_svarint_iterator<int64_t>;
/// Forward iterator for iterating over uint64 (varint) values.
using const_uint64_iterator = const_varint_iterator<uint64_t>;
/// Forward iterator for iterating over fixed32 values.
using const_fixed32_iterator = const_fixed_iterator<uint32_t>;
/// Forward iterator for iterating over sfixed32 values.
using const_sfixed32_iterator = const_fixed_iterator<int32_t>;
/// Forward iterator for iterating over fixed64 values.
using const_fixed64_iterator = const_fixed_iterator<uint64_t>;
/// Forward iterator for iterating over sfixed64 values.
using const_sfixed64_iterator = const_fixed_iterator<int64_t>;
/// Forward iterator for iterating over float values.
using const_float_iterator = const_fixed_iterator<float>;
/// Forward iterator for iterating over double values.
using const_double_iterator = const_fixed_iterator<double>;
///@{
/**
* @name Repeated packed field accessor functions
*/
/**
* Consume current "repeated packed bool" field.
*
* @returns a pair of iterators to the beginning and one past the end of
* the data.
* @pre There must be a current field (ie. next() must have returned `true`).
* @pre The current field must be of type "repeated packed bool".
* @post The current field was consumed and there is no current field now.
*/
iterator_range<pbf_reader::const_bool_iterator> get_packed_bool() {
return get_packed<pbf_reader::const_bool_iterator>();
}
/**
* Consume current "repeated packed enum" field.
*
* @returns a pair of iterators to the beginning and one past the end of
* the data.
* @pre There must be a current field (ie. next() must have returned `true`).
* @pre The current field must be of type "repeated packed enum".
* @post The current field was consumed and there is no current field now.
*/
iterator_range<pbf_reader::const_enum_iterator> get_packed_enum() {
return get_packed<pbf_reader::const_enum_iterator>();
}
/**
* Consume current "repeated packed int32" field.
*
* @returns a pair of iterators to the beginning and one past the end of
* the data.
* @pre There must be a current field (ie. next() must have returned `true`).
* @pre The current field must be of type "repeated packed int32".
* @post The current field was consumed and there is no current field now.
*/
iterator_range<pbf_reader::const_int32_iterator> get_packed_int32() {
return get_packed<pbf_reader::const_int32_iterator>();
}
/**
* Consume current "repeated packed sint32" field.
*
* @returns a pair of iterators to the beginning and one past the end of
* the data.
* @pre There must be a current field (ie. next() must have returned `true`).
* @pre The current field must be of type "repeated packed sint32".
* @post The current field was consumed and there is no current field now.
*/
iterator_range<pbf_reader::const_sint32_iterator> get_packed_sint32() {
return get_packed<pbf_reader::const_sint32_iterator>();
}
/**
* Consume current "repeated packed uint32" field.
*
* @returns a pair of iterators to the beginning and one past the end of
* the data.
* @pre There must be a current field (ie. next() must have returned `true`).
* @pre The current field must be of type "repeated packed uint32".
* @post The current field was consumed and there is no current field now.
*/
iterator_range<pbf_reader::const_uint32_iterator> get_packed_uint32() {
return get_packed<pbf_reader::const_uint32_iterator>();
}
/**
* Consume current "repeated packed int64" field.
*
* @returns a pair of iterators to the beginning and one past the end of
* the data.
* @pre There must be a current field (ie. next() must have returned `true`).
* @pre The current field must be of type "repeated packed int64".
* @post The current field was consumed and there is no current field now.
*/
iterator_range<pbf_reader::const_int64_iterator> get_packed_int64() {
return get_packed<pbf_reader::const_int64_iterator>();
}
/**
* Consume current "repeated packed sint64" field.
*
* @returns a pair of iterators to the beginning and one past the end of
* the data.
* @pre There must be a current field (ie. next() must have returned `true`).
* @pre The current field must be of type "repeated packed sint64".
* @post The current field was consumed and there is no current field now.
*/
iterator_range<pbf_reader::const_sint64_iterator> get_packed_sint64() {
return get_packed<pbf_reader::const_sint64_iterator>();
}
/**
* Consume current "repeated packed uint64" field.
*
* @returns a pair of iterators to the beginning and one past the end of
* the data.
* @pre There must be a current field (ie. next() must have returned `true`).
* @pre The current field must be of type "repeated packed uint64".
* @post The current field was consumed and there is no current field now.
*/
iterator_range<pbf_reader::const_uint64_iterator> get_packed_uint64() {
return get_packed<pbf_reader::const_uint64_iterator>();
}
/**
* Consume current "repeated packed fixed32" field.
*
* @returns a pair of iterators to the beginning and one past the end of
* the data.
* @pre There must be a current field (ie. next() must have returned `true`).
* @pre The current field must be of type "repeated packed fixed32".
* @post The current field was consumed and there is no current field now.
*/
iterator_range<pbf_reader::const_fixed32_iterator> get_packed_fixed32() {
return packed_fixed<uint32_t>();
}
/**
* Consume current "repeated packed sfixed32" field.
*
* @returns a pair of iterators to the beginning and one past the end of
* the data.
* @pre There must be a current field (ie. next() must have returned `true`).
* @pre The current field must be of type "repeated packed sfixed32".
* @post The current field was consumed and there is no current field now.
*/
iterator_range<pbf_reader::const_sfixed32_iterator> get_packed_sfixed32() {
return packed_fixed<int32_t>();
}
/**
* Consume current "repeated packed fixed64" field.
*
* @returns a pair of iterators to the beginning and one past the end of
* the data.
* @pre There must be a current field (ie. next() must have returned `true`).
* @pre The current field must be of type "repeated packed fixed64".
* @post The current field was consumed and there is no current field now.
*/
iterator_range<pbf_reader::const_fixed64_iterator> get_packed_fixed64() {
return packed_fixed<uint64_t>();
}
/**
* Consume current "repeated packed sfixed64" field.
*
* @returns a pair of iterators to the beginning and one past the end of
* the data.
* @pre There must be a current field (ie. next() must have returned `true`).
* @pre The current field must be of type "repeated packed sfixed64".
* @post The current field was consumed and there is no current field now.
*/
iterator_range<pbf_reader::const_sfixed64_iterator> get_packed_sfixed64() {
return packed_fixed<int64_t>();
}
/**
* Consume current "repeated packed float" field.
*
* @returns a pair of iterators to the beginning and one past the end of
* the data.
* @pre There must be a current field (ie. next() must have returned `true`).
* @pre The current field must be of type "repeated packed float".
* @post The current field was consumed and there is no current field now.
*/
iterator_range<pbf_reader::const_float_iterator> get_packed_float() {
return packed_fixed<float>();
}
/**
* Consume current "repeated packed double" field.
*
* @returns a pair of iterators to the beginning and one past the end of
* the data.
* @pre There must be a current field (ie. next() must have returned `true`).
* @pre The current field must be of type "repeated packed double".
* @post The current field was consumed and there is no current field now.
*/
iterator_range<pbf_reader::const_double_iterator> get_packed_double() {
return packed_fixed<double>();
}
///@}
}; // class pbf_reader
/**
* Swap two pbf_reader objects.
*
* @param lhs First object.
* @param rhs Second object.
*/
inline void swap(pbf_reader& lhs, pbf_reader& rhs) noexcept {
lhs.swap(rhs);
}
} // end namespace protozero
#endif // PROTOZERO_PBF_READER_HPP
+76
View File
@@ -0,0 +1,76 @@
#ifndef PROTOZERO_PBF_WRITER_HPP
#define PROTOZERO_PBF_WRITER_HPP
/*****************************************************************************
protozero - Minimalistic protocol buffer decoder and encoder in C++.
This file is from https://github.com/mapbox/protozero where you can find more
documentation.
*****************************************************************************/
/**
* @file pbf_writer.hpp
*
* @brief Contains the pbf_writer class.
*/
#include "basic_pbf_writer.hpp"
#include "buffer_string.hpp"
#include <cstdint>
#include <string>
namespace protozero {
/**
* Specialization of basic_pbf_writer using std::string as buffer type.
*/
using pbf_writer = basic_pbf_writer<std::string>;
/// Class for generating packed repeated bool fields.
using packed_field_bool = detail::packed_field_varint<std::string, bool>;
/// Class for generating packed repeated enum fields.
using packed_field_enum = detail::packed_field_varint<std::string, int32_t>;
/// Class for generating packed repeated int32 fields.
using packed_field_int32 = detail::packed_field_varint<std::string, int32_t>;
/// Class for generating packed repeated sint32 fields.
using packed_field_sint32 = detail::packed_field_svarint<std::string, int32_t>;
/// Class for generating packed repeated uint32 fields.
using packed_field_uint32 = detail::packed_field_varint<std::string, uint32_t>;
/// Class for generating packed repeated int64 fields.
using packed_field_int64 = detail::packed_field_varint<std::string, int64_t>;
/// Class for generating packed repeated sint64 fields.
using packed_field_sint64 = detail::packed_field_svarint<std::string, int64_t>;
/// Class for generating packed repeated uint64 fields.
using packed_field_uint64 = detail::packed_field_varint<std::string, uint64_t>;
/// Class for generating packed repeated fixed32 fields.
using packed_field_fixed32 = detail::packed_field_fixed<std::string, uint32_t>;
/// Class for generating packed repeated sfixed32 fields.
using packed_field_sfixed32 = detail::packed_field_fixed<std::string, int32_t>;
/// Class for generating packed repeated fixed64 fields.
using packed_field_fixed64 = detail::packed_field_fixed<std::string, uint64_t>;
/// Class for generating packed repeated sfixed64 fields.
using packed_field_sfixed64 = detail::packed_field_fixed<std::string, int64_t>;
/// Class for generating packed repeated float fields.
using packed_field_float = detail::packed_field_fixed<std::string, float>;
/// Class for generating packed repeated double fields.
using packed_field_double = detail::packed_field_fixed<std::string, double>;
} // end namespace protozero
#endif // PROTOZERO_PBF_WRITER_HPP
+66
View File
@@ -0,0 +1,66 @@
#ifndef PROTOZERO_TYPES_HPP
#define PROTOZERO_TYPES_HPP
/*****************************************************************************
protozero - Minimalistic protocol buffer decoder and encoder in C++.
This file is from https://github.com/mapbox/protozero where you can find more
documentation.
*****************************************************************************/
/**
* @file types.hpp
*
* @brief Contains the declaration of low-level types used in the pbf format.
*/
#include "config.hpp"
#include <algorithm>
#include <cstddef>
#include <cstdint>
#include <cstring>
#include <string>
#include <utility>
namespace protozero {
/**
* The type used for field tags (field numbers).
*/
using pbf_tag_type = uint32_t;
/**
* The type used to encode type information.
* See the table on
* https://developers.google.com/protocol-buffers/docs/encoding
*/
enum class pbf_wire_type : uint32_t {
varint = 0, // int32/64, uint32/64, sint32/64, bool, enum
fixed64 = 1, // fixed64, sfixed64, double
length_delimited = 2, // string, bytes, nested messages, packed repeated fields
fixed32 = 5, // fixed32, sfixed32, float
unknown = 99 // used for default setting in this library
};
/**
* Get the tag and wire type of the current field in one integer suitable
* for comparison with a switch statement.
*
* See pbf_reader.tag_and_type() for an example how to use this.
*/
template <typename T>
constexpr uint32_t tag_and_type(T tag, pbf_wire_type wire_type) noexcept {
return (static_cast<uint32_t>(static_cast<pbf_tag_type>(tag)) << 3U) | static_cast<uint32_t>(wire_type);
}
/**
* The type used for length values, such as the length of a field.
*/
using pbf_length_type = uint32_t;
} // end namespace protozero
#endif // PROTOZERO_TYPES_HPP
+244
View File
@@ -0,0 +1,244 @@
#ifndef PROTOZERO_VARINT_HPP
#define PROTOZERO_VARINT_HPP
/*****************************************************************************
protozero - Minimalistic protocol buffer decoder and encoder in C++.
This file is from https://github.com/mapbox/protozero where you can find more
documentation.
*****************************************************************************/
/**
* @file varint.hpp
*
* @brief Contains low-level varint and zigzag encoding and decoding functions.
*/
#include "buffer_tmpl.hpp"
#include "exception.hpp"
#include <cstdint>
namespace protozero {
/**
* The maximum length of a 64 bit varint.
*/
constexpr const int8_t max_varint_length = (sizeof(uint64_t) * 8 / 7) + 1;
namespace detail {
// from https://github.com/facebook/folly/blob/master/folly/Varint.h
inline uint64_t decode_varint_impl(const char** data, const char* end) {
const auto* begin = reinterpret_cast<const int8_t*>(*data);
const auto* iend = reinterpret_cast<const int8_t*>(end);
const int8_t* p = begin;
uint64_t val = 0;
if (iend - begin >= max_varint_length) { // fast path
do {
int64_t b = *p++;
val = ((static_cast<uint64_t>(b) & 0x7fU) ); if (b >= 0) { break; }
b = *p++; val |= ((static_cast<uint64_t>(b) & 0x7fU) << 7U); if (b >= 0) { break; }
b = *p++; val |= ((static_cast<uint64_t>(b) & 0x7fU) << 14U); if (b >= 0) { break; }
b = *p++; val |= ((static_cast<uint64_t>(b) & 0x7fU) << 21U); if (b >= 0) { break; }
b = *p++; val |= ((static_cast<uint64_t>(b) & 0x7fU) << 28U); if (b >= 0) { break; }
b = *p++; val |= ((static_cast<uint64_t>(b) & 0x7fU) << 35U); if (b >= 0) { break; }
b = *p++; val |= ((static_cast<uint64_t>(b) & 0x7fU) << 42U); if (b >= 0) { break; }
b = *p++; val |= ((static_cast<uint64_t>(b) & 0x7fU) << 49U); if (b >= 0) { break; }
b = *p++; val |= ((static_cast<uint64_t>(b) & 0x7fU) << 56U); if (b >= 0) { break; }
b = *p++; val |= ((static_cast<uint64_t>(b) & 0x01U) << 63U); if (b >= 0) { break; }
throw varint_too_long_exception{};
} while (false);
} else {
unsigned int shift = 0;
while (p != iend && *p < 0) {
val |= (static_cast<uint64_t>(*p++) & 0x7fU) << shift;
shift += 7;
}
if (p == iend) {
throw end_of_buffer_exception{};
}
val |= static_cast<uint64_t>(*p++) << shift;
}
*data = reinterpret_cast<const char*>(p);
return val;
}
} // end namespace detail
/**
* Decode a 64 bit varint.
*
* Strong exception guarantee: if there is an exception the data pointer will
* not be changed.
*
* @param[in,out] data Pointer to pointer to the input data. After the function
* returns this will point to the next data to be read.
* @param[in] end Pointer one past the end of the input data.
* @returns The decoded integer
* @throws varint_too_long_exception if the varint is longer then the maximum
* length that would fit in a 64 bit int. Usually this means your data
* is corrupted or you are trying to read something as a varint that
* isn't.
* @throws end_of_buffer_exception if the *end* of the buffer was reached
* before the end of the varint.
*/
inline uint64_t decode_varint(const char** data, const char* end) {
// If this is a one-byte varint, decode it here.
if (end != *data && ((static_cast<uint64_t>(**data) & 0x80U) == 0)) {
const auto val = static_cast<uint64_t>(**data);
++(*data);
return val;
}
// If this varint is more than one byte, defer to complete implementation.
return detail::decode_varint_impl(data, end);
}
/**
* Skip over a varint.
*
* Strong exception guarantee: if there is an exception the data pointer will
* not be changed.
*
* @param[in,out] data Pointer to pointer to the input data. After the function
* returns this will point to the next data to be read.
* @param[in] end Pointer one past the end of the input data.
* @throws end_of_buffer_exception if the *end* of the buffer was reached
* before the end of the varint.
*/
inline void skip_varint(const char** data, const char* end) {
const auto* begin = reinterpret_cast<const int8_t*>(*data);
const auto* iend = reinterpret_cast<const int8_t*>(end);
const int8_t* p = begin;
while (p != iend && *p < 0) {
++p;
}
if (p - begin >= max_varint_length) {
throw varint_too_long_exception{};
}
if (p == iend) {
throw end_of_buffer_exception{};
}
++p;
*data = reinterpret_cast<const char*>(p);
}
/**
* Varint encode a 64 bit integer.
*
* @tparam T An output iterator type.
* @param data Output iterator the varint encoded value will be written to
* byte by byte.
* @param value The integer that will be encoded.
* @returns the number of bytes written
* @throws Any exception thrown by increment or dereference operator on data.
* @deprecated Use add_varint_to_buffer() instead.
*/
template <typename T>
inline int write_varint(T data, uint64_t value) {
int n = 1;
while (value >= 0x80U) {
*data++ = static_cast<char>((value & 0x7fU) | 0x80U);
value >>= 7U;
++n;
}
*data = static_cast<char>(value);
return n;
}
/**
* Varint encode a 64 bit integer.
*
* @tparam TBuffer A buffer type.
* @param buffer Output buffer the varint will be written to.
* @param value The integer that will be encoded.
* @throws Any exception thrown by calling the buffer_push_back() function.
*/
template <typename TBuffer>
inline void add_varint_to_buffer(TBuffer* buffer, uint64_t value) {
while (value >= 0x80U) {
buffer_customization<TBuffer>::push_back(buffer, static_cast<char>((value & 0x7fU) | 0x80U));
value >>= 7U;
}
buffer_customization<TBuffer>::push_back(buffer, static_cast<char>(value));
}
/**
* Varint encode a 64 bit integer.
*
* @param data Where to add the varint. There must be enough space available!
* @param value The integer that will be encoded.
* @returns the number of bytes written
*/
inline int add_varint_to_buffer(char* data, uint64_t value) noexcept {
int n = 1;
while (value >= 0x80U) {
*data++ = static_cast<char>((value & 0x7fU) | 0x80U);
value >>= 7U;
++n;
}
*data = static_cast<char>(value);
return n;
}
/**
* Get the length of the varint the specified value would produce.
*
* @param value The integer to be encoded.
* @returns the number of bytes the varint would have if we created it.
*/
inline int length_of_varint(uint64_t value) noexcept {
int n = 1;
while (value >= 0x80U) {
value >>= 7U;
++n;
}
return n;
}
/**
* ZigZag encodes a 32 bit integer.
*/
constexpr uint32_t encode_zigzag32(int32_t value) noexcept {
return (static_cast<uint32_t>(value) << 1U) ^ static_cast<uint32_t>(-static_cast<int32_t>(static_cast<uint32_t>(value) >> 31U));
}
/**
* ZigZag encodes a 64 bit integer.
*/
constexpr uint64_t encode_zigzag64(int64_t value) noexcept {
return (static_cast<uint64_t>(value) << 1U) ^ static_cast<uint64_t>(-static_cast<int64_t>(static_cast<uint64_t>(value) >> 63U));
}
/**
* Decodes a 32 bit ZigZag-encoded integer.
*/
constexpr int32_t decode_zigzag32(uint32_t value) noexcept {
return static_cast<int32_t>((value >> 1U) ^ static_cast<uint32_t>(-static_cast<int32_t>(value & 1U)));
}
/**
* Decodes a 64 bit ZigZag-encoded integer.
*/
constexpr int64_t decode_zigzag64(uint64_t value) noexcept {
return static_cast<int64_t>((value >> 1U) ^ static_cast<uint64_t>(-static_cast<int64_t>(value & 1U)));
}
} // end namespace protozero
#endif // PROTOZERO_VARINT_HPP
+34
View File
@@ -0,0 +1,34 @@
#ifndef PROTOZERO_VERSION_HPP
#define PROTOZERO_VERSION_HPP
/*****************************************************************************
protozero - Minimalistic protocol buffer decoder and encoder in C++.
This file is from https://github.com/mapbox/protozero where you can find more
documentation.
*****************************************************************************/
/**
* @file version.hpp
*
* @brief Contains macros defining the protozero version.
*/
/// The major version number
#define PROTOZERO_VERSION_MAJOR 1
/// The minor version number
#define PROTOZERO_VERSION_MINOR 8
/// The patch number
#define PROTOZERO_VERSION_PATCH 1
/// The complete version number
#define PROTOZERO_VERSION_CODE (PROTOZERO_VERSION_MAJOR * 10000 + PROTOZERO_VERSION_MINOR * 100 + PROTOZERO_VERSION_PATCH)
/// Version number as string
#define PROTOZERO_VERSION_STRING "1.8.1"
#endif // PROTOZERO_VERSION_HPP
+51
View File
@@ -0,0 +1,51 @@
#-----------------------------------------------------------------------------
#
# CMake config
#
# protozero tools
#
#-----------------------------------------------------------------------------
# Needs getopt, which is not available on Windows
if(NOT MSVC)
add_executable(pbf-decoder pbf-decoder.cpp)
add_test(NAME pbf-decoder-no-args
COMMAND pbf-decoder)
set_tests_properties(pbf-decoder-no-args PROPERTIES
PASS_REGULAR_EXPRESSION "^Usage:")
add_test(NAME pbf-decoder-help
COMMAND pbf-decoder --help)
set_tests_properties(pbf-decoder-help PROPERTIES
PASS_REGULAR_EXPRESSION "^Usage:")
add_test(NAME pbf-decoder-empty
COMMAND pbf-decoder "${CMAKE_SOURCE_DIR}/test/t/message/data-opt-empty.pbf")
set_tests_properties(pbf-decoder-empty PROPERTIES
PASS_REGULAR_EXPRESSION "^$")
add_test(NAME pbf-decoder-data
COMMAND pbf-decoder "${CMAKE_SOURCE_DIR}/test/t/message/data-message.pbf")
set_tests_properties(pbf-decoder-data PROPERTIES
PASS_REGULAR_EXPRESSION "^1:")
add_test(NAME pbf-decoder-vt
COMMAND pbf-decoder -l 999999 -o 0 "${CMAKE_SOURCE_DIR}/test/t/vector_tile/data.vector.pbf")
set_tests_properties(pbf-decoder-vt PROPERTIES
PASS_REGULAR_EXPRESSION "^3:")
add_test(NAME pbf-decoder-fail
COMMAND pbf-decoder -l 1 "${CMAKE_SOURCE_DIR}/test/t/vector_tile/data.vector.pbf")
set_tests_properties(pbf-decoder-fail PROPERTIES
WILL_FAIL true)
add_test(NAME pbf-decoder-fail-msg
COMMAND pbf-decoder -l 1 "${CMAKE_SOURCE_DIR}/test/t/vector_tile/data.vector.pbf")
set_tests_properties(pbf-decoder-fail-msg PROPERTIES
PASS_REGULAR_EXPRESSION "^end of buffer exception")
endif()
#-----------------------------------------------------------------------------
+278
View File
@@ -0,0 +1,278 @@
/*****************************************************************************
Protobuf decoder tool
Tool to decode unknown protocol buffer encoded messages. The protocol buffer
format doesn't contain enough information about the contents of a file to make
it decodable without the format description usually found in a `.proto` file,
so this tool does some informed guessing.
Usage:
pbf-decoder [OPTIONS] [FILENAME]
Use "-" as a file name to read from STDIN.
The output always goes to STDOUT.
Call with --help/-h to see more options.
*****************************************************************************/
#include <protozero/pbf_reader.hpp>
#include <algorithm>
#include <cctype>
#include <cstddef>
#include <exception>
#include <fstream>
#include <getopt.h>
#include <iostream>
#include <limits>
#include <sstream>
#include <stdexcept>
#include <string>
#include <vector>
namespace {
std::string decode(const char* data, std::size_t len, const std::string& indent);
// Try decoding as a nested message
bool decode_message(std::stringstream& out, const std::string& indent, const protozero::data_view view) {
try {
const auto nested = decode(view.data(), view.size(), indent + " ");
out << '\n' << nested;
return true;
} catch (const protozero::exception&) { // NOLINT(bugprone-empty-catch)
}
return false;
}
// Try decoding as a string (only printable characters allowed).
bool decode_printable_string(std::stringstream& out, const protozero::data_view view) {
static constexpr const std::size_t max_string_length = 60;
const std::string str{view.data(), view.size()};
if (str.find_first_not_of("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_:-") != std::string::npos) {
return false;
}
if (str.size() > max_string_length) {
out << '"' << str.substr(0, max_string_length) << "\"...\n";
} else {
out << '"' << str << '"' << '\n';
}
return true;
}
// Try decoding as a string.
bool decode_string(std::stringstream& out, const protozero::data_view view) {
static constexpr const std::size_t max_string_length = 60;
const std::string str{view.data(), std::min(view.size(), max_string_length)};
out << '"';
for (const auto c : str) {
if (std::isprint(c) != 0) {
out << c;
} else {
out << '.';
}
}
out << '"' << '\n';
return true;
}
// Print a list of numbers from a range
template <typename TRange>
void print_number_range(std::stringstream& out, const TRange& range) {
bool first = true;
for (auto val : range) {
if (first) {
first = false;
} else {
out << ',';
}
out << val;
}
out << '\n';
}
// Try decoding as packed repeated double
bool decode_packed_double(std::stringstream& out, std::size_t size, protozero::pbf_reader& message) {
if (size % 8 != 0) {
return false;
}
try {
print_number_range(out, message.get_packed_double());
return true;
} catch (const protozero::exception&) { // NOLINT(bugprone-empty-catch)
}
return false;
}
// Try decoding as packed repeated float
bool decode_packed_float(std::stringstream& out, std::size_t size, protozero::pbf_reader& message) {
if (size % 4 != 0) {
return false;
}
try {
print_number_range(out, message.get_packed_float());
return true;
} catch (const protozero::exception&) { // NOLINT(bugprone-empty-catch)
}
return false;
}
// Try decoding as packed repeated varint
bool decode_packed_varint(std::stringstream& out, protozero::pbf_reader& message) {
try {
print_number_range(out, message.get_packed_int64());
return true;
} catch (const protozero::exception&) { // NOLINT(bugprone-empty-catch)
}
return false;
}
std::string decode(const char* data, std::size_t len, const std::string& indent) {
std::stringstream stream;
protozero::pbf_reader message{data, len};
while (message.next()) {
stream << indent << message.tag() << ": ";
switch (message.wire_type()) {
case protozero::pbf_wire_type::varint: {
// This is int32, int64, uint32, uint64, sint32, sint64, bool, or enum.
// Try decoding as int64.
stream << message.get_int64() << '\n';
break;
}
case protozero::pbf_wire_type::fixed64:
// This is fixed64, sfixed64, or double.
// Try decoding as a double, because int64_t or uint64_t
// would probably be encoded as varint.
stream << message.get_double() << '\n';
break;
case protozero::pbf_wire_type::length_delimited: {
// This is string, bytes, embedded messages, or packed repeated fields.
protozero::pbf_reader message_copy{message};
const auto view = message.get_view();
decode_message(stream, indent, view) ||
decode_printable_string(stream, view) ||
decode_packed_double(stream, view.size(), message_copy) ||
decode_packed_float(stream, view.size(), message_copy) ||
decode_packed_varint(stream, message_copy) ||
decode_string(stream, view);
break;
}
case protozero::pbf_wire_type::fixed32:
// This is fixed32, sfixed32, or float.
// Try decoding as a float, because int32_t or uint32_t
// would probably be encoded as varint.
stream << message.get_float() << '\n';
break;
default:
throw protozero::unknown_pbf_wire_type_exception{};
}
}
return stream.str();
}
void print_help() {
std::cout << "Usage: pbf-decoder [OPTIONS] [INPUT_FILE]\n\n"
<< "Dump raw contents of protobuf encoded file.\n"
<< "To read from STDIN use '-' as INPUT_FILE.\n"
<< "\nOptions:\n"
<< " -h, --help This help message\n"
<< " -l, --length=LENGTH Read only LENGTH bytes\n"
<< " -o, --offset=OFFSET Start reading from OFFSET bytes\n";
}
std::vector<char> read_from_file(const char* filename) {
const std::ifstream file{filename, std::ios::binary};
return std::vector<char>{std::istreambuf_iterator<char>(file.rdbuf()),
std::istreambuf_iterator<char>()};
}
std::vector<char> read_from_stdin() {
return std::vector<char>{std::istreambuf_iterator<char>(std::cin.rdbuf()),
std::istreambuf_iterator<char>()};
}
} // anonymous namespace
int main(int argc, char* argv[]) {
static struct option long_options[] = {
{"help", no_argument, nullptr, 'h'},
{"length", required_argument, nullptr, 'l'},
{"offset", required_argument, nullptr, 'o'},
{nullptr, 0, nullptr, 0}
};
std::size_t offset = 0;
std::size_t length = std::numeric_limits<std::size_t>::max();
while (true) {
const int c = getopt_long(argc, argv, "hl:o:", long_options, nullptr); // NOLINT(concurrency-mt-unsafe) no threads here
if (c == -1) {
break;
}
switch (c) {
case 'h':
print_help();
return 0;
case 'l':
length = std::atoll(optarg); // NOLINT(cert-err34-c)
// good enough for a limited-use tool
break;
case 'o':
offset = std::atoll(optarg); // NOLINT(cert-err34-c)
// good enough for a limited-use tool
break;
default:
return 1;
}
}
const int remaining_args = argc - optind;
if (remaining_args != 1) {
std::cerr << "Usage: " << argv[0] << " [OPTIONS] [INPUT_FILE]\n\n"
<< "Call with --help/-h to see options.\n";
return 1;
}
const std::string filename{argv[optind]};
try {
std::vector<char> buffer{filename == "-" ? read_from_stdin() :
read_from_file(argv[optind])};
if (offset > buffer.size()) {
throw std::runtime_error{"offset is larger than file size"};
}
if (offset > 0) {
buffer.erase(buffer.begin(), buffer.begin() + offset);
}
if (length < buffer.size()) {
buffer.resize(length);
}
std::cout << decode(buffer.data(), buffer.size(), "");
} catch (const std::exception& ex) {
std::cerr << ex.what() << '\n';
return 1;
}
return 0;
}
+85
View File
@@ -0,0 +1,85 @@
#include <protozero/pbf_reader.hpp>
// From Google Benchmark library.
// See https://github.com/google/benchmark/blob/main/LICENSE
template <class Tp>
inline __attribute__((always_inline)) void DoNotOptimize(Tp const& value) {
asm volatile("" : : "r,m"(value) : "memory");
}
template <typename T>
int read_packed(protozero::iterator_range<T> range) {
try {
for (const auto& item : range) {
DoNotOptimize(item);
}
} catch (const protozero::exception&) {
// no-op. This is probably not a packed field of that type.
}
return 0;
}
template <typename Fn>
void try_field(const protozero::pbf_reader& reader, Fn&& fn) {
try {
DoNotOptimize(fn(reader));
} catch (const protozero::exception&) {
// no-op. This is probably not a field of that type.
}
}
int try_message(protozero::pbf_reader reader) {
try {
while (reader.next()) {
if (reader.has_wire_type(protozero::pbf_wire_type::varint)) {
// Try to decode this field as any of the types that can be encoded as varint.
try_field(reader, [](protozero::pbf_reader r) { return r.get_bool(); });
try_field(reader, [](protozero::pbf_reader r) { return r.get_enum(); });
try_field(reader, [](protozero::pbf_reader r) { return r.get_int32(); });
try_field(reader, [](protozero::pbf_reader r) { return r.get_sint32(); });
try_field(reader, [](protozero::pbf_reader r) { return r.get_uint32(); });
try_field(reader, [](protozero::pbf_reader r) { return r.get_int64(); });
try_field(reader, [](protozero::pbf_reader r) { return r.get_sint64(); });
try_field(reader, [](protozero::pbf_reader r) { return r.get_uint64(); });
} else if (reader.has_wire_type(protozero::pbf_wire_type::length_delimited)) {
// Try to decode this field as any of the types that can be encoded as length-delimited.
try_field(reader, [](protozero::pbf_reader r) { return try_message(r.get_message()); });
try_field(reader, [](protozero::pbf_reader r) { return read_packed(r.get_packed_bool()); });
try_field(reader, [](protozero::pbf_reader r) { return read_packed(r.get_packed_double()); });
try_field(reader, [](protozero::pbf_reader r) { return read_packed(r.get_packed_enum()); });
try_field(reader, [](protozero::pbf_reader r) { return read_packed(r.get_packed_fixed32()); });
try_field(reader, [](protozero::pbf_reader r) { return read_packed(r.get_packed_fixed64()); });
try_field(reader, [](protozero::pbf_reader r) { return read_packed(r.get_packed_float()); });
try_field(reader, [](protozero::pbf_reader r) { return read_packed(r.get_packed_int32()); });
try_field(reader, [](protozero::pbf_reader r) { return read_packed(r.get_packed_int64()); });
try_field(reader, [](protozero::pbf_reader r) { return read_packed(r.get_packed_sfixed32()); });
try_field(reader, [](protozero::pbf_reader r) { return read_packed(r.get_packed_sfixed64()); });
try_field(reader, [](protozero::pbf_reader r) { return read_packed(r.get_packed_sint32()); });
try_field(reader, [](protozero::pbf_reader r) { return read_packed(r.get_packed_sint64()); });
try_field(reader, [](protozero::pbf_reader r) { return read_packed(r.get_packed_uint32()); });
try_field(reader, [](protozero::pbf_reader r) { return read_packed(r.get_packed_uint64()); });
} else if (reader.has_wire_type(protozero::pbf_wire_type::fixed64)) {
// Try to decode this field as any of the types that can be encoded as fixed64.
try_field(reader, [](protozero::pbf_reader r) { return r.get_double(); });
try_field(reader, [](protozero::pbf_reader r) { return r.get_fixed64(); });
try_field(reader, [](protozero::pbf_reader r) { return r.get_sfixed64(); });
} else if (reader.has_wire_type(protozero::pbf_wire_type::fixed32)) {
// Try to decode this field as any of the types that can be encoded as fixed32.
try_field(reader, [](protozero::pbf_reader r) { return r.get_float(); });
try_field(reader, [](protozero::pbf_reader r) { return r.get_fixed32(); });
try_field(reader, [](protozero::pbf_reader r) { return r.get_sfixed32(); });
}
reader.skip();
}
} catch (const protozero::exception&) {
// no-op. This is probably not a valid message.
}
return 0;
}
extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
try_message(protozero::pbf_reader{reinterpret_cast<const char*>(data), size});
return 0;
}