989 lines
52 KiB
C++
989 lines
52 KiB
C++
#ifndef OSMIUM_IO_DETAIL_PBF_DECODER_HPP
|
|
#define OSMIUM_IO_DETAIL_PBF_DECODER_HPP
|
|
|
|
/*
|
|
|
|
This file is part of Osmium (https://osmcode.org/libosmium).
|
|
|
|
Copyright 2013-2023 Jochen Topf <jochen@topf.org> and others (see README).
|
|
|
|
Boost Software License - Version 1.0 - August 17th, 2003
|
|
|
|
Permission is hereby granted, free of charge, to any person or organization
|
|
obtaining a copy of the software and accompanying documentation covered by
|
|
this license (the "Software") to use, reproduce, display, distribute,
|
|
execute, and transmit the Software, and to prepare derivative works of the
|
|
Software, and to permit third-parties to whom the Software is furnished to
|
|
do so, all subject to the following:
|
|
|
|
The copyright notices in the Software and this entire statement, including
|
|
the above license grant, this restriction and the following disclaimer,
|
|
must be included in all copies of the Software, in whole or in part, and
|
|
all derivative works of the Software, unless such copies or derivative
|
|
works are solely in the form of machine-executable object code generated by
|
|
a source language processor.
|
|
|
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
|
|
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
|
|
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
|
|
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
|
DEALINGS IN THE SOFTWARE.
|
|
|
|
*/
|
|
|
|
#include <cstdint>
|
|
#include <cstring>
|
|
#include <limits>
|
|
#include <memory>
|
|
#include <stdexcept>
|
|
#include <string>
|
|
#include <utility>
|
|
#include <vector>
|
|
|
|
#include <osmium/builder/osm_object_builder.hpp>
|
|
#include <osmium/io/detail/pbf.hpp> // IWYU pragma: export
|
|
#include <osmium/io/detail/protobuf_tags.hpp>
|
|
#include <osmium/io/detail/zlib.hpp>
|
|
#include <osmium/io/file_format.hpp>
|
|
#include <osmium/io/header.hpp>
|
|
#include <osmium/memory/buffer.hpp>
|
|
#include <osmium/osm/box.hpp>
|
|
#include <osmium/osm/entity_bits.hpp>
|
|
#include <osmium/osm/item_type.hpp>
|
|
#include <osmium/osm/location.hpp>
|
|
#include <osmium/osm/node.hpp>
|
|
#include <osmium/osm/object.hpp>
|
|
#include <osmium/osm/relation.hpp>
|
|
#include <osmium/osm/timestamp.hpp>
|
|
#include <osmium/osm/types.hpp>
|
|
#include <osmium/osm/way.hpp>
|
|
#include <osmium/util/delta.hpp>
|
|
|
|
#ifdef OSMIUM_WITH_LZ4
|
|
# include <osmium/io/detail/lz4.hpp>
|
|
#endif
|
|
|
|
#include <protozero/iterators.hpp>
|
|
#include <protozero/pbf_message.hpp>
|
|
#include <protozero/types.hpp>
|
|
|
|
namespace osmium {
|
|
|
|
namespace builder {
|
|
class Builder;
|
|
} // namespace builder
|
|
|
|
namespace io {
|
|
|
|
namespace detail {
|
|
|
|
using protozero::data_view;
|
|
|
|
class varint_range {
|
|
|
|
const char* m_data = nullptr;
|
|
const char* m_end = nullptr;
|
|
|
|
std::uint64_t next() {
|
|
return protozero::decode_varint(&m_data, m_end);
|
|
}
|
|
|
|
public:
|
|
|
|
varint_range() = default;
|
|
|
|
explicit varint_range(const data_view& data) :
|
|
m_data(data.data()),
|
|
m_end(data.data() + data.size()) {
|
|
}
|
|
|
|
bool empty() const noexcept {
|
|
return m_data == m_end;
|
|
}
|
|
|
|
std::size_t size() const noexcept {
|
|
if (!m_data) {
|
|
return 0;
|
|
}
|
|
|
|
// We know that each varint contains exactly one byte with the most
|
|
// significant bit not set. We can use this to quickly figure out
|
|
// how many varints there are without actually decoding the varints.
|
|
return std::count_if(m_data, m_end, [](char c) noexcept {
|
|
return (static_cast<unsigned char>(c) & 0x80U) == 0;
|
|
});
|
|
}
|
|
|
|
std::int32_t next_int32() {
|
|
return static_cast<int32_t>(next());
|
|
}
|
|
|
|
std::uint32_t next_uint32() {
|
|
return static_cast<uint32_t>(next());
|
|
}
|
|
|
|
std::int32_t next_sint32() {
|
|
return protozero::decode_zigzag32(static_cast<uint32_t>(next()));
|
|
}
|
|
|
|
std::int64_t next_sint64() {
|
|
return protozero::decode_zigzag64(next());
|
|
}
|
|
|
|
}; // class varint_range
|
|
|
|
using osm_string_len_type = std::pair<const char*, osmium::string_size_type>;
|
|
|
|
class PBFPrimitiveBlockDecoder {
|
|
|
|
enum {
|
|
initial_buffer_size = 64UL * 1024UL
|
|
};
|
|
|
|
data_view m_data;
|
|
std::vector<osm_string_len_type> m_stringtable;
|
|
|
|
int64_t m_lon_offset = 0;
|
|
int64_t m_lat_offset = 0;
|
|
int64_t m_date_factor = 1000;
|
|
int32_t m_granularity = 100;
|
|
|
|
osmium::osm_entity_bits::type m_read_types;
|
|
|
|
osmium::memory::Buffer m_buffer{initial_buffer_size, osmium::memory::Buffer::auto_grow::internal};
|
|
|
|
osmium::io::read_meta m_read_metadata;
|
|
|
|
void decode_stringtable(const data_view& data) {
|
|
if (!m_stringtable.empty()) {
|
|
throw osmium::pbf_error{"more than one stringtable in pbf file"};
|
|
}
|
|
|
|
protozero::pbf_message<OSMFormat::StringTable> pbf_string_table{data};
|
|
while (pbf_string_table.next(OSMFormat::StringTable::repeated_bytes_s, protozero::pbf_wire_type::length_delimited)) {
|
|
const auto str_view = pbf_string_table.get_view();
|
|
if (str_view.size() > osmium::max_osm_string_length) {
|
|
const std::string start_of_string(str_view.data(), 20);
|
|
throw osmium::pbf_error{"overlong string (" + start_of_string + "...) in string table"};
|
|
}
|
|
m_stringtable.emplace_back(str_view.data(), static_cast<osmium::string_size_type>(str_view.size()));
|
|
}
|
|
}
|
|
|
|
void decode_primitive_block_metadata() {
|
|
protozero::pbf_message<OSMFormat::PrimitiveBlock> pbf_primitive_block{m_data};
|
|
while (pbf_primitive_block.next()) {
|
|
switch (pbf_primitive_block.tag_and_type()) {
|
|
case protozero::tag_and_type(OSMFormat::PrimitiveBlock::required_StringTable_stringtable, protozero::pbf_wire_type::length_delimited):
|
|
decode_stringtable(pbf_primitive_block.get_view());
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::PrimitiveBlock::optional_int32_granularity, protozero::pbf_wire_type::varint):
|
|
m_granularity = pbf_primitive_block.get_int32();
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::PrimitiveBlock::optional_int32_date_granularity, protozero::pbf_wire_type::varint):
|
|
m_date_factor = pbf_primitive_block.get_int32();
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::PrimitiveBlock::optional_int64_lat_offset, protozero::pbf_wire_type::varint):
|
|
m_lat_offset = pbf_primitive_block.get_int64();
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::PrimitiveBlock::optional_int64_lon_offset, protozero::pbf_wire_type::varint):
|
|
m_lon_offset = pbf_primitive_block.get_int64();
|
|
break;
|
|
default:
|
|
pbf_primitive_block.skip();
|
|
}
|
|
}
|
|
}
|
|
|
|
void decode_primitive_block_data() {
|
|
protozero::pbf_message<OSMFormat::PrimitiveBlock> pbf_primitive_block{m_data};
|
|
while (pbf_primitive_block.next(OSMFormat::PrimitiveBlock::repeated_PrimitiveGroup_primitivegroup, protozero::pbf_wire_type::length_delimited)) {
|
|
protozero::pbf_message<OSMFormat::PrimitiveGroup> pbf_primitive_group = pbf_primitive_block.get_message();
|
|
while (pbf_primitive_group.next()) {
|
|
switch (pbf_primitive_group.tag_and_type()) {
|
|
case protozero::tag_and_type(OSMFormat::PrimitiveGroup::repeated_Node_nodes, protozero::pbf_wire_type::length_delimited):
|
|
if (m_read_types & osmium::osm_entity_bits::node) {
|
|
decode_node(pbf_primitive_group.get_view());
|
|
m_buffer.commit();
|
|
} else {
|
|
pbf_primitive_group.skip();
|
|
}
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::PrimitiveGroup::optional_DenseNodes_dense, protozero::pbf_wire_type::length_delimited):
|
|
if (m_read_types & osmium::osm_entity_bits::node) {
|
|
if (m_read_metadata == osmium::io::read_meta::yes) {
|
|
decode_dense_nodes(pbf_primitive_group.get_view());
|
|
} else {
|
|
decode_dense_nodes_without_metadata(pbf_primitive_group.get_view());
|
|
}
|
|
m_buffer.commit();
|
|
} else {
|
|
pbf_primitive_group.skip();
|
|
}
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::PrimitiveGroup::repeated_Way_ways, protozero::pbf_wire_type::length_delimited):
|
|
if (m_read_types & osmium::osm_entity_bits::way) {
|
|
decode_way(pbf_primitive_group.get_view());
|
|
m_buffer.commit();
|
|
} else {
|
|
pbf_primitive_group.skip();
|
|
}
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::PrimitiveGroup::repeated_Relation_relations, protozero::pbf_wire_type::length_delimited):
|
|
if (m_read_types & osmium::osm_entity_bits::relation) {
|
|
decode_relation(pbf_primitive_group.get_view());
|
|
m_buffer.commit();
|
|
} else {
|
|
pbf_primitive_group.skip();
|
|
}
|
|
break;
|
|
default:
|
|
pbf_primitive_group.skip();
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
osm_string_len_type decode_info(const data_view& data, osmium::OSMObject& object) {
|
|
osm_string_len_type user{"", 0};
|
|
|
|
protozero::pbf_message<OSMFormat::Info> pbf_info{data};
|
|
while (pbf_info.next()) {
|
|
switch (pbf_info.tag_and_type()) {
|
|
case protozero::tag_and_type(OSMFormat::Info::optional_int32_version, protozero::pbf_wire_type::varint):
|
|
{
|
|
const auto version = pbf_info.get_int32();
|
|
if (version < -1) {
|
|
throw osmium::pbf_error{"object version must not be negative"};
|
|
}
|
|
|
|
if (version == -1) {
|
|
object.set_version(0U);
|
|
} else {
|
|
object.set_version(static_cast<object_version_type>(version));
|
|
}
|
|
}
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::Info::optional_int64_timestamp, protozero::pbf_wire_type::varint):
|
|
object.set_timestamp(pbf_info.get_int64() * m_date_factor / 1000);
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::Info::optional_int64_changeset, protozero::pbf_wire_type::varint):
|
|
{
|
|
const auto changeset_id = pbf_info.get_int64();
|
|
if (changeset_id < -1 || changeset_id >= std::numeric_limits<changeset_id_type>::max()) {
|
|
throw osmium::pbf_error{"object changeset_id must be between 0 and 2^32-1"};
|
|
}
|
|
|
|
if (changeset_id == -1) {
|
|
object.set_changeset(0U);
|
|
} else {
|
|
object.set_changeset(static_cast<changeset_id_type>(changeset_id));
|
|
}
|
|
}
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::Info::optional_int32_uid, protozero::pbf_wire_type::varint):
|
|
object.set_uid_from_signed(pbf_info.get_int32());
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::Info::optional_uint32_user_sid, protozero::pbf_wire_type::varint):
|
|
user = m_stringtable.at(pbf_info.get_uint32());
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::Info::optional_bool_visible, protozero::pbf_wire_type::varint):
|
|
object.set_visible(pbf_info.get_bool());
|
|
break;
|
|
default:
|
|
pbf_info.skip();
|
|
}
|
|
}
|
|
|
|
return user;
|
|
}
|
|
|
|
void build_tag_list(osmium::builder::Builder& parent, varint_range& keys, varint_range& vals) {
|
|
if (keys.empty() || vals.empty()) {
|
|
return;
|
|
}
|
|
|
|
osmium::builder::TagListBuilder builder{parent};
|
|
do {
|
|
const auto& k = m_stringtable.at(keys.next_uint32());
|
|
const auto& v = m_stringtable.at(vals.next_uint32());
|
|
builder.add_tag(k.first, k.second, v.first, v.second);
|
|
} while (!keys.empty() && !vals.empty());
|
|
}
|
|
|
|
int32_t convert_pbf_lon(const int64_t c) const noexcept {
|
|
return static_cast<int32_t>((c * m_granularity + m_lon_offset) / resolution_convert);
|
|
}
|
|
|
|
int32_t convert_pbf_lat(const int64_t c) const noexcept {
|
|
return static_cast<int32_t>((c * m_granularity + m_lat_offset) / resolution_convert);
|
|
}
|
|
|
|
void decode_node(const data_view& data) {
|
|
osmium::builder::NodeBuilder builder{m_buffer};
|
|
osmium::Node& node = builder.object();
|
|
|
|
varint_range keys;
|
|
varint_range vals;
|
|
int64_t lon = std::numeric_limits<int64_t>::max();
|
|
int64_t lat = std::numeric_limits<int64_t>::max();
|
|
|
|
osm_string_len_type user{"", 0};
|
|
|
|
protozero::pbf_message<OSMFormat::Node> pbf_node{data};
|
|
while (pbf_node.next()) {
|
|
switch (pbf_node.tag_and_type()) {
|
|
case protozero::tag_and_type(OSMFormat::Node::required_sint64_id, protozero::pbf_wire_type::varint):
|
|
node.set_id(pbf_node.get_sint64());
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::Node::packed_uint32_keys, protozero::pbf_wire_type::length_delimited):
|
|
keys = varint_range{pbf_node.get_view()};
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::Node::packed_uint32_vals, protozero::pbf_wire_type::length_delimited):
|
|
vals = varint_range{pbf_node.get_view()};
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::Node::optional_Info_info, protozero::pbf_wire_type::length_delimited):
|
|
if (m_read_metadata == osmium::io::read_meta::yes) {
|
|
user = decode_info(pbf_node.get_view(), builder.object());
|
|
} else {
|
|
pbf_node.skip();
|
|
}
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::Node::required_sint64_lat, protozero::pbf_wire_type::varint):
|
|
lat = pbf_node.get_sint64();
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::Node::required_sint64_lon, protozero::pbf_wire_type::varint):
|
|
lon = pbf_node.get_sint64();
|
|
break;
|
|
default:
|
|
pbf_node.skip();
|
|
}
|
|
}
|
|
|
|
if (node.visible()) {
|
|
if (lon == std::numeric_limits<int64_t>::max() ||
|
|
lat == std::numeric_limits<int64_t>::max()) {
|
|
throw osmium::pbf_error{"illegal coordinate format"};
|
|
}
|
|
node.set_location(osmium::Location{
|
|
convert_pbf_lon(lon),
|
|
convert_pbf_lat(lat)
|
|
});
|
|
}
|
|
|
|
builder.set_user(user.first, user.second);
|
|
|
|
build_tag_list(builder, keys, vals);
|
|
}
|
|
|
|
void decode_way(const data_view& data) {
|
|
osmium::builder::WayBuilder builder{m_buffer};
|
|
|
|
varint_range keys;
|
|
varint_range vals;
|
|
varint_range refs;
|
|
varint_range lats;
|
|
varint_range lons;
|
|
|
|
osm_string_len_type user{"", 0};
|
|
|
|
protozero::pbf_message<OSMFormat::Way> pbf_way{data};
|
|
while (pbf_way.next()) {
|
|
switch (pbf_way.tag_and_type()) {
|
|
case protozero::tag_and_type(OSMFormat::Way::required_int64_id, protozero::pbf_wire_type::varint):
|
|
builder.object().set_id(pbf_way.get_int64());
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::Way::packed_uint32_keys, protozero::pbf_wire_type::length_delimited):
|
|
keys = varint_range{pbf_way.get_view()};
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::Way::packed_uint32_vals, protozero::pbf_wire_type::length_delimited):
|
|
vals = varint_range{pbf_way.get_view()};
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::Way::optional_Info_info, protozero::pbf_wire_type::length_delimited):
|
|
if (m_read_metadata == osmium::io::read_meta::yes) {
|
|
user = decode_info(pbf_way.get_view(), builder.object());
|
|
} else {
|
|
pbf_way.skip();
|
|
}
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::Way::packed_sint64_refs, protozero::pbf_wire_type::length_delimited):
|
|
refs = varint_range{pbf_way.get_view()};
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::Way::packed_sint64_lat, protozero::pbf_wire_type::length_delimited):
|
|
lats = varint_range{pbf_way.get_view()};
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::Way::packed_sint64_lon, protozero::pbf_wire_type::length_delimited):
|
|
lons = varint_range{pbf_way.get_view()};
|
|
break;
|
|
default:
|
|
pbf_way.skip();
|
|
}
|
|
}
|
|
|
|
builder.set_user(user.first, user.second);
|
|
|
|
if (!refs.empty()) {
|
|
osmium::builder::WayNodeListBuilder wnl_builder{builder};
|
|
osmium::DeltaDecode<int64_t> ref;
|
|
if (lats.empty()) {
|
|
while (!refs.empty()) {
|
|
wnl_builder.add_node_ref(ref.update(refs.next_sint64()));
|
|
}
|
|
} else {
|
|
osmium::DeltaDecode<int64_t> lon;
|
|
osmium::DeltaDecode<int64_t> lat;
|
|
while (!refs.empty() && !lons.empty() && !lats.empty()) {
|
|
wnl_builder.add_node_ref(
|
|
ref.update(refs.next_sint64()),
|
|
osmium::Location{convert_pbf_lon(lon.update(lons.next_sint64())),
|
|
convert_pbf_lat(lat.update(lats.next_sint64()))}
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
build_tag_list(builder, keys, vals);
|
|
}
|
|
|
|
void decode_relation(const data_view& data) {
|
|
osmium::builder::RelationBuilder builder{m_buffer};
|
|
|
|
varint_range keys;
|
|
varint_range vals;
|
|
varint_range roles;
|
|
varint_range refs;
|
|
varint_range types;
|
|
|
|
osm_string_len_type user{"", 0};
|
|
|
|
protozero::pbf_message<OSMFormat::Relation> pbf_relation{data};
|
|
while (pbf_relation.next()) {
|
|
switch (pbf_relation.tag_and_type()) {
|
|
case protozero::tag_and_type(OSMFormat::Relation::required_int64_id, protozero::pbf_wire_type::varint):
|
|
builder.object().set_id(pbf_relation.get_int64());
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::Relation::packed_uint32_keys, protozero::pbf_wire_type::length_delimited):
|
|
keys = varint_range{pbf_relation.get_view()};
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::Relation::packed_uint32_vals, protozero::pbf_wire_type::length_delimited):
|
|
vals = varint_range{pbf_relation.get_view()};
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::Relation::optional_Info_info, protozero::pbf_wire_type::length_delimited):
|
|
if (m_read_metadata == osmium::io::read_meta::yes) {
|
|
user = decode_info(pbf_relation.get_view(), builder.object());
|
|
} else {
|
|
pbf_relation.skip();
|
|
}
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::Relation::packed_int32_roles_sid, protozero::pbf_wire_type::length_delimited):
|
|
roles = varint_range{pbf_relation.get_view()};
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::Relation::packed_sint64_memids, protozero::pbf_wire_type::length_delimited):
|
|
refs = varint_range{pbf_relation.get_view()};
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::Relation::packed_MemberType_types, protozero::pbf_wire_type::length_delimited):
|
|
types = varint_range{pbf_relation.get_view()};
|
|
break;
|
|
default:
|
|
pbf_relation.skip();
|
|
}
|
|
}
|
|
|
|
builder.set_user(user.first, user.second);
|
|
|
|
if (!refs.empty()) {
|
|
osmium::builder::RelationMemberListBuilder rml_builder{builder};
|
|
osmium::DeltaDecode<int64_t> ref;
|
|
while (!roles.empty() && !refs.empty() && !types.empty()) {
|
|
const auto& r = m_stringtable.at(roles.next_int32());
|
|
const int type = types.next_int32();
|
|
if (type < 0 || type > 2) {
|
|
throw osmium::pbf_error{"unknown relation member type"};
|
|
}
|
|
rml_builder.add_member(
|
|
static_cast<osmium::item_type>(type + 1),
|
|
ref.update(refs.next_sint64()),
|
|
r.first,
|
|
r.second
|
|
);
|
|
}
|
|
}
|
|
|
|
build_tag_list(builder, keys, vals);
|
|
}
|
|
|
|
void build_tag_list_from_dense_nodes(osmium::builder::NodeBuilder& builder, varint_range& tags) {
|
|
osmium::builder::TagListBuilder tl_builder{builder};
|
|
while (!tags.empty()) {
|
|
const auto idx = tags.next_int32();
|
|
if (idx == 0) {
|
|
return;
|
|
}
|
|
const auto& k = m_stringtable.at(idx);
|
|
if (tags.empty()) {
|
|
throw osmium::pbf_error{"PBF format error"}; // this is against the spec, keys/vals must come in pairs
|
|
}
|
|
const auto& v = m_stringtable.at(tags.next_int32());
|
|
tl_builder.add_tag(k.first, k.second, v.first, v.second);
|
|
}
|
|
}
|
|
|
|
void decode_dense_nodes_without_metadata(const data_view& data) {
|
|
varint_range ids;
|
|
varint_range lats;
|
|
varint_range lons;
|
|
varint_range tags;
|
|
|
|
protozero::pbf_message<OSMFormat::DenseNodes> pbf_dense_nodes{data};
|
|
while (pbf_dense_nodes.next()) {
|
|
switch (pbf_dense_nodes.tag_and_type()) {
|
|
case protozero::tag_and_type(OSMFormat::DenseNodes::packed_sint64_id, protozero::pbf_wire_type::length_delimited):
|
|
ids = varint_range{pbf_dense_nodes.get_view()};
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::DenseNodes::packed_sint64_lat, protozero::pbf_wire_type::length_delimited):
|
|
lats = varint_range{pbf_dense_nodes.get_view()};
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::DenseNodes::packed_sint64_lon, protozero::pbf_wire_type::length_delimited):
|
|
lons = varint_range{pbf_dense_nodes.get_view()};
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::DenseNodes::packed_int32_keys_vals, protozero::pbf_wire_type::length_delimited):
|
|
tags = varint_range{pbf_dense_nodes.get_view()};
|
|
break;
|
|
default:
|
|
pbf_dense_nodes.skip();
|
|
}
|
|
}
|
|
|
|
osmium::DeltaDecode<int64_t> dense_id;
|
|
osmium::DeltaDecode<int64_t> dense_latitude;
|
|
osmium::DeltaDecode<int64_t> dense_longitude;
|
|
|
|
while (!ids.empty()) {
|
|
if (lons.empty() ||
|
|
lats.empty()) {
|
|
// this is against the spec, must have same number of elements
|
|
throw osmium::pbf_error{"PBF format error"};
|
|
}
|
|
|
|
{
|
|
osmium::builder::NodeBuilder builder{m_buffer};
|
|
osmium::Node& node = builder.object();
|
|
|
|
node.set_id(dense_id.update(ids.next_sint64()));
|
|
|
|
const auto lon = dense_longitude.update(lons.next_sint64());
|
|
const auto lat = dense_latitude.update(lats.next_sint64());
|
|
builder.object().set_location(osmium::Location{
|
|
convert_pbf_lon(lon),
|
|
convert_pbf_lat(lat)
|
|
});
|
|
|
|
if (!tags.empty()) {
|
|
build_tag_list_from_dense_nodes(builder, tags);
|
|
}
|
|
}
|
|
m_buffer.commit();
|
|
}
|
|
|
|
}
|
|
|
|
void decode_dense_nodes(const data_view& data) {
|
|
bool has_info = false;
|
|
|
|
varint_range ids;
|
|
varint_range lats;
|
|
varint_range lons;
|
|
varint_range tags;
|
|
varint_range versions;
|
|
varint_range timestamps;
|
|
varint_range changesets;
|
|
varint_range uids;
|
|
varint_range user_sids;
|
|
varint_range visibles;
|
|
|
|
protozero::pbf_message<OSMFormat::DenseNodes> pbf_dense_nodes{data};
|
|
while (pbf_dense_nodes.next()) {
|
|
switch (pbf_dense_nodes.tag_and_type()) {
|
|
case protozero::tag_and_type(OSMFormat::DenseNodes::packed_sint64_id, protozero::pbf_wire_type::length_delimited):
|
|
ids = varint_range{pbf_dense_nodes.get_view()};
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::DenseNodes::optional_DenseInfo_denseinfo, protozero::pbf_wire_type::length_delimited):
|
|
{
|
|
has_info = true;
|
|
protozero::pbf_message<OSMFormat::DenseInfo> pbf_dense_info{pbf_dense_nodes.get_message()};
|
|
while (pbf_dense_info.next()) {
|
|
switch (pbf_dense_info.tag_and_type()) {
|
|
case protozero::tag_and_type(OSMFormat::DenseInfo::packed_int32_version, protozero::pbf_wire_type::length_delimited):
|
|
versions = varint_range{pbf_dense_info.get_view()};
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::DenseInfo::packed_sint64_timestamp, protozero::pbf_wire_type::length_delimited):
|
|
timestamps = varint_range{pbf_dense_info.get_view()};
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::DenseInfo::packed_sint64_changeset, protozero::pbf_wire_type::length_delimited):
|
|
changesets = varint_range{pbf_dense_info.get_view()};
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::DenseInfo::packed_sint32_uid, protozero::pbf_wire_type::length_delimited):
|
|
uids = varint_range{pbf_dense_info.get_view()};
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::DenseInfo::packed_sint32_user_sid, protozero::pbf_wire_type::length_delimited):
|
|
user_sids = varint_range{pbf_dense_info.get_view()};
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::DenseInfo::packed_bool_visible, protozero::pbf_wire_type::length_delimited):
|
|
visibles = varint_range{pbf_dense_info.get_view()};
|
|
break;
|
|
default:
|
|
pbf_dense_info.skip();
|
|
}
|
|
}
|
|
}
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::DenseNodes::packed_sint64_lat, protozero::pbf_wire_type::length_delimited):
|
|
lats = varint_range{pbf_dense_nodes.get_view()};
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::DenseNodes::packed_sint64_lon, protozero::pbf_wire_type::length_delimited):
|
|
lons = varint_range{pbf_dense_nodes.get_view()};
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::DenseNodes::packed_int32_keys_vals, protozero::pbf_wire_type::length_delimited):
|
|
tags = varint_range{pbf_dense_nodes.get_view()};
|
|
break;
|
|
default:
|
|
pbf_dense_nodes.skip();
|
|
}
|
|
}
|
|
|
|
osmium::DeltaDecode<int64_t> dense_id;
|
|
osmium::DeltaDecode<int64_t> dense_latitude;
|
|
osmium::DeltaDecode<int64_t> dense_longitude;
|
|
osmium::DeltaDecode<int64_t> dense_uid;
|
|
osmium::DeltaDecode<int64_t> dense_user_sid;
|
|
osmium::DeltaDecode<int64_t> dense_changeset;
|
|
osmium::DeltaDecode<int64_t> dense_timestamp;
|
|
|
|
while (!ids.empty()) {
|
|
if (lons.empty() ||
|
|
lats.empty()) {
|
|
// this is against the spec, must have same number of elements
|
|
throw osmium::pbf_error{"PBF format error"};
|
|
}
|
|
|
|
{
|
|
bool visible = true;
|
|
|
|
osmium::builder::NodeBuilder builder{m_buffer};
|
|
osmium::Node& node = builder.object();
|
|
|
|
node.set_id(dense_id.update(ids.next_sint64()));
|
|
|
|
if (has_info) {
|
|
if (!versions.empty()) {
|
|
const auto version = versions.next_int32();
|
|
if (version < -1) {
|
|
throw osmium::pbf_error{"object version must not be negative"};
|
|
}
|
|
|
|
if (version == -1) {
|
|
node.set_version(0U);
|
|
} else {
|
|
node.set_version(static_cast<osmium::object_version_type>(version));
|
|
}
|
|
}
|
|
|
|
if (!changesets.empty()) {
|
|
const auto changeset_id = dense_changeset.update(changesets.next_sint64());
|
|
if (changeset_id < -1 || changeset_id >= std::numeric_limits<changeset_id_type>::max()) {
|
|
throw osmium::pbf_error{"object changeset_id must be between 0 and 2^32-1"};
|
|
}
|
|
|
|
if (changeset_id == -1) {
|
|
node.set_changeset(0U);
|
|
} else {
|
|
node.set_changeset(static_cast<osmium::changeset_id_type>(changeset_id));
|
|
}
|
|
}
|
|
|
|
if (!timestamps.empty()) {
|
|
node.set_timestamp(dense_timestamp.update(timestamps.next_sint64()) * m_date_factor / 1000);
|
|
}
|
|
|
|
if (!uids.empty()) {
|
|
node.set_uid_from_signed(static_cast<osmium::signed_user_id_type>(dense_uid.update(uids.next_sint32())));
|
|
}
|
|
|
|
if (!visibles.empty()) {
|
|
visible = (visibles.next_int32() != 0);
|
|
}
|
|
node.set_visible(visible);
|
|
|
|
if (!user_sids.empty()) {
|
|
const auto& u = m_stringtable.at(dense_user_sid.update(user_sids.next_sint32()));
|
|
builder.set_user(u.first, u.second);
|
|
}
|
|
}
|
|
|
|
// even if the node isn't visible, there's still a record
|
|
// of its lat/lon in the dense arrays.
|
|
const auto lon = dense_longitude.update(lons.next_sint64());
|
|
const auto lat = dense_latitude.update(lats.next_sint64());
|
|
if (visible) {
|
|
builder.object().set_location(osmium::Location{
|
|
convert_pbf_lon(lon),
|
|
convert_pbf_lat(lat)
|
|
});
|
|
}
|
|
|
|
if (!tags.empty()) {
|
|
build_tag_list_from_dense_nodes(builder, tags);
|
|
}
|
|
}
|
|
m_buffer.commit();
|
|
}
|
|
}
|
|
|
|
public:
|
|
|
|
PBFPrimitiveBlockDecoder(const data_view& data, const osmium::osm_entity_bits::type read_types, const osmium::io::read_meta read_metadata) :
|
|
m_data(data),
|
|
m_read_types(read_types),
|
|
m_read_metadata(read_metadata) {
|
|
}
|
|
|
|
PBFPrimitiveBlockDecoder(const PBFPrimitiveBlockDecoder&) = delete;
|
|
PBFPrimitiveBlockDecoder& operator=(const PBFPrimitiveBlockDecoder&) = delete;
|
|
|
|
PBFPrimitiveBlockDecoder(PBFPrimitiveBlockDecoder&&) = delete;
|
|
PBFPrimitiveBlockDecoder& operator=(PBFPrimitiveBlockDecoder&&) = delete;
|
|
|
|
~PBFPrimitiveBlockDecoder() noexcept = default;
|
|
|
|
osmium::memory::Buffer operator()() {
|
|
try {
|
|
decode_primitive_block_metadata();
|
|
decode_primitive_block_data();
|
|
} catch (const std::out_of_range&) {
|
|
throw osmium::pbf_error{"string id out of range"};
|
|
}
|
|
|
|
return std::move(m_buffer);
|
|
}
|
|
|
|
}; // class PBFPrimitiveBlockDecoder
|
|
|
|
inline data_view decode_blob(const std::string& blob_data, std::string& output) {
|
|
int32_t raw_size = 0;
|
|
protozero::data_view compressed_data;
|
|
pbf_compression use_compression = pbf_compression::none;
|
|
|
|
protozero::pbf_message<FileFormat::Blob> pbf_blob{blob_data};
|
|
while (pbf_blob.next()) {
|
|
switch (pbf_blob.tag_and_type()) {
|
|
case protozero::tag_and_type(FileFormat::Blob::optional_bytes_raw, protozero::pbf_wire_type::length_delimited):
|
|
{
|
|
const auto data_len = pbf_blob.get_view();
|
|
if (data_len.size() > max_uncompressed_blob_size) {
|
|
throw osmium::pbf_error{"illegal blob size"};
|
|
}
|
|
return data_len;
|
|
}
|
|
case protozero::tag_and_type(FileFormat::Blob::optional_int32_raw_size, protozero::pbf_wire_type::varint):
|
|
raw_size = pbf_blob.get_int32();
|
|
if (raw_size <= 0 || static_cast<uint32_t>(raw_size) > max_uncompressed_blob_size) {
|
|
throw osmium::pbf_error{"illegal blob size"};
|
|
}
|
|
break;
|
|
case protozero::tag_and_type(FileFormat::Blob::optional_bytes_zlib_data, protozero::pbf_wire_type::length_delimited):
|
|
use_compression = pbf_compression::zlib;
|
|
compressed_data = pbf_blob.get_view();
|
|
break;
|
|
case protozero::tag_and_type(FileFormat::Blob::optional_bytes_lzma_data, protozero::pbf_wire_type::length_delimited):
|
|
throw osmium::pbf_error{"lzma blobs not supported"};
|
|
case protozero::tag_and_type(FileFormat::Blob::optional_bytes_lz4_data, protozero::pbf_wire_type::length_delimited):
|
|
#ifdef OSMIUM_WITH_LZ4
|
|
use_compression = pbf_compression::lz4;
|
|
compressed_data = pbf_blob.get_view();
|
|
break;
|
|
#else
|
|
throw osmium::pbf_error{"lz4 blobs not supported"};
|
|
#endif
|
|
case protozero::tag_and_type(FileFormat::Blob::optional_bytes_zstd_data, protozero::pbf_wire_type::length_delimited):
|
|
throw osmium::pbf_error{"zstd blobs not supported"};
|
|
default:
|
|
pbf_blob.skip();
|
|
}
|
|
}
|
|
|
|
if (compressed_data.empty()) {
|
|
throw osmium::pbf_error{"blob contains no data or unknown compression method"};
|
|
}
|
|
|
|
if (raw_size == 0) {
|
|
throw osmium::pbf_error{"missing raw_size in compressed blob"};
|
|
}
|
|
|
|
switch (use_compression) {
|
|
case pbf_compression::none:
|
|
break;
|
|
case pbf_compression::zlib:
|
|
return osmium::io::detail::zlib_uncompress_string(
|
|
compressed_data.data(),
|
|
static_cast<unsigned long>(compressed_data.size()), // NOLINT(google-runtime-int)
|
|
static_cast<unsigned long>(raw_size), // NOLINT(google-runtime-int)
|
|
output
|
|
);
|
|
case pbf_compression::lz4:
|
|
#ifdef OSMIUM_WITH_LZ4
|
|
return osmium::io::detail::lz4_uncompress_string(
|
|
compressed_data.data(),
|
|
static_cast<unsigned long>(compressed_data.size()), // NOLINT(google-runtime-int)
|
|
static_cast<unsigned long>(raw_size), // NOLINT(google-runtime-int)
|
|
output
|
|
);
|
|
#else
|
|
break;
|
|
#endif
|
|
}
|
|
std::abort(); // should never be here
|
|
}
|
|
|
|
inline osmium::Box decode_header_bbox(const data_view& data) {
|
|
int64_t left = std::numeric_limits<int64_t>::max();
|
|
int64_t right = std::numeric_limits<int64_t>::max();
|
|
int64_t top = std::numeric_limits<int64_t>::max();
|
|
int64_t bottom = std::numeric_limits<int64_t>::max();
|
|
|
|
protozero::pbf_message<OSMFormat::HeaderBBox> pbf_header_bbox{data};
|
|
while (pbf_header_bbox.next()) {
|
|
switch (pbf_header_bbox.tag_and_type()) {
|
|
case protozero::tag_and_type(OSMFormat::HeaderBBox::required_sint64_left, protozero::pbf_wire_type::varint):
|
|
left = pbf_header_bbox.get_sint64();
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::HeaderBBox::required_sint64_right, protozero::pbf_wire_type::varint):
|
|
right = pbf_header_bbox.get_sint64();
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::HeaderBBox::required_sint64_top, protozero::pbf_wire_type::varint):
|
|
top = pbf_header_bbox.get_sint64();
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::HeaderBBox::required_sint64_bottom, protozero::pbf_wire_type::varint):
|
|
bottom = pbf_header_bbox.get_sint64();
|
|
break;
|
|
default:
|
|
pbf_header_bbox.skip();
|
|
}
|
|
}
|
|
|
|
if (left == std::numeric_limits<int64_t>::max() ||
|
|
right == std::numeric_limits<int64_t>::max() ||
|
|
top == std::numeric_limits<int64_t>::max() ||
|
|
bottom == std::numeric_limits<int64_t>::max()) {
|
|
throw osmium::pbf_error{"invalid bbox"};
|
|
}
|
|
|
|
osmium::Box box;
|
|
box.extend(osmium::Location{left / resolution_convert, bottom / resolution_convert});
|
|
box.extend(osmium::Location{right / resolution_convert, top / resolution_convert});
|
|
|
|
return box;
|
|
}
|
|
|
|
inline osmium::io::Header decode_header_block(const data_view& data) {
|
|
osmium::io::Header header;
|
|
int i = 0;
|
|
|
|
protozero::pbf_message<OSMFormat::HeaderBlock> pbf_header_block{data};
|
|
while (pbf_header_block.next()) {
|
|
switch (pbf_header_block.tag_and_type()) {
|
|
case protozero::tag_and_type(OSMFormat::HeaderBlock::optional_HeaderBBox_bbox, protozero::pbf_wire_type::length_delimited):
|
|
header.add_box(decode_header_bbox(pbf_header_block.get_view()));
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::HeaderBlock::repeated_string_required_features, protozero::pbf_wire_type::length_delimited):
|
|
{
|
|
auto feature = pbf_header_block.get_view();
|
|
if (!std::strncmp("OsmSchema-V0.6", feature.data(), feature.size())) {
|
|
// intentionally left blank
|
|
} else if (!std::strncmp("DenseNodes", feature.data(), feature.size())) {
|
|
header.set("pbf_dense_nodes", true);
|
|
} else if (!std::strncmp("HistoricalInformation", feature.data(), feature.size())) {
|
|
header.set_has_multiple_object_versions(true);
|
|
} else {
|
|
std::string msg{"required feature not supported: "};
|
|
msg.append(feature.data(), feature.size());
|
|
throw osmium::pbf_error{msg};
|
|
}
|
|
}
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::HeaderBlock::repeated_string_optional_features, protozero::pbf_wire_type::length_delimited): {
|
|
const auto opt = pbf_header_block.get_string();
|
|
header.set("pbf_optional_feature_" + std::to_string(i++), opt);
|
|
if (opt == "Sort.Type_then_ID") {
|
|
header.set("sorting", "Type_then_ID");
|
|
}
|
|
}
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::HeaderBlock::optional_string_writingprogram, protozero::pbf_wire_type::length_delimited):
|
|
header.set("generator", pbf_header_block.get_string());
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::HeaderBlock::optional_int64_osmosis_replication_timestamp, protozero::pbf_wire_type::varint):
|
|
{
|
|
const auto timestamp = osmium::Timestamp{pbf_header_block.get_int64()}.to_iso();
|
|
header.set("osmosis_replication_timestamp", timestamp);
|
|
header.set("timestamp", timestamp);
|
|
}
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::HeaderBlock::optional_int64_osmosis_replication_sequence_number, protozero::pbf_wire_type::varint):
|
|
header.set("osmosis_replication_sequence_number", std::to_string(pbf_header_block.get_int64()));
|
|
break;
|
|
case protozero::tag_and_type(OSMFormat::HeaderBlock::optional_string_osmosis_replication_base_url, protozero::pbf_wire_type::length_delimited):
|
|
header.set("osmosis_replication_base_url", pbf_header_block.get_string());
|
|
break;
|
|
default:
|
|
pbf_header_block.skip();
|
|
}
|
|
}
|
|
|
|
return header;
|
|
}
|
|
|
|
/**
|
|
* Decode HeaderBlock.
|
|
*
|
|
* @param header_block_data Input data
|
|
* @returns Header object
|
|
* @throws osmium::pbf_error If there was a parsing error
|
|
*/
|
|
inline osmium::io::Header decode_header(const std::string& header_block_data) {
|
|
std::string output;
|
|
|
|
return decode_header_block(decode_blob(header_block_data, output));
|
|
}
|
|
|
|
class PBFDataBlobDecoder {
|
|
|
|
std::shared_ptr<std::string> m_input_buffer;
|
|
osmium::osm_entity_bits::type m_read_types;
|
|
osmium::io::read_meta m_read_metadata;
|
|
|
|
public:
|
|
|
|
PBFDataBlobDecoder(std::string&& input_buffer, const osmium::osm_entity_bits::type read_types, const osmium::io::read_meta read_metadata) :
|
|
m_input_buffer(std::make_shared<std::string>(std::move(input_buffer))),
|
|
m_read_types(read_types),
|
|
m_read_metadata(read_metadata) {
|
|
}
|
|
|
|
osmium::memory::Buffer operator()() {
|
|
std::string output;
|
|
PBFPrimitiveBlockDecoder decoder{decode_blob(*m_input_buffer, output), m_read_types, m_read_metadata};
|
|
return decoder();
|
|
}
|
|
|
|
}; // class PBFDataBlobDecoder
|
|
|
|
} // namespace detail
|
|
|
|
} // namespace io
|
|
|
|
} // namespace osmium
|
|
|
|
#endif // OSMIUM_IO_DETAIL_PBF_DECODER_HPP
|