Skip to content

Serialize non-native BEVE map/pair keys via a generic-array fallback #2669

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
stephenberry wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Serialize non-native BEVE map/pair keys via a generic-array fallback #2669

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions docs/binary.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -438,6 +438,23 @@ struct glz::meta<ModuleID> {
};
```

### Keys that are not numeric or string

A BEVE object (map) header encodes a *single shared key type*, so the compact map encoding requires keys that reduce to a numeric or string type. When a key does not, such as a multi-field struct, the map/pair serializers automatically fall back to a **generic array of `[key, value]` entries**, where each key is written as a full BEVE value. The container still round-trips with no extra code:

```c++
struct CompositeKey {
int32_t id{};
std::string name{};
auto operator<=>(const CompositeKey&) const = default;
};

std::unordered_map<CompositeKey, int> m{{{1, "a"}, 10}, {{2, "b"}, 20}};
glz::write_beve(m, beve); // a generic array of [key, value] entries
```

This applies to `std::map`, `std::unordered_map`, `std::vector<std::pair<Key, Value>>`, and a standalone `std::pair<Key, Value>`. The fallback is portable and self-describing, and each entry is encoded identically to `std::tuple<Key, Value>` (so a non-native `std::pair` and the corresponding `std::tuple` produce the same bytes). Native (numeric/string) keys are unaffected and keep the compact shared-header encoding.

## Partial Objects

It is sometimes desirable to write out only a portion of an object. This is permitted via an array of JSON pointers, which indicate which parts of the object should be written out.
Expand Down
4 changes: 4 additions & 0 deletions include/glaze/beve/key_traits.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,10 @@ namespace glz
using numeric_type = typename beve_numeric_type<underlying>::type;

static constexpr bool numeric = std::is_arithmetic_v<numeric_type>;
// A BEVE object header encodes a single shared key type, which must be numeric or string-like.
// Native keys use the compact object/map encoding; keys that reduce to neither (e.g. multi-field
// structs) fall back to a generic array of [key, value] entries in the map/pair serializers.
static constexpr bool native = numeric || str_t<underlying>;
static constexpr bool as_string = str_t<underlying> || !numeric;
static constexpr bool as_number = !as_string;

Expand Down
248 changes: 197 additions & 51 deletions include/glaze/beve/read.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1470,47 +1470,86 @@ namespace glz
using Element = typename T::value_type;
using Key = typename Element::first_type;

constexpr uint8_t header = beve_key_traits<Key>::header;
if constexpr (beve_key_traits<Key>::native) {
constexpr uint8_t header = beve_key_traits<Key>::header;

if (invalid_end(ctx, it, end)) {
return;
}
const auto tag = uint8_t(*it);
if (tag != header) [[unlikely]] {
if constexpr (check_allow_conversions(Opts)) {
const auto key_type = tag & 0b000'11'000;
if constexpr (beve_key_traits<Key>::as_string) {
if (key_type != 0) {
ctx.error = error_code::syntax_error;
return;
if (invalid_end(ctx, it, end)) {
return;
}
const auto tag = uint8_t(*it);
if (tag != header) [[unlikely]] {
if constexpr (check_allow_conversions(Opts)) {
const auto key_type = tag & 0b000'11'000;
if constexpr (beve_key_traits<Key>::as_string) {
if (key_type != 0) {
ctx.error = error_code::syntax_error;
return;
}
}
else {
if (key_type == 0) {
ctx.error = error_code::syntax_error;
return;
}
}
}
else {
if (key_type == 0) {
ctx.error = error_code::syntax_error;
return;
}
ctx.error = error_code::syntax_error;
return;
}
}
else {
ctx.error = error_code::syntax_error;

++it;
const size_t n = int_from_compressed(ctx, it, end);
if (bool(ctx.error)) [[unlikely]] {
return;
}
}

++it;
const size_t n = int_from_compressed(ctx, it, end);
if (bool(ctx.error)) [[unlikely]] {
return;
// Validate count against remaining buffer size (minimum 1 byte per key-value pair)
if (n > size_t(end - it)) [[unlikely]] {
ctx.error = error_code::unexpected_end;
return;
}

value.clear();

constexpr uint8_t key_tag = beve_key_traits<Key>::key_tag;
for (size_t i = 0; i < n; ++i) {
auto& item = value.emplace_back();
parse<BEVE>::op<no_header_on<Opts>()>(item.first, key_tag, ctx, it, end);
parse<BEVE>::op<Opts>(item.second, ctx, it, end);
}
}
else {
// Non-native key: read the generic-array fallback (an array of [key, value] entries; see
// the writer). Each entry is itself a generic array, identical to std::tuple<Key, Value>.
if (invalid_end(ctx, it, end)) {
return;
}
if (uint8_t(*it) != tag::generic_array) [[unlikely]] {
ctx.error = error_code::syntax_error;
return;
}
++it;

value.clear();
const size_t n = int_from_compressed(ctx, it, end);
if (bool(ctx.error)) [[unlikely]] {
return;
}
// Validate count against remaining buffer size (minimum 1 byte per entry)
if (n > size_t(end - it)) [[unlikely]] {
ctx.error = error_code::unexpected_end;
return;
}

constexpr uint8_t key_tag = beve_key_traits<Key>::key_tag;
for (size_t i = 0; i < n; ++i) {
auto& item = value.emplace_back();
parse<BEVE>::op<no_header_on<Opts>()>(item.first, key_tag, ctx, it, end);
parse<BEVE>::op<Opts>(item.second, ctx, it, end);
value.clear();
for (size_t i = 0; i < n; ++i) {
auto& item = value.emplace_back();
parse<BEVE>::op<Opts>(item, ctx, it, end);
if (bool(ctx.error)) [[unlikely]] {
return;
}
}
}
}
};
Expand All @@ -1523,31 +1562,60 @@ namespace glz
{
using Key = typename T::first_type;

constexpr uint8_t header = beve_key_traits<Key>::header;
if constexpr (beve_key_traits<Key>::native) {
constexpr uint8_t header = beve_key_traits<Key>::header;

if (invalid_end(ctx, it, end)) {
return;
}
const auto tag = uint8_t(*it);
if (tag != header) [[unlikely]] {
ctx.error = error_code::syntax_error;
return;
}
if (invalid_end(ctx, it, end)) {
return;
}
const auto tag = uint8_t(*it);
if (tag != header) [[unlikely]] {
ctx.error = error_code::syntax_error;
return;
}

++it;
const auto n = int_from_compressed(ctx, it, end);
if (bool(ctx.error)) [[unlikely]] {
return;
}
++it;
const auto n = int_from_compressed(ctx, it, end);
if (bool(ctx.error)) [[unlikely]] {
return;
}

if (n != 1) [[unlikely]] {
ctx.error = error_code::syntax_error;
return;
if (n != 1) [[unlikely]] {
ctx.error = error_code::syntax_error;
return;
}

constexpr uint8_t key_tag = beve_key_traits<Key>::key_tag;
parse<BEVE>::op<no_header_on<Opts>()>(value.first, key_tag, ctx, it, end);
parse<BEVE>::op<Opts>(value.second, ctx, it, end);
}
else {
// Non-native key: read the generic-array fallback [key, value], identical to
// std::tuple<Key, Value>. Each element carries its own header.
if (invalid_end(ctx, it, end)) {
return;
}
if (uint8_t(*it) != tag::generic_array) [[unlikely]] {
ctx.error = error_code::syntax_error;
return;
}
++it;

const auto n = int_from_compressed(ctx, it, end);
if (bool(ctx.error)) [[unlikely]] {
return;
}
if (n != 2) [[unlikely]] {
ctx.error = error_code::syntax_error;
return;
}

constexpr uint8_t key_tag = beve_key_traits<Key>::key_tag;
parse<BEVE>::op<no_header_on<Opts>()>(value.first, key_tag, ctx, it, end);
parse<BEVE>::op<Opts>(value.second, ctx, it, end);
parse<BEVE>::op<Opts>(value.first, ctx, it, end);
if (bool(ctx.error)) [[unlikely]] {
return;
}
parse<BEVE>::op<Opts>(value.second, ctx, it, end);
}
}
};

Expand All @@ -1559,6 +1627,84 @@ namespace glz
{
using Key = typename T::key_type;

// Non-native key (e.g. a multi-field struct): a BEVE object header can't carry a struct key,
// so the writer emits a generic array of [key, value] entries. Read that fallback form here;
// native keys use the compact object/map encoding below.
if constexpr (not beve_key_traits<Key>::native) {
if (invalid_end(ctx, it, end)) {
return;
}
if (uint8_t(*it) != tag::generic_array) [[unlikely]] {
ctx.error = error_code::syntax_error;
return;
}
++it;

const size_t n = int_from_compressed(ctx, it, end);
if (bool(ctx.error)) [[unlikely]] {
return;
}
// Validate count against remaining buffer size (minimum 1 byte per entry)
if (n > size_t(end - it)) [[unlikely]] {
ctx.error = error_code::unexpected_end;
return;
}
if constexpr (check_max_map_size(Opts) > 0) {
if (n > check_max_map_size(Opts)) [[unlikely]] {
ctx.error = error_code::invalid_length;
return;
}
}
if constexpr (has_runtime_max_map_size<std::decay_t<decltype(ctx)>>) {
if (ctx.max_map_size > 0 && n > ctx.max_map_size) [[unlikely]] {
ctx.error = error_code::invalid_length;
return;
}
}

for (size_t i = 0; i < n; ++i) {
// each entry is a generic array [key, value]
if (invalid_end(ctx, it, end)) {
return;
}
if (uint8_t(*it) != tag::generic_array) [[unlikely]] {
ctx.error = error_code::syntax_error;
return;
}
++it;
const auto m = int_from_compressed(ctx, it, end);
if (bool(ctx.error)) [[unlikely]] {
return;
}
if (m != 2) [[unlikely]] {
ctx.error = error_code::syntax_error;
return;
}

Key key{};
parse<BEVE>::op<Opts>(key, ctx, it, end);
if (bool(ctx.error)) [[unlikely]] {
return;
}
if constexpr (Opts.partial_read) {
if (auto element = value.find(key); element != value.end()) {
parse<BEVE>::op<Opts>(element->second, ctx, it, end);
}
else {
typename T::mapped_type discard{};
parse<BEVE>::op<Opts>(discard, ctx, it, end);
}
}
else {
parse<BEVE>::op<Opts>(value[key], ctx, it, end);
}
if (bool(ctx.error)) [[unlikely]] {
return;
}
}
return;
}

constexpr uint8_t header = beve_key_traits<Key>::header;

if (invalid_end(ctx, it, end)) {
Expand Down Expand Up @@ -1618,7 +1764,7 @@ namespace glz
}
}

constexpr uint8_t key_tag = beve_key_traits<Key>::key_tag;
[[maybe_unused]] constexpr uint8_t key_tag = beve_key_traits<Key>::key_tag;

if constexpr (beve_key_traits<Key>::as_number) {
Key key{}; // Value-initialize to silence false positive -Wmaybe-uninitialized
Expand Down Expand Up @@ -1651,7 +1797,7 @@ namespace glz
}
}
}
else {
else if constexpr (beve_key_traits<Key>::native) {
Key key;
for (size_t i = 0; i < n; ++i) {
if constexpr (Opts.partial_read) {
Expand Down
Loading
Loading