From b48289bffa148a93988623036e0aee9a38a6ba14 Mon Sep 17 00:00:00 2001 From: Stephen Berry Date: Fri, 31 Jul 2026 12:11:03 -0500 Subject: [PATCH 1/2] Route streaming buffers through read_streaming in glz::read A streaming buffer satisfies `contiguous`, so it bound to the buffered `glz::read` and was parsed as a flat span: one window, with `null_terminated` on, over memory that carries no terminator and ends wherever the last fill stopped. That reads past the window. std::istringstream iss{json}; // larger than the window glz::basic_istream_buffer buf{iss}; std::vector v; glz::read_jsonc(v, buf); ERROR: AddressSanitizer: heap-buffer-overflow READ of size 1 ... 0 bytes after 512-byte region #0 glz::parse_int atoi.hpp:488 #7 glz::read_jsonc, glz::basic_istream_buffer<...,512>> `read_json` was given its own `is_input_streaming` overloads and was safe. Nothing else was: `read_jsonc`, `read_beve`, the other per-format helpers and direct `glz::read` calls all funnel into the same buffered overload, so each had to remember a rule none of them stated. Fixed where they meet rather than one helper at a time. The buffered overloads now exclude streaming buffers and a pair of dispatching overloads forwards them to `read_streaming`, which reads with `null_terminated` off and refills as it goes. Every helper that funnels through `read` is fixed by that, and a format added later inherits it instead of having to opt in. The context-taking overload has nowhere to hold the streaming state the parsers refill through, so it runs the read on a `streaming_context` and reports the outcome back through the caller's context. --- include/glaze/core/read.hpp | 43 +++++++++-- .../istream_buffer_test.cpp | 74 +++++++++++++++++++ 2 files changed, 111 insertions(+), 6 deletions(-) diff --git a/include/glaze/core/read.hpp b/include/glaze/core/read.hpp index bd71c74d5b..ab41dc992c 100644 --- a/include/glaze/core/read.hpp +++ b/include/glaze/core/read.hpp @@ -85,9 +85,9 @@ namespace glz finalize_read_context(ctx); } - template - requires read_supported - [[nodiscard]] error_ctx read(T& value, contiguous auto&& buffer, is_context auto&& ctx) + template + requires read_supported && (!is_input_streaming) + [[nodiscard]] error_ctx read(T& value, Buf&& buffer, is_context auto&& ctx) { static_assert(sizeof(decltype(*buffer.data())) == 1); using Buffer = std::remove_reference_t; @@ -159,9 +159,9 @@ namespace glz return {size_t(it - start), ctx.error, ctx.custom_error_message}; } - template - requires read_supported - [[nodiscard]] error_ctx read(T& value, contiguous auto&& buffer) + template + requires read_supported && (!is_input_streaming) + [[nodiscard]] error_ctx read(T& value, Buf&& buffer) { format_context_t ctx{}; return read(value, buffer, ctx); @@ -271,4 +271,35 @@ namespace glz streaming_context ctx{}; return read_streaming(value, std::forward(buffer), ctx); } + + // A streaming buffer satisfies `contiguous`, but the span it exposes is one window of a larger + // stream: it ends wherever the last fill stopped, it carries no terminator, and reading it as a + // flat buffer parses that window alone. Route it to read_streaming rather than let it bind to + // the overloads above, which would run with null_terminated on and read past the window's end. + // Doing this here rather than per format is what keeps read_jsonc, read_beve and every other + // helper that funnels through `read` from having to remember on its own. + template + requires read_supported && is_input_streaming> + [[nodiscard]] error_ctx read(T& value, Buffer&& buffer, is_context auto&& ctx) + { + if constexpr (has_streaming_state) { + return read_streaming(value, std::forward(buffer), ctx); + } + else { + // The caller's context has nowhere to hold the streaming state the parsers refill + // through, so the read runs on one that does and reports back through theirs. + streaming_context stream_ctx{}; + const error_ctx ec = read_streaming(value, std::forward(buffer), stream_ctx); + ctx.error = stream_ctx.error; + ctx.custom_error_message = stream_ctx.custom_error_message; + return ec; + } + } + + template + requires read_supported && is_input_streaming> + [[nodiscard]] error_ctx read(T& value, Buffer&& buffer) + { + return read_streaming(value, std::forward(buffer)); + } } diff --git a/tests/istream_buffer_test/istream_buffer_test.cpp b/tests/istream_buffer_test/istream_buffer_test.cpp index 3d4243a26f..e5e2675f54 100644 --- a/tests/istream_buffer_test/istream_buffer_test.cpp +++ b/tests/istream_buffer_test/istream_buffer_test.cpp @@ -4144,4 +4144,78 @@ suite additional_edge_cases = [] { }; }; +// A streaming buffer satisfies `contiguous`, so before these entry points routed to read_streaming +// they bound to the buffered read: one window, parsed with null_terminated on, read past its end. +// Only read_json carried its own streaming overload; everything funnelling through `read` did not. +// A document larger than the window is what makes the difference visible -- it both overruns the +// window and, once routed correctly, needs the refills to complete. +namespace +{ + std::string oversized_array() + { + std::string json = "["; + for (int i = 0; i < 400; ++i) { + if (i) json += ','; + json += std::to_string(i); + } + return json + "]"; + } + + constexpr size_t narrow_window = 512; // smaller than oversized_array() +} + +suite streaming_buffer_dispatch_tests = [] { + "read_jsonc streams instead of parsing one window"_test = [] { + std::istringstream iss{oversized_array()}; + glz::basic_istream_buffer buffer(iss); + + std::vector v{}; + expect(!glz::read_jsonc(v, buffer)); + expect(v.size() == 400u); + expect(v[399] == 399); + }; + + "generic read streams instead of parsing one window"_test = [] { + std::istringstream iss{oversized_array()}; + glz::basic_istream_buffer buffer(iss); + + std::vector v{}; + expect(!glz::read(v, buffer)); + expect(v.size() == 400u); + }; + + // The context-taking overload has nowhere to put the streaming state, so it runs on one that + // does. The caller's context must still come back carrying the outcome. + "generic read with a caller supplied context"_test = [] { + std::istringstream iss{oversized_array()}; + glz::basic_istream_buffer buffer(iss); + + std::vector v{}; + glz::context ctx{}; + expect(!glz::read(v, buffer, ctx)); + expect(v.size() == 400u); + expect(ctx.error == glz::error_code::none); + }; + + "caller supplied context reports the error"_test = [] { + std::istringstream iss{R"([1,2,)"}; + glz::basic_istream_buffer buffer(iss); + + std::vector v{}; + glz::context ctx{}; + expect(glz::read(v, buffer, ctx)) << "truncated input must fail"; + expect(ctx.error != glz::error_code::none) << "the caller's context must carry the error"; + }; + + // read_json already had streaming overloads; nothing here may disturb them. + "read_json is unchanged"_test = [] { + std::istringstream iss{oversized_array()}; + glz::basic_istream_buffer buffer(iss); + + std::vector v{}; + expect(!glz::read_json(v, buffer)); + expect(v.size() == 400u); + }; +}; + int main() { return 0; } From 35f0fa570382f2e8aad3ddc26c164510247c7106 Mon Sep 17 00:00:00 2001 From: Stephen Berry Date: Fri, 31 Jul 2026 13:42:41 -0500 Subject: [PATCH 2/2] Report a short read when a format's reader cannot refill Routing streaming buffers to read_streaming removed the overrun, but it did not give a reader refill points it never had. Only the JSON reader has them, so every other format parsed whatever the first window held and stopped at its edge -- and reported that in terms that blamed the document: - NDJSON returned success carrying only the elements that fit. A 400 element document read back 156, with error_code::none. Silent data loss. - BEVE returned unexpected_end, which reads as malformed input. Neither told the caller the window was the cause, and nothing in the API said which formats could stream at all. format_supports_streaming names that, next to the format identifiers so it is visible where formats are. It is a property of the reader rather than of the grammar: NDJSON is false because its line loop cannot refill between lines, even though the per-line values it delegates to the JSON reader can. read_streaming reports error_code::streaming_unsupported when a non-refilling reader was not shown the whole document. The check asks whether the *source* is exhausted, not whether the window is drained, which is why streaming_state gains source_at_eof: at_eof also demands a drained window, and would call a value that legitimately left trailing bytes a truncated read. Once the source is exhausted the window holds everything there will ever be, so whatever the reader concluded it concluded on the full input and stands -- genuinely malformed input keeps its own error. The new error code is appended to the enum so no existing value shifts; REPE puts these on the wire. Tests cover both short reads, the three legitimate cases they must not touch (a document that fits, a value with trailing bytes, genuinely malformed input), and that JSON still streams to completion. --- include/glaze/core/context.hpp | 4 +- include/glaze/core/error_category.hpp | 7 +- include/glaze/core/istream_buffer.hpp | 6 ++ include/glaze/core/read.hpp | 27 +++++ include/glaze/core/streaming_state.hpp | 17 ++++ include/glaze/forward.hpp | 14 +++ .../istream_buffer_test.cpp | 99 +++++++++++++++++++ 7 files changed, 171 insertions(+), 3 deletions(-) diff --git a/include/glaze/core/context.hpp b/include/glaze/core/context.hpp index c545168b70..5420e63f1b 100644 --- a/include/glaze/core/context.hpp +++ b/include/glaze/core/context.hpp @@ -96,7 +96,9 @@ namespace glz buffer_overflow, // Write would exceed fixed buffer capacity invalid_length, // Length exceeds allowed limit (buffer size or user-configured max) // Encoding errors - invalid_utf8 // Malformed UTF-8 in a string; always checked on read + invalid_utf8, // Malformed UTF-8 in a string; always checked on read + // Streaming errors + streaming_unsupported // Document outruns the buffer window and this format's reader cannot refill }; // Unified error context for all read/write operations diff --git a/include/glaze/core/error_category.hpp b/include/glaze/core/error_category.hpp index 7b034d79c7..b49d42bb8d 100644 --- a/include/glaze/core/error_category.hpp +++ b/include/glaze/core/error_category.hpp @@ -80,7 +80,8 @@ struct glz::meta "patch_test_failed", "buffer_overflow", "invalid_length", - "invalid_utf8"}; + "invalid_utf8", + "streaming_unsupported"}; static constexpr std::array value{none, // version_mismatch, // invalid_header, // @@ -159,5 +160,7 @@ struct glz::meta buffer_overflow, // invalid_length, // // Encoding errors - invalid_utf8}; + invalid_utf8, // + // Streaming errors + streaming_unsupported}; }; diff --git a/include/glaze/core/istream_buffer.hpp b/include/glaze/core/istream_buffer.hpp index d7ecae2d40..b8b4c49de2 100644 --- a/include/glaze/core/istream_buffer.hpp +++ b/include/glaze/core/istream_buffer.hpp @@ -150,6 +150,12 @@ namespace glz // Check if stream is exhausted and buffer is empty bool eof() const noexcept { return eof_reached_ && empty(); } + // Whether the underlying stream has been read to its end, regardless of how much of the + // current window is still unread. Distinct from eof(): once this is true every byte the + // stream will ever produce is already in the buffer, which is what a reader needs to know + // to tell "this input is malformed" from "I have not been shown all of it yet". + bool source_exhausted() const noexcept { return eof_reached_; } + // Reset for reuse with same stream void reset() { diff --git a/include/glaze/core/read.hpp b/include/glaze/core/read.hpp index ab41dc992c..1710433f55 100644 --- a/include/glaze/core/read.hpp +++ b/include/glaze/core/read.hpp @@ -257,6 +257,33 @@ namespace glz finalize_read_context(ctx); } + // Only the JSON reader has refill points (see format_supports_streaming). Every other reader + // parses whatever the first window happens to hold and stops at its edge, and what it reports + // there does not name the window as the cause: NDJSON returns success carrying only the + // elements that fit, which is silent data loss, and BEVE returns unexpected_end, which reads + // as malformed input. Say what actually happened instead. + // + // The question is whether the reader was shown the whole document, so it asks source_at_eof + // rather than at_eof: at_eof also demands a drained window, which would call a value that + // legitimately left trailing bytes behind a truncated read. Once the source is exhausted the + // window holds everything there will ever be, so whatever the reader concluded, it concluded + // on the full input and stands. + // + // With input still pending, two outcomes are the window's doing rather than the document's: + // - an error, because a reader that cannot refill has no way to distinguish input that is + // malformed from input it simply has not been shown. Its own code would assert the first. + // - success with it == end, which is the reader having run out of window and called that + // the end. A parse that stopped short of the edge found a real end and is left alone. + // `end` is still the window's edge to compare against: a reader with no refill points never + // moved it. + if constexpr (!format_supports_streaming) { + if (!ctx.stream.source_at_eof() && (bool(ctx.error) || it == end)) { + ctx.error = error_code::streaming_unsupported; + ctx.custom_error_message = + "the document is larger than the buffer window and this format's reader cannot refill"; + } + } + if (bool(ctx.error)) { return {buffer.bytes_consumed(), ctx.error, ctx.custom_error_message}; } diff --git a/include/glaze/core/streaming_state.hpp b/include/glaze/core/streaming_state.hpp index 36c93b3665..948740d23d 100644 --- a/include/glaze/core/streaming_state.hpp +++ b/include/glaze/core/streaming_state.hpp @@ -35,6 +35,7 @@ namespace glz void (*consume)(void*, size_t) = nullptr; bool (*refill)(void*) = nullptr; bool (*eof)(void*) = nullptr; + bool (*source_eof)(void*) = nullptr; // Check if streaming is enabled bool enabled() const noexcept { return buffer_ptr != nullptr; } @@ -54,6 +55,11 @@ namespace glz // Check if at end of stream bool at_eof() const noexcept { return eof(buffer_ptr); } + // Whether the source has been read to its end, whether or not the window is drained. + // at_eof() requires both; this asks only the first, which is how a reader tells input it has + // seen in full from input the window cut short. + bool source_at_eof() const noexcept { return source_eof(buffer_ptr); } + // Consume up to current position and refill // Returns new iterators via out parameters // it_offset is how far into current buffer we've parsed @@ -78,6 +84,17 @@ namespace glz state.consume = [](void* p, size_t n) { static_cast(p)->consume(n); }; state.refill = [](void* p) -> bool { return static_cast(p)->refill(); }; state.eof = [](void* p) -> bool { return static_cast(p)->eof(); }; + if constexpr (requires(Buffer& b) { + { b.source_exhausted() } -> std::convertible_to; + }) { + state.source_eof = [](void* p) -> bool { return static_cast(p)->source_exhausted(); }; + } + else { + // A buffer that does not distinguish the two can only offer the stricter answer. That is + // safe in the direction that matters: it never claims the source is exhausted when it is + // not, so nothing concludes it has seen input it has not. + state.source_eof = [](void* p) -> bool { return static_cast(p)->eof(); }; + } return state; } diff --git a/include/glaze/forward.hpp b/include/glaze/forward.hpp index c25257ecbe..a2bdc27dba 100644 --- a/include/glaze/forward.hpp +++ b/include/glaze/forward.hpp @@ -48,6 +48,20 @@ namespace glz inline constexpr std::uint32_t REST = 30100; inline constexpr std::uint32_t JSONRPC = 30200; + // Whether a format's reader can refill from an input stream mid-parse, and so read a document + // larger than the buffer window. Only the JSON reader has refill points today; every other + // reader sees one window and stops at its edge. Reading a longer document through such a format + // fails with error_code::streaming_unsupported rather than silently returning what fit. + // + // This is a property of the reader, not of the format's grammar: NDJSON is false because its + // line loop cannot refill between lines, even though the per-line values it delegates to the + // JSON reader can. A user-defined format that gives its reader refill points specializes this. + template + inline constexpr bool format_supports_streaming = false; + + template <> + inline constexpr bool format_supports_streaming = true; + // Reflection metadata customization point. template struct meta; diff --git a/tests/istream_buffer_test/istream_buffer_test.cpp b/tests/istream_buffer_test/istream_buffer_test.cpp index e5e2675f54..5c57215e69 100644 --- a/tests/istream_buffer_test/istream_buffer_test.cpp +++ b/tests/istream_buffer_test/istream_buffer_test.cpp @@ -4218,4 +4218,103 @@ suite streaming_buffer_dispatch_tests = [] { }; }; +// Routing every format to read_streaming only removes the overrun; it does not give a reader +// refill points it never had. JSON is the only reader that has them, so for the rest a document +// longer than the window used to end in an answer that blamed the document: NDJSON reported +// success carrying the elements that fit, BEVE reported unexpected_end. Both now say which it was. +suite non_streaming_format_reporting = [] { + static_assert(glz::format_supports_streaming); + static_assert(!glz::format_supports_streaming, + "NDJSON's line loop cannot refill between lines, whatever its per-line values do"); + static_assert(!glz::format_supports_streaming); + + "NDJSON larger than the window does not silently truncate"_test = [] { + std::vector src{}; + for (int i = 0; i < 400; ++i) src.push_back(i); + std::string doc{}; + expect(!glz::write_ndjson(src, doc)); + expect(doc.size() > narrow_window) << "the document has to outrun the window to test anything"; + + std::istringstream iss{doc}; + glz::basic_istream_buffer buffer(iss); + + std::vector v{}; + const auto ec = glz::read_streaming(v, buffer); + expect(ec == glz::error_code::streaming_unsupported) << "a short read must be reported, not returned as success"; + }; + + "BEVE larger than the window names the window"_test = [] { + std::vector src{}; + for (int i = 0; i < 400; ++i) src.push_back(i); + std::string doc{}; + expect(!glz::write_beve(src, doc)); + expect(doc.size() > narrow_window); + + std::istringstream iss{doc}; + glz::basic_istream_buffer buffer(iss); + + std::vector v{}; + const auto ec = glz::read_streaming(v, buffer); + expect(ec == glz::error_code::streaming_unsupported) + << "unexpected_end would blame the document for a window that was too small"; + }; + + // The three ways a non-streaming read is legitimate. Each one must keep its own answer, because + // the check keys on whether the source was exhausted rather than on whether the window was. + "a document that fits still reads"_test = [] { + std::vector src{1, 2, 3, 4, 5}; + std::string doc{}; + expect(!glz::write_beve(src, doc)); + expect(doc.size() < narrow_window); + + std::istringstream iss{doc}; + glz::basic_istream_buffer buffer(iss); + + std::vector v{}; + expect(!glz::read_streaming(v, buffer)); + expect(v.size() == 5u); + }; + + "a value followed by trailing bytes is not a short read"_test = [] { + std::vector src{1, 2, 3, 4, 5}; + std::string doc{}; + expect(!glz::write_beve(src, doc)); + const std::string two = doc + doc; // second document left unread in the same window + expect(two.size() < narrow_window); + + std::istringstream iss{two}; + glz::basic_istream_buffer buffer(iss); + + std::vector v{}; + expect(!glz::read_streaming(v, buffer)) + << "the window is undrained but the source is exhausted, so the read saw everything it needed"; + expect(v.size() == 5u); + }; + + "genuinely malformed input keeps its own error"_test = [] { + std::vector src{1, 2, 3, 4, 5}; + std::string doc{}; + expect(!glz::write_beve(src, doc)); + const std::string truncated = doc.substr(0, doc.size() - 3); + + std::istringstream iss{truncated}; + glz::basic_istream_buffer buffer(iss); + + std::vector v{}; + const auto ec = glz::read_streaming(v, buffer); + expect(ec == glz::error_code::unexpected_end) + << "the whole source fit in the window, so the document really is truncated"; + }; + + // The trait is what exempts JSON, so pin that the exemption is live. + "JSON larger than the window still streams to completion"_test = [] { + std::istringstream iss{oversized_array()}; + glz::basic_istream_buffer buffer(iss); + + std::vector v{}; + expect(!glz::read_streaming(v, buffer)); + expect(v.size() == 400u); + }; +}; + int main() { return 0; }