Route streaming buffers through read_streaming in glz::read

include/glaze/core/context.hpp

@@ -96,7 +96,9 @@ namespace glz
       buffer_overflow, // Write would exceed fixed buffer capacity
       invalid_length, // Length exceeds allowed limit (buffer size or user-configured max)
       // Encoding errors
-      invalid_utf8 // Malformed UTF-8 in a string; always checked on read
+      invalid_utf8, // Malformed UTF-8 in a string; always checked on read
+      // Streaming errors
+      streaming_unsupported // Document outruns the buffer window and this format's reader cannot refill
    };
 
    // Unified error context for all read/write operations

include/glaze/core/error_category.hpp

@@ -80,7 +80,8 @@ struct glz::meta<glz::error_code>
                                     "patch_test_failed",
                                     "buffer_overflow",
                                     "invalid_length",
-                                    "invalid_utf8"};
+                                    "invalid_utf8",
+                                    "streaming_unsupported"};
    static constexpr std::array value{none, //
                                      version_mismatch, //
                                      invalid_header, //
@@ -159,5 +160,7 @@ struct glz::meta<glz::error_code>
                                      buffer_overflow, //
                                      invalid_length, //
                                      // Encoding errors
-                                     invalid_utf8};
+                                     invalid_utf8, //
+                                     // Streaming errors
+                                     streaming_unsupported};
 };

include/glaze/core/istream_buffer.hpp

@@ -150,6 +150,12 @@ namespace glz
       // Check if stream is exhausted and buffer is empty
       bool eof() const noexcept { return eof_reached_ && empty(); }
 
+      // Whether the underlying stream has been read to its end, regardless of how much of the
+      // current window is still unread. Distinct from eof(): once this is true every byte the
+      // stream will ever produce is already in the buffer, which is what a reader needs to know
+      // to tell "this input is malformed" from "I have not been shown all of it yet".
+      bool source_exhausted() const noexcept { return eof_reached_; }
+
       // Reset for reuse with same stream
       void reset()
       {

include/glaze/core/read.hpp

@@ -85,9 +85,9 @@ namespace glz
       finalize_read_context<Opts>(ctx);
    }
 
-   template <auto Opts, class T>
-      requires read_supported<T, Opts.format>
-   [[nodiscard]] error_ctx read(T& value, contiguous auto&& buffer, is_context auto&& ctx)
+   template <auto Opts, class T, contiguous Buf>
+      requires read_supported<T, Opts.format> && (!is_input_streaming<Buf>)
+   [[nodiscard]] error_ctx read(T& value, Buf&& buffer, is_context auto&& ctx)
    {
       static_assert(sizeof(decltype(*buffer.data())) == 1);
       using Buffer = std::remove_reference_t<decltype(buffer)>;
@@ -159,9 +159,9 @@ namespace glz
       return {size_t(it - start), ctx.error, ctx.custom_error_message};
    }
 
-   template <auto Opts, class T>
-      requires read_supported<T, Opts.format>
-   [[nodiscard]] error_ctx read(T& value, contiguous auto&& buffer)
+   template <auto Opts, class T, contiguous Buf>
+      requires read_supported<T, Opts.format> && (!is_input_streaming<Buf>)
+   [[nodiscard]] error_ctx read(T& value, Buf&& buffer)
    {
       format_context_t<Opts.format> ctx{};
       return read<Opts>(value, buffer, ctx);
@@ -257,6 +257,33 @@ namespace glz
          finalize_read_context<StreamingOpts>(ctx);
       }
 
+      // Only the JSON reader has refill points (see format_supports_streaming). Every other reader
+      // parses whatever the first window happens to hold and stops at its edge, and what it reports
+      // there does not name the window as the cause: NDJSON returns success carrying only the
+      // elements that fit, which is silent data loss, and BEVE returns unexpected_end, which reads
+      // as malformed input. Say what actually happened instead.
+      //
+      // The question is whether the reader was shown the whole document, so it asks source_at_eof
+      // rather than at_eof: at_eof also demands a drained window, which would call a value that
+      // legitimately left trailing bytes behind a truncated read. Once the source is exhausted the
+      // window holds everything there will ever be, so whatever the reader concluded, it concluded
+      // on the full input and stands.
+      //
+      // With input still pending, two outcomes are the window's doing rather than the document's:
+      //   - an error, because a reader that cannot refill has no way to distinguish input that is
+      //     malformed from input it simply has not been shown. Its own code would assert the first.
+      //   - success with it == end, which is the reader having run out of window and called that
+      //     the end. A parse that stopped short of the edge found a real end and is left alone.
+      // `end` is still the window's edge to compare against: a reader with no refill points never
+      // moved it.
+      if constexpr (!format_supports_streaming<Opts.format>) {
+         if (!ctx.stream.source_at_eof() && (bool(ctx.error) || it == end)) {
+            ctx.error = error_code::streaming_unsupported;
+            ctx.custom_error_message =
+               "the document is larger than the buffer window and this format's reader cannot refill";
+         }
+      }
+
       if (bool(ctx.error)) {
          return {buffer.bytes_consumed(), ctx.error, ctx.custom_error_message};
       }
@@ -271,4 +298,35 @@ namespace glz
       streaming_context ctx{};
       return read_streaming<Opts>(value, std::forward<Buffer>(buffer), ctx);
    }
+
+   // A streaming buffer satisfies `contiguous`, but the span it exposes is one window of a larger
+   // stream: it ends wherever the last fill stopped, it carries no terminator, and reading it as a
+   // flat buffer parses that window alone. Route it to read_streaming rather than let it bind to
+   // the overloads above, which would run with null_terminated on and read past the window's end.
+   // Doing this here rather than per format is what keeps read_jsonc, read_beve and every other
+   // helper that funnels through `read` from having to remember on its own.
+   template <auto Opts, class T, class Buffer>
+      requires read_supported<T, Opts.format> && is_input_streaming<std::remove_reference_t<Buffer>>
+   [[nodiscard]] error_ctx read(T& value, Buffer&& buffer, is_context auto&& ctx)
+   {
+      if constexpr (has_streaming_state<decltype(ctx)>) {
+         return read_streaming<Opts>(value, std::forward<Buffer>(buffer), ctx);
+      }
+      else {
+         // The caller's context has nowhere to hold the streaming state the parsers refill
+         // through, so the read runs on one that does and reports back through theirs.
+         streaming_context stream_ctx{};
+         const error_ctx ec = read_streaming<Opts>(value, std::forward<Buffer>(buffer), stream_ctx);
+         ctx.error = stream_ctx.error;
+         ctx.custom_error_message = stream_ctx.custom_error_message;
+         return ec;
+      }
+   }
+
+   template <auto Opts, class T, class Buffer>
+      requires read_supported<T, Opts.format> && is_input_streaming<std::remove_reference_t<Buffer>>
+   [[nodiscard]] error_ctx read(T& value, Buffer&& buffer)
+   {
+      return read_streaming<Opts>(value, std::forward<Buffer>(buffer));
+   }
 }

include/glaze/core/streaming_state.hpp

@@ -35,6 +35,7 @@ namespace glz
       void (*consume)(void*, size_t) = nullptr;
       bool (*refill)(void*) = nullptr;
       bool (*eof)(void*) = nullptr;
+      bool (*source_eof)(void*) = nullptr;
 
       // Check if streaming is enabled
       bool enabled() const noexcept { return buffer_ptr != nullptr; }
@@ -54,6 +55,11 @@ namespace glz
       // Check if at end of stream
       bool at_eof() const noexcept { return eof(buffer_ptr); }
 
+      // Whether the source has been read to its end, whether or not the window is drained.
+      // at_eof() requires both; this asks only the first, which is how a reader tells input it has
+      // seen in full from input the window cut short.
+      bool source_at_eof() const noexcept { return source_eof(buffer_ptr); }
+
       // Consume up to current position and refill
       // Returns new iterators via out parameters
       // it_offset is how far into current buffer we've parsed
@@ -78,6 +84,17 @@ namespace glz
       state.consume = [](void* p, size_t n) { static_cast<Buffer*>(p)->consume(n); };
       state.refill = [](void* p) -> bool { return static_cast<Buffer*>(p)->refill(); };
       state.eof = [](void* p) -> bool { return static_cast<Buffer*>(p)->eof(); };
+      if constexpr (requires(Buffer& b) {
+                       { b.source_exhausted() } -> std::convertible_to<bool>;
+                    }) {
+         state.source_eof = [](void* p) -> bool { return static_cast<Buffer*>(p)->source_exhausted(); };
+      }
+      else {
+         // A buffer that does not distinguish the two can only offer the stricter answer. That is
+         // safe in the direction that matters: it never claims the source is exhausted when it is
+         // not, so nothing concludes it has seen input it has not.
+         state.source_eof = [](void* p) -> bool { return static_cast<Buffer*>(p)->eof(); };
+      }
       return state;
    }
 

include/glaze/forward.hpp

@@ -48,6 +48,20 @@ namespace glz
    inline constexpr std::uint32_t REST = 30100;
    inline constexpr std::uint32_t JSONRPC = 30200;
 
+   // Whether a format's reader can refill from an input stream mid-parse, and so read a document
+   // larger than the buffer window. Only the JSON reader has refill points today; every other
+   // reader sees one window and stops at its edge. Reading a longer document through such a format
+   // fails with error_code::streaming_unsupported rather than silently returning what fit.
+   //
+   // This is a property of the reader, not of the format's grammar: NDJSON is false because its
+   // line loop cannot refill between lines, even though the per-line values it delegates to the
+   // JSON reader can. A user-defined format that gives its reader refill points specializes this.
+   template <std::uint32_t Format>
+   inline constexpr bool format_supports_streaming = false;
+
+   template <>
+   inline constexpr bool format_supports_streaming<JSON> = true;
+
    // Reflection metadata customization point.
    template <class T>
    struct meta;

tests/istream_buffer_test/istream_buffer_test.cpp

@@ -4144,4 +4144,177 @@ suite additional_edge_cases = [] {
    };
 };
 
+// A streaming buffer satisfies `contiguous`, so before these entry points routed to read_streaming
+// they bound to the buffered read: one window, parsed with null_terminated on, read past its end.
+// Only read_json carried its own streaming overload; everything funnelling through `read` did not.
+// A document larger than the window is what makes the difference visible -- it both overruns the
+// window and, once routed correctly, needs the refills to complete.
+namespace
+{
+   std::string oversized_array()
+   {
+      std::string json = "[";
+      for (int i = 0; i < 400; ++i) {
+         if (i) json += ',';
+         json += std::to_string(i);
+      }
+      return json + "]";
+   }
+
+   constexpr size_t narrow_window = 512; // smaller than oversized_array()
+}
+
+suite streaming_buffer_dispatch_tests = [] {
+   "read_jsonc streams instead of parsing one window"_test = [] {
+      std::istringstream iss{oversized_array()};
+      glz::basic_istream_buffer<std::istringstream, narrow_window> buffer(iss);
+
+      std::vector<int> v{};
+      expect(!glz::read_jsonc(v, buffer));
+      expect(v.size() == 400u);
+      expect(v[399] == 399);
+   };
+
+   "generic read streams instead of parsing one window"_test = [] {
+      std::istringstream iss{oversized_array()};
+      glz::basic_istream_buffer<std::istringstream, narrow_window> buffer(iss);
+
+      std::vector<int> v{};
+      expect(!glz::read<glz::opts{}>(v, buffer));
+      expect(v.size() == 400u);
+   };
+
+   // The context-taking overload has nowhere to put the streaming state, so it runs on one that
+   // does. The caller's context must still come back carrying the outcome.
+   "generic read with a caller supplied context"_test = [] {
+      std::istringstream iss{oversized_array()};
+      glz::basic_istream_buffer<std::istringstream, narrow_window> buffer(iss);
+
+      std::vector<int> v{};
+      glz::context ctx{};
+      expect(!glz::read<glz::opts{}>(v, buffer, ctx));
+      expect(v.size() == 400u);
+      expect(ctx.error == glz::error_code::none);
+   };
+
+   "caller supplied context reports the error"_test = [] {
+      std::istringstream iss{R"([1,2,)"};
+      glz::basic_istream_buffer<std::istringstream, narrow_window> buffer(iss);
+
+      std::vector<int> v{};
+      glz::context ctx{};
+      expect(glz::read<glz::opts{}>(v, buffer, ctx)) << "truncated input must fail";
+      expect(ctx.error != glz::error_code::none) << "the caller's context must carry the error";
+   };
+
+   // read_json already had streaming overloads; nothing here may disturb them.
+   "read_json is unchanged"_test = [] {
+      std::istringstream iss{oversized_array()};
+      glz::basic_istream_buffer<std::istringstream, narrow_window> buffer(iss);
+
+      std::vector<int> v{};
+      expect(!glz::read_json(v, buffer));
+      expect(v.size() == 400u);
+   };
+};
+
+// Routing every format to read_streaming only removes the overrun; it does not give a reader
+// refill points it never had. JSON is the only reader that has them, so for the rest a document
+// longer than the window used to end in an answer that blamed the document: NDJSON reported
+// success carrying the elements that fit, BEVE reported unexpected_end. Both now say which it was.
+suite non_streaming_format_reporting = [] {
+   static_assert(glz::format_supports_streaming<glz::JSON>);
+   static_assert(!glz::format_supports_streaming<glz::NDJSON>,
+                 "NDJSON's line loop cannot refill between lines, whatever its per-line values do");
+   static_assert(!glz::format_supports_streaming<glz::BEVE>);
+
+   "NDJSON larger than the window does not silently truncate"_test = [] {
+      std::vector<int> src{};
+      for (int i = 0; i < 400; ++i) src.push_back(i);
+      std::string doc{};
+      expect(!glz::write_ndjson(src, doc));
+      expect(doc.size() > narrow_window) << "the document has to outrun the window to test anything";
+
+      std::istringstream iss{doc};
+      glz::basic_istream_buffer<std::istringstream, narrow_window> buffer(iss);
+
+      std::vector<int> v{};
+      const auto ec = glz::read_streaming<glz::opts{.format = glz::NDJSON}>(v, buffer);
+      expect(ec == glz::error_code::streaming_unsupported) << "a short read must be reported, not returned as success";
+   };
+
+   "BEVE larger than the window names the window"_test = [] {
+      std::vector<int> src{};
+      for (int i = 0; i < 400; ++i) src.push_back(i);
+      std::string doc{};
+      expect(!glz::write_beve(src, doc));
+      expect(doc.size() > narrow_window);
+
+      std::istringstream iss{doc};
+      glz::basic_istream_buffer<std::istringstream, narrow_window> buffer(iss);
+
+      std::vector<int> v{};
+      const auto ec = glz::read_streaming<glz::opts{.format = glz::BEVE}>(v, buffer);
+      expect(ec == glz::error_code::streaming_unsupported)
+         << "unexpected_end would blame the document for a window that was too small";
+   };
+
+   // The three ways a non-streaming read is legitimate. Each one must keep its own answer, because
+   // the check keys on whether the source was exhausted rather than on whether the window was.
+   "a document that fits still reads"_test = [] {
+      std::vector<int> src{1, 2, 3, 4, 5};
+      std::string doc{};
+      expect(!glz::write_beve(src, doc));
+      expect(doc.size() < narrow_window);
+
+      std::istringstream iss{doc};
+      glz::basic_istream_buffer<std::istringstream, narrow_window> buffer(iss);
+
+      std::vector<int> v{};
+      expect(!glz::read_streaming<glz::opts{.format = glz::BEVE}>(v, buffer));
+      expect(v.size() == 5u);
+   };
+
+   "a value followed by trailing bytes is not a short read"_test = [] {
+      std::vector<int> src{1, 2, 3, 4, 5};
+      std::string doc{};
+      expect(!glz::write_beve(src, doc));
+      const std::string two = doc + doc; // second document left unread in the same window
+      expect(two.size() < narrow_window);
+
+      std::istringstream iss{two};
+      glz::basic_istream_buffer<std::istringstream, narrow_window> buffer(iss);
+
+      std::vector<int> v{};
+      expect(!glz::read_streaming<glz::opts{.format = glz::BEVE}>(v, buffer))
+         << "the window is undrained but the source is exhausted, so the read saw everything it needed";
+      expect(v.size() == 5u);
+   };
+
+   "genuinely malformed input keeps its own error"_test = [] {
+      std::vector<int> src{1, 2, 3, 4, 5};
+      std::string doc{};
+      expect(!glz::write_beve(src, doc));
+      const std::string truncated = doc.substr(0, doc.size() - 3);
+
+      std::istringstream iss{truncated};
+      glz::basic_istream_buffer<std::istringstream, narrow_window> buffer(iss);
+
+      std::vector<int> v{};
+      const auto ec = glz::read_streaming<glz::opts{.format = glz::BEVE}>(v, buffer);
+      expect(ec == glz::error_code::unexpected_end)
+         << "the whole source fit in the window, so the document really is truncated";
+   };
+
+   // The trait is what exempts JSON, so pin that the exemption is live.
+   "JSON larger than the window still streams to completion"_test = [] {
+      std::istringstream iss{oversized_array()};
+      glz::basic_istream_buffer<std::istringstream, narrow_window> buffer(iss);
+
+      std::vector<int> v{};
+      expect(!glz::read_streaming<glz::opts{}>(v, buffer));
+      expect(v.size() == 400u);
+   };
+};
+
 int main() { return 0; }