feat: implement virtio based host-guest communication

.gitmodules

@@ -2,3 +2,6 @@
 	path = src/hyperlight_libc/third_party/picolibc
 	url = https://github.com/hyperlight-dev/picolibc-bsd.git
 	shallow = true
+[submodule "src/hyperlight_libc/third_party/mimalloc"]
+	path = src/hyperlight_libc/third_party/mimalloc
+	url = https://github.com/microsoft/mimalloc

Cargo.toml

@@ -50,6 +50,7 @@ hyperlight-component-macro = { path = "src/hyperlight_component_macro", version
 
 [workspace.lints.rust]
 unsafe_op_in_unsafe_fn = "deny"
+unexpected_cfgs = { level = "warn", check-cfg = [ 'cfg(loom)' ] }
 
 # this will generate symbols for release builds
 # so is handy for debugging issues in release builds

Justfile

@@ -225,7 +225,7 @@ test-unit target=default-target features="":
 test-isolated target=default-target features="" :
     {{ cargo-cmd }} test {{ if features =="" {''} else if features=="no-default-features" {"--no-default-features" } else {"--no-default-features -F " + features } }} --profile={{ if target == "debug" { "dev" } else { target } }} {{ target-triple-flag }} -p hyperlight-host --lib -- sandbox::uninitialized::tests::test_log_trace --exact --ignored
     {{ cargo-cmd }} test {{ if features =="" {''} else if features=="no-default-features" {"--no-default-features" } else {"--no-default-features -F " + features } }} --profile={{ if target == "debug" { "dev" } else { target } }} {{ target-triple-flag }} -p hyperlight-host --lib -- sandbox::outb::tests::test_log_outb_log --exact --ignored
-    {{ cargo-cmd }} test {{ if features =="" {''} else if features=="no-default-features" {"--no-default-features" } else {"--no-default-features -F " + features } }} --profile={{ if target == "debug" { "dev" } else { target } }} {{ target-triple-flag }} -p hyperlight-host --test integration_test -- log_message --exact --ignored
+    {{ cargo-cmd }} test {{ if features =="" {''} else if features=="no-default-features" {"--no-default-features" } else {"--no-default-features -F " + features } }} --profile={{ if target == "debug" { "dev" } else { target } }} {{ target-triple-flag }} -p hyperlight-host --test integration_test -- --test-threads=1 --ignored
     @# metrics tests
     {{ cargo-cmd }} test {{ if features =="" {''} else if features=="no-default-features" {"--no-default-features" } else {"--no-default-features -F function_call_metrics," + features } }} --profile={{ if target == "debug" { "dev" } else { target } }} {{ target-triple-flag }} -p hyperlight-host --lib -- metrics::tests::test_metrics_are_emitted --exact
 

docs/README.md

@@ -34,6 +34,7 @@ This project is composed internally of several components, depicted in the below
 
 * [Security guidance for developers](./security-guidance-for-developers.md)
 * [Paging Development Notes](./paging-development-notes.md)
+* [Virtio host/guest communication](./virtio-host-guest-communication.md)
 * [How to debug a Hyperlight guest](./how-to-debug-a-hyperlight-guest.md)
 * [How to use Flatbuffers in Hyperlight](./how-to-use-flatbuffers.md)
 * [How to make a Hyperlight release](./how-to-make-releases.md)

docs/virtio-host-guest-communication.md

@@ -0,0 +1,173 @@
+# Virtio host/guest communication
+
+Hyperlight's virtio communication path is built on packed virtqueues. It is
+intended to replace stack-oriented host/guest calls with a transport that can
+support request/response calls, async calls, streams, batching, and large
+payloads.
+
+The implementation is layered so that low-level virtqueue mechanics stay
+separate from the protocol and dispatch logic that uses them.
+
+## Queue directions
+
+Hyperlight uses two logical queues:
+
+| Queue | Producer | Consumer | Typical payloads |
+| --- | --- | --- | --- |
+| Guest-to-host (G2H) | Guest | Host | Host function calls, guest logs, H2G function-call responses |
+| Host-to-guest (H2G) | Guest, via prefilled writable chains | Host | Host-to-guest function calls/events |
+
+The H2G queue is intentionally subtle: the guest owns the producer side and
+prefills the queue with device-writable receive capacity. The host consumes
+those chains, writes H2G requests into the writable buffers, and completes the
+chains. The guest then observes those completed chains as incoming data through
+its producer.
+
+## Layering
+
+The virtqueue code is split into these layers:
+
+| Layer | Main types | Responsibility |
+| --- | --- | --- |
+| Ring primitives | `RingProducer`, `RingConsumer`, `BufferChain`, `BufferElement`, `Descriptor` | Packed-ring descriptor publication, polling, event suppression, and memory ordering. |
+| Allocation | `BufferProvider`, `Allocation`, `BufferPool`, `RecyclePool`, `PoolAlloc`, `BufferOwner` | Allocate shared-memory buffers and keep pool ownership tied to chain lifetime. |
+| High-level chain API | `VirtqProducer`, `VirtqConsumer`, `SendChain`, `RecvChain`, `ReplyChain`, `WritableChain`, `AckChain`, `UsedChain` | Manage buffer allocation, chain lifecycle, safe payload views, and notifications. |
+| Payload view | `Segments`, `SegmentsBuf` | Preserve ordered byte segment boundaries and provide explicit contiguous conversion when needed. |
+| Protocol | `VirtqMsgHeader`, FlatBuffer wrappers | Interpret bytes as calls, results, logs, and future stream/batch messages. |
+
+The low-level `BufferChain` is descriptor-oriented: it contains addresses,
+lengths, and writable flags. The high-level `Segments` type is payload-oriented:
+it contains ordered `Bytes` values after shared-memory access has been resolved.
+These are deliberately separate abstractions.
+
+## Readable and writable buffers
+
+Readable and writable are named from the device/consumer perspective:
+
+- A **readable** buffer is written by the producer before submission and read by
+  the consumer after polling.
+- A **writable** buffer is reserved by the producer and filled by the consumer
+  before completion.
+
+Virtio requires readable descriptors to appear before writable descriptors in a
+chain. The high-level API preserves that ordering.
+
+## Chain lifecycle
+
+The high-level lifecycle uses these names:
+
+| Type | Owner | Meaning |
+| --- | --- | --- |
+| `SendChain` | Producer | A chain being prepared for submission. |
+| `RecvChain` | Consumer | A received chain whose readable payload has been copied into `Segments`. |
+| `ReplyChain` | Consumer | The capability to complete a `RecvChain`; either writable capacity or ack-only. |
+| `WritableChain` | Consumer | The writable form of `ReplyChain`; used to write response bytes. |
+| `AckChain` | Consumer | The ack-only form of `ReplyChain`; used for chains with no writable buffers. |
+| `UsedChain` | Producer | The producer-observed result after the consumer completes a chain. |
+
+In the common request/response case:
+
+```text
+producer.chain()
+    -> SendChain
+    -> producer.submit(SendChain) -> Token
+
+consumer.poll()
+    -> (RecvChain, ReplyChain)
+    -> consumer.complete(ReplyChain)
+
+producer.poll()
+    -> UsedChain
+```
+
+For fire-and-forget chains with no writable buffers, the producer receives
+`UsedChain::Ack(Token)`. For chains with writable buffers, the producer receives
+`UsedChain::Data(Token, Segments)`.
+
+## Payload segments
+
+`Segments` is the high-level payload shape. It preserves ordered byte segment
+boundaries instead of forcing every chain into one contiguous allocation.
+
+Use segment-aware APIs when possible:
+
+```rust
+let segments = recv_chain.segments();
+let mut cursor = segments.cursor();
+```
+
+Use explicit contiguous conversion only when a compatibility path needs it:
+
+```rust
+let bytes = recv_chain.to_bytes();
+let bytes = used_chain.into_bytes();
+```
+
+`Segments::to_bytes()` and `Segments::into_bytes()` are O(1) for zero or one
+segment. They allocate and copy when multiple segments must be flattened.
+
+`SegmentsBuf` implements `bytes::Buf`, so protocol code can parse across segment
+boundaries without collecting the entire payload. This is useful for fixed-size
+headers such as `VirtqMsgHeader`.
+
+## Writable replies and used lengths
+
+The virtio used ring reports one aggregate written length for a completed
+descriptor chain. It does not report per-writable-descriptor lengths.
+
+For that reason, `WritableChain` writes sequentially across writable buffers.
+When the producer later observes `UsedChain::Data`, it reconstructs returned
+segments greedily from the aggregate length:
+
+```text
+writable capacities: [4096, 4096, 4096]
+used length:          5000
+returned segments:    [4096, 904]
+```
+
+Random writes into arbitrary writable descriptors would make this reconstruction
+ambiguous, so the public write path preserves a sequential-write invariant.
+
+## Notifications and batching
+
+Submission and completion notification use virtio event suppression:
+
+- `VirtqProducer::submit` publishes one `SendChain` and notifies if suppression
+  allows.
+- `VirtqProducer::batch` publishes multiple chains and kicks at most once when
+  `finish` is called.
+- `VirtqConsumer::complete` marks one received chain used and notifies the
+  producer if suppression allows.
+- `notify_backpressure` bypasses suppression when the peer needs to drain work
+  to free descriptors or pool buffers.
+
+This keeps the common path simple while allowing higher-level code to batch
+calls, logs, stream chunks, or prefilled receive buffers.
+
+## FlatBuffer boundary
+
+FlatBuffer roots still need contiguous bytes. The current dispatch paths may
+explicitly call `to_bytes()` before verifying or decoding a FlatBuffer envelope.
+
+Large payloads should not depend on a single large contiguous allocation. The
+intended direction is:
+
+```text
+small contiguous envelope/header
+ordered Segments for large bytes, strings, stream data, or chunk messages
+```
+
+If a future schema references external payload segments, those segments need
+their own validation and ownership rules. FlatBuffer verification only validates
+the bytes inside the FlatBuffer buffer.
+
+## Current limitations and follow-ups
+
+- Some host/guest dispatch paths still collect `Segments` into contiguous
+  `Bytes` for existing FlatBuffer wrappers.
+- Large payload chunking is a protocol/schema follow-up; it is not part of the
+  chain lifecycle itself.
+- The allocator can become simpler once large logical payloads consistently use
+  bounded `Segments` instead of requiring large contiguous allocations.
+- Snapshot/reset ownership for outstanding `UsedChain::Data` buffers remains a
+  separate design point.

fuzz/fuzz_targets/guest_trace.rs

@@ -69,8 +69,8 @@ impl<'a> Arbitrary<'a> for FuzzInput {
 fuzz_target!(
     init: {
         let mut cfg = SandboxConfiguration::default();
-        // In local tests, 256 KiB seemed sufficient for deep recursion
-        cfg.set_scratch_size(256 * 1024);
+        // In local tests, 512 KiB seemed sufficient for deep recursion
+        cfg.set_scratch_size(512 * 1024);
         let path = simple_guest_for_fuzzing_as_string().expect("Guest Binary Missing");
         let u_sbox = UninitializedSandbox::new(
             GuestBinary::FilePath(path),

fuzz/fuzz_targets/host_call.rs

@@ -33,9 +33,9 @@ static SANDBOX: OnceLock<Mutex<MultiUseSandbox>> = OnceLock::new();
 fuzz_target!(
     init: {
         let mut cfg = SandboxConfiguration::default();
-        cfg.set_output_data_size(64 * 1024); // 64 KB output buffer
-        cfg.set_input_data_size(64 * 1024); // 64 KB input buffer
-        cfg.set_scratch_size(512 * 1024); // large scratch region to contain those buffers, any data copies, etc.
+        cfg.set_g2h_pool_pages(16); // 64 KB / 4096 = 16 pages
+        cfg.set_h2g_pool_pages(16); // 64 KB / 4096 = 16 pages
+        cfg.set_scratch_size(512 * 1024); // large scratch region
         let u_sbox = UninitializedSandbox::new(
             GuestBinary::FilePath(simple_guest_for_fuzzing_as_string().expect("Guest Binary Missing")),
             Some(cfg)
@@ -61,6 +61,8 @@ fuzz_target!(
                 HyperlightError::GuestError(ErrorCode::HostFunctionError, msg) if msg.contains("The number of arguments to the function is wrong") => {}
                 HyperlightError::ParameterValueConversionFailure(_, _) => {},
                 HyperlightError::GuestError(ErrorCode::HostFunctionError, msg) if msg.contains("Failed To Convert Parameter Value") => {}
+                HyperlightError::GuestError(ErrorCode::HostFunctionError, msg) if msg.contains("The parameter value type is unexpected") => {}
+                HyperlightError::GuestError(ErrorCode::HostFunctionError, msg) if msg.contains("The return value type is unexpected") => {}
 
                 // any other error should be reported
                 _ => panic!("Guest Aborted with Unexpected Error: {:?}", e),

src/hyperlight_common/Cargo.toml

@@ -19,6 +19,8 @@ arbitrary = {version = "1.4.2", optional = true, features = ["derive"]}
 anyhow = { version = "1.0.102", default-features = false }
 bitflags = "2.10.0"
 bytemuck = { version = "1.24", features = ["derive"] }
+bytes = { version = "1", default-features = false }
+fixedbitset = { version = "0.5.7", default-features = false }
 flatbuffers = { version = "25.12.19", default-features = false }
 log = "0.4.29"
 smallvec = "1.15.1"
@@ -39,9 +41,18 @@ nanvix-unstable = ["i686-guest"]
 guest-counter = []
 
 [dev-dependencies]
+criterion = "0.8.1"
+hyperlight-testing = { workspace = true }
 quickcheck = "1.0.3"
 rand = "0.9.2"
 
+[target.'cfg(loom)'.dev-dependencies]
+loom = "0.7"
+
 [lib]
 bench = false # see https://bheisler.github.io/criterion.rs/book/faq.html#cargo-bench-gives-unrecognized-option-errors-for-valid-command-line-options
 doctest = false # reduce noise in test output
+
+[[bench]]
+name = "buffer_pool"
+harness = false