From 24aa5a53e61f04ddd695a2e357d9cbb4074e5401 Mon Sep 17 00:00:00 2001 From: Nikita Kalyazin Date: Fri, 8 May 2026 16:04:06 +0100 Subject: [PATCH 1/3] feat(balloon): advertise VIRTIO_BALLOON_F_HINT_WAIT_ON_ACK with FPH Whenever free-page hinting is enabled, also advertise the new VIRTIO_BALLOON_F_HINT_WAIT_ON_ACK feature bit (6). When negotiated, the guest driver waits for the device to signal-used each hint buffer before pushing the just-hinted page onto vb->free_page_list, closing a stale-hint data-loss race where the shrinker could recycle a page back to the buddy allocator before discard_range completed on the host. Guests without kernel support for bit 6 simply do not negotiate it (the driver self-clears the bit if VIRTIO_BALLOON_F_FREE_PAGE_HINT is not also negotiated), so this is forward-compatible with stock guests. No host-side protocol change is required: process_free_page_hinting_queue already calls signal_used_queue once per drain, which serves as the ACK the guest waits on. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Nikita Kalyazin --- src/vmm/src/devices/virtio/balloon/device.rs | 34 ++++++++++++++++++- src/vmm/src/devices/virtio/balloon/mod.rs | 1 + src/vmm/src/devices/virtio/balloon/persist.rs | 25 ++++++++++++++ 3 files changed, 59 insertions(+), 1 deletion(-) diff --git a/src/vmm/src/devices/virtio/balloon/device.rs b/src/vmm/src/devices/virtio/balloon/device.rs index 411b84bc7be..db6da845919 100644 --- a/src/vmm/src/devices/virtio/balloon/device.rs +++ b/src/vmm/src/devices/virtio/balloon/device.rs @@ -20,7 +20,8 @@ use super::{ FREE_PAGE_HINT_STOP, INFLATE_INDEX, MAX_PAGE_COMPACT_BUFFER, MAX_PAGES_IN_DESC, MIB_TO_4K_PAGES, STATS_INDEX, VIRTIO_BALLOON_F_DEFLATE_ON_OOM, VIRTIO_BALLOON_F_FREE_PAGE_HINTING, VIRTIO_BALLOON_F_FREE_PAGE_REPORTING, - VIRTIO_BALLOON_F_STATS_VQ, VIRTIO_BALLOON_PFN_SHIFT, VIRTIO_BALLOON_S_ALLOC_STALL, + VIRTIO_BALLOON_F_HINT_WAIT_ON_ACK, VIRTIO_BALLOON_F_STATS_VQ, VIRTIO_BALLOON_PFN_SHIFT, + VIRTIO_BALLOON_S_ALLOC_STALL, VIRTIO_BALLOON_S_ASYNC_RECLAIM, VIRTIO_BALLOON_S_ASYNC_SCAN, VIRTIO_BALLOON_S_AVAIL, VIRTIO_BALLOON_S_CACHES, VIRTIO_BALLOON_S_DIRECT_RECLAIM, VIRTIO_BALLOON_S_DIRECT_SCAN, VIRTIO_BALLOON_S_HTLB_PGALLOC, VIRTIO_BALLOON_S_HTLB_PGFAIL, VIRTIO_BALLOON_S_MAJFLT, @@ -299,6 +300,7 @@ impl Balloon { if free_page_hinting { log_dev_preview_warning("Free Page Hinting", None); avail_features |= 1u64 << VIRTIO_BALLOON_F_FREE_PAGE_HINTING; + avail_features |= 1u64 << VIRTIO_BALLOON_F_HINT_WAIT_ON_ACK; queue_count += 1; } @@ -741,6 +743,10 @@ impl Balloon { self.avail_features & (1u64 << VIRTIO_BALLOON_F_FREE_PAGE_HINTING) != 0 } + pub fn free_page_hinting_wait_ack(&self) -> bool { + self.avail_features & (1u64 << VIRTIO_BALLOON_F_HINT_WAIT_ON_ACK) != 0 + } + pub fn free_page_hinting_idx(&self) -> usize { let mut idx = BALLOON_MIN_NUM_QUEUES; @@ -1169,6 +1175,7 @@ pub(crate) mod tests { | (u64::from(*deflate_on_oom) << VIRTIO_BALLOON_F_DEFLATE_ON_OOM) | ((u64::from(*reporting)) << VIRTIO_BALLOON_F_FREE_PAGE_REPORTING) | ((u64::from(*hinting)) << VIRTIO_BALLOON_F_FREE_PAGE_HINTING) + | ((u64::from(*hinting)) << VIRTIO_BALLOON_F_HINT_WAIT_ON_ACK) | ((u64::from(*stats_interval)) << VIRTIO_BALLOON_F_STATS_VQ); assert_eq!( @@ -1188,6 +1195,31 @@ pub(crate) mod tests { } } + #[test] + fn test_wait_on_ack_advertised_with_fph() { + let balloon = Balloon::new(0, false, 0, true, false).unwrap(); + assert!(balloon.free_page_hinting()); + assert!(balloon.free_page_hinting_wait_ack()); + assert_ne!( + balloon.avail_features & (1u64 << VIRTIO_BALLOON_F_HINT_WAIT_ON_ACK), + 0 + ); + } + + #[test] + fn test_wait_on_ack_not_advertised_without_fph() { + for (deflate, stats_interval, reporting) in + iproduct!(&[true, false], &[0u16, 1], &[true, false]) + { + let balloon = Balloon::new(0, *deflate, *stats_interval, false, *reporting).unwrap(); + assert!(!balloon.free_page_hinting_wait_ack()); + assert_eq!( + balloon.avail_features & (1u64 << VIRTIO_BALLOON_F_HINT_WAIT_ON_ACK), + 0 + ); + } + } + #[test] fn test_virtio_read_config() { let balloon = Balloon::new(0x10, true, 0, false, false).unwrap(); diff --git a/src/vmm/src/devices/virtio/balloon/mod.rs b/src/vmm/src/devices/virtio/balloon/mod.rs index d222989a1cb..7c31e937d38 100644 --- a/src/vmm/src/devices/virtio/balloon/mod.rs +++ b/src/vmm/src/devices/virtio/balloon/mod.rs @@ -54,6 +54,7 @@ const VIRTIO_BALLOON_F_STATS_VQ: u32 = 1; // Enable statistics. const VIRTIO_BALLOON_F_DEFLATE_ON_OOM: u32 = 2; // Deflate balloon on OOM. const VIRTIO_BALLOON_F_FREE_PAGE_HINTING: u32 = 3; // Enable free page hinting const VIRTIO_BALLOON_F_FREE_PAGE_REPORTING: u32 = 5; // Enable free page reporting +pub(crate) const VIRTIO_BALLOON_F_HINT_WAIT_ON_ACK: u32 = 6; // Hinting waits on device ack // The statistics tags. defined in linux "include/uapi/linux/virtio_balloon.h". const VIRTIO_BALLOON_S_SWAP_IN: u16 = 0; diff --git a/src/vmm/src/devices/virtio/balloon/persist.rs b/src/vmm/src/devices/virtio/balloon/persist.rs index f044c99494b..f990a83aaaf 100644 --- a/src/vmm/src/devices/virtio/balloon/persist.rs +++ b/src/vmm/src/devices/virtio/balloon/persist.rs @@ -263,4 +263,29 @@ mod tests { assert_eq!(restored_balloon.stats_desc_index, balloon.stats_desc_index); assert_eq!(restored_balloon.latest_stats, balloon.latest_stats); } + + #[test] + fn test_wait_on_ack_round_trips_snapshot() { + let guest_mem = default_mem(); + let mut mem = vec![0; 4096]; + + let balloon = Balloon::new(0, false, 0, true, false).unwrap(); + assert!(balloon.free_page_hinting_wait_ack()); + + Snapshot::new(balloon.save()) + .save(&mut mem.as_mut_slice()) + .unwrap(); + + let restored_balloon = Balloon::restore( + BalloonConstructorArgs { mem: guest_mem }, + &Snapshot::load_without_crc_check(mem.as_slice()) + .unwrap() + .data, + ) + .unwrap(); + + assert!(restored_balloon.free_page_hinting()); + assert!(restored_balloon.free_page_hinting_wait_ack()); + assert_eq!(restored_balloon.avail_features, balloon.avail_features); + } } From 01f182628afd7e1395280ce8a8e432f08dfeb3bf Mon Sep 17 00:00:00 2001 From: Nikita Kalyazin Date: Fri, 8 May 2026 16:09:34 +0100 Subject: [PATCH 2/3] test(balloon): integration test for WAIT_ON_ACK negotiation Adds a guest-side check that the negotiated balloon features in /sys/bus/virtio/devices/virtioN/features include bit 3 (FREE_PAGE_HINT) and bit 6 (HINT_WAIT_ON_ACK) when free_page_hinting is enabled. The test is gated on a new dedicated marker, requires_patched_kernel, which is registered in tests/pytest.ini and added to the default -m exclusion filter so the test is auto-skipped by every CI run (regular and nightly). To run it, replace the 6.1 artifact vmlinux with a build that carries Jack Thomson's wait-on-ACK patch and invoke: tools/devtool -y test -- -m requires_patched_kernel \ tests/integration_tests/functional/test_balloon_wait_on_ack.py If the kernel is not patched, the bit-6 assertion fails with a clear "did you replace the kernel?" message. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Nikita Kalyazin --- .../functional/test_balloon_wait_on_ack.py | 77 +++++++++++++++++++ tests/pytest.ini | 3 +- 2 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 tests/integration_tests/functional/test_balloon_wait_on_ack.py diff --git a/tests/integration_tests/functional/test_balloon_wait_on_ack.py b/tests/integration_tests/functional/test_balloon_wait_on_ack.py new file mode 100644 index 00000000000..77cb3ebdf4e --- /dev/null +++ b/tests/integration_tests/functional/test_balloon_wait_on_ack.py @@ -0,0 +1,77 @@ +# Copyright 2026 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Verify VIRTIO_BALLOON_F_HINT_WAIT_ON_ACK negotiation with a patched guest. + +This test is gated on `@pytest.mark.requires_patched_kernel` and is excluded +from every CI run (regular and nightly) by `tests/pytest.ini`. It assumes the +6.1 artifact `vmlinux` has been replaced in place with a build that carries +Jack Thomson's `virtio_balloon: Support wait on ACK for hinting` patch (see +fc-kernels/patches/6.1.158/0001-virtio_balloon-Support-wait-on-ACK-for-hinting.patch). + +To run it after swapping in the patched kernel: + + tools/devtool -y test -- -m requires_patched_kernel \\ + tests/integration_tests/functional/test_balloon_wait_on_ack.py + +If the kernel is *not* patched, the bit-6 assertion fails with a clear +"did you replace the kernel?" message — that's the expected signal that +the prerequisite is missing. +""" + +import pytest + +VIRTIO_ID_BALLOON = 5 +VIRTIO_BALLOON_F_FREE_PAGE_HINT = 3 +VIRTIO_BALLOON_F_HINT_WAIT_ON_ACK = 6 + + +def _read_balloon_features(vm): + """Return the negotiated features bitstring of the balloon virtio device. + + Linux exposes negotiated features via + `/sys/bus/virtio/devices/virtio*/features` as one ASCII char per bit, + LSB-first, plus a trailing newline. See `features_show` in upstream + `drivers/virtio/virtio.c`. + """ + # /sys/bus/virtio/devices/virtio*/device is a 0x%04x string with + # trailing newline, e.g. "0x0005\n" for balloon. Match by hex value. + cmd = ( + "for d in /sys/bus/virtio/devices/virtio*; do " + ' if [ "$(cat "$d/device")" = "0x0005" ]; then ' + ' cat "$d/features"; ' + " exit 0; " + " fi; " + "done; " + "exit 1" + ) + rc, stdout, stderr = vm.ssh.run(cmd) + assert rc == 0, f"balloon virtio device not found in guest sysfs: {stderr}" + return stdout.strip() + + +@pytest.mark.requires_patched_kernel +def test_fph_wait_on_ack_negotiated(uvm_plain_6_1): + """The guest negotiates bit 6 (WAIT_ON_ACK) when FPH is enabled.""" + vm = uvm_plain_6_1 + vm.spawn() + vm.basic_config(vcpu_count=1, mem_size_mib=256) + vm.add_net_iface() + vm.api.balloon.put( + amount_mib=0, + deflate_on_oom=False, + free_page_hinting=True, + ) + vm.start() + + features = _read_balloon_features(vm) + + # Format: LSB-first '0'/'1' string. + assert features[VIRTIO_BALLOON_F_FREE_PAGE_HINT] == "1", ( + f"FREE_PAGE_HINT (bit 3) not negotiated; features={features!r}" + ) + assert features[VIRTIO_BALLOON_F_HINT_WAIT_ON_ACK] == "1", ( + f"HINT_WAIT_ON_ACK (bit 6) not negotiated; features={features!r}. " + "The guest kernel likely lacks the wait-on-ACK patch — did you " + "forget to replace the 6.1 artifact vmlinux with a patched build? " + "See the module docstring." + ) diff --git a/tests/pytest.ini b/tests/pytest.ini index 5656c8eee4d..8abceaf5c17 100644 --- a/tests/pytest.ini +++ b/tests/pytest.ini @@ -5,12 +5,13 @@ addopts = -vv --durations=10 --showlocals - -m 'not nonci and not no_block_pr' + -m 'not nonci and not no_block_pr and not requires_patched_kernel' --json-report --json-report-file=../test_results/test-report.json markers = no_block_pr: tests whose failure does not block PR merging. nonci: mark test as nonci. + requires_patched_kernel: tests that require a kernel binary with out-of-tree patches; never run by CI, run manually with -m requires_patched_kernel. ; Overwrite the default norecursedirs, which includes 'build'. norecursedirs = .* From f9009a9ba217eb60be878615b75233a1c9560476 Mon Sep 17 00:00:00 2001 From: Nikita Kalyazin Date: Fri, 8 May 2026 16:10:35 +0100 Subject: [PATCH 3/3] docs(balloon): document WAIT_ON_ACK feature Add a subsection under free_page_hinting describing the behaviour of VIRTIO_BALLOON_F_HINT_WAIT_ON_ACK: always advertised alongside FPH, self-cleared by guests without the supporting kernel patch, no separate config knob, and a note on the per-buffer round-trip cost on supported guests. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Nikita Kalyazin --- docs/ballooning.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/ballooning.md b/docs/ballooning.md index 0928646f0a8..6f7cf5daf69 100644 --- a/docs/ballooning.md +++ b/docs/ballooning.md @@ -465,6 +465,26 @@ your scenario. > > This will prevent ranges which have been reclaimed from being freed. +#### `VIRTIO_BALLOON_F_HINT_WAIT_ON_ACK` + +Whenever `free_page_hinting` is enabled, Firecracker also advertises +`VIRTIO_BALLOON_F_HINT_WAIT_ON_ACK` (bit 6). When negotiated, the guest +driver waits for the device to signal-used each hint buffer before +pushing the corresponding page onto its internal free list — closing +the data-loss race described in the warning above without any host-side +protocol change. + +The bit only takes effect on guests whose kernel carries the supporting +patch (Jack Thomson's `virtio_balloon: Support wait on ACK for hinting`, +not yet upstream as of this writing). On unsupported guests the driver +self-clears the bit during `validate`, so the advertise is ignored and +hinting falls back to the unsynchronised behaviour. There is no separate +configuration knob — opting into `free_page_hinting` is sufficient. + +Note that the per-buffer round trip introduces extra wait time per hint +cycle on supported guests; the safety/perf trade-off is intentional and +documented at the kernel-patch level. + ## Balloon Caveats - Firecracker has no control over the speed of inflation or deflation; this is