From 44f95ae0ac748939846070f6015aec1ac28ca5c4 Mon Sep 17 00:00:00 2001 From: Francesco Lavra Date: Mon, 6 Jul 2026 11:50:14 +0200 Subject: [PATCH 1/5] xen: Embed struct vector inside struct xen_plaform_info This avoids a separate memory allocation for the vector struct. As part of this change, add the vector_init() and vector_deinit() generic functions to the runtime library. --- src/runtime/vector.c | 15 +++++++++++++++ src/runtime/vector.h | 5 ++++- src/xen/xen.c | 19 +++++++++++-------- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/src/runtime/vector.c b/src/runtime/vector.c index 6f98221d9..3e992361f 100644 --- a/src/runtime/vector.c +++ b/src/runtime/vector.c @@ -11,3 +11,18 @@ void init_vectors(heap h, heap init) { vheap = h; } + +boolean vector_init(vector v, heap h, int len) +{ + bytes s = len * sizeof(void *); + void *contents = allocate(h, s); + if (contents == INVALID_ADDRESS) + return false; + init_buffer(v, s, false, h, contents); + return true; +} + +void vector_deinit(vector v) +{ + deallocate(v->h, v->contents, v->length); +} diff --git a/src/runtime/vector.h b/src/runtime/vector.h index ab15f9ab7..e252315ec 100644 --- a/src/runtime/vector.h +++ b/src/runtime/vector.h @@ -1,4 +1,4 @@ -typedef buffer vector; +#define vector buffer static inline void *vector_get(vector v, int offset) { @@ -58,6 +58,9 @@ static inline int vector_delete_range(vector v, int start, int end) return (end_offset - start_offset) / sizeof(void *); } +boolean vector_init(vector v, heap h, int len); +void vector_deinit(vector v); + static inline vector allocate_vector(heap h, int length) { return allocate_buffer(h, length * sizeof (void *)); diff --git a/src/xen/xen.c b/src/xen/xen.c index 3d2e9f37b..d3fd3e9d7 100644 --- a/src/xen/xen.c +++ b/src/xen/xen.c @@ -47,7 +47,8 @@ typedef struct xen_platform_info { u64 xenstore_paddr; evtchn_port_t xenstore_evtchn; struct spinlock xenstore_lock; - vector evtchn_handlers; + + struct vector evtchn_handlers; closure_struct(xenstore_watch_handler, watch_handler); closure_struct(thunk, scan_service); @@ -120,7 +121,7 @@ closure_function(0, 0, void, xen_interrupt) bitmap_word_foreach_set(l2_pending, bit2, i2, l2_offset) { (void)i2; xenint_debug(" int %d pending", i2); - thunk handler = vector_get(xen_info.evtchn_handlers, i2); + thunk handler = vector_get(&xen_info.evtchn_handlers, i2); if (handler) { xenint_debug(" evtchn %d: applying handler %p", i2, handler); apply(handler); @@ -138,7 +139,7 @@ closure_function(0, 0, void, xen_interrupt) void xen_register_evtchn_handler(evtchn_port_t evtchn, thunk handler) { - assert(vector_set(xen_info.evtchn_handlers, evtchn, handler)); + assert(vector_set(&xen_info.evtchn_handlers, evtchn, handler)); } int xen_unmask_evtchn(evtchn_port_t evtchn) @@ -152,7 +153,7 @@ int xen_unmask_evtchn(evtchn_port_t evtchn) int xen_close_evtchn(evtchn_port_t evtchn) { - vector_set(xen_info.evtchn_handlers, evtchn, 0); + vector_set(&xen_info.evtchn_handlers, evtchn, 0); evtchn_op_t eop; eop.cmd = EVTCHNOP_close; eop.u.close.port = evtchn; @@ -539,11 +540,11 @@ boolean xen_detect(kernel_heaps kh) goto out_unregister_irq; } - xen_info.evtchn_handlers = allocate_vector(xen_info.h, 1); - assert(xen_info.evtchn_handlers != INVALID_ADDRESS); + if (!vector_init(&xen_info->evtchn_handlers, h, 1)) + goto out_unregister_irq; if (xen_setup_vcpu(0, shared_info_phys) < 0) - goto out_unregister_irq; + goto out_deinit_evtchn_handlers; register_platform_clock_timer(closure_func(xen_info.h, clock_timer, xen_runloop_timer), closure(xen_info.h, xen_per_cpu_init, shared_info_phys)); @@ -560,7 +561,7 @@ boolean xen_detect(kernel_heaps kh) if (!xen_grant_init(kh)) { msg_err("xen: failed to set up grant tables"); - goto out_unregister_irq; + goto out_deinit_evtchn_handlers; } init_closure(&xen_info.shutdown_handler, xen_shutdown_handler, allocate_buffer(xen_info.h, 16)); @@ -574,6 +575,8 @@ boolean xen_detect(kernel_heaps kh) list_init(&xen_info.driver_list); xen_info.initialized = true; return true; + out_deinit_evtchn_handlers: + vector_deinit(&xen_info->evtchn_handlers); out_unregister_irq: unregister_interrupt(irq); out_dealloc_shared_page: From 32a5c5a011f26e52b624f10f94b0f65df86b6920 Mon Sep 17 00:00:00 2001 From: Francesco Lavra Date: Mon, 6 Jul 2026 12:10:01 +0200 Subject: [PATCH 2/5] xen: Remove unneeded fields from struct xen_platform_info Remove fields that are only used during initialization. This will allow avoiding early allocation of the struct xen_platform_info when the static structure is replaced with a dynamically allocated one in a future commit. --- src/xen/xen.c | 39 ++++++++++++--------------------------- 1 file changed, 12 insertions(+), 27 deletions(-) diff --git a/src/xen/xen.c b/src/xen/xen.c index d3fd3e9d7..2938bdcb6 100644 --- a/src/xen/xen.c +++ b/src/xen/xen.c @@ -32,13 +32,6 @@ status xenstore_watch(buffer path, xenstore_watch_handler handler, boolean watch typedef struct xen_platform_info { heap h; /* general heap for internal use */ - /* version and features */ - u16 xen_major; - u16 xen_minor; - u32 last_leaf; - u32 msr_base; - u32 features; - /* event channel interface / PV interrupts */ volatile struct shared_info *shared_info; @@ -85,10 +78,6 @@ struct xen_platform_info xen_info; #define xenstore_lock() u64 _irqflags = spin_lock_irq(&xen_info.xenstore_lock) #define xenstore_unlock() spin_unlock_irq(&xen_info.xenstore_lock, _irqflags) -boolean xen_feature_supported(int feature) -{ - return (xen_info.features & U64_FROM_BIT(feature)) != 0; -} boolean xen_detected(void) { @@ -162,7 +151,7 @@ int xen_close_evtchn(evtchn_port_t evtchn) #define GTAB_RESERVED_ENTRIES 8 -static boolean xen_grant_init(kernel_heaps kh) +static boolean xen_grant_init(kernel_heaps kh, u32 features) { struct gtab *gt = &xen_info.gtab; struct gnttab_query_size qs; @@ -198,7 +187,7 @@ static boolean xen_grant_init(kernel_heaps kh) /* We have this feature on the platforms we care about now, but we can add support for the other case if need be. */ - if (!xen_feature_supported(XENFEAT_auto_translated_physmap)) { + if (!(features & U32_FROM_BIT(XENFEAT_auto_translated_physmap))) { msg_err("%s error: auto translated physmap feature required", func_ss); goto fail_dealloc_heap; } @@ -422,20 +411,16 @@ boolean xen_detect(kernel_heaps kh) return false; } - xen_info.last_leaf = v[0]; - cpuid(XEN_CPUID_LEAF(1), 0, v); - xen_info.xen_major = v[0] >> 16; - xen_info.xen_minor = v[0] & MASK(16); - xen_debug("xen version %d.%d detected", xen_info.xen_major, xen_info.xen_minor); + xen_debug("xen version %d.%d detected", v[0] >> 16, v[0] & MASK(16)); cpuid(XEN_CPUID_LEAF(2), 0, v); if (v[0] != 1) { msg_err("xen reporting %d hypercall pages; not supported", v[0]); return false; } - xen_info.msr_base = v[1]; - xen_debug("msr base 0x%x, features 1 0x%x, features 2 0x%x", xen_info.msr_base, v[2], v[3]); + u32 msr_base = v[1]; + xen_debug("msr base 0x%x, features 1 0x%x, features 2 0x%x", msr_base, v[2], v[3]); #if 0 cpuid(XEN_CPUID_LEAF(3), 0, v); @@ -453,7 +438,7 @@ boolean xen_detect(kernel_heaps kh) /* install hypercall page */ u64 hp_phys = physical_from_virtual(&hypercall_page); xen_debug("hypercall_page: v 0x%lx, p 0x%lx", u64_from_pointer(&hypercall_page), hp_phys); - write_msr(xen_info.msr_base, hp_phys); + write_msr(msr_base, hp_phys); /* get xen features */ build_assert(XENFEAT_NR_SUBMAPS == 1); @@ -464,10 +449,10 @@ boolean xen_detect(kernel_heaps kh) msg_err("xen: failed to get features map (rv %d)", rv); return false; } - xen_info.features = xfi.submap; - xen_debug("reported features map 0x%x", xen_info.features); + u32 features = xfi.submap; + xen_debug("reported features map 0x%x", features); - if (!xen_feature_supported(XENFEAT_hvm_safe_pvclock)) { + if (!(features & U32_FROM_BIT(XENFEAT_hvm_safe_pvclock))) { msg_err("xen failed to init; XENFEAT_hvm_safe_pvclock required"); return false; } @@ -520,9 +505,9 @@ boolean xen_detect(kernel_heaps kh) } xen_debug("shared info page: %p, phys 0x%lx", shared_info, shared_info_phys); - if (!xen_feature_supported(XENFEAT_hvm_callback_vector)) { + if (!(features & U32_FROM_BIT(XENFEAT_hvm_callback_vector))) { msg_err("xen setup failed: HVM callback vector must be supported (features mask 0x%x)", - xen_info.features); + features); goto out_dealloc_shared_page; } @@ -559,7 +544,7 @@ boolean xen_detect(kernel_heaps kh) closure_func(xen_info.h, thunk, xenstore_evtchn_handler)); assert(xen_unmask_evtchn(xen_info.xenstore_evtchn) == 0); - if (!xen_grant_init(kh)) { + if (!xen_grant_init(kh, features)) { msg_err("xen: failed to set up grant tables"); goto out_deinit_evtchn_handlers; } From 8ecff5ad9854d4ec1896b5887dd1980077c53f44 Mon Sep 17 00:00:00 2001 From: Francesco Lavra Date: Mon, 6 Jul 2026 12:31:17 +0200 Subject: [PATCH 3/5] xen: Replace static struct xen_platform_info with dynamically allocated one This change, which decreases memory requirements on non-Xen platforms, is being made in preparation for a commit that will add a new (somewhat large) field (`vcpu_evtchn_enable`) to this struct. --- src/xen/xen.c | 152 +++++++++++++++++++++++++------------------------- 1 file changed, 77 insertions(+), 75 deletions(-) diff --git a/src/xen/xen.c b/src/xen/xen.c index 2938bdcb6..39d62f187 100644 --- a/src/xen/xen.c +++ b/src/xen/xen.c @@ -62,9 +62,6 @@ typedef struct xen_platform_info { tuple device_tree; struct list xenbus_list; struct list driver_list; - - /* XXX could make generalized status */ - boolean initialized; } *xen_platform_info; typedef struct xen_driver { @@ -73,15 +70,14 @@ typedef struct xen_driver { xen_device_probe probe; } *xen_driver; -struct xen_platform_info xen_info; - -#define xenstore_lock() u64 _irqflags = spin_lock_irq(&xen_info.xenstore_lock) -#define xenstore_unlock() spin_unlock_irq(&xen_info.xenstore_lock, _irqflags) +static BSS_RO_AFTER_INIT xen_platform_info xen_info; +#define xenstore_lock() u64 _irqflags = spin_lock_irq(&xen_info->xenstore_lock) +#define xenstore_unlock() spin_unlock_irq(&xen_info->xenstore_lock, _irqflags) boolean xen_detected(void) { - return xen_info.initialized; + return !!xen_info; } extern u64 hypercall_page; @@ -89,8 +85,8 @@ extern u64 hypercall_page; closure_function(0, 0, void, xen_interrupt) { int vcpu = current_cpu()->id; - volatile struct shared_info *si = xen_info.shared_info; - volatile struct vcpu_info *vci = &xen_info.shared_info->vcpu_info[vcpu]; + volatile struct shared_info *si = xen_info->shared_info; + volatile struct vcpu_info *vci = &si->vcpu_info[vcpu]; xenint_debug("xen_interrupt enter"); assert(vci->evtchn_upcall_pending); @@ -110,7 +106,7 @@ closure_function(0, 0, void, xen_interrupt) bitmap_word_foreach_set(l2_pending, bit2, i2, l2_offset) { (void)i2; xenint_debug(" int %d pending", i2); - thunk handler = vector_get(&xen_info.evtchn_handlers, i2); + thunk handler = vector_get(&xen_info->evtchn_handlers, i2); if (handler) { xenint_debug(" evtchn %d: applying handler %p", i2, handler); apply(handler); @@ -128,7 +124,7 @@ closure_function(0, 0, void, xen_interrupt) void xen_register_evtchn_handler(evtchn_port_t evtchn, thunk handler) { - assert(vector_set(&xen_info.evtchn_handlers, evtchn, handler)); + assert(vector_set(&xen_info->evtchn_handlers, evtchn, handler)); } int xen_unmask_evtchn(evtchn_port_t evtchn) @@ -142,7 +138,7 @@ int xen_unmask_evtchn(evtchn_port_t evtchn) int xen_close_evtchn(evtchn_port_t evtchn) { - vector_set(&xen_info.evtchn_handlers, evtchn, 0); + vector_set(&xen_info->evtchn_handlers, evtchn, 0); evtchn_op_t eop; eop.cmd = EVTCHNOP_close; eop.u.close.port = evtchn; @@ -153,7 +149,7 @@ int xen_close_evtchn(evtchn_port_t evtchn) static boolean xen_grant_init(kernel_heaps kh, u32 features) { - struct gtab *gt = &xen_info.gtab; + struct gtab *gt = &xen_info->gtab; struct gnttab_query_size qs; qs.dom = DOMID_SELF; int rv = HYPERVISOR_grant_table_op(GNTTABOP_query_size, &qs, 1); @@ -222,7 +218,7 @@ static boolean xen_grant_init(kernel_heaps kh, u32 features) /* optimization: create a gntref free list / fifo of sorts */ grant_ref_t xen_grant_page_access(u16 domid, u64 phys, boolean readonly) { - struct gtab *gt = &xen_info.gtab; + struct gtab *gt = &xen_info->gtab; u64 ref = allocate_u64(gt->entry_heap, 1); if (ref == INVALID_PHYSICAL) return 0; @@ -236,9 +232,9 @@ grant_ref_t xen_grant_page_access(u16 domid, u64 phys, boolean readonly) void xen_revoke_page_access(grant_ref_t ref) { assert(ref > 8 && ref != -1); - xen_info.gtab.table[ref].flags = 0; + xen_info->gtab.table[ref].flags = 0; memory_barrier(); - deallocate_u64(xen_info.gtab.entry_heap, ref, 1); + deallocate_u64(xen_info->gtab.entry_heap, ref, 1); } /* Reportedly, Xen timers can fire up to 100us early. */ @@ -314,10 +310,10 @@ closure_func_basic(thunk, void, xenstore_evtchn_handler) xenstore_watch_event(msg); } else { /* unexpected message type: discard message data */ - volatile struct xenstore_domain_interface *xsdi = xen_info.xenstore_interface; + volatile struct xenstore_domain_interface *xsdi = xen_info->xenstore_interface; xsdi->rsp_cons += msg->len; write_barrier(); - xen_notify_evtchn(xen_info.xenstore_evtchn); + xen_notify_evtchn(xen_info->xenstore_evtchn); } out: xenstore_unlock(); @@ -360,7 +356,7 @@ static int xen_setup_vcpu(int vcpu, u64 shared_info_phys) evtchn_port_t timer_evtchn = eop.u.bind_virq.port; xen_debug("cpu %d timer event channel %d", vcpu, timer_evtchn); xen_register_evtchn_handler(timer_evtchn, - closure_func(xen_info.h, thunk, xen_runloop_timer_handler)); + closure_func(xen_info->h, thunk, xen_runloop_timer_handler)); assert(xen_unmask_evtchn(timer_evtchn) == 0); return 0; } @@ -394,14 +390,13 @@ define_closure_function(1, 0, void, xen_shutdown_handler, closure_func_basic(xenstore_watch_handler, void, xen_shutdown_watcher, sstring path) { - async_apply_bh((thunk)&xen_info.shutdown_handler); + async_apply_bh((thunk)&xen_info->shutdown_handler); } boolean xen_detect(kernel_heaps kh) { u32 v[4]; - xen_info.initialized = false; - xen_info.h = heap_locked(kh); + heap h = heap_locked(kh); xen_debug("checking for xen cpuid leaves"); cpuid(XEN_CPUID_FIRST_LEAF, 0, v); if (!(v[1] == XEN_CPUID_SIGNATURE_EBX && @@ -457,6 +452,9 @@ boolean xen_detect(kernel_heaps kh) return false; } + xen_info = mem_alloc(h, sizeof(*xen_info), MEM_ZERO | MEM_NOWAIT | MEM_NOFAIL); + xen_info->h = h; + /* get store page, map it, and retrieve event channel */ xen_debug("retrieving xenstore page"); struct xen_hvm_param xen_hvm_param; @@ -465,16 +463,16 @@ boolean xen_detect(kernel_heaps kh) rv = HYPERVISOR_hvm_op(HVMOP_get_param, &xen_hvm_param); if (rv < 0) { msg_err("xen: failed to get xenstore page address (rv %d)", rv); - return false; + goto out_dealloc_xen_info; } - xen_info.xenstore_paddr = xen_hvm_param.value << PAGELOG; + xen_info->xenstore_paddr = xen_hvm_param.value << PAGELOG; - xen_debug("xenstore page at phys 0x%lx; allocating virtual page and mapping", xen_info.xenstore_paddr); - xen_info.xenstore_interface = mem_alloc((heap)heap_virtual_page(kh), PAGESIZE, + xen_debug("xenstore page at phys 0x%lx; allocating virtual page", xen_info->xenstore_paddr); + xen_info->xenstore_interface = mem_alloc((heap)heap_virtual_page(kh), PAGESIZE, MEM_NOWAIT | MEM_NOFAIL); - map(u64_from_pointer(xen_info.xenstore_interface), xen_info.xenstore_paddr, PAGESIZE, + map(u64_from_pointer(xen_info->xenstore_interface), xen_info->xenstore_paddr, PAGESIZE, pageflags_writable(pageflags_memory())); - xen_debug("xenstore page mapped at %p", xen_info.xenstore_interface); + xen_debug("xenstore page mapped at %p", xen_info->xenstore_interface); xen_debug("retrieving store event channel"); xen_hvm_param.domid = DOMID_SELF; @@ -482,17 +480,17 @@ boolean xen_detect(kernel_heaps kh) rv = HYPERVISOR_hvm_op(HVMOP_get_param, &xen_hvm_param); if (rv < 0) { msg_err("xen: failed to get xenstore event channel (rv %d)", rv); - return false; + goto out_dealloc_xenstore; } - xen_info.xenstore_evtchn = xen_hvm_param.value; - xen_debug("event channel %ld, allocating and mapping shared info page", xen_info.xenstore_evtchn); + xen_info->xenstore_evtchn = xen_hvm_param.value; + xen_debug("event channel %ld, allocating and mapping shared info page", xen_info->xenstore_evtchn); heap shared_info_heap = (heap)heap_linear_backed(kh); void *shared_info = mem_alloc(shared_info_heap, PAGESIZE, MEM_ZERO | MEM_NOWAIT | MEM_NOFAIL); u64 shared_info_phys = physical_from_virtual(shared_info); assert(shared_info_phys != INVALID_PHYSICAL); - xen_info.shared_info = shared_info; + xen_info->shared_info = shared_info; xen_add_to_physmap_t xatp; xatp.domid = DOMID_SELF; xatp.space = XENMAPSPACE_shared_info; @@ -514,7 +512,7 @@ boolean xen_detect(kernel_heaps kh) /* set up interrupt handling path */ int irq = allocate_interrupt(); xen_debug("interrupt vector %d; registering", irq); - register_interrupt(irq, closure(xen_info.h, xen_interrupt), ss("xen")); + register_interrupt(irq, closure(h, xen_interrupt), ss("xen")); xen_hvm_param.domid = DOMID_SELF; xen_hvm_param.index = HVM_PARAM_CALLBACK_IRQ; @@ -531,42 +529,46 @@ boolean xen_detect(kernel_heaps kh) if (xen_setup_vcpu(0, shared_info_phys) < 0) goto out_deinit_evtchn_handlers; - register_platform_clock_timer(closure_func(xen_info.h, clock_timer, xen_runloop_timer), - closure(xen_info.h, xen_per_cpu_init, shared_info_phys)); + register_platform_clock_timer(closure_func(h, clock_timer, xen_runloop_timer), + closure(h, xen_per_cpu_init, shared_info_phys)); /* register pvclock (feature verified above) */ - init_pvclock(xen_info.h, (struct pvclock_vcpu_time_info *)&xen_info.shared_info->vcpu_info[0].time, - (struct pvclock_wall_clock *)&xen_info.shared_info->wc_version); + init_pvclock(h, (struct pvclock_vcpu_time_info *)&xen_info->shared_info->vcpu_info[0].time, + (struct pvclock_wall_clock *)&xen_info->shared_info->wc_version); xen_debug("unmasking xenstore event channel"); - spin_lock_init(&xen_info.xenstore_lock); - xen_register_evtchn_handler(xen_info.xenstore_evtchn, - closure_func(xen_info.h, thunk, xenstore_evtchn_handler)); - assert(xen_unmask_evtchn(xen_info.xenstore_evtchn) == 0); + spin_lock_init(&xen_info->xenstore_lock); + xen_register_evtchn_handler(xen_info->xenstore_evtchn, + closure_func(h, thunk, xenstore_evtchn_handler)); + assert(xen_unmask_evtchn(xen_info->xenstore_evtchn) == 0); if (!xen_grant_init(kh, features)) { msg_err("xen: failed to set up grant tables"); goto out_deinit_evtchn_handlers; } - init_closure(&xen_info.shutdown_handler, xen_shutdown_handler, allocate_buffer(xen_info.h, 16)); + init_closure(&xen_info->shutdown_handler, xen_shutdown_handler, allocate_buffer(h, 16)); if (!is_ok(xenstore_watch(alloca_wrap_cstring("control/shutdown"), - init_closure_func(&xen_info.shutdown_watcher, xenstore_watch_handler, + init_closure_func(&xen_info->shutdown_watcher, xenstore_watch_handler, xen_shutdown_watcher), true))) msg_err("xen: failed to register shutdown handler"); xen_debug("xen initialization complete"); - list_init(&xen_info.driver_list); - xen_info.initialized = true; + list_init(&xen_info->driver_list); return true; out_deinit_evtchn_handlers: vector_deinit(&xen_info->evtchn_handlers); out_unregister_irq: unregister_interrupt(irq); out_dealloc_shared_page: - deallocate(shared_info_heap, xen_info.shared_info, PAGESIZE); - xen_info.shared_info = 0; + deallocate(shared_info_heap, xen_info->shared_info, PAGESIZE); + out_dealloc_xenstore: + unmap(u64_from_pointer(xen_info->xenstore_interface), PAGESIZE); + deallocate((heap)heap_virtual_page(kh), xen_info->xenstore_interface, PAGESIZE); + out_dealloc_xen_info: + deallocate(h, xen_info, sizeof(*xen_info)); + xen_info = 0; return false; } @@ -597,7 +599,7 @@ int xen_notify_evtchn(evtchn_port_t evtchn) /* returns bytes written; doesn't block */ static s64 xenstore_write_internal(const void * data, s64 length) { - volatile struct xenstore_domain_interface *xsdi = xen_info.xenstore_interface; + volatile struct xenstore_domain_interface *xsdi = xen_info->xenstore_interface; if (length == 0) return 0; assert(length > 0); @@ -620,7 +622,7 @@ static s64 xenstore_write_internal(const void * data, s64 length) write_barrier(); xsdi->req_prod += nwrite; write_barrier(); - int rv = xen_notify_evtchn(xen_info.xenstore_evtchn); + int rv = xen_notify_evtchn(xen_info->xenstore_evtchn); if (rv < 0) { result = rv; break; @@ -633,7 +635,7 @@ static s64 xenstore_write_internal(const void * data, s64 length) static s64 xenstore_read_internal(buffer b, s64 length) { - volatile struct xenstore_domain_interface *xsdi = xen_info.xenstore_interface; + volatile struct xenstore_domain_interface *xsdi = xen_info->xenstore_interface; if (length == 0) return 0; assert(length > 0); @@ -656,7 +658,7 @@ static s64 xenstore_read_internal(buffer b, s64 length) write_barrier(); xsdi->rsp_cons += nread; write_barrier(); - int rv = xen_notify_evtchn(xen_info.xenstore_evtchn); + int rv = xen_notify_evtchn(xen_info->xenstore_evtchn); if (rv < 0) { result = rv; break; @@ -703,7 +705,7 @@ status xenstore_sync_request(u32 tx_id, enum xsd_sockmsg_type type, buffer reque { status s = STATUS_OK; struct xsd_sockmsg msg; - buffer rbuf = allocate_buffer(xen_info.h, PAGESIZE); // XXX + buffer rbuf = allocate_buffer(xen_info->h, PAGESIZE); assert(rbuf != INVALID_ADDRESS); u32 len = request ? buffer_length(request) : 1; void * data = request ? buffer_ref(request, 0) : ""; @@ -758,7 +760,7 @@ status xenstore_sync_request(u32 tx_id, enum xsd_sockmsg_type type, buffer reque status xenstore_transaction_start(u32 *tx_id) { - buffer response = allocate_buffer(xen_info.h, 16); + buffer response = allocate_buffer(xen_info->h, 16); status s = xenstore_sync_request(0, XS_TRANSACTION_START, 0, response); if (!is_ok(s)) goto out; @@ -776,7 +778,7 @@ status xenstore_transaction_start(u32 *tx_id) status xenstore_transaction_end(u32 tx_id, boolean abort) { buffer request = alloca_wrap_buffer(abort ? "F" : "T", 2); - buffer response = allocate_buffer(xen_info.h, 8); /* for error capture */ + buffer response = allocate_buffer(xen_info->h, 8); /* for error capture */ status s = xenstore_sync_request(tx_id, XS_TRANSACTION_END, request, response); deallocate_buffer(response); return s; @@ -784,7 +786,7 @@ status xenstore_transaction_end(u32 tx_id, boolean abort) status xenstore_sync_printf(u32 tx_id, buffer path, sstring node, sstring format, ...) { - buffer request = allocate_buffer(xen_info.h, PAGESIZE); + buffer request = allocate_buffer(xen_info->h, PAGESIZE); assert(push_buffer(request, path)); push_u8(request, '/'); assert(buffer_write_sstring(request, node)); @@ -794,7 +796,7 @@ status xenstore_sync_printf(u32 tx_id, buffer path, sstring node, sstring format vbprintf(request, format, &a); xenstore_debug("%s: request: \"%b\"", func_ss, request); - buffer response = allocate_buffer(xen_info.h, 8); /* for error capture */ + buffer response = allocate_buffer(xen_info->h, 8); /* for error capture */ status s = xenstore_sync_request(tx_id, XS_WRITE, request, response); deallocate_buffer(request); deallocate_buffer(response); @@ -803,7 +805,7 @@ status xenstore_sync_printf(u32 tx_id, buffer path, sstring node, sstring format status xenstore_read_u64(u32 tx_id, buffer path, sstring node, u64 *result) { - buffer response = allocate_buffer(xen_info.h, 16); + buffer response = allocate_buffer(xen_info->h, 16); status s = xenstore_read_string(tx_id, path, node, response); if (!is_ok(s)) goto out; @@ -820,7 +822,7 @@ status xenstore_read_u64(u32 tx_id, buffer path, sstring node, u64 *result) status xenstore_read_string(u32 tx_id, buffer path, sstring node, buffer response) { - buffer request = allocate_buffer(xen_info.h, 64); + buffer request = allocate_buffer(xen_info->h, 64); assert(push_buffer(request, path)); push_u8(request, '/'); assert(buffer_write_sstring(request, node)); @@ -837,7 +839,7 @@ status xenstore_read_string(u32 tx_id, buffer path, sstring node, buffer respons status xenstore_watch(buffer path, xenstore_watch_handler handler, boolean watch) { buffer request = little_stack_buffer(buffer_length(path) + 18); - buffer response = allocate_buffer(xen_info.h, 8); + buffer response = allocate_buffer(xen_info->h, 8); if (response == INVALID_ADDRESS) return timm("result", "failed to allocate memory"); push_buffer(request, path); @@ -1011,7 +1013,7 @@ closure_function(4, 2, boolean, xen_probe_id_each, return false; } xen_debug("driver match, id %d, value %v", id, v); - buffer frontend = allocate_buffer(xen_info.h, buffer_length(bound(name)) + 10); + buffer frontend = allocate_buffer(xen_info->h, buffer_length(bound(name)) + 10); bprintf(frontend, "device/%b/%d", bound(name), id); boolean bound = apply(bound(xd)->probe, (int)id, frontend, v); if (bound) @@ -1028,7 +1030,7 @@ closure_function(1, 2, boolean, xen_probe_devices_each, assert(is_symbol(k)); if (!is_tuple(v)) return true; - list_foreach(&xen_info.driver_list, l) { + list_foreach(&xen_info->driver_list, l) { xen_driver xd = struct_from_list(l, xen_driver, l); /* XXX must be a cleaner way to compare symbols? */ string name = symbol_string(k); @@ -1043,13 +1045,13 @@ closure_function(1, 2, boolean, xen_probe_devices_each, static status xen_scan(void) { - status s = traverse_directory(xen_info.h, "device", &xen_info.device_tree); + status s = traverse_directory(xen_info->h, "device", &xen_info->device_tree); if (!is_ok(s)) return s; - if (!xen_info.device_tree) + if (!xen_info->device_tree) return timm("result", "failed to parse directory"); - xen_debug("scan result: %v", xen_info.device_tree); - iterate(xen_info.device_tree, stack_closure(xen_probe_devices_each, &s)); + xen_debug("scan result: %v", xen_info->device_tree); + iterate(xen_info->device_tree, stack_closure(xen_probe_devices_each, &s)); return s; } @@ -1066,7 +1068,7 @@ closure_func_basic(xenstore_watch_handler, void, xen_watch_handler, } if (depth == 0) return; - async_apply_bh((thunk)&xen_info.scan_service); + async_apply_bh((thunk)&xen_info->scan_service); } closure_func_basic(thunk, void, xen_scan_service) @@ -1074,13 +1076,13 @@ closure_func_basic(thunk, void, xen_scan_service) xenstore_debug("%s", func_ss); /* Avoid concurrent scans. */ - if (atomic_test_and_set_bit(&xen_info.scanning, 0)) { - async_apply((thunk)&xen_info.scan_service); + if (atomic_test_and_set_bit(&xen_info->scanning, 0)) { + async_apply((thunk)&xen_info->scan_service); return; } status s = xen_scan(); - atomic_clear_bit(&xen_info.scanning, 0); + atomic_clear_bit(&xen_info->scanning, 0); if (!is_ok(s)) { msg_warn("xen: cannot scan devices: %v", s); timm_dealloc(s); @@ -1090,12 +1092,12 @@ closure_func_basic(thunk, void, xen_scan_service) status xen_probe_devices(void) { xen_debug("probing xen device tree from xenstored"); - assert(xen_info.device_tree == 0); + assert(xen_info->device_tree == 0); status s = xen_scan(); if (is_ok(s)) { - init_closure_func(&xen_info.scan_service, thunk, xen_scan_service); + init_closure_func(&xen_info->scan_service, thunk, xen_scan_service); s = xenstore_watch(alloca_wrap_cstring("device"), - init_closure_func(&xen_info.watch_handler, xenstore_watch_handler, + init_closure_func(&xen_info->watch_handler, xenstore_watch_handler, xen_watch_handler), true); if (!is_ok(s)) { @@ -1109,8 +1111,8 @@ status xen_probe_devices(void) void register_xen_driver(sstring name, xen_device_probe probe) { - xen_driver xd = mem_alloc(xen_info.h, sizeof(struct xen_driver), MEM_NOWAIT | MEM_NOFAIL); + xen_driver xd = mem_alloc(xen_info->h, sizeof(struct xen_driver), MEM_NOWAIT | MEM_NOFAIL); xd->name = name; xd->probe = probe; - list_insert_before(&xen_info.driver_list, &xd->l); + list_insert_before(&xen_info->driver_list, &xd->l); } From 862ab9c8b430f0624753e583ddb3214d8c0bc598 Mon Sep 17 00:00:00 2001 From: Francesco Lavra Date: Mon, 6 Jul 2026 12:36:46 +0200 Subject: [PATCH 4/5] xen: Fix dispatching of event handlers on multi-CPU platforms The Xen interrupt handler is invoking the handlers of all pending events whose port numbers belong to the same `evtchn_pending` array element as the element for which there is a pending event for the current vCPU. This means that the handler for an event bound to a given vCPU can potentially be executed on a different vCPU. This creates issues in event handlers (such as `xennet_event_handler()` that assume they are always run sequentially and cannot run in parallel on different vCPUs. Fix the Xen interrupt handler so that it only invokes handlers for events bound to the current vCPU. The issue with seemingly spurious interrupts mentioned in a comment in the interrupt handler function cannot be reproduced (tested on t2.nano, t2.medium and t2.2xlarge AWS instances), thus the `evtchn_mask` array is no longer used. In addition, allow device drivers to request that their event handler is bound to a specific vCPU (via the newly defined xen_bind_evtchn() function); this allows spreading the interrupt load among different vCPUs. --- src/xen/xen.c | 39 +++++++++++++++++++++++---------------- src/xen/xen_internal.h | 5 +++-- src/xen/xenblk.c | 23 ++++++++++++++++++++--- src/xen/xennet.c | 10 +++++++++- 4 files changed, 55 insertions(+), 22 deletions(-) diff --git a/src/xen/xen.c b/src/xen/xen.c index 39d62f187..9b3051ba2 100644 --- a/src/xen/xen.c +++ b/src/xen/xen.c @@ -41,6 +41,7 @@ typedef struct xen_platform_info { evtchn_port_t xenstore_evtchn; struct spinlock xenstore_lock; + u64 vcpu_evtchn_enable[XEN_LEGACY_MAX_VCPUS][sizeof(u64) * 8]; struct vector evtchn_handlers; closure_struct(xenstore_watch_handler, watch_handler); @@ -87,6 +88,7 @@ closure_function(0, 0, void, xen_interrupt) int vcpu = current_cpu()->id; volatile struct shared_info *si = xen_info->shared_info; volatile struct vcpu_info *vci = &si->vcpu_info[vcpu]; + u64 *evtchn_enable = xen_info->vcpu_evtchn_enable[vcpu]; xenint_debug("xen_interrupt enter"); assert(vci->evtchn_upcall_pending); @@ -97,24 +99,16 @@ closure_function(0, 0, void, xen_interrupt) /* this may not process in the right order, or it might not matter - care later */ bitmap_word_foreach_set(l1_pending, bit1, i1, 0) { (void)i1; - /* TODO: any per-cpu event mask would be also applied here... */ - u64 l2_pending = si->evtchn_pending[bit1] & ~si->evtchn_mask[bit1]; + u64 l2_pending = si->evtchn_pending[bit1] & evtchn_enable[bit1]; xenint_debug("pending 0x%lx, mask 0x%lx, masked 0x%lx", - si->evtchn_pending[bit1], si->evtchn_mask[bit1], l2_pending); + si->evtchn_pending[bit1], ~evtchn_enable[bit1], l2_pending); __sync_and_and_fetch(&si->evtchn_pending[bit1], ~l2_pending); u64 l2_offset = bit1 << 6; bitmap_word_foreach_set(l2_pending, bit2, i2, l2_offset) { (void)i2; xenint_debug(" int %d pending", i2); thunk handler = vector_get(&xen_info->evtchn_handlers, i2); - if (handler) { - xenint_debug(" evtchn %d: applying handler %p", i2, handler); - apply(handler); - } else { - /* XXX we have an issue with seemingly spurious interrupts at evtchn >= 2048... */ - xenint_debug(" evtchn %d: spurious interrupt", i2); - si->evtchn_mask[bit1] |= U64_FROM_BIT(bit2); - } + apply(handler); } } vci->evtchn_upcall_mask = 0; @@ -127,17 +121,30 @@ void xen_register_evtchn_handler(evtchn_port_t evtchn, thunk handler) assert(vector_set(&xen_info->evtchn_handlers, evtchn, handler)); } -int xen_unmask_evtchn(evtchn_port_t evtchn) +int xen_bind_evtchn(evtchn_port_t evtchn, int vcpu) +{ + evtchn_op_t eop; + eop.cmd = EVTCHNOP_bind_vcpu; + eop.u.bind_vcpu.port = evtchn; + eop.u.bind_vcpu.vcpu = vcpu; + return HYPERVISOR_event_channel_op(&eop); +} + +int xen_unmask_evtchn(evtchn_port_t evtchn, int vcpu) { assert(evtchn > 0 && evtchn < EVTCHN_2L_NR_CHANNELS); evtchn_op_t eop; eop.cmd = EVTCHNOP_unmask; eop.u.unmask.port = evtchn; - return HYPERVISOR_event_channel_op(&eop); + int rv = HYPERVISOR_event_channel_op(&eop); + if (rv == 0) + atomic_set_bit(&xen_info->vcpu_evtchn_enable[vcpu][evtchn >> 6], evtchn & MASK(6)); + return rv; } -int xen_close_evtchn(evtchn_port_t evtchn) +int xen_close_evtchn(evtchn_port_t evtchn, int vcpu) { + atomic_clear_bit(&xen_info->vcpu_evtchn_enable[vcpu][evtchn >> 6], evtchn & MASK(6)); vector_set(&xen_info->evtchn_handlers, evtchn, 0); evtchn_op_t eop; eop.cmd = EVTCHNOP_close; @@ -357,7 +364,7 @@ static int xen_setup_vcpu(int vcpu, u64 shared_info_phys) xen_debug("cpu %d timer event channel %d", vcpu, timer_evtchn); xen_register_evtchn_handler(timer_evtchn, closure_func(xen_info->h, thunk, xen_runloop_timer_handler)); - assert(xen_unmask_evtchn(timer_evtchn) == 0); + assert(xen_unmask_evtchn(timer_evtchn, vcpu) == 0); return 0; } @@ -540,7 +547,7 @@ boolean xen_detect(kernel_heaps kh) spin_lock_init(&xen_info->xenstore_lock); xen_register_evtchn_handler(xen_info->xenstore_evtchn, closure_func(h, thunk, xenstore_evtchn_handler)); - assert(xen_unmask_evtchn(xen_info->xenstore_evtchn) == 0); + assert(xen_unmask_evtchn(xen_info->xenstore_evtchn, 0) == 0); if (!xen_grant_init(kh, features)) { msg_err("xen: failed to set up grant tables"); diff --git a/src/xen/xen_internal.h b/src/xen/xen_internal.h index 383d6947c..d0cb934c2 100644 --- a/src/xen/xen_internal.h +++ b/src/xen/xen_internal.h @@ -38,8 +38,9 @@ typedef struct xen_dev { status xen_allocate_evtchn(domid_t other_id, evtchn_port_t *evtchn); void xen_register_evtchn_handler(evtchn_port_t evtchn, thunk handler); int xen_notify_evtchn(evtchn_port_t evtchn); -int xen_unmask_evtchn(evtchn_port_t evtchn); -int xen_close_evtchn(evtchn_port_t evtchn); +int xen_bind_evtchn(evtchn_port_t evtchn, int vcpu); +int xen_unmask_evtchn(evtchn_port_t evtchn, int vcpu); +int xen_close_evtchn(evtchn_port_t evtchn, int vcpu); grant_ref_t xen_grant_page_access(u16 domid, u64 phys, boolean readonly); void xen_revoke_page_access(grant_ref_t ref); diff --git a/src/xen/xenblk.c b/src/xen/xenblk.c index 750c209bb..c31b80753 100644 --- a/src/xen/xenblk.c +++ b/src/xen/xenblk.c @@ -39,6 +39,7 @@ struct xenblk_dev { blkif_front_ring_t ring; grant_ref_t ring_gntref; evtchn_port_t evtchn; + s32 evtchn_cpu; closure_struct(xenblk_io, read); closure_struct(xenblk_io, write); closure_struct(storage_simple_req_handler, req_handler); @@ -275,7 +276,12 @@ static void xenblk_remove(xenblk_dev xbd) } xen_driver_unbind(xbd->meta); xenbus_set_state(0, xbd->dev.frontend, XenbusStateClosed); - xen_close_evtchn(xbd->evtchn); + s32 evtchn_cpu = xbd->evtchn_cpu; + if (evtchn_cpu >= 0) + irq_put_target_cpu(evtchn_cpu); + else + evtchn_cpu = 0; + xen_close_evtchn(xbd->evtchn, evtchn_cpu); xen_revoke_page_access(xbd->ring_gntref); deallocate(xbd->contiguous, xbd->ring.sring, PAGESIZE); deallocate_vector(xbd->rreqs); @@ -336,7 +342,18 @@ closure_func_basic(thunk, void, xenblk_watch_service) timm_dealloc(s); goto remove; } - int rv = xen_unmask_evtchn(xbd->evtchn); + u32 evtchn_cpu = irq_get_target_cpu(irange(0, 0)); + int rv = xen_bind_evtchn(xbd->evtchn, evtchn_cpu); + if (rv < 0) { + msg_warn("%s: failed to bind event channel %d to vCPU %d: rv %d", func_ss, xbd->evtchn, + evtchn_cpu, rv); + irq_put_target_cpu(evtchn_cpu); + xbd->evtchn_cpu = -1; + evtchn_cpu = 0; + } else { + xbd->evtchn_cpu = evtchn_cpu; + } + rv = xen_unmask_evtchn(xbd->evtchn, evtchn_cpu); if (rv < 0) { msg_err("%s: failed to unmask event channel %d: rv %d", func_ss, xbd->evtchn, rv); goto remove; @@ -472,7 +489,7 @@ static status xenblk_enable(xenblk_dev xbd) } return STATUS_OK; out_evtchn: - xen_close_evtchn(xbd->evtchn); + xen_close_evtchn(xbd->evtchn, 0); out_revoke: xen_revoke_page_access(xbd->ring_gntref); out_dealloc: diff --git a/src/xen/xennet.c b/src/xen/xennet.c index c1174f53c..caa821411 100644 --- a/src/xen/xennet.c +++ b/src/xen/xennet.c @@ -724,7 +724,15 @@ static status xennet_enable(xennet_dev xd) /* we're kind of always up ... start rx now */ xd->rx_ring.sring->rsp_event = xd->rx_ring.rsp_cons + 1; write_barrier(); - int rv = xen_unmask_evtchn(xd->evtchn); + u32 evtchn_cpu = irq_get_target_cpu(irange(0, 0)); + int rv = xen_bind_evtchn(xd->evtchn, evtchn_cpu); + if (rv < 0) { + msg_warn("%s: failed to bind event channel %d to vCPU %d: rv %d", func_ss, xd->evtchn, + evtchn_cpu, rv); + irq_put_target_cpu(evtchn_cpu); + evtchn_cpu = 0; + } + rv = xen_unmask_evtchn(xd->evtchn, evtchn_cpu); if (rv < 0) { s = timm("result", "failed to unmask event channel %d: rv %d", xd->evtchn, rv); goto out_dealloc_rx_buffers; From 2e85d1ed92c13cffa6be61ee74db34b3acd5404c Mon Sep 17 00:00:00 2001 From: Francesco Lavra Date: Mon, 6 Jul 2026 13:00:05 +0200 Subject: [PATCH 5/5] random_early_u64(): fix compilation error with clang Initialize the `retval` variable to fix "error: variable 'retval' is uninitialized when passed as a const pointer argument here [-Werror,-Wuninitialized-const-pointer]" when compiling with clang on a Mac host. --- src/runtime/random.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/random.c b/src/runtime/random.c index 0399a77bc..df51ffe57 100644 --- a/src/runtime/random.c +++ b/src/runtime/random.c @@ -150,7 +150,7 @@ u64 random_early_u64(void) { u8 key[CHACHA20_KEYBYTES]; get_seed_complete(key, CHACHA20_KEYBYTES); - u64 retval; + u64 retval = 0; chacha_keysetup(&chacha20inst.ctx, key, CHACHA20_KEYBYTES * 8); chacha_ivsetup(&chacha20inst.ctx, (u8 *)&retval, (u8 *)&retval); chacha_encrypt_bytes(&chacha20inst.ctx, chacha20inst.m_buffer, (u8 *)&retval, sizeof(retval));