diff --git a/src/runtime/random.c b/src/runtime/random.c index 0399a77bc..df51ffe57 100644 --- a/src/runtime/random.c +++ b/src/runtime/random.c @@ -150,7 +150,7 @@ u64 random_early_u64(void) { u8 key[CHACHA20_KEYBYTES]; get_seed_complete(key, CHACHA20_KEYBYTES); - u64 retval; + u64 retval = 0; chacha_keysetup(&chacha20inst.ctx, key, CHACHA20_KEYBYTES * 8); chacha_ivsetup(&chacha20inst.ctx, (u8 *)&retval, (u8 *)&retval); chacha_encrypt_bytes(&chacha20inst.ctx, chacha20inst.m_buffer, (u8 *)&retval, sizeof(retval)); diff --git a/src/runtime/vector.c b/src/runtime/vector.c index 6f98221d9..3e992361f 100644 --- a/src/runtime/vector.c +++ b/src/runtime/vector.c @@ -11,3 +11,18 @@ void init_vectors(heap h, heap init) { vheap = h; } + +boolean vector_init(vector v, heap h, int len) +{ + bytes s = len * sizeof(void *); + void *contents = allocate(h, s); + if (contents == INVALID_ADDRESS) + return false; + init_buffer(v, s, false, h, contents); + return true; +} + +void vector_deinit(vector v) +{ + deallocate(v->h, v->contents, v->length); +} diff --git a/src/runtime/vector.h b/src/runtime/vector.h index ab15f9ab7..e252315ec 100644 --- a/src/runtime/vector.h +++ b/src/runtime/vector.h @@ -1,4 +1,4 @@ -typedef buffer vector; +#define vector buffer static inline void *vector_get(vector v, int offset) { @@ -58,6 +58,9 @@ static inline int vector_delete_range(vector v, int start, int end) return (end_offset - start_offset) / sizeof(void *); } +boolean vector_init(vector v, heap h, int len); +void vector_deinit(vector v); + static inline vector allocate_vector(heap h, int length) { return allocate_buffer(h, length * sizeof (void *)); diff --git a/src/xen/xen.c b/src/xen/xen.c index 3d2e9f37b..9b3051ba2 100644 --- a/src/xen/xen.c +++ b/src/xen/xen.c @@ -32,13 +32,6 @@ status xenstore_watch(buffer path, xenstore_watch_handler handler, boolean watch typedef struct xen_platform_info { heap h; /* general heap for internal use */ - /* version and features */ - u16 xen_major; - u16 xen_minor; - u32 last_leaf; - u32 msr_base; - u32 features; - /* event channel interface / PV interrupts */ volatile struct shared_info *shared_info; @@ -47,7 +40,9 @@ typedef struct xen_platform_info { u64 xenstore_paddr; evtchn_port_t xenstore_evtchn; struct spinlock xenstore_lock; - vector evtchn_handlers; + + u64 vcpu_evtchn_enable[XEN_LEGACY_MAX_VCPUS][sizeof(u64) * 8]; + struct vector evtchn_handlers; closure_struct(xenstore_watch_handler, watch_handler); closure_struct(thunk, scan_service); @@ -68,9 +63,6 @@ typedef struct xen_platform_info { tuple device_tree; struct list xenbus_list; struct list driver_list; - - /* XXX could make generalized status */ - boolean initialized; } *xen_platform_info; typedef struct xen_driver { @@ -79,19 +71,14 @@ typedef struct xen_driver { xen_device_probe probe; } *xen_driver; -struct xen_platform_info xen_info; +static BSS_RO_AFTER_INIT xen_platform_info xen_info; -#define xenstore_lock() u64 _irqflags = spin_lock_irq(&xen_info.xenstore_lock) -#define xenstore_unlock() spin_unlock_irq(&xen_info.xenstore_lock, _irqflags) - -boolean xen_feature_supported(int feature) -{ - return (xen_info.features & U64_FROM_BIT(feature)) != 0; -} +#define xenstore_lock() u64 _irqflags = spin_lock_irq(&xen_info->xenstore_lock) +#define xenstore_unlock() spin_unlock_irq(&xen_info->xenstore_lock, _irqflags) boolean xen_detected(void) { - return xen_info.initialized; + return !!xen_info; } extern u64 hypercall_page; @@ -99,8 +86,9 @@ extern u64 hypercall_page; closure_function(0, 0, void, xen_interrupt) { int vcpu = current_cpu()->id; - volatile struct shared_info *si = xen_info.shared_info; - volatile struct vcpu_info *vci = &xen_info.shared_info->vcpu_info[vcpu]; + volatile struct shared_info *si = xen_info->shared_info; + volatile struct vcpu_info *vci = &si->vcpu_info[vcpu]; + u64 *evtchn_enable = xen_info->vcpu_evtchn_enable[vcpu]; xenint_debug("xen_interrupt enter"); assert(vci->evtchn_upcall_pending); @@ -111,24 +99,16 @@ closure_function(0, 0, void, xen_interrupt) /* this may not process in the right order, or it might not matter - care later */ bitmap_word_foreach_set(l1_pending, bit1, i1, 0) { (void)i1; - /* TODO: any per-cpu event mask would be also applied here... */ - u64 l2_pending = si->evtchn_pending[bit1] & ~si->evtchn_mask[bit1]; + u64 l2_pending = si->evtchn_pending[bit1] & evtchn_enable[bit1]; xenint_debug("pending 0x%lx, mask 0x%lx, masked 0x%lx", - si->evtchn_pending[bit1], si->evtchn_mask[bit1], l2_pending); + si->evtchn_pending[bit1], ~evtchn_enable[bit1], l2_pending); __sync_and_and_fetch(&si->evtchn_pending[bit1], ~l2_pending); u64 l2_offset = bit1 << 6; bitmap_word_foreach_set(l2_pending, bit2, i2, l2_offset) { (void)i2; xenint_debug(" int %d pending", i2); - thunk handler = vector_get(xen_info.evtchn_handlers, i2); - if (handler) { - xenint_debug(" evtchn %d: applying handler %p", i2, handler); - apply(handler); - } else { - /* XXX we have an issue with seemingly spurious interrupts at evtchn >= 2048... */ - xenint_debug(" evtchn %d: spurious interrupt", i2); - si->evtchn_mask[bit1] |= U64_FROM_BIT(bit2); - } + thunk handler = vector_get(&xen_info->evtchn_handlers, i2); + apply(handler); } } vci->evtchn_upcall_mask = 0; @@ -138,21 +118,34 @@ closure_function(0, 0, void, xen_interrupt) void xen_register_evtchn_handler(evtchn_port_t evtchn, thunk handler) { - assert(vector_set(xen_info.evtchn_handlers, evtchn, handler)); + assert(vector_set(&xen_info->evtchn_handlers, evtchn, handler)); } -int xen_unmask_evtchn(evtchn_port_t evtchn) +int xen_bind_evtchn(evtchn_port_t evtchn, int vcpu) +{ + evtchn_op_t eop; + eop.cmd = EVTCHNOP_bind_vcpu; + eop.u.bind_vcpu.port = evtchn; + eop.u.bind_vcpu.vcpu = vcpu; + return HYPERVISOR_event_channel_op(&eop); +} + +int xen_unmask_evtchn(evtchn_port_t evtchn, int vcpu) { assert(evtchn > 0 && evtchn < EVTCHN_2L_NR_CHANNELS); evtchn_op_t eop; eop.cmd = EVTCHNOP_unmask; eop.u.unmask.port = evtchn; - return HYPERVISOR_event_channel_op(&eop); + int rv = HYPERVISOR_event_channel_op(&eop); + if (rv == 0) + atomic_set_bit(&xen_info->vcpu_evtchn_enable[vcpu][evtchn >> 6], evtchn & MASK(6)); + return rv; } -int xen_close_evtchn(evtchn_port_t evtchn) +int xen_close_evtchn(evtchn_port_t evtchn, int vcpu) { - vector_set(xen_info.evtchn_handlers, evtchn, 0); + atomic_clear_bit(&xen_info->vcpu_evtchn_enable[vcpu][evtchn >> 6], evtchn & MASK(6)); + vector_set(&xen_info->evtchn_handlers, evtchn, 0); evtchn_op_t eop; eop.cmd = EVTCHNOP_close; eop.u.close.port = evtchn; @@ -161,9 +154,9 @@ int xen_close_evtchn(evtchn_port_t evtchn) #define GTAB_RESERVED_ENTRIES 8 -static boolean xen_grant_init(kernel_heaps kh) +static boolean xen_grant_init(kernel_heaps kh, u32 features) { - struct gtab *gt = &xen_info.gtab; + struct gtab *gt = &xen_info->gtab; struct gnttab_query_size qs; qs.dom = DOMID_SELF; int rv = HYPERVISOR_grant_table_op(GNTTABOP_query_size, &qs, 1); @@ -197,7 +190,7 @@ static boolean xen_grant_init(kernel_heaps kh) /* We have this feature on the platforms we care about now, but we can add support for the other case if need be. */ - if (!xen_feature_supported(XENFEAT_auto_translated_physmap)) { + if (!(features & U32_FROM_BIT(XENFEAT_auto_translated_physmap))) { msg_err("%s error: auto translated physmap feature required", func_ss); goto fail_dealloc_heap; } @@ -232,7 +225,7 @@ static boolean xen_grant_init(kernel_heaps kh) /* optimization: create a gntref free list / fifo of sorts */ grant_ref_t xen_grant_page_access(u16 domid, u64 phys, boolean readonly) { - struct gtab *gt = &xen_info.gtab; + struct gtab *gt = &xen_info->gtab; u64 ref = allocate_u64(gt->entry_heap, 1); if (ref == INVALID_PHYSICAL) return 0; @@ -246,9 +239,9 @@ grant_ref_t xen_grant_page_access(u16 domid, u64 phys, boolean readonly) void xen_revoke_page_access(grant_ref_t ref) { assert(ref > 8 && ref != -1); - xen_info.gtab.table[ref].flags = 0; + xen_info->gtab.table[ref].flags = 0; memory_barrier(); - deallocate_u64(xen_info.gtab.entry_heap, ref, 1); + deallocate_u64(xen_info->gtab.entry_heap, ref, 1); } /* Reportedly, Xen timers can fire up to 100us early. */ @@ -324,10 +317,10 @@ closure_func_basic(thunk, void, xenstore_evtchn_handler) xenstore_watch_event(msg); } else { /* unexpected message type: discard message data */ - volatile struct xenstore_domain_interface *xsdi = xen_info.xenstore_interface; + volatile struct xenstore_domain_interface *xsdi = xen_info->xenstore_interface; xsdi->rsp_cons += msg->len; write_barrier(); - xen_notify_evtchn(xen_info.xenstore_evtchn); + xen_notify_evtchn(xen_info->xenstore_evtchn); } out: xenstore_unlock(); @@ -370,8 +363,8 @@ static int xen_setup_vcpu(int vcpu, u64 shared_info_phys) evtchn_port_t timer_evtchn = eop.u.bind_virq.port; xen_debug("cpu %d timer event channel %d", vcpu, timer_evtchn); xen_register_evtchn_handler(timer_evtchn, - closure_func(xen_info.h, thunk, xen_runloop_timer_handler)); - assert(xen_unmask_evtchn(timer_evtchn) == 0); + closure_func(xen_info->h, thunk, xen_runloop_timer_handler)); + assert(xen_unmask_evtchn(timer_evtchn, vcpu) == 0); return 0; } @@ -404,14 +397,13 @@ define_closure_function(1, 0, void, xen_shutdown_handler, closure_func_basic(xenstore_watch_handler, void, xen_shutdown_watcher, sstring path) { - async_apply_bh((thunk)&xen_info.shutdown_handler); + async_apply_bh((thunk)&xen_info->shutdown_handler); } boolean xen_detect(kernel_heaps kh) { u32 v[4]; - xen_info.initialized = false; - xen_info.h = heap_locked(kh); + heap h = heap_locked(kh); xen_debug("checking for xen cpuid leaves"); cpuid(XEN_CPUID_FIRST_LEAF, 0, v); if (!(v[1] == XEN_CPUID_SIGNATURE_EBX && @@ -421,20 +413,16 @@ boolean xen_detect(kernel_heaps kh) return false; } - xen_info.last_leaf = v[0]; - cpuid(XEN_CPUID_LEAF(1), 0, v); - xen_info.xen_major = v[0] >> 16; - xen_info.xen_minor = v[0] & MASK(16); - xen_debug("xen version %d.%d detected", xen_info.xen_major, xen_info.xen_minor); + xen_debug("xen version %d.%d detected", v[0] >> 16, v[0] & MASK(16)); cpuid(XEN_CPUID_LEAF(2), 0, v); if (v[0] != 1) { msg_err("xen reporting %d hypercall pages; not supported", v[0]); return false; } - xen_info.msr_base = v[1]; - xen_debug("msr base 0x%x, features 1 0x%x, features 2 0x%x", xen_info.msr_base, v[2], v[3]); + u32 msr_base = v[1]; + xen_debug("msr base 0x%x, features 1 0x%x, features 2 0x%x", msr_base, v[2], v[3]); #if 0 cpuid(XEN_CPUID_LEAF(3), 0, v); @@ -452,7 +440,7 @@ boolean xen_detect(kernel_heaps kh) /* install hypercall page */ u64 hp_phys = physical_from_virtual(&hypercall_page); xen_debug("hypercall_page: v 0x%lx, p 0x%lx", u64_from_pointer(&hypercall_page), hp_phys); - write_msr(xen_info.msr_base, hp_phys); + write_msr(msr_base, hp_phys); /* get xen features */ build_assert(XENFEAT_NR_SUBMAPS == 1); @@ -463,14 +451,17 @@ boolean xen_detect(kernel_heaps kh) msg_err("xen: failed to get features map (rv %d)", rv); return false; } - xen_info.features = xfi.submap; - xen_debug("reported features map 0x%x", xen_info.features); + u32 features = xfi.submap; + xen_debug("reported features map 0x%x", features); - if (!xen_feature_supported(XENFEAT_hvm_safe_pvclock)) { + if (!(features & U32_FROM_BIT(XENFEAT_hvm_safe_pvclock))) { msg_err("xen failed to init; XENFEAT_hvm_safe_pvclock required"); return false; } + xen_info = mem_alloc(h, sizeof(*xen_info), MEM_ZERO | MEM_NOWAIT | MEM_NOFAIL); + xen_info->h = h; + /* get store page, map it, and retrieve event channel */ xen_debug("retrieving xenstore page"); struct xen_hvm_param xen_hvm_param; @@ -479,16 +470,16 @@ boolean xen_detect(kernel_heaps kh) rv = HYPERVISOR_hvm_op(HVMOP_get_param, &xen_hvm_param); if (rv < 0) { msg_err("xen: failed to get xenstore page address (rv %d)", rv); - return false; + goto out_dealloc_xen_info; } - xen_info.xenstore_paddr = xen_hvm_param.value << PAGELOG; + xen_info->xenstore_paddr = xen_hvm_param.value << PAGELOG; - xen_debug("xenstore page at phys 0x%lx; allocating virtual page and mapping", xen_info.xenstore_paddr); - xen_info.xenstore_interface = mem_alloc((heap)heap_virtual_page(kh), PAGESIZE, + xen_debug("xenstore page at phys 0x%lx; allocating virtual page", xen_info->xenstore_paddr); + xen_info->xenstore_interface = mem_alloc((heap)heap_virtual_page(kh), PAGESIZE, MEM_NOWAIT | MEM_NOFAIL); - map(u64_from_pointer(xen_info.xenstore_interface), xen_info.xenstore_paddr, PAGESIZE, + map(u64_from_pointer(xen_info->xenstore_interface), xen_info->xenstore_paddr, PAGESIZE, pageflags_writable(pageflags_memory())); - xen_debug("xenstore page mapped at %p", xen_info.xenstore_interface); + xen_debug("xenstore page mapped at %p", xen_info->xenstore_interface); xen_debug("retrieving store event channel"); xen_hvm_param.domid = DOMID_SELF; @@ -496,17 +487,17 @@ boolean xen_detect(kernel_heaps kh) rv = HYPERVISOR_hvm_op(HVMOP_get_param, &xen_hvm_param); if (rv < 0) { msg_err("xen: failed to get xenstore event channel (rv %d)", rv); - return false; + goto out_dealloc_xenstore; } - xen_info.xenstore_evtchn = xen_hvm_param.value; - xen_debug("event channel %ld, allocating and mapping shared info page", xen_info.xenstore_evtchn); + xen_info->xenstore_evtchn = xen_hvm_param.value; + xen_debug("event channel %ld, allocating and mapping shared info page", xen_info->xenstore_evtchn); heap shared_info_heap = (heap)heap_linear_backed(kh); void *shared_info = mem_alloc(shared_info_heap, PAGESIZE, MEM_ZERO | MEM_NOWAIT | MEM_NOFAIL); u64 shared_info_phys = physical_from_virtual(shared_info); assert(shared_info_phys != INVALID_PHYSICAL); - xen_info.shared_info = shared_info; + xen_info->shared_info = shared_info; xen_add_to_physmap_t xatp; xatp.domid = DOMID_SELF; xatp.space = XENMAPSPACE_shared_info; @@ -519,16 +510,16 @@ boolean xen_detect(kernel_heaps kh) } xen_debug("shared info page: %p, phys 0x%lx", shared_info, shared_info_phys); - if (!xen_feature_supported(XENFEAT_hvm_callback_vector)) { + if (!(features & U32_FROM_BIT(XENFEAT_hvm_callback_vector))) { msg_err("xen setup failed: HVM callback vector must be supported (features mask 0x%x)", - xen_info.features); + features); goto out_dealloc_shared_page; } /* set up interrupt handling path */ int irq = allocate_interrupt(); xen_debug("interrupt vector %d; registering", irq); - register_interrupt(irq, closure(xen_info.h, xen_interrupt), ss("xen")); + register_interrupt(irq, closure(h, xen_interrupt), ss("xen")); xen_hvm_param.domid = DOMID_SELF; xen_hvm_param.index = HVM_PARAM_CALLBACK_IRQ; @@ -539,46 +530,52 @@ boolean xen_detect(kernel_heaps kh) goto out_unregister_irq; } - xen_info.evtchn_handlers = allocate_vector(xen_info.h, 1); - assert(xen_info.evtchn_handlers != INVALID_ADDRESS); + if (!vector_init(&xen_info->evtchn_handlers, h, 1)) + goto out_unregister_irq; if (xen_setup_vcpu(0, shared_info_phys) < 0) - goto out_unregister_irq; + goto out_deinit_evtchn_handlers; - register_platform_clock_timer(closure_func(xen_info.h, clock_timer, xen_runloop_timer), - closure(xen_info.h, xen_per_cpu_init, shared_info_phys)); + register_platform_clock_timer(closure_func(h, clock_timer, xen_runloop_timer), + closure(h, xen_per_cpu_init, shared_info_phys)); /* register pvclock (feature verified above) */ - init_pvclock(xen_info.h, (struct pvclock_vcpu_time_info *)&xen_info.shared_info->vcpu_info[0].time, - (struct pvclock_wall_clock *)&xen_info.shared_info->wc_version); + init_pvclock(h, (struct pvclock_vcpu_time_info *)&xen_info->shared_info->vcpu_info[0].time, + (struct pvclock_wall_clock *)&xen_info->shared_info->wc_version); xen_debug("unmasking xenstore event channel"); - spin_lock_init(&xen_info.xenstore_lock); - xen_register_evtchn_handler(xen_info.xenstore_evtchn, - closure_func(xen_info.h, thunk, xenstore_evtchn_handler)); - assert(xen_unmask_evtchn(xen_info.xenstore_evtchn) == 0); + spin_lock_init(&xen_info->xenstore_lock); + xen_register_evtchn_handler(xen_info->xenstore_evtchn, + closure_func(h, thunk, xenstore_evtchn_handler)); + assert(xen_unmask_evtchn(xen_info->xenstore_evtchn, 0) == 0); - if (!xen_grant_init(kh)) { + if (!xen_grant_init(kh, features)) { msg_err("xen: failed to set up grant tables"); - goto out_unregister_irq; + goto out_deinit_evtchn_handlers; } - init_closure(&xen_info.shutdown_handler, xen_shutdown_handler, allocate_buffer(xen_info.h, 16)); + init_closure(&xen_info->shutdown_handler, xen_shutdown_handler, allocate_buffer(h, 16)); if (!is_ok(xenstore_watch(alloca_wrap_cstring("control/shutdown"), - init_closure_func(&xen_info.shutdown_watcher, xenstore_watch_handler, + init_closure_func(&xen_info->shutdown_watcher, xenstore_watch_handler, xen_shutdown_watcher), true))) msg_err("xen: failed to register shutdown handler"); xen_debug("xen initialization complete"); - list_init(&xen_info.driver_list); - xen_info.initialized = true; + list_init(&xen_info->driver_list); return true; + out_deinit_evtchn_handlers: + vector_deinit(&xen_info->evtchn_handlers); out_unregister_irq: unregister_interrupt(irq); out_dealloc_shared_page: - deallocate(shared_info_heap, xen_info.shared_info, PAGESIZE); - xen_info.shared_info = 0; + deallocate(shared_info_heap, xen_info->shared_info, PAGESIZE); + out_dealloc_xenstore: + unmap(u64_from_pointer(xen_info->xenstore_interface), PAGESIZE); + deallocate((heap)heap_virtual_page(kh), xen_info->xenstore_interface, PAGESIZE); + out_dealloc_xen_info: + deallocate(h, xen_info, sizeof(*xen_info)); + xen_info = 0; return false; } @@ -609,7 +606,7 @@ int xen_notify_evtchn(evtchn_port_t evtchn) /* returns bytes written; doesn't block */ static s64 xenstore_write_internal(const void * data, s64 length) { - volatile struct xenstore_domain_interface *xsdi = xen_info.xenstore_interface; + volatile struct xenstore_domain_interface *xsdi = xen_info->xenstore_interface; if (length == 0) return 0; assert(length > 0); @@ -632,7 +629,7 @@ static s64 xenstore_write_internal(const void * data, s64 length) write_barrier(); xsdi->req_prod += nwrite; write_barrier(); - int rv = xen_notify_evtchn(xen_info.xenstore_evtchn); + int rv = xen_notify_evtchn(xen_info->xenstore_evtchn); if (rv < 0) { result = rv; break; @@ -645,7 +642,7 @@ static s64 xenstore_write_internal(const void * data, s64 length) static s64 xenstore_read_internal(buffer b, s64 length) { - volatile struct xenstore_domain_interface *xsdi = xen_info.xenstore_interface; + volatile struct xenstore_domain_interface *xsdi = xen_info->xenstore_interface; if (length == 0) return 0; assert(length > 0); @@ -668,7 +665,7 @@ static s64 xenstore_read_internal(buffer b, s64 length) write_barrier(); xsdi->rsp_cons += nread; write_barrier(); - int rv = xen_notify_evtchn(xen_info.xenstore_evtchn); + int rv = xen_notify_evtchn(xen_info->xenstore_evtchn); if (rv < 0) { result = rv; break; @@ -715,7 +712,7 @@ status xenstore_sync_request(u32 tx_id, enum xsd_sockmsg_type type, buffer reque { status s = STATUS_OK; struct xsd_sockmsg msg; - buffer rbuf = allocate_buffer(xen_info.h, PAGESIZE); // XXX + buffer rbuf = allocate_buffer(xen_info->h, PAGESIZE); assert(rbuf != INVALID_ADDRESS); u32 len = request ? buffer_length(request) : 1; void * data = request ? buffer_ref(request, 0) : ""; @@ -770,7 +767,7 @@ status xenstore_sync_request(u32 tx_id, enum xsd_sockmsg_type type, buffer reque status xenstore_transaction_start(u32 *tx_id) { - buffer response = allocate_buffer(xen_info.h, 16); + buffer response = allocate_buffer(xen_info->h, 16); status s = xenstore_sync_request(0, XS_TRANSACTION_START, 0, response); if (!is_ok(s)) goto out; @@ -788,7 +785,7 @@ status xenstore_transaction_start(u32 *tx_id) status xenstore_transaction_end(u32 tx_id, boolean abort) { buffer request = alloca_wrap_buffer(abort ? "F" : "T", 2); - buffer response = allocate_buffer(xen_info.h, 8); /* for error capture */ + buffer response = allocate_buffer(xen_info->h, 8); /* for error capture */ status s = xenstore_sync_request(tx_id, XS_TRANSACTION_END, request, response); deallocate_buffer(response); return s; @@ -796,7 +793,7 @@ status xenstore_transaction_end(u32 tx_id, boolean abort) status xenstore_sync_printf(u32 tx_id, buffer path, sstring node, sstring format, ...) { - buffer request = allocate_buffer(xen_info.h, PAGESIZE); + buffer request = allocate_buffer(xen_info->h, PAGESIZE); assert(push_buffer(request, path)); push_u8(request, '/'); assert(buffer_write_sstring(request, node)); @@ -806,7 +803,7 @@ status xenstore_sync_printf(u32 tx_id, buffer path, sstring node, sstring format vbprintf(request, format, &a); xenstore_debug("%s: request: \"%b\"", func_ss, request); - buffer response = allocate_buffer(xen_info.h, 8); /* for error capture */ + buffer response = allocate_buffer(xen_info->h, 8); /* for error capture */ status s = xenstore_sync_request(tx_id, XS_WRITE, request, response); deallocate_buffer(request); deallocate_buffer(response); @@ -815,7 +812,7 @@ status xenstore_sync_printf(u32 tx_id, buffer path, sstring node, sstring format status xenstore_read_u64(u32 tx_id, buffer path, sstring node, u64 *result) { - buffer response = allocate_buffer(xen_info.h, 16); + buffer response = allocate_buffer(xen_info->h, 16); status s = xenstore_read_string(tx_id, path, node, response); if (!is_ok(s)) goto out; @@ -832,7 +829,7 @@ status xenstore_read_u64(u32 tx_id, buffer path, sstring node, u64 *result) status xenstore_read_string(u32 tx_id, buffer path, sstring node, buffer response) { - buffer request = allocate_buffer(xen_info.h, 64); + buffer request = allocate_buffer(xen_info->h, 64); assert(push_buffer(request, path)); push_u8(request, '/'); assert(buffer_write_sstring(request, node)); @@ -849,7 +846,7 @@ status xenstore_read_string(u32 tx_id, buffer path, sstring node, buffer respons status xenstore_watch(buffer path, xenstore_watch_handler handler, boolean watch) { buffer request = little_stack_buffer(buffer_length(path) + 18); - buffer response = allocate_buffer(xen_info.h, 8); + buffer response = allocate_buffer(xen_info->h, 8); if (response == INVALID_ADDRESS) return timm("result", "failed to allocate memory"); push_buffer(request, path); @@ -1023,7 +1020,7 @@ closure_function(4, 2, boolean, xen_probe_id_each, return false; } xen_debug("driver match, id %d, value %v", id, v); - buffer frontend = allocate_buffer(xen_info.h, buffer_length(bound(name)) + 10); + buffer frontend = allocate_buffer(xen_info->h, buffer_length(bound(name)) + 10); bprintf(frontend, "device/%b/%d", bound(name), id); boolean bound = apply(bound(xd)->probe, (int)id, frontend, v); if (bound) @@ -1040,7 +1037,7 @@ closure_function(1, 2, boolean, xen_probe_devices_each, assert(is_symbol(k)); if (!is_tuple(v)) return true; - list_foreach(&xen_info.driver_list, l) { + list_foreach(&xen_info->driver_list, l) { xen_driver xd = struct_from_list(l, xen_driver, l); /* XXX must be a cleaner way to compare symbols? */ string name = symbol_string(k); @@ -1055,13 +1052,13 @@ closure_function(1, 2, boolean, xen_probe_devices_each, static status xen_scan(void) { - status s = traverse_directory(xen_info.h, "device", &xen_info.device_tree); + status s = traverse_directory(xen_info->h, "device", &xen_info->device_tree); if (!is_ok(s)) return s; - if (!xen_info.device_tree) + if (!xen_info->device_tree) return timm("result", "failed to parse directory"); - xen_debug("scan result: %v", xen_info.device_tree); - iterate(xen_info.device_tree, stack_closure(xen_probe_devices_each, &s)); + xen_debug("scan result: %v", xen_info->device_tree); + iterate(xen_info->device_tree, stack_closure(xen_probe_devices_each, &s)); return s; } @@ -1078,7 +1075,7 @@ closure_func_basic(xenstore_watch_handler, void, xen_watch_handler, } if (depth == 0) return; - async_apply_bh((thunk)&xen_info.scan_service); + async_apply_bh((thunk)&xen_info->scan_service); } closure_func_basic(thunk, void, xen_scan_service) @@ -1086,13 +1083,13 @@ closure_func_basic(thunk, void, xen_scan_service) xenstore_debug("%s", func_ss); /* Avoid concurrent scans. */ - if (atomic_test_and_set_bit(&xen_info.scanning, 0)) { - async_apply((thunk)&xen_info.scan_service); + if (atomic_test_and_set_bit(&xen_info->scanning, 0)) { + async_apply((thunk)&xen_info->scan_service); return; } status s = xen_scan(); - atomic_clear_bit(&xen_info.scanning, 0); + atomic_clear_bit(&xen_info->scanning, 0); if (!is_ok(s)) { msg_warn("xen: cannot scan devices: %v", s); timm_dealloc(s); @@ -1102,12 +1099,12 @@ closure_func_basic(thunk, void, xen_scan_service) status xen_probe_devices(void) { xen_debug("probing xen device tree from xenstored"); - assert(xen_info.device_tree == 0); + assert(xen_info->device_tree == 0); status s = xen_scan(); if (is_ok(s)) { - init_closure_func(&xen_info.scan_service, thunk, xen_scan_service); + init_closure_func(&xen_info->scan_service, thunk, xen_scan_service); s = xenstore_watch(alloca_wrap_cstring("device"), - init_closure_func(&xen_info.watch_handler, xenstore_watch_handler, + init_closure_func(&xen_info->watch_handler, xenstore_watch_handler, xen_watch_handler), true); if (!is_ok(s)) { @@ -1121,8 +1118,8 @@ status xen_probe_devices(void) void register_xen_driver(sstring name, xen_device_probe probe) { - xen_driver xd = mem_alloc(xen_info.h, sizeof(struct xen_driver), MEM_NOWAIT | MEM_NOFAIL); + xen_driver xd = mem_alloc(xen_info->h, sizeof(struct xen_driver), MEM_NOWAIT | MEM_NOFAIL); xd->name = name; xd->probe = probe; - list_insert_before(&xen_info.driver_list, &xd->l); + list_insert_before(&xen_info->driver_list, &xd->l); } diff --git a/src/xen/xen_internal.h b/src/xen/xen_internal.h index 383d6947c..d0cb934c2 100644 --- a/src/xen/xen_internal.h +++ b/src/xen/xen_internal.h @@ -38,8 +38,9 @@ typedef struct xen_dev { status xen_allocate_evtchn(domid_t other_id, evtchn_port_t *evtchn); void xen_register_evtchn_handler(evtchn_port_t evtchn, thunk handler); int xen_notify_evtchn(evtchn_port_t evtchn); -int xen_unmask_evtchn(evtchn_port_t evtchn); -int xen_close_evtchn(evtchn_port_t evtchn); +int xen_bind_evtchn(evtchn_port_t evtchn, int vcpu); +int xen_unmask_evtchn(evtchn_port_t evtchn, int vcpu); +int xen_close_evtchn(evtchn_port_t evtchn, int vcpu); grant_ref_t xen_grant_page_access(u16 domid, u64 phys, boolean readonly); void xen_revoke_page_access(grant_ref_t ref); diff --git a/src/xen/xenblk.c b/src/xen/xenblk.c index 750c209bb..c31b80753 100644 --- a/src/xen/xenblk.c +++ b/src/xen/xenblk.c @@ -39,6 +39,7 @@ struct xenblk_dev { blkif_front_ring_t ring; grant_ref_t ring_gntref; evtchn_port_t evtchn; + s32 evtchn_cpu; closure_struct(xenblk_io, read); closure_struct(xenblk_io, write); closure_struct(storage_simple_req_handler, req_handler); @@ -275,7 +276,12 @@ static void xenblk_remove(xenblk_dev xbd) } xen_driver_unbind(xbd->meta); xenbus_set_state(0, xbd->dev.frontend, XenbusStateClosed); - xen_close_evtchn(xbd->evtchn); + s32 evtchn_cpu = xbd->evtchn_cpu; + if (evtchn_cpu >= 0) + irq_put_target_cpu(evtchn_cpu); + else + evtchn_cpu = 0; + xen_close_evtchn(xbd->evtchn, evtchn_cpu); xen_revoke_page_access(xbd->ring_gntref); deallocate(xbd->contiguous, xbd->ring.sring, PAGESIZE); deallocate_vector(xbd->rreqs); @@ -336,7 +342,18 @@ closure_func_basic(thunk, void, xenblk_watch_service) timm_dealloc(s); goto remove; } - int rv = xen_unmask_evtchn(xbd->evtchn); + u32 evtchn_cpu = irq_get_target_cpu(irange(0, 0)); + int rv = xen_bind_evtchn(xbd->evtchn, evtchn_cpu); + if (rv < 0) { + msg_warn("%s: failed to bind event channel %d to vCPU %d: rv %d", func_ss, xbd->evtchn, + evtchn_cpu, rv); + irq_put_target_cpu(evtchn_cpu); + xbd->evtchn_cpu = -1; + evtchn_cpu = 0; + } else { + xbd->evtchn_cpu = evtchn_cpu; + } + rv = xen_unmask_evtchn(xbd->evtchn, evtchn_cpu); if (rv < 0) { msg_err("%s: failed to unmask event channel %d: rv %d", func_ss, xbd->evtchn, rv); goto remove; @@ -472,7 +489,7 @@ static status xenblk_enable(xenblk_dev xbd) } return STATUS_OK; out_evtchn: - xen_close_evtchn(xbd->evtchn); + xen_close_evtchn(xbd->evtchn, 0); out_revoke: xen_revoke_page_access(xbd->ring_gntref); out_dealloc: diff --git a/src/xen/xennet.c b/src/xen/xennet.c index c1174f53c..caa821411 100644 --- a/src/xen/xennet.c +++ b/src/xen/xennet.c @@ -724,7 +724,15 @@ static status xennet_enable(xennet_dev xd) /* we're kind of always up ... start rx now */ xd->rx_ring.sring->rsp_event = xd->rx_ring.rsp_cons + 1; write_barrier(); - int rv = xen_unmask_evtchn(xd->evtchn); + u32 evtchn_cpu = irq_get_target_cpu(irange(0, 0)); + int rv = xen_bind_evtchn(xd->evtchn, evtchn_cpu); + if (rv < 0) { + msg_warn("%s: failed to bind event channel %d to vCPU %d: rv %d", func_ss, xd->evtchn, + evtchn_cpu, rv); + irq_put_target_cpu(evtchn_cpu); + evtchn_cpu = 0; + } + rv = xen_unmask_evtchn(xd->evtchn, evtchn_cpu); if (rv < 0) { s = timm("result", "failed to unmask event channel %d: rv %d", xd->evtchn, rv); goto out_dealloc_rx_buffers;