diff --git a/MAINTAINERS b/MAINTAINERS index ad215eced84..0961d6c2af9 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -475,7 +475,9 @@ ARM KVM CPUs M: Peter Maydell L: qemu-arm@nongnu.org S: Supported +F: include/hw/arm/rme-da.h F: target/arm/kvm.c +F: target/arm/kvm-rme.c F: tests/functional/aarch64/test_kvm.py MIPS KVM CPUs @@ -1115,6 +1117,7 @@ S: Supported F: hw/arm/virt* F: include/hw/arm/virt.h F: docs/system/arm/virt.rst +F: tests/qtest/arm-virt-machine-test.c F: tests/functional/aarch64/test_*virt*.py F: tests/functional/aarch64/test_tuxrun.py F: tests/functional/arm/test_tuxrun.py diff --git a/accel/kvm/kvm-all.c b/accel/kvm/kvm-all.c index 92af42503b1..32f72704d79 100644 --- a/accel/kvm/kvm-all.c +++ b/accel/kvm/kvm-all.c @@ -1665,6 +1665,12 @@ static void kvm_set_phys_mem(KVMMemoryListener *kml, ram_start_offset = memory_region_get_ram_addr(mr) + mr_offset; if (!add) { + bool clear_attrs = + machine_has_assigned_device_memory(current_machine) && + memory_region_is_ram_device(mr) && + (kvm_supported_memory_attributes & + KVM_MEMORY_ATTRIBUTE_PRIVATE); + do { slot_size = MIN(kvm_max_slot_size, size); mem = kvm_lookup_matching_slot(kml, start_addr, slot_size); @@ -1708,6 +1714,23 @@ static void kvm_set_phys_mem(KVMMemoryListener *kml, __func__, strerror(-err)); abort(); } + /* + * Drop any VDEV-installed PRIVATE attributes for this slot. The + * mem_attr_array is per-VM and persists across slot lifecycle, + * so without this clear a re-mapped or recycled gfn range could + * inherit stale VDEV-locked state. The attribute-clear also + * tears down any leftover ASSIGNED-DEV S2 entries via + * kvm_arch_post_set_memory_attributes() / kvm_unmap_gfn_range(). + */ + if (clear_attrs) { + if (kvm_set_memory_attributes_shared(start_addr, slot_size)) { + error_report("failed to clear memory attributes while " + "removing slot [0x%" HWADDR_PRIx + ", 0x%" HWADDR_PRIx ")", + start_addr, start_addr + slot_size); + exit(EXIT_FAILURE); + } + } start_addr += slot_size; size -= slot_size; kml->nr_slots_used--; @@ -2701,6 +2724,14 @@ static int do_kvm_create_vm(KVMState *s, int type) error_printf("PPC KVM module is not loaded. Try modprobe kvm_%s.\n", (type == 2) ? "pr" : "hv"); } +#elif defined(TARGET_AARCH64) + if (ret == -EINVAL && + (type & KVM_VM_TYPE_ARM_MASK) == KVM_VM_TYPE_ARM_REALM && + KVM_VM_TYPE_ARM_IPA_SIZE(type)) { + error_printf("The requested Realm IPA size (%u bits) may exceed " + "the RMM S2SZ limit.\n", + (unsigned int)KVM_VM_TYPE_ARM_IPA_SIZE(type)); + } #endif } @@ -3377,18 +3408,51 @@ int kvm_convert_memory(hwaddr start, hwaddr size, bool to_private) return ret; } + if (!int128_eq(section.size, int128_make64(size))) { + error_report("Convert memory range (0x%" HWADDR_PRIx + " + 0x%" HWADDR_PRIx ") crosses a memory-region " + "boundary", start, size); + goto out_unref; + } + if (!memory_region_has_guest_memfd(mr)) { /* - * Because vMMIO region must be shared, guest TD may convert vMMIO - * region to shared explicitly. Don't complain such case. See + * Because vMMIO region may be shared, guest TD may convert vMMIO + * region to shared explicitly. Don't complain such case. See * memory_region_type() for checking if the region is MMIO region. */ if (!to_private && - !memory_region_is_ram(mr) && - !memory_region_is_ram_device(mr) && + (!memory_region_is_ram(mr) || + (machine_has_assigned_device_memory(current_machine) && + memory_region_is_ram_device(mr))) && !memory_region_is_rom(mr) && !memory_region_is_romd(mr)) { - ret = 0; + /* + * For ram_device regions (e.g. VFIO BARs), the realm may + * have marked individual gfns PRIVATE through + * iommufd_tsm_dev_memmap_exit() when validating a VDEV + * mapping. When the realm later asks to release the + * mapping with RSI_IPA_STATE_SET(EMPTY), the kernel + * surfaces it here as a !to_private convert. A silent + * ret=0 would leave PRIVATE set with no DEV mapping + * behind it; honour the release by actually clearing the + * attribute. The kvm_arch_post_set_memory_attributes() + * side-effect then tears down any leftover ASSIGNED-DEV + * S2 via kvm_unmap_gfn_range(KVM_FILTER_PRIVATE), keeping + * userspace's view in sync with the kernel's. + * + * For the non-ram_device legs of this branch (e.g. TDX + * vMMIO traps) no PRIVATE attribute was ever installed, + * so preserve the original ret=0 fast path. + */ + if (machine_has_assigned_device_memory(current_machine) && + memory_region_is_ram_device(mr) && + (kvm_supported_memory_attributes & + KVM_MEMORY_ATTRIBUTE_PRIVATE)) { + ret = kvm_set_memory_attributes_shared(start, size); + } else { + ret = 0; + } } else { error_report("Convert non guest_memfd backed memory region " "(0x%"HWADDR_PRIx" ,+ 0x%"HWADDR_PRIx") to %s", @@ -4763,6 +4827,11 @@ void kvm_mark_guest_state_protected(void) kvm_state->guest_state_protected = true; } +bool kvm_guest_state_protected(void) +{ + return kvm_state && kvm_state->guest_state_protected; +} + int kvm_create_guest_memfd(uint64_t size, uint64_t flags, Error **errp) { int fd; diff --git a/accel/stubs/kvm-stub.c b/accel/stubs/kvm-stub.c index c4617caac6b..c249c242b31 100644 --- a/accel/stubs/kvm-stub.c +++ b/accel/stubs/kvm-stub.c @@ -143,3 +143,13 @@ int kvm_create_guest_memfd(uint64_t size, uint64_t flags, Error **errp) { return -ENOSYS; } + +int kvm_set_memory_attributes_private(hwaddr start, uint64_t size) +{ + return -ENOSYS; +} + +int kvm_set_memory_attributes_shared(hwaddr start, uint64_t size) +{ + return -ENOSYS; +} diff --git a/docs/interop/firmware.json b/docs/interop/firmware.json index 421bee0e5ed..2850b91078d 100644 --- a/docs/interop/firmware.json +++ b/docs/interop/firmware.json @@ -161,6 +161,9 @@ # options related to this feature are documented in # "docs/system/i386/amd-memory-encryption.rst". # +# @arm-rme: The firmware supports running in a Realm, under the Arm Realm +# Management Extension (RME). +# # @intel-tdx: The firmware supports running under Intel Trust Domain # Extensions (TDX). # @@ -229,7 +232,7 @@ { 'enum' : 'FirmwareFeature', 'data' : [ 'acpi-s3', 'acpi-s4', 'amd-sev', 'amd-sev-es', 'amd-sev-snp', - 'intel-tdx', + 'arm-rme', 'intel-tdx', 'enrolled-keys', 'requires-smm', 'secure-boot', 'host-uefi-vars', 'verbose-dynamic', 'verbose-static' ] } diff --git a/docs/system/arm/virt.rst b/docs/system/arm/virt.rst index fbe3ca9e129..fb66667be21 100644 --- a/docs/system/arm/virt.rst +++ b/docs/system/arm/virt.rst @@ -218,10 +218,11 @@ dtb-randomness rng-seed and kaslr-seed nodes (in both "/chosen" and "/secure-chosen") to use for features like the random number generator and address space randomisation. The default is - ``on``. You will want to disable it if your trusted boot chain - will verify the DTB it is passed, since this option causes the - DTB to be non-deterministic. It would be the responsibility of - the firmware to come up with a seed and pass it on if it wants to. + ``off`` for confidential VMs, and ``on`` otherwise. You will want + to disable it if your trusted boot chain will verify the DTB it is + passed, since this option causes the DTB to be non-deterministic. + It would be the responsibility of the firmware to come up with a + seed and pass it on if it wants to. dtb-kaslr-seed A deprecated synonym for dtb-randomness. diff --git a/docs/system/confidential-guest-support.rst b/docs/system/confidential-guest-support.rst index 562a7c3c285..abb56923ad1 100644 --- a/docs/system/confidential-guest-support.rst +++ b/docs/system/confidential-guest-support.rst @@ -42,5 +42,6 @@ Currently supported confidential guest mechanisms are: * POWER Protected Execution Facility (PEF) (see :ref:`power-papr-protected-execution-facility-pef`) * s390x Protected Virtualization (PV) (see :doc:`s390x/protvirt`) * AWS Nitro Enclaves (see :doc:`nitro`) +* Arm Realm Management Extension (RME) Other mechanisms may be supported in future. diff --git a/hw/arm/boot.c b/hw/arm/boot.c index 878b80cf681..3c438a8ec85 100644 --- a/hw/arm/boot.c +++ b/hw/arm/boot.c @@ -581,7 +581,42 @@ int arm_load_dtb(hwaddr addr, const struct arm_boot_info *binfo, g_autoptr(MemoryDeviceInfoList) md_list = NULL; Error *err = NULL; - if (binfo->dtb_filename) { + if (binfo->dtb_filename && binfo->confidential) { + g_autofree char *filename = NULL; + int64_t file_size; + + /* + * If the user is providing a DTB for a confidential VM, it is already + * tailored to this configuration and measured. Load it as is, without + * any modification. + */ + filename = qemu_find_file(QEMU_FILE_TYPE_DTB, binfo->dtb_filename); + if (!filename) { + fprintf(stderr, "Couldn't open dtb file %s\n", + binfo->dtb_filename); + goto fail; + } + + file_size = get_image_size(filename, &err); + if (file_size <= 0 || file_size > INT_MAX) { + if (err) { + error_report_err(err); + err = NULL; + } else { + error_report("Invalid DTB file size for %s", filename); + } + goto fail; + } + + if (addr_limit > addr && file_size > addr_limit - addr) { + return 0; + } + + if (rom_add_file_fixed_as(filename, addr, -1, as) < 0) { + goto fail; + } + return file_size; + } else if (binfo->dtb_filename) { char *filename; filename = qemu_find_file(QEMU_FILE_TYPE_DTB, binfo->dtb_filename); if (!filename) { @@ -834,7 +869,13 @@ static void do_cpu_reset(void *opaque) if (cpu == info->primary_cpu) { AddressSpace *as = arm_boot_address_space(cpu, info); - cpu_set_pc(cs, info->loader_start); + if (info->confidential) { + assert(is_a64(env)); + env->xregs[0] = info->dtb_start; + cpu_set_pc(cs, info->entry); + } else { + cpu_set_pc(cs, info->loader_start); + } if (!have_dtb(info)) { set_kernel_args(info, as); @@ -924,7 +965,8 @@ static ssize_t arm_load_elf(struct arm_boot_info *info, uint64_t *pentry, } static uint64_t load_aarch64_image(const char *filename, hwaddr mem_base, - hwaddr *entry, AddressSpace *as) + hwaddr *entry, AddressSpace *as, + bool confidential) { const size_t max_bytes = LOAD_IMAGE_MAX_DECOMPRESSED_BYTES; hwaddr kernel_load_offset = KERNEL64_LOAD_ADDR; @@ -976,7 +1018,8 @@ static uint64_t load_aarch64_image(const char *filename, hwaddr mem_base, * bootloader, we can just load it starting at 2MB+offset rather * than 0MB + offset. */ - if (kernel_load_offset < BOOTLOADER_MAX_SIZE) { + if (kernel_load_offset < BOOTLOADER_MAX_SIZE && + !confidential) { kernel_load_offset += 2 * MiB; } } @@ -1060,7 +1103,8 @@ static void arm_setup_direct_kernel_boot(ARMCPU *cpu, } if (arm_feature(&cpu->env, ARM_FEATURE_AARCH64) && kernel_size < 0) { kernel_size = load_aarch64_image(info->kernel_filename, - info->loader_start, &entry, as); + info->loader_start, &entry, as, + info->confidential); is_linux = 1; if (kernel_size >= 0) { image_low_addr = entry; @@ -1201,8 +1245,11 @@ static void arm_setup_direct_kernel_boot(ARMCPU *cpu, fixupcontext[FIXUP_ENTRYPOINT_LO] = entry; fixupcontext[FIXUP_ENTRYPOINT_HI] = entry >> 32; - arm_write_bootloader("bootloader", as, info->loader_start, - primary_loader, fixupcontext); + /* Immediately jump to the kernel when guest is a Realm */ + if (!info->confidential) { + arm_write_bootloader("bootloader", as, info->loader_start, + primary_loader, fixupcontext); + } if (info->write_board_setup) { info->write_board_setup(cpu, info); @@ -1222,7 +1269,41 @@ static void arm_setup_direct_kernel_boot(ARMCPU *cpu, } } -static void arm_setup_firmware_boot(ARMCPU *cpu, struct arm_boot_info *info) +static void arm_setup_confidential_firmware_boot(ARMCPU *cpu, + struct arm_boot_info *info, + const char *firmware_filename) +{ + ssize_t fw_size; + g_autofree char *fname = NULL; + AddressSpace *as = arm_boot_address_space(cpu, info); + + if (!firmware_filename) { + error_report("a confidential Arm guest requires either a kernel " + "or a firmware image"); + exit(1); + } + + fname = qemu_find_file(QEMU_FILE_TYPE_BIOS, firmware_filename); + if (!fname) { + error_report("Could not find firmware image '%s'", firmware_filename); + exit(1); + } + + /* + * Load the firmware image in the Realm's address space. Mapping of the + * firmware area in the Realm's address space is done in function + * virt_confidential_firmware_init(). + */ + fw_size = load_image_targphys_as(fname, info->firmware_base, + info->firmware_max_size, as, NULL); + if (fw_size <= 0) { + error_report("could not load firmware '%s'", firmware_filename); + exit(1); + } +} + +static void arm_setup_firmware_boot(ARMCPU *cpu, struct arm_boot_info *info, + const char *firmware_filename) { /* Set up for booting firmware (which might load a kernel via fw_cfg) */ @@ -1273,6 +1354,10 @@ static void arm_setup_firmware_boot(ARMCPU *cpu, struct arm_boot_info *info) } } + if (info->confidential) { + arm_setup_confidential_firmware_boot(cpu, info, firmware_filename); + } + /* * We will start from address 0 (typically a boot ROM image) in the * same way as hardware. Leave env->boot_info NULL, so that @@ -1316,7 +1401,7 @@ void arm_load_kernel(ARMCPU *cpu, MachineState *ms, struct arm_boot_info *info) /* Load the kernel. */ if (!info->kernel_filename || info->firmware_loaded) { - arm_setup_firmware_boot(cpu, info); + arm_setup_firmware_boot(cpu, info, ms->firmware); } else { arm_setup_direct_kernel_boot(cpu, info); } diff --git a/hw/arm/virt.c b/hw/arm/virt.c index 137a0a7df77..b3f29686767 100644 --- a/hw/arm/virt.c +++ b/hw/arm/virt.c @@ -62,6 +62,7 @@ #include "hw/pci-host/gpex.h" #include "hw/pci-bridge/pci_expander_bridge.h" #include "hw/virtio/virtio-pci.h" +#include "hw/virtio/virtio-mmio.h" #include "hw/core/sysbus-fdt.h" #include "hw/core/platform-bus.h" #include "hw/core/qdev-properties.h" @@ -263,6 +264,17 @@ static const int a15irqmap[] = { [VIRT_PLATFORM_BUS] = 112, /* ...to 112 + PLATFORM_BUS_NUM_IRQS -1 */ }; +static bool virt_machine_is_confidential(VirtMachineState *vms) +{ + return MACHINE(vms)->cgs; +} + +static bool virt_dtb_randomness_enabled(VirtMachineState *vms) +{ + return vms->dtb_randomness && + (vms->dtb_randomness_set || !virt_machine_is_confidential(vms)); +} + static void create_randomness(MachineState *ms, const char *node) { struct { @@ -293,6 +305,7 @@ static bool ns_el2_virt_timer_present(void) static void create_fdt(VirtMachineState *vms) { + bool dtb_randomness = true; MachineState *ms = MACHINE(vms); int nb_numa_nodes = ms->numa_state->num_nodes; void *fdt = create_device_tree(&vms->fdt_size); @@ -302,6 +315,14 @@ static void create_fdt(VirtMachineState *vms) exit(1); } + /* + * Including random data in the DTB causes random intial measurement on CCA, + * so disable it for confidential VMs. + */ + if (!virt_dtb_randomness_enabled(vms)) { + dtb_randomness = false; + } + ms->fdt = fdt; /* Header */ @@ -323,13 +344,13 @@ static void create_fdt(VirtMachineState *vms) /* /chosen must exist for load_dtb to fill in necessary properties later */ qemu_fdt_add_subnode(fdt, "/chosen"); - if (vms->dtb_randomness) { + if (dtb_randomness) { create_randomness(ms, "/chosen"); } if (vms->secure) { qemu_fdt_add_subnode(fdt, "/secure-chosen"); - if (vms->dtb_randomness) { + if (dtb_randomness) { create_randomness(ms, "/secure-chosen"); } } @@ -1186,6 +1207,7 @@ static void create_gpio_devices(const VirtMachineState *vms, int gpio, static void create_virtio_devices(const VirtMachineState *vms) { + AddressSpace *dma_as = kvm_arm_rme_get_dma_as(); int i; hwaddr size = vms->memmap[VIRT_MMIO].size; MachineState *ms = MACHINE(vms); @@ -1220,9 +1242,18 @@ static void create_virtio_devices(const VirtMachineState *vms) for (i = 0; i < vms->virtio_transports; i++) { int irq = vms->irqmap[VIRT_MMIO] + i; hwaddr base = vms->memmap[VIRT_MMIO].base + i * size; + DeviceState *dev = qdev_new(TYPE_VIRTIO_MMIO); + SysBusDevice *s = SYS_BUS_DEVICE(dev); - sysbus_create_simple("virtio-mmio", base, - qdev_get_gpio_in(vms->gic, irq)); + if (dma_as) { + /* Legacy virtio-mmio cannot negotiate IOMMU_PLATFORM. */ + qdev_prop_set_bit(dev, "force-legacy", false); + virtio_mmio_set_dma_as(dev, dma_as); + } + + sysbus_realize_and_unref(s, &error_fatal); + sysbus_mmio_map(s, 0, base); + sysbus_connect_irq(s, 0, qdev_get_gpio_in(vms->gic, irq)); } /* We add dtb nodes in reverse order so that they appear in the finished @@ -1284,9 +1315,7 @@ static void virt_flash_create(VirtMachineState *vms) vms->flash[1] = virt_flash_create1(vms, "virt.flash1", "pflash1"); } -static void virt_flash_map1(PFlashCFI01 *flash, - hwaddr base, hwaddr size, - MemoryRegion *sysmem) +static void virt_flash_realize1(PFlashCFI01 *flash, hwaddr size) { DeviceState *dev = DEVICE(flash); @@ -1294,10 +1323,18 @@ static void virt_flash_map1(PFlashCFI01 *flash, assert(size / VIRT_FLASH_SECTOR_SIZE <= UINT32_MAX); qdev_prop_set_uint32(dev, "num-blocks", size / VIRT_FLASH_SECTOR_SIZE); sysbus_realize_and_unref(SYS_BUS_DEVICE(dev), &error_fatal); +} + +static void virt_flash_map1(PFlashCFI01 *flash, + hwaddr base, hwaddr size, + MemoryRegion *sysmem) +{ + SysBusDevice *sbd = SYS_BUS_DEVICE(flash); + + virt_flash_realize1(flash, size); memory_region_add_subregion(sysmem, base, - sysbus_mmio_get_region(SYS_BUS_DEVICE(dev), - 0)); + sysbus_mmio_get_region(sbd, 0)); } static void virt_flash_map(VirtMachineState *vms, @@ -1330,6 +1367,16 @@ static void virt_flash_fdt(VirtMachineState *vms, MachineState *ms = MACHINE(vms); char *nodename; + /* + * For Realms the firmware images are stored in the guest's address + * space. As such there is no need for flash configuration in the FDT. + * See function virt_confidential_firmware_init() and + * arm_setup_confidential_firmware_boot() for details. + */ + if (virt_machine_is_confidential(vms)) { + return; + } + if (sysmem == secure_sysmem) { /* Report both flash devices as a single node in the DT */ nodename = g_strdup_printf("/flash@%" PRIx64, flashbase); @@ -1365,6 +1412,32 @@ static void virt_flash_fdt(VirtMachineState *vms, } } +static bool virt_confidential_firmware_init(VirtMachineState *vms, + MemoryRegion *sysmem) +{ + MemoryRegion *fw_ram; + hwaddr fw_base = vms->memmap[VIRT_FLASH].base; + hwaddr fw_size = vms->memmap[VIRT_FLASH].size; + + if (!MACHINE(vms)->firmware) { + return false; + } + + assert(machine_require_guest_memfd(MACHINE(vms))); + + fw_ram = g_new(MemoryRegion, 1); + memory_region_init_ram_guest_memfd(fw_ram, NULL, "fw_ram", fw_size, + &error_fatal); + /* + * Map the guest's firmware image directly in its address space. + * Copying of the firmware image itself is done in function + * arm_setup_confidential_firmware_boot(). + */ + memory_region_add_subregion(sysmem, fw_base, fw_ram); + + return true; +} + static bool virt_firmware_init(VirtMachineState *vms, MemoryRegion *sysmem, MemoryRegion *secure_sysmem) @@ -1373,6 +1446,28 @@ static bool virt_firmware_init(VirtMachineState *vms, const char *bios_name; BlockBackend *pflash_blk0; + /* + * For a confidential VM, the firmware image and any boot information, + * including EFI variables, are stored in RAM in order to be measurable and + * private. Create a RAM region and load the firmware image there. + */ + if (virt_machine_is_confidential(vms)) { + hwaddr flashsize = vms->memmap[VIRT_FLASH].size / 2; + + for (i = 0; i < ARRAY_SIZE(vms->flash); i++) { + if (pflash_cfi01_get_blk(vms->flash[i]) || + drive_get(IF_PFLASH, 0, i)) { + error_report("pflash is not supported for Realm VMs; " + "use -bios to provide Realm firmware"); + exit(EXIT_FAILURE); + } + } + + virt_flash_realize1(vms->flash[0], flashsize); + virt_flash_realize1(vms->flash[1], flashsize); + return virt_confidential_firmware_init(vms, sysmem); + } + /* Map legacy -drive if=pflash to machine properties */ for (i = 0; i < ARRAY_SIZE(vms->flash); i++) { pflash_cfi01_legacy_drive(vms->flash[i], @@ -1664,6 +1759,12 @@ static void create_pcie(VirtMachineState *vms) pci->bypass_iommu = vms->default_bus_bypass_iommu; vms->bus = pci->bus; if (vms->bus) { + /* + * Some PCI devices query their IOMMU address space while they are + * realized. Install the Realm DMA address-space selector before + * creating even the default NIC so every endpoint sees it. + */ + kvm_arm_rme_init_gpa_space(vms->rme_ipa_bits, vms->bus); pci_init_nic_devices(pci->bus, mc->default_nic); } @@ -2318,6 +2419,23 @@ static void machvirt_init(MachineState *machine) unsigned int smp_cpus = machine->smp.cpus; unsigned int max_cpus = machine->smp.max_cpus; + if (virt_machine_is_confidential(vms) && !kvm_enabled()) { + error_report("Realm VMs require KVM acceleration"); + exit(EXIT_FAILURE); + } + + if (virt_machine_is_confidential(vms) && vms->iommu != VIRT_IOMMU_NONE) { + error_report("guest IOMMUs are not supported for Realm VMs"); + exit(EXIT_FAILURE); + } + + if (virt_machine_is_confidential(vms) && + vms->default_bus_bypass_iommu) { + error_report("default-bus-bypass-iommu is not supported for Realm " + "VMs"); + exit(EXIT_FAILURE); + } + possible_cpus = mc->possible_cpu_arch_ids(machine); /* @@ -2377,10 +2495,11 @@ static void machvirt_init(MachineState *machine) * if the guest has EL2 then we will use SMC as the conduit, * and otherwise we will use HVC (for backwards compatibility and * because if we're using KVM then we must use HVC). + * Realm guests must also use SMC. */ if (vms->secure && firmware_loaded) { vms->psci_conduit = QEMU_PSCI_CONDUIT_DISABLED; - } else if (vms->virt) { + } else if (vms->virt || virt_machine_is_confidential(vms)) { vms->psci_conduit = QEMU_PSCI_CONDUIT_SMC; } else { vms->psci_conduit = QEMU_PSCI_CONDUIT_HVC; @@ -2636,7 +2755,8 @@ static void machvirt_init(MachineState *machine) */ create_virtio_devices(vms); - vms->fw_cfg = create_fw_cfg(vms, &address_space_memory); + vms->fw_cfg = create_fw_cfg(vms, kvm_arm_rme_get_dma_as() ?: + &address_space_memory); rom_set_fw(vms->fw_cfg); create_platform_bus(vms); @@ -2659,7 +2779,10 @@ static void machvirt_init(MachineState *machine) vms->bootinfo.get_dtb = machvirt_dtb; vms->bootinfo.skip_dtb_autoload = true; vms->bootinfo.firmware_loaded = firmware_loaded; + vms->bootinfo.firmware_base = vms->memmap[VIRT_FLASH].base; + vms->bootinfo.firmware_max_size = vms->memmap[VIRT_FLASH].size; vms->bootinfo.psci_conduit = vms->psci_conduit; + vms->bootinfo.confidential = virt_machine_is_confidential(vms); arm_load_kernel(ARM_CPU(first_cpu), machine, &vms->bootinfo); vms->machine_done.notify = virt_machine_done; @@ -2906,6 +3029,11 @@ static bool virt_get_dtb_randomness(Object *obj, Error **errp) { VirtMachineState *vms = VIRT_MACHINE(obj); + /* + * Report the value the user set, not the effective one. A confidential + * VM defaults to no randomness (see virt_dtb_randomness_enabled()), but + * a getter that did not round-trip its setter would be surprising. + */ return vms->dtb_randomness; } @@ -2914,6 +3042,7 @@ static void virt_set_dtb_randomness(Object *obj, bool value, Error **errp) VirtMachineState *vms = VIRT_MACHINE(obj); vms->dtb_randomness = value; + vms->dtb_randomness_set = true; } static char *virt_get_oem_id(Object *obj, Error **errp) @@ -3212,6 +3341,13 @@ static void virt_machine_device_pre_plug_cb(HotplugHandler *hotplug_dev, { VirtMachineState *vms = VIRT_MACHINE(hotplug_dev); + if (virt_machine_is_confidential(vms) && + (object_dynamic_cast(OBJECT(dev), TYPE_VIRTIO_IOMMU_PCI) || + object_dynamic_cast(OBJECT(dev), TYPE_ARM_SMMUV3))) { + error_setg(errp, "guest IOMMUs are not supported for Realm VMs"); + return; + } + if (object_dynamic_cast(OBJECT(dev), TYPE_PC_DIMM)) { virt_memory_pre_plug(hotplug_dev, dev, errp); } else if (object_dynamic_cast(OBJECT(dev), TYPE_VIRTIO_MD_PCI)) { @@ -3431,14 +3567,33 @@ static int virt_kvm_type(MachineState *ms, const char *type_str) { VirtMachineState *vms = VIRT_MACHINE(ms); int max_vm_pa_size, requested_pa_size; + int rme_reserve_bit = 0; bool fixed_ipa; + int vm_type; + + vm_type = (ms->cgs ? QEMU_KVM_ARM_VM_TYPE_REALM : + QEMU_KVM_ARM_VM_TYPE_NORMAL); + + if (ms->cgs) { + /* + * With RME, the upper GPA bit differentiates Realm from NS memory. + * Reserve the upper bit to ensure that highmem devices will fit. + */ + rme_reserve_bit = 1; + } - max_vm_pa_size = kvm_arm_get_max_vm_ipa_size(ms, &fixed_ipa); + max_vm_pa_size = kvm_arm_get_max_vm_ipa_size(ms, &fixed_ipa) - + rme_reserve_bit; /* we freeze the memory map to compute the highest gpa */ virt_set_memmap(vms, max_vm_pa_size); - requested_pa_size = 64 - clz64(vms->highest_gpa); + if (ms->cgs) { + /* Keep the Realm shared IPA bit stable across PCI layout changes. */ + requested_pa_size = max_vm_pa_size + rme_reserve_bit; + } else { + requested_pa_size = 64 - clz64(vms->highest_gpa); + } /* * KVM requires the IPA size to be at least 32 bits. @@ -3447,19 +3602,26 @@ static int virt_kvm_type(MachineState *ms, const char *type_str) requested_pa_size = 32; } - if (requested_pa_size > max_vm_pa_size) { + if (requested_pa_size > max_vm_pa_size + rme_reserve_bit) { error_report("-m and ,maxmem option values " "require an IPA range (%d bits) larger than " "the one supported by the host (%d bits)", - requested_pa_size, max_vm_pa_size); + requested_pa_size, max_vm_pa_size + rme_reserve_bit); return -1; } + + vms->rme_ipa_bits = ms->cgs ? requested_pa_size : 0; + /* - * We return the requested PA log size, unless KVM only supports - * the implicit legacy 40b IPA setting, in which case the kvm_type - * must be 0. + * Return the requested PA log size unless KVM only supports the implicit + * legacy 40-bit IPA setting. In that case, leave the IPA-size bits clear + * while preserving the Realm VM-type field. */ - return fixed_ipa ? 0 : requested_pa_size; + if (fixed_ipa) { + return vm_type; + } + + return requested_pa_size | vm_type; } static int virt_get_physical_address_range(MachineState *ms, @@ -3764,13 +3926,14 @@ static void virt_instance_init(Object *obj) /* MTE is disabled by default. */ vms->mte = false; - /* Supply kaslr-seed and rng-seed by default */ + /* Supply kaslr-seed and rng-seed by default. */ vms->dtb_randomness = true; vms->irqmap = a15irqmap; vms->virtio_transports = NUM_VIRTIO_TRANSPORTS; + /* Machine properties must exist before machine options are parsed. */ virt_flash_create(vms); vms->oem_id = g_strndup(ACPI_BUILD_APPNAME6, 6); diff --git a/hw/core/loader.c b/hw/core/loader.c index 5cbfba0a86d..6ed609fc4ea 100644 --- a/hw/core/loader.c +++ b/hw/core/loader.c @@ -74,6 +74,8 @@ #endif static int roms_loaded; +static NotifierList rom_loader_notifier = + NOTIFIER_LIST_INITIALIZER(rom_loader_notifier); /* return the size or -1 if error */ int64_t get_image_size(const char *filename, Error **errp) @@ -1201,6 +1203,11 @@ MemoryRegion *rom_add_blob(const char *name, const void *blob, size_t len, return mr; } +void rom_add_load_notifier(Notifier *notifier) +{ + notifier_list_add(&rom_loader_notifier, notifier); +} + /* This function is specific for elf program because we don't need to allocate * all the rom. We just allocate the first part and the rest is just zeros. This * is why romsize and datasize are different. Also, this function takes its own @@ -1242,6 +1249,7 @@ ssize_t rom_add_option(const char *file, int32_t bootindex) static void rom_reset(void *unused) { Rom *rom; + RomLoaderNotifyData notify; QTAILQ_FOREACH(rom, &roms, next) { if (rom->fw_file) { @@ -1277,10 +1285,6 @@ static void rom_reset(void *unused) rom->romsize - rom->datasize, MEMTXATTRS_UNSPECIFIED); } - if (rom->isrom) { - /* rom needs to be written only once */ - rom_free_data(rom); - } /* * The rom loader is really on the same level as firmware in the guest * shadowing a ROM into RAM. Such a shadowing mechanism needs to ensure @@ -1290,6 +1294,21 @@ static void rom_reset(void *unused) address_space_flush_icache_range(rom->as, rom->addr, rom->datasize); trace_loader_write_rom(rom->name, rom->addr, rom->datasize, rom->isrom); + + if (!notifier_list_empty(&rom_loader_notifier)) { + notify = (RomLoaderNotifyData) { + .addr = rom->addr, + .len = rom->romsize, + .data_len = rom->datasize, + .data = rom->data, + }; + notifier_list_notify(&rom_loader_notifier, ¬ify); + } + + if (rom->isrom) { + /* rom needs to be written only once */ + rom_free_data(rom); + } } } diff --git a/hw/core/machine.c b/hw/core/machine.c index d4a6aefb282..344385b58fe 100644 --- a/hw/core/machine.c +++ b/hw/core/machine.c @@ -1321,6 +1321,11 @@ bool machine_require_guest_memfd(MachineState *machine) return machine->cgs && machine->cgs->require_guest_memfd; } +bool machine_has_assigned_device_memory(MachineState *machine) +{ + return machine->cgs && machine->cgs->assigned_device_memory; +} + static char *cpu_slot_to_string(const CPUArchId *cpu) { GString *s = g_string_new(NULL); diff --git a/hw/vfio/iommufd-stubs.c b/hw/vfio/iommufd-stubs.c index 0be52761753..7585668f35d 100644 --- a/hw/vfio/iommufd-stubs.c +++ b/hw/vfio/iommufd-stubs.c @@ -5,8 +5,10 @@ */ #include "qemu/osdep.h" +#include "qapi/error.h" #include "migration/cpr.h" #include "migration/vmstate.h" +#include "system/iommufd.h" const VMStateDescription vmstate_cpr_vfio_devices = { .name = CPR_STATE "/vfio devices", @@ -16,3 +18,61 @@ const VMStateDescription vmstate_cpr_vfio_devices = { VMSTATE_END_OF_LIST() } }; + +/* + * Arm RME device assignment. target/arm/kvm.c dispatches the RHI + * device-assignment hypercalls unconditionally, but the implementations live + * in iommufd.c which is only built for CONFIG_VFIO && CONFIG_IOMMUFD. Without + * IOMMUFD there can be no assigned device, so every lookup fails with -ENODEV + * and the guest sees RHI_DA_ERROR_INVALID_VDEV_ID. + */ +int iommufd_vdevice_register(VFIODevice *vbasedev, Error **errp) +{ + error_setg(errp, "IOMMUFD support is not compiled in"); + return -ENOSYS; +} + +int iommufd_tsm_bind(uint32_t rid) +{ + return -ENODEV; +} + +int iommufd_tsm_unbind(uint32_t rid) +{ + return -ENODEV; +} + +int iommufd_tsm_da_set_tdi_state_run(uint32_t rid) +{ + return -ENODEV; +} + +int iommufd_tsm_get_da_object_size(uint32_t rid, uint32_t object_type, + uint32_t *object_size) +{ + return -ENODEV; +} + +int iommufd_tsm_da_object_read(uint32_t rid, uint32_t object_type, + uint64_t offset, void *buf, uint32_t max_len, + uint32_t *resp_len) +{ + return -ENODEV; +} + +int iommufd_tsm_da_get_interface_report(uint32_t rid) +{ + return -ENODEV; +} + +int iommufd_tsm_da_get_measurement(uint32_t rid, + struct rhi_vdev_measurement_params *param) +{ + return -ENODEV; +} + +bool iommufd_tsm_dev_memmap_exit(uint32_t rid, uint64_t gpa_base, + uint64_t gpa_top, uint64_t pa_base) +{ + return false; +} diff --git a/hw/vfio/iommufd.c b/hw/vfio/iommufd.c index 50710640dae..9e5e0567049 100644 --- a/hw/vfio/iommufd.c +++ b/hw/vfio/iommufd.c @@ -15,12 +15,15 @@ #include #include +#include "hw/arm/rme-da.h" #include "hw/vfio/vfio-device.h" #include "qemu/error-report.h" #include "trace.h" #include "qapi/error.h" #include "system/iommufd.h" #include "hw/core/qdev.h" +#include "system/kvm.h" +#include "system/memory.h" #include "hw/vfio/vfio-cpr.h" #include "system/reset.h" #include "qemu/cutils.h" @@ -34,6 +37,308 @@ #define TYPE_HOST_IOMMU_DEVICE_IOMMUFD_VFIO \ TYPE_HOST_IOMMU_DEVICE_IOMMUFD "-vfio" +/* + * Arm SMMUv3 stream-table-entry bits used to build a stage-1 bypass STE for + * the nested (VIOMMU) domain in the RME device-assignment flow. + */ +#define VFIO_STRTAB_STE_0_V (1UL << 0) +#define VFIO_STRTAB_STE_0_CFG_BYPASS 4 + +static bool iommufd_tsm_vdevice_is_registered(VFIODevice *vbasedev, + uint32_t rid) +{ + return vbasedev && vbasedev->iommufd_vdevice && vbasedev->vdevice_id && + vbasedev->vdevice_rid == rid; +} + +static int iommufd_tsm_op(uint32_t rid, uint32_t op) +{ + VFIODevice *vbasedev = vfio_find_bdf(rid); + struct iommu_vdevice_tsm_op tsm_op; + + if (!iommufd_tsm_vdevice_is_registered(vbasedev, rid)) { + return -ENODEV; + } + + tsm_op.size = sizeof(struct iommu_vdevice_tsm_op); + tsm_op.flags = 0; + tsm_op.type = op; + tsm_op.vdevice_id = vbasedev->vdevice_id; + + if (ioctl(vbasedev->iommufd->fd, IOMMU_VDEVICE_TSM_OP, &tsm_op)) { + int ret = -errno; + + trace_iommufd_tsm_op_failed(rid, op, errno); + return ret; + } + + return 0; +} + +int iommufd_tsm_bind(uint32_t rid) +{ + return iommufd_tsm_op(rid, IOMMU_VDEVICE_TSM_BIND); +} + +int iommufd_tsm_unbind(uint32_t rid) +{ + return iommufd_tsm_op(rid, IOMMU_VDEVICE_TSM_UNBIND); +} + +static int iommufd_tsm_guest_request(VFIODevice *vbasedev, + uint32_t vdevice_id, uint32_t scope, + void *req, uint32_t req_len, + void *resp, uint32_t resp_len, + uint32_t *actual_resp_len) +{ + struct iommu_vdevice_tsm_guest_request guest_req = { + .size = sizeof(guest_req), + .vdevice_id = vdevice_id, + .scope = scope, + .req_uptr = (uintptr_t)req, + .req_len = req_len, + .resp_uptr = (uintptr_t)resp, + .resp_len = resp_len, + }; + int ret; + + ret = ioctl(vbasedev->iommufd->fd, IOMMU_VDEVICE_TSM_GUEST_REQUEST, + &guest_req); + if (ret < 0) { + int err = -errno; + + /* + * These requests are issued on behalf of the guest, so a failure is + * not necessarily a host problem and must not be reportable at will + * by the guest. Trace rather than warn_report(). + */ + trace_iommufd_tsm_guest_request_failed(vdevice_id, scope, errno); + return err; + } + + /* + * The return value is response residue when a response buffer is present, + * or unconsumed request bytes for a request-only operation. Every caller + * requires the complete request to be consumed, and a response residue + * cannot exceed the supplied response buffer. + */ + if ((uint32_t)ret > resp_len) { + trace_iommufd_tsm_guest_request_bad_residue(vdevice_id, ret, resp_len); + return -EIO; + } + if (actual_resp_len) { + *actual_resp_len = resp_len - ret; + } + + return 0; +} + +int iommufd_tsm_da_set_tdi_state_run(uint32_t rid) +{ + VFIODevice *vbasedev = vfio_find_bdf(rid); + struct arm64_vdev_set_tdi_state_guest_req req = { + .req_type = __RHI_DA_VDEV_SET_TDI_STATE, + .tdi_state = RHI_DA_TDI_CONFIG_RUN, + }; + + if (!iommufd_tsm_vdevice_is_registered(vbasedev, rid)) { + return -ENODEV; + } + + return iommufd_tsm_guest_request(vbasedev, vbasedev->vdevice_id, + PCI_TSM_REQ_STATE_CHANGE, + &req, sizeof(req), + NULL, 0, NULL); +} + +int iommufd_tsm_get_da_object_size(uint32_t rid, uint32_t object_type, + uint32_t *object_size) +{ + VFIODevice *vbasedev = vfio_find_bdf(rid); + struct arm64_vdev_object_size_guest_req req = { + .req_type = __RHI_DA_OBJECT_SIZE, + .object_type = object_type, + }; + uint32_t resp_len = 0; + int ret; + + if (!iommufd_tsm_vdevice_is_registered(vbasedev, rid)) { + return -ENODEV; + } + + ret = iommufd_tsm_guest_request(vbasedev, vbasedev->vdevice_id, + PCI_TSM_REQ_INFO, + &req, sizeof(req), + object_size, sizeof(*object_size), + &resp_len); + if (ret) { + return ret; + } + if (resp_len != sizeof(*object_size)) { + return -EINVAL; + } + return 0; +} + +int iommufd_tsm_da_object_read(uint32_t rid, uint32_t object_type, + uint64_t offset, void *buf, uint32_t max_len, + uint32_t *resp_len) +{ + VFIODevice *vbasedev = vfio_find_bdf(rid); + struct arm64_vdev_object_read_guest_req req = { + .req_type = __RHI_DA_OBJECT_READ, + .object_type = object_type, + .offset = offset, + }; + + if (!iommufd_tsm_vdevice_is_registered(vbasedev, rid)) { + return -ENODEV; + } + + return iommufd_tsm_guest_request(vbasedev, vbasedev->vdevice_id, + PCI_TSM_REQ_INFO, + &req, sizeof(req), + buf, max_len, resp_len); +} + +int iommufd_tsm_da_get_interface_report(uint32_t rid) +{ + VFIODevice *vbasedev = vfio_find_bdf(rid); + uint32_t req_type; + + if (!iommufd_tsm_vdevice_is_registered(vbasedev, rid)) { + return -ENODEV; + } + + req_type = __RHI_DA_VDEV_UPDATE_INTERFACE_REPORT; + return iommufd_tsm_guest_request(vbasedev, vbasedev->vdevice_id, + PCI_TSM_REQ_INFO, + &req_type, sizeof(req_type), + NULL, 0, NULL); +} + +int iommufd_tsm_da_get_measurement(uint32_t rid, + struct rhi_vdev_measurement_params *param) +{ + VFIODevice *vbasedev = vfio_find_bdf(rid); + struct arm64_vdev_device_measurement_guest_req req = { + .req_type = __RHI_DA_VDEV_UPDATE_MEASUREMENTS, + .flags = param->flags, + .nonce = (uintptr_t)¶m->nonce[0], + }; + + if (!iommufd_tsm_vdevice_is_registered(vbasedev, rid)) { + return -ENODEV; + } + + return iommufd_tsm_guest_request(vbasedev, vbasedev->vdevice_id, + PCI_TSM_REQ_INFO, + &req, sizeof(req), + NULL, 0, NULL); +} + +static bool iommufd_tsm_range_is_ram_device(hwaddr start, uint64_t size) +{ + MemoryRegion *mr; + hwaddr xlat; + hwaddr len; + + if (!size || size - 1 > HWADDR_MAX - start) { + return false; + } + + RCU_READ_LOCK_GUARD(); + while (size) { + len = size; + mr = address_space_translate(&address_space_memory, start, &xlat, + &len, false, + MEMTXATTRS_UNSPECIFIED); + if (!len || !memory_region_is_ram_device(mr)) { + return false; + } + + start += len; + size -= len; + } + + return true; +} + +bool iommufd_tsm_dev_memmap_exit(uint32_t rid, uint64_t gpa_base, + uint64_t gpa_top, uint64_t pa_base) +{ + VFIODevice *vbasedev = vfio_find_bdf(rid); + struct arm64_vdev_device_memmap_guest_req req = { + .req_type = __REC_DA_VDEV_MAP, + .gpa_base = gpa_base, + .gpa_top = gpa_top, + .pa_base = pa_base, + }; + uint64_t range_size; + bool ok; + + if (!iommufd_tsm_vdevice_is_registered(vbasedev, rid)) { + return false; + } + if (gpa_top <= gpa_base || + !QEMU_IS_ALIGNED(gpa_base, qemu_real_host_page_size()) || + !QEMU_IS_ALIGNED(gpa_top, qemu_real_host_page_size()) || + !QEMU_IS_ALIGNED(pa_base, qemu_real_host_page_size())) { + return false; + } + range_size = gpa_top - gpa_base; + + /* + * Treat the exit fields as untrusted. In particular, do not let a Realm + * apply VDEV attributes to ordinary guest RAM, or to a range which only + * partially resolves to a VFIO RAM-device region. + */ + if (!iommufd_tsm_range_is_ram_device(gpa_base, range_size)) { + return false; + } + + /* + * Mark the IPA window PRIVATE before the iommufd guest-request reaches + * the host TSM/iommufd and the kernel installs ASSIGNED-DEV S2 entries + * via realm_dev_mem_map(). This is the VDEV-side equivalent of the + * RIPAS-change handshake used for RAM (rec_exit_ripas_change -> + * KVM_EXIT_MEMORY_FAULT -> kvm_convert_memory) and gives the kernel's + * realm_clamp_order() the neighbour signal it needs to refuse a 2 MiB + * unprotected coalescing over VDEV-locked PAs. + * + * Set the attribute *before* the iommufd call: the + * kvm_arch_post_set_memory_attributes() callback sweeps away any stale + * NS S2 entries on those gfns (KVM_FILTER_SHARED) before the DEV + * mapping lands, closing the race against other vCPUs faulting on the + * unprotected alias in the window between VDEV_MAP exit and the host + * actually installing the DEV S2. + */ + if (kvm_set_memory_attributes_private(gpa_base, range_size)) { + return false; + } + + ok = iommufd_tsm_guest_request(vbasedev, vbasedev->vdevice_id, + PCI_TSM_REQ_STATE_CHANGE, + &req, sizeof(req), + NULL, 0, NULL) == 0; + if (!ok) { + /* + * KVM does not expose the previous attributes for this range. It may + * overlap an existing device mapping, so setting it SHARED here could + * destroy state which this request did not create. Continuing with a + * PRIVATE range but no matching host mapping is also unsafe. Stop the + * VM rather than guessing at the prior state. + */ + error_report("host rejected RME device mapping after PRIVATE " + "attributes were installed [0x%" PRIx64 + ", 0x%" PRIx64 ")", gpa_base, gpa_top); + exit(EXIT_FAILURE); + } + + trace_iommufd_tsm_dev_memmap(rid, gpa_base, gpa_top, pa_base, ok); + return ok; +} + static int iommufd_cdev_map(const VFIOContainer *bcontainer, hwaddr iova, uint64_t size, void *vaddr, bool readonly, MemoryRegion *mr) @@ -338,6 +643,66 @@ static int iommufd_cdev_attach_ioas_hwpt(VFIODevice *vbasedev, uint32_t id, return 0; } +int iommufd_vdevice_register(VFIODevice *vbasedev, Error **errp) +{ + IOMMUFDBackend *iommufd; + struct iommu_vdevice_alloc alloc_vdev; + VFIOPCIDevice *vdev; + int ret; + + if (!vbasedev || vbasedev->type != VFIO_DEVICE_TYPE_PCI) { + error_setg(errp, "vdevice registration is only supported for PCI"); + return -EINVAL; + } + if (!vbasedev->iommufd) { + error_setg(errp, "vdevice registration requires an IOMMUFD backend"); + return -EINVAL; + } + if (vbasedev->mdev) { + error_setg(errp, "vdevice registration is not supported for mdevs"); + return -EINVAL; + } + if (!vbasedev->hwpt || !vbasedev->hwpt->viommu_id || + !vbasedev->hwpt->nested_hwpt_id) { + error_setg(errp, + "vdevice registration requires a VIOMMU and nested HWPT"); + return -EINVAL; + } + + iommufd = vbasedev->iommufd; + alloc_vdev = (struct iommu_vdevice_alloc) { + .size = sizeof(alloc_vdev), + .viommu_id = vbasedev->hwpt->viommu_id, + .dev_id = vbasedev->devid, + }; + + vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev); + + /* Guest-visible RID: segment (0) in bits [31:16], BDF in bits [15:0]. */ + alloc_vdev.virt_id = pci_get_bdf(&vdev->parent_obj); + + do { + ret = ioctl(iommufd->fd, IOMMU_VDEVICE_ALLOC, &alloc_vdev); + } while (ret < 0 && errno == EINTR); + if (ret) { + ret = -errno; + error_setg_errno(errp, errno, "failed to allocate vdevice"); + return ret; + } + + ret = iommufd_cdev_attach_ioas_hwpt(vbasedev, + vbasedev->hwpt->nested_hwpt_id, errp); + if (ret) { + iommufd_backend_free_id(iommufd, alloc_vdev.out_vdevice_id); + return ret; + } + + vbasedev->vdevice_id = alloc_vdev.out_vdevice_id; + vbasedev->vdevice_rid = alloc_vdev.virt_id; + + return 0; +} + static bool iommufd_cdev_detach_ioas_hwpt(VFIODevice *vbasedev, Error **errp) { int iommufd = vbasedev->iommufd->fd; @@ -345,8 +710,13 @@ static bool iommufd_cdev_detach_ioas_hwpt(VFIODevice *vbasedev, Error **errp) .argsz = sizeof(detach_data), .flags = 0, }; + int ret; - if (ioctl(vbasedev->fd, VFIO_DEVICE_DETACH_IOMMUFD_PT, &detach_data)) { + do { + ret = ioctl(vbasedev->fd, VFIO_DEVICE_DETACH_IOMMUFD_PT, + &detach_data); + } while (ret < 0 && errno == EINTR); + if (ret) { error_setg_errno(errp, errno, "detach %s failed", vbasedev->name); return false; } @@ -355,6 +725,93 @@ static bool iommufd_cdev_detach_ioas_hwpt(VFIODevice *vbasedev, Error **errp) return true; } +/* + * Allocate the nested-translation topology used for RME device assignment: a + * stage-2 nesting-parent HWPT, a VIOMMU on top of it, and a stage-1 bypass + * HWPT nested under the VIOMMU. Returns the parent VFIOIOASHwpt (with + * nested_hwpt_id populated) on success, or NULL with @errp set on failure. + */ +static VFIOIOASHwpt * +iommufd_cdev_alloc_viommu_hwpt(VFIODevice *vbasedev, + VFIOIOMMUFDContainer *container, + Error **errp) +{ + IOMMUFDBackend *iommufd = vbasedev->iommufd; + struct iommu_hwpt_arm_smmuv3 bypass_ste = { + .ste = { + VFIO_STRTAB_STE_0_V | (VFIO_STRTAB_STE_0_CFG_BYPASS << 1), + 0, + }, + }; + struct iommu_viommu_alloc alloc_viommu = { + .size = sizeof(alloc_viommu), + .flags = 0, + .type = IOMMU_VIOMMU_TYPE_ARM_REALM_SMMUV3, + .dev_id = vbasedev->devid, + }; + VFIOIOASHwpt *hwpt; + Error *detach_err = NULL; + uint32_t hwpt_id; + bool parent_attached = false; + bool viommu_allocated = false; + int ret; + + if (!iommufd_backend_alloc_hwpt(iommufd, vbasedev->devid, + container->ioas_id, + IOMMU_HWPT_ALLOC_NEST_PARENT, + IOMMU_HWPT_DATA_NONE, 0, NULL, + &hwpt_id, errp)) { + return NULL; + } + + hwpt = g_malloc0(sizeof(*hwpt)); + hwpt->hwpt_id = hwpt_id; + hwpt->hwpt_flags = IOMMU_HWPT_ALLOC_NEST_PARENT; + QLIST_INIT(&hwpt->device_list); + + ret = iommufd_cdev_attach_ioas_hwpt(vbasedev, hwpt->hwpt_id, errp); + if (ret) { + goto err_free; + } + parent_attached = true; + + alloc_viommu.hwpt_id = hwpt->hwpt_id; + if (ioctl(iommufd->fd, IOMMU_VIOMMU_ALLOC, &alloc_viommu)) { + error_setg_errno(errp, errno, "failed to allocate VIOMMU"); + goto err_free; + } + viommu_allocated = true; + + if (!iommufd_backend_alloc_hwpt(iommufd, vbasedev->devid, + alloc_viommu.out_viommu_id, 0, + IOMMU_HWPT_DATA_ARM_SMMUV3, + sizeof(bypass_ste), &bypass_ste, + &hwpt_id, errp)) { + goto err_free; + } + + hwpt->viommu_id = alloc_viommu.out_viommu_id; + hwpt->nested_hwpt_id = hwpt_id; + + return hwpt; + +err_free: + if (viommu_allocated) { + iommufd_backend_free_id(container->be, + alloc_viommu.out_viommu_id); + } + if (parent_attached && + !iommufd_cdev_detach_ioas_hwpt(vbasedev, &detach_err)) { + error_reportf_err(detach_err, + "failed to detach device while unwinding: "); + /* Continuing would discard ownership of an attached kernel HWPT. */ + exit(EXIT_FAILURE); + } + iommufd_backend_free_id(container->be, hwpt->hwpt_id); + g_free(hwpt); + return NULL; +} + static bool iommufd_cdev_autodomains_get(VFIODevice *vbasedev, VFIOIOMMUFDContainer *container, Error **errp) @@ -372,6 +829,13 @@ static bool iommufd_cdev_autodomains_get(VFIODevice *vbasedev, /* Try to find a domain */ QLIST_FOREACH(hwpt, &container->hwpt_list, next) { + bool hwpt_has_vdevice = hwpt->viommu_id && hwpt->nested_hwpt_id; + + /* Never mix regular devices and Realm vDevices in one HWPT. */ + if (vbasedev->iommufd_vdevice != hwpt_has_vdevice) { + continue; + } + if (!cpr_is_incoming()) { ret = iommufd_cdev_attach_ioas_hwpt(vbasedev, hwpt->hwpt_id, errp); } else if (vbasedev->cpr.hwpt_id == hwpt->hwpt_id) { @@ -403,67 +867,75 @@ static bool iommufd_cdev_autodomains_get(VFIODevice *vbasedev, } } - /* - * This is quite early and VFIO Migration state isn't yet fully - * initialized, thus rely only on IOMMU hardware capabilities as to - * whether IOMMU dirty tracking is going to be requested. Later - * vfio_migration_realize() may decide to use VF dirty tracking - * instead. - */ - if (!iommufd_backend_get_device_info(vbasedev->iommufd, vbasedev->devid, - &type, &caps, sizeof(caps), &hw_caps, - NULL, errp)) { - return false; - } + if (vbasedev->iommufd_vdevice) { + hwpt = iommufd_cdev_alloc_viommu_hwpt(vbasedev, container, errp); + if (!hwpt) { + return false; + } + } else { + /* + * This is quite early and VFIO Migration state isn't yet fully + * initialized, thus rely only on IOMMU hardware capabilities as to + * whether IOMMU dirty tracking is going to be requested. Later + * vfio_migration_realize() may decide to use VF dirty tracking + * instead. + */ + if (!iommufd_backend_get_device_info(vbasedev->iommufd, + vbasedev->devid, &type, &caps, + sizeof(caps), &hw_caps, + NULL, errp)) { + return false; + } - viommu_nesting = vfio_device_get_viommu_flags_want_nesting(vbasedev); - viommu_nesting_dirty = - vfio_device_get_viommu_flags_want_nesting_dirty(vbasedev); + viommu_nesting = vfio_device_get_viommu_flags_want_nesting(vbasedev); + viommu_nesting_dirty = + vfio_device_get_viommu_flags_want_nesting_dirty(vbasedev); - if (hw_caps & IOMMU_HW_CAP_DIRTY_TRACKING) { - if (!viommu_nesting || viommu_nesting_dirty) { - flags |= IOMMU_HWPT_ALLOC_DIRTY_TRACKING; + if (hw_caps & IOMMU_HW_CAP_DIRTY_TRACKING) { + if (!viommu_nesting || viommu_nesting_dirty) { + flags |= IOMMU_HWPT_ALLOC_DIRTY_TRACKING; + } } - } - /* - * If vIOMMU requests VFIO's cooperation to create nesting parent HWPT, - * force to create it so that it could be reused by vIOMMU to create - * nested HWPT. - */ - if (viommu_nesting) { - flags |= IOMMU_HWPT_ALLOC_NEST_PARENT; + /* + * If vIOMMU requests VFIO's cooperation to create nesting parent HWPT, + * force to create it so that it could be reused by vIOMMU to create + * nested HWPT. + */ + if (viommu_nesting) { + flags |= IOMMU_HWPT_ALLOC_NEST_PARENT; - if (vfio_device_get_host_iommu_quirk_bypass_ro(vbasedev, type, - &caps, sizeof(caps))) { - bcontainer->bypass_ro = true; + if (vfio_device_get_host_iommu_quirk_bypass_ro(vbasedev, type, + &caps, + sizeof(caps))) { + bcontainer->bypass_ro = true; + } } - } - if (cpr_is_incoming()) { - hwpt_id = vbasedev->cpr.hwpt_id; - goto skip_alloc; - } + if (cpr_is_incoming()) { + hwpt_id = vbasedev->cpr.hwpt_id; + goto skip_alloc; + } - if (!iommufd_backend_alloc_hwpt(iommufd, vbasedev->devid, - container->ioas_id, flags, - IOMMU_HWPT_DATA_NONE, 0, NULL, - &hwpt_id, errp)) { - return false; - } + if (!iommufd_backend_alloc_hwpt(iommufd, vbasedev->devid, + container->ioas_id, flags, + IOMMU_HWPT_DATA_NONE, 0, NULL, + &hwpt_id, errp)) { + return false; + } - ret = iommufd_cdev_attach_ioas_hwpt(vbasedev, hwpt_id, errp); - if (ret) { - iommufd_backend_free_id(container->be, hwpt_id); - return false; - } + ret = iommufd_cdev_attach_ioas_hwpt(vbasedev, hwpt_id, errp); + if (ret) { + iommufd_backend_free_id(container->be, hwpt_id); + return false; + } skip_alloc: - hwpt = g_malloc0(sizeof(*hwpt)); - hwpt->hwpt_id = hwpt_id; - hwpt->hwpt_flags = flags; - QLIST_INIT(&hwpt->device_list); - + hwpt = g_malloc0(sizeof(*hwpt)); + hwpt->hwpt_id = hwpt_id; + hwpt->hwpt_flags = flags; + QLIST_INIT(&hwpt->device_list); + } vbasedev->hwpt = hwpt; vbasedev->cpr.hwpt_id = hwpt->hwpt_id; vbasedev->iommu_dirty_tracking = iommufd_hwpt_dirty_tracking(hwpt); @@ -489,6 +961,17 @@ static void iommufd_cdev_autodomains_put(VFIODevice *vbasedev, if (QLIST_EMPTY(&hwpt->device_list)) { QLIST_REMOVE(hwpt, next); + /* + * Tear down the RME device-assignment nested topology (if any) in the + * reverse order it was allocated: stage-1 bypass HWPT, then the VIOMMU, + * then the stage-2 nesting-parent HWPT below. + */ + if (hwpt->nested_hwpt_id) { + iommufd_backend_free_id(container->be, hwpt->nested_hwpt_id); + } + if (hwpt->viommu_id) { + iommufd_backend_free_id(container->be, hwpt->viommu_id); + } iommufd_backend_free_id(container->be, hwpt->hwpt_id); g_free(hwpt); } @@ -517,10 +1000,19 @@ static void iommufd_cdev_detach_container(VFIODevice *vbasedev, error_report_err(err); } + /* + * Destroy the VDEVICE before the VIOMMU it belongs to (freed in + * iommufd_cdev_autodomains_put() below), as required by the iommufd UAPI. + */ + if (vbasedev->iommufd_vdevice && vbasedev->vdevice_id) { + iommufd_backend_free_id(container->be, vbasedev->vdevice_id); + vbasedev->vdevice_id = 0; + vbasedev->vdevice_rid = 0; + } + if (vbasedev->hwpt) { iommufd_cdev_autodomains_put(vbasedev, container); } - } static void iommufd_cdev_container_destroy(VFIOIOMMUFDContainer *container) diff --git a/hw/vfio/listener.c b/hw/vfio/listener.c index 8981348277f..31b26cab5c9 100644 --- a/hw/vfio/listener.c +++ b/hw/vfio/listener.c @@ -76,6 +76,32 @@ static bool vfio_log_sync_needed(const VFIOContainer *bcontainer) return true; } +/* + * Look up an assigned PCI device by its guest-visible Routing ID: PCI segment + * in bits [31:16], BDF in bits [15:0]. Only segment 0 is modelled today, so + * anything above bit 15 never matches. + */ +VFIODevice *vfio_find_bdf(uint32_t rid) +{ + VFIOPCIDevice *pcidev; + VFIODevice *vbasedev; + + /* vfio_device_list insertion and removal are protected by the BQL. */ + assert(bql_locked()); + + QLIST_FOREACH(vbasedev, &vfio_device_list, global_next) { + if (vbasedev->type != VFIO_DEVICE_TYPE_PCI) { + continue; + } + pcidev = container_of(vbasedev, VFIOPCIDevice, vbasedev); + if ((uint32_t)pci_get_bdf(&pcidev->parent_obj) == rid) { + return vbasedev; + } + } + + return NULL; +} + static bool vfio_listener_skipped_section(MemoryRegionSection *section, bool bypass_ro) { @@ -127,6 +153,31 @@ static MemoryRegion *vfio_translate_iotlb(IOMMUTLBEntry *iotlb, hwaddr *xlat_p, return mr; } +static bool vfio_iommu_notify_error(VFIOGuestIOMMU *giommu, Error *err) +{ + VFIOContainer *bcontainer = giommu->bcontainer; + IOMMUMemoryRegionClass *imrc = + IOMMU_MEMORY_REGION_GET_CLASS(giommu->iommu_mr); + + if (!imrc->require_notifier_success) { + return false; + } + + error_prepend(&err, "IOMMU region %s: ", + memory_region_name(MEMORY_REGION(giommu->iommu_mr))); + if (!bcontainer->initialized) { + if (!bcontainer->error) { + error_propagate(&bcontainer->error, err); + } else { + error_free(err); + } + return true; + } + + error_report_err(err); + hw_error("vfio: IOMMU mapping update failed, unable to continue"); +} + static void vfio_iommu_map_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb) { VFIOGuestIOMMU *giommu = container_of(n, VFIOGuestIOMMU, n); @@ -145,6 +196,9 @@ static void vfio_iommu_map_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb) error_setg(&local_err, "Wrong target AS \"%s\", only system memory is allowed", iotlb->target_as->name ? iotlb->target_as->name : "none"); + if (vfio_iommu_notify_error(giommu, local_err)) { + return; + } if (migration_is_running()) { migration_file_set_error(-EINVAL, local_err); } else { @@ -160,7 +214,9 @@ static void vfio_iommu_map_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb) mr = vfio_translate_iotlb(iotlb, &xlat, &local_err); if (!mr) { - error_report_err(local_err); + if (!vfio_iommu_notify_error(giommu, local_err)) { + error_report_err(local_err); + } goto out; } vaddr = memory_region_get_ram_ptr(mr) + xlat; @@ -177,10 +233,14 @@ static void vfio_iommu_map_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb) iotlb->addr_mask + 1, vaddr, read_only, mr); if (ret) { - error_report("vfio_container_dma_map(%p, 0x%"HWADDR_PRIx", " - "0x%"HWADDR_PRIx", %p) = %d (%s)", - bcontainer, iova, - iotlb->addr_mask + 1, vaddr, ret, strerror(-ret)); + error_setg(&local_err, + "vfio_container_dma_map(%p, 0x%" HWADDR_PRIx ", " + "0x%" HWADDR_PRIx ", %p) = %d (%s)", + bcontainer, iova, iotlb->addr_mask + 1, vaddr, ret, + strerror(-ret)); + if (!vfio_iommu_notify_error(giommu, local_err)) { + error_report_err(local_err); + } } } else { ret = vfio_container_dma_unmap(bcontainer, iova, @@ -191,6 +251,9 @@ static void vfio_iommu_map_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb) "0x%"HWADDR_PRIx") = %d (%s)", bcontainer, iova, iotlb->addr_mask + 1, ret, strerror(-ret)); + if (vfio_iommu_notify_error(giommu, local_err)) { + goto out; + } if (migration_is_running()) { migration_file_set_error(ret, local_err); } else { diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c index d8ab00ee8fa..4cbb068e80f 100644 --- a/hw/vfio/pci.c +++ b/hw/vfio/pci.c @@ -23,6 +23,7 @@ #include #include +#include "hw/core/boards.h" #include "hw/core/hw-error.h" #include "hw/core/iommu.h" #include "hw/cxl/cxl_component.h" @@ -58,6 +59,11 @@ static KVMRouteChange vfio_route_change; static void vfio_disable_interrupts(VFIOPCIDevice *vdev); static void vfio_mmap_set_enabled(VFIOPCIDevice *vdev, bool enabled); static void vfio_msi_disable_common(VFIOPCIDevice *vdev); +#ifdef CONFIG_IOMMUFD +static bool vfio_register_bdf(PCIDevice *pci_dev, bool require_running, + Error **errp); +static void vfio_register_bdf_deferred(PCIDevice *pci_dev); +#endif /* Create new or reuse existing eventfd */ static bool vfio_notifier_init(VFIOPCIDevice *vdev, EventNotifier *e, @@ -1399,6 +1405,11 @@ uint32_t vfio_pci_read_config(PCIDevice *pdev, uint32_t addr, int len) VFIODevice *vbasedev = &vdev->vbasedev; uint32_t emu_bits = 0, emu_val = 0, phys_val = 0, val; +#ifdef CONFIG_IOMMUFD + /* Attempt registering device info to kernel. No-op if done already */ + vfio_register_bdf_deferred(pdev); +#endif + memcpy(&emu_bits, vdev->emulated_config_bits + addr, len); emu_bits = le32_to_cpu(emu_bits); @@ -1436,6 +1447,11 @@ void vfio_pci_write_config(PCIDevice *pdev, trace_vfio_pci_write_config(vdev->vbasedev.name, addr, val, len); +#ifdef CONFIG_IOMMUFD + /* Attempt registering device info to kernel. No-op if done already */ + vfio_register_bdf_deferred(pdev); +#endif + /* Write everything to VFIO, let it filter out what we can't write */ ret = vfio_pci_config_space_write(vdev, addr, len, &val_le); if (ret != len) { @@ -3253,6 +3269,19 @@ bool vfio_pci_populate_device(VFIOPCIDevice *vdev, Error **errp) void vfio_pci_put_device(VFIOPCIDevice *vdev) { +#ifdef CONFIG_IOMMUFD + /* + * Only TYPE_VFIO_PCI installs this handler (see vfio_pci_init()). Sibling + * subclasses of TYPE_VFIO_PCI_DEVICE such as TYPE_VFIO_USER_PCI have their + * own instance_init and share this teardown path, so ->vmstate may be NULL + * here. vfio_user_pci_realize() also calls us on its error path, before + * vfio_user_pci_finalize() calls us again, so clear the pointer too. + */ + if (vdev->vmstate) { + qemu_del_vm_change_state_handler(vdev->vmstate); + vdev->vmstate = NULL; + } +#endif vfio_display_finalize(vdev); vfio_bars_finalize(vdev); @@ -4075,6 +4104,23 @@ static void vfio_pci_realize(PCIDevice *pdev, Error **errp) trace_vfio_mdev(vbasedev->name, vbasedev->mdev); +#ifdef CONFIG_IOMMUFD + if (vbasedev->iommufd_vdevice && !vbasedev->iommufd) { + error_setg(errp, "iommufd-vdevice requires an iommufd backend"); + goto error; + } + if (vbasedev->iommufd_vdevice && + !machine_has_assigned_device_memory(current_machine)) { + error_setg(errp, "iommufd-vdevice requires an Arm Realm with " + "RME device-assignment support"); + goto error; + } + if (vbasedev->iommufd_vdevice && vbasedev->mdev) { + error_setg(errp, "iommufd-vdevice is not supported for mdevs"); + goto error; + } +#endif + if (vbasedev->ram_block_discard_allowed && !vbasedev->mdev) { error_setg(errp, "x-balloon-allowed only potentially compatible " "with mdev devices"); @@ -4171,6 +4217,17 @@ static void vfio_pci_realize(PCIDevice *pdev, Error **errp) } } +#ifdef CONFIG_IOMMUFD + /* + * Register while realize can still propagate failure if the guest-visible + * BDF is already stable. Devices behind an unnumbered bridge are deferred + * until the guest assigns the secondary bus number. + */ + if (!vfio_register_bdf(pdev, false, errp)) { + goto out_deregister; + } +#endif + vfio_pci_register_err_notifier(vdev); vfio_pci_register_req_notifier(vdev); vfio_setup_resetfn_quirk(vdev); @@ -4275,6 +4332,91 @@ static void vfio_pci_reset(DeviceState *dev) vfio_pci_post_reset(vdev); } +#ifdef CONFIG_IOMMUFD +static bool vfio_register_bdf(PCIDevice *pci_dev, bool require_running, + Error **errp) +{ + VFIOPCIDevice *vdev = VFIO_PCI_DEVICE(pci_dev); + PCIBus *bus = pci_get_bus(pci_dev); + + /* + * The root-bus number is fixed before realize. A secondary bus can remain + * zero until the guest programs its bridge, so defer registration there. + */ + if (!vdev->vbasedev.iommufd_vdevice) { + return true; + } + if (vdev->has_info_set) { + uint32_t current_rid = pci_get_bdf(pci_dev); + + if (vdev->vbasedev.vdevice_rid != current_rid) { + error_setg(errp, + "guest BDF for IOMMUFD vdevice %s changed " + "from %02x:%02x.%x to %02x:%02x.%x", + vdev->vbasedev.name, + PCI_BUS_NUM(vdev->vbasedev.vdevice_rid), + PCI_SLOT(vdev->vbasedev.vdevice_rid), + PCI_FUNC(vdev->vbasedev.vdevice_rid), + PCI_BUS_NUM(current_rid), PCI_SLOT(current_rid), + PCI_FUNC(current_rid)); + return false; + } + return true; + } + if ((require_running && !vdev->is_running) || + (!pci_bus_is_root(bus) && + (pci_bus_num(bus) == 0))) { + return true; + } + + if (iommufd_vdevice_register(&vdev->vbasedev, errp)) { + return false; + } + + vdev->has_info_set = true; + return true; +} + +static void vfio_register_bdf_deferred(PCIDevice *pci_dev) +{ + VFIOPCIDevice *vdev = VFIO_PCI_DEVICE(pci_dev); + Error *err = NULL; + + /* Config accesses also occur inside realize, before failure can unwind. */ + if (!DEVICE(pci_dev)->realized || vdev->info_set_failed) { + return; + } + + if (!vfio_register_bdf(pci_dev, true, &err)) { + /* + * The requested vDevice mode cannot operate without registration. + * Latch only while the asynchronous VM-stop request is pending, so + * repeated config accesses cannot reissue the ioctl or flood the log. + */ + vdev->info_set_failed = true; + error_reportf_err(err, "Failed to register IOMMUFD vdevice for %s: ", + vdev->vbasedev.name); + qemu_system_vmstop_request_prepare(); + qemu_system_vmstop_request(RUN_STATE_INTERNAL_ERROR); + } +} + +static void vfio_register_bdf_notifier(void *opaque, bool running, + RunState state) +{ + VFIOPCIDevice *vdev = VFIO_PCI_DEVICE(opaque); + + vdev->is_running = running; + if (!running) { + /* The asynchronous stop has consumed the deferred-failure latch. */ + vdev->info_set_failed = false; + return; + } + + vfio_register_bdf_deferred(opaque); +} +#endif + static void vfio_pci_init(Object *obj) { PCIDevice *pci_dev = PCI_DEVICE(obj); @@ -4289,6 +4431,12 @@ static void vfio_pci_init(Object *obj) vdev->host.slot = ~0U; vdev->host.function = ~0U; +#ifdef CONFIG_IOMMUFD + vdev->is_running = runstate_is_running(); + vdev->has_info_set = false; + vdev->info_set_failed = false; +#endif + vfio_device_init(vbasedev, VFIO_DEVICE_TYPE_PCI, &vfio_pci_ops, DEVICE(vdev), false); @@ -4304,6 +4452,11 @@ static void vfio_pci_init(Object *obj) * may be lost. */ pci_dev->cap_present |= QEMU_PCI_SKIP_RESET_ON_CPR; + +#ifdef CONFIG_IOMMUFD + vdev->vmstate = qemu_add_vm_change_state_handler_prio( + vfio_register_bdf_notifier, obj, 10); +#endif } static void vfio_pci_device_class_init(ObjectClass *klass, const void *data) @@ -4401,6 +4554,8 @@ static const Property vfio_pci_properties[] = { #ifdef CONFIG_IOMMUFD DEFINE_PROP_LINK("iommufd", VFIOPCIDevice, vbasedev.iommufd, TYPE_IOMMUFD_BACKEND, IOMMUFDBackend *), + DEFINE_PROP_BOOL("iommufd-vdevice", VFIOPCIDevice, vbasedev.iommufd_vdevice, + false), #endif DEFINE_PROP_BOOL("skip-vsc-check", VFIOPCIDevice, skip_vsc_check, true), DEFINE_PROP_UINT16("x-vpasid-cap-offset", VFIOPCIDevice, @@ -4530,6 +4685,10 @@ static void vfio_pci_class_init(ObjectClass *klass, const void *data) object_class_property_set_description(klass, /* 9.0 */ "iommufd", "Set host IOMMUFD backend device"); + object_class_property_set_description(klass, /* 10.0 */ + "iommufd-vdevice", + "Register the device as an IOMMUFD " + "vDevice"); #endif object_class_property_set_description(klass, /* 9.1 */ "x-device-dirty-page-tracking", diff --git a/hw/vfio/pci.h b/hw/vfio/pci.h index 72a187b7452..abc205ea1b0 100644 --- a/hw/vfio/pci.h +++ b/hw/vfio/pci.h @@ -203,10 +203,14 @@ struct VFIOPCIDevice { bool clear_parent_atomics_on_exit; bool skip_vsc_check; uint16_t vpasid_cap_offset; + bool has_info_set; + bool info_set_failed; + bool is_running; VFIODisplay *dpy; Notifier irqchip_change_notifier; VFIOPCICPR cpr; VFIOCXL cxl; + VMChangeStateEntry *vmstate; }; /* Use uin32_t for vendor & device so PCI_ANY_ID expands and cannot match hw */ diff --git a/hw/vfio/trace-events b/hw/vfio/trace-events index 740283afbe9..51fa8431ceb 100644 --- a/hw/vfio/trace-events +++ b/hw/vfio/trace-events @@ -189,6 +189,10 @@ iommufd_cdev_fail_attach_existing_container(const char *msg) " %s" iommufd_cdev_alloc_ioas(int iommufd, int ioas_id) " [iommufd=%d] new IOMMUFD container with ioasid=%d" iommufd_cdev_device_info(char *name, int devfd, int num_irqs, int num_regions, int flags) " %s (%d) num_irqs=%d num_regions=%d flags=%d" iommufd_cdev_pci_hot_reset_dep_devices(int domain, int bus, int slot, int function, int dev_id) "\t%04x:%02x:%02x.%x devid %d" +iommufd_tsm_op_failed(uint32_t rid, uint32_t op, int err) " rid 0x%04x op %u failed, errno=%d" +iommufd_tsm_guest_request_failed(uint32_t vdevice_id, uint32_t scope, int err) " vdevice_id %u scope %u failed, errno=%d" +iommufd_tsm_guest_request_bad_residue(uint32_t vdevice_id, int residue, uint32_t resp_len) " vdevice_id %u returned invalid residue %d for a %u-byte response" +iommufd_tsm_dev_memmap(uint32_t rid, uint64_t gpa_base, uint64_t gpa_top, uint64_t pa_base, bool ok) " rid 0x%04x [0x%"PRIx64", 0x%"PRIx64") pa 0x%"PRIx64" accepted=%d" # cpr-iommufd.c vfio_cpr_find_device(uint32_t ioas_id, int devid, uint32_t hwpt_id) "ioas_id %u, devid %d, hwpt_id %u" diff --git a/hw/vfio/vfio-iommufd.h b/hw/vfio/vfio-iommufd.h index 6b28e1ff7bb..45f6ac2de4d 100644 --- a/hw/vfio/vfio-iommufd.h +++ b/hw/vfio/vfio-iommufd.h @@ -15,6 +15,8 @@ typedef struct VFIODevice VFIODevice; typedef struct VFIOIOASHwpt { uint32_t hwpt_id; + uint32_t nested_hwpt_id; + uint32_t viommu_id; uint32_t hwpt_flags; QLIST_HEAD(, VFIODevice) device_list; QLIST_ENTRY(VFIOIOASHwpt) next; diff --git a/hw/virtio/virtio-mmio.c b/hw/virtio/virtio-mmio.c index 742ca3d3e4d..4e5fda13895 100644 --- a/hw/virtio/virtio-mmio.c +++ b/hw/virtio/virtio-mmio.c @@ -28,6 +28,7 @@ #include "migration/qemu-file-types.h" #include "qemu/host-utils.h" #include "qemu/module.h" +#include "system/address-spaces.h" #include "system/kvm.h" #include "system/replay.h" #include "hw/virtio/virtio-mmio.h" @@ -761,6 +762,27 @@ static void virtio_mmio_pre_plugged(DeviceState *d, Error **errp) } } +static AddressSpace *virtio_mmio_get_dma_as(DeviceState *d) +{ + VirtIOMMIOProxy *proxy = VIRTIO_MMIO(d); + + return proxy->dma_as ?: &address_space_memory; +} + +static bool virtio_mmio_iommu_enabled(DeviceState *d) +{ + return virtio_mmio_get_dma_as(d) != &address_space_memory; +} + +void virtio_mmio_set_dma_as(DeviceState *d, AddressSpace *dma_as) +{ + VirtIOMMIOProxy *proxy = VIRTIO_MMIO(d); + + assert(!d->realized); + assert(dma_as); + proxy->dma_as = dma_as; +} + /* virtio-mmio device */ static const Property virtio_mmio_properties[] = { @@ -868,6 +890,8 @@ static void virtio_mmio_bus_class_init(ObjectClass *klass, const void *data) k->ioeventfd_assign = virtio_mmio_ioeventfd_assign; k->pre_plugged = virtio_mmio_pre_plugged; k->vmstate_change = virtio_mmio_vmstate_change; + k->get_dma_as = virtio_mmio_get_dma_as; + k->iommu_enabled = virtio_mmio_iommu_enabled; k->has_variable_vring_alignment = true; bus_class->max_dev = 1; bus_class->get_dev_path = virtio_mmio_bus_get_dev_path; diff --git a/include/hw/arm/boot.h b/include/hw/arm/boot.h index a2e22bda8a5..8a1bb110693 100644 --- a/include/hw/arm/boot.h +++ b/include/hw/arm/boot.h @@ -112,6 +112,10 @@ struct arm_boot_info { */ bool firmware_loaded; + /* Used when loading firmware into RAM */ + hwaddr firmware_base; + hwaddr firmware_max_size; + /* Address at which board specific loader/setup code exists. If enabled, * this code-blob will run before anything else. It must return to the * caller via the link register. There is no stack set up. Enabled by @@ -135,6 +139,11 @@ struct arm_boot_info { /* CPU having load the kernel and that should be the first to boot. */ ARMCPU *primary_cpu; + + /* + * Confidential guest boot loads everything into RAM so it can be measured. + */ + bool confidential; }; /** diff --git a/include/hw/arm/rme-da.h b/include/hw/arm/rme-da.h new file mode 100644 index 00000000000..c5f0d9d0459 --- /dev/null +++ b/include/hw/arm/rme-da.h @@ -0,0 +1,133 @@ +/* + * Arm RME device-assignment ABI used by QEMU + * + * Copyright (c) 2026 NVIDIA Corporation + * + * SPDX-License-Identifier: GPL-2.0-or-later + * + * These definitions mirror the RHI device-assignment interface and the + * request payloads consumed by the Arm RME IOMMUFD implementation. Keep the + * values and layouts synchronized with the corresponding Arm specifications + * and kernel interfaces. + */ + +#ifndef HW_ARM_RME_DA_H +#define HW_ARM_RME_DA_H + +#define ARM_SMCCC_FAST_CALL 1U +#define ARM_SMCCC_SMC_64 1U +#define ARM_SMCCC_TYPE_SHIFT 31 +#define ARM_SMCCC_CALL_CONV_SHIFT 30 +#define ARM_SMCCC_OWNER_MASK 0x3fU +#define ARM_SMCCC_OWNER_SHIFT 24 +#define ARM_SMCCC_FUNC_MASK 0xffffU +#define ARM_SMCCC_OWNER_STANDARD_HYP 5U + +#define ARM_SMCCC_CALL_VAL(type, calling_convention, owner, func_num) \ + (((type) << ARM_SMCCC_TYPE_SHIFT) | \ + ((calling_convention) << ARM_SMCCC_CALL_CONV_SHIFT) | \ + (((owner) & ARM_SMCCC_OWNER_MASK) << ARM_SMCCC_OWNER_SHIFT) | \ + ((func_num) & ARM_SMCCC_FUNC_MASK)) + +#define SMC_RHI_CALL(func) \ + ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, ARM_SMCCC_SMC_64, \ + ARM_SMCCC_OWNER_STANDARD_HYP, (func)) + +#define RHI_DA_FEATURE_OBJECT_SIZE 0b000001 +#define RHI_DA_FEATURE_OBJECT_READ 0b000010 +#define RHI_DA_FEATURE_VDEV_CONTINUE 0b000100 +#define RHI_DA_FEATURE_VDEV_GET_MEASUREMENT 0b001000 +#define RHI_DA_FEATURE_VDEV_GET_INTF_REPORT 0b010000 +#define RHI_DA_FEATURE_VDEV_SET_TDI_STATE 0b100000 + +#define RHI_DA_BASE_FEATURE \ + (RHI_DA_FEATURE_OBJECT_SIZE | RHI_DA_FEATURE_OBJECT_READ | \ + RHI_DA_FEATURE_VDEV_GET_INTF_REPORT | \ + RHI_DA_FEATURE_VDEV_GET_MEASUREMENT | \ + RHI_DA_FEATURE_VDEV_SET_TDI_STATE) + +#define RHI_DA_VERSION SMC_RHI_CALL(0x004a) +#define RHI_DA_FEATURES SMC_RHI_CALL(0x004b) +#define RHI_DA_OBJECT_SIZE SMC_RHI_CALL(0x004c) +#define RHI_DA_OBJECT_READ SMC_RHI_CALL(0x004d) +#define RHI_DA_VDEV_GET_MEASUREMENTS SMC_RHI_CALL(0x0052) +#define RHI_DA_VDEV_GET_INTERFACE_REPORT SMC_RHI_CALL(0x0053) +#define RHI_DA_VDEV_SET_TDI_STATE SMC_RHI_CALL(0x0054) + +#define RHI_DA_VERSION_1_0 0x10000 + +#define SMCCC_RET_NOT_SUPPORTED -1 + +#define RHI_DA_SUCCESS 0x0 +#define RHI_DA_INCOMPLETE 0x1 +#define RHI_DA_ERROR_DATA_NOT_AVAILABLE 0x2 +#define RHI_DA_ERROR_INVALID_VDEV_ID 0x3 +#define RHI_DA_ERROR_INVALID_OBJECT 0x4 +#define RHI_DA_ERROR_INPUT 0x5 +#define RHI_DA_ERROR_DEVICE 0x6 +#define RHI_DA_ERROR_INVALID_OFFSET 0x7 +#define RHI_DA_ERROR_ACCESS_FAILED 0x8 +#define RHI_DA_ERROR_BUSY 0x9 + +#define RHI_DA_TDI_CONFIG_UNLOCKED 0x0 +#define RHI_DA_TDI_CONFIG_LOCKED 0x1 +#define RHI_DA_TDI_CONFIG_RUN 0x2 + +#define PCI_TSM_REQ_INFO 0 +#define PCI_TSM_REQ_STATE_CHANGE 1 + +/* Guest request operation numbers from the RME device-assignment ABI. */ +#define __RHI_DA_OBJECT_SIZE 0x1 +#define __RHI_DA_OBJECT_READ 0x2 +#define __RHI_DA_VDEV_UPDATE_INTERFACE_REPORT 0x3 +#define __RHI_DA_VDEV_UPDATE_MEASUREMENTS 0x4 +#define __REC_DA_VDEV_MAP 0x5 +#define __RHI_DA_VDEV_SET_TDI_STATE 0x6 + +struct rhi_vdev_measurement_params { + union { + uint64_t flags; + uint8_t padding0[256]; + }; + uint8_t nonce[32]; +}; + +struct arm64_vdev_object_size_guest_req { + uint32_t req_type; + uint32_t object_type; +}; + +struct arm64_vdev_object_read_guest_req { + uint32_t req_type; + uint32_t object_type; + uint64_t offset QEMU_ALIGNED(8); +}; + +struct arm64_vdev_device_measurement_guest_req { + uint32_t req_type; + uint32_t reserved; + uint64_t flags QEMU_ALIGNED(8); + uint64_t nonce QEMU_ALIGNED(8); +}; + +struct arm64_vdev_device_memmap_guest_req { + uint32_t req_type; + uint32_t reserved; + uint64_t gpa_base QEMU_ALIGNED(8); + uint64_t gpa_top QEMU_ALIGNED(8); + uint64_t pa_base QEMU_ALIGNED(8); +}; + +struct arm64_vdev_set_tdi_state_guest_req { + uint32_t req_type; + uint32_t tdi_state; +}; + +QEMU_BUILD_BUG_ON(sizeof(struct rhi_vdev_measurement_params) != 288); +QEMU_BUILD_BUG_ON(sizeof(struct arm64_vdev_object_size_guest_req) != 8); +QEMU_BUILD_BUG_ON(sizeof(struct arm64_vdev_object_read_guest_req) != 16); +QEMU_BUILD_BUG_ON(sizeof(struct arm64_vdev_device_measurement_guest_req) != 24); +QEMU_BUILD_BUG_ON(sizeof(struct arm64_vdev_device_memmap_guest_req) != 32); +QEMU_BUILD_BUG_ON(sizeof(struct arm64_vdev_set_tdi_state_guest_req) != 8); + +#endif /* HW_ARM_RME_DA_H */ diff --git a/include/hw/arm/virt.h b/include/hw/arm/virt.h index ed8e7c4b4ab..8df2236741f 100644 --- a/include/hw/arm/virt.h +++ b/include/hw/arm/virt.h @@ -157,6 +157,7 @@ struct VirtMachineState { bool ras; bool mte; bool dtb_randomness; + bool dtb_randomness_set; bool second_ns_uart_present; OnOffAuto acpi; VirtGICType gic_version; @@ -176,6 +177,7 @@ struct VirtMachineState { int psci_conduit; uint8_t virtio_transports; hwaddr highest_gpa; + uint8_t rme_ipa_bits; DeviceState *gic; DeviceState *acpi_dev; Notifier powerdown_notifier; diff --git a/include/hw/core/boards.h b/include/hw/core/boards.h index b8dad0a1074..2e7c099f42e 100644 --- a/include/hw/core/boards.h +++ b/include/hw/core/boards.h @@ -43,6 +43,7 @@ int machine_phandle_start(MachineState *machine); bool machine_dump_guest_core(MachineState *machine); bool machine_mem_merge(MachineState *machine); bool machine_require_guest_memfd(MachineState *machine); +bool machine_has_assigned_device_memory(MachineState *machine); HotpluggableCPUList *machine_query_hotpluggable_cpus(MachineState *machine); void machine_set_cpu_numa_node(MachineState *machine, const CpuInstanceProperties *props, diff --git a/include/hw/core/loader.h b/include/hw/core/loader.h index d9431e8a8d1..121cca9bcbe 100644 --- a/include/hw/core/loader.h +++ b/include/hw/core/loader.h @@ -342,6 +342,25 @@ void *rom_ptr_for_as(AddressSpace *as, hwaddr addr, size_t size); ssize_t rom_add_vga(const char *file); ssize_t rom_add_option(const char *file, int32_t bootindex); +typedef struct RomLoaderNotifyData { + /* Address of the blob in guest memory */ + hwaddr addr; + /* Length of the blob, including any zero-filled tail */ + size_t len; + /* Length of the data physically present in @data */ + size_t data_len; + /* File-backed data, valid only for the duration of the notification */ + uint8_t *data; +} RomLoaderNotifyData; + +/** + * rom_add_load_notifier - Add a notifier for loaded images + * + * Add a notifier that will be invoked with a RomLoaderNotifyData structure for + * each blob loaded into guest memory, after the blob is loaded. + */ +void rom_add_load_notifier(Notifier *notifier); + /* This is the usual maximum in uboot, so if a uImage overflows this, it would * overflow on real hardware too. */ #define UBOOT_MAX_DECOMPRESSED_BYTES (64 << 20) diff --git a/include/hw/vfio/vfio-device.h b/include/hw/vfio/vfio-device.h index a95c5bf5030..ecd6168a8c4 100644 --- a/include/hw/vfio/vfio-device.h +++ b/include/hw/vfio/vfio-device.h @@ -85,6 +85,9 @@ typedef struct VFIODevice { bool iommu_dirty_tracking; HostIOMMUDevice *hiod; int devid; + uint32_t vdevice_id; + uint32_t vdevice_rid; + bool iommufd_vdevice; IOMMUFDBackend *iommufd; VFIOIOASHwpt *hwpt; QLIST_ENTRY(VFIODevice) hwpt_next; @@ -171,6 +174,9 @@ VFIODevice *vfio_get_vfio_device(Object *obj); typedef QLIST_HEAD(VFIODeviceList, VFIODevice) VFIODeviceList; extern VFIODeviceList vfio_device_list; +/* Caller must hold the BQL while using the returned device. */ +VFIODevice *vfio_find_bdf(uint32_t rid); + #ifdef CONFIG_LINUX /* * How devices communicate with the server. The default option is through diff --git a/include/hw/virtio/virtio-mmio.h b/include/hw/virtio/virtio-mmio.h index 1644d098105..fcafc769ea1 100644 --- a/include/hw/virtio/virtio-mmio.h +++ b/include/hw/virtio/virtio-mmio.h @@ -64,6 +64,7 @@ struct VirtIOMMIOProxy { uint32_t host_features_sel; uint32_t guest_features_sel; uint32_t guest_page_shift; + AddressSpace *dma_as; /* virtio-bus */ VirtioBusState bus; /* Fields only used for non-legacy (v2) devices */ @@ -71,4 +72,6 @@ struct VirtIOMMIOProxy { VirtIOMMIOQueue vqs[VIRTIO_QUEUE_MAX]; }; +void virtio_mmio_set_dma_as(DeviceState *dev, AddressSpace *dma_as); + #endif diff --git a/include/system/confidential-guest-support.h b/include/system/confidential-guest-support.h index 5dca7173088..5763a38aad8 100644 --- a/include/system/confidential-guest-support.h +++ b/include/system/confidential-guest-support.h @@ -69,6 +69,12 @@ struct ConfidentialGuestSupport { */ bool require_guest_memfd; + /* + * True when the confidential-guest implementation assigns device-memory + * ranges and needs KVM memory attributes maintained across VFIO slots. + */ + bool assigned_device_memory; + /* * ready: flag set by CGS initialization code once it's ready to * start executing instructions in a potentially-secure diff --git a/include/system/iommufd.h b/include/system/iommufd.h index da68ba0037b..3defad848c8 100644 --- a/include/system/iommufd.h +++ b/include/system/iommufd.h @@ -19,6 +19,8 @@ #include "system/ram_addr.h" #include "system/host_iommu_device.h" +typedef struct VFIODevice VFIODevice; + #define TYPE_IOMMUFD_BACKEND "iommufd" OBJECT_DECLARE_TYPE(IOMMUFDBackend, IOMMUFDBackendClass, IOMMUFD_BACKEND) @@ -129,6 +131,29 @@ bool iommufd_backend_invalidate_cache(IOMMUFDBackend *be, uint32_t id, bool iommufd_change_process_capable(IOMMUFDBackend *be); bool iommufd_change_process(IOMMUFDBackend *be, Error **errp); +struct rhi_vdev_measurement_params; + +/* + * Arm RME device assignment. @rid is the guest-visible Routing ID of the + * assigned device: PCI segment in bits [31:16], BDF in bits [15:0]. All of + * these return 0 (or true) on success, and -ENODEV if @rid does not name a + * device that has been registered as an iommufd vDevice. + */ +int iommufd_vdevice_register(VFIODevice *vbasedev, Error **errp); +int iommufd_tsm_bind(uint32_t rid); +int iommufd_tsm_unbind(uint32_t rid); +int iommufd_tsm_da_set_tdi_state_run(uint32_t rid); +int iommufd_tsm_get_da_object_size(uint32_t rid, uint32_t object_type, + uint32_t *object_size); +int iommufd_tsm_da_object_read(uint32_t rid, uint32_t object_type, + uint64_t offset, void *buf, uint32_t max_len, + uint32_t *resp_len); +int iommufd_tsm_da_get_interface_report(uint32_t rid); +int iommufd_tsm_da_get_measurement(uint32_t rid, + struct rhi_vdev_measurement_params *param); +bool iommufd_tsm_dev_memmap_exit(uint32_t rid, uint64_t gpa_base, + uint64_t gpa_top, uint64_t pa_base); + #define TYPE_HOST_IOMMU_DEVICE_IOMMUFD TYPE_HOST_IOMMU_DEVICE "-iommufd" OBJECT_DECLARE_TYPE(HostIOMMUDeviceIOMMUFD, HostIOMMUDeviceIOMMUFDClass, HOST_IOMMU_DEVICE_IOMMUFD) diff --git a/include/system/kvm.h b/include/system/kvm.h index 5fa33eddda3..4494236da9f 100644 --- a/include/system/kvm.h +++ b/include/system/kvm.h @@ -553,6 +553,8 @@ bool kvm_dirty_ring_enabled(void); uint32_t kvm_dirty_ring_size(void); +bool kvm_guest_state_protected(void); + void kvm_mark_guest_state_protected(void); /** diff --git a/include/system/memory.h b/include/system/memory.h index 3812cfdf262..a2a30cc53ad 100644 --- a/include/system/memory.h +++ b/include/system/memory.h @@ -489,6 +489,15 @@ struct IOMMUMemoryRegionClass { */ void (*replay)(IOMMUMemoryRegion *iommu, IOMMUNotifier *notifier); + /** + * @require_notifier_success: + * + * The IOMMU's translation state must not diverge from mappings installed + * by notifier consumers. A consumer that cannot apply a MAP or UNMAP must + * reject device realization or stop the VM. + */ + bool require_notifier_success; + /** * @get_attr: * diff --git a/include/system/ramblock.h b/include/system/ramblock.h index 4435f8d55fe..8a9425c1d4d 100644 --- a/include/system/ramblock.h +++ b/include/system/ramblock.h @@ -15,6 +15,7 @@ #define SYSTEM_RAMBLOCK_H #include "qemu/rcu.h" +#include "qemu/thread.h" #include "system/ram_addr.h" #include "system/ramlist.h" #include "system/hostmem.h" @@ -96,6 +97,12 @@ struct RamBlockAttributes { RAMBlock *ram_block; + /* + * Protects the bitmap and listener list. Listener callbacks run while + * this lock is held to order notifications and replay with state changes. + */ + QemuMutex lock; + /* 1-setting of the bitmap represents ram is populated (shared) */ unsigned bitmap_size; unsigned long *bitmap; diff --git a/linux-headers/asm-arm64/kvm.h b/linux-headers/asm-arm64/kvm.h index 46ffbddab54..4bca1482c94 100644 --- a/linux-headers/asm-arm64/kvm.h +++ b/linux-headers/asm-arm64/kvm.h @@ -208,6 +208,15 @@ struct kvm_arm_counter_offset { __u64 reserved; }; +#define KVM_ARM_RMI_POPULATE_FLAGS_MEASURE (1 << 0) +struct kvm_arm_rmi_populate { + __u64 base; + __u64 size; + __u64 source_uaddr; + __u32 flags; + __u32 reserved; +}; + #define KVM_ARM_TAGS_TO_GUEST 0 #define KVM_ARM_TAGS_FROM_GUEST 1 @@ -543,4 +552,7 @@ struct reg_mask_range { #endif +/* arm specific KVM_EXIT_ARM64_TIO nr value*/ +#define RMI_EXIT_VDEV_MAP 0x08 + #endif /* __ARM_KVM_H__ */ diff --git a/linux-headers/linux/iommufd.h b/linux-headers/linux/iommufd.h index 384183a4039..ec3d865996f 100644 --- a/linux-headers/linux/iommufd.h +++ b/linux-headers/linux/iommufd.h @@ -57,6 +57,8 @@ enum { IOMMUFD_CMD_IOAS_CHANGE_PROCESS = 0x92, IOMMUFD_CMD_VEVENTQ_ALLOC = 0x93, IOMMUFD_CMD_HW_QUEUE_ALLOC = 0x94, + IOMMUFD_CMD_VDEVICE_TSM_OP = 0x95, + IOMMUFD_CMD_VDEVICE_TSM_GUEST_REQUEST = 0x96, }; /** @@ -1014,6 +1016,7 @@ enum iommu_viommu_type { IOMMU_VIOMMU_TYPE_DEFAULT = 0, IOMMU_VIOMMU_TYPE_ARM_SMMUV3 = 1, IOMMU_VIOMMU_TYPE_TEGRA241_CMDQV = 2, + IOMMU_VIOMMU_TYPE_ARM_REALM_SMMUV3 = 3, }; /** @@ -1299,4 +1302,27 @@ struct iommu_hw_queue_alloc { __aligned_u64 length; }; #define IOMMU_HW_QUEUE_ALLOC _IO(IOMMUFD_TYPE, IOMMUFD_CMD_HW_QUEUE_ALLOC) + +struct iommu_vdevice_tsm_op { + __u32 size; + __u32 type; + __u32 flags; + __u32 vdevice_id; +}; +#define IOMMU_VDEVICE_TSM_OP _IO(IOMMUFD_TYPE, IOMMUFD_CMD_VDEVICE_TSM_OP) +#define IOMMU_VDEVICE_TSM_BIND 0x1 +#define IOMMU_VDEVICE_TSM_UNBIND 0x2 + +struct iommu_vdevice_tsm_guest_request { + __u32 size; + __u32 vdevice_id; + __u32 scope; + __u32 req_len; + __u32 resp_len; + __u32 __reserved; + __aligned_u64 req_uptr; + __aligned_u64 resp_uptr; +}; +#define IOMMU_VDEVICE_TSM_GUEST_REQUEST _IO(IOMMUFD_TYPE, IOMMUFD_CMD_VDEVICE_TSM_GUEST_REQUEST) + #endif diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h index a4ab42dcba9..39e45933e36 100644 --- a/linux-headers/linux/kvm.h +++ b/linux-headers/linux/kvm.h @@ -180,6 +180,7 @@ struct kvm_xen_exit { #define KVM_EXIT_MEMORY_FAULT 39 #define KVM_EXIT_TDX 40 #define KVM_EXIT_ARM_SEA 41 +#define KVM_EXIT_ARM64_TIO 44 /* For KVM_EXIT_INTERNAL_ERROR */ /* Emulate instruction failed. */ @@ -474,6 +475,16 @@ struct kvm_run { __u64 gva; __u64 gpa; } arm_sea; + /* KVM_EXIT_ARM64_TIO*/ + struct { + __u64 flags; + __u64 nr; + __u64 vdev_id; + __u64 gpa_base; + __u64 gpa_top; /* input and output */ + __u64 pa_base; + __u64 response; + } cca_exit; /* Fix the size of the union. */ char padding[256]; }; @@ -677,10 +688,20 @@ struct kvm_enable_cap { * address size for the VM. Bits[7-0] are reserved for the guest * PA size shift (i.e, log2(PA_Size)). For backward compatibility, * value 0 implies the default IPA size, 40bits. + * + * Bits[11-8] are reserved for the VM type, bit[31] for protected VMs. */ #define KVM_VM_TYPE_ARM_IPA_SIZE_MASK 0xffULL #define KVM_VM_TYPE_ARM_IPA_SIZE(x) \ ((x) & KVM_VM_TYPE_ARM_IPA_SIZE_MASK) +#define KVM_VM_TYPE_ARM_SHIFT 8 +#define KVM_VM_TYPE_ARM_MASK (0xfULL << KVM_VM_TYPE_ARM_SHIFT) +#define KVM_VM_TYPE_ARM(_type) \ + (((_type) << KVM_VM_TYPE_ARM_SHIFT) & KVM_VM_TYPE_ARM_MASK) +#define KVM_VM_TYPE_ARM_NORMAL KVM_VM_TYPE_ARM(0) +#define KVM_VM_TYPE_ARM_REALM KVM_VM_TYPE_ARM(1) +#define KVM_VM_TYPE_ARM_PROTECTED (1UL << 31) + /* * ioctls for /dev/kvm fds: */ @@ -702,6 +723,8 @@ struct kvm_enable_cap { #define KVM_GET_EMULATED_CPUID _IOWR(KVMIO, 0x09, struct kvm_cpuid2) #define KVM_GET_MSR_FEATURE_INDEX_LIST _IOWR(KVMIO, 0x0a, struct kvm_msr_list) +#define KVM_ARM_RMI_POPULATE _IOWR(KVMIO, 0xd7, struct kvm_arm_rmi_populate) + /* * Extension capability list. */ @@ -966,6 +989,12 @@ struct kvm_enable_cap { #define KVM_CAP_GUEST_MEMFD_FLAGS 244 #define KVM_CAP_ARM_SEA_TO_USER 245 #define KVM_CAP_S390_USER_OPEREXEC 246 +#define KVM_CAP_S390_KEYOP 247 +#define KVM_CAP_S390_VSIE_ESAMODE 248 +#define KVM_CAP_S390_HPAGE_2G 249 +#define KVM_CAP_PPC_COMPAT_CAPS 250 +#define KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES 251 +#define KVM_CAP_ARM_RMI 252 struct kvm_irq_routing_irqchip { __u32 irqchip; diff --git a/qapi/misc-arm.json b/qapi/misc-arm.json index 4dc66d00e5c..fa8114f3b59 100644 --- a/qapi/misc-arm.json +++ b/qapi/misc-arm.json @@ -89,3 +89,47 @@ ## { 'enum': 'OasMode', 'data': [ 'auto', '32', '36', '40', '42', '44', '48', '52', '56' ] } + +## +# @CcaMeasurementAlgo: +# +# A measurement algorithm supported by Arm CCA. +# +# @measurement-algo: name of the measurement algorithm +# +# Since: 11.0 +## +{ 'struct': 'CcaMeasurementAlgo', + 'data': { 'measurement-algo': 'str' } } + +## +# @CcaCapability: +# +# Capabilities reported for Arm CCA. +# +# @sections: supported measurement algorithms +# +# Since: 11.0 +## +{ 'struct': 'CcaCapability', + 'data': { 'sections': ['CcaMeasurementAlgo'] } } + +## +# @query-cca-capabilities: +# +# Report Arm CCA capabilities. This command is available on Arm +# system emulators. It reports an error when KVM does not advertise +# Realm support. +# +# Returns: the Arm CCA capabilities +# +# Since: 11.0 +# +# .. qmp-example:: +# +# -> { "execute": "query-cca-capabilities" } +# <- { "return": { "sections": [ +# { "measurement-algo": "sha256" } +# ] } } +## +{ 'command': 'query-cca-capabilities', 'returns': 'CcaCapability' } diff --git a/qapi/qom.json b/qapi/qom.json index b86eef19c5f..0c37c611a47 100644 --- a/qapi/qom.json +++ b/qapi/qom.json @@ -1194,6 +1194,16 @@ 'data': { '*cpu-affinity': ['uint16'], '*node-affinity': ['uint16'] } } +## +# @RmeGuestProperties: +# +# Properties for rme-guest objects. +# +# Since: 11.0 +## +{ 'struct': 'RmeGuestProperties', + 'data': {} } + ## # @ObjectType: # @@ -1249,6 +1259,7 @@ { 'name': 'pr-manager-helper', 'if': 'CONFIG_LINUX' }, 'qtest', + 'rme-guest', 'rng-builtin', 'rng-egd', { 'name': 'rng-random', @@ -1327,6 +1338,7 @@ 'pr-manager-helper': { 'type': 'PrManagerHelperProperties', 'if': 'CONFIG_LINUX' }, 'qtest': 'QtestProperties', + 'rme-guest': 'RmeGuestProperties', 'rng-builtin': 'RngProperties', 'rng-egd': 'RngEgdProperties', 'rng-random': { 'type': 'RngRandomProperties', diff --git a/stubs/monitor-arm-gic.c b/stubs/monitor-arm-gic.c index b3429243ef8..e936347d60f 100644 --- a/stubs/monitor-arm-gic.c +++ b/stubs/monitor-arm-gic.c @@ -10,3 +10,9 @@ GICCapabilityList *qmp_query_gic_capabilities(Error **errp) error_setg(errp, "GIC hardware is not available on this target"); return NULL; } + +CcaCapability *qmp_query_cca_capabilities(Error **errp) +{ + error_setg(errp, "CCA is not available on this target"); + return NULL; +} diff --git a/system/ram-block-attributes.c b/system/ram-block-attributes.c index 630b0fda126..5f2ef45e6b7 100644 --- a/system/ram-block-attributes.c +++ b/system/ram-block-attributes.c @@ -37,16 +37,21 @@ static bool ram_block_attributes_rdm_is_populated(const RamDiscardManager *rdm, const MemoryRegionSection *section) { - const RamBlockAttributes *attr = RAM_BLOCK_ATTRIBUTES(rdm); + RamBlockAttributes *attr = RAM_BLOCK_ATTRIBUTES(rdm); const size_t block_size = ram_block_attributes_get_block_size(); const uint64_t first_bit = section->offset_within_region / block_size; const uint64_t last_bit = first_bit + int128_get64(section->size) / block_size - 1; unsigned long first_discarded_bit; + bool populated; + qemu_mutex_lock(&attr->lock); first_discarded_bit = find_next_zero_bit(attr->bitmap, last_bit + 1, first_bit); - return first_discarded_bit > last_bit; + populated = first_discarded_bit > last_bit; + qemu_mutex_unlock(&attr->lock); + + return populated; } typedef int (*ram_block_attributes_section_cb)(MemoryRegionSection *s, @@ -165,6 +170,7 @@ ram_block_attributes_rdm_register_listener(RamDiscardManager *rdm, g_assert(section->mr == attr->ram_block->mr); rdl->section = memory_region_section_new_copy(section); + qemu_mutex_lock(&attr->lock); QLIST_INSERT_HEAD(&attr->rdl_list, rdl, next); ret = ram_block_attributes_for_each_populated_section(attr, section, rdl, @@ -174,6 +180,7 @@ ram_block_attributes_rdm_register_listener(RamDiscardManager *rdm, __func__, strerror(-ret)); exit(1); } + qemu_mutex_unlock(&attr->lock); } static void @@ -185,11 +192,13 @@ ram_block_attributes_rdm_unregister_listener(RamDiscardManager *rdm, g_assert(rdl->section); g_assert(rdl->section->mr == attr->ram_block->mr); + qemu_mutex_lock(&attr->lock); rdl->notify_discard(rdl, rdl->section); memory_region_section_free_copy(rdl->section); rdl->section = NULL; QLIST_REMOVE(rdl, next); + qemu_mutex_unlock(&attr->lock); } typedef struct RamBlockAttributesReplayData { @@ -213,10 +222,15 @@ ram_block_attributes_rdm_replay_populated(const RamDiscardManager *rdm, { RamBlockAttributes *attr = RAM_BLOCK_ATTRIBUTES(rdm); RamBlockAttributesReplayData data = { .fn = replay_fn, .opaque = opaque }; + int ret; g_assert(section->mr == attr->ram_block->mr); - return ram_block_attributes_for_each_populated_section(attr, section, &data, - ram_block_attributes_rdm_replay_cb); + qemu_mutex_lock(&attr->lock); + ret = ram_block_attributes_for_each_populated_section( + attr, section, &data, ram_block_attributes_rdm_replay_cb); + qemu_mutex_unlock(&attr->lock); + + return ret; } static int @@ -227,10 +241,15 @@ ram_block_attributes_rdm_replay_discarded(const RamDiscardManager *rdm, { RamBlockAttributes *attr = RAM_BLOCK_ATTRIBUTES(rdm); RamBlockAttributesReplayData data = { .fn = replay_fn, .opaque = opaque }; + int ret; g_assert(section->mr == attr->ram_block->mr); - return ram_block_attributes_for_each_discarded_section(attr, section, &data, - ram_block_attributes_rdm_replay_cb); + qemu_mutex_lock(&attr->lock); + ret = ram_block_attributes_for_each_discarded_section( + attr, section, &data, ram_block_attributes_rdm_replay_cb); + qemu_mutex_unlock(&attr->lock); + + return ret; } static bool @@ -300,13 +319,8 @@ int ram_block_attributes_state_change(RamBlockAttributes *attr, bool to_discard) { const size_t block_size = ram_block_attributes_get_block_size(); - const unsigned long first_bit = offset / block_size; - const unsigned long nbits = size / block_size; - const unsigned long last_bit = first_bit + nbits - 1; - const bool is_discarded = find_next_bit(attr->bitmap, attr->bitmap_size, - first_bit) > last_bit; - const bool is_populated = find_next_zero_bit(attr->bitmap, - attr->bitmap_size, first_bit) > last_bit; + unsigned long first_bit, nbits, last_bit; + bool is_discarded, is_populated; unsigned long bit; int ret = 0; @@ -316,6 +330,16 @@ int ram_block_attributes_state_change(RamBlockAttributes *attr, return -EINVAL; } + first_bit = offset / block_size; + nbits = size / block_size; + last_bit = first_bit + nbits - 1; + + qemu_mutex_lock(&attr->lock); + is_discarded = find_next_bit(attr->bitmap, attr->bitmap_size, + first_bit) > last_bit; + is_populated = find_next_zero_bit(attr->bitmap, attr->bitmap_size, + first_bit) > last_bit; + trace_ram_block_attributes_state_change(offset, size, is_discarded ? "discarded" : is_populated ? "populated" : @@ -364,6 +388,7 @@ int ram_block_attributes_state_change(RamBlockAttributes *attr, } } + qemu_mutex_unlock(&attr->lock); return ret; } @@ -399,11 +424,15 @@ static void ram_block_attributes_init(Object *obj) { RamBlockAttributes *attr = RAM_BLOCK_ATTRIBUTES(obj); + qemu_mutex_init(&attr->lock); QLIST_INIT(&attr->rdl_list); } static void ram_block_attributes_finalize(Object *obj) { + RamBlockAttributes *attr = RAM_BLOCK_ATTRIBUTES(obj); + + qemu_mutex_destroy(&attr->lock); } static void ram_block_attributes_class_init(ObjectClass *klass, diff --git a/target/arm/arm-qmp-cmds.c b/target/arm/arm-qmp-cmds.c index 83ec95c290f..93a2a9647d3 100644 --- a/target/arm/arm-qmp-cmds.c +++ b/target/arm/arm-qmp-cmds.c @@ -76,6 +76,7 @@ static const char *cpu_model_advertised_features[] = { "sve1408", "sve1536", "sve1664", "sve1792", "sve1920", "sve2048", "kvm-no-adjvtime", "kvm-steal-time", "pauth", "pauth-impdef", "pauth-qarma3", "pauth-qarma5", + "num-breakpoints", "num-watchpoints", "num-pmu-counters", NULL }; @@ -227,3 +228,30 @@ CpuDefinitionInfoList *qmp_query_cpu_definitions(Error **errp) return cpu_list; } + +CcaCapability *qmp_query_cca_capabilities(Error **errp) +{ + CcaMeasurementAlgoList *head = NULL; + CcaMeasurementAlgoList **tail = &head; + CcaCapability *info; + CcaMeasurementAlgo *malgo; + + if (!kvm_enabled()) { + error_setg(errp, "KVM is not enabled"); + return NULL; + } + + if (!kvm_arm_rme_available()) { + error_setg(errp, "RME is not enabled in KVM"); + return NULL; + } + + /* KVM currently creates Realms using SHA-256. */ + malgo = g_new0(CcaMeasurementAlgo, 1); + malgo->measurement_algo = g_strdup("sha256"); + QAPI_LIST_APPEND(tail, malgo); + + info = g_new0(CcaCapability, 1); + info->sections = head; + return info; +} diff --git a/target/arm/cpu.c b/target/arm/cpu.c index ccc47c8a9ad..8acf41e7ac4 100644 --- a/target/arm/cpu.c +++ b/target/arm/cpu.c @@ -850,6 +850,11 @@ static void aarch64_cpu_dump_state(CPUState *cs, FILE *f, int flags) const char *ns_status; bool sve; + if (cpu->kvm_rme) { + qemu_fprintf(f, "the CPU registers are confidential to the realm\n"); + return; + } + qemu_fprintf(f, " PC=%016" PRIx64 " ", env->pc); for (i = 0; i < 32; i++) { if (i == 31) { @@ -2399,7 +2404,9 @@ static void arm_cpu_class_init(ObjectClass *oc, const void *data) static void arm_cpu_instance_init(Object *obj) { ARMCPUClass *acc = ARM_CPU_GET_CLASS(obj); + ARMCPU *cpu = ARM_CPU(obj); + cpu->num_pmu_ctrs = -1; acc->info->initfn(obj); arm_cpu_post_init(obj); } diff --git a/target/arm/cpu.h b/target/arm/cpu.h index 657ff4ab20b..049ba297dec 100644 --- a/target/arm/cpu.h +++ b/target/arm/cpu.h @@ -1046,6 +1046,9 @@ struct ArchCPU { /* KVM steal time */ OnOffAuto kvm_steal_time; + /* Realm Management Extension */ + bool kvm_rme; + /* Uniprocessor system with MP extensions */ bool mp_is_up; @@ -1160,6 +1163,18 @@ struct ArchCPU { /* Generic timer counter frequency, in Hz */ uint64_t gt_cntfrq_hz; + + /* Allows to override the default configuration */ + uint8_t num_bps; + uint8_t num_wps; + int8_t num_pmu_ctrs; + + /* + * Set once the above have been pushed to KVM. They can only be applied + * before the VM runs, so kvm_arm_reset_vcpu() must not retry on a later + * reset. + */ + bool kvm_vcpu_regs_configured; }; typedef struct ARMCPUInfo { @@ -2094,6 +2109,8 @@ FIELD(MFAR, FPA, 12, 40) FIELD(MFAR, NSE, 62, 1) FIELD(MFAR, NS, 63, 1) +FIELD(PMCR, N, 11, 5) + QEMU_BUILD_BUG_ON(ARRAY_SIZE(((ARMCPU *)0)->ccsidr) <= R_V7M_CSSELR_INDEX_MASK); /* If adding a feature bit which corresponds to a Linux ELF diff --git a/target/arm/cpu64.c b/target/arm/cpu64.c index d6feba220e8..550b50a5e33 100644 --- a/target/arm/cpu64.c +++ b/target/arm/cpu64.c @@ -666,6 +666,142 @@ void aarch64_add_pauth_properties(Object *obj) } } +#if defined(CONFIG_KVM) +static void arm_cpu_get_num_wps(Object *obj, Visitor *v, const char *name, + void *opaque, Error **errp) +{ + uint8_t val; + ARMCPU *cpu = ARM_CPU(obj); + + val = cpu->num_wps; + if (val == 0) { + val = FIELD_EX64(cpu->isar.idregs[ID_AA64DFR0_EL1_IDX], + ID_AA64DFR0, WRPS) + 1; + } + + visit_type_uint8(v, name, &val, errp); +} + +static void arm_cpu_set_num_wps(Object *obj, Visitor *v, const char *name, + void *opaque, Error **errp) +{ + uint8_t val; + ARMCPU *cpu = ARM_CPU(obj); + uint8_t max_wps = FIELD_EX64(cpu->isar.idregs[ID_AA64DFR0_EL1_IDX], + ID_AA64DFR0, WRPS) + 1; + + if (!visit_type_uint8(v, name, &val, errp)) { + return; + } + + if (cpu->kvm_vcpu_regs_configured) { + error_setg(errp, "cannot change the number of watchpoints after " + "KVM vCPU register configuration"); + return; + } + + if (val < 2 || val > max_wps) { + error_setg(errp, "invalid number of watchpoints"); + return; + } + + cpu->num_wps = val; +} + +static void arm_cpu_get_num_bps(Object *obj, Visitor *v, const char *name, + void *opaque, Error **errp) +{ + uint8_t val; + ARMCPU *cpu = ARM_CPU(obj); + + val = cpu->num_bps; + if (val == 0) { + val = FIELD_EX64(cpu->isar.idregs[ID_AA64DFR0_EL1_IDX], + ID_AA64DFR0, BRPS) + 1; + } + + visit_type_uint8(v, name, &val, errp); +} + +static void arm_cpu_set_num_bps(Object *obj, Visitor *v, const char *name, + void *opaque, Error **errp) +{ + uint8_t val; + ARMCPU *cpu = ARM_CPU(obj); + uint8_t max_bps = FIELD_EX64(cpu->isar.idregs[ID_AA64DFR0_EL1_IDX], + ID_AA64DFR0, BRPS) + 1; + + if (!visit_type_uint8(v, name, &val, errp)) { + return; + } + + if (cpu->kvm_vcpu_regs_configured) { + error_setg(errp, "cannot change the number of breakpoints after " + "KVM vCPU register configuration"); + return; + } + + if (val < 2 || val > max_bps) { + error_setg(errp, "invalid number of breakpoints"); + return; + } + + cpu->num_bps = val; +} + +static void arm_cpu_get_num_pmu_ctrs(Object *obj, Visitor *v, const char *name, + void *opaque, Error **errp) +{ + uint8_t val; + ARMCPU *cpu = ARM_CPU(obj); + + if (cpu->num_pmu_ctrs == -1) { + val = FIELD_EX64(cpu->isar.reset_pmcr_el0, PMCR, N); + } else { + val = cpu->num_pmu_ctrs; + } + + visit_type_uint8(v, name, &val, errp); +} + +static void arm_cpu_set_num_pmu_ctrs(Object *obj, Visitor *v, const char *name, + void *opaque, Error **errp) +{ + uint8_t val; + ARMCPU *cpu = ARM_CPU(obj); + uint8_t max_ctrs = FIELD_EX64(cpu->isar.reset_pmcr_el0, PMCR, N); + + if (!visit_type_uint8(v, name, &val, errp)) { + return; + } + + if (cpu->kvm_vcpu_regs_configured) { + error_setg(errp, "cannot change the number of PMU counters after " + "KVM vCPU register configuration"); + return; + } + + if (val > max_ctrs) { + error_setg(errp, "invalid number of PMU counters"); + return; + } + + cpu->num_pmu_ctrs = val; +} + +static void aarch64_add_kvm_writable_properties(Object *obj) +{ + object_property_add(obj, "num-breakpoints", "uint8", arm_cpu_get_num_bps, + arm_cpu_set_num_bps, NULL, NULL); + object_property_add(obj, "num-watchpoints", "uint8", arm_cpu_get_num_wps, + arm_cpu_set_num_wps, NULL, NULL); + + object_property_add(obj, "num-pmu-counters", "uint8", + arm_cpu_get_num_pmu_ctrs, arm_cpu_set_num_pmu_ctrs, + NULL, NULL); +} +#endif /* CONFIG_KVM */ + void arm_cpu_lpa2_finalize(ARMCPU *cpu, Error **errp) { uint64_t t; @@ -824,6 +960,7 @@ static void aarch64_host_initfn(Object *obj) #if defined(CONFIG_KVM) kvm_arm_set_cpu_features_from_host(cpu); aarch64_add_sve_properties(obj); + aarch64_add_kvm_writable_properties(obj); #elif defined(CONFIG_HVF) hvf_arm_set_cpu_features_from_host(cpu); #elif defined(CONFIG_WHPX) diff --git a/target/arm/kvm-rme.c b/target/arm/kvm-rme.c new file mode 100644 index 00000000000..044d99e72ac --- /dev/null +++ b/target/arm/kvm-rme.c @@ -0,0 +1,981 @@ +/* + * QEMU Arm RME support + * + * SPDX-License-Identifier: GPL-2.0-or-later + * + * Copyright Linaro 2026 + */ + +#include "qemu/osdep.h" + +#include "hw/core/boards.h" +#include "hw/core/cpu.h" +#include "hw/core/loader.h" +#include "hw/pci/pci.h" +#include "kvm_arm.h" +#include "migration/blocker.h" +#include "qapi/error.h" +#include "qemu/error-report.h" +#include "qemu/memalign.h" +#include "qemu/units.h" +#include "qom/object_interfaces.h" +#include "system/confidential-guest-support.h" +#include "system/kvm.h" +#include "system/runstate.h" + +#define TYPE_RME_GUEST "rme-guest" +OBJECT_DECLARE_SIMPLE_TYPE(RmeGuest, RME_GUEST) + +#define RME_PAGE_SIZE qemu_real_host_page_size() + +/* + * Realms have a split guest-physical address space: the bottom half is private + * to the realm, and the top half is shared with the host. Within QEMU, we use a + * merged view of both halves. Most of RAM is private to the guest and not + * accessible to us, but the guest shares some pages with us. + * + * RealmDmaRegion performs remapping of top-half accesses to system memory. + */ +struct RealmDmaRegion { + IOMMUMemoryRegion parent_obj; +}; + +#define TYPE_REALM_DMA_REGION "realm-dma-region" +OBJECT_DECLARE_SIMPLE_TYPE(RealmDmaRegion, REALM_DMA_REGION) +OBJECT_DEFINE_SIMPLE_TYPE(RealmDmaRegion, realm_dma_region, + REALM_DMA_REGION, IOMMU_MEMORY_REGION); + +typedef struct { + hwaddr base; + hwaddr size; + size_t data_size; + uint8_t *data; +} RmeRamRegion; + +typedef struct RealmRamDiscardListener { + RmeGuest *guest; + MemoryRegion *mr; + hwaddr offset_within_address_space; + RamDiscardManager *rdm; + uint64_t granularity; + RamDiscardListener listener; + QLIST_ENTRY(RealmRamDiscardListener) next; +} RealmRamDiscardListener; + +struct RmeGuest { + ConfidentialGuestSupport parent_obj; + Notifier rom_load_notifier; + VMChangeStateEntry *vm_state_handler; + Error *migration_blocker; + GSList *ram_regions; + bool rom_load_notifier_registered; + bool activated; + uint8_t ipa_bits; + + RealmDmaRegion dma_region; + QLIST_HEAD(, RealmRamDiscardListener) ram_discard_list; + /* + * Lock order: ram_discard_lock nests outside RamBlockAttributes::lock. + * RamDiscardManager callbacks must not acquire ram_discard_lock. + */ + QemuMutex ram_discard_lock; + MemoryListener memory_listener; + bool memory_listener_registered; + AddressSpace *dma_as; +}; + +OBJECT_DEFINE_SIMPLE_TYPE_WITH_INTERFACES(RmeGuest, rme_guest, RME_GUEST, + CONFIDENTIAL_GUEST_SUPPORT, + { TYPE_USER_CREATABLE }, { }) + +static RmeGuest *rme_get_machine_guest(void) +{ + MachineState *machine = MACHINE(qdev_get_machine()); + + if (!machine->cgs || + !object_dynamic_cast(OBJECT(machine->cgs), TYPE_RME_GUEST)) { + return NULL; + } + + return RME_GUEST(machine->cgs); +} + +static void rme_ram_region_free(gpointer opaque) +{ + RmeRamRegion *region = opaque; + + qemu_vfree(region->data); + g_free(region); +} + +static uint8_t *rme_alloc_page_buffer(hwaddr size, Error **errp) +{ + size_t buffer_size = size; + uint8_t *buffer; + + if (buffer_size != size) { + error_setg(errp, "Realm population range is too large"); + return NULL; + } + + buffer = qemu_try_memalign(RME_PAGE_SIZE, buffer_size); + if (!buffer) { + error_setg_errno(errp, ENOMEM, + "failed to allocate Realm population buffer"); + return NULL; + } + memset(buffer, 0, buffer_size); + return buffer; +} + +static int rme_population_chunk_size(hwaddr base, hwaddr size, + hwaddr *chunk_size, Error **errp) +{ + MemoryRegionSection section; + MemoryRegion *mr; + + section = memory_region_find(get_system_memory(), base, size); + mr = section.mr; + if (!mr) { + error_setg(errp, "Realm population GPA 0x%" HWADDR_PRIx + " is not backed by RAM", base); + return -EINVAL; + } + + *chunk_size = int128_get64(section.size); + if (!memory_region_is_ram(mr) || !memory_region_has_guest_memfd(mr)) { + error_setg(errp, "Realm population GPA 0x%" HWADDR_PRIx + " is not backed by guestmemfd RAM", base); + memory_region_unref(mr); + return -EINVAL; + } + memory_region_unref(mr); + + if (!*chunk_size || + !QEMU_IS_ALIGNED(*chunk_size, qemu_real_host_page_size())) { + error_setg(errp, "Realm population section at GPA 0x%" HWADDR_PRIx + " is not host-page-aligned", base); + return -EINVAL; + } + + return 0; +} + +static int rme_populate_range(const RmeRamRegion *region, bool measure, + Error **errp) +{ + const hwaddr end = region->base + region->size; + hwaddr offset; + int ret; + + if (region->size > HWADDR_MAX - region->base || + !QEMU_IS_ALIGNED(region->base, RME_PAGE_SIZE) || + !QEMU_IS_ALIGNED(region->size, RME_PAGE_SIZE) || + !QEMU_PTR_IS_ALIGNED(region->data, RME_PAGE_SIZE)) { + error_setg(errp, + "Realm range [0x%" HWADDR_PRIx ", 0x%" HWADDR_PRIx + ") is not page-aligned", + region->base, end); + return -EINVAL; + } + + /* + * Preflight every leaf before changing attributes. Guest-visible RAM can + * be contiguous across adjacent NUMA memory backends, but each conversion + * must stay within one leaf MemoryRegion. + */ + for (offset = 0; offset < region->size; ) { + hwaddr chunk_size; + + ret = rme_population_chunk_size(region->base + offset, + region->size - offset, + &chunk_size, errp); + if (ret) { + return ret; + } + offset += chunk_size; + } + + for (offset = 0; offset < region->size; ) { + struct kvm_arm_rmi_populate populate_args; + hwaddr chunk_size; + + ret = rme_population_chunk_size(region->base + offset, + region->size - offset, + &chunk_size, errp); + assert(ret == 0); + + /* + * Keep KVM's attributes, QEMU's RamDiscardManager state, and the + * shared host mapping synchronized before handing the private copy + * to KVM. + */ + ret = kvm_convert_memory(region->base + offset, chunk_size, true); + if (ret) { + error_setg_errno(errp, -ret, + "failed to configure private Realm range " + "[0x%" HWADDR_PRIx ", 0x%" HWADDR_PRIx ")", + region->base + offset, + region->base + offset + chunk_size); + return ret; + } + + populate_args = (struct kvm_arm_rmi_populate) { + .base = region->base + offset, + .size = chunk_size, + .source_uaddr = (uintptr_t)region->data + offset, + .flags = measure ? KVM_ARM_RMI_POPULATE_FLAGS_MEASURE : 0, + }; + + while (populate_args.size > 0) { + hwaddr size = populate_args.size; + + ret = kvm_vm_ioctl(kvm_state, KVM_ARM_RMI_POPULATE, + &populate_args, 0); + if (ret) { + error_setg_errno(errp, -ret, + "failed to populate Realm " + "[0x%" HWADDR_PRIx ", 0x%" HWADDR_PRIx ")", + region->base, end); + return ret; + } + if (populate_args.size >= size) { + error_setg(errp, + "KVM made no progress populating Realm range " + "[0x%" HWADDR_PRIx ", 0x%" HWADDR_PRIx ")", + region->base, end); + return -EIO; + } + } + + offset += chunk_size; + } + + return 0; +} + +static void rme_populate_ram_region(gpointer data, gpointer err) +{ + Error **errp = err; + const RmeRamRegion *region = data; + + if (*errp) { + return; + } + + rme_populate_range(region, /* measure */ true, errp); +} + +static bool rme_coalesce_ram_regions(RmeGuest *guest, Error **errp) +{ + GSList *regions = g_steal_pointer(&guest->ram_regions); + GSList *result = NULL; + RmeRamRegion *merged = NULL; + hwaddr previous_end = 0; + bool have_previous = false; + + while (regions) { + GSList *node = regions; + RmeRamRegion *region = node->data; + RmeRamRegion *new_region; + uint8_t *new_data; + hwaddr region_end; + hwaddr start; + hwaddr end; + + regions = regions->next; + g_slist_free_1(node); + + if (region->size > HWADDR_MAX - region->base) { + error_setg(errp, "Realm image at 0x%" HWADDR_PRIx + " is too large", region->base); + goto error; + } + region_end = region->base + region->size; + if (region_end > HWADDR_MAX - (RME_PAGE_SIZE - 1)) { + error_setg(errp, "Realm image at 0x%" HWADDR_PRIx + " cannot be page-aligned", region->base); + goto error; + } + if (have_previous && region->base < previous_end) { + error_setg(errp, "overlapping Realm images at GPA 0x%" + HWADDR_PRIx, region->base); + goto error; + } + + start = QEMU_ALIGN_DOWN(region->base, RME_PAGE_SIZE); + end = QEMU_ALIGN_UP(region_end, RME_PAGE_SIZE); + + if (merged && start <= merged->base + merged->size) { + hwaddr merged_end = merged->base + merged->size; + + if (end > merged_end) { + new_data = rme_alloc_page_buffer(end - merged->base, errp); + if (!new_data) { + goto error; + } + memcpy(new_data, merged->data, merged->size); + qemu_vfree(merged->data); + merged->data = new_data; + merged->size = end - merged->base; + merged->data_size = merged->size; + } + if (region->data_size) { + memcpy(merged->data + (region->base - merged->base), + region->data, region->data_size); + } + } else { + new_region = g_new0(RmeRamRegion, 1); + new_region->base = start; + new_region->size = end - start; + new_region->data = rme_alloc_page_buffer(new_region->size, errp); + if (!new_region->data) { + g_free(new_region); + goto error; + } + new_region->data_size = new_region->size; + if (region->data_size) { + memcpy(new_region->data + (region->base - start), + region->data, region->data_size); + } + result = g_slist_append(result, new_region); + merged = new_region; + } + + previous_end = region_end; + have_previous = true; + rme_ram_region_free(region); + continue; + +error: + rme_ram_region_free(region); + g_slist_free_full(regions, rme_ram_region_free); + g_slist_free_full(result, rme_ram_region_free); + return false; + } + + guest->ram_regions = result; + return true; +} + +static void rme_vm_state_change(void *opaque, bool running, RunState state) +{ + RmeGuest *guest = opaque; + Error *errp = NULL; + + if (!running || guest->activated) { + return; + } + + if (rme_coalesce_ram_regions(guest, &errp)) { + g_slist_foreach(guest->ram_regions, rme_populate_ram_region, &errp); + } + g_slist_free_full(g_steal_pointer(&guest->ram_regions), + rme_ram_region_free); + if (errp) { + error_report_err(errp); + exit(EXIT_FAILURE); + } + + guest->activated = true; + if (guest->rom_load_notifier_registered) { + notifier_remove(&guest->rom_load_notifier); + guest->rom_load_notifier_registered = false; + } + kvm_mark_guest_state_protected(); +} + +static gint rme_compare_ram_regions(gconstpointer a, gconstpointer b) +{ + const RmeRamRegion *ra = a; + const RmeRamRegion *rb = b; + + if (ra->base == rb->base) { + return 0; + } + return ra->base < rb->base ? -1 : 1; +} + +static void rme_rom_load_notify(Notifier *notifier, void *data) +{ + RmeGuest *guest = container_of(notifier, RmeGuest, rom_load_notifier); + GSList *entry; + RmeRamRegion *region; + RomLoaderNotifyData *rom = data; + uint8_t *copy; + + if (guest->activated) { + return; + } + + if (rom->addr == -1) { + /* + * These blobs (ACPI tables) are not loaded into guest RAM at reset. + * Instead the firmware will load them via fw_cfg and measure them + * itself. + */ + return; + } + if (!rom->len) { + return; + } + if (rom->data_len > rom->len) { + error_report("Realm image at 0x%" HWADDR_PRIx + " has invalid data length", rom->addr); + exit(EXIT_FAILURE); + } + if (rom->data_len && !rom->data) { + error_report("Realm image at 0x%" HWADDR_PRIx " has no data", + rom->addr); + exit(EXIT_FAILURE); + } + if (rom->len > HWADDR_MAX - rom->addr) { + error_report("Realm image at 0x%" HWADDR_PRIx " is too large", + rom->addr); + exit(EXIT_FAILURE); + } + + copy = NULL; + if (rom->data_len) { + copy = qemu_try_memalign(RME_PAGE_SIZE, rom->data_len); + if (!copy) { + error_report("failed to copy Realm image at 0x%" HWADDR_PRIx, + rom->addr); + exit(EXIT_FAILURE); + } + memcpy(copy, rom->data, rom->data_len); + } + + /* + * rom_reset() notifies listeners on every reset. Before the Realm is + * activated, replace a previous snapshot of the same ROM instead of + * adding a duplicate which would later look like an overlapping image. + */ + for (entry = guest->ram_regions; entry; entry = entry->next) { + region = entry->data; + if (region->base == rom->addr && region->size == rom->len) { + qemu_vfree(region->data); + region->data = copy; + region->data_size = rom->data_len; + return; + } + } + + region = g_new0(RmeRamRegion, 1); + region->base = rom->addr; + region->size = rom->len; + region->data_size = rom->data_len; + region->data = copy; + + /* + * The Realm Initial Measurement (RIM) depends on the order in which we + * initialize and populate the RAM regions. To help a verifier + * independently calculate the RIM, sort regions by GPA. + */ + guest->ram_regions = g_slist_insert_sorted(guest->ram_regions, region, + rme_compare_ram_regions); +} + +#define KVM_CAP_ARM_RMI_SYSFS_PATH "/sys/module/kvm/parameters/kvm_cap_arm_rmi" + +/* + * Returns the KVM CCA capability number for the running kernel. + * + * The capability number is not stable: it shifts whenever other KVM + * capabilities land ahead of it, so the value in linux-headers only matches + * hosts built from the same snapshot. NVIDIA kernels export the live value + * as a module parameter; prefer it and fall back to the compile-time + * constant. + * + * FIXME: this is a downstream-only workaround and must be dropped before the + * series is posted upstream, where KVM_CAP_ARM_RMI will have a fixed value. + */ +#define KVM_CAP_ARM_RMI_MAX 4095 + +static unsigned int kvm_arm_rme_get_cap(void) +{ + static unsigned int rme_cap; + static bool detected; + + if (!detected) { + FILE *f; + int cap; + + rme_cap = KVM_CAP_ARM_RMI; + + f = fopen(KVM_CAP_ARM_RMI_SYSFS_PATH, "r"); + if (f) { + /* + * Bound the value: a garbage module parameter would otherwise + * turn into a wild KVM_CHECK_EXTENSION argument. + */ + if (fscanf(f, "%d", &cap) == 1 && + cap > 0 && cap <= KVM_CAP_ARM_RMI_MAX) { + rme_cap = cap; + } else { + warn_report("ignoring out-of-range %s, falling back to " + "KVM_CAP_ARM_RMI=%d", + KVM_CAP_ARM_RMI_SYSFS_PATH, KVM_CAP_ARM_RMI); + } + fclose(f); + } else { + warn_report("cannot read %s: %s; falling back to " + "KVM_CAP_ARM_RMI=%d", + KVM_CAP_ARM_RMI_SYSFS_PATH, strerror(errno), + KVM_CAP_ARM_RMI); + } + detected = true; + } + + return rme_cap; +} + +bool kvm_arm_rme_available(void) +{ + return kvm_enabled() && + kvm_vm_check_extension(kvm_state, kvm_arm_rme_get_cap()); +} + +static int kvm_arm_rme_init(ConfidentialGuestSupport *cgs, Error **errp) +{ + RmeGuest *guest = RME_GUEST(cgs); + + if (!kvm_arm_rme_available()) { + error_setg(errp, "VM doesn't support Realms"); + return -ENODEV; + } + + error_setg(&guest->migration_blocker, + "RME: migration is not implemented"); + migrate_add_blocker(&guest->migration_blocker, &error_fatal); + + guest->rom_load_notifier.notify = rme_rom_load_notify; + rom_add_load_notifier(&guest->rom_load_notifier); + guest->rom_load_notifier_registered = true; + + /* + * The realm activation is done last, when the VM starts, after all images + * have been loaded and all vcpus finalized. + */ + guest->vm_state_handler = + qemu_add_vm_change_state_handler(rme_vm_state_change, guest); + + cgs->require_guest_memfd = true; + cgs->assigned_device_memory = true; + cgs->ready = true; + return 0; +} + +void kvm_arm_rme_vcpu_init(ARMCPU *cpu) +{ + if (!rme_get_machine_guest()) { + return; + } + + cpu->kvm_rme = true; +} + +static bool rme_guest_can_be_deleted(UserCreatable *uc) +{ + MachineState *machine = MACHINE(qdev_get_machine()); + + /* The unparent hook tears down state that an active machine still uses. */ + return machine->cgs != CONFIDENTIAL_GUEST_SUPPORT(uc); +} + +static void rme_guest_unparent(Object *obj); + +static void rme_guest_class_init(ObjectClass *oc, const void *data) +{ + ConfidentialGuestSupportClass *klass = CONFIDENTIAL_GUEST_SUPPORT_CLASS(oc); + UserCreatableClass *ucc = USER_CREATABLE_CLASS(oc); + + oc->unparent = rme_guest_unparent; + klass->kvm_init = kvm_arm_rme_init; + ucc->can_be_deleted = rme_guest_can_be_deleted; +} + +static void rme_guest_init(Object *obj) +{ + RmeGuest *guest = RME_GUEST(obj); + + QLIST_INIT(&guest->ram_discard_list); + qemu_mutex_init(&guest->ram_discard_lock); +} + +static void rme_guest_cleanup(RmeGuest *guest) +{ + if (guest->rom_load_notifier_registered) { + notifier_remove(&guest->rom_load_notifier); + guest->rom_load_notifier_registered = false; + } + if (guest->vm_state_handler) { + qemu_del_vm_change_state_handler(guest->vm_state_handler); + guest->vm_state_handler = NULL; + } + if (guest->memory_listener_registered) { + memory_listener_unregister(&guest->memory_listener); + guest->memory_listener_registered = false; + } + g_clear_pointer(&guest->dma_as, address_space_destroy_free); + migrate_del_blocker(&guest->migration_blocker); +} + +static void rme_guest_unparent(Object *obj) +{ + /* + * dma_as references the RmeGuest through its root MemoryRegion owner. + * Break that reference before the objects container drops its reference; + * waiting until instance_finalize() would leave a reference cycle. + */ + rme_guest_cleanup(RME_GUEST(obj)); +} + +static void rme_guest_finalize(Object *obj) +{ + RmeGuest *guest = RME_GUEST(obj); + + /* Also cover objects destroyed after only partial initialization. */ + rme_guest_cleanup(guest); + assert(QLIST_EMPTY(&guest->ram_discard_list)); + qemu_mutex_destroy(&guest->ram_discard_lock); + g_slist_free_full(guest->ram_regions, rme_ram_region_free); +} + +static void rme_dma_notify_section(RmeGuest *guest, + MemoryRegionSection *section, + uint64_t granularity, bool populate, + IOMMUNotifier *notifier) +{ + const hwaddr shared_bit = 1ULL << (guest->ipa_bits - 1); + const hwaddr end = section->offset_within_address_space + + int128_get64(section->size); + hwaddr gpa, next; + IOMMUTLBEvent event = { + .type = populate ? IOMMU_NOTIFIER_MAP : IOMMU_NOTIFIER_UNMAP, + .entry = { + .target_as = &address_space_memory, + .perm = populate ? IOMMU_RW : IOMMU_NONE, + .addr_mask = granularity - 1, + }, + }; + + assert(guest->dma_as); + assert(end <= shared_bit); + + for (gpa = section->offset_within_address_space; gpa < end; gpa = next) { + next = ROUND_UP(gpa + 1, granularity); + next = MIN(next, end); + + event.entry.translated_addr = gpa; + + /* Devices must use the shared-bit alias produced by the DMA API. */ + event.entry.iova = gpa | shared_bit; + if (notifier) { + memory_region_notify_iommu_one(notifier, &event); + } else { + memory_region_notify_iommu(IOMMU_MEMORY_REGION(&guest->dma_region), + 0, event); + } + } +} + +static int rme_ram_discard_notify(RamDiscardListener *rdl, + MemoryRegionSection *section, + bool populate) +{ + RealmRamDiscardListener *rrdl = + container_of(rdl, RealmRamDiscardListener, listener); + + rme_dma_notify_section(rrdl->guest, section, rrdl->granularity, + populate, NULL); + return 0; +} + +static int rme_ram_discard_notify_populate(RamDiscardListener *rdl, + MemoryRegionSection *section) +{ + return rme_ram_discard_notify(rdl, section, true); +} + +static void rme_ram_discard_notify_discard(RamDiscardListener *rdl, + MemoryRegionSection *section) +{ + rme_ram_discard_notify(rdl, section, false); +} + +static void rme_listener_region_add(MemoryListener *listener, + MemoryRegionSection *section) +{ + RmeGuest *guest = container_of(listener, RmeGuest, memory_listener); + RamDiscardManager *rdm = memory_region_get_ram_discard_manager(section->mr); + RealmRamDiscardListener *rrdl; + + if (!rdm) { + return; + } + + rrdl = g_new0(RealmRamDiscardListener, 1); + rrdl->guest = guest; + rrdl->mr = section->mr; + rrdl->offset_within_address_space = section->offset_within_address_space; + rrdl->rdm = rdm; + rrdl->granularity = + ram_discard_manager_get_min_granularity(rdm, section->mr); + + ram_discard_listener_init(&rrdl->listener, + rme_ram_discard_notify_populate, + rme_ram_discard_notify_discard); + ram_discard_manager_register_listener(rdm, &rrdl->listener, section); + + qemu_mutex_lock(&guest->ram_discard_lock); + QLIST_INSERT_HEAD(&guest->ram_discard_list, rrdl, next); + qemu_mutex_unlock(&guest->ram_discard_lock); +} + +static void rme_listener_region_del(MemoryListener *listener, + MemoryRegionSection *section) +{ + RmeGuest *guest = container_of(listener, RmeGuest, memory_listener); + RealmRamDiscardListener *rrdl = NULL; + + qemu_mutex_lock(&guest->ram_discard_lock); + QLIST_FOREACH(rrdl, &guest->ram_discard_list, next) { + if (rrdl->mr == section->mr && + rrdl->offset_within_address_space == + section->offset_within_address_space) { + QLIST_REMOVE(rrdl, next); + break; + } + } + qemu_mutex_unlock(&guest->ram_discard_lock); + + if (rrdl) { + ram_discard_manager_unregister_listener(rrdl->rdm, &rrdl->listener); + g_free(rrdl); + } +} + +static AddressSpace *rme_dma_get_address_space(PCIBus *bus, void *opaque, + int devfn) +{ + RmeGuest *guest = opaque; + + return guest->dma_as; +} + +static const PCIIOMMUOps rme_dma_ops = { + .get_address_space = rme_dma_get_address_space, +}; + +void kvm_arm_rme_init_gpa_space(unsigned int ipa_bits, PCIBus *pci_bus) +{ + RmeGuest *guest = rme_get_machine_guest(); + + if (!guest || !ipa_bits) { + return; + } + + assert(ipa_bits < 64); + assert(!guest->dma_as); + + /* + * Setup a DMA translation from the shared top half of the guest-physical + * address space to our merged view of RAM. + */ + memory_region_init_iommu(&guest->dma_region, sizeof(guest->dma_region), + TYPE_REALM_DMA_REGION, OBJECT(guest), + "realm-dma-region", 1ULL << ipa_bits); + guest->dma_as = g_new0(AddressSpace, 1); + address_space_init(guest->dma_as, MEMORY_REGION(&guest->dma_region), + TYPE_REALM_DMA_REGION); + guest->ipa_bits = ipa_bits; + + pci_setup_iommu(pci_bus, &rme_dma_ops, guest); + + guest->memory_listener = (MemoryListener) { + .name = "rme", + .region_add = rme_listener_region_add, + .region_del = rme_listener_region_del, + }; + memory_listener_register(&guest->memory_listener, &address_space_memory); + guest->memory_listener_registered = true; +} + +AddressSpace *kvm_arm_rme_get_dma_as(void) +{ + RmeGuest *guest = rme_get_machine_guest(); + + return guest ? guest->dma_as : NULL; +} + +static void realm_dma_region_init(Object *obj) +{ +} + +static bool realm_dma_access_allowed(RmeGuest *guest, hwaddr gpa) +{ + RealmRamDiscardListener *rrdl; + MemoryRegionSection section = { 0 }; + MemoryRegion *target; + hwaddr target_offset; + hwaddr len = 1; + bool shared = false; + bool tracked = false; + + /* + * IOMMU translations can run without the BQL. Use the listener cache + * instead of walking the global address-space topology and keep each + * listener alive while its RamDiscardManager is queried. + */ + qemu_mutex_lock(&guest->ram_discard_lock); + QLIST_FOREACH(rrdl, &guest->ram_discard_list, next) { + const MemoryRegionSection *registered = rrdl->listener.section; + const hwaddr as_start = registered->offset_within_address_space; + const hwaddr region_start = registered->offset_within_region; + const uint64_t registered_size = int128_get64(registered->size); + const uint64_t granularity = rrdl->granularity; + uint64_t granule_offset; + hwaddr translated; + + if (gpa < as_start || gpa - as_start >= registered_size) { + continue; + } + if (!granularity || gpa - as_start > HWADDR_MAX - region_start) { + continue; + } + + translated = region_start + (gpa - as_start); + granule_offset = translated / granularity * granularity; + + /* RamDiscardManager queries operate on complete tracking granules. */ + if (granule_offset < region_start || + granule_offset - region_start > registered_size || + granularity > + registered_size - (granule_offset - region_start)) { + continue; + } + + section.mr = registered->mr; + section.offset_within_address_space = + as_start + (granule_offset - region_start); + section.offset_within_region = granule_offset; + section.size = int128_make64(granularity); + shared = ram_discard_manager_is_populated(rrdl->rdm, §ion); + tracked = true; + break; + } + qemu_mutex_unlock(&guest->ram_discard_lock); + + if (tracked) { + return shared; + } + + /* + * PCI DMA also carries interrupt writes to MMIO targets such as the GIC + * ITS doorbell. The shared/private state applies only to RAM, so let the + * target MemoryRegion validate non-RAM accesses. Absence from the cache + * still fails closed for RAM, deliberately disallowing peer DMA to a + * VFIO BAR through the Realm DMA address space. + * + * IOMMU translations run under either the BQL or an RCU read lock. + */ + target = address_space_translate(&address_space_memory, gpa, &target_offset, + &len, false, MEMTXATTRS_UNSPECIFIED); + return !memory_region_is_ram(target); +} + +static IOMMUTLBEntry realm_dma_region_translate(IOMMUMemoryRegion *mr, + hwaddr addr, + IOMMUAccessFlags flag, + int iommu_idx) +{ + RmeGuest *guest = RME_GUEST(memory_region_owner(MEMORY_REGION(mr))); + const hwaddr shared_bit = 1ULL << (guest->ipa_bits - 1); + const hwaddr address_mask = shared_bit - 1; + const hwaddr translated_addr = addr & address_mask; + IOMMUTLBEntry entry = { + .target_as = &address_space_memory, + .iova = addr, + .translated_addr = translated_addr, + /* + * Somewhat arbitrary granule for users that need one, such as + * address_space_get_iotlb_entry(). Should be relatively large to + * avoid frequent TLB misses. It can't be larger than memory region + * alignment (eg. address_mask) because that would mask the whole + * address, preventing vhost from finding the correct memory region. + */ + .addr_mask = 4 * KiB - 1, + /* + * Firmware can use the canonical IPA for a page that it has made + * shared with the RMM, while Linux DMA addresses carry shared_bit. + * Accept both spellings for RAM only while the RAM discard manager + * records the page as shared. Permit non-RAM transactions such as + * PCI interrupt writes, while RAM without a discard manager remains + * inaccessible. + */ + .perm = realm_dma_access_allowed(guest, translated_addr) ? + IOMMU_RW : IOMMU_NONE, + }; + + return entry; +} + +typedef struct RealmDmaReplayData { + RmeGuest *guest; + IOMMUNotifier *notifier; + uint64_t granularity; +} RealmDmaReplayData; + +static int realm_dma_replay_populated(MemoryRegionSection *section, + void *opaque) +{ + RealmDmaReplayData *data = opaque; + + rme_dma_notify_section(data->guest, section, data->granularity, true, + data->notifier); + return 0; +} + +static void realm_dma_region_replay(IOMMUMemoryRegion *mr, IOMMUNotifier *n) +{ + RmeGuest *guest = RME_GUEST(memory_region_owner(MEMORY_REGION(mr))); + RealmRamDiscardListener *rrdl; + + if (!(n->notifier_flags & IOMMU_NOTIFIER_MAP)) { + return; + } + + qemu_mutex_lock(&guest->ram_discard_lock); + QLIST_FOREACH(rrdl, &guest->ram_discard_list, next) { + RealmDmaReplayData data = { + .guest = guest, + .notifier = n, + .granularity = rrdl->granularity, + }; + + /* + * RamBlockAttributes serializes this callback with bitmap changes and + * their MAP/UNMAP notifications. This ensures that a discard UNMAP + * cannot be overtaken by a stale replay MAP. + */ + ram_discard_manager_replay_populated(rrdl->rdm, + rrdl->listener.section, + realm_dma_replay_populated, + &data); + } + qemu_mutex_unlock(&guest->ram_discard_lock); +} + +static void realm_dma_region_finalize(Object *obj) +{ +} + +static void realm_dma_region_class_init(ObjectClass *oc, const void *data) +{ + IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_CLASS(oc); + + imrc->translate = realm_dma_region_translate; + imrc->replay = realm_dma_region_replay; + imrc->require_notifier_success = true; +} diff --git a/target/arm/kvm-stub.c b/target/arm/kvm-stub.c index 88cbe8d85c4..bad33a169d1 100644 --- a/target/arm/kvm-stub.c +++ b/target/arm/kvm-stub.c @@ -42,6 +42,15 @@ bool kvm_arm_el2_supported(void) return false; } +void kvm_arm_rme_init_gpa_space(unsigned int ipa_bits, PCIBus *pci_bus) +{ +} + +AddressSpace *kvm_arm_rme_get_dma_as(void) +{ + return NULL; +} + /* * These functions should never actually be called without KVM support. */ @@ -119,3 +128,13 @@ char *kvm_print_register_name(uint64_t regidx) { g_assert_not_reached(); } + +bool kvm_arm_rme_available(void) +{ + return false; +} + +void kvm_arm_rme_vcpu_init(ARMCPU *cpu) +{ + g_assert_not_reached(); +} diff --git a/target/arm/kvm.c b/target/arm/kvm.c index d4a68874b88..d9a2497a28f 100644 --- a/target/arm/kvm.c +++ b/target/arm/kvm.c @@ -18,6 +18,7 @@ #include "qemu/timer.h" #include "qemu/error-report.h" #include "qemu/main-loop.h" +#include "qemu/units.h" #include "qom/object.h" #include "qapi/error.h" #include "system/system.h" @@ -33,6 +34,7 @@ #include "hw/pci/pci.h" #include "exec/memattrs.h" #include "system/address-spaces.h" +#include "system/confidential-guest-support.h" #include "gdbstub/enums.h" #include "hw/core/boards.h" #include "hw/core/irq.h" @@ -40,8 +42,12 @@ #include "qemu/log.h" #include "hw/acpi/acpi.h" #include "hw/acpi/ghes.h" +#include "hw/arm/rme-da.h" #include "target/arm/gtimer.h" #include "migration/blocker.h" +#include "system/iommufd.h" + +QEMU_BUILD_BUG_ON(QEMU_KVM_ARM_VM_TYPE_REALM != KVM_VM_TYPE_ARM_REALM); const KVMCapabilityInfo kvm_arch_required_capabilities[] = { KVM_CAP_INFO(DEVICE_CTRL), @@ -52,6 +58,9 @@ static bool cap_has_mp_state; static bool cap_has_inject_serror_esr; static bool cap_has_inject_ext_dabt; +#define KVM_REG_ARM_ID_AA64DFR0_EL1 ARM64_SYS_REG(3, 0, 0, 5, 0) +#define KVM_REG_ARM_PMCR_EL0 ARM64_SYS_REG(3, 3, 9, 12, 0) + /** * ARMHostCPUFeatures: information about the host CPU (identified * by asking the host kernel) @@ -107,7 +116,9 @@ bool kvm_arm_create_scratch_host_vcpu(int *fdarray, struct kvm_vcpu_init *init) { int ret = 0, kvmfd = -1, vmfd = -1, cpufd = -1; + MachineState *ms = MACHINE(qdev_get_machine()); int max_vm_pa_size; + int vm_type; kvmfd = qemu_open_old("/dev/kvm", O_RDWR); if (kvmfd < 0) { @@ -117,10 +128,18 @@ bool kvm_arm_create_scratch_host_vcpu(int *fdarray, if (max_vm_pa_size < 0) { max_vm_pa_size = 0; } + + vm_type = (ms->cgs ? KVM_VM_TYPE_ARM_REALM : KVM_VM_TYPE_ARM_NORMAL); do { - vmfd = ioctl(kvmfd, KVM_CREATE_VM, max_vm_pa_size); + vmfd = ioctl(kvmfd, KVM_CREATE_VM, max_vm_pa_size | vm_type); } while (vmfd == -1 && errno == EINTR); if (vmfd < 0) { + if (errno == EINVAL && max_vm_pa_size && + (vm_type & KVM_VM_TYPE_ARM_MASK) == KVM_VM_TYPE_ARM_REALM) { + error_report("KVM rejected a scratch Realm VM with a %d-bit IPA; " + "the requested size may exceed the RMM S2SZ limit", + max_vm_pa_size); + } goto err; } @@ -217,8 +236,8 @@ static int read_sys_reg64(int fd, uint64_t *pret, uint64_t id) static bool kvm_arm_pauth_supported(void) { - return (kvm_check_extension(kvm_state, KVM_CAP_ARM_PTRAUTH_ADDRESS) && - kvm_check_extension(kvm_state, KVM_CAP_ARM_PTRAUTH_GENERIC)); + return (kvm_vm_check_extension(kvm_state, KVM_CAP_ARM_PTRAUTH_ADDRESS) && + kvm_vm_check_extension(kvm_state, KVM_CAP_ARM_PTRAUTH_GENERIC)); } @@ -297,7 +316,7 @@ static bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf) * Ask for SVE if supported, so that we can query ID_AA64ZFR0, * which is otherwise RAZ. */ - sve_supported = kvm_check_extension(kvm_state, KVM_CAP_ARM_SVE); + sve_supported = kvm_vm_check_extension(kvm_state, KVM_CAP_ARM_SVE); if (sve_supported) { init.features[0] |= 1 << KVM_ARM_VCPU_SVE; } @@ -319,7 +338,7 @@ static bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf) 1 << KVM_ARM_VCPU_PTRAUTH_GENERIC); } - if (kvm_check_extension(kvm_state, KVM_CAP_ARM_PMU_V3)) { + if (kvm_vm_check_extension(kvm_state, KVM_CAP_ARM_PMU_V3)) { init.features[0] |= 1 << KVM_ARM_VCPU_PMU_V3; pmu_supported = true; features |= 1ULL << ARM_FEATURE_PMU; @@ -588,8 +607,63 @@ int kvm_arch_get_default_type(MachineState *ms) return fixed_ipa ? 0 : size; } +/* + * RHI device-assignment hypercalls live in the SMCCC Standard Hypervisor + * range and must be forwarded to userspace. Each entry covers a block of + * consecutive function IDs (.nr_functions is the count of IDs starting at + * .base): RHI_DA_VERSION/FEATURES/OBJECT_SIZE/OBJECT_READ (0x4A..0x4D) and + * RHI_DA_VDEV_GET_MEASUREMENTS/GET_INTERFACE_REPORT/SET_TDI_STATE + * (0x52..0x54). + */ +static struct kvm_smccc_filter rhi_da_smccc_filters[] = { + { + .base = RHI_DA_VDEV_GET_MEASUREMENTS, + .nr_functions = 0x3, + .action = KVM_SMCCC_FILTER_FWD_TO_USER, + }, + { + .base = RHI_DA_VERSION, + .nr_functions = 0x4, + .action = KVM_SMCCC_FILTER_FWD_TO_USER, + }, +}; + +/* + * Forward the RHI device-assignment hypercalls to userspace. Only needed for + * realms with assigned devices; the filter is inert for other guests, which + * never issue these function IDs. + */ +static bool kvm_arm_install_rhi_da_filter(KVMState *s) +{ + struct kvm_device_attr attr = { + .group = KVM_ARM_VM_SMCCC_CTRL, + .attr = KVM_ARM_VM_SMCCC_FILTER, + }; + bool installed = true; + unsigned int i; + + if (kvm_vm_ioctl(s, KVM_HAS_DEVICE_ATTR, &attr)) { + warn_report("KVM SMCCC filter not supported; " + "RME device assignment will not work"); + return false; + } + + for (i = 0; i < ARRAY_SIZE(rhi_da_smccc_filters); i++) { + attr.addr = (uintptr_t)&rhi_da_smccc_filters[i]; + + if (kvm_vm_ioctl(s, KVM_SET_DEVICE_ATTR, &attr)) { + warn_report("Failed to install RHI device-assignment " + "SMCCC filter"); + installed = false; + } + } + + return installed; +} + int kvm_arch_init(MachineState *ms, KVMState *s) { + Error *local_err = NULL; int ret = 0; /* For ARM interrupt delivery is always asynchronous, * whether we are using an in-kernel VGIC or not. @@ -604,6 +678,15 @@ int kvm_arch_init(MachineState *ms, KVMState *s) cap_has_mp_state = kvm_check_extension(s, KVM_CAP_MP_STATE); + /* Initialize confidential guest (Realm) if needed */ + if (ms->cgs) { + ret = confidential_guest_kvm_init(ms->cgs, &local_err); + if (ret < 0) { + error_report_err(local_err); + return ret; + } + } + /* Check whether user space can specify guest syndrome value */ cap_has_inject_serror_esr = kvm_check_extension(s, KVM_CAP_ARM_INJECT_SERROR_ESR); @@ -612,10 +695,22 @@ int kvm_arch_init(MachineState *ms, KVMState *s) !kvm_check_extension(s, KVM_CAP_ARM_IRQ_LINE_LAYOUT_2)) { error_report("Using more than 256 vcpus requires a host kernel " "with KVM_CAP_ARM_IRQ_LINE_LAYOUT_2"); - ret = -EINVAL; + return -EINVAL; } - if (kvm_check_extension(s, KVM_CAP_ARM_NISV_TO_USER)) { + /* + * KVM_CAP_ARM_NISV_TO_USER is not on the realm-ext-allowed list + * (kvm_realm_ext_allowed() rejects anything not relevant to + * confidential VMs), so KVM_ENABLE_CAP returns -EINVAL for a realm VM. + * The cap has no effect on realms anyway -- realm faults go through the + * RMM path, not the NISV-to-user path -- so skip the attempt instead of + * producing a spurious error_report. + * + * ms->cgs is the same realm test the series uses for the VM type and for + * the confidential_guest_kvm_init() call above; kvm_arm_rme_vm_type() was + * removed in RFC v2. + */ + if (!ms->cgs && kvm_vm_check_extension(s, KVM_CAP_ARM_NISV_TO_USER)) { if (kvm_vm_enable_cap(s, KVM_CAP_ARM_NISV_TO_USER, 0)) { error_report("Failed to enable KVM_CAP_ARM_NISV_TO_USER cap"); } else { @@ -634,25 +729,31 @@ int kvm_arch_init(MachineState *ms, KVMState *s) warn_report("Eager Page Split support not available"); } else if (!(s->kvm_eager_split_size & sizes)) { error_report("Eager Page Split requested chunk size not valid"); - ret = -EINVAL; + return -EINVAL; } else { ret = kvm_vm_enable_cap(s, KVM_CAP_ARM_EAGER_SPLIT_CHUNK_SIZE, 0, s->kvm_eager_split_size); if (ret < 0) { error_report("Enabling of Eager Page Split failed: %s", strerror(-ret)); + return ret; } } } - max_hw_wps = kvm_check_extension(s, KVM_CAP_GUEST_DEBUG_HW_WPS); + max_hw_wps = kvm_vm_check_extension(s, KVM_CAP_GUEST_DEBUG_HW_WPS); hw_watchpoints = g_array_sized_new(true, true, sizeof(HWWatchpoint), max_hw_wps); - max_hw_bps = kvm_check_extension(s, KVM_CAP_GUEST_DEBUG_HW_BPS); + max_hw_bps = kvm_vm_check_extension(s, KVM_CAP_GUEST_DEBUG_HW_BPS); hw_breakpoints = g_array_sized_new(true, true, sizeof(HWBreakpoint), max_hw_bps); + if (ms->cgs && !kvm_arm_install_rhi_da_filter(s)) { + /* CPU-only Realms remain usable, but device assignment cannot. */ + ms->cgs->assigned_device_memory = false; + } + return ret; } @@ -801,8 +902,18 @@ static uint64_t *kvm_arm_get_cpreg_ptr(ARMCPU *cpu, uint64_t regidx) * cpreg list of arbitrary system registers, false if it is synchronized * by hand using code in kvm_arch_get/put_registers(). */ -static bool kvm_arm_reg_syncs_via_cpreg_list(uint64_t regidx) +static bool kvm_arm_reg_syncs_via_cpreg_list(ARMCPU *cpu, uint64_t regidx) { + /* + * Realm KVM exposes PMCR_EL0 so userspace can discover the counter + * count, but the Realm SET_ONE_REG allow-list does not permit writing + * it. PMU configuration is handled explicitly through the PMU device + * attribute instead. + */ + if (cpu->kvm_rme && regidx == KVM_REG_ARM_PMCR_EL0) { + return false; + } + switch (regidx & KVM_REG_ARM_COPROC_MASK) { case KVM_REG_ARM_CORE: case KVM_REG_ARM64_SVE: @@ -846,7 +957,7 @@ static int kvm_arm_init_cpreg_list(ARMCPU *cpu) qsort(&rlp->reg, rlp->n, sizeof(rlp->reg[0]), compare_u64); for (i = 0, arraylen = 0; i < rlp->n; i++) { - if (!kvm_arm_reg_syncs_via_cpreg_list(rlp->reg[i])) { + if (!kvm_arm_reg_syncs_via_cpreg_list(cpu, rlp->reg[i])) { continue; } switch (rlp->reg[i] & KVM_REG_SIZE_MASK) { @@ -868,7 +979,7 @@ static int kvm_arm_init_cpreg_list(ARMCPU *cpu) for (i = 0, arraylen = 0; i < rlp->n; i++) { uint64_t regidx = rlp->reg[i]; - if (!kvm_arm_reg_syncs_via_cpreg_list(regidx)) { + if (!kvm_arm_reg_syncs_via_cpreg_list(cpu, regidx)) { continue; } cpu->cpreg_indexes[arraylen] = regidx; @@ -890,6 +1001,140 @@ static int kvm_arm_init_cpreg_list(ARMCPU *cpu) return ret; } +static bool kvm_arm_configure_aa64dfr0(ARMCPU *cpu) +{ + int ret; + uint64_t val, newval; + CPUState *cs = CPU(cpu); + + if (!cpu->num_bps && !cpu->num_wps) { + return true; + } + + newval = cpu->isar.idregs[ID_AA64DFR0_EL1_IDX]; + if (cpu->num_bps) { + uint64_t ctx_cmps = FIELD_EX64(newval, ID_AA64DFR0, CTX_CMPS); + + /* CTX_CMPs is never greater than BRPs */ + ctx_cmps = MIN(ctx_cmps, cpu->num_bps - 1); + newval = FIELD_DP64(newval, ID_AA64DFR0, BRPS, cpu->num_bps - 1); + newval = FIELD_DP64(newval, ID_AA64DFR0, CTX_CMPS, ctx_cmps); + } + if (cpu->num_wps) { + newval = FIELD_DP64(newval, ID_AA64DFR0, WRPS, cpu->num_wps - 1); + } + ret = kvm_set_one_reg(cs, KVM_REG_ARM_ID_AA64DFR0_EL1, &newval); + if (ret) { + error_report("Failed to set KVM_REG_ARM_ID_AA64DFR0_EL1: %s", + strerror(-ret)); + return false; + } + + /* + * Check if the write succeeded. KVM does offer the writable mask for this + * register, but this way we also check if the value we wrote was sane. + */ + ret = kvm_get_one_reg(cs, KVM_REG_ARM_ID_AA64DFR0_EL1, &val); + if (ret) { + error_report("Failed to get KVM_REG_ARM_ID_AA64DFR0_EL1: %s", + strerror(-ret)); + return false; + } + + if (val != newval) { + error_report("Failed to update KVM_REG_ARM_ID_AA64DFR0_EL1"); + return false; + } + + return true; +} + +static bool kvm_arm_configure_pmcr(ARMCPU *cpu) +{ + unsigned int nr_counters = cpu->num_pmu_ctrs; + struct kvm_device_attr attr = { + .group = KVM_ARM_VCPU_PMU_V3_CTRL, + .attr = KVM_ARM_VCPU_PMU_V3_SET_NR_COUNTERS, + .addr = (uintptr_t)&nr_counters, + }; + int ret; + uint64_t val, newval; + CPUState *cs = CPU(cpu); + + if (cpu->num_pmu_ctrs == -1) { + return true; + } + + /* An explicit zero is already satisfied when this vCPU has no PMU. */ + if (!cpu->has_pmu && cpu->num_pmu_ctrs == 0) { + return true; + } + + /* + * Realms restrict which registers userspace may write. Prefer the PMU + * device attribute, which configures the VM-wide PMCR_EL0.N value without + * requiring a SET_ONE_REG allow-list entry. Fall back for older kernels. + */ + ret = kvm_vcpu_ioctl(cs, KVM_HAS_DEVICE_ATTR, &attr); + if (!ret) { + ret = kvm_vcpu_ioctl(cs, KVM_SET_DEVICE_ATTR, &attr); + if (ret) { + error_report("Failed to set number of KVM PMU counters: %s", + strerror(-ret)); + return false; + } + return true; + } + + newval = FIELD_DP64(cpu->isar.reset_pmcr_el0, PMCR, N, cpu->num_pmu_ctrs); + ret = kvm_set_one_reg(cs, KVM_REG_ARM_PMCR_EL0, &newval); + if (ret) { + error_report("Failed to set KVM_REG_ARM_PMCR_EL0: %s", + strerror(-ret)); + return false; + } + + /* + * Check if the write succeeded, since older versions of KVM ignore it. + */ + ret = kvm_get_one_reg(cs, KVM_REG_ARM_PMCR_EL0, &val); + if (ret) { + error_report("Failed to get KVM_REG_ARM_PMCR_EL0: %s", + strerror(-ret)); + return false; + } + + if (val != newval) { + error_report("Failed to update KVM_REG_ARM_PMCR_EL0"); + return false; + } + + return true; +} + +/* + * Apply the user-requested overrides for the number of breakpoints, + * watchpoints and PMU counters. + * + * These settings are one-shot in KVM. ID register writes are rejected after + * the VM has run, while KVM_ARM_VCPU_PMU_V3_SET_NR_COUNTERS is rejected once + * KVM_ARM_VCPU_PMU_V3_INIT has created the PMU. kvm_arm_reset_vcpu() runs on + * every guest reset, so only configure them on the first one. + */ +static bool kvm_arm_configure_vcpu_regs(ARMCPU *cpu) +{ + if (cpu->kvm_vcpu_regs_configured) { + return true; + } + + if (!kvm_arm_configure_aa64dfr0(cpu) || !kvm_arm_configure_pmcr(cpu)) { + return false; + } + + cpu->kvm_vcpu_regs_configured = true; + return true; +} + /** * kvm_arm_cpreg_level: * @regidx: KVM register index @@ -1106,6 +1351,20 @@ void kvm_arm_reset_vcpu(ARMCPU *cpu) fprintf(stderr, "kvm_arm_vcpu_init failed: %s\n", strerror(-ret)); abort(); } + + /* + * Before loading the KVM values into CPUState, update the KVM configuration + */ + if (!kvm_arm_configure_vcpu_regs(cpu)) { + /* + * The individual helpers have already reported what went wrong. This + * is a configuration failure rather than an internal inconsistency, + * so exit cleanly instead of dumping core. + */ + error_report("Failed to apply the requested vCPU configuration"); + exit(1); + } + if (!write_kvmstate_to_list(cpu)) { fprintf(stderr, "write_kvmstate_to_list failed\n"); abort(); @@ -1409,6 +1668,10 @@ static void kvm_arm_vm_state_change(void *opaque, bool running, RunState state) { ARMCPU *cpu = opaque; + if (kvm_guest_state_protected()) { + return; + } + if (running) { if (cpu->kvm_adjvtime) { kvm_arm_put_virtual_time(cpu); @@ -1537,6 +1800,467 @@ static bool kvm_arm_handle_debug(ARMCPU *cpu, return false; } +/* + * The RHI device-assignment arguments below all come straight from guest + * registers, so every one of them has to be validated before use. + * + * A vDevice is named by a 32-bit Routing ID (segment:BDF). Reject anything + * wider instead of silently truncating, which would otherwise let a guest + * address a device it did not name. + */ +static bool rhi_da_get_rid(uint64_t reg, uint32_t *rid) +{ + if (reg > UINT32_MAX) { + return false; + } + *rid = reg; + return true; +} + +/* + * The address space RHI buffer IPAs are resolved in. + * + * The guest hands over an IPA, and for a buffer the host has to read or write + * that IPA carries the "shared" top bit. The Realm DMA address space strips + * that bit and targets the host-visible RAM alias, and passes canonical + * addresses through unchanged, so both spellings of the same page work. + */ +static AddressSpace *rhi_guest_buffer_as(void) +{ + return kvm_arm_rme_get_dma_as() ?: &address_space_memory; +} + +static bool rhi_guest_buffer_is_ram(AddressSpace *as, hwaddr addr, + hwaddr size, bool is_write) +{ + MemoryRegion *mr; + hwaddr xlat, len; + + if (!size || size - 1 > HWADDR_MAX - addr) { + return false; + } + + RCU_READ_LOCK_GUARD(); + while (size > 0) { + len = size; + mr = address_space_translate(as, addr, &xlat, &len, is_write, + MEMTXATTRS_UNSPECIFIED); + if (!len || !memory_region_is_ram(mr)) { + return false; + } + size -= len; + addr += len; + } + + return true; +} + +/* + * Map a guest buffer passed to an RHI hypercall. + * + * Refuse anything that is not plain RAM: these buffers are only ever guest + * memory, and letting one land on an emulated device would turn a TSM + * response into arbitrary MMIO writes. + * + * Returns the host address of the whole range, or NULL. On success the + * caller must release it with rhi_unmap_guest_buffer() and the same @len. + */ +static void *rhi_map_guest_buffer(uint64_t ipa, hwaddr len, bool is_write) +{ + AddressSpace *as = rhi_guest_buffer_as(); + hwaddr mapped = len; + void *hva; + + if (!rhi_guest_buffer_is_ram(as, ipa, len, is_write)) { + return NULL; + } + + hva = address_space_map(as, ipa, &mapped, is_write, MEMTXATTRS_UNSPECIFIED); + if (!hva) { + return NULL; + } + if (mapped != len) { + address_space_unmap(as, hva, mapped, is_write, 0); + return NULL; + } + + return hva; +} + +static void rhi_unmap_guest_buffer(void *hva, hwaddr len, bool is_write, + hwaddr access_len) +{ + address_space_unmap(rhi_guest_buffer_as(), hva, len, is_write, access_len); +} + +/* Number of GP registers carrying SMCCC arguments (x0-x5) / results (x0-x3). */ +#define SMCCC_NUM_ARG_REGS 6 +#define SMCCC_NUM_RES_REGS 4 + +/* + * A forwarded SMCCC call. + * + * @in holds the guest's x0-x5, captured before any result is produced, and + * @out holds x0-x3 as they will be written back. Keeping the two apart + * matters: x1-x3 are both argument and result registers, so a handler that + * wrote its status into the vCPU's registers directly would clobber arguments + * it had not read yet. + */ +typedef struct SmcccCall { + uint64_t fn; + uint64_t in[SMCCC_NUM_ARG_REGS]; + uint64_t out[SMCCC_NUM_RES_REGS]; +} SmcccCall; + +static int handle_da_vdev_set_tdi_state(SmcccCall *call) +{ + uint64_t target_state = call->in[2]; + int ret = -EINVAL; + uint32_t guest_rid; + + if (!rhi_da_get_rid(call->in[1], &guest_rid)) { + call->out[0] = RHI_DA_ERROR_INVALID_VDEV_ID; + return 0; + } + + if (target_state == RHI_DA_TDI_CONFIG_LOCKED) { + ret = iommufd_tsm_bind(guest_rid); + } else if (target_state == RHI_DA_TDI_CONFIG_UNLOCKED) { + ret = iommufd_tsm_unbind(guest_rid); + } else if (target_state == RHI_DA_TDI_CONFIG_RUN) { + ret = iommufd_tsm_da_set_tdi_state_run(guest_rid); + } + + if (!ret) { + call->out[0] = RHI_DA_SUCCESS; + } else if (ret == -ENODEV) { + call->out[0] = RHI_DA_ERROR_INVALID_VDEV_ID; + } else { + call->out[0] = RHI_DA_ERROR_INVALID_OBJECT; + } + + /* return to guest. */ + return 0; +} + +static int handle_da_version(SmcccCall *call) +{ + call->out[0] = RHI_DA_VERSION_1_0; + + return 0; +} + +static int handle_da_features(SmcccCall *call) +{ + call->out[0] = RHI_DA_BASE_FEATURE; + + return 0; +} + +/* Match the maximum object size accepted by the RME-DA guest ABI. */ +#define RME_DA_OBJECT_MAX_SIZE (16 * MiB) + +static int handle_da_object_size(SmcccCall *call) +{ + uint64_t object_type = call->in[2]; + uint32_t object_size; + uint32_t guest_rid; + int ret; + + if (!rhi_da_get_rid(call->in[1], &guest_rid)) { + call->out[0] = RHI_DA_ERROR_INVALID_VDEV_ID; + return 0; + } + if (object_type > UINT32_MAX) { + call->out[0] = RHI_DA_ERROR_INVALID_OBJECT; + return 0; + } + + ret = iommufd_tsm_get_da_object_size(guest_rid, object_type, &object_size); + if (!ret) { + call->out[0] = RHI_DA_SUCCESS; + call->out[1] = object_size; + } else if (ret == -ENODEV) { + call->out[0] = RHI_DA_ERROR_INVALID_VDEV_ID; + } else if (ret == -EINVAL) { + call->out[0] = RHI_DA_ERROR_INVALID_OBJECT; + } else { + call->out[0] = RHI_DA_ERROR_DATA_NOT_AVAILABLE; + } + /* return to guest. */ + return 0; +} + +static int handle_da_object_read(SmcccCall *call) +{ + uint64_t object_type = call->in[2]; + uint64_t guest_ipa = call->in[3]; + uint64_t max_len = call->in[4]; + uint64_t offset = call->in[5]; + g_autofree uint8_t *object_data = NULL; + AddressSpace *as = rhi_guest_buffer_as(); + hwaddr destination; + uint32_t object_size; + uint32_t resp_len = 0; + uint32_t guest_rid; + int ret; + + if (!rhi_da_get_rid(call->in[1], &guest_rid)) { + call->out[0] = RHI_DA_ERROR_INVALID_VDEV_ID; + return 0; + } + if (object_type > UINT32_MAX) { + call->out[0] = RHI_DA_ERROR_INVALID_OBJECT; + return 0; + } + + if (!max_len || max_len > RME_DA_OBJECT_MAX_SIZE) { + call->out[0] = RHI_DA_ERROR_INPUT; + return 0; + } + + ret = iommufd_tsm_get_da_object_size(guest_rid, object_type, + &object_size); + if (ret) { + if (ret == -ENODEV) { + call->out[0] = RHI_DA_ERROR_INVALID_VDEV_ID; + } else if (ret == -EINVAL) { + call->out[0] = RHI_DA_ERROR_INVALID_OBJECT; + } else { + call->out[0] = RHI_DA_ERROR_DATA_NOT_AVAILABLE; + } + return 0; + } + if (!object_size || object_size > RME_DA_OBJECT_MAX_SIZE) { + call->out[0] = RHI_DA_ERROR_DATA_NOT_AVAILABLE; + return 0; + } + if (offset > max_len || object_size > max_len - offset) { + call->out[0] = RHI_DA_ERROR_INVALID_OFFSET; + return 0; + } + if (offset > HWADDR_MAX - guest_ipa) { + call->out[0] = RHI_DA_ERROR_ACCESS_FAILED; + return 0; + } + destination = guest_ipa + offset; + if (!rhi_guest_buffer_is_ram(as, destination, object_size, true)) { + call->out[0] = RHI_DA_ERROR_ACCESS_FAILED; + return 0; + } + + object_data = g_try_malloc0(object_size); + if (!object_data) { + call->out[0] = RHI_DA_ERROR_DEVICE; + return 0; + } + + /* + * Fetch from offset zero into a QEMU-owned buffer. The target kernel's + * cached-object helper does not validate a nonzero offset before adding + * it to the source pointer, so never pass guest-controlled offsets to it. + */ + ret = iommufd_tsm_da_object_read(guest_rid, object_type, 0, object_data, + object_size, &resp_len); + if (!ret && resp_len == object_size) { + MemTxResult txret; + + txret = address_space_write(as, destination, MEMTXATTRS_UNSPECIFIED, + object_data, object_size); + if (txret == MEMTX_OK) { + call->out[0] = RHI_DA_SUCCESS; + call->out[1] = object_size; + } else { + call->out[0] = RHI_DA_ERROR_ACCESS_FAILED; + } + } else if (!ret) { + call->out[0] = RHI_DA_ERROR_DATA_NOT_AVAILABLE; + } else if (ret == -EFAULT) { + call->out[0] = RHI_DA_ERROR_ACCESS_FAILED; + } else if (ret == -ENODEV) { + call->out[0] = RHI_DA_ERROR_INVALID_VDEV_ID; + } else if (ret == -EINVAL) { + call->out[0] = RHI_DA_ERROR_INVALID_OBJECT; + } else { + call->out[0] = RHI_DA_ERROR_DATA_NOT_AVAILABLE; + } + return 0; +} + +static int handle_da_get_interface_report(SmcccCall *call) +{ + uint32_t guest_rid; + int ret; + + if (!rhi_da_get_rid(call->in[1], &guest_rid)) { + call->out[0] = RHI_DA_ERROR_INVALID_VDEV_ID; + return 0; + } + + ret = iommufd_tsm_da_get_interface_report(guest_rid); + if (!ret) { + call->out[0] = RHI_DA_SUCCESS; + } else if (ret == -ENODEV) { + call->out[0] = RHI_DA_ERROR_INVALID_VDEV_ID; + } else { + call->out[0] = RHI_DA_ERROR_INPUT; + } + /* return to guest. */ + return 0; +} + +static int handle_da_get_measurements(SmcccCall *call) +{ + struct rhi_vdev_measurement_params *param_hva; + const hwaddr len = sizeof(*param_hva); + uint32_t guest_rid; + int ret; + + if (!rhi_da_get_rid(call->in[1], &guest_rid)) { + call->out[0] = RHI_DA_ERROR_INVALID_VDEV_ID; + return 0; + } + + param_hva = rhi_map_guest_buffer(call->in[2], len, false); + if (!param_hva) { + call->out[0] = RHI_DA_ERROR_ACCESS_FAILED; + return 0; + } + + /* + * The kernel copies the nonce while servicing the ioctl. A concurrent + * guest update can only change the nonce selected by that same guest. + */ + ret = iommufd_tsm_da_get_measurement(guest_rid, param_hva); + if (!ret) { + call->out[0] = RHI_DA_SUCCESS; + } else if (ret == -EFAULT) { + call->out[0] = RHI_DA_ERROR_ACCESS_FAILED; + } else if (ret == -ENODEV) { + call->out[0] = RHI_DA_ERROR_INVALID_VDEV_ID; + } else { + call->out[0] = RHI_DA_ERROR_INPUT; + } + + /* Read-only mapping of the guest parameter block; nothing to flush back. */ + rhi_unmap_guest_buffer(param_hva, len, false, 0); + return 0; +} + +static int handle_std_hyp_call(SmcccCall *call) +{ + /* + * SMCCC leaves x1-x3 as result registers. Start from zero so that a + * handler which only produces a status code does not echo the guest's own + * arguments back to it. + */ + memset(call->out, 0, sizeof(call->out)); + + switch (call->fn) { + case RHI_DA_VERSION: + return handle_da_version(call); + case RHI_DA_FEATURES: + return handle_da_features(call); + case RHI_DA_OBJECT_SIZE: + return handle_da_object_size(call); + case RHI_DA_OBJECT_READ: + return handle_da_object_read(call); + case RHI_DA_VDEV_GET_INTERFACE_REPORT: + return handle_da_get_interface_report(call); + case RHI_DA_VDEV_GET_MEASUREMENTS: + return handle_da_get_measurements(call); + case RHI_DA_VDEV_SET_TDI_STATE: + return handle_da_vdev_set_tdi_state(call); + default: + /* + * The SMCCC filter installed by kvm_arm_install_rhi_da_filter() + * should keep this unreachable, but never leave the function ID + * itself sitting in x0 as if it were a result. + */ + qemu_log_mask(LOG_UNIMP, "%s: unhandled SMCCC function 0x%" PRIx64 "\n", + __func__, call->fn); + call->out[0] = (uint64_t)SMCCC_RET_NOT_SUPPORTED; + return 0; + } +} + +static int handle_arm64_tio_exit(struct kvm_run *kvm_run) +{ + uint64_t gpa_base, gpa_top, pa_base; + uint32_t rid; + bool accepted; + + if (kvm_run->cca_exit.nr != RMI_EXIT_VDEV_MAP) { + error_report("unsupported KVM_EXIT_ARM64_TIO operation 0x%" PRIx64, + (uint64_t)kvm_run->cca_exit.nr); + return -EINVAL; + } + + gpa_base = kvm_run->cca_exit.gpa_base; + gpa_top = kvm_run->cca_exit.gpa_top; + pa_base = kvm_run->cca_exit.pa_base; + + if (kvm_run->cca_exit.vdev_id > UINT32_MAX) { + accepted = false; + } else { + rid = kvm_run->cca_exit.vdev_id; + accepted = iommufd_tsm_dev_memmap_exit(rid, gpa_base, gpa_top, + pa_base); + } + + /* KVM treats response as a boolean and constructs the RMM flag itself. */ + kvm_run->cca_exit.response = accepted ? 0 : 1; + return 0; +} + +#define AARCH64_CORE_REG(x) (KVM_REG_ARM64 | KVM_REG_SIZE_U64 | \ + KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(x)) + +/* + * SMCCC Standard Hypervisor (ARM_SMCCC_OWNER_STANDARD_HYP) call forwarded to + * userspace by the filter installed in kvm_arm_install_rhi_da_filter(). + * + * KVM does not sync the GP registers on a hypercall exit, so read the + * registers carrying the SMCCC arguments before dispatch and write the + * results back afterwards. + * + * TODO: replace this with first-class SMCCC register handling. + */ +static int kvm_arm_handle_hypercall(CPUState *cs, struct kvm_run *run) +{ + SmcccCall call = { .fn = run->hypercall.nr }; + int ret; + int i; + + for (i = 0; i < SMCCC_NUM_ARG_REGS; i++) { + ret = kvm_get_one_reg(cs, AARCH64_CORE_REG(regs.regs[i]), &call.in[i]); + if (ret) { + return ret; + } + } + + /* + * RHI dispatch looks up and operates on VFIO devices. The global VFIO + * device list and device lifetime are protected by the BQL, which is not + * held while kvm_cpu_exec() handles architecture-specific exits. + */ + bql_lock(); + ret = handle_std_hyp_call(&call); + bql_unlock(); + if (ret) { + return ret; + } + + for (i = 0; i < SMCCC_NUM_RES_REGS; i++) { + ret = kvm_set_one_reg(cs, AARCH64_CORE_REG(regs.regs[i]), &call.out[i]); + if (ret) { + return ret; + } + } + + return 0; +} + int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run) { ARMCPU *cpu = ARM_CPU(cs); @@ -1553,11 +2277,24 @@ int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run) ret = kvm_arm_handle_dabt_nisv(cpu, run->arm_nisv.esr_iss, run->arm_nisv.fault_ipa); break; + case KVM_EXIT_ARM64_TIO: + /* + * See the VFIO device-lifetime comment in + * kvm_arm_handle_hypercall(). + */ + bql_lock(); + ret = handle_arm64_tio_exit(run); + bql_unlock(); + break; + case KVM_EXIT_HYPERCALL: + ret = kvm_arm_handle_hypercall(cs, run); + break; default: qemu_log_mask(LOG_UNIMP, "%s: un-handled exit reason %d\n", __func__, run->exit_reason); break; } + return ret; } @@ -1888,7 +2625,7 @@ void kvm_arm_pvtime_init(ARMCPU *cpu, uint64_t ipa) void kvm_arm_steal_time_finalize(ARMCPU *cpu, Error **errp) { - bool has_steal_time = kvm_check_extension(kvm_state, KVM_CAP_STEAL_TIME); + bool has_steal_time = kvm_vm_check_extension(kvm_state, KVM_CAP_STEAL_TIME); if (cpu->kvm_steal_time == ON_OFF_AUTO_AUTO) { if (!has_steal_time || !arm_feature(&cpu->env, ARM_FEATURE_AARCH64)) { @@ -1918,17 +2655,17 @@ void kvm_arm_steal_time_finalize(ARMCPU *cpu, Error **errp) bool kvm_arm_aarch32_supported(void) { - return kvm_check_extension(kvm_state, KVM_CAP_ARM_EL1_32BIT); + return kvm_vm_check_extension(kvm_state, KVM_CAP_ARM_EL1_32BIT); } bool kvm_arm_el2_supported(void) { - return kvm_check_extension(kvm_state, KVM_CAP_ARM_EL2); + return kvm_vm_check_extension(kvm_state, KVM_CAP_ARM_EL2); } bool kvm_arm_mte_supported(void) { - return kvm_check_extension(kvm_state, KVM_CAP_ARM_MTE); + return kvm_vm_check_extension(kvm_state, KVM_CAP_ARM_MTE); } QEMU_BUILD_BUG_ON(KVM_ARM64_SVE_VQ_MIN != 1); @@ -1994,6 +2731,8 @@ int kvm_arch_init_vcpu(CPUState *cs) cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_HAS_EL2; } + kvm_arm_rme_vcpu_init(cpu); + /* Do KVM_ARM_VCPU_INIT ioctl */ ret = kvm_arm_vcpu_init(cpu); if (ret) { @@ -2076,9 +2815,6 @@ static void kvm_inject_arm_sea(CPUState *c) arm_cpu_do_interrupt(c); } -#define AARCH64_CORE_REG(x) (KVM_REG_ARM64 | KVM_REG_SIZE_U64 | \ - KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(x)) - #define AARCH64_SIMD_CORE_REG(x) (KVM_REG_ARM64 | KVM_REG_SIZE_U128 | \ KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(x)) @@ -2148,7 +2884,30 @@ static int kvm_arch_put_sve(CPUState *cs, uint32_t vq, bool have_ffr) return 0; } -int kvm_arch_put_registers(CPUState *cs, KvmPutState level, Error **errp) +static int kvm_arm_rme_put_core_regs(CPUState *cs, Error **errp) +{ + int i, ret; + ARMCPU *cpu = ARM_CPU(cs); + CPUARMState *env = &cpu->env; + + /* The RME ABI only allows us to set 8 GPRs and the PC */ + for (i = 0; i < 8; i++) { + ret = kvm_set_one_reg(cs, AARCH64_CORE_REG(regs.regs[i]), + &env->xregs[i]); + if (ret) { + return ret; + } + } + + ret = kvm_set_one_reg(cs, AARCH64_CORE_REG(regs.pc), &env->pc); + if (ret) { + return ret; + } + + return 0; +} + +static int kvm_arm_put_core_regs(CPUState *cs, Error **errp) { uint64_t val; uint32_t fpr; @@ -2251,6 +3010,23 @@ int kvm_arch_put_registers(CPUState *cs, KvmPutState level, Error **errp) return ret; } + return 0; +} + +int kvm_arch_put_registers(CPUState *cs, KvmPutState level, Error **errp) +{ + int ret; + ARMCPU *cpu = ARM_CPU(cs); + + if (cpu->kvm_rme) { + ret = kvm_arm_rme_put_core_regs(cs, errp); + } else { + ret = kvm_arm_put_core_regs(cs, errp); + } + if (ret) { + return ret; + } + write_cpustate_to_list(cpu, true); if (!write_list_to_kvmstate(cpu, level)) { @@ -2333,7 +3109,24 @@ static int kvm_arch_get_sve(CPUState *cs, uint32_t vq, bool have_ffr) return 0; } -int kvm_arch_get_registers(CPUState *cs, Error **errp) +static int kvm_arm_rme_get_core_regs(CPUState *cs, Error **errp) +{ + int i, ret; + ARMCPU *cpu = ARM_CPU(cs); + CPUARMState *env = &cpu->env; + + for (i = 0; i < 8; i++) { + ret = kvm_get_one_reg(cs, AARCH64_CORE_REG(regs.regs[i]), + &env->xregs[i]); + if (ret) { + return ret; + } + } + + return 0; +} + +static int kvm_arm_get_core_regs(CPUState *cs, Error **errp) { uint64_t val; unsigned int el; @@ -2436,6 +3229,23 @@ int kvm_arch_get_registers(CPUState *cs, Error **errp) } vfp_set_fpcr(env, fpr); + return 0; +} + +int kvm_arch_get_registers(CPUState *cs, Error **errp) +{ + int ret; + ARMCPU *cpu = ARM_CPU(cs); + + if (cpu->kvm_rme) { + ret = kvm_arm_rme_get_core_regs(cs, errp); + } else { + ret = kvm_arm_get_core_regs(cs, errp); + } + if (ret) { + return ret; + } + ret = kvm_get_vcpu_events(cpu); if (ret) { return ret; diff --git a/target/arm/kvm_arm.h b/target/arm/kvm_arm.h index e7c40fb003e..8b0706175de 100644 --- a/target/arm/kvm_arm.h +++ b/target/arm/kvm_arm.h @@ -18,6 +18,17 @@ #define KVM_ARM_VGIC_V2 (1 << 0) #define KVM_ARM_VGIC_V3 (1 << 1) +/* + * Keep the machine model independent of host Linux headers. These values + * mirror KVM_VM_TYPE_ARM_NORMAL and KVM_VM_TYPE_ARM_REALM. + */ +#define QEMU_KVM_ARM_VM_TYPE_SHIFT 8 +#define QEMU_KVM_ARM_VM_TYPE(type) \ + (((type) << QEMU_KVM_ARM_VM_TYPE_SHIFT) & \ + (0xfULL << QEMU_KVM_ARM_VM_TYPE_SHIFT)) +#define QEMU_KVM_ARM_VM_TYPE_NORMAL QEMU_KVM_ARM_VM_TYPE(0) +#define QEMU_KVM_ARM_VM_TYPE_REALM QEMU_KVM_ARM_VM_TYPE(1) + /** * kvm_arm_register_device: * @mr: memory region for this device @@ -240,4 +251,39 @@ void arm_gic_cap_kvm_probe(GICCapability *v2, GICCapability *v3); */ char *kvm_print_register_name(uint64_t regidx); +/** + * kvm_arm_rme_available: + * + * Return whether the current KVM VM supports Arm Realms. + */ +bool kvm_arm_rme_available(void); + +/** + * kvm_arm_rme_vcpu_init + * @cs: the CPU + * + * If the user requested a Realm, setup the given vCPU accordingly. Realm vCPUs + * behave a little differently, for example most of their register state is + * hidden from the host. + */ +void kvm_arm_rme_vcpu_init(ARMCPU *cpu); + +/** + * kvm_arm_rme_init_gpa_space + * @ipa_bits: size of the full Realm IPA space, including the shared bit + * @pci_bus: The main PCI bus, for which PCI queries DMA address spaces + * + * Setup the guest-physical address space for a Realm. Install a memory region + * and notifier to manage the shared upper half of the address space. + */ +void kvm_arm_rme_init_gpa_space(unsigned int ipa_bits, PCIBus *pci_bus); + +/** + * kvm_arm_rme_get_dma_as: + * + * Return the shared-IPA DMA address space for a Realm, or NULL for a + * non-Realm machine or before the Realm address space has been initialized. + */ +AddressSpace *kvm_arm_rme_get_dma_as(void); + #endif diff --git a/target/arm/meson.build b/target/arm/meson.build index 6e0e504a403..a0d25c2d4c4 100644 --- a/target/arm/meson.build +++ b/target/arm/meson.build @@ -19,7 +19,10 @@ arm_common_ss.add(files( arm_common_system_ss.add(files( 'arm-qmp-cmds.c', )) -arm_system_ss.add(when: 'CONFIG_KVM', if_true: files('hyp_gdbstub.c', 'kvm.c')) +arm_system_ss.add(when: 'CONFIG_KVM', if_true: files( + 'hyp_gdbstub.c', + 'kvm.c', + 'kvm-rme.c')) arm_system_ss.add(when: 'CONFIG_HVF', if_true: files('hyp_gdbstub.c')) arm_user_ss.add(files('cpu.c')) diff --git a/tests/qtest/arm-virt-machine-test.c b/tests/qtest/arm-virt-machine-test.c new file mode 100644 index 00000000000..311a144951f --- /dev/null +++ b/tests/qtest/arm-virt-machine-test.c @@ -0,0 +1,45 @@ +/* + * Arm virt machine tests + * + * SPDX-License-Identifier: GPL-2.0-or-later + */ + +#include "qemu/osdep.h" +#include "qemu/units.h" +#include "libqtest.h" + +#define VIRT_FLASH_BANK_SIZE (64 * MiB) + +static void test_pflash_property(gconstpointer data) +{ + const char *property = data; + g_autofree char *path = NULL; + QTestState *qts; + int fd; + + fd = g_file_open_tmp("qtest-arm-virt-pflash-XXXXXX", &path, NULL); + g_assert_cmpint(fd, >=, 0); + g_assert_cmpint(ftruncate(fd, VIRT_FLASH_BANK_SIZE), ==, 0); + close(fd); + + qts = qtest_initf("-machine virt,%s=flash -accel qtest -nodefaults " + "-drive if=none,id=flash,format=raw,file=%s", + property, path); + qtest_quit(qts); + + unlink(path); +} + +int main(int argc, char **argv) +{ + g_test_init(&argc, &argv, NULL); + + if (qtest_has_machine("virt")) { + qtest_add_data_func("/arm/virt/pflash0-property", "pflash0", + test_pflash_property); + qtest_add_data_func("/arm/virt/pflash1-property", "pflash1", + test_pflash_property); + } + + return g_test_run(); +} diff --git a/tests/qtest/meson.build b/tests/qtest/meson.build index be4fa627b5f..c690c418749 100644 --- a/tests/qtest/meson.build +++ b/tests/qtest/meson.build @@ -249,6 +249,7 @@ qtests_arm = \ (config_all_devices.has_key('CONFIG_STM32L4X5_SOC') and config_all_devices.has_key('CONFIG_DM163')? ['dm163-test'] : []) + \ ['arm-cpu-features', + 'arm-virt-machine-test', 'boot-serial-test'] # TODO: once aarch64 TCG is fixed on ARM 32 bit host, make bios-tables-test unconditional @@ -268,6 +269,7 @@ qtests_aarch64 = \ ['iommu-smmuv3-test'] : []) + \ qtests_cxl + \ ['arm-cpu-features', + 'arm-virt-machine-test', 'numa-test', 'boot-serial-test', 'migration-test']