From 1d831a32a914c42a9fc467b63ca88f98b1e5d211 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gabriel=20Beims=20Br=C3=A4scher?= Date: Thu, 15 Jul 2021 04:32:44 -0300 Subject: [PATCH] kvmk: KVM NFS disk IO driver supporting IO_URING (#5012) Currently there is no disk IO driver configuration for VMs running on KVM. That's OK for most the cases; however, recently there have been added some quite interesting optimizations with the IO driver io_uring. Note that IO URING requires: Qemu >= 5.0, and Libvirt >= 6.3.0. By using io_uring we can see a massive I/O performance improvement within Virtual Machines running from Local and/or NFS storage. This implementation enhances the KVM disk configuration by adding workflow for setting the disk IO drivers. Additionally, if the Qemu and Libvirt versions matches with the required for having io_uring we are going to set it on the VM. If there is no support for such driver we keep it as it is nowadays, without any IO driver configured. Fixes: #4883 --- .../resource/LibvirtComputingResource.java | 24 +++++++++++++ .../hypervisor/kvm/resource/LibvirtVMDef.java | 34 ++++++++++++++++++ .../LibvirtComputingResourceTest.java | 36 +++++++++++++++++++ 3 files changed, 94 insertions(+) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 8b7932652c0..8255afe514d 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -253,6 +253,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv protected static final String DEFAULT_OVS_VIF_DRIVER_CLASS_NAME = "com.cloud.hypervisor.kvm.resource.OvsVifDriver"; protected static final String DEFAULT_BRIDGE_VIF_DRIVER_CLASS_NAME = "com.cloud.hypervisor.kvm.resource.BridgeVifDriver"; + private final static long HYPERVISOR_LIBVIRT_VERSION_SUPPORTS_IO_URING = 6003000; + private final static long HYPERVISOR_QEMU_VERSION_SUPPORTS_IO_URING = 5000000; protected HypervisorType _hypervisorType; protected String _hypervisorURI; @@ -342,6 +344,14 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv protected MemStat _memStat = new MemStat(_dom0MinMem, _dom0OvercommitMem); private final LibvirtUtilitiesHelper libvirtUtilitiesHelper = new LibvirtUtilitiesHelper(); + protected long getHypervisorLibvirtVersion() { + return _hypervisorLibvirtVersion; + } + + protected long getHypervisorQemuVersion() { + return _hypervisorQemuVersion; + } + @Override public ExecutionResult executeInVR(final String routerIp, final String script, final String args) { return executeInVR(routerIp, script, args, _timeout); @@ -2613,6 +2623,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv disk.setDiscard(DiscardType.UNMAP); } + setDiskIoDriver(disk); + if (pool.getType() == StoragePoolType.RBD) { /* For RBD pools we use the secret mechanism in libvirt. @@ -2713,6 +2725,18 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv return storagePool.getPhysicalDisk(data.getPath()); } + /** + * Set Disk IO Driver, if supported by the Libvirt/Qemu version. + * IO Driver works for: + * (i) Qemu >= 5.0; + * (ii) Libvirt >= 6.3.0 + */ + protected void setDiskIoDriver(DiskDef disk) { + if (getHypervisorLibvirtVersion() >= HYPERVISOR_LIBVIRT_VERSION_SUPPORTS_IO_URING && getHypervisorQemuVersion() >= HYPERVISOR_QEMU_VERSION_SUPPORTS_IO_URING) { + disk.setIoDriver(DiskDef.IoDriver.IOURING); + } + } + private KVMPhysicalDisk getPhysicalDiskFromNfsStore(String dataStoreUrl, DataTO data) { final String volPath = dataStoreUrl + File.separator + data.getPath(); final int index = volPath.lastIndexOf("/"); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java index 1927ddd3569..43e840cc548 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java @@ -651,6 +651,26 @@ public class LibvirtVMDef { } + /** + * This enum specifies IO Drivers, each option controls specific policies on I/O. + * Qemu guests support "threads" and "native" options Since 0.8.8 ; "io_uring" is supported Since 6.3.0 (QEMU 5.0). + */ + public enum IoDriver { + NATIVE("native"), + THREADS("threads"), + IOURING("io_uring"); + String ioDriver; + + IoDriver(String driver) { + ioDriver = driver; + } + + @Override + public String toString() { + return ioDriver; + } + } + private DeviceType _deviceType; /* floppy, disk, cdrom */ private DiskType _diskType; private DiskProtocol _diskProtocol; @@ -681,6 +701,7 @@ public class LibvirtVMDef { private String _serial; private boolean qemuDriver = true; private DiscardType _discard = DiscardType.IGNORE; + private IoDriver ioDriver; public DiscardType getDiscard() { return _discard; @@ -690,6 +711,14 @@ public class LibvirtVMDef { this._discard = discard; } + public DiskDef.IoDriver getIoDriver() { + return ioDriver; + } + + public void setIoDriver(IoDriver ioDriver) { + this.ioDriver = ioDriver; + } + public void setDeviceType(DeviceType deviceType) { _deviceType = deviceType; } @@ -1004,6 +1033,11 @@ public class LibvirtVMDef { if(_discard != null && _discard != DiscardType.IGNORE) { diskBuilder.append("discard='" + _discard.toString() + "' "); } + + if(ioDriver != null) { + diskBuilder.append(String.format("io='%s'", ioDriver)); + } + diskBuilder.append("/>\n"); } diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java index 84d08531c2b..1020ff86e3a 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java @@ -202,6 +202,9 @@ public class LibvirtComputingResourceTest { @Mock LibvirtVMDef vmDef; + private final static long HYPERVISOR_LIBVIRT_VERSION_SUPPORTS_IOURING = 6003000; + private final static long HYPERVISOR_QEMU_VERSION_SUPPORTS_IOURING = 5000000; + String hyperVisorType = "kvm"; Random random = new Random(); final String memInfo = "MemTotal: 5830236 kB\n" + @@ -5209,4 +5212,37 @@ public class LibvirtComputingResourceTest { libvirtComputingResource.addExtraConfigComponent(extraConfig, vmDef); Mockito.verify(vmDef, times(1)).addComp(any()); } + + @Test + public void setDiskIoDriverTestIoUring() { + DiskDef diskDef = configureAndTestSetDiskIoDriverTest(HYPERVISOR_LIBVIRT_VERSION_SUPPORTS_IOURING, HYPERVISOR_QEMU_VERSION_SUPPORTS_IOURING); + Assert.assertEquals(DiskDef.IoDriver.IOURING, diskDef.getIoDriver()); + } + + @Test + public void setDiskIoDriverTestLibvirtSupportsIoUring() { + DiskDef diskDef = configureAndTestSetDiskIoDriverTest(123l, HYPERVISOR_QEMU_VERSION_SUPPORTS_IOURING); + Assert.assertNotEquals(DiskDef.IoDriver.IOURING, diskDef.getIoDriver()); + } + + @Test + public void setDiskIoDriverTestQemuSupportsIoUring() { + DiskDef diskDef = configureAndTestSetDiskIoDriverTest(HYPERVISOR_LIBVIRT_VERSION_SUPPORTS_IOURING, 123l); + Assert.assertNotEquals(DiskDef.IoDriver.IOURING, diskDef.getIoDriver()); + } + + @Test + public void setDiskIoDriverTestNoSupportToIoUring() { + DiskDef diskDef = configureAndTestSetDiskIoDriverTest(123l, 123l); + Assert.assertNotEquals(DiskDef.IoDriver.IOURING, diskDef.getIoDriver()); + } + + private DiskDef configureAndTestSetDiskIoDriverTest(long hypervisorLibvirtVersion, long hypervisorQemuVersion) { + DiskDef diskDef = new DiskDef(); + LibvirtComputingResource libvirtComputingResourceSpy = Mockito.spy(new LibvirtComputingResource()); + Mockito.when(libvirtComputingResourceSpy.getHypervisorLibvirtVersion()).thenReturn(hypervisorLibvirtVersion); + Mockito.when(libvirtComputingResourceSpy.getHypervisorQemuVersion()).thenReturn(hypervisorQemuVersion); + libvirtComputingResourceSpy.setDiskIoDriver(diskDef); + return diskDef; + } }