CLOUDSTACK-8715: Add channel to Instances for Qemu Guest Agent

This commit adds a additional VirtIO channel with the name
'org.qemu.guest_agent.0' to all Instances.

With the Qemu Guest Agent the Hypervisor gains more control over the Instance if
these tools are present inside the Instance, for example:

* Power control
* Flushing filesystems
* Fetching Network information

In the future this should allow safer snapshots on KVM since we can instruct the
Instance to flush the filesystems prior to snapshotting the disk.

More information: http://wiki.qemu.org/Features/QAPI/GuestAgent

Keep in mind that on Ubuntu AppArmor still needs to be disabled since the default
AppArmor profile doesn't allow libvirt to write into /var/lib/libvirt/qemu

This commit does not add any communication methods through API-calls, it merely
adds the channel to the Instances and installs the Guest Agent in the SSVMs.

With the addition of the Qemu Guest Agent channel a second channel appears in /dev
on a SSVM as a VirtIO port.

The order in which the ports are defined in the XML matters for the naming inside
the SSVM VM and by not relying on /dev/vportXX but looking for a static name the
SSVM still boots properly if the order in the XML definition is changed.

A SSVM with both ports attached will have something like this:

  root@v-215-VM:~# ls -l /dev/virtio-ports
  total 0
  lrwxrwxrwx 1 root root 11 May 13 21:41 org.qemu.guest_agent.0 -> ../vport0p2
  lrwxrwxrwx 1 root root 11 May 13 21:41 v-215-VM.vport -> ../vport0p1
  root@v-215-VM:~# ls -l /dev/vport*
  crw------- 1 root root 251, 1 May 13 21:41 /dev/vport0p1
  crw------- 1 root root 251, 2 May 13 21:41 /dev/vport0p2
  root@v-215-VM:~#

In this case the SSVM port points to /dev/vport0p1, but if the order in the XML
is different it might point to /dev/vport0p2

By looking for a portname with a pre-defined pattern in /dev/virtio-ports we
do not rely on the order in the XML definition.

Signed-off-by: Wido den Hollander <wido@widodh.nl>
This commit is contained in:
Wido den Hollander 2016-05-13 11:04:49 +02:00
parent 9bec7032be
commit 2a5f37c1b1
No known key found for this signature in database
GPG Key ID: 019B582DDB3ECA42
9 changed files with 208 additions and 20 deletions

View File

@ -56,6 +56,11 @@ zone=default
# local storage path, by default, it's /var/lib/libvirt/images/
#local.storage.path=/var/lib/libvirt/images/
# Qemu socket path, directory where Qemu sockets are placed.
# These sockets are for the Qemu Guest Agent and SSVM privisioning
# Make sure that AppArmor or SELinux allow libvirt to write there
#qemu.sockets.path=/var/lib/libvirt/qemu
# The UUID for the local storage pool, this is mandatory!
# Generate with "uuidgen"
local.storage.uuid=

View File

@ -97,6 +97,7 @@ import com.cloud.dc.Vlan;
import com.cloud.exception.InternalErrorException;
import com.cloud.host.Host.Type;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.ChannelDef;
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.ClockDef;
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.ConsoleDef;
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.CpuModeDef;
@ -116,7 +117,6 @@ import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.InterfaceDef.GuestNetType;
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.SerialDef;
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.TermPolicy;
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.VideoDef;
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.VirtioSerialDef;
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.RngDef;
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.RngDef.RngBackendModel;
import com.cloud.hypervisor.kvm.resource.wrapper.LibvirtRequestWrapper;
@ -139,6 +139,7 @@ import com.cloud.storage.resource.StorageSubsystemCommandHandler;
import com.cloud.storage.resource.StorageSubsystemCommandHandlerBase;
import com.cloud.utils.ExecutionResult;
import com.cloud.utils.NumbersUtil;
import com.cloud.utils.StringUtils;
import com.cloud.utils.Pair;
import com.cloud.utils.PropertiesUtil;
import com.cloud.utils.exception.CloudRuntimeException;
@ -253,6 +254,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
protected String _rngPath = "/dev/random";
protected int _rngRatePeriod = 1000;
protected int _rngRateBytes = 2048;
private File _qemuSocketsPath;
private final String _qemuGuestAgentSocketName = "org.qemu.guest_agent.0";
private final Map <String, String> _pifs = new HashMap<String, String>();
private final Map<String, VmStats> _vmStats = new ConcurrentHashMap<String, VmStats>();
@ -777,6 +780,13 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
_localStoragePath = "/var/lib/libvirt/images/";
}
/* Directory to use for Qemu sockets like for the Qemu Guest Agent */
_qemuSocketsPath = new File("/var/lib/libvirt/qemu");
String _qemuSocketsPathVar = (String)params.get("qemu.sockets.path");
if (_qemuSocketsPathVar != null && StringUtils.isNotBlank(_qemuSocketsPathVar)) {
_qemuSocketsPath = new File(_qemuSocketsPathVar);
}
final File storagePath = new File(_localStoragePath);
_localStoragePath = storagePath.getAbsolutePath();
@ -2008,9 +2018,10 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
final SerialDef serial = new SerialDef("pty", null, (short)0);
devices.addDevice(serial);
/* Add a VirtIO channel for SystemVMs for communication and provisioning */
if (vmTO.getType() != VirtualMachine.Type.User) {
final VirtioSerialDef vserial = new VirtioSerialDef(vmTO.getName(), null);
devices.addDevice(vserial);
devices.addDevice(new ChannelDef(vmTO.getName() + ".vport", ChannelDef.ChannelType.UNIX,
new File(_qemuSocketsPath + "/" + vmTO.getName() + ".agent")));
}
if (_rngEnable) {
@ -2018,6 +2029,10 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
devices.addDevice(rngDevice);
}
/* Add a VirtIO channel for the Qemu Guest Agent tools */
devices.addDevice(new ChannelDef(_qemuGuestAgentSocketName, ChannelDef.ChannelType.UNIX,
new File(_qemuSocketsPath + "/" + vmTO.getName() + "." + _qemuGuestAgentSocketName)));
final VideoDef videoCard = new VideoDef(_videoHw, _videoRam);
devices.addDevice(videoCard);

View File

@ -16,9 +16,11 @@
// under the License.
package com.cloud.hypervisor.kvm.resource;
import java.io.File;
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import javax.xml.parsers.DocumentBuilder;
@ -34,6 +36,8 @@ import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import com.cloud.utils.StringUtils;
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.ChannelDef;
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.DiskDef;
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.InterfaceDef;
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.InterfaceDef.NicModel;
@ -45,6 +49,7 @@ public class LibvirtDomainXMLParser {
private final List<InterfaceDef> interfaces = new ArrayList<InterfaceDef>();
private final List<DiskDef> diskDefs = new ArrayList<DiskDef>();
private final List<RngDef> rngDefs = new ArrayList<RngDef>();
private final List<ChannelDef> channels = new ArrayList<ChannelDef>();
private Integer vncPort;
private String desc;
@ -175,6 +180,26 @@ public class LibvirtDomainXMLParser {
interfaces.add(def);
}
NodeList ports = devices.getElementsByTagName("channel");
for (int i = 0; i < ports.getLength(); i++) {
Element channel = (Element)ports.item(i);
String type = channel.getAttribute("type");
String path = getAttrValue("source", "path", channel);
String name = getAttrValue("target", "name", channel);
String state = getAttrValue("target", "state", channel);
ChannelDef def = null;
if (!StringUtils.isNotBlank(state)) {
def = new ChannelDef(name, ChannelDef.ChannelType.valueOf(type.toUpperCase()), new File(path));
} else {
def = new ChannelDef(name, ChannelDef.ChannelType.valueOf(type.toUpperCase()),
ChannelDef.ChannelState.valueOf(state.toUpperCase()), new File(path));
}
channels.add(def);
}
Element graphic = (Element)devices.getElementsByTagName("graphics").item(0);
if (graphic != null) {
@ -261,6 +286,10 @@ public class LibvirtDomainXMLParser {
return rngDefs;
}
public List<ChannelDef> getChannels() {
return Collections.unmodifiableList(channels);
}
public String getDescription() {
return desc;
}

View File

@ -20,6 +20,7 @@ import com.google.common.collect.Maps;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.log4j.Logger;
import java.io.File;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@ -1209,25 +1210,95 @@ public class LibvirtVMDef {
}
}
public static class VirtioSerialDef {
private final String _name;
private String _path;
public final static class ChannelDef {
enum ChannelType {
UNIX("unix"), SERIAL("serial");
String type;
public VirtioSerialDef(String name, String path) {
_name = name;
_path = path;
ChannelType(String type) {
this.type = type;
}
@Override
public String toString() {
return this.type;
}
}
enum ChannelState {
DISCONNECTED("disconnected"), CONNECTED("connected");
String type;
ChannelState(String type) {
this.type = type;
}
@Override
public String toString() {
return type;
}
}
private final String name;
private File path = new File("");
private final ChannelType type;
private ChannelState state;
public ChannelDef(String name, ChannelType type) {
this.name = name;
this.type = type;
}
public ChannelDef(String name, ChannelType type, File path) {
this.name = name;
this.path = path;
this.type = type;
}
public ChannelDef(String name, ChannelType type, ChannelState state) {
this.name = name;
this.state = state;
this.type = type;
}
public ChannelDef(String name, ChannelType type, ChannelState state, File path) {
this.name = name;
this.path = path;
this.state = state;
this.type = type;
}
public ChannelType getChannelType() {
return type;
}
public ChannelState getChannelState() {
return state;
}
public String getName() {
return name;
}
public File getPath() {
return path;
}
@Override
public String toString() {
StringBuilder virtioSerialBuilder = new StringBuilder();
if (_path == null) {
_path = "/var/lib/libvirt/qemu";
virtioSerialBuilder.append("<channel type='" + type.toString() + "'>\n");
if (path == null) {
virtioSerialBuilder.append("<source mode='bind'/>\n");
} else {
virtioSerialBuilder.append("<source mode='bind' path='" + path.toString() + "'/>\n");
}
virtioSerialBuilder.append("<channel type='unix'>\n");
virtioSerialBuilder.append("<source mode='bind' path='" + _path + "/" + _name + ".agent'/>\n");
virtioSerialBuilder.append("<target type='virtio' name='" + _name + ".vport'/>\n");
virtioSerialBuilder.append("<address type='virtio-serial'/>\n");
if (state == null) {
virtioSerialBuilder.append("<target type='virtio' name='" + name + "'/>\n");
} else {
virtioSerialBuilder.append("<target type='virtio' name='" + name + "' state='" + state.toString() + "'/>\n");
}
virtioSerialBuilder.append("</channel>\n");
return virtioSerialBuilder.toString();
}

View File

@ -150,6 +150,7 @@ import com.cloud.agent.api.to.VolumeTO;
import com.cloud.agent.resource.virtualnetwork.VirtualRoutingResource;
import com.cloud.exception.InternalErrorException;
import com.cloud.hypervisor.kvm.resource.KVMHABase.NfsStoragePool;
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.ChannelDef;
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.DiskDef;
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.InterfaceDef;
import com.cloud.hypervisor.kvm.resource.wrapper.LibvirtRequestWrapper;
@ -336,6 +337,19 @@ public class LibvirtComputingResourceTest {
assertXpath(domainDoc, "/domain/devices/input/@type", "tablet");
assertXpath(domainDoc, "/domain/devices/input/@bus", "usb");
assertNodeExists(domainDoc, "/domain/devices/channel");
assertXpath(domainDoc, "/domain/devices/channel/@type", ChannelDef.ChannelType.UNIX.toString());
/*
The configure() method of LibvirtComputingResource has not been called, so the default path for the sockets
hasn't been initialized. That's why we check for 'null'
Calling configure is also not possible since that looks for certain files on the system which are not present
during testing
*/
assertXpath(domainDoc, "/domain/devices/channel/source/@path", "null/" + to.getName() + ".org.qemu.guest_agent.0");
assertXpath(domainDoc, "/domain/devices/channel/target/@name", "org.qemu.guest_agent.0");
assertXpath(domainDoc, "/domain/memory/text()", String.valueOf( to.getMaxRam() / 1024 ));
assertXpath(domainDoc, "/domain/currentMemory/text()", String.valueOf( to.getMinRam() / 1024 ));

View File

@ -20,10 +20,13 @@
package com.cloud.hypervisor.kvm.resource;
import junit.framework.TestCase;
import java.io.File;
import java.util.List;
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.DiskDef;
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.InterfaceDef;
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.RngDef;
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.ChannelDef;
public class LibvirtDomainXMLParserTest extends TestCase {
@ -39,6 +42,13 @@ public class LibvirtDomainXMLParserTest extends TestCase {
InterfaceDef.NicModel ifModel = InterfaceDef.NicModel.VIRTIO;
InterfaceDef.GuestNetType ifType = InterfaceDef.GuestNetType.BRIDGE;
ChannelDef.ChannelType channelType = ChannelDef.ChannelType.UNIX;
ChannelDef.ChannelState channelState = ChannelDef.ChannelState.DISCONNECTED;
String ssvmAgentPath = "/var/lib/libvirt/qemu/s-2970-VM.agent";
String ssvmAgentName = "s-2970-VM.vport";
String guestAgentPath = "/var/lib/libvirt/qemu/guest-agent.org.qemu.guest_agent.0";
String guestAgentName = "org.qemu.guest_agent.0";
String diskLabel ="vda";
String diskPath = "/var/lib/libvirt/images/my-test-image.qcow2";
@ -145,7 +155,7 @@ public class LibvirtDomainXMLParserTest extends TestCase {
"</console>" +
"<channel type='unix'>" +
"<source mode='bind' path='/var/lib/libvirt/qemu/s-2970-VM.agent'/>" +
"<target type='virtio' name='s-2970-VM.vport'/>" +
"<target type='virtio' name='s-2970-VM.vport' state='disconnected'/>" +
"<alias name='channel0'/>" +
"<address type='virtio-serial' controller='0' bus='0' port='1'/>" +
"</channel>" +
@ -169,6 +179,12 @@ public class LibvirtDomainXMLParserTest extends TestCase {
"<rate period='5000' bytes='4096' />" +
"<backend model='random'>/dev/random</backend>" +
"</rng>" +
"<channel type='unix'>" +
"<source mode='bind' path='" + guestAgentPath + "'/>" +
"<target type='virtio' name='" + guestAgentName + "'/>" +
"<alias name='channel0'/>" +
"<address type='virtio-serial' controller='0' bus='0' port='1'/>" +
"</channel>" +
"</devices>" +
"<seclabel type='none'/>" +
"</domain>";
@ -190,6 +206,21 @@ public class LibvirtDomainXMLParserTest extends TestCase {
assertEquals(deviceType, disks.get(diskId).getDeviceType());
assertEquals(diskFormat, disks.get(diskId).getDiskFormatType());
List<ChannelDef> channels = parser.getChannels();
for (int i = 0; i < channels.size(); i++) {
assertEquals(channelType, channels.get(i).getChannelType());
assertEquals(channelType, channels.get(i).getChannelType());
}
/* SSVM provisioning port/channel */
assertEquals(channelState, channels.get(0).getChannelState());
assertEquals(new File(ssvmAgentPath), channels.get(0).getPath());
assertEquals(ssvmAgentName, channels.get(0).getName());
/* Qemu Guest Agent port/channel */
assertEquals(new File(guestAgentPath), channels.get(1).getPath());
assertEquals(guestAgentName, channels.get(1).getName());
List<InterfaceDef> ifs = parser.getInterfaces();
for (int i = 0; i < ifs.size(); i++) {
assertEquals(ifModel, ifs.get(i).getModel());

View File

@ -21,8 +21,11 @@ package com.cloud.hypervisor.kvm.resource;
import junit.framework.TestCase;
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.DiskDef;
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.ChannelDef;
import com.cloud.utils.Pair;
import java.io.File;
public class LibvirtVMDefTest extends TestCase {
public void testInterfaceEtehrnet() {
@ -118,7 +121,7 @@ public class LibvirtVMDefTest extends TestCase {
assertTrue((hostOsVersion.first() == 6 && hostOsVersion.second() >= 5) || (hostOsVersion.first() >= 7));
}
public void testChannelDef() {
public void testRngDef() {
LibvirtVMDef.RngDef.RngBackendModel backendModel = LibvirtVMDef.RngDef.RngBackendModel.RANDOM;
String path = "/dev/random";
int period = 2000;
@ -132,4 +135,18 @@ public class LibvirtVMDefTest extends TestCase {
assertEquals(def.getRngRatePeriod(), period);
}
public void testChannelDef() {
ChannelDef.ChannelType type = ChannelDef.ChannelType.UNIX;
ChannelDef.ChannelState state = ChannelDef.ChannelState.CONNECTED;
String name = "v-136-VM.vport";
File path = new File("/var/lib/libvirt/qemu/" + name);
ChannelDef channelDef = new ChannelDef(name, type, state, path);
assertEquals(state, channelDef.getChannelState());
assertEquals(type, channelDef.getChannelType());
assertEquals(name, channelDef.getName());
assertEquals(path, channelDef.getPath());
}
}

View File

@ -114,8 +114,14 @@ get_boot_params() {
sed -i "s/%/ /g" /var/cache/cloud/cmdline
;;
kvm)
if [ ! -e /dev/vport0p1 ]; then
log_it "/dev/vport0p1 not loaded, perhaps guest kernel is too old." && exit 2
VPORT=$(find /dev/virtio-ports -type l -name '*.vport' 2>/dev/null|head -1)
if [ -z "$VPORT" ]; then
log_it "No suitable VirtIO port was found in /dev/virtio-ports" && exit 2
fi
if [ ! -e "$VPORT" ]; then
log_it "${VPORT} not loaded, perhaps guest kernel is too old." && exit 2
fi
local factor=2
@ -131,7 +137,7 @@ get_boot_params() {
echo $pubkey > /var/cache/cloud/authorized_keys
echo $pubkey > /root/.ssh/authorized_keys
fi
done < /dev/vport0p1
done < $VPORT
# In case of reboot we do not send the boot args again.
# So, no need to wait for them, as the boot args are already set at startup
if [ -s /var/cache/cloud/cmdline ]

View File

@ -75,7 +75,7 @@ function install_packages() {
radvd \
sharutils
${apt_get} -t wheezy-backports install keepalived irqbalance open-vm-tools
${apt_get} -t wheezy-backports install keepalived irqbalance open-vm-tools qemu-guest-agent
# hold on installed openswan version, upgrade rest of the packages (if any)
apt-mark hold openswan