mirror of https://github.com/apache/cloudstack.git
Summary: KVM - use virtio socket to communicate config to system vms
Detail: This gets rid of the patchdisk method of passing cmdline and authorized_keys to KVM system VMs. It instead passes them to a virtio socket, which the KVM guest reads from the character device /dev/vport0p1 during cloud-early-config. Tested to work on CentOS 6.3 and Ubuntu 12.04. Should work with even older versions of libvirt. Signed-off-by: Marcus Sorensen <marcus@betterservers.com> 1362691685 -0700
This commit is contained in:
Marcus Sorensen
parent
c9ee05517d
commit
9ad54a082c
|
|
@ -22,7 +22,7 @@ Description: CloudStack server library
|
|||
|
||||
Package: cloudstack-agent
|
||||
Architecture: all
|
||||
Depends: openjdk-6-jre | openjdk-7-jre, cloudstack-common (= ${source:Version}), lsb-base (>= 3.2), libcommons-daemon-java, libjna-java, openssh-client, libvirt0, sysvinit-utils, chkconfig, qemu-kvm, libvirt-bin, uuid-runtime, rsync, grep, iproute, ebtables, vlan, wget, jsvc
|
||||
Depends: openjdk-6-jre | openjdk-7-jre, cloudstack-common (= ${source:Version}), lsb-base (>= 3.2), libcommons-daemon-java, libjna-java, openssh-client, libvirt0, sysvinit-utils, chkconfig, qemu-kvm, libvirt-bin, uuid-runtime, rsync, grep, iproute, perl-base, perl-modules, ebtables, vlan, wget, jsvc
|
||||
Conflicts: cloud-agent, cloud-agent-libs, cloud-agent-deps, cloud-agent-scripts
|
||||
Description: CloudStack agent
|
||||
The CloudStack agent is in charge of managing shared computing resources in
|
||||
|
|
|
|||
|
|
@ -116,6 +116,7 @@ Requires: ebtables
|
|||
Requires: jsvc
|
||||
Requires: jakarta-commons-daemon
|
||||
Requires: jakarta-commons-daemon-jsvc
|
||||
Requires: perl
|
||||
Provides: cloud-agent
|
||||
Obsoletes: cloud-agent < 4.1.0
|
||||
Obsoletes: cloud-test < 4.1.0
|
||||
|
|
|
|||
|
|
@ -108,14 +108,17 @@ get_boot_params() {
|
|||
sed -i "s/%/ /g" /var/cache/cloud/cmdline
|
||||
;;
|
||||
kvm)
|
||||
# KVM needs to mount another disk, to get cmdline
|
||||
mkdir -p $EXTRA_MOUNT
|
||||
mount /dev/vdb $EXTRA_MOUNT
|
||||
cp -f $EXTRA_MOUNT/cmdline /var/cache/cloud/cmdline
|
||||
cp -f $EXTRA_MOUNT/authorized_keys /var/cache/cloud/authorized_keys
|
||||
privkey=/var/cache/cloud/authorized_keys
|
||||
umount $EXTRA_MOUNT
|
||||
cp -f $privkey /root/.ssh/ && chmod go-rwx /root/.ssh/authorized_keys
|
||||
while read line; do
|
||||
if [[ $line == cmdline:* ]]; then
|
||||
cmd=${line//cmdline:/}
|
||||
echo $cmd > /var/cache/cloud/cmdline
|
||||
elif [[ $line == pubkey:* ]]; then
|
||||
pubkey=${line//pubkey:/}
|
||||
echo $pubkey > /var/cache/cloud/authorized_keys
|
||||
echo $pubkey > /root/.ssh/authorized_keys
|
||||
fi
|
||||
done < /dev/vport0p1
|
||||
chmod go-rwx /root/.ssh/authorized_keys
|
||||
;;
|
||||
vmware)
|
||||
vmtoolsd --cmd 'machine.id.get' > /var/cache/cloud/cmdline
|
||||
|
|
|
|||
|
|
@ -255,7 +255,7 @@ ServerResource {
|
|||
|
||||
private String _modifyVlanPath;
|
||||
private String _versionstringpath;
|
||||
private String _patchdomrPath;
|
||||
private String _patchViaSocketPath;
|
||||
private String _createvmPath;
|
||||
private String _manageSnapshotPath;
|
||||
private String _resizeVolumePath;
|
||||
|
|
@ -521,10 +521,10 @@ ServerResource {
|
|||
throw new ConfigurationException("Unable to find versions.sh");
|
||||
}
|
||||
|
||||
_patchdomrPath = Script.findScript(kvmScriptsDir + "/patch/",
|
||||
"rundomrpre.sh");
|
||||
if (_patchdomrPath == null) {
|
||||
throw new ConfigurationException("Unable to find rundomrpre.sh");
|
||||
_patchViaSocketPath = Script.findScript(kvmScriptsDir + "/patch/",
|
||||
"patchviasocket.pl");
|
||||
if (_patchViaSocketPath == null) {
|
||||
throw new ConfigurationException("Unable to find patchviasocket.pl");
|
||||
}
|
||||
|
||||
_heartBeatPath = Script.findScript(kvmScriptsDir, "kvmheartbeat.sh");
|
||||
|
|
@ -1014,13 +1014,11 @@ ServerResource {
|
|||
return vnetId;
|
||||
}
|
||||
|
||||
private void patchSystemVm(String cmdLine, String dataDiskPath,
|
||||
String vmName) throws InternalErrorException {
|
||||
private void passCmdLine(String vmName, String cmdLine)
|
||||
throws InternalErrorException {
|
||||
final Script command = new Script(_patchViaSocketPath, _timeout, s_logger);
|
||||
String result;
|
||||
final Script command = new Script(_patchdomrPath, _timeout, s_logger);
|
||||
command.add("-l", vmName);
|
||||
command.add("-t", "all");
|
||||
command.add("-d", dataDiskPath);
|
||||
command.add("-n",vmName);
|
||||
command.add("-p", cmdLine.replaceAll(" ", "%"));
|
||||
result = command.execute();
|
||||
if (result != null) {
|
||||
|
|
@ -1460,24 +1458,6 @@ ServerResource {
|
|||
pool.deletePhysicalDisk(vol.getPath());
|
||||
String vmName = cmd.getVmName();
|
||||
String poolPath = pool.getLocalPath();
|
||||
|
||||
/* if vol is a root disk for a system vm, try to remove accompanying patch disk as well
|
||||
this is a bit tricky since the patchdisk is only a LibvirtComputingResource construct
|
||||
and not tracked anywhere in cloudstack */
|
||||
if (vol.getType() == Volume.Type.ROOT && vmName.matches("^[rsv]-\\d+-.+$")) {
|
||||
File patchVbd = new File(poolPath + File.separator + vmName + "-patchdisk");
|
||||
if(patchVbd.exists()){
|
||||
try {
|
||||
_storagePoolMgr.deleteVbdByPath(vol.getPoolType(),patchVbd.getAbsolutePath());
|
||||
} catch(CloudRuntimeException e) {
|
||||
s_logger.warn("unable to destroy patch disk '" + patchVbd.getAbsolutePath() +
|
||||
"' while removing root disk for " + vmName + " : " + e);
|
||||
}
|
||||
} else {
|
||||
s_logger.debug("file '" +patchVbd.getAbsolutePath()+ "' not found");
|
||||
}
|
||||
}
|
||||
|
||||
return new Answer(cmd, true, "Success");
|
||||
} catch (CloudRuntimeException e) {
|
||||
s_logger.debug("Failed to delete volume: " + e.toString());
|
||||
|
|
@ -3121,6 +3101,11 @@ ServerResource {
|
|||
}
|
||||
}
|
||||
|
||||
// pass cmdline info to system vms
|
||||
if (vmSpec.getType() != VirtualMachine.Type.User) {
|
||||
passCmdLine(vmName, vmSpec.getBootArgs() );
|
||||
}
|
||||
|
||||
state = State.Running;
|
||||
return new StartAnswer(cmd);
|
||||
} catch (LibvirtException e) {
|
||||
|
|
@ -3248,8 +3233,6 @@ ServerResource {
|
|||
iso.defISODisk(_sysvmISOPath);
|
||||
vm.getDevices().addDevice(iso);
|
||||
}
|
||||
|
||||
createPatchVbd(conn, vmName, vm, vmSpec);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -3263,64 +3246,6 @@ ServerResource {
|
|||
return null;
|
||||
}
|
||||
|
||||
private void createPatchVbd(Connect conn, String vmName, LibvirtVMDef vm,
|
||||
VirtualMachineTO vmSpec) throws LibvirtException,
|
||||
InternalErrorException {
|
||||
|
||||
List<DiskDef> disks = vm.getDevices().getDisks();
|
||||
DiskDef rootDisk = disks.get(0);
|
||||
VolumeTO rootVol = getVolume(vmSpec, Volume.Type.ROOT);
|
||||
String patchName = vmName + "-patchdisk";
|
||||
KVMStoragePool pool = _storagePoolMgr.getStoragePool(
|
||||
rootVol.getPoolType(),
|
||||
rootVol.getPoolUuid());
|
||||
String patchDiskPath = pool.getLocalPath() + "/" + patchName;
|
||||
|
||||
List<KVMPhysicalDisk> phyDisks = pool.listPhysicalDisks();
|
||||
boolean foundDisk = false;
|
||||
|
||||
for (KVMPhysicalDisk phyDisk : phyDisks) {
|
||||
if (phyDisk.getPath().equals(patchDiskPath)) {
|
||||
foundDisk = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!foundDisk) {
|
||||
s_logger.debug("generating new patch disk for " + vmName + " since none was found");
|
||||
KVMPhysicalDisk disk = pool.createPhysicalDisk(patchName, KVMPhysicalDisk.PhysicalDiskFormat.RAW,
|
||||
10L * 1024 * 1024);
|
||||
} else {
|
||||
s_logger.debug("found existing patch disk at " + patchDiskPath + " using it for " + vmName);
|
||||
}
|
||||
|
||||
/* Format/create fs on this disk */
|
||||
final Script command = new Script(_createvmPath, _timeout, s_logger);
|
||||
command.add("-f", patchDiskPath);
|
||||
String result = command.execute();
|
||||
if (result != null) {
|
||||
s_logger.debug("Failed to create data disk: " + result);
|
||||
throw new InternalErrorException("Failed to create data disk: "
|
||||
+ result);
|
||||
}
|
||||
|
||||
/* add patch disk */
|
||||
DiskDef patchDisk = new DiskDef();
|
||||
|
||||
if (pool.getType() == StoragePoolType.CLVM) {
|
||||
patchDisk.defBlockBasedDisk(patchDiskPath, 1, rootDisk.getBusType());
|
||||
} else {
|
||||
patchDisk.defFileBasedDisk(patchDiskPath, 1, rootDisk.getBusType(),
|
||||
DiskDef.diskFmtType.RAW);
|
||||
}
|
||||
|
||||
disks.add(patchDisk);
|
||||
|
||||
String bootArgs = vmSpec.getBootArgs();
|
||||
|
||||
patchSystemVm(bootArgs, patchDiskPath, vmName);
|
||||
}
|
||||
|
||||
private void createVif(LibvirtVMDef vm, NicTO nic)
|
||||
throws InternalErrorException, LibvirtException {
|
||||
vm.getDevices().addDevice(
|
||||
|
|
|
|||
|
|
@ -864,8 +864,8 @@ public class LibvirtVMDef {
|
|||
virtioSerialBuilder.append("<channel type='unix'>\n");
|
||||
virtioSerialBuilder.append("<source mode='bind' path='" + _path
|
||||
+ "/" + _name + ".agent'/>\n");
|
||||
virtioSerialBuilder.append("<target type='virtio' name='org.qemu.guest_agent.0'/>\n");
|
||||
virtioSerialBuilder.append("<address type='virtio-serial' controller='0' bus='0' port='1'/>\n");
|
||||
virtioSerialBuilder.append("<target type='virtio' name='" + _name + ".vport'/>\n");
|
||||
virtioSerialBuilder.append("<address type='virtio-serial'/>\n");
|
||||
virtioSerialBuilder.append("</channel>\n");
|
||||
return virtioSerialBuilder.toString();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,58 @@
|
|||
#!/usr/bin/perl -w
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
#############################################################
|
||||
# This script connects to the system vm socket and writes the
|
||||
# authorized_keys and cmdline data to it. The system VM then
|
||||
# reads it from /dev/vport0p1 in cloud_early_config
|
||||
#############################################################
|
||||
|
||||
use strict;
|
||||
use Getopt::Std;
|
||||
use IO::Socket;
|
||||
$|=1;
|
||||
|
||||
my $opts = {};
|
||||
getopt('pn',$opts);
|
||||
my $name = $opts->{n};
|
||||
my $cmdline = $opts->{p};
|
||||
my $sockfile = "/var/lib/libvirt/qemu/$name.agent";
|
||||
my $pubkeyfile = "/root/.ssh/id_rsa.pub.cloud";
|
||||
|
||||
if (! -S $sockfile) {
|
||||
print "ERROR: $sockfile socket not found\n";
|
||||
exit 1;
|
||||
}
|
||||
|
||||
if (! -f $pubkeyfile) {
|
||||
print "ERROR: ssh public key not found on host at $pubkeyfile\n";
|
||||
exit 1;
|
||||
}
|
||||
|
||||
open(FILE,$pubkeyfile) or die "ERROR: unable to open $pubkeyfile - $^E";
|
||||
my $key = <FILE>;
|
||||
close FILE;
|
||||
|
||||
$cmdline =~ s/%/ /g;
|
||||
my $msg = "pubkey:" . $key . "\ncmdline:" . $cmdline;
|
||||
|
||||
my $socket = IO::Socket::UNIX->new(Peer=>$sockfile,Type=>SOCK_STREAM)
|
||||
or die "ERROR: unable to connect to $sockfile - $^E\n";
|
||||
print $socket "$msg\r\n";
|
||||
close $socket;
|
||||
|
||||
|
|
@ -1,147 +0,0 @@
|
|||
#!/bin/bash
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
|
||||
# $Id: rundomrpre.sh 10427 2010-07-09 03:30:48Z edison $ $HeadURL: svn://svn.lab.vmops.com/repos/vmdev/java/scripts/vm/hypervisor/kvm/rundomrpre.sh $
|
||||
|
||||
set -x
|
||||
pubKey="/root/.ssh/id_rsa.pub.cloud"
|
||||
mntpath() {
|
||||
local vmname=$1
|
||||
if [ ! -d /mnt/$vmname ]
|
||||
then
|
||||
mkdir -p /mnt/$vmname
|
||||
fi
|
||||
echo "/mnt/$vmname"
|
||||
}
|
||||
|
||||
mount_raw_disk() {
|
||||
local vmname=$1
|
||||
local datadisk=$2
|
||||
local path=$(mntpath $vmname)
|
||||
if [ ! -f $datadisk -a ! -b $datadisk ]
|
||||
then
|
||||
printf "$datadisk doesn't exist" >&2
|
||||
return 2
|
||||
fi
|
||||
|
||||
retry=10
|
||||
while [ $retry -gt 0 ]
|
||||
do
|
||||
if [ -b $datadisk ]; then
|
||||
mount $datadisk $path &>/dev/null
|
||||
ret=$?
|
||||
else
|
||||
mount $datadisk $path -o loop &>/dev/null
|
||||
ret=$?
|
||||
fi
|
||||
sleep 10
|
||||
if [ $ret -gt 0 ]
|
||||
then
|
||||
sleep 5
|
||||
else
|
||||
break
|
||||
fi
|
||||
retry=$(($retry-1))
|
||||
done
|
||||
return 0
|
||||
}
|
||||
|
||||
umount_raw_disk() {
|
||||
local vmname=$1
|
||||
local datadisk=$2
|
||||
local path=$(mntpath $vmname)
|
||||
|
||||
retry=10
|
||||
sync
|
||||
while [ $retry -gt 0 ]
|
||||
do
|
||||
umount -d $path &>/dev/null
|
||||
if [ $? -gt 0 ]
|
||||
then
|
||||
sleep 5
|
||||
else
|
||||
rm -rf $path
|
||||
break
|
||||
fi
|
||||
retry=$(($retry-1))
|
||||
done
|
||||
return $?
|
||||
}
|
||||
|
||||
patch_all() {
|
||||
local vmname=$1
|
||||
local cmdline=$2
|
||||
local datadisk=$3
|
||||
local path=$(mntpath $vmname)
|
||||
|
||||
|
||||
if [ -f $pubKey ]
|
||||
then
|
||||
cp $pubKey $path/authorized_keys
|
||||
fi
|
||||
echo $cmdline > $path/cmdline
|
||||
sed -i "s/%/\ /g" $path/cmdline
|
||||
return 0
|
||||
}
|
||||
|
||||
lflag=
|
||||
dflag=
|
||||
|
||||
while getopts 't:v:i:m:e:E:a:A:g:l:n:d:b:B:p:I:N:Mx:X:' OPTION
|
||||
do
|
||||
case $OPTION in
|
||||
l) lflag=1
|
||||
vmname="$OPTARG"
|
||||
;;
|
||||
t) tflag=1
|
||||
vmtype="$OPTARG"
|
||||
;;
|
||||
d) dflag=1
|
||||
rootdisk="$OPTARG"
|
||||
;;
|
||||
p) pflag=1
|
||||
cmdline="$OPTARG"
|
||||
;;
|
||||
*) ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ "$lflag$tflag$dflag" != "111" ]
|
||||
then
|
||||
printf "Error: No enough parameter\n" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$vmtype" = "all" ]
|
||||
then
|
||||
mount_raw_disk $vmname $rootdisk
|
||||
if [ $? -gt 0 ]
|
||||
then
|
||||
printf "Failed to mount $rootdisk"
|
||||
exit $?
|
||||
fi
|
||||
|
||||
patch_all $vmname $cmdline $rootdisk
|
||||
|
||||
umount_raw_disk $vmname $rootdisk
|
||||
exit $?
|
||||
fi
|
||||
|
||||
|
||||
exit $?
|
||||
Loading…
Reference in New Issue