mirror of https://github.com/apache/cloudstack.git
kvm: Properly report available memory to Management Server (#2795)
The KVM Agent had two mechanisms for reporting its capabilities and memory to the Management Server. On startup it would ask libvirt the amount of Memory the Host has and subtract and add the reserved and overcommit memory. When the HostStats were however reported to the Management Server these two configured values on the Agent were no longer reported in the statistics thus showing all the available memory in the Agent/Host to the Management Server. This commit unifies this by using the same logic on Agent Startup and during statistics reporting. memory=3069636608, reservedMemory=1073741824 This was reported by a 4GB Hypervisor with this setting: host.reserved.mem.mb=1024 The GUI (thus API) would then show: Memory Total 2.86 GB This way the Agent properly 'lies' to the Management Server about its capabilities in terms of Memory. This is very helpful if you want to overprovision or undercommit machines for various reasons. Overcommitting can be done when KSM or ZSwap or a fast SWAP device is installed in the machine. Underprovisioning is done when the Host might run other tasks then a KVM hypervisor, for example when it runs in a hyperconverged setup with Ceph. In addition internally many values have been changed from a Double to a Long and also store the amount of bytes instead of Kilobytes. Signed-off-by: Wido den Hollander <wido@widodh.nl>
This commit is contained in:
Wido den Hollander
Gabriel Beims Bräscher
parent
323f791efc
commit
c496c84c6c
|
|
@ -19,9 +19,7 @@ package com.cloud.hypervisor.kvm.resource;
|
|||
import java.io.BufferedReader;
|
||||
import java.io.File;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.FileReader;
|
||||
import java.io.IOException;
|
||||
import java.io.Reader;
|
||||
import java.io.StringReader;
|
||||
import java.net.InetAddress;
|
||||
import java.net.URI;
|
||||
|
|
@ -55,12 +53,12 @@ import org.apache.cloudstack.storage.to.TemplateObjectTO;
|
|||
import org.apache.cloudstack.storage.to.VolumeObjectTO;
|
||||
import org.apache.cloudstack.utils.hypervisor.HypervisorUtils;
|
||||
import org.apache.cloudstack.utils.linux.CPUStat;
|
||||
import org.apache.cloudstack.utils.linux.KVMHostInfo;
|
||||
import org.apache.cloudstack.utils.linux.MemStat;
|
||||
import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat;
|
||||
import org.apache.cloudstack.utils.security.KeyStoreUtils;
|
||||
import org.apache.commons.collections.MapUtils;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.commons.lang.ArrayUtils;
|
||||
import org.apache.commons.lang.math.NumberUtils;
|
||||
import org.apache.log4j.Logger;
|
||||
|
|
@ -311,8 +309,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
|
|||
protected int _cmdsTimeout;
|
||||
protected int _stopTimeout;
|
||||
protected CPUStat _cpuStat = new CPUStat();
|
||||
protected MemStat _memStat = new MemStat();
|
||||
|
||||
protected MemStat _memStat = new MemStat(_dom0MinMem, _dom0OvercommitMem);
|
||||
private final LibvirtUtilitiesHelper libvirtUtilitiesHelper = new LibvirtUtilitiesHelper();
|
||||
|
||||
@Override
|
||||
|
|
@ -871,7 +868,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
|
|||
|
||||
value = (String)params.get("host.reserved.mem.mb");
|
||||
// Reserve 1GB unless admin overrides
|
||||
_dom0MinMem = NumbersUtil.parseInt(value, 1024) * 1024 * 1024L;
|
||||
_dom0MinMem = NumbersUtil.parseInt(value, 1024) * 1024* 1024L;
|
||||
|
||||
value = (String)params.get("host.overcommit.mem.mb");
|
||||
// Support overcommit memory for host if host uses ZSWAP, KSM and other memory
|
||||
|
|
@ -2661,12 +2658,14 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
|
|||
@Override
|
||||
public StartupCommand[] initialize() {
|
||||
|
||||
final List<Object> info = getHostInfo();
|
||||
final KVMHostInfo info = new KVMHostInfo(_dom0MinMem, _dom0OvercommitMem);
|
||||
|
||||
final String capabilities = String.join(",", info.getCapabilities());
|
||||
|
||||
final StartupRoutingCommand cmd =
|
||||
new StartupRoutingCommand((Integer)info.get(0), (Long)info.get(1), (Long)info.get(2), (Long)info.get(4), (String)info.get(3), _hypervisorType,
|
||||
new StartupRoutingCommand(info.getCpus(), info.getCpuSpeed(), info.getTotalMemory(), info.getReservedMemory(), capabilities, _hypervisorType,
|
||||
RouterPrivateIpStrategy.HostLocal);
|
||||
cmd.setCpuSockets((Integer)info.get(5));
|
||||
cmd.setCpuSockets(info.getCpuSockets());
|
||||
fillNetworkInformation(cmd);
|
||||
_privateIp = cmd.getPrivateIpAddress();
|
||||
cmd.getHostDetails().putAll(getVersionStrings());
|
||||
|
|
@ -2886,71 +2885,6 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
|
|||
return vmStates;
|
||||
}
|
||||
|
||||
protected List<Object> getHostInfo() {
|
||||
final ArrayList<Object> info = new ArrayList<Object>();
|
||||
long speed = 0;
|
||||
long cpus = 0;
|
||||
long ram = 0;
|
||||
int cpuSockets = 0;
|
||||
String cap = null;
|
||||
try {
|
||||
final Connect conn = LibvirtConnection.getConnection();
|
||||
final NodeInfo hosts = conn.nodeInfo();
|
||||
speed = getCpuSpeed(hosts);
|
||||
|
||||
/*
|
||||
* Some CPUs report a single socket and multiple NUMA cells.
|
||||
* We need to multiply them to get the correct socket count.
|
||||
*/
|
||||
cpuSockets = hosts.sockets;
|
||||
if (hosts.nodes > 0) {
|
||||
cpuSockets = hosts.sockets * hosts.nodes;
|
||||
}
|
||||
cpus = hosts.cpus;
|
||||
ram = hosts.memory * 1024L;
|
||||
final LibvirtCapXMLParser parser = new LibvirtCapXMLParser();
|
||||
parser.parseCapabilitiesXML(conn.getCapabilities());
|
||||
final ArrayList<String> oss = parser.getGuestOsType();
|
||||
for (final String s : oss) {
|
||||
/*
|
||||
* Even host supports guest os type more than hvm, we only
|
||||
* report hvm to management server
|
||||
*/
|
||||
if (s.equalsIgnoreCase("hvm")) {
|
||||
cap = "hvm";
|
||||
}
|
||||
}
|
||||
} catch (final LibvirtException e) {
|
||||
s_logger.trace("Ignoring libvirt error.", e);
|
||||
}
|
||||
|
||||
if (isSnapshotSupported()) {
|
||||
cap = cap + ",snapshot";
|
||||
}
|
||||
|
||||
info.add((int)cpus);
|
||||
info.add(speed);
|
||||
// Report system's RAM as actual RAM minus host OS reserved RAM
|
||||
ram = ram - _dom0MinMem + _dom0OvercommitMem;
|
||||
info.add(ram);
|
||||
info.add(cap);
|
||||
info.add(_dom0MinMem);
|
||||
info.add(cpuSockets);
|
||||
s_logger.debug("cpus=" + cpus + ", speed=" + speed + ", ram=" + ram + ", _dom0MinMem=" + _dom0MinMem + ", _dom0OvercommitMem=" + _dom0OvercommitMem + ", cpu sockets=" + cpuSockets);
|
||||
|
||||
return info;
|
||||
}
|
||||
|
||||
protected static long getCpuSpeed(final NodeInfo nodeInfo) {
|
||||
try (final Reader reader = new FileReader(
|
||||
"/sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_max_freq")) {
|
||||
return Long.parseLong(IOUtils.toString(reader).trim()) / 1000;
|
||||
} catch (IOException | NumberFormatException e) {
|
||||
s_logger.warn("Could not read cpuinfo_max_freq");
|
||||
return nodeInfo.mhz;
|
||||
}
|
||||
}
|
||||
|
||||
public String rebootVM(final Connect conn, final String vmName) {
|
||||
Domain dm = null;
|
||||
String msg = null;
|
||||
|
|
|
|||
|
|
@ -42,13 +42,10 @@ public final class LibvirtGetHostStatsCommandWrapper extends CommandWrapper<GetH
|
|||
MemStat memStat = libvirtComputingResource.getMemStat();
|
||||
|
||||
final double cpuUtil = cpuStat.getCpuUsedPercent();
|
||||
memStat.refresh();
|
||||
double totMem = memStat.getTotal();
|
||||
double freeMem = memStat.getAvailable();
|
||||
|
||||
final Pair<Double, Double> nicStats = libvirtComputingResource.getNicStats(libvirtComputingResource.getPublicBridgeName());
|
||||
|
||||
final HostStatsEntry hostStats = new HostStatsEntry(command.getHostId(), cpuUtil, nicStats.first() / 1024, nicStats.second() / 1024, "host", totMem, freeMem, 0, 0);
|
||||
final HostStatsEntry hostStats = new HostStatsEntry(command.getHostId(), cpuUtil, nicStats.first() / 1024, nicStats.second() / 1024, "host", memStat.getTotal() / 1024, memStat.getAvailable() / 1024, 0, 0);
|
||||
return new GetHostStatsAnswer(command, hostStats);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,132 @@
|
|||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
package org.apache.cloudstack.utils.linux;
|
||||
|
||||
import com.cloud.hypervisor.kvm.resource.LibvirtCapXMLParser;
|
||||
import com.cloud.hypervisor.kvm.resource.LibvirtConnection;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.log4j.Logger;
|
||||
import org.libvirt.Connect;
|
||||
import org.libvirt.LibvirtException;
|
||||
import org.libvirt.NodeInfo;
|
||||
|
||||
import java.io.FileReader;
|
||||
import java.io.IOException;
|
||||
import java.io.Reader;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class KVMHostInfo {
|
||||
|
||||
private static final Logger LOGGER = Logger.getLogger(KVMHostInfo.class);
|
||||
|
||||
private int cpus;
|
||||
private int cpusockets;
|
||||
private long cpuSpeed;
|
||||
private long totalMemory;
|
||||
private long reservedMemory;
|
||||
private long overCommitMemory;
|
||||
private List<String> capabilities = new ArrayList<>();
|
||||
|
||||
public KVMHostInfo(long reservedMemory, long overCommitMemory) {
|
||||
this.reservedMemory = reservedMemory;
|
||||
this.overCommitMemory = overCommitMemory;
|
||||
this.getHostInfoFromLibvirt();
|
||||
this.totalMemory = new MemStat(this.getReservedMemory(), this.getOverCommitMemory()).getTotal();
|
||||
}
|
||||
|
||||
public int getCpus() {
|
||||
return this.cpus;
|
||||
}
|
||||
|
||||
public int getCpuSockets() {
|
||||
return this.cpusockets;
|
||||
}
|
||||
|
||||
public long getCpuSpeed() {
|
||||
return this.cpuSpeed;
|
||||
}
|
||||
|
||||
public long getTotalMemory() {
|
||||
return this.totalMemory;
|
||||
}
|
||||
|
||||
public long getReservedMemory() {
|
||||
return this.reservedMemory;
|
||||
}
|
||||
|
||||
public long getOverCommitMemory() {
|
||||
return this.overCommitMemory;
|
||||
}
|
||||
|
||||
public List<String> getCapabilities() {
|
||||
return this.capabilities;
|
||||
}
|
||||
|
||||
protected static long getCpuSpeed(final NodeInfo nodeInfo) {
|
||||
try (final Reader reader = new FileReader(
|
||||
"/sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_max_freq")) {
|
||||
return Long.parseLong(IOUtils.toString(reader).trim()) / 1000;
|
||||
} catch (IOException | NumberFormatException e) {
|
||||
LOGGER.info("Could not read cpuinfo_max_freq, falling back on libvirt");
|
||||
return nodeInfo.mhz;
|
||||
}
|
||||
}
|
||||
|
||||
private void getHostInfoFromLibvirt() {
|
||||
try {
|
||||
final Connect conn = LibvirtConnection.getConnection();
|
||||
final NodeInfo hosts = conn.nodeInfo();
|
||||
this.cpuSpeed = getCpuSpeed(hosts);
|
||||
|
||||
/*
|
||||
* Some CPUs report a single socket and multiple NUMA cells.
|
||||
* We need to multiply them to get the correct socket count.
|
||||
*/
|
||||
this.cpusockets = hosts.sockets;
|
||||
if (hosts.nodes > 0) {
|
||||
this.cpusockets = hosts.sockets * hosts.nodes;
|
||||
}
|
||||
this.cpus = hosts.cpus;
|
||||
|
||||
final LibvirtCapXMLParser parser = new LibvirtCapXMLParser();
|
||||
parser.parseCapabilitiesXML(conn.getCapabilities());
|
||||
final ArrayList<String> oss = parser.getGuestOsType();
|
||||
for (final String s : oss) {
|
||||
/*
|
||||
* Even host supports guest os type more than hvm, we only
|
||||
* report hvm to management server
|
||||
*/
|
||||
String hvmCapability = "hvm";
|
||||
if (s.equalsIgnoreCase(hvmCapability)) {
|
||||
if (!this.capabilities.contains(hvmCapability)) {
|
||||
this.capabilities.add(hvmCapability);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
Any modern Qemu/KVM supports snapshots
|
||||
We used to check if this was supported, but that is no longer required
|
||||
*/
|
||||
this.capabilities.add("snapshot");
|
||||
conn.close();
|
||||
} catch (final LibvirtException e) {
|
||||
LOGGER.error("Caught libvirt exception while fetching host information", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -22,30 +22,47 @@ import java.util.HashMap;
|
|||
import java.util.Map;
|
||||
import java.util.Scanner;
|
||||
|
||||
|
||||
public class MemStat {
|
||||
/*
|
||||
Gather Memory Statistics of the current node by opening /proc/meminfo
|
||||
which contains the memory information in KiloBytes.
|
||||
|
||||
Convert this all to bytes and return Long as a type with the information
|
||||
in bytes
|
||||
*/
|
||||
protected final static String MEMINFO_FILE = "/proc/meminfo";
|
||||
protected final static String FREE_KEY = "MemFree";
|
||||
protected final static String CACHE_KEY = "Cached";
|
||||
protected final static String TOTAL_KEY = "MemTotal";
|
||||
long reservedMemory;
|
||||
long overCommitMemory;
|
||||
|
||||
private final Map<String, Double> _memStats = new HashMap<String, Double>();
|
||||
private final Map<String, Long> _memStats = new HashMap<>();
|
||||
|
||||
public MemStat() {
|
||||
this(0,0);
|
||||
}
|
||||
|
||||
public Double getTotal() {
|
||||
return _memStats.get(TOTAL_KEY);
|
||||
public MemStat(long reservedMemory, long overCommitMemory) {
|
||||
this.reservedMemory = reservedMemory;
|
||||
this.overCommitMemory = overCommitMemory;
|
||||
this.refresh();
|
||||
}
|
||||
|
||||
public Double getAvailable() {
|
||||
public long getTotal() {
|
||||
return _memStats.get(TOTAL_KEY) - reservedMemory + overCommitMemory;
|
||||
}
|
||||
|
||||
public long getAvailable() {
|
||||
return getFree() + getCache();
|
||||
}
|
||||
|
||||
public Double getFree() {
|
||||
return _memStats.get(FREE_KEY);
|
||||
public long getFree() {
|
||||
return _memStats.get(FREE_KEY) - reservedMemory + overCommitMemory;
|
||||
}
|
||||
|
||||
public Double getCache() {
|
||||
public long getCache() {
|
||||
return _memStats.get(CACHE_KEY);
|
||||
}
|
||||
|
||||
|
|
@ -63,7 +80,7 @@ public class MemStat {
|
|||
while(scanner.hasNext()) {
|
||||
String[] stats = scanner.next().split("\\:\\s+");
|
||||
if (stats.length == 2) {
|
||||
_memStats.put(stats[0], Double.valueOf(stats[1].replaceAll("\\s+\\w+","")));
|
||||
_memStats.put(stats[0], Long.valueOf(stats[1].replaceAll("\\s+\\w+","")) * 1024L);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -534,13 +534,6 @@ public class LibvirtComputingResourceTest {
|
|||
Assert.assertTrue(vmStat.getTargetMemoryKBs() >= vmStat.getMemoryKBs());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void getCpuSpeed() {
|
||||
Assume.assumeTrue(SystemUtils.IS_OS_LINUX);
|
||||
final NodeInfo nodeInfo = Mockito.mock(NodeInfo.class);
|
||||
LibvirtComputingResource.getCpuSpeed(nodeInfo);
|
||||
}
|
||||
|
||||
/*
|
||||
* New Tests
|
||||
*/
|
||||
|
|
@ -988,8 +981,8 @@ public class LibvirtComputingResourceTest {
|
|||
when(libvirtComputingResource.getMemStat()).thenReturn(memStat);
|
||||
when(libvirtComputingResource.getNicStats(Mockito.anyString())).thenReturn(new Pair<Double, Double>(1.0d, 1.0d));
|
||||
when(cpuStat.getCpuUsedPercent()).thenReturn(0.5d);
|
||||
when(memStat.getAvailable()).thenReturn(1500.5d);
|
||||
when(memStat.getTotal()).thenReturn(15000d);
|
||||
when(memStat.getAvailable()).thenReturn(1500L);
|
||||
when(memStat.getTotal()).thenReturn(15000L);
|
||||
|
||||
final LibvirtRequestWrapper wrapper = LibvirtRequestWrapper.getInstance();
|
||||
assertNotNull(wrapper);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,37 @@
|
|||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
package org.apache.cloudstack.utils.linux;
|
||||
|
||||
import org.apache.commons.lang.SystemUtils;
|
||||
|
||||
import org.hamcrest.Matchers;
|
||||
import org.junit.Test;
|
||||
import org.junit.Assume;
|
||||
import org.junit.Assert;
|
||||
import org.mockito.Mockito;
|
||||
|
||||
import org.libvirt.NodeInfo;
|
||||
|
||||
public class KVMHostInfoTest {
|
||||
@Test
|
||||
public void getCpuSpeed() {
|
||||
Assume.assumeTrue(SystemUtils.IS_OS_LINUX);
|
||||
NodeInfo nodeInfo = Mockito.mock(NodeInfo.class);
|
||||
nodeInfo.mhz = 1000;
|
||||
Assert.assertThat(KVMHostInfo.getCpuSpeed(nodeInfo), Matchers.greaterThan(0l));
|
||||
}
|
||||
}
|
||||
|
|
@ -22,16 +22,16 @@ import org.junit.Test;
|
|||
import java.util.Scanner;
|
||||
|
||||
public class MemStatTest {
|
||||
final String memInfo = "MemTotal: 5830236 kB\n" +
|
||||
"MemFree: 156752 kB\n" +
|
||||
"Buffers: 326836 kB\n" +
|
||||
"Cached: 2606764 kB\n" +
|
||||
"SwapCached: 0 kB\n" +
|
||||
"Active: 4260808 kB\n" +
|
||||
"Inactive: 949392 kB\n";
|
||||
|
||||
@Test
|
||||
public void getMemInfoParseTest() {
|
||||
String memInfo = "MemTotal: 5830236 kB\n" +
|
||||
"MemFree: 156752 kB\n" +
|
||||
"Buffers: 326836 kB\n" +
|
||||
"Cached: 2606764 kB\n" +
|
||||
"SwapCached: 0 kB\n" +
|
||||
"Active: 4260808 kB\n" +
|
||||
"Inactive: 949392 kB\n";
|
||||
|
||||
MemStat memStat = null;
|
||||
try {
|
||||
memStat = new MemStat();
|
||||
|
|
@ -46,9 +46,25 @@ public class MemStatTest {
|
|||
Scanner scanner = new Scanner(memInfo);
|
||||
memStat.parseFromScanner(scanner);
|
||||
|
||||
Assert.assertEquals(memStat.getTotal(), Double.valueOf(5830236));
|
||||
Assert.assertEquals(memStat.getAvailable(), Double.valueOf(2763516));
|
||||
Assert.assertEquals(memStat.getFree(), Double.valueOf(156752));
|
||||
Assert.assertEquals(memStat.getCache(), Double.valueOf(2606764));
|
||||
Assert.assertEquals(memStat.getTotal(), 5970161664L);
|
||||
Assert.assertEquals(memStat.getAvailable(), 2829840384L);
|
||||
Assert.assertEquals(memStat.getFree(), 160514048L);
|
||||
Assert.assertEquals(memStat.getCache(), 2669326336L);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void reservedMemoryTest() {
|
||||
MemStat memStat = null;
|
||||
try {
|
||||
memStat = new MemStat(1024, 2048);
|
||||
} catch (RuntimeException ex) {
|
||||
if (memStat == null) {
|
||||
throw ex;
|
||||
}
|
||||
}
|
||||
Scanner scanner = new Scanner(memInfo);
|
||||
memStat.parseFromScanner(scanner);
|
||||
|
||||
Assert.assertEquals(memStat.getTotal(), 5970162688L);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue