etan39
/
cloudstack
mirror of https://github.com/apache/cloudstack.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

server: Fix update capacity for hosts take long time if there are many service offerings (#4623) 

Steps to reproduce the issue:

(1)Create 10000 service offerings (by db changes below or cloudmonkey).

```
DROP PROCEDURE IF EXISTS cloud.insert_service_offering;

DELIMITER $$
CREATE PROCEDURE cloud.insert_service_offering()
BEGIN
  DECLARE count INT DEFAULT 10000;
  SET @offeringid = (select max(id)+1 from disk_offering);

  WHILE count > 0 DO
    INSERT INTO disk_offering (id,name,uuid,display_text,disk_size,type,created) values (@offeringid,'test-offering-wei',uuid(), 'test-offering-wei',0,'Service',now());
    INSERT INTO service_offering (id,cpu,speed,ram_size) values (@offeringid, 1, 500,256);
    SET @offeringid = @offeringid + 1;
    SET count = count - 1;
  END WHILE;
END $$
DELIMITER ;

CALL cloud.insert_service_offering();

mysql> CALL cloud.insert_service_offering();
Query OK, 0 rows affected (2 min 30.85 sec)
```

(2) Check the total time of periodical capacity check in cloudstack.

Without this patch, it spend 2.5 seconds (2 hosts)
```
2021-01-15 16:10:12,793 DEBUG [c.c.a.AlertManagerImpl] (CapacityChecker:ctx-5d5f3b3b) (logid:f5eb68ba) Running Capacity Checker ...
2021-01-15 16:10:15,287 DEBUG [c.c.a.AlertManagerImpl] (CapacityChecker:ctx-5d5f3b3b) (logid:f5eb68ba) Done running Capacity Checker ...
```

With this patch ,it spend 1.3 seconds (2 hosts)
```
2021-01-15 16:12:43,604 DEBUG [c.c.a.AlertManagerImpl] (CapacityChecker:ctx-a2a7f3f1) (logid:f7e0a4c5) Running Capacity Checker ...
2021-01-15 16:12:44,927 DEBUG [c.c.a.AlertManagerImpl] (CapacityChecker:ctx-a2a7f3f1) (logid:f7e0a4c5) Done running Capacity Checker ...
```

If there are 100 hosts, the total time will be reduced from 100+ seconds to around 10 seconds.
This commit is contained in:
Wei Zhou 2021-02-04 10:13:57 +01:00 committed by GitHub
parent 05301b1e6a
commit 78f73c1bc6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 27 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java
View File

@ -16,10 +16,13 @@
// under the License.
package com.cloud.capacity;
import java.util.Map;
import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
import com.cloud.host.Host;
import com.cloud.service.ServiceOfferingVO;
import com.cloud.storage.VMTemplateVO;
import com.cloud.vm.VirtualMachine;
@ -99,6 +102,8 @@ public interface CapacityManager {
void updateCapacityForHost(Host host);
void updateCapacityForHost(Host host, Map<Long, ServiceOfferingVO> offeringsMap);
/**
* @param pool storage pool
* @param templateForVmCreation template that will be used for vm creation

12
server/src/main/java/com/cloud/alert/AlertManagerImpl.java
View File

@ -75,6 +75,8 @@ import com.cloud.host.HostVO;
import com.cloud.network.dao.IPAddressDao;
import com.cloud.org.Grouping.AllocationState;
import com.cloud.resource.ResourceManager;
import com.cloud.service.ServiceOfferingVO;
import com.cloud.service.dao.ServiceOfferingDao;
import com.cloud.storage.StorageManager;
import com.cloud.utils.NumbersUtil;
import com.cloud.utils.component.ManagerBase;
@ -121,6 +123,8 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager, Confi
private ConfigurationManager _configMgr;
@Inject
protected ConfigDepot _configDepot;
@Inject
ServiceOfferingDao _offeringsDao;
private Timer _timer = null;
private long _capacityCheckPeriod = 60L * 60L * 1000L; // One hour by default.
@ -275,8 +279,14 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager, Confi
// get all hosts...even if they are not in 'UP' state
List<HostVO> hosts = _resourceMgr.listAllNotInMaintenanceHostsInOneZone(Host.Type.Routing, null);
if (hosts != null) {
// prepare the service offerings
List<ServiceOfferingVO> offerings = _offeringsDao.listAllIncludingRemoved();
Map<Long, ServiceOfferingVO> offeringsMap = new HashMap<Long, ServiceOfferingVO>();
for (ServiceOfferingVO offering : offerings) {
offeringsMap.put(offering.getId(), offering);
}
for (HostVO host : hosts) {
_capacityMgr.updateCapacityForHost(host);
_capacityMgr.updateCapacityForHost(host, offeringsMap);
}
}
if (s_logger.isDebugEnabled()) {

11
server/src/main/java/com/cloud/capacity/CapacityManagerImpl.java
View File

@ -627,7 +627,12 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager,
for (ServiceOfferingVO offering : offerings) {
offeringsMap.put(offering.getId(), offering);
}
updateCapacityForHost(host, offeringsMap);
}
@DB
@Override
public void updateCapacityForHost(final Host host, final Map<Long, ServiceOfferingVO> offeringsMap) {
long usedCpuCore = 0;
long reservedCpuCore = 0;
long usedCpu = 0;
@ -664,6 +669,9 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager,
ramOvercommitRatio = Float.parseFloat(vmDetailRam);
}
ServiceOffering so = offeringsMap.get(vm.getServiceOfferingId());
if (so == null) {
so = _offeringsDao.findByIdIncludingRemoved(vm.getServiceOfferingId());
}
if (so.isDynamic()) {
usedMemory +=
((Integer.parseInt(vmDetails.get(UsageEventVO.DynamicParameters.memory.name())) * 1024L * 1024L) / ramOvercommitRatio) *
@ -703,6 +711,9 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager,
}
ServiceOffering so = offeringsMap.get(vm.getServiceOfferingId());
Map<String, String> vmDetails = _userVmDetailsDao.listDetailsKeyPairs(vm.getId());
if (so == null) {
so = _offeringsDao.findByIdIncludingRemoved(vm.getServiceOfferingId());
}
if (so.isDynamic()) {
reservedMemory +=
((Integer.parseInt(vmDetails.get(UsageEventVO.DynamicParameters.memory.name())) * 1024L * 1024L) / ramOvercommitRatio) *