From 4e2f461b3154c35c92dff71442cffacf240a903e Mon Sep 17 00:00:00 2001 From: Sina Kashipazha Date: Fri, 30 Sep 2022 17:02:01 +0200 Subject: [PATCH] Prometheus exporter enhancement (#4438) * Export count of total/up/down hosts by tags * Export count of vms by state and host tag. * Add host tags to host cpu/cores/memory usage in Prometheus exporter * Cloudstack Prometheus exporter: Add allocated capacity group by host tag. * Show count of Active domains on grafana. * Show count of Active accounts and vms by size on grafana * Use prepared statement to query database for a number of VM who use a specific tag. * Extract repeated codes to new methods. --- .../com/cloud/capacity/dao/CapacityDao.java | 3 + .../cloud/capacity/dao/CapacityDaoImpl.java | 52 +++- .../java/com/cloud/user/dao/AccountDao.java | 1 + .../com/cloud/user/dao/AccountDaoImpl.java | 16 +- .../main/java/com/cloud/vm/dao/UserVmDao.java | 5 + .../java/com/cloud/vm/dao/UserVmDaoImpl.java | 42 +++ .../java/com/cloud/vm/dao/VMInstanceDao.java | 2 + .../com/cloud/vm/dao/VMInstanceDaoImpl.java | 25 ++ .../metrics/PrometheusExporterImpl.java | 263 +++++++++++++++--- .../metrics/PrometheusExporterServer.java | 3 + .../metrics/PrometheusExporterServerImpl.java | 3 +- 11 files changed, 376 insertions(+), 39 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDao.java b/engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDao.java index dc04f76f2f2..459a63a7ba1 100644 --- a/engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDao.java +++ b/engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDao.java @@ -22,6 +22,7 @@ import java.util.Map; import com.cloud.capacity.CapacityVO; import com.cloud.capacity.dao.CapacityDaoImpl.SummedCapacity; import com.cloud.utils.Pair; +import com.cloud.utils.Ternary; import com.cloud.utils.db.GenericDao; public interface CapacityDao extends GenericDao { @@ -39,6 +40,8 @@ public interface CapacityDao extends GenericDao { Pair, Map> orderClustersByAggregateCapacity(long id, long vmId, short capacityType, boolean isZone); + Ternary findCapacityByZoneAndHostTag(Long zoneId, String hostTag); + List findCapacityBy(Integer capacityType, Long zoneId, Long podId, Long clusterId); List listPodsByHostCapacities(long zoneId, int requiredCpu, long requiredRam, short capacityType); diff --git a/engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDaoImpl.java b/engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDaoImpl.java index fe6f2f4ce7b..302ffd8e760 100644 --- a/engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDaoImpl.java @@ -27,6 +27,7 @@ import java.util.Map; import javax.inject.Inject; import org.apache.log4j.Logger; +import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; @@ -37,6 +38,7 @@ import com.cloud.capacity.CapacityVO; import com.cloud.dc.ClusterDetailsDao; import com.cloud.storage.Storage; import com.cloud.utils.Pair; +import com.cloud.utils.Ternary; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.GenericSearchBuilder; import com.cloud.utils.db.JoinBuilder.JoinType; @@ -203,11 +205,15 @@ public class CapacityDaoImpl extends GenericDaoBase implements "FROM (SELECT vi.data_center_id, (CASE WHEN ISNULL(service_offering.cpu) THEN custom_cpu.value ELSE service_offering.cpu end) AS cpu, " + "(CASE WHEN ISNULL(service_offering.speed) THEN custom_speed.value ELSE service_offering.speed end) AS speed, " + "(CASE WHEN ISNULL(service_offering.ram_size) THEN custom_ram_size.value ELSE service_offering.ram_size end) AS ram_size " + - "FROM (((vm_instance vi LEFT JOIN service_offering ON(((vi.service_offering_id = service_offering.id))) " + - "LEFT JOIN user_vm_details custom_cpu ON(((custom_cpu.vm_id = vi.id) AND (custom_cpu.name = 'CpuNumber')))) " + - "LEFT JOIN user_vm_details custom_speed ON(((custom_speed.vm_id = vi.id) AND (custom_speed.name = 'CpuSpeed')))) " + - "LEFT JOIN user_vm_details custom_ram_size ON(((custom_ram_size.vm_id = vi.id) AND (custom_ram_size.name = 'memory')))) " + - "WHERE ISNULL(vi.removed) AND vi.state NOT IN ('Destroyed', 'Error', 'Expunging')"; + "FROM vm_instance vi LEFT JOIN service_offering ON(((vi.service_offering_id = service_offering.id))) " + + "LEFT JOIN user_vm_details custom_cpu ON(((custom_cpu.vm_id = vi.id) AND (custom_cpu.name = 'CpuNumber'))) " + + "LEFT JOIN user_vm_details custom_speed ON(((custom_speed.vm_id = vi.id) AND (custom_speed.name = 'CpuSpeed'))) " + + "LEFT JOIN user_vm_details custom_ram_size ON(((custom_ram_size.vm_id = vi.id) AND (custom_ram_size.name = 'memory'))) "; + + private static final String WHERE_STATE_IS_NOT_DESTRUCTIVE = + "WHERE ISNULL(vi.removed) AND vi.state NOT IN ('Destroyed', 'Error', 'Expunging')"; + + private static final String LEFT_JOIN_VM_TEMPLATE = "LEFT JOIN vm_template ON vm_template.id = vi.vm_template_id "; public CapacityDaoImpl() { _hostIdTypeSearch = createSearchBuilder(); @@ -422,6 +428,41 @@ public class CapacityDaoImpl extends GenericDaoBase implements } + @Override + public Ternary findCapacityByZoneAndHostTag(Long zoneId, String hostTag) { + TransactionLegacy txn = TransactionLegacy.currentTxn(); + PreparedStatement pstmt; + + StringBuilder allocatedSql = new StringBuilder(LIST_ALLOCATED_CAPACITY_GROUP_BY_CAPACITY_AND_ZONE); + if (StringUtils.isNotEmpty(hostTag)) { + allocatedSql.append(LEFT_JOIN_VM_TEMPLATE); + } + allocatedSql.append(WHERE_STATE_IS_NOT_DESTRUCTIVE); + if (zoneId != null) { + allocatedSql.append(" AND vi.data_center_id = ?"); + } + if (StringUtils.isNotEmpty(hostTag)) { + allocatedSql.append(" AND (vm_template.template_tag = '").append(hostTag).append("'"); + allocatedSql.append(" OR service_offering.host_tag = '").append(hostTag).append("')"); + } + allocatedSql.append(" ) AS v GROUP BY v.data_center_id"); + + try { + // add allocated capacity of zone in result + pstmt = txn.prepareAutoCloseStatement(allocatedSql.toString()); + if (zoneId != null) { + pstmt.setLong(1, zoneId); + } + ResultSet rs = pstmt.executeQuery(); + if (rs.next()) { + return new Ternary<>(rs.getLong(2), rs.getLong(3), rs.getLong(4)); // cpu cores, cpu, memory + } + return new Ternary<>(0L, 0L, 0L); + } catch (SQLException e) { + throw new CloudRuntimeException("DB Exception on: " + allocatedSql, e); + } + } + @Override public List findCapacityBy(Integer capacityType, Long zoneId, Long podId, Long clusterId) { @@ -430,6 +471,7 @@ public class CapacityDaoImpl extends GenericDaoBase implements List results = new ArrayList(); StringBuilder allocatedSql = new StringBuilder(LIST_ALLOCATED_CAPACITY_GROUP_BY_CAPACITY_AND_ZONE); + allocatedSql.append(WHERE_STATE_IS_NOT_DESTRUCTIVE); HashMap sumCpuCore = new HashMap(); HashMap sumCpu = new HashMap(); diff --git a/engine/schema/src/main/java/com/cloud/user/dao/AccountDao.java b/engine/schema/src/main/java/com/cloud/user/dao/AccountDao.java index d7754e1b935..17b07496731 100644 --- a/engine/schema/src/main/java/com/cloud/user/dao/AccountDao.java +++ b/engine/schema/src/main/java/com/cloud/user/dao/AccountDao.java @@ -78,4 +78,5 @@ public interface AccountDao extends GenericDao { */ long getDomainIdForGivenAccountId(long id); + int getActiveDomains(); } diff --git a/engine/schema/src/main/java/com/cloud/user/dao/AccountDaoImpl.java b/engine/schema/src/main/java/com/cloud/user/dao/AccountDaoImpl.java index 62d409c9af0..3dacbb70f39 100644 --- a/engine/schema/src/main/java/com/cloud/user/dao/AccountDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/user/dao/AccountDaoImpl.java @@ -28,6 +28,7 @@ import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.GenericSearchBuilder; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.db.SearchCriteria.Func; import com.cloud.utils.db.SearchCriteria.Op; import org.apache.commons.lang3.StringUtils; import com.cloud.utils.db.TransactionLegacy; @@ -54,6 +55,7 @@ public class AccountDaoImpl extends GenericDaoBase implements A protected final SearchBuilder NonProjectAccountSearch; protected final SearchBuilder AccountByRoleSearch; protected final GenericSearchBuilder AccountIdsSearch; + protected final GenericSearchBuilder ActiveDomainCount; public AccountDaoImpl() { AllFieldsSearch = createSearchBuilder(); @@ -101,6 +103,13 @@ public class AccountDaoImpl extends GenericDaoBase implements A AccountByRoleSearch = createSearchBuilder(); AccountByRoleSearch.and("roleId", AccountByRoleSearch.entity().getRoleId(), SearchCriteria.Op.EQ); AccountByRoleSearch.done(); + + ActiveDomainCount = createSearchBuilder(Long.class); + ActiveDomainCount.select(null, Func.COUNT, null); + ActiveDomainCount.and("domain", ActiveDomainCount.entity().getDomainId(), SearchCriteria.Op.EQ); + ActiveDomainCount.and("state", ActiveDomainCount.entity().getState(), SearchCriteria.Op.EQ); + ActiveDomainCount.groupBy(ActiveDomainCount.entity().getDomainId()); + ActiveDomainCount.done(); } @Override @@ -318,5 +327,10 @@ public class AccountDaoImpl extends GenericDaoBase implements A } } - + @Override + public int getActiveDomains() { + SearchCriteria sc = ActiveDomainCount.create(); + sc.setParameters("state", "enabled"); + return customSearch(sc, null).size(); + } } diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/UserVmDao.java b/engine/schema/src/main/java/com/cloud/vm/dao/UserVmDao.java index c53936f7d2d..f4fa5c1d405 100644 --- a/engine/schema/src/main/java/com/cloud/vm/dao/UserVmDao.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/UserVmDao.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.Set; import com.cloud.utils.Pair; +import com.cloud.utils.Ternary; import com.cloud.utils.db.GenericDao; import com.cloud.vm.UserVmVO; import com.cloud.vm.VirtualMachine; @@ -92,4 +93,8 @@ public interface UserVmDao extends GenericDao { List listByIsoId(Long isoId); List, Pair>> getVmsDetailByNames(Set vmNames, String detail); + + List> countVmsBySize(long dcId, int limit); + + int getActiveAccounts(final long dcId); } diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/UserVmDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/UserVmDaoImpl.java index f6b99754c80..af8c0a0988a 100644 --- a/engine/schema/src/main/java/com/cloud/vm/dao/UserVmDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/UserVmDaoImpl.java @@ -40,6 +40,7 @@ import com.cloud.server.ResourceTag.ResourceObjectType; import com.cloud.tags.dao.ResourceTagDao; import com.cloud.user.Account; import com.cloud.utils.Pair; +import com.cloud.utils.Ternary; import com.cloud.utils.db.Attribute; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.GenericSearchBuilder; @@ -75,6 +76,7 @@ public class UserVmDaoImpl extends GenericDaoBase implements Use protected SearchBuilder AccountDataCenterVirtualSearch; protected GenericSearchBuilder CountByAccountPod; protected GenericSearchBuilder CountByAccount; + protected GenericSearchBuilder CountActiveAccount; protected GenericSearchBuilder PodsHavingVmsForAccount; protected SearchBuilder UserVmSearch; @@ -192,6 +194,15 @@ public class UserVmDaoImpl extends GenericDaoBase implements Use CountByAccount.and("displayVm", CountByAccount.entity().isDisplayVm(), SearchCriteria.Op.EQ); CountByAccount.done(); + CountActiveAccount = createSearchBuilder(Long.class); + CountActiveAccount.select(null, Func.COUNT, null); + CountActiveAccount.and("account", CountActiveAccount.entity().getAccountId(), SearchCriteria.Op.EQ); + CountActiveAccount.and("type", CountActiveAccount.entity().getType(), SearchCriteria.Op.EQ); + CountActiveAccount.and("dataCenterId", CountActiveAccount.entity().getDataCenterId(), SearchCriteria.Op.EQ); + CountActiveAccount.and("state", CountActiveAccount.entity().getState(), SearchCriteria.Op.NIN); + CountActiveAccount.groupBy(CountActiveAccount.entity().getAccountId()); + CountActiveAccount.done(); + SearchBuilder nicSearch = _nicDao.createSearchBuilder(); nicSearch.and("networkId", nicSearch.entity().getNetworkId(), SearchCriteria.Op.EQ); nicSearch.and("ip4Address", nicSearch.entity().getIPv4Address(), SearchCriteria.Op.NNULL); @@ -721,4 +732,35 @@ public class UserVmDaoImpl extends GenericDaoBase implements Use return vmsDetailByNames; } + + @Override + public List> countVmsBySize(long dcId, int limit) { + TransactionLegacy txn = TransactionLegacy.currentTxn(); + String sql = "SELECT cpu,ram_size,count(1) AS count FROM (SELECT * FROM user_vm_view WHERE data_center_id = ? AND state NOT IN ('Destroyed', 'Error', 'Expunging') GROUP BY id) AS uvv GROUP BY cpu,ram_size ORDER BY count DESC "; + if (limit >= 0) + sql = sql + "limit " + limit; + PreparedStatement pstmt = null; + List> result = new ArrayList<>(); + try { + pstmt = txn.prepareAutoCloseStatement(sql); + pstmt.setLong(1, dcId); + ResultSet rs = pstmt.executeQuery(); + while (rs.next()) { + result.add(new Ternary(rs.getInt(1), rs.getInt(2), rs.getInt(3))); + } + } catch (Exception e) { + s_logger.warn("Error counting vms by size for dcId= " + dcId, e); + } + return result; + } + + @Override + public int getActiveAccounts(final long dcId) { + SearchCriteria sc = CountActiveAccount.create(); + sc.setParameters("type", VirtualMachine.Type.User); + sc.setParameters("state", State.Destroyed, State.Error, State.Expunging, State.Stopped); + sc.setParameters("dataCenterId", dcId); + + return customSearch(sc, null).size(); + } } diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java index 4db07c0dcac..bdb2534b62d 100755 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java @@ -133,6 +133,8 @@ public interface VMInstanceDao extends GenericDao, StateDao< Long countByZoneAndState(long zoneId, State state); + Long countByZoneAndStateAndHostTag(long dcId, State state, String hostTag); + List listNonRemovedVmsByTypeAndNetwork(long networkId, VirtualMachine.Type... types); /** diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java index 5701d7e3989..0e3d4bdde8f 100755 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java @@ -129,6 +129,9 @@ public class VMInstanceDaoImpl extends GenericDaoBase implem private static final String UPDATE_SYSTEM_VM_TEMPLATE_ID_FOR_HYPERVISOR = "UPDATE `cloud`.`vm_instance` SET vm_template_id = ? WHERE type <> 'User' AND hypervisor_type = ? AND removed is NULL"; + private static final String COUNT_VMS_BY_ZONE_AND_STATE_AND_HOST_TAG = "SELECT COUNT(1) FROM vm_instance vi JOIN service_offering so ON vi.service_offering_id=so.id " + + "JOIN vm_template vt ON vi.vm_template_id = vt.id WHERE vi.data_center_id = ? AND vi.state = ? AND vi.removed IS NULL AND (so.host_tag = ? OR vt.template_tag = ?)"; + @Inject protected HostDao _hostDao; @@ -807,6 +810,28 @@ public class VMInstanceDaoImpl extends GenericDaoBase implem return customSearch(sc, null).get(0); } + @Override + public Long countByZoneAndStateAndHostTag(long dcId, State state, String hostTag) { + TransactionLegacy txn = TransactionLegacy.currentTxn(); + PreparedStatement pstmt = null; + try { + pstmt = txn.prepareAutoCloseStatement(COUNT_VMS_BY_ZONE_AND_STATE_AND_HOST_TAG); + + pstmt.setLong(1, dcId); + pstmt.setString(2, String.valueOf(state)); + pstmt.setString(3, hostTag); + pstmt.setString(4, hostTag); + + ResultSet rs = pstmt.executeQuery(); + if (rs.next()) { + return rs.getLong(1); + } + } catch (Exception e) { + s_logger.warn(String.format("Error counting vms by host tag for dcId= %s, hostTag= %s", dcId, hostTag), e); + } + return 0L; + } + @Override public List listNonRemovedVmsByTypeAndNetwork(long networkId, VirtualMachine.Type... types) { if (NetworkTypeSearch == null) { diff --git a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java index 8609cd9829b..de9a5a40c3b 100644 --- a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java +++ b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java @@ -18,7 +18,10 @@ package org.apache.cloudstack.metrics; import java.math.BigDecimal; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; import javax.inject.Inject; @@ -34,7 +37,6 @@ import org.apache.log4j.Logger; import com.cloud.alert.AlertManager; import com.cloud.api.ApiDBUtils; import com.cloud.api.query.dao.DomainJoinDao; -import com.cloud.api.query.dao.HostJoinDao; import com.cloud.api.query.dao.StoragePoolJoinDao; import com.cloud.api.query.vo.DomainJoinVO; import com.cloud.api.query.vo.StoragePoolJoinVO; @@ -52,17 +54,20 @@ import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.host.Status; import com.cloud.host.dao.HostDao; +import com.cloud.host.dao.HostTagsDao; import com.cloud.network.dao.IPAddressDao; import com.cloud.storage.ImageStore; import com.cloud.storage.StorageStats; import com.cloud.storage.Volume; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.VolumeDao; +import org.apache.commons.lang3.StringUtils; +import com.cloud.utils.Ternary; import com.cloud.utils.component.Manager; import com.cloud.utils.component.ManagerBase; import com.cloud.vm.VirtualMachine.State; +import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.VMInstanceDao; -import org.apache.commons.lang3.StringUtils; public class PrometheusExporterImpl extends ManagerBase implements PrometheusExporter, Manager { private static final Logger LOG = Logger.getLogger(PrometheusExporterImpl.class); @@ -81,10 +86,10 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp @Inject private HostDao hostDao; @Inject - private HostJoinDao hostJoinDao; - @Inject private VMInstanceDao vmDao; @Inject + private UserVmDao uservmDao; + @Inject private VolumeDao volumeDao; @Inject private IPAddressDao publicIpAddressDao; @@ -106,6 +111,8 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp private AccountDao _accountDao; @Inject private ResourceCountDao _resourceCountDao; + @Inject + private HostTagsDao _hostTagsDao; public PrometheusExporterImpl() { super(); @@ -115,6 +122,10 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp int total = 0; int up = 0; int down = 0; + Map totalHosts = new HashMap<>(); + Map upHosts = new HashMap<>(); + Map downHosts = new HashMap<>(); + for (final HostVO host : hostDao.listAll()) { if (host == null || host.getType() != Host.Type.Routing || host.getDataCenterId() != dcId) { continue; @@ -131,6 +142,8 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp int isDedicated = (dr != null) ? 1 : 0; metricsList.add(new ItemHostIsDedicated(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), isDedicated)); + String hostTags = markTagMaps(host, totalHosts, upHosts, downHosts); + // Get account, domain details for dedicated hosts if (isDedicated == 1) { String accountName; @@ -144,53 +157,112 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp final String cpuFactor = String.valueOf(CapacityManager.CpuOverprovisioningFactor.valueIn(host.getClusterId())); final CapacityVO cpuCapacity = capacityDao.findByHostIdType(host.getId(), Capacity.CAPACITY_TYPE_CPU); if (cpuCapacity != null) { - metricsList.add(new ItemHostCpu(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), cpuFactor, USED, cpuCapacity.getUsedCapacity())); - metricsList.add(new ItemHostCpu(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), cpuFactor, TOTAL, cpuCapacity.getTotalCapacity())); + metricsList.add(new ItemHostCpu(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), cpuFactor, USED, cpuCapacity.getUsedCapacity(), isDedicated, hostTags)); + metricsList.add(new ItemHostCpu(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), cpuFactor, TOTAL, cpuCapacity.getTotalCapacity(), isDedicated, hostTags)); } else { - metricsList.add(new ItemHostCpu(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), cpuFactor, USED, 0L)); - metricsList.add(new ItemHostCpu(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), cpuFactor, TOTAL, 0L)); + metricsList.add(new ItemHostCpu(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), cpuFactor, USED, 0L, isDedicated, hostTags)); + metricsList.add(new ItemHostCpu(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), cpuFactor, TOTAL, 0L, isDedicated, hostTags)); } final String memoryFactor = String.valueOf(CapacityManager.MemOverprovisioningFactor.valueIn(host.getClusterId())); final CapacityVO memCapacity = capacityDao.findByHostIdType(host.getId(), Capacity.CAPACITY_TYPE_MEMORY); if (memCapacity != null) { - metricsList.add(new ItemHostMemory(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), memoryFactor, USED, memCapacity.getUsedCapacity(), isDedicated)); - metricsList.add(new ItemHostMemory(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), memoryFactor, TOTAL, memCapacity.getTotalCapacity(), isDedicated)); + metricsList.add(new ItemHostMemory(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), memoryFactor, USED, memCapacity.getUsedCapacity(), isDedicated, hostTags)); + metricsList.add(new ItemHostMemory(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), memoryFactor, TOTAL, memCapacity.getTotalCapacity(), isDedicated, hostTags)); } else { - metricsList.add(new ItemHostMemory(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), memoryFactor, USED, 0L, isDedicated)); - metricsList.add(new ItemHostMemory(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), memoryFactor, TOTAL, 0L, isDedicated)); + metricsList.add(new ItemHostMemory(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), memoryFactor, USED, 0L, isDedicated, hostTags)); + metricsList.add(new ItemHostMemory(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), memoryFactor, TOTAL, 0L, isDedicated, hostTags)); } metricsList.add(new ItemHostVM(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), vmDao.listByHostId(host.getId()).size())); final CapacityVO coreCapacity = capacityDao.findByHostIdType(host.getId(), Capacity.CAPACITY_TYPE_CPU_CORE); if (coreCapacity != null) { - metricsList.add(new ItemVMCore(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), USED, coreCapacity.getUsedCapacity(), isDedicated)); - metricsList.add(new ItemVMCore(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), TOTAL, coreCapacity.getTotalCapacity(), isDedicated)); + metricsList.add(new ItemVMCore(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), USED, coreCapacity.getUsedCapacity(), isDedicated, hostTags)); + metricsList.add(new ItemVMCore(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), TOTAL, coreCapacity.getTotalCapacity(), isDedicated, hostTags)); } else { - metricsList.add(new ItemVMCore(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), USED, 0L, isDedicated)); - metricsList.add(new ItemVMCore(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), TOTAL, 0L, isDedicated)); + metricsList.add(new ItemVMCore(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), USED, 0L, isDedicated, hostTags)); + metricsList.add(new ItemVMCore(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), TOTAL, 0L, isDedicated, hostTags)); } } final List cpuCapacity = capacityDao.findCapacityBy((int) Capacity.CAPACITY_TYPE_CPU, dcId, null, null); if (cpuCapacity != null && cpuCapacity.size() > 0) { - metricsList.add(new ItemHostCpu(zoneName, zoneUuid, null, null, null, null, ALLOCATED, cpuCapacity.get(0).getAllocatedCapacity() != null ? cpuCapacity.get(0).getAllocatedCapacity() : 0)); + metricsList.add(new ItemHostCpu(zoneName, zoneUuid, null, null, null, null, ALLOCATED, cpuCapacity.get(0).getAllocatedCapacity() != null ? cpuCapacity.get(0).getAllocatedCapacity() : 0, 0, "")); } final List memCapacity = capacityDao.findCapacityBy((int) Capacity.CAPACITY_TYPE_MEMORY, dcId, null, null); if (memCapacity != null && memCapacity.size() > 0) { - metricsList.add(new ItemHostMemory(zoneName, zoneUuid, null, null, null, null, ALLOCATED, memCapacity.get(0).getAllocatedCapacity() != null ? memCapacity.get(0).getAllocatedCapacity() : 0, 0)); + metricsList.add(new ItemHostMemory(zoneName, zoneUuid, null, null, null, null, ALLOCATED, memCapacity.get(0).getAllocatedCapacity() != null ? memCapacity.get(0).getAllocatedCapacity() : 0, 0, "")); } final List coreCapacity = capacityDao.findCapacityBy((int) Capacity.CAPACITY_TYPE_CPU_CORE, dcId, null, null); if (coreCapacity != null && coreCapacity.size() > 0) { - metricsList.add(new ItemVMCore(zoneName, zoneUuid, null, null, null, ALLOCATED, coreCapacity.get(0).getAllocatedCapacity() != null ? coreCapacity.get(0).getAllocatedCapacity() : 0, 0)); + metricsList.add(new ItemVMCore(zoneName, zoneUuid, null, null, null, ALLOCATED, coreCapacity.get(0).getAllocatedCapacity() != null ? coreCapacity.get(0).getAllocatedCapacity() : 0, 0, "")); } - metricsList.add(new ItemHost(zoneName, zoneUuid, ONLINE, up)); - metricsList.add(new ItemHost(zoneName, zoneUuid, OFFLINE, down)); - metricsList.add(new ItemHost(zoneName, zoneUuid, TOTAL, total)); + metricsList.add(new ItemHost(zoneName, zoneUuid, ONLINE, up, null)); + metricsList.add(new ItemHost(zoneName, zoneUuid, OFFLINE, down, null)); + metricsList.add(new ItemHost(zoneName, zoneUuid, TOTAL, total, null)); + + addHostTagsMetrics(metricsList, dcId, zoneName, zoneUuid, totalHosts, upHosts, downHosts, total, up, down); + } + + private String markTagMaps(HostVO host, Map totalHosts, Map upHosts, Map downHosts) { + List hostTags = _hostTagsDao.getHostTags(host.getId()); + markTags(hostTags,totalHosts); + if (host.getStatus() == Status.Up) { + markTags(hostTags, upHosts); + } else if (host.getStatus() == Status.Disconnected || host.getStatus() == Status.Down) { + markTags(hostTags, downHosts); + } + return StringUtils.join(hostTags, ","); + } + + private void markTags(List tags, Map tagMap) { + tags.forEach(tag -> { + int current = tagMap.get(tag) != null ? tagMap.get(tag) : 0; + tagMap.put(tag, current + 1); + }); + } + + private void addHostTagsMetrics(final List metricsList, final long dcId, final String zoneName, final String zoneUuid, Map totalHosts, Map upHosts, Map downHosts, int total, int up, int down) { + + for (Map.Entry entry : totalHosts.entrySet()) { + String tag = entry.getKey(); + Integer count = entry.getValue(); + metricsList.add(new ItemHost(zoneName, zoneUuid, TOTAL, count, tag)); + if (upHosts.get(tag) != null) { + metricsList.add(new ItemHost(zoneName, zoneUuid, ONLINE, upHosts.get(tag), tag)); + } else { + metricsList.add(new ItemHost(zoneName, zoneUuid, ONLINE, 0, tag)); + } + if (downHosts.get(tag) != null) { + metricsList.add(new ItemHost(zoneName, zoneUuid, OFFLINE, downHosts.get(tag), tag)); + } else { + metricsList.add(new ItemHost(zoneName, zoneUuid, OFFLINE, 0, tag)); + } + } + + totalHosts.keySet() + .forEach( tag -> { + Ternary allocatedCapacityByTag = capacityDao.findCapacityByZoneAndHostTag(dcId, tag); + metricsList.add(new ItemVMCore(zoneName, zoneUuid, null, null, null, ALLOCATED, allocatedCapacityByTag.first(), 0, tag)); + metricsList.add(new ItemHostCpu(zoneName, zoneUuid, null, null, null, null, ALLOCATED, allocatedCapacityByTag.second(), 0, tag)); + metricsList.add(new ItemHostMemory(zoneName, zoneUuid, null, null, null, null, ALLOCATED, allocatedCapacityByTag.third(), 0, tag)); + }); + + List allHostTags = hostDao.listAll().stream() + .flatMap( h -> _hostTagsDao.getHostTags(h.getId()).stream()) + .distinct() + .collect(Collectors.toList()); + + for (final State state : State.values()) { + for (final String hostTag : allHostTags) { + final Long count = vmDao.countByZoneAndStateAndHostTag(dcId, state, hostTag); + metricsList.add(new ItemVMByTag(zoneName, zoneUuid, state.name().toLowerCase(), count, hostTag)); + } + } } private void addVMMetrics(final List metricsList, final long dcId, final String zoneName, final String zoneUuid) { @@ -335,6 +407,21 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp } } + private void addDomainMetrics(final List metricsList, final String zoneName, final String zoneUuid) { + metricsList.add(new ItemActiveDomains(zoneName, zoneUuid, _accountDao.getActiveDomains())); + } + + private void addAccountMetrics(final List metricsList, final long dcId, final String zoneName, final String zoneUuid) { + metricsList.add(new ItemActiveAccounts(zoneName, zoneUuid, uservmDao.getActiveAccounts(dcId))); + } + + private void addVMsBySizeMetrics(final List metricsList, final long dcId, final String zoneName, final String zoneUuid) { + List> vms = uservmDao.countVmsBySize(dcId, PrometheusExporterServer.PrometheusExporterOfferingCountLimit.value()); + for (Ternary vm : vms) { + metricsList.add(new ItemVMsBySize(zoneName, zoneUuid, vm.first(), vm.second(), vm.third())); + } + } + @Override public void updateMetrics() { final List latestMetricsItems = new ArrayList(); @@ -349,6 +436,9 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp addStorageMetrics(latestMetricsItems, dc.getId(), zoneName, zoneUuid); addIpAddressMetrics(latestMetricsItems, dc.getId(), zoneName, zoneUuid); addVlanMetrics(latestMetricsItems, dc.getId(), zoneName, zoneUuid); + addDomainMetrics(latestMetricsItems, zoneName, zoneUuid); + addAccountMetrics(latestMetricsItems, dc.getId(), zoneName, zoneUuid); + addVMsBySizeMetrics(latestMetricsItems, dc.getId(), zoneName, zoneUuid); } addDomainLimits(latestMetricsItems); addDomainResourceCount(latestMetricsItems); @@ -398,6 +488,28 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp } } + class ItemVMByTag extends Item { + String zoneName; + String zoneUuid; + String filter; + long total; + String hosttags; + + public ItemVMByTag(final String zn, final String zu, final String st, long cnt, final String tags) { + super("cloudstack_vms_total_by_tag"); + zoneName = zn; + zoneUuid = zu; + filter = st; + total = cnt; + hosttags = tags; + } + + @Override + public String toMetricsString() { + return String.format("%s{zone=\"%s\",filter=\"%s\",tags=\"%s\"} %d", name, zoneName, filter, hosttags, total); + } + } + class ItemVolume extends Item { String zoneName; String zoneUuid; @@ -423,17 +535,23 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp String zoneUuid; String state; int total; + String hosttags; - public ItemHost(final String zn, final String zu, final String st, int cnt) { + public ItemHost(final String zn, final String zu, final String st, int cnt, final String tags) { super("cloudstack_hosts_total"); zoneName = zn; zoneUuid = zu; state = st; total = cnt; + hosttags = tags; } @Override public String toMetricsString() { + if (StringUtils.isNotEmpty(hosttags)) { + name = "cloudstack_hosts_total_by_tag"; + return String.format("%s{zone=\"%s\",filter=\"%s\",tags=\"%s\"} %d", name, zoneName, state, hosttags, total); + } return String.format("%s{zone=\"%s\",filter=\"%s\"} %d", name, zoneName, state, total); } } @@ -447,8 +565,9 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp String filter; long core = 0; int isDedicated; + String hosttags; - public ItemVMCore(final String zn, final String zu, final String hn, final String hu, final String hip, final String fl, final Long cr, final int dedicated) { + public ItemVMCore(final String zn, final String zu, final String hn, final String hu, final String hip, final String fl, final Long cr, final int dedicated, final String tags) { super("cloudstack_host_vms_cores_total"); zoneName = zn; zoneUuid = zu; @@ -460,14 +579,20 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp core = cr; } isDedicated = dedicated; + hosttags = tags; } @Override public String toMetricsString() { if (StringUtils.isAllEmpty(hostName, ip)) { - return String.format("%s{zone=\"%s\",filter=\"%s\"} %d", name, zoneName, filter, core); + if (StringUtils.isEmpty(hosttags)) { + return String.format("%s{zone=\"%s\",filter=\"%s\"} %d", name, zoneName, filter, core); + } else { + name = "cloudstack_host_vms_cores_total_by_tag"; + return String.format("%s{zone=\"%s\",filter=\"%s\",tags=\"%s\"} %d", name, zoneName, filter, hosttags, core); + } } - return String.format("%s{zone=\"%s\",hostname=\"%s\",ip=\"%s\",filter=\"%s\",dedicated=\"%d\"} %d", name, zoneName, hostName, ip, filter, isDedicated, core); + return String.format("%s{zone=\"%s\",hostname=\"%s\",ip=\"%s\",filter=\"%s\",dedicated=\"%d\",tags=\"%s\"} %d", name, zoneName, hostName, ip, filter, isDedicated, hosttags, core); } } @@ -480,8 +605,10 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp String overProvisioningFactor; String filter; double mhertz; + int isDedicated; + String hosttags; - public ItemHostCpu(final String zn, final String zu, final String hn, final String hu, final String hip, final String of, final String fl, final double mh) { + public ItemHostCpu(final String zn, final String zu, final String hn, final String hu, final String hip, final String of, final String fl, final double mh, final int dedicated, final String tags) { super("cloudstack_host_cpu_usage_mhz_total"); zoneName = zn; zoneUuid = zu; @@ -491,14 +618,21 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp overProvisioningFactor = of; filter = fl; mhertz = mh; + isDedicated = dedicated; + hosttags = tags; } @Override public String toMetricsString() { if (StringUtils.isAllEmpty(hostName, ip)) { - return String.format("%s{zone=\"%s\",filter=\"%s\"} %.2f", name, zoneName, filter, mhertz); + if (StringUtils.isEmpty(hosttags)) { + return String.format("%s{zone=\"%s\",filter=\"%s\"} %.2f", name, zoneName, filter, mhertz); + } else { + name = "cloudstack_host_cpu_usage_mhz_total_by_tag"; + return String.format("%s{zone=\"%s\",filter=\"%s\",tags=\"%s\"} %.2f", name, zoneName, filter, hosttags, mhertz); + } } - return String.format("%s{zone=\"%s\",hostname=\"%s\",ip=\"%s\",overprovisioningfactor=\"%s\",filter=\"%s\"} %.2f", name, zoneName, hostName, ip, overProvisioningFactor, filter, mhertz); + return String.format("%s{zone=\"%s\",hostname=\"%s\",ip=\"%s\",overprovisioningfactor=\"%s\",filter=\"%s\",tags=\"%s\"} %.2f", name, zoneName, hostName, ip, overProvisioningFactor, filter, hosttags, mhertz); } } @@ -512,8 +646,9 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp String filter; double miBytes; int isDedicated; + String hosttags; - public ItemHostMemory(final String zn, final String zu, final String hn, final String hu, final String hip, final String of, final String fl, final double membytes, final int dedicated) { + public ItemHostMemory(final String zn, final String zu, final String hn, final String hu, final String hip, final String of, final String fl, final double membytes, final int dedicated, final String tags) { super("cloudstack_host_memory_usage_mibs_total"); zoneName = zn; zoneUuid = zu; @@ -524,14 +659,20 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp filter = fl; miBytes = membytes / (1024.0 * 1024.0); isDedicated = dedicated; + hosttags = tags; } @Override public String toMetricsString() { if (StringUtils.isAllEmpty(hostName, ip)) { - return String.format("%s{zone=\"%s\",filter=\"%s\"} %.2f", name, zoneName, filter, miBytes); + if (StringUtils.isEmpty(hosttags)) { + return String.format("%s{zone=\"%s\",filter=\"%s\"} %.2f", name, zoneName, filter, miBytes); + } else { + name = "cloudstack_host_memory_usage_mibs_total_by_tag"; + return String.format("%s{zone=\"%s\",filter=\"%s\",tags=\"%s\"} %.2f", name, zoneName, filter, hosttags, miBytes); + } } - return String.format("%s{zone=\"%s\",hostname=\"%s\",ip=\"%s\",overprovisioningfactor=\"%s\",filter=\"%s\",dedicated=\"%d\"} %.2f", name, zoneName, hostName, ip, overProvisioningFactor, filter, isDedicated, miBytes); + return String.format("%s{zone=\"%s\",hostname=\"%s\",ip=\"%s\",overprovisioningfactor=\"%s\",filter=\"%s\",dedicated=\"%d\",tags=\"%s\"} %.2f", name, zoneName, hostName, ip, overProvisioningFactor, filter, isDedicated, hosttags, miBytes); } } @@ -723,6 +864,24 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp } + class ItemActiveDomains extends Item { + String zoneName; + String zoneUuid; + int total; + + public ItemActiveDomains(final String zn, final String zu, final int cnt) { + super("cloudstack_active_domains_total"); + zoneName = zn; + zoneUuid = zu; + total = cnt; + } + + @Override + public String toMetricsString() { + return String.format("%s{zone=\"%s\"} %d", name, zoneName, total); + } + } + class ItemHostDedicatedToAccount extends Item { String zoneName; String hostName; @@ -782,4 +941,44 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp return String.format("%s{domain=\"%s\", type=\"%s\"} %d", name, domainName, resourceType, miBytes); } } + + class ItemActiveAccounts extends Item { + String zoneName; + String zoneUuid; + int total; + + public ItemActiveAccounts(final String zn, final String zu, final int cnt) { + super("cloudstack_active_accounts_total"); + zoneName = zn; + zoneUuid = zu; + total = cnt; + } + + @Override + public String toMetricsString() { + return String.format("%s{zone=\"%s\"} %d", name, zoneName, total); + } + } + + class ItemVMsBySize extends Item { + String zoneName; + String zoneUuid; + int cpu; + int memory; + int total; + + public ItemVMsBySize(final String zn, final String zu, final int c, final int m, int cnt) { + super("cloudstack_vms_total_by_size"); + zoneName = zn; + zoneUuid = zu; + cpu = c; + memory = m; + total = cnt; + } + + @Override + public String toMetricsString() { + return String.format("%s{zone=\"%s\",cpu=\"%d\",memory=\"%d\"} %d", name, zoneName, cpu, memory, total); + } + } } diff --git a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterServer.java b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterServer.java index 12d513c92d7..0ec83066f61 100644 --- a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterServer.java +++ b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterServer.java @@ -30,4 +30,7 @@ public interface PrometheusExporterServer extends Manager { ConfigKey PrometheusExporterAllowedAddresses = new ConfigKey<>("Advanced", String.class, "prometheus.exporter.allowed.ips", "127.0.0.1", "List of comma separated prometheus server ips (with no spaces) that should be allowed to access the URLs", true); + + ConfigKey PrometheusExporterOfferingCountLimit = new ConfigKey<>("Advanced", Integer.class, "prometheus.exporter.offering.output.limit", "-1", + "Limit the number of output for cloudstack_vms_total_by_size to the provided value. -1 for unlimited output.", true); } diff --git a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterServerImpl.java b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterServerImpl.java index a615c65766b..d550e2a3c91 100644 --- a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterServerImpl.java +++ b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterServerImpl.java @@ -112,7 +112,8 @@ public class PrometheusExporterServerImpl extends ManagerBase implements Prometh return new ConfigKey[] { EnablePrometheusExporter, PrometheusExporterServerPort, - PrometheusExporterAllowedAddresses + PrometheusExporterAllowedAddresses, + PrometheusExporterOfferingCountLimit }; } }