From 4c6c8216d5b4d86d8feeee8dea07d8ff63d1d0df Mon Sep 17 00:00:00 2001 From: Vishesh Date: Thu, 14 Mar 2024 17:49:35 +0530 Subject: [PATCH] Use join instead of views (#365) * Use join instead of views for filtering volumes * Use join instead of views for filtering events * Use join instead of views for filtering accounts * Use join instead of views for filtering domains * Use join instead of views for filtering hosts * Use join instead of views for filtering storage pools * Use join instead of views for filtering service offerings * Use join instead of views for filtering disk offerings * Remove unused code * Fix unit test * Use disk_offering instead of disk_offering_view in service_offering_view * Fixup * Fix listing of diskoffering & serviceoffering * Use constants instead of strings * Make changes to prevent sql injection * Remove commented code * Prevent n+1 queries for template's response * remove unused import * refactor some code * Add missing check for service offering's join with disk offering * Fix n+1 queries for stoage pool metrics * Remove n+1 queries from list accounts * Remove unused imports * remove todo * Remove unused import * Fixup query generation for nested joins * Fixups * Fix DB exception on ClientPreparedStatement * events,alerts: Add missing indexes (#366) * Fixup --- .../com/cloud/offering/ServiceOffering.java | 2 +- .../apache/cloudstack/acl/RoleService.java | 2 + .../cloudstack/api/InternalIdentity.java | 14 + .../com/cloud/service/ServiceOfferingVO.java | 4 +- .../upgrade/dao/DatabaseAccessObject.java | 9 +- .../com/cloud/upgrade/dao/DbUpgradeUtils.java | 6 +- .../upgrade/dao/Upgrade41800to41810.java | 5 + .../apache/cloudstack/acl/dao/RoleDao.java | 2 + .../cloudstack/acl/dao/RoleDaoImpl.java | 17 + .../resourcedetail/DiskOfferingDetailVO.java | 4 + .../storage/datastore/db/ImageStoreDao.java | 2 + .../datastore/db/ImageStoreDaoImpl.java | 17 + .../datastore/db/PrimaryDataStoreDao.java | 8 + .../datastore/db/PrimaryDataStoreDaoImpl.java | 98 ++ .../META-INF/db/schema-41800to41810.sql | 96 ++ .../upgrade/dao/DatabaseAccessObjectTest.java | 15 +- .../java/com/cloud/utils/db/Attribute.java | 9 + .../java/com/cloud/utils/db/GenericDao.java | 2 + .../com/cloud/utils/db/GenericDaoBase.java | 154 ++- .../cloud/utils/db/GenericSearchBuilder.java | 22 + .../java/com/cloud/utils/db/JoinBuilder.java | 62 +- .../java/com/cloud/utils/db/SearchBase.java | 93 +- .../com/cloud/utils/db/SearchCriteria.java | 12 +- .../cloud/utils/db/GenericDaoBaseTest.java | 35 +- .../metrics/MetricsServiceImpl.java | 9 +- .../main/java/com/cloud/api/ApiDBUtils.java | 243 ++-- .../com/cloud/api/query/QueryManagerImpl.java | 1164 +++++++++++------ .../cloud/api/query/ViewResponseHelper.java | 6 +- .../cloud/api/query/dao/AccountJoinDao.java | 2 + .../api/query/dao/AccountJoinDaoImpl.java | 53 + .../api/query/dao/DiskOfferingJoinDao.java | 2 + .../query/dao/DiskOfferingJoinDaoImpl.java | 54 + .../cloud/api/query/dao/DomainJoinDao.java | 2 + .../api/query/dao/DomainJoinDaoImpl.java | 53 + .../api/query/dao/ServiceOfferingJoinDao.java | 2 + .../query/dao/ServiceOfferingJoinDaoImpl.java | 54 + .../api/query/dao/StoragePoolJoinDao.java | 6 - .../api/query/dao/StoragePoolJoinDaoImpl.java | 76 -- .../api/query/dao/TemplateJoinDaoImpl.java | 11 +- .../ConfigurationManagerImpl.java | 2 +- .../com/cloud/network/NetworkServiceImpl.java | 4 +- .../cloudstack/acl/RoleManagerImpl.java | 27 + .../cloud/api/query/QueryManagerImplTest.java | 38 +- .../cloud/network/NetworkServiceImplTest.java | 4 +- .../com/cloud/user/MockUsageEventDao.java | 5 + 45 files changed, 1828 insertions(+), 679 deletions(-) diff --git a/api/src/main/java/com/cloud/offering/ServiceOffering.java b/api/src/main/java/com/cloud/offering/ServiceOffering.java index 278acbe231e..58c7b0dbaf9 100644 --- a/api/src/main/java/com/cloud/offering/ServiceOffering.java +++ b/api/src/main/java/com/cloud/offering/ServiceOffering.java @@ -102,7 +102,7 @@ public interface ServiceOffering extends InfrastructureEntity, InternalIdentity, boolean getDefaultUse(); - String getSystemVmType(); + String getVmType(); String getDeploymentPlanner(); diff --git a/api/src/main/java/org/apache/cloudstack/acl/RoleService.java b/api/src/main/java/org/apache/cloudstack/acl/RoleService.java index 578c13ef6fd..d18c25f43df 100644 --- a/api/src/main/java/org/apache/cloudstack/acl/RoleService.java +++ b/api/src/main/java/org/apache/cloudstack/acl/RoleService.java @@ -40,6 +40,8 @@ public interface RoleService { */ Role findRole(Long id); + List findRoles(List ids, boolean ignorePrivateRoles); + Role createRole(String name, RoleType roleType, String description); Role createRole(String name, Role role, String description); diff --git a/api/src/main/java/org/apache/cloudstack/api/InternalIdentity.java b/api/src/main/java/org/apache/cloudstack/api/InternalIdentity.java index 4149dd1c846..ce214e05b4f 100644 --- a/api/src/main/java/org/apache/cloudstack/api/InternalIdentity.java +++ b/api/src/main/java/org/apache/cloudstack/api/InternalIdentity.java @@ -25,4 +25,18 @@ import java.io.Serializable; public interface InternalIdentity extends Serializable { long getId(); + + /* + Helper method to add conditions in joins where some column name is equal to a string value + */ + default Object setString(String str) { + return null; + } + + /* + Helper method to add conditions in joins where some column name is equal to a long value + */ + default Object setLong(Long l) { + return null; + } } diff --git a/engine/schema/src/main/java/com/cloud/service/ServiceOfferingVO.java b/engine/schema/src/main/java/com/cloud/service/ServiceOfferingVO.java index 31e4b073c13..7f5c1a7afa1 100644 --- a/engine/schema/src/main/java/com/cloud/service/ServiceOfferingVO.java +++ b/engine/schema/src/main/java/com/cloud/service/ServiceOfferingVO.java @@ -194,7 +194,7 @@ public class ServiceOfferingVO implements ServiceOffering { limitCpuUse = offering.getLimitCpuUse(); volatileVm = offering.isVolatileVm(); hostTag = offering.getHostTag(); - vmType = offering.getSystemVmType(); + vmType = offering.getVmType(); systemUse = offering.isSystemUse(); dynamicScalingEnabled = offering.isDynamicScalingEnabled(); diskOfferingStrictness = offering.diskOfferingStrictness; @@ -278,7 +278,7 @@ public class ServiceOfferingVO implements ServiceOffering { } @Override - public String getSystemVmType() { + public String getVmType() { return vmType; } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java index 0b38acb5c21..f1e62b6f0fa 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java @@ -21,6 +21,7 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; public class DatabaseAccessObject { @@ -85,8 +86,8 @@ public class DatabaseAccessObject { return columnExists; } - public String generateIndexName(String tableName, String columnName) { - return String.format("i_%s__%s", tableName, columnName); + public String generateIndexName(String tableName, String... columnNames) { + return String.format("i_%s__%s", tableName, StringUtils.join(columnNames, "__")); } public boolean indexExists(Connection conn, String tableName, String indexName) { @@ -101,8 +102,8 @@ public class DatabaseAccessObject { return false; } - public void createIndex(Connection conn, String tableName, String columnName, String indexName) { - String stmt = String.format("CREATE INDEX %s on %s (%s)", indexName, tableName, columnName); + public void createIndex(Connection conn, String tableName, String indexName, String... columnNames) { + String stmt = String.format("CREATE INDEX %s ON %s (%s)", indexName, tableName, StringUtils.join(columnNames, ", ")); s_logger.debug("Statement: " + stmt); try (PreparedStatement pstmt = conn.prepareStatement(stmt)) { pstmt.execute(); diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/DbUpgradeUtils.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/DbUpgradeUtils.java index 6b4e1814de0..51e6ac7b9a1 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/DbUpgradeUtils.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/DbUpgradeUtils.java @@ -23,11 +23,11 @@ public class DbUpgradeUtils { private static DatabaseAccessObject dao = new DatabaseAccessObject(); - public static void addIndexIfNeeded(Connection conn, String tableName, String columnName) { - String indexName = dao.generateIndexName(tableName, columnName); + public static void addIndexIfNeeded(Connection conn, String tableName, String... columnNames) { + String indexName = dao.generateIndexName(tableName, columnNames); if (!dao.indexExists(conn, tableName, indexName)) { - dao.createIndex(conn, tableName, columnName, indexName); + dao.createIndex(conn, tableName, indexName, columnNames); } } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41800to41810.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41800to41810.java index a58d9965259..20bd0458d9e 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41800to41810.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41800to41810.java @@ -245,6 +245,11 @@ public class Upgrade41800to41810 implements DbUpgrade, DbUpgradeSystemVmTemplate } private void addIndexes(Connection conn) { + DbUpgradeUtils.addIndexIfNeeded(conn, "alert", "archived", "created"); + DbUpgradeUtils.addIndexIfNeeded(conn, "alert", "type", "data_center_id", "pod_id"); + + DbUpgradeUtils.addIndexIfNeeded(conn, "event", "resource_type", "resource_id"); + DbUpgradeUtils.addIndexIfNeeded(conn, "cluster_details", "name"); } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/acl/dao/RoleDao.java b/engine/schema/src/main/java/org/apache/cloudstack/acl/dao/RoleDao.java index 36833d5e790..3f3675900ca 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/acl/dao/RoleDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/acl/dao/RoleDao.java @@ -35,4 +35,6 @@ public interface RoleDao extends GenericDao { RoleVO findByNameAndType(String roleName, RoleType type); Pair, Integer> findAllByRoleType(RoleType type, Long offset, Long limit); + + List searchByIds(Long... ids); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/acl/dao/RoleDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/acl/dao/RoleDaoImpl.java index b4938a1e833..89b8ae7fd41 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/acl/dao/RoleDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/acl/dao/RoleDaoImpl.java @@ -28,10 +28,13 @@ import org.apache.cloudstack.acl.RoleVO; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; +import java.util.Collections; import java.util.List; @Component public class RoleDaoImpl extends GenericDaoBase implements RoleDao { + + private final SearchBuilder RoleByIdsSearch; private final SearchBuilder RoleByNameSearch; private final SearchBuilder RoleByTypeSearch; private final SearchBuilder RoleByNameAndTypeSearch; @@ -39,6 +42,10 @@ public class RoleDaoImpl extends GenericDaoBase implements RoleDao public RoleDaoImpl() { super(); + RoleByIdsSearch = createSearchBuilder(); + RoleByIdsSearch.and("idIN", RoleByIdsSearch.entity().getId(), SearchCriteria.Op.IN); + RoleByIdsSearch.done(); + RoleByNameSearch = createSearchBuilder(); RoleByNameSearch.and("roleName", RoleByNameSearch.entity().getName(), SearchCriteria.Op.LIKE); RoleByNameSearch.done(); @@ -96,4 +103,14 @@ public class RoleDaoImpl extends GenericDaoBase implements RoleDao sc.setParameters("roleType", type); return findOneBy(sc); } + + @Override + public List searchByIds(Long... ids) { + if (ids == null || ids.length == 0) { + return Collections.emptyList(); + } + SearchCriteria sc = RoleByIdsSearch.create(); + sc.setParameters("idIN", ids); + return listBy(sc); + } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/DiskOfferingDetailVO.java b/engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/DiskOfferingDetailVO.java index f4d98c3fb7f..7b0500680b9 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/DiskOfferingDetailVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/DiskOfferingDetailVO.java @@ -65,6 +65,10 @@ public class DiskOfferingDetailVO implements ResourceDetail { return name; } + public void setName(String name) { + this.name = name; + } + @Override public String getValue() { return value; diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDao.java index ba9825c3c86..5a572b1ef11 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDao.java @@ -49,4 +49,6 @@ public interface ImageStoreDao extends GenericDao { List findByProtocol(String protocol); ImageStoreVO findOneByZoneAndProtocol(long zoneId, String protocol); + + List listByIds(List ids); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDaoImpl.java index 3468b6008d9..d19b06073ea 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDaoImpl.java @@ -18,12 +18,14 @@ */ package org.apache.cloudstack.storage.datastore.db; +import java.util.Collections; import java.util.List; import java.util.Map; import javax.naming.ConfigurationException; import com.cloud.utils.db.Filter; +import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Component; import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; @@ -42,6 +44,7 @@ public class ImageStoreDaoImpl extends GenericDaoBase implem private SearchBuilder storeSearch; private SearchBuilder protocolSearch; private SearchBuilder zoneProtocolSearch; + private SearchBuilder IdsSearch; public ImageStoreDaoImpl() { super(); @@ -55,6 +58,10 @@ public class ImageStoreDaoImpl extends GenericDaoBase implem zoneProtocolSearch.and("protocol", zoneProtocolSearch.entity().getProtocol(), SearchCriteria.Op.EQ); zoneProtocolSearch.and("role", zoneProtocolSearch.entity().getRole(), SearchCriteria.Op.EQ); zoneProtocolSearch.done(); + + IdsSearch = createSearchBuilder(); + IdsSearch.and("ids", IdsSearch.entity().getId(), SearchCriteria.Op.IN); + IdsSearch.done(); } @Override public boolean configure(String name, Map params) throws ConfigurationException { @@ -191,4 +198,14 @@ public class ImageStoreDaoImpl extends GenericDaoBase implem List results = listBy(sc, filter); return results.size() == 0 ? null : results.get(0); } + + @Override + public List listByIds(List ids) { + if (CollectionUtils.isEmpty(ids)) { + return Collections.emptyList(); + } + SearchCriteria sc = IdsSearch.create(); + sc.setParameters("ids", ids.toArray()); + return listBy(sc); + } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java index 9a62f5d3c74..c23943ed8e9 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java @@ -23,6 +23,8 @@ import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.storage.ScopeType; import com.cloud.storage.Storage; import com.cloud.storage.StoragePoolStatus; +import com.cloud.utils.Pair; +import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDao; /** @@ -134,4 +136,10 @@ public interface PrimaryDataStoreDao extends GenericDao { List findPoolsByStorageType(Storage.StoragePoolType storageType); List listStoragePoolsWithActiveVolumesByOfferingId(long offeringid); + + Pair, Integer> searchForIdsAndCount(Long storagePoolId, String storagePoolName, Long zoneId, + String path, Long podId, Long clusterId, String address, ScopeType scopeType, StoragePoolStatus status, + String keyword, Filter searchFilter); + + List listByIds(List ids); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java index 58de405751c..58d5ec47dfd 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java @@ -20,12 +20,16 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.utils.Pair; +import com.cloud.utils.db.Filter; import org.apache.commons.collections.CollectionUtils; import com.cloud.host.Status; @@ -58,6 +62,7 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase private final SearchBuilder DcLocalStorageSearch; private final GenericSearchBuilder StatusCountSearch; private final SearchBuilder ClustersSearch; + private final SearchBuilder IdsSearch; @Inject private StoragePoolDetailsDao _detailsDao; @@ -144,6 +149,11 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase ClustersSearch = createSearchBuilder(); ClustersSearch.and("clusterIds", ClustersSearch.entity().getClusterId(), Op.IN); ClustersSearch.and("status", ClustersSearch.entity().getStatus(), Op.EQ); + ClustersSearch.done(); + + IdsSearch = createSearchBuilder(); + IdsSearch.and("ids", IdsSearch.entity().getId(), SearchCriteria.Op.IN); + IdsSearch.done(); } @@ -616,4 +626,92 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase throw new CloudRuntimeException("Caught: " + sql, e); } } + + @Override + public Pair, Integer> searchForIdsAndCount(Long storagePoolId, String storagePoolName, Long zoneId, + String path, Long podId, Long clusterId, String address, ScopeType scopeType, StoragePoolStatus status, + String keyword, Filter searchFilter) { + SearchCriteria sc = createStoragePoolSearchCriteria(storagePoolId, storagePoolName, zoneId, path, podId, clusterId, address, scopeType, status, keyword); + Pair, Integer> uniquePair = searchAndCount(sc, searchFilter); + List idList = uniquePair.first().stream().map(StoragePoolVO::getId).collect(Collectors.toList()); + return new Pair<>(idList, uniquePair.second()); + } + + @Override + public List listByIds(List ids) { + if (CollectionUtils.isEmpty(ids)) { + return Collections.emptyList(); + } + SearchCriteria sc = IdsSearch.create(); + sc.setParameters("ids", ids.toArray()); + return listBy(sc); + } + + private SearchCriteria createStoragePoolSearchCriteria(Long storagePoolId, String storagePoolName, + Long zoneId, String path, Long podId, Long clusterId, String address, ScopeType scopeType, + StoragePoolStatus status, String keyword) { + SearchBuilder sb = createSearchBuilder(); + sb.select(null, SearchCriteria.Func.DISTINCT, sb.entity().getId()); // select distinct + // ids + sb.and("id", sb.entity().getId(), SearchCriteria.Op.EQ); + sb.and("name", sb.entity().getName(), SearchCriteria.Op.EQ); + sb.and("path", sb.entity().getPath(), SearchCriteria.Op.EQ); + sb.and("dataCenterId", sb.entity().getDataCenterId(), SearchCriteria.Op.EQ); + sb.and("podId", sb.entity().getPodId(), SearchCriteria.Op.EQ); + sb.and("clusterId", sb.entity().getClusterId(), SearchCriteria.Op.EQ); + sb.and("hostAddress", sb.entity().getHostAddress(), SearchCriteria.Op.EQ); + sb.and("scope", sb.entity().getScope(), SearchCriteria.Op.EQ); + sb.and("status", sb.entity().getStatus(), SearchCriteria.Op.EQ); + sb.and("parent", sb.entity().getParent(), SearchCriteria.Op.EQ); + + SearchCriteria sc = sb.create(); + + if (keyword != null) { + SearchCriteria ssc = createSearchCriteria(); + ssc.addOr("name", SearchCriteria.Op.LIKE, "%" + keyword + "%"); + ssc.addOr("poolType", SearchCriteria.Op.LIKE, new Storage.StoragePoolType("%" + keyword + "%")); + + sc.addAnd("name", SearchCriteria.Op.SC, ssc); + } + + if (storagePoolId != null) { + sc.setParameters("id", storagePoolId); + } + + if (storagePoolName != null) { + sc.setParameters("name", storagePoolName); + } + + if (path != null) { + sc.setParameters("path", path); + } + if (zoneId != null) { + sc.setParameters("dataCenterId", zoneId); + } + if (podId != null) { + SearchCriteria ssc = createSearchCriteria(); + ssc.addOr("podId", SearchCriteria.Op.EQ, podId); + ssc.addOr("podId", SearchCriteria.Op.NULL); + + sc.addAnd("podId", SearchCriteria.Op.SC, ssc); + } + if (address != null) { + sc.setParameters("hostAddress", address); + } + if (clusterId != null) { + SearchCriteria ssc = createSearchCriteria(); + ssc.addOr("clusterId", SearchCriteria.Op.EQ, clusterId); + ssc.addOr("clusterId", SearchCriteria.Op.NULL); + + sc.addAnd("clusterId", SearchCriteria.Op.SC, ssc); + } + if (scopeType != null) { + sc.setParameters("scope", scopeType.toString()); + } + if (status != null) { + sc.setParameters("status", status.toString()); + } + sc.setParameters("parent", 0); + return sc; + } } diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41800to41810.sql b/engine/schema/src/main/resources/META-INF/db/schema-41800to41810.sql index fee2b038e8d..7c5e2ac1c8e 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41800to41810.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41800to41810.sql @@ -48,6 +48,102 @@ WHERE so.default_use = 1 AND so.vm_type IN ('domainrouter', 'secondarystoragevm' UPDATE `cloud`.`guest_os_hypervisor` SET guest_os_name = 'rhel9_64Guest' WHERE guest_os_name = 'rhel9_64Guest,'; CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.guest_os', 'display', 'tinyint(1) DEFAULT ''1'' COMMENT ''should this guest_os be shown to the end user'' '); +-- +DROP VIEW IF EXISTS `cloud`.`service_offering_view`; +CREATE VIEW `cloud`.`service_offering_view` AS +SELECT + `service_offering`.`id` AS `id`, + `service_offering`.`uuid` AS `uuid`, + `service_offering`.`name` AS `name`, + `service_offering`.`display_text` AS `display_text`, + `disk_offering`.`provisioning_type` AS `provisioning_type`, + `service_offering`.`created` AS `created`, + `disk_offering`.`tags` AS `tags`, + `service_offering`.`removed` AS `removed`, + `disk_offering`.`use_local_storage` AS `use_local_storage`, + `service_offering`.`system_use` AS `system_use`, + `disk_offering`.`id` AS `disk_offering_id`, + `disk_offering`.`name` AS `disk_offering_name`, + `disk_offering`.`uuid` AS `disk_offering_uuid`, + `disk_offering`.`display_text` AS `disk_offering_display_text`, + `disk_offering`.`customized_iops` AS `customized_iops`, + `disk_offering`.`min_iops` AS `min_iops`, + `disk_offering`.`max_iops` AS `max_iops`, + `disk_offering`.`hv_ss_reserve` AS `hv_ss_reserve`, + `disk_offering`.`bytes_read_rate` AS `bytes_read_rate`, + `disk_offering`.`bytes_read_rate_max` AS `bytes_read_rate_max`, + `disk_offering`.`bytes_read_rate_max_length` AS `bytes_read_rate_max_length`, + `disk_offering`.`bytes_write_rate` AS `bytes_write_rate`, + `disk_offering`.`bytes_write_rate_max` AS `bytes_write_rate_max`, + `disk_offering`.`bytes_write_rate_max_length` AS `bytes_write_rate_max_length`, + `disk_offering`.`iops_read_rate` AS `iops_read_rate`, + `disk_offering`.`iops_read_rate_max` AS `iops_read_rate_max`, + `disk_offering`.`iops_read_rate_max_length` AS `iops_read_rate_max_length`, + `disk_offering`.`iops_write_rate` AS `iops_write_rate`, + `disk_offering`.`iops_write_rate_max` AS `iops_write_rate_max`, + `disk_offering`.`iops_write_rate_max_length` AS `iops_write_rate_max_length`, + `disk_offering`.`cache_mode` AS `cache_mode`, + `disk_offering`.`disk_size` AS `root_disk_size`, + `disk_offering`.`encrypt` AS `encrypt_root`, + `service_offering`.`cpu` AS `cpu`, + `service_offering`.`speed` AS `speed`, + `service_offering`.`ram_size` AS `ram_size`, + `service_offering`.`nw_rate` AS `nw_rate`, + `service_offering`.`mc_rate` AS `mc_rate`, + `service_offering`.`ha_enabled` AS `ha_enabled`, + `service_offering`.`limit_cpu_use` AS `limit_cpu_use`, + `service_offering`.`host_tag` AS `host_tag`, + `service_offering`.`default_use` AS `default_use`, + `service_offering`.`vm_type` AS `vm_type`, + `service_offering`.`sort_key` AS `sort_key`, + `service_offering`.`is_volatile` AS `is_volatile`, + `service_offering`.`deployment_planner` AS `deployment_planner`, + `service_offering`.`dynamic_scaling_enabled` AS `dynamic_scaling_enabled`, + `service_offering`.`disk_offering_strictness` AS `disk_offering_strictness`, + `vsphere_storage_policy`.`value` AS `vsphere_storage_policy`, + GROUP_CONCAT(DISTINCT(domain.id)) AS domain_id, + GROUP_CONCAT(DISTINCT(domain.uuid)) AS domain_uuid, + GROUP_CONCAT(DISTINCT(domain.name)) AS domain_name, + GROUP_CONCAT(DISTINCT(domain.path)) AS domain_path, + GROUP_CONCAT(DISTINCT(zone.id)) AS zone_id, + GROUP_CONCAT(DISTINCT(zone.uuid)) AS zone_uuid, + GROUP_CONCAT(DISTINCT(zone.name)) AS zone_name, + IFNULL(`min_compute_details`.`value`, `cpu`) AS min_cpu, + IFNULL(`max_compute_details`.`value`, `cpu`) AS max_cpu, + IFNULL(`min_memory_details`.`value`, `ram_size`) AS min_memory, + IFNULL(`max_memory_details`.`value`, `ram_size`) AS max_memory +FROM + `cloud`.`service_offering` + INNER JOIN + `cloud`.`disk_offering` ON service_offering.disk_offering_id = disk_offering.id AND `disk_offering`.`state`='Active' + LEFT JOIN + `cloud`.`service_offering_details` AS `domain_details` ON `domain_details`.`service_offering_id` = `service_offering`.`id` AND `domain_details`.`name`='domainid' + LEFT JOIN + `cloud`.`domain` AS `domain` ON FIND_IN_SET(`domain`.`id`, `domain_details`.`value`) + LEFT JOIN + `cloud`.`service_offering_details` AS `zone_details` ON `zone_details`.`service_offering_id` = `service_offering`.`id` AND `zone_details`.`name`='zoneid' + LEFT JOIN + `cloud`.`data_center` AS `zone` ON FIND_IN_SET(`zone`.`id`, `zone_details`.`value`) + LEFT JOIN + `cloud`.`service_offering_details` AS `min_compute_details` ON `min_compute_details`.`service_offering_id` = `service_offering`.`id` + AND `min_compute_details`.`name` = 'mincpunumber' + LEFT JOIN + `cloud`.`service_offering_details` AS `max_compute_details` ON `max_compute_details`.`service_offering_id` = `service_offering`.`id` + AND `max_compute_details`.`name` = 'maxcpunumber' + LEFT JOIN + `cloud`.`service_offering_details` AS `min_memory_details` ON `min_memory_details`.`service_offering_id` = `service_offering`.`id` + AND `min_memory_details`.`name` = 'minmemory' + LEFT JOIN + `cloud`.`service_offering_details` AS `max_memory_details` ON `max_memory_details`.`service_offering_id` = `service_offering`.`id` + AND `max_memory_details`.`name` = 'maxmemory' + LEFT JOIN + `cloud`.`service_offering_details` AS `vsphere_storage_policy` ON `vsphere_storage_policy`.`service_offering_id` = `service_offering`.`id` + AND `vsphere_storage_policy`.`name` = 'storagepolicy' +WHERE + `service_offering`.`state`='Active' +GROUP BY + `service_offering`.`id`; + -- Idempotent ADD COLUMN DROP PROCEDURE IF EXISTS `cloud`.`IDEMPOTENT_ADD_COLUMN`; CREATE PROCEDURE `cloud`.`IDEMPOTENT_ADD_COLUMN` ( diff --git a/engine/schema/src/test/java/com/cloud/upgrade/dao/DatabaseAccessObjectTest.java b/engine/schema/src/test/java/com/cloud/upgrade/dao/DatabaseAccessObjectTest.java index 7e78c3ec3e9..0679888cca0 100644 --- a/engine/schema/src/test/java/com/cloud/upgrade/dao/DatabaseAccessObjectTest.java +++ b/engine/schema/src/test/java/com/cloud/upgrade/dao/DatabaseAccessObjectTest.java @@ -91,8 +91,14 @@ public class DatabaseAccessObjectTest { @Test public void generateIndexNameTest() { - String indexName = dao.generateIndexName("mytable","mycolumn"); - Assert.assertEquals( "i_mytable__mycolumn", indexName); + String indexName = dao.generateIndexName("mytable","mycolumn1", "mycolumn2"); + Assert.assertEquals( "i_mytable__mycolumn1__mycolumn2", indexName); + } + + @Test + public void generateIndexNameTestSingleColumn() { + String indexName = dao.generateIndexName("mytable","mycolumn1"); + Assert.assertEquals( "i_mytable__mycolumn1", indexName); } @Test @@ -134,10 +140,11 @@ public class DatabaseAccessObjectTest { Connection conn = connectionMock; String tableName = "mytable"; - String columnName = "mycolumn"; + String columnName1 = "mycolumn1"; + String columnName2 = "mycolumn2"; String indexName = "myindex"; - dao.createIndex(conn, tableName, columnName, indexName); + dao.createIndex(conn, tableName, indexName, columnName1, columnName2); verify(connectionMock, times(1)).prepareStatement(anyString()); verify(preparedStatementMock, times(1)).execute(); verify(preparedStatementMock, times(1)).close(); diff --git a/framework/db/src/main/java/com/cloud/utils/db/Attribute.java b/framework/db/src/main/java/com/cloud/utils/db/Attribute.java index 82c2bdbaa79..3e5128d97b4 100644 --- a/framework/db/src/main/java/com/cloud/utils/db/Attribute.java +++ b/framework/db/src/main/java/com/cloud/utils/db/Attribute.java @@ -81,6 +81,7 @@ public class Attribute { protected String table; protected String columnName; + protected Object value; protected Field field; protected int flags; protected Column column; @@ -100,6 +101,10 @@ public class Attribute { this.column = null; } + public Attribute(Object value) { + this.value = value; + } + protected void setupColumnInfo(Class clazz, AttributeOverride[] overrides, String tableName, boolean isEmbedded, boolean isId) { flags = Flag.Selectable.setTrue(flags); GeneratedValue gv = field.getAnnotation(GeneratedValue.class); @@ -214,6 +219,10 @@ public class Attribute { return field; } + public Object getValue() { + return value; + } + public Object get(Object entity) { try { return field.get(entity); diff --git a/framework/db/src/main/java/com/cloud/utils/db/GenericDao.java b/framework/db/src/main/java/com/cloud/utils/db/GenericDao.java index 1eae0edd9c3..65288f4a577 100644 --- a/framework/db/src/main/java/com/cloud/utils/db/GenericDao.java +++ b/framework/db/src/main/java/com/cloud/utils/db/GenericDao.java @@ -284,4 +284,6 @@ public interface GenericDao { Pair, Integer> searchAndDistinctCount(final SearchCriteria sc, final Filter filter, final String[] distinctColumns); Integer countAll(); + + List findByUuids(String... uuidArray); } diff --git a/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java b/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java index 4683dfb12e0..f7abe39d8fd 100644 --- a/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java +++ b/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java @@ -59,6 +59,7 @@ import javax.persistence.Enumerated; import javax.persistence.Table; import javax.persistence.TableGenerator; +import com.amazonaws.util.CollectionUtils; import org.apache.log4j.Logger; import com.cloud.utils.DateUtil; @@ -378,10 +379,11 @@ public abstract class GenericDaoBase extends Compone } Collection>> joins = null; + List joinAttrList = null; if (sc != null) { joins = sc.getJoins(); if (joins != null) { - addJoins(str, joins); + joinAttrList = addJoins(str, joins); } } @@ -401,6 +403,13 @@ public abstract class GenericDaoBase extends Compone try { pstmt = txn.prepareAutoCloseStatement(sql); int i = 1; + + if (!CollectionUtils.isNullOrEmpty(joinAttrList)) { + for (Attribute attr : joinAttrList) { + prepareAttribute(i++, pstmt, attr, null); + } + } + if (clause != null) { for (final Pair value : sc.getValues()) { prepareAttribute(i++, pstmt, value.first(), value.second()); @@ -453,8 +462,9 @@ public abstract class GenericDaoBase extends Compone Collection>> joins = null; joins = sc.getJoins(); + List joinAttrList = null; if (joins != null) { - addJoins(str, joins); + joinAttrList = addJoins(str, joins); } List groupByValues = addGroupBy(str, sc); @@ -467,6 +477,13 @@ public abstract class GenericDaoBase extends Compone try { pstmt = txn.prepareAutoCloseStatement(sql); int i = 1; + + if (!CollectionUtils.isNullOrEmpty(joinAttrList)) { + for (Attribute attr : joinAttrList) { + prepareAttribute(i++, pstmt, attr, null); + } + } + if (clause != null) { for (final Pair value : sc.getValues()) { prepareAttribute(i++, pstmt, value.first(), value.second()); @@ -1270,12 +1287,13 @@ public abstract class GenericDaoBase extends Compone } @DB() - protected void addJoins(StringBuilder str, Collection>> joins) { - addJoins(str, joins, new HashMap<>()); + protected List addJoins(StringBuilder str, Collection>> joins) { + return addJoins(str, joins, new HashMap<>()); } @DB() - protected void addJoins(StringBuilder str, Collection>> joins, Map joinedTableNames) { + protected List addJoins(StringBuilder str, Collection>> joins, Map joinedTableNames) { + List joinAttrList = new ArrayList<>(); boolean hasWhereClause = true; int fromIndex = str.lastIndexOf("WHERE"); if (fromIndex == -1) { @@ -1286,8 +1304,14 @@ public abstract class GenericDaoBase extends Compone } for (JoinBuilder> join : joins) { - String joinTableName = join.getSecondAttribute().table; - String joinTableAlias = findNextJoinTableName(joinTableName, joinedTableNames); + String joinTableName = join.getSecondAttribute()[0].table; + String joinTableAlias; + if (StringUtils.isNotEmpty(join.getName())) { + joinTableAlias = join.getName(); + joinedTableNames.put(joinTableName, joinTableAlias); + } else { + joinTableAlias = joinedTableNames.getOrDefault(joinTableName, joinTableName); + } StringBuilder onClause = new StringBuilder(); onClause.append(" ") .append(join.getType().getName()) @@ -1296,21 +1320,36 @@ public abstract class GenericDaoBase extends Compone if (!joinTableAlias.equals(joinTableName)) { onClause.append(" ").append(joinTableAlias); } - onClause.append(" ON ") - .append(join.getFirstAttribute().table) - .append(".") - .append(join.getFirstAttribute().columnName) - .append("="); - if(!joinTableAlias.equals(joinTableName)) { - onClause.append(joinTableAlias); - } else { - onClause.append(joinTableName); + onClause.append(" ON "); + for (int i = 0; i < join.getFirstAttributes().length; i++) { + if (i > 0) { + onClause.append(join.getCondition().getName()); + } + if (join.getFirstAttributes()[i].getValue() != null) { + onClause.append("?"); + joinAttrList.add(join.getFirstAttributes()[i]); + } else { + onClause.append(joinedTableNames.getOrDefault(join.getFirstAttributes()[i].table, join.getFirstAttributes()[i].table)) + .append(".") + .append(join.getFirstAttributes()[i].columnName); + } + onClause.append("="); + if (join.getSecondAttribute()[i].getValue() != null) { + onClause.append("?"); + joinAttrList.add(join.getSecondAttribute()[i]); + } else { + if(!joinTableAlias.equals(joinTableName)) { + onClause.append(joinTableAlias); + } else { + onClause.append(joinTableName); + } + onClause.append(".") + .append(join.getSecondAttribute()[i].columnName); + } } - onClause.append(".") - .append(join.getSecondAttribute().columnName) - .append(" "); + onClause.append(" "); str.insert(fromIndex, onClause); - String whereClause = join.getT().getWhereClause(); + String whereClause = join.getT().getWhereClause(joinTableAlias); if (StringUtils.isNotEmpty(whereClause)) { if (!hasWhereClause) { str.append(" WHERE "); @@ -1327,20 +1366,10 @@ public abstract class GenericDaoBase extends Compone for (JoinBuilder> join : joins) { if (join.getT().getJoins() != null) { - addJoins(str, join.getT().getJoins(), joinedTableNames); + joinAttrList.addAll(addJoins(str, join.getT().getJoins(), joinedTableNames)); } } - } - - protected static String findNextJoinTableName(String tableName, Map usedTableNames) { - if (usedTableNames.containsKey(tableName)) { - Integer tableCounter = usedTableNames.get(tableName); - usedTableNames.put(tableName, ++tableCounter); - tableName = tableName + tableCounter; - } else { - usedTableNames.put(tableName, 0); - } - return tableName; + return joinAttrList; } private void removeAndClause(StringBuilder sql) { @@ -1593,13 +1622,29 @@ public abstract class GenericDaoBase extends Compone return; } } - if(attr.field.getDeclaredAnnotation(Convert.class) != null) { + + if (attr.getValue() != null && attr.getValue() instanceof String) { + pstmt.setString(j, (String)attr.getValue()); + } else if (attr.getValue() != null && attr.getValue() instanceof Long) { + pstmt.setLong(j, (Long)attr.getValue()); + } else if(attr.field.getDeclaredAnnotation(Convert.class) != null) { Object val = _conversionSupport.convertToDatabaseColumn(attr.field, value); pstmt.setObject(j, val); + } else if (attr.field.getType() == String.class) { - final String str = (String)value; - if (str == null) { - pstmt.setString(j, null); + final String str; + try { + str = (String) value; + if (str == null) { + pstmt.setString(j, null); + return; + } + } catch (ClassCastException ex) { + // This happens when we pass in an integer, long or any other object which can't be cast to String. + // Converting to string in case of integer or long can result in different results. Required specifically for details tables. + // So, we set the value for the object directly. + s_logger.debug("ClassCastException when casting value to String. Setting the value of the object directly."); + pstmt.setObject(j, value); return; } final Column column = attr.field.getAnnotation(Column.class); @@ -2019,10 +2064,11 @@ public abstract class GenericDaoBase extends Compone } Collection>> joins = null; + List joinAttrList = null; if (sc != null) { joins = sc.getJoins(); if (joins != null) { - addJoins(str, joins); + joinAttrList = addJoins(str, joins); } } @@ -2035,6 +2081,13 @@ public abstract class GenericDaoBase extends Compone try { pstmt = txn.prepareAutoCloseStatement(sql); int i = 1; + + if (!CollectionUtils.isNullOrEmpty(joinAttrList)) { + for (Attribute attr : joinAttrList) { + prepareAttribute(i++, pstmt, attr, null); + } + } + if (clause != null) { for (final Pair value : sc.getValues()) { prepareAttribute(i++, pstmt, value.first(), value.second()); @@ -2082,10 +2135,11 @@ public abstract class GenericDaoBase extends Compone } Collection>> joins = null; + List joinAttrList = null; if (sc != null) { joins = sc.getJoins(); if (joins != null) { - addJoins(str, joins); + joinAttrList = addJoins(str, joins); } } @@ -2094,6 +2148,13 @@ public abstract class GenericDaoBase extends Compone try (PreparedStatement pstmt = txn.prepareAutoCloseStatement(sql)) { int i = 1; + + if (!CollectionUtils.isNullOrEmpty(joinAttrList)) { + for (Attribute attr : joinAttrList) { + prepareAttribute(i++, pstmt, attr, null); + } + } + if (clause != null) { for (final Pair value : sc.getValues()) { prepareAttribute(i++, pstmt, value.first(), value.second()); @@ -2120,6 +2181,13 @@ public abstract class GenericDaoBase extends Compone return getCount(null); } + @Override + public List findByUuids(String... uuidArray) { + SearchCriteria sc = createSearchCriteria(); + sc.addAnd("uuid", SearchCriteria.Op.IN, uuidArray); + return search(sc, null); + } + public Integer getCount(SearchCriteria sc) { sc = checkAndSetRemovedIsNull(sc); return getCountIncludingRemoved(sc); @@ -2137,10 +2205,11 @@ public abstract class GenericDaoBase extends Compone } Collection>> joins = null; + List joinAttrList = null; if (sc != null) { joins = sc.getJoins(); if (joins != null) { - addJoins(str, joins); + joinAttrList = addJoins(str, joins); } } @@ -2151,6 +2220,13 @@ public abstract class GenericDaoBase extends Compone try { pstmt = txn.prepareAutoCloseStatement(sql); int i = 1; + + if (!CollectionUtils.isNullOrEmpty(joinAttrList)) { + for (Attribute attr : joinAttrList) { + prepareAttribute(i++, pstmt, attr, null); + } + } + if (clause != null) { for (final Pair value : sc.getValues()) { prepareAttribute(i++, pstmt, value.first(), value.second()); diff --git a/framework/db/src/main/java/com/cloud/utils/db/GenericSearchBuilder.java b/framework/db/src/main/java/com/cloud/utils/db/GenericSearchBuilder.java index df6f1f7602f..f4385efb3b8 100644 --- a/framework/db/src/main/java/com/cloud/utils/db/GenericSearchBuilder.java +++ b/framework/db/src/main/java/com/cloud/utils/db/GenericSearchBuilder.java @@ -80,6 +80,12 @@ public class GenericSearchBuilder extends SearchBase and(String joinName, String name, Object field, Op op) { + SearchBase join = _joins.get(joinName).getT(); + constructCondition(joinName, name, " AND ", join._specifiedAttrs.get(0), op); + return this; + } + /** * Adds an AND condition. Some prefer this method because it looks like * the actual SQL query. @@ -134,6 +140,12 @@ public class GenericSearchBuilder extends SearchBase left(String joinName, Object field, Op op, String name) { + SearchBase joinSb = _joins.get(joinName).getT(); + constructCondition(joinName, name, " ( ", joinSb._specifiedAttrs.get(0), op); + return this; + } + protected Preset left(Object field, Op op) { Condition condition = constructCondition(UUID.randomUUID().toString(), " ( ", _specifiedAttrs.get(0), op); return new Preset(this, condition); @@ -169,6 +181,10 @@ public class GenericSearchBuilder extends SearchBase op(String joinName, String name, Object field, Op op) { + return left(joinName, field, op, name); + } + /** * Adds an OR condition to the SearchBuilder. * @@ -182,6 +198,12 @@ public class GenericSearchBuilder extends SearchBase or(String joinName, String name, Object field, Op op) { + SearchBase join = _joins.get(joinName).getT(); + constructCondition(joinName, name, " OR ", join._specifiedAttrs.get(0), op); + return this; + } + /** * Adds an OR condition * diff --git a/framework/db/src/main/java/com/cloud/utils/db/JoinBuilder.java b/framework/db/src/main/java/com/cloud/utils/db/JoinBuilder.java index 60e392170be..f0a3ad82596 100644 --- a/framework/db/src/main/java/com/cloud/utils/db/JoinBuilder.java +++ b/framework/db/src/main/java/com/cloud/utils/db/JoinBuilder.java @@ -31,19 +31,57 @@ public class JoinBuilder { return _name; } } + public enum JoinCondition { + AND(" AND "), OR(" OR "); + + private final String name; + + JoinCondition(String name) { + this.name = name; + } + + public String getName() { + return name; + } + } private final T t; + private final String name; private JoinType type; - private Attribute firstAttribute; - private Attribute secondAttribute; - public JoinBuilder(T t, Attribute firstAttribute, Attribute secondAttribute, JoinType type) { + private JoinCondition condition; + private Attribute[] firstAttributes; + private Attribute[] secondAttribute; + + public JoinBuilder(String name, T t, Attribute firstAttributes, Attribute secondAttribute, JoinType type) { + this.name = name; this.t = t; - this.firstAttribute = firstAttribute; + this.firstAttributes = new Attribute[]{firstAttributes}; + this.secondAttribute = new Attribute[]{secondAttribute}; + this.type = type; + } + + public JoinBuilder(String name, T t, Attribute[] firstAttributes, Attribute[] secondAttribute, JoinType type) { + this.name = name; + this.t = t; + this.firstAttributes = firstAttributes; this.secondAttribute = secondAttribute; this.type = type; } + public JoinBuilder(String name, T t, Attribute[] firstAttributes, Attribute[] secondAttribute, JoinType type, JoinCondition condition) { + this.name = name; + this.t = t; + this.firstAttributes = firstAttributes; + this.secondAttribute = secondAttribute; + this.type = type; + this.condition = condition; + } + + public String getName() { + return name; + } + public T getT() { return t; } @@ -56,19 +94,23 @@ public class JoinBuilder { this.type = type; } - public Attribute getFirstAttribute() { - return firstAttribute; + public JoinCondition getCondition() { + return condition; } - public void setFirstAttribute(Attribute firstAttribute) { - this.firstAttribute = firstAttribute; + public Attribute[] getFirstAttributes() { + return firstAttributes; } - public Attribute getSecondAttribute() { + public void setFirstAttributes(Attribute[] firstAttributes) { + this.firstAttributes = firstAttributes; + } + + public Attribute[] getSecondAttribute() { return secondAttribute; } - public void setSecondAttribute(Attribute secondAttribute) { + public void setSecondAttribute(Attribute[] secondAttribute) { this.secondAttribute = secondAttribute; } diff --git a/framework/db/src/main/java/com/cloud/utils/db/SearchBase.java b/framework/db/src/main/java/com/cloud/utils/db/SearchBase.java index 3d41a62b43c..fcc9ded684d 100644 --- a/framework/db/src/main/java/com/cloud/utils/db/SearchBase.java +++ b/framework/db/src/main/java/com/cloud/utils/db/SearchBase.java @@ -36,6 +36,7 @@ import com.cloud.utils.exception.CloudRuntimeException; import net.sf.cglib.proxy.Factory; import net.sf.cglib.proxy.MethodInterceptor; import net.sf.cglib.proxy.MethodProxy; +import org.apache.commons.lang3.StringUtils; /** * SearchBase contains the methods that are used to build up search @@ -70,6 +71,14 @@ public abstract class SearchBase, T, K> { init(entityType, resultType); } + public SearchBase getJoinSB(String name) { + JoinBuilder> jb = null; + if (_joins != null) { + jb = _joins.get(name); + } + return jb == null ? null : jb.getT(); + } + protected void init(final Class entityType, final Class resultType) { _dao = (GenericDaoBase)GenericDaoBase.getDao(entityType); if (_dao == null) { @@ -194,15 +203,45 @@ public abstract class SearchBase, T, K> { * @param joinType type of join * @return itself */ - @SuppressWarnings("unchecked") public J join(final String name, final SearchBase builder, final Object joinField1, final Object joinField2, final JoinBuilder.JoinType joinType) { - assert _entity != null : "SearchBuilder cannot be modified once it has been setup"; - assert _specifiedAttrs.size() == 1 : "You didn't select the attribute."; - assert builder._entity != null : "SearchBuilder cannot be modified once it has been setup"; - assert builder._specifiedAttrs.size() == 1 : "You didn't select the attribute."; - assert builder != this : "You can't add yourself, can you? Really think about it!"; + if (_specifiedAttrs.size() != 1) + throw new CloudRuntimeException("You didn't select the attribute."); + if (builder._specifiedAttrs.size() != 1) + throw new CloudRuntimeException("You didn't select the attribute."); - final JoinBuilder> t = new JoinBuilder>(builder, _specifiedAttrs.get(0), builder._specifiedAttrs.get(0), joinType); + return join(name, builder, joinType, null, joinField1, joinField2); + } + + + /** + * joins this search with another search with multiple conditions in the join clause + * + * @param name name given to the other search. used for setJoinParameters. + * @param builder The other search + * @param joinType type of join + * @param condition condition to be used for multiple conditions in the join clause + * @param joinFields fields the first and second table used to perform the join. + * The fields should be in the order of the checks between the two tables. + * + * @return + */ + public J join(final String name, final SearchBase builder, final JoinBuilder.JoinType joinType, final + JoinBuilder.JoinCondition condition, final Object... joinFields) { + if (_entity == null) + throw new CloudRuntimeException("SearchBuilder cannot be modified once it has been setup"); + if (_specifiedAttrs.isEmpty()) + throw new CloudRuntimeException("Attribute not specified."); + if (builder._entity == null) + throw new CloudRuntimeException("SearchBuilder cannot be modified once it has been setup"); + if (builder._specifiedAttrs.isEmpty()) + throw new CloudRuntimeException("Attribute not specified."); + if (builder == this) + throw new CloudRuntimeException("Can't join with itself. Create a new SearchBuilder for the same entity and use that."); + if (_specifiedAttrs.size() != builder._specifiedAttrs.size()) + throw new CloudRuntimeException("Number of attributes to join on must be the same."); + + final JoinBuilder> t = new JoinBuilder<>(name, builder, _specifiedAttrs.toArray(new Attribute[0]), + builder._specifiedAttrs.toArray(new Attribute[0]), joinType, condition); if (_joins == null) { _joins = new HashMap>>(); } @@ -223,6 +262,16 @@ public abstract class SearchBase, T, K> { _specifiedAttrs.add(attr); } + /* + Allows to set conditions in join where one entity is equivalent to a string or a long + e.g. join("vm", vmSearch, VmDetailVO.class, entity.getName(), "vm.id", SearchCriteria.Op.EQ); + will create a condition 'vm.name = "vm.id"' + */ + protected void setAttr(final Object obj) { + final Attribute attr = new Attribute(obj); + _specifiedAttrs.add(attr); + } + /** * @return entity object. This allows the caller to use the entity return * to specify the field to be selected in many of the search parameters. @@ -242,17 +291,26 @@ public abstract class SearchBase, T, K> { return _specifiedAttrs; } - protected Condition constructCondition(final String conditionName, final String cond, final Attribute attr, final Op op) { + protected Condition constructCondition(final String joinName, final String conditionName, final String cond, final Attribute attr, final Op op) { assert _entity != null : "SearchBuilder cannot be modified once it has been setup"; assert op == null || _specifiedAttrs.size() == 1 : "You didn't select the attribute."; assert op != Op.SC : "Call join"; final Condition condition = new Condition(conditionName, cond, attr, op); + if (StringUtils.isNotEmpty(joinName)) { + condition.setJoinName(joinName); + } _conditions.add(condition); _specifiedAttrs.clear(); return condition; } + + protected Condition constructCondition(final String conditionName, final String cond, final Attribute attr, final Op op) { + return constructCondition(null, conditionName, cond, attr, op); + } + + /** * creates the SearchCriteria so the actual values can be filled in. * @@ -364,6 +422,7 @@ public abstract class SearchBase, T, K> { protected static class Condition { protected final String name; protected final String cond; + protected String joinName; protected final Op op; protected final Attribute attr; protected Object[] presets; @@ -388,11 +447,15 @@ public abstract class SearchBase, T, K> { this.presets = presets; } + public void setJoinName(final String joinName) { + this.joinName = joinName; + } + public Object[] getPresets() { return presets; } - public void toSql(final StringBuilder sql, final Object[] params, final int count) { + public void toSql(final StringBuilder sql, String tableAlias, final Object[] params, final int count) { if (count > 0) { sql.append(cond); } @@ -414,7 +477,15 @@ public abstract class SearchBase, T, K> { sql.append(" FIND_IN_SET(?, "); } - sql.append(attr.table).append(".").append(attr.columnName).append(op.toString()); + if (tableAlias == null) { + if (joinName != null) { + tableAlias = joinName; + } else { + tableAlias = attr.table; + } + } + + sql.append(tableAlias).append(".").append(attr.columnName).append(op.toString()); if (op == Op.IN && params.length == 1) { sql.delete(sql.length() - op.toString().length(), sql.length()); sql.append("=?"); @@ -485,6 +556,8 @@ public abstract class SearchBase, T, K> { final String fieldName = Character.toLowerCase(name.charAt(2)) + name.substring(3); set(fieldName); return null; + } else if (name.equals("setLong") || name.equals("setString")) { + setAttr(args[0]); } else { final Column ann = method.getAnnotation(Column.class); if (ann != null) { diff --git a/framework/db/src/main/java/com/cloud/utils/db/SearchCriteria.java b/framework/db/src/main/java/com/cloud/utils/db/SearchCriteria.java index 6f90d2391e6..8affbd5300a 100644 --- a/framework/db/src/main/java/com/cloud/utils/db/SearchCriteria.java +++ b/framework/db/src/main/java/com/cloud/utils/db/SearchCriteria.java @@ -106,7 +106,7 @@ public class SearchCriteria { for (Map.Entry>> entry : sb._joins.entrySet()) { JoinBuilder> value = entry.getValue(); _joins.put(entry.getKey(), - new JoinBuilder>(value.getT().create(), value.getFirstAttribute(), value.getSecondAttribute(), value.getType())); + new JoinBuilder>(entry.getKey(), value.getT().create(), value.getFirstAttributes(), value.getSecondAttribute(), value.getType(), value.getCondition())); } } _selects = sb._selects; @@ -250,7 +250,7 @@ public class SearchCriteria { _additionals.add(condition); } - public String getWhereClause() { + public String getWhereClause(String tableAlias) { StringBuilder sql = new StringBuilder(); int i = 0; for (Condition condition : _conditions) { @@ -259,7 +259,7 @@ public class SearchCriteria { } Object[] params = _params.get(condition.name); if ((condition.op == null || condition.op.params == 0) || (params != null)) { - condition.toSql(sql, params, i++); + condition.toSql(sql, tableAlias, params, i++); } } @@ -269,13 +269,17 @@ public class SearchCriteria { } Object[] params = _params.get(condition.name); if ((condition.op.params == 0) || (params != null)) { - condition.toSql(sql, params, i++); + condition.toSql(sql, tableAlias, params, i++); } } return sql.toString(); } + public String getWhereClause() { + return getWhereClause(null); + } + public List> getValues() { ArrayList> params = new ArrayList>(_params.size()); for (Condition condition : _conditions) { diff --git a/framework/db/src/test/java/com/cloud/utils/db/GenericDaoBaseTest.java b/framework/db/src/test/java/com/cloud/utils/db/GenericDaoBaseTest.java index b950501337b..308600341c3 100644 --- a/framework/db/src/test/java/com/cloud/utils/db/GenericDaoBaseTest.java +++ b/framework/db/src/test/java/com/cloud/utils/db/GenericDaoBaseTest.java @@ -20,8 +20,6 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; import java.util.Collection; -import java.util.HashMap; -import java.util.Map; import org.junit.Assert; import org.junit.Before; @@ -229,12 +227,16 @@ public class GenericDaoBaseTest { Attribute attr2 = new Attribute("table2", "column2"); Attribute attr3 = new Attribute("table3", "column1"); Attribute attr4 = new Attribute("table4", "column2"); + Attribute attr5 = new Attribute("table3", "column1"); + Attribute attr6 = new Attribute("XYZ"); - joins.add(new JoinBuilder<>(dbTestDao.createSearchCriteria(), attr1, attr2, JoinBuilder.JoinType.INNER)); - joins.add(new JoinBuilder<>(dbTestDao.createSearchCriteria(), attr3, attr4, JoinBuilder.JoinType.INNER)); + joins.add(new JoinBuilder<>("", dbTestDao.createSearchCriteria(), attr1, attr2, JoinBuilder.JoinType.INNER)); + joins.add(new JoinBuilder<>("", dbTestDao.createSearchCriteria(), + new Attribute[]{attr3, attr5}, new Attribute[]{attr4, attr6}, JoinBuilder.JoinType.INNER, JoinBuilder.JoinCondition.OR)); dbTestDao.addJoins(joinString, joins); - Assert.assertEquals(" INNER JOIN table2 ON table1.column1=table2.column2 INNER JOIN table4 ON table3.column1=table4.column2 ", joinString.toString()); + Assert.assertEquals(" INNER JOIN table2 ON table1.column1=table2.column2 " + + " INNER JOIN table4 ON table3.column1=table4.column2 OR table3.column1=? ", joinString.toString()); } @Test @@ -248,22 +250,17 @@ public class GenericDaoBaseTest { Attribute tBc2 = new Attribute("tableB", "column2"); Attribute tCc3 = new Attribute("tableC", "column3"); Attribute tDc4 = new Attribute("tableD", "column4"); + Attribute tDc5 = new Attribute("tableD", "column5"); + Attribute attr = new Attribute(123); - joins.add(new JoinBuilder<>(dbTestDao.createSearchCriteria(), tBc2, tAc1, JoinBuilder.JoinType.INNER)); - joins.add(new JoinBuilder<>(dbTestDao.createSearchCriteria(), tCc3, tAc2, JoinBuilder.JoinType.INNER)); - joins.add(new JoinBuilder<>(dbTestDao.createSearchCriteria(), tDc4, tAc3, JoinBuilder.JoinType.INNER)); + joins.add(new JoinBuilder<>("tableA1Alias", dbTestDao.createSearchCriteria(), tBc2, tAc1, JoinBuilder.JoinType.INNER)); + joins.add(new JoinBuilder<>("tableA2Alias", dbTestDao.createSearchCriteria(), tCc3, tAc2, JoinBuilder.JoinType.INNER)); + joins.add(new JoinBuilder<>("tableA3Alias", dbTestDao.createSearchCriteria(), + new Attribute[]{tDc4, tDc5}, new Attribute[]{tAc3, attr}, JoinBuilder.JoinType.INNER, JoinBuilder.JoinCondition.AND)); dbTestDao.addJoins(joinString, joins); - Assert.assertEquals(" INNER JOIN tableA ON tableB.column2=tableA.column1 INNER JOIN tableA tableA1 ON tableC.column3=tableA1.column2 INNER JOIN tableA tableA2 ON tableD.column4=tableA2.column3 ", joinString.toString()); - } - - @Test - public void findNextTableNameTest() { - Map usedTables = new HashMap<>(); - - Assert.assertEquals("tableA", GenericDaoBase.findNextJoinTableName("tableA", usedTables)); - Assert.assertEquals("tableA1", GenericDaoBase.findNextJoinTableName("tableA", usedTables)); - Assert.assertEquals("tableA2", GenericDaoBase.findNextJoinTableName("tableA", usedTables)); - Assert.assertEquals("tableA3", GenericDaoBase.findNextJoinTableName("tableA", usedTables)); + Assert.assertEquals(" INNER JOIN tableA tableA1Alias ON tableB.column2=tableA1Alias.column1 " + + " INNER JOIN tableA tableA2Alias ON tableC.column3=tableA2Alias.column2 " + + " INNER JOIN tableA tableA3Alias ON tableD.column4=tableA3Alias.column3 AND tableD.column5=? ", joinString.toString()); } } diff --git a/plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsServiceImpl.java b/plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsServiceImpl.java index 979e4aacbe7..4bbe7627f2c 100644 --- a/plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsServiceImpl.java +++ b/plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsServiceImpl.java @@ -27,9 +27,11 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.stream.Collectors; import javax.inject.Inject; +import com.cloud.dc.ClusterVO; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.ListClustersMetricsCmd; import org.apache.cloudstack.api.ListDbMetricsCmd; @@ -626,6 +628,7 @@ public class MetricsServiceImpl extends MutualExclusiveIdsManagerBase implements @Override public List listStoragePoolMetrics(List poolResponses) { final List metricsResponses = new ArrayList<>(); + Map clusterUuidToIdMap = clusterDao.findByUuids(poolResponses.stream().map(StoragePoolResponse::getClusterId).toArray(String[]::new)).stream().collect(Collectors.toMap(ClusterVO::getUuid, ClusterVO::getId)); for (final StoragePoolResponse poolResponse: poolResponses) { StoragePoolMetricsResponse metricsResponse = new StoragePoolMetricsResponse(); @@ -635,11 +638,7 @@ public class MetricsServiceImpl extends MutualExclusiveIdsManagerBase implements throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to generate storagepool metrics response"); } - Long poolClusterId = null; - final Cluster cluster = clusterDao.findByUuid(poolResponse.getClusterId()); - if (cluster != null) { - poolClusterId = cluster.getId(); - } + Long poolClusterId = clusterUuidToIdMap.get(poolResponse.getClusterId()); final Double storageThreshold = AlertManager.StorageCapacityThreshold.valueIn(poolClusterId); final Double storageDisableThreshold = CapacityManager.StorageCapacityDisableThreshold.valueIn(poolClusterId); diff --git a/server/src/main/java/com/cloud/api/ApiDBUtils.java b/server/src/main/java/com/cloud/api/ApiDBUtils.java index c5f7a8911b9..4f64043c35e 100644 --- a/server/src/main/java/com/cloud/api/ApiDBUtils.java +++ b/server/src/main/java/com/cloud/api/ApiDBUtils.java @@ -16,75 +16,6 @@ // under the License. package com.cloud.api; -import java.util.ArrayList; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.List; -import java.util.ListIterator; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -import javax.annotation.PostConstruct; -import javax.inject.Inject; - -import org.apache.cloudstack.acl.Role; -import org.apache.cloudstack.acl.RoleService; -import org.apache.cloudstack.affinity.AffinityGroup; -import org.apache.cloudstack.affinity.AffinityGroupResponse; -import org.apache.cloudstack.affinity.dao.AffinityGroupDao; -import org.apache.cloudstack.api.ApiCommandResourceType; -import org.apache.cloudstack.api.ApiConstants.DomainDetails; -import org.apache.cloudstack.api.ApiConstants.HostDetails; -import org.apache.cloudstack.api.ApiConstants.VMDetails; -import org.apache.cloudstack.api.ResponseObject.ResponseView; -import org.apache.cloudstack.api.response.AccountResponse; -import org.apache.cloudstack.api.response.AsyncJobResponse; -import org.apache.cloudstack.api.response.BackupOfferingResponse; -import org.apache.cloudstack.api.response.BackupResponse; -import org.apache.cloudstack.api.response.BackupScheduleResponse; -import org.apache.cloudstack.api.response.DiskOfferingResponse; -import org.apache.cloudstack.api.response.DomainResponse; -import org.apache.cloudstack.api.response.DomainRouterResponse; -import org.apache.cloudstack.api.response.EventResponse; -import org.apache.cloudstack.api.response.HostForMigrationResponse; -import org.apache.cloudstack.api.response.HostResponse; -import org.apache.cloudstack.api.response.HostTagResponse; -import org.apache.cloudstack.api.response.ImageStoreResponse; -import org.apache.cloudstack.api.response.InstanceGroupResponse; -import org.apache.cloudstack.api.response.NetworkOfferingResponse; -import org.apache.cloudstack.api.response.ProjectAccountResponse; -import org.apache.cloudstack.api.response.ProjectInvitationResponse; -import org.apache.cloudstack.api.response.ProjectResponse; -import org.apache.cloudstack.api.response.ResourceIconResponse; -import org.apache.cloudstack.api.response.ResourceTagResponse; -import org.apache.cloudstack.api.response.SecurityGroupResponse; -import org.apache.cloudstack.api.response.ServiceOfferingResponse; -import org.apache.cloudstack.api.response.StoragePoolResponse; -import org.apache.cloudstack.api.response.StorageTagResponse; -import org.apache.cloudstack.api.response.TemplateResponse; -import org.apache.cloudstack.api.response.UserResponse; -import org.apache.cloudstack.api.response.UserVmResponse; -import org.apache.cloudstack.api.response.VolumeResponse; -import org.apache.cloudstack.api.response.VpcOfferingResponse; -import org.apache.cloudstack.api.response.ZoneResponse; -import org.apache.cloudstack.backup.Backup; -import org.apache.cloudstack.backup.BackupOffering; -import org.apache.cloudstack.backup.BackupSchedule; -import org.apache.cloudstack.backup.dao.BackupDao; -import org.apache.cloudstack.backup.dao.BackupOfferingDao; -import org.apache.cloudstack.backup.dao.BackupScheduleDao; -import org.apache.cloudstack.context.CallContext; -import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; -import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; -import org.apache.cloudstack.framework.config.dao.ConfigurationDao; -import org.apache.cloudstack.framework.jobs.AsyncJob; -import org.apache.cloudstack.framework.jobs.AsyncJobManager; -import org.apache.cloudstack.framework.jobs.dao.AsyncJobDao; -import org.apache.cloudstack.resourcedetail.dao.DiskOfferingDetailsDao; -import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; - import com.cloud.agent.api.VgpuTypesInfo; import com.cloud.api.query.dao.AccountJoinDao; import com.cloud.api.query.dao.AffinityGroupJoinDao; @@ -344,6 +275,75 @@ import com.cloud.vm.dao.UserVmDetailsDao; import com.cloud.vm.dao.VMInstanceDao; import com.cloud.vm.snapshot.VMSnapshot; import com.cloud.vm.snapshot.dao.VMSnapshotDao; +import org.apache.cloudstack.acl.Role; +import org.apache.cloudstack.acl.RoleService; +import org.apache.cloudstack.affinity.AffinityGroup; +import org.apache.cloudstack.affinity.AffinityGroupResponse; +import org.apache.cloudstack.affinity.dao.AffinityGroupDao; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants.DomainDetails; +import org.apache.cloudstack.api.ApiConstants.HostDetails; +import org.apache.cloudstack.api.ApiConstants.VMDetails; +import org.apache.cloudstack.api.ResponseObject.ResponseView; +import org.apache.cloudstack.api.response.AccountResponse; +import org.apache.cloudstack.api.response.AsyncJobResponse; +import org.apache.cloudstack.api.response.BackupOfferingResponse; +import org.apache.cloudstack.api.response.BackupResponse; +import org.apache.cloudstack.api.response.BackupScheduleResponse; +import org.apache.cloudstack.api.response.DiskOfferingResponse; +import org.apache.cloudstack.api.response.DomainResponse; +import org.apache.cloudstack.api.response.DomainRouterResponse; +import org.apache.cloudstack.api.response.EventResponse; +import org.apache.cloudstack.api.response.HostForMigrationResponse; +import org.apache.cloudstack.api.response.HostResponse; +import org.apache.cloudstack.api.response.HostTagResponse; +import org.apache.cloudstack.api.response.ImageStoreResponse; +import org.apache.cloudstack.api.response.InstanceGroupResponse; +import org.apache.cloudstack.api.response.NetworkOfferingResponse; +import org.apache.cloudstack.api.response.ProjectAccountResponse; +import org.apache.cloudstack.api.response.ProjectInvitationResponse; +import org.apache.cloudstack.api.response.ProjectResponse; +import org.apache.cloudstack.api.response.ResourceIconResponse; +import org.apache.cloudstack.api.response.ResourceTagResponse; +import org.apache.cloudstack.api.response.SecurityGroupResponse; +import org.apache.cloudstack.api.response.ServiceOfferingResponse; +import org.apache.cloudstack.api.response.StoragePoolResponse; +import org.apache.cloudstack.api.response.StorageTagResponse; +import org.apache.cloudstack.api.response.TemplateResponse; +import org.apache.cloudstack.api.response.UserResponse; +import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.api.response.VolumeResponse; +import org.apache.cloudstack.api.response.VpcOfferingResponse; +import org.apache.cloudstack.api.response.ZoneResponse; +import org.apache.cloudstack.backup.Backup; +import org.apache.cloudstack.backup.BackupOffering; +import org.apache.cloudstack.backup.BackupSchedule; +import org.apache.cloudstack.backup.dao.BackupDao; +import org.apache.cloudstack.backup.dao.BackupOfferingDao; +import org.apache.cloudstack.backup.dao.BackupScheduleDao; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; +import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.framework.jobs.AsyncJob; +import org.apache.cloudstack.framework.jobs.AsyncJobManager; +import org.apache.cloudstack.framework.jobs.dao.AsyncJobDao; +import org.apache.cloudstack.resourcedetail.dao.DiskOfferingDetailsDao; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; + +import javax.annotation.PostConstruct; +import javax.inject.Inject; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; public class ApiDBUtils { private static ManagementServer s_ms; @@ -792,7 +792,7 @@ public class ApiDBUtils { s_configDao = configDao; s_consoleProxyDao = consoleProxyDao; s_firewallCidrsDao = firewallCidrsDao; - s_firewallDcidrsDao = firewalDcidrsDao; + s_firewallDcidrsDao = firewalDcidrsDao; s_vmDao = vmDao; s_resourceLimitMgr = resourceLimitMgr; s_projectMgr = projectMgr; @@ -917,7 +917,8 @@ public class ApiDBUtils { return s_resourceLimitMgr.findCorrectResourceLimitForDomain(domain, type, null); } - public static long findCorrectResourceLimitForDomain(Long limit, boolean isRootDomain, ResourceType type, long domainId) { + public static long findCorrectResourceLimitForDomain(Long limit, boolean isRootDomain, ResourceType type, + long domainId) { long max = Resource.RESOURCE_UNLIMITED; // if resource limit is not found, then we treat it as unlimited // No limits for Root domain @@ -997,7 +998,7 @@ public class ApiDBUtils { return s_statsCollector.getManagementServerHostStats(mgmtSrvrUuid); } - public static Map getDbStatistics() { + public static Map getDbStatistics() { return s_statsCollector.getDbStats(); } @@ -1077,6 +1078,7 @@ public class ApiDBUtils { ServiceOfferingVO off = s_serviceOfferingDao.findServiceOfferingByComputeOnlyDiskOffering(diskOfferingId); return off; } + public static DomainVO findDomainById(Long domainId) { return s_domainDao.findByIdIncludingRemoved(domainId); } @@ -1249,7 +1251,7 @@ public class ApiDBUtils { return s_volumeDao.getHypervisorType(volumeId); } - public static HypervisorType getHypervisorTypeFromFormat(long dcId, ImageFormat format){ + public static HypervisorType getHypervisorTypeFromFormat(long dcId, ImageFormat format) { HypervisorType type = s_storageMgr.getHypervisorTypeFromFormat(format); if (format == ImageFormat.VHD) { // Xenserver and Hyperv both support vhd format. Additionally hyperv is only supported @@ -1260,7 +1262,8 @@ public class ApiDBUtils { if (xenClusters.isEmpty()) { type = HypervisorType.Hyperv; } - } if (format == ImageFormat.RAW) { + } + if (format == ImageFormat.RAW) { // Currently, KVM only supports RBD and PowerFlex images of type RAW. // This results in a weird collision with OVM volumes which // can only be raw, thus making KVM RBD volumes show up as OVM @@ -1271,16 +1274,16 @@ public class ApiDBUtils { // This would be better implemented at a cluster level. List pools = s_storagePoolDao.listByDataCenterId(dcId); ListIterator itr = pools.listIterator(); - while(itr.hasNext()) { + while (itr.hasNext()) { StoragePoolVO pool = itr.next(); - if(pool.getPoolType() == StoragePoolType.RBD || - pool.getPoolType() == StoragePoolType.PowerFlex || - pool.getPoolType() == StoragePoolType.CLVM || - pool.getPoolType() == StoragePoolType.Linstor) { - // This case will note the presence of non-qcow2 primary stores, suggesting KVM without NFS. Otherwse, - // If this check is not passed, the hypervisor type will remain OVM. - type = HypervisorType.KVM; - break; + if (pool.getPoolType() == StoragePoolType.RBD || + pool.getPoolType() == StoragePoolType.PowerFlex || + pool.getPoolType() == StoragePoolType.CLVM || + pool.getPoolType() == StoragePoolType.Linstor) { + // This case will note the presence of non-qcow2 primary stores, suggesting KVM without NFS. Otherwse, + // If this check is not passed, the hypervisor type will remain OVM. + type = HypervisorType.KVM; + break; } else if (pool.getHypervisor() == HypervisorType.Custom) { type = HypervisorType.Custom; break; @@ -1428,7 +1431,7 @@ public class ApiDBUtils { public static boolean isExtractionDisabled() { String disableExtractionString = s_configDao.getValue(Config.DisableExtraction.toString()); - boolean disableExtraction = (disableExtractionString == null) ? false : Boolean.parseBoolean(disableExtractionString); + boolean disableExtraction = (disableExtractionString == null) ? false : Boolean.parseBoolean(disableExtractionString); return disableExtraction; } @@ -1444,7 +1447,7 @@ public class ApiDBUtils { return s_firewallCidrsDao.getSourceCidrs(id); } - public static List findFirewallDestCidrs(long id){ + public static List findFirewallDestCidrs(long id) { return s_firewallDcidrsDao.getDestCidrs(id); } @@ -1554,7 +1557,8 @@ public class ApiDBUtils { return conditions; } - public static void getAutoScaleVmGroupPolicyIds(long vmGroupId, List scaleUpPolicyIds, List scaleDownPolicyIds) { + public static void getAutoScaleVmGroupPolicyIds(long vmGroupId, List scaleUpPolicyIds, + List scaleDownPolicyIds) { List vos = s_asVmGroupPolicyMapDao.listByVmGroupId(vmGroupId); for (AutoScaleVmGroupPolicyMapVO vo : vos) { AutoScalePolicy autoScalePolicy = s_asPolicyDao.findById(vo.getPolicyId()); @@ -1575,11 +1579,12 @@ public class ApiDBUtils { return null; } - public static UserVmDetailVO findPublicKeyByVmId(long vmId) { + public static UserVmDetailVO findPublicKeyByVmId(long vmId) { return s_userVmDetailsDao.findDetail(vmId, VmDetailConstants.SSH_PUBLIC_KEY); } - public static void getAutoScaleVmGroupPolicies(long vmGroupId, List scaleUpPolicies, List scaleDownPolicies) { + public static void getAutoScaleVmGroupPolicies(long vmGroupId, List scaleUpPolicies, + List scaleDownPolicies) { List vos = s_asVmGroupPolicyMapDao.listByVmGroupId(vmGroupId); for (AutoScaleVmGroupPolicyMapVO vo : vos) { AutoScalePolicy autoScalePolicy = s_asPolicyDao.findById(vo.getPolicyId()); @@ -1691,7 +1696,7 @@ public class ApiDBUtils { jobInstanceId = template.getUuid(); } } else if (jobInstanceType == ApiCommandResourceType.VirtualMachine || jobInstanceType == ApiCommandResourceType.ConsoleProxy || - jobInstanceType == ApiCommandResourceType.SystemVm || jobInstanceType == ApiCommandResourceType.DomainRouter) { + jobInstanceType == ApiCommandResourceType.SystemVm || jobInstanceType == ApiCommandResourceType.DomainRouter) { VMInstanceVO vm = ApiDBUtils.findVMInstanceById(job.getInstanceId()); if (vm != null) { jobInstanceId = vm.getUuid(); @@ -1788,7 +1793,7 @@ public class ApiDBUtils { } } else if (jobInstanceType == ApiCommandResourceType.Network) { NetworkVO networkVO = ApiDBUtils.findNetworkById(job.getInstanceId()); - if(networkVO != null) { + if (networkVO != null) { jobInstanceId = networkVO.getUuid(); } } else if (jobInstanceType != ApiCommandResourceType.None) { @@ -1815,11 +1820,13 @@ public class ApiDBUtils { return s_domainRouterJoinDao.newDomainRouterView(vr); } - public static UserVmResponse newUserVmResponse(ResponseView view, String objectName, UserVmJoinVO userVm, Set details, Account caller) { + public static UserVmResponse newUserVmResponse(ResponseView view, String objectName, UserVmJoinVO userVm, + Set details, Account caller) { return s_userVmJoinDao.newUserVmResponse(view, objectName, userVm, details, null, null, caller); } - public static UserVmResponse newUserVmResponse(ResponseView view, String objectName, UserVmJoinVO userVm, Set details, Boolean accumulateStats, + public static UserVmResponse newUserVmResponse(ResponseView view, String objectName, UserVmJoinVO userVm, + Set details, Boolean accumulateStats, Boolean showUserData, Account caller) { return s_userVmJoinDao.newUserVmResponse(view, objectName, userVm, details, accumulateStats, showUserData, caller); } @@ -1836,7 +1843,8 @@ public class ApiDBUtils { return s_securityGroupJoinDao.newSecurityGroupResponse(vsg, caller); } - public static SecurityGroupResponse fillSecurityGroupDetails(SecurityGroupResponse vsgData, SecurityGroupJoinVO sg) { + public static SecurityGroupResponse fillSecurityGroupDetails(SecurityGroupResponse vsgData, + SecurityGroupJoinVO sg) { return s_securityGroupJoinDao.setSecurityGroupResponse(vsgData, sg); } @@ -1886,12 +1894,12 @@ public class ApiDBUtils { public static UserResponse newUserResponse(UserAccountJoinVO usr, Long domainId) { UserResponse response = s_userAccountJoinDao.newUserResponse(usr); - if(!AccountManager.UseSecretKeyInResponse.value()){ + if (!AccountManager.UseSecretKeyInResponse.value()) { response.setSecretKey(null); } // Populate user account role information if (usr.getAccountRoleId() != null) { - Role role = s_roleService.findRole( usr.getAccountRoleId()); + Role role = s_roleService.findRole(usr.getAccountRoleId()); if (role != null) { response.setRoleId(role.getUuid()); response.setRoleType(role.getRoleType()); @@ -1987,7 +1995,8 @@ public class ApiDBUtils { return s_poolJoinDao.newStoragePoolForMigrationResponse(vr); } - public static StoragePoolResponse fillStoragePoolForMigrationDetails(StoragePoolResponse vrData, StoragePoolJoinVO vr) { + public static StoragePoolResponse fillStoragePoolForMigrationDetails(StoragePoolResponse vrData, + StoragePoolJoinVO vr) { return s_poolJoinDao.setStoragePoolForMigrationResponse(vrData, vr); } @@ -2011,7 +2020,8 @@ public class ApiDBUtils { return s_domainJoinDao.newDomainResponse(view, details, ve); } - public static AccountResponse newAccountResponse(ResponseView view, EnumSet details, AccountJoinVO ve) { + public static AccountResponse newAccountResponse(ResponseView view, EnumSet details, + AccountJoinVO ve) { AccountResponse response = s_accountJoinDao.newAccountResponse(view, details, ve); // Populate account role information if (ve.getRoleId() != null) { @@ -2025,6 +2035,30 @@ public class ApiDBUtils { return response; } + public static List newAccountResponses(ResponseView view, EnumSet details, + AccountJoinVO... accounts) { + List responseList = new ArrayList<>(); + + List roleIdList = Arrays.stream(accounts).map(AccountJoinVO::getRoleId).collect(Collectors.toList()); + Map roleIdMap = s_roleService.findRoles(roleIdList, false).stream().collect(Collectors.toMap(Role::getId, Function.identity())); + + for (AccountJoinVO account : accounts) { + AccountResponse response = s_accountJoinDao.newAccountResponse(view, details, account); + // Populate account role information + if (account.getRoleId() != null) { + Role role = roleIdMap.get(account.getRoleId()); + if (role != null) { + response.setRoleId(role.getUuid()); + response.setRoleType(role.getRoleType()); + response.setRoleName(role.getName()); + } + } + responseList.add(response); + } + + return responseList; + } + public static AccountJoinVO newAccountView(Account e) { return s_accountJoinDao.newAccountView(e); } @@ -2073,7 +2107,8 @@ public class ApiDBUtils { return s_serviceOfferingJoinDao.newServiceOfferingView(offering); } - public static ZoneResponse newDataCenterResponse(ResponseView view, DataCenterJoinVO dc, Boolean showCapacities, Boolean showResourceImage) { + public static ZoneResponse newDataCenterResponse(ResponseView view, DataCenterJoinVO dc, Boolean showCapacities, + Boolean showResourceImage) { return s_dcJoinDao.newDataCenterResponse(view, dc, showCapacities, showResourceImage); } @@ -2093,7 +2128,8 @@ public class ApiDBUtils { return s_templateJoinDao.newUpdateResponse(vr); } - public static TemplateResponse newTemplateResponse(EnumSet detailsView, ResponseView view, TemplateJoinVO vr) { + public static TemplateResponse newTemplateResponse(EnumSet detailsView, ResponseView view, + TemplateJoinVO vr) { return s_templateJoinDao.newTemplateResponse(detailsView, view, vr); } @@ -2101,7 +2137,8 @@ public class ApiDBUtils { return s_templateJoinDao.newIsoResponse(vr); } - public static TemplateResponse fillTemplateDetails(EnumSet detailsView, ResponseView view, TemplateResponse vrData, TemplateJoinVO vr) { + public static TemplateResponse fillTemplateDetails(EnumSet detailsView, ResponseView view, + TemplateResponse vrData, TemplateJoinVO vr) { return s_templateJoinDao.setTemplateResponse(detailsView, view, vrData, vr); } @@ -2121,7 +2158,8 @@ public class ApiDBUtils { return s_affinityGroupJoinDao.newAffinityGroupResponse(group); } - public static AffinityGroupResponse fillAffinityGroupDetails(AffinityGroupResponse resp, AffinityGroupJoinVO group) { + public static AffinityGroupResponse fillAffinityGroupDetails(AffinityGroupResponse resp, + AffinityGroupJoinVO group) { return s_affinityGroupJoinDao.setAffinityGroupResponse(resp, group); } @@ -2149,7 +2187,8 @@ public class ApiDBUtils { return s_accountService.isAdmin(account.getId()); } - public static List listResourceTagViewByResourceUUID(String resourceUUID, ResourceObjectType resourceType) { + public static List listResourceTagViewByResourceUUID(String resourceUUID, + ResourceObjectType resourceType) { return s_tagJoinDao.listBy(resourceUUID, resourceType); } diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java index 5f1bb9a7248..c1b21104e1c 100644 --- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java @@ -35,14 +35,20 @@ import java.util.stream.Stream; import javax.inject.Inject; +import com.cloud.event.EventVO; +import com.cloud.event.dao.EventDao; import com.cloud.host.Host; +import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; import com.cloud.network.as.AutoScaleVmGroupVmMapVO; import com.cloud.network.as.dao.AutoScaleVmGroupDao; import com.cloud.network.as.dao.AutoScaleVmGroupVmMapDao; import com.cloud.network.dao.NetworkDao; import com.cloud.network.dao.NetworkVO; +import com.cloud.offering.ServiceOffering; +import com.cloud.service.ServiceOfferingDetailsVO; import com.cloud.storage.VolumeVO; +import com.cloud.user.AccountVO; import com.cloud.user.SSHKeyPairVO; import com.cloud.user.dao.SSHKeyPairDao; import com.cloud.vm.InstanceGroupVMMapVO; @@ -143,13 +149,17 @@ import org.apache.cloudstack.engine.subsystem.api.storage.TemplateState; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO; +import org.apache.cloudstack.outofbandmanagement.OutOfBandManagementVO; +import org.apache.cloudstack.outofbandmanagement.dao.OutOfBandManagementDao; import org.apache.cloudstack.query.QueryService; +import org.apache.cloudstack.resourcedetail.DiskOfferingDetailVO; import org.apache.cloudstack.resourcedetail.dao.DiskOfferingDetailsDao; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailVO; import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.EnumUtils; import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; @@ -314,6 +324,9 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q @Inject private UserAccountJoinDao _userAccountJoinDao; + @Inject + private EventDao eventDao; + @Inject private EventJoinDao _eventJoinDao; @@ -487,6 +500,9 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q @Inject private HostDao hostDao; + @Inject + private OutOfBandManagementDao outOfBandManagementDao; + @Inject private InstanceGroupVMMapDao instanceGroupVMMapDao; @@ -718,9 +734,23 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q } private Pair, Integer> searchForEventsInternal(ListEventsCmd cmd) { + Pair, Integer> eventIdPage = searchForEventIdsAndCount(cmd); + + Integer count = eventIdPage.second(); + Long[] idArray = eventIdPage.first().toArray(new Long[0]); + + if (count == 0) { + return new Pair<>(new ArrayList<>(), count); + } + + List events = _eventJoinDao.searchByIds(idArray); + return new Pair<>(events, count); + } + + private Pair, Integer> searchForEventIdsAndCount(ListEventsCmd cmd) { Account caller = CallContext.current().getCallingAccount(); boolean isRootAdmin = accountMgr.isRootAdmin(caller.getId()); - List permittedAccounts = new ArrayList(); + List permittedAccounts = new ArrayList<>(); Long id = cmd.getId(); String type = cmd.getType(); @@ -769,32 +799,40 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q Boolean isRecursive = domainIdRecursiveListProject.second(); ListProjectResourcesCriteria listProjectResourcesCriteria = domainIdRecursiveListProject.third(); - Filter searchFilter = new Filter(EventJoinVO.class, "createDate", false, cmd.getStartIndex(), cmd.getPageSizeVal()); + Filter searchFilter = new Filter(EventVO.class, "createDate", false, cmd.getStartIndex(), cmd.getPageSizeVal()); // additional order by since createdDate does not have milliseconds // and two events, created within one second can be incorrectly ordered (for example VM.CREATE Completed before Scheduled) - searchFilter.addOrderBy(EventJoinVO.class, "id", false); + searchFilter.addOrderBy(EventVO.class, "id", false); - SearchBuilder sb = _eventJoinDao.createSearchBuilder(); - accountMgr.buildACLViewSearchBuilder(sb, domainId, isRecursive, permittedAccounts, listProjectResourcesCriteria); + SearchBuilder eventSearchBuilder = eventDao.createSearchBuilder(); + eventSearchBuilder.select(null, Func.DISTINCT, eventSearchBuilder.entity().getId()); + accountMgr.buildACLSearchBuilder(eventSearchBuilder, domainId, isRecursive, permittedAccounts, listProjectResourcesCriteria); - sb.and("id", sb.entity().getId(), SearchCriteria.Op.EQ); - sb.and("levelL", sb.entity().getLevel(), SearchCriteria.Op.LIKE); - sb.and("levelEQ", sb.entity().getLevel(), SearchCriteria.Op.EQ); - sb.and("type", sb.entity().getType(), SearchCriteria.Op.EQ); - sb.and("createDateB", sb.entity().getCreateDate(), SearchCriteria.Op.BETWEEN); - sb.and("createDateG", sb.entity().getCreateDate(), SearchCriteria.Op.GTEQ); - sb.and("createDateL", sb.entity().getCreateDate(), SearchCriteria.Op.LTEQ); - sb.and("state", sb.entity().getState(), SearchCriteria.Op.NEQ); - sb.or("startId", sb.entity().getStartId(), SearchCriteria.Op.EQ); - sb.and("createDate", sb.entity().getCreateDate(), SearchCriteria.Op.BETWEEN); - sb.and("displayEvent", sb.entity().getDisplay(), SearchCriteria.Op.EQ); - sb.and("archived", sb.entity().getArchived(), SearchCriteria.Op.EQ); - sb.and("resourceId", sb.entity().getResourceId(), SearchCriteria.Op.EQ); - sb.and("resourceType", sb.entity().getResourceType(), SearchCriteria.Op.EQ); + eventSearchBuilder.and("id", eventSearchBuilder.entity().getId(), SearchCriteria.Op.EQ); + eventSearchBuilder.and("levelL", eventSearchBuilder.entity().getLevel(), SearchCriteria.Op.LIKE); + eventSearchBuilder.and("levelEQ", eventSearchBuilder.entity().getLevel(), SearchCriteria.Op.EQ); + eventSearchBuilder.and("type", eventSearchBuilder.entity().getType(), SearchCriteria.Op.EQ); + eventSearchBuilder.and("createDateB", eventSearchBuilder.entity().getCreateDate(), SearchCriteria.Op.BETWEEN); + eventSearchBuilder.and("createDateG", eventSearchBuilder.entity().getCreateDate(), SearchCriteria.Op.GTEQ); + eventSearchBuilder.and("createDateL", eventSearchBuilder.entity().getCreateDate(), SearchCriteria.Op.LTEQ); + eventSearchBuilder.and("state", eventSearchBuilder.entity().getState(), SearchCriteria.Op.NEQ); + eventSearchBuilder.or("startId", eventSearchBuilder.entity().getStartId(), SearchCriteria.Op.EQ); + eventSearchBuilder.and("createDate", eventSearchBuilder.entity().getCreateDate(), SearchCriteria.Op.BETWEEN); + eventSearchBuilder.and("displayEvent", eventSearchBuilder.entity().isDisplay(), SearchCriteria.Op.EQ); + eventSearchBuilder.and("archived", eventSearchBuilder.entity().getArchived(), SearchCriteria.Op.EQ); + eventSearchBuilder.and("resourceId", eventSearchBuilder.entity().getResourceId(), SearchCriteria.Op.EQ); + eventSearchBuilder.and("resourceType", eventSearchBuilder.entity().getResourceType(), SearchCriteria.Op.EQ); - SearchCriteria sc = sb.create(); + if (keyword != null) { + eventSearchBuilder.and().op("keywordType", eventSearchBuilder.entity().getType(), SearchCriteria.Op.LIKE); + eventSearchBuilder.or("keywordDescription", eventSearchBuilder.entity().getDescription(), SearchCriteria.Op.LIKE); + eventSearchBuilder.or("keywordLevel", eventSearchBuilder.entity().getLevel(), SearchCriteria.Op.LIKE); + eventSearchBuilder.cp(); + } + + SearchCriteria sc = eventSearchBuilder.create(); // building ACL condition - accountMgr.buildACLViewSearchCriteria(sc, domainId, isRecursive, permittedAccounts, listProjectResourcesCriteria); + accountMgr.buildACLSearchCriteria(sc, domainId, isRecursive, permittedAccounts, listProjectResourcesCriteria); // For end users display only enabled events if (!accountMgr.isRootAdmin(caller.getId())) { @@ -813,11 +851,9 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q } if (keyword != null) { - SearchCriteria ssc = _eventJoinDao.createSearchCriteria(); - ssc.addOr("type", SearchCriteria.Op.LIKE, "%" + keyword + "%"); - ssc.addOr("description", SearchCriteria.Op.LIKE, "%" + keyword + "%"); - ssc.addOr("level", SearchCriteria.Op.LIKE, "%" + keyword + "%"); - sc.addAnd("level", SearchCriteria.Op.SC, ssc); + sc.setParameters("keywordType", "%" + keyword + "%"); + sc.setParameters("keywordDescription", "%" + keyword + "%"); + sc.setParameters("keywordLevel", "%" + keyword + "%"); } if (level != null) { @@ -846,7 +882,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q sc.setParameters("archived", false); - Pair, Integer> eventPair = null; + Pair, Integer> eventPair = null; // event_view will not have duplicate rows for each event, so // searchAndCount should be good enough. if ((entryTime != null) && (duration != null)) { @@ -870,11 +906,14 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q * _eventDao.findCompletedEvent(event.getId()); if (completedEvent * == null) { pendingEvents.add(event); } } return pendingEvents; */ + eventPair = new Pair<>(new ArrayList<>(), 0); } else { - eventPair = _eventJoinDao.searchAndCount(sc, searchFilter); + Pair, Integer> uniqueEventPair = eventDao.searchAndCount(sc, searchFilter); + Integer count = uniqueEventPair.second(); + List eventIds = uniqueEventPair.first().stream().map(EventVO::getId).collect(Collectors.toList()); + eventPair = new Pair<>(eventIds, count); } return eventPair; - } @Override @@ -1053,7 +1092,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q } // search vm details by ids - List vms = _userVmJoinDao.searchByIds( idArray); + List vms = _userVmJoinDao.searchByIds(idArray); return new Pair<>(vms, count); } @@ -2071,7 +2110,19 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q } public Pair, Integer> searchForServersInternal(ListHostsCmd cmd) { + Pair, Integer> serverIdPage = searchForServerIdsAndCount(cmd); + Integer count = serverIdPage.second(); + Long[] idArray = serverIdPage.first().toArray(new Long[0]); + + if (count == 0) { + return new Pair<>(new ArrayList<>(), count); + } + + List servers = hostJoinDao.searchByIds(idArray); + return new Pair<>(servers, count); + } + public Pair, Integer> searchForServerIdsAndCount(ListHostsCmd cmd) { Long zoneId = accountMgr.checkAccessAndSpecifyAuthority(CallContext.current().getCallingAccount(), cmd.getZoneId()); Object name = cmd.getHostName(); Object type = cmd.getType(); @@ -2088,44 +2139,55 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q Long pageSize = cmd.getPageSizeVal(); Hypervisor.HypervisorType hypervisorType = cmd.getHypervisor(); - Filter searchFilter = new Filter(HostJoinVO.class, "id", Boolean.TRUE, startIndex, pageSize); + Filter searchFilter = new Filter(HostVO.class, "id", Boolean.TRUE, startIndex, pageSize); - SearchBuilder sb = hostJoinDao.createSearchBuilder(); - sb.select(null, Func.DISTINCT, sb.entity().getId()); // select distinct + SearchBuilder hostSearchBuilder = hostDao.createSearchBuilder(); + hostSearchBuilder.select(null, Func.DISTINCT, hostSearchBuilder.entity().getId()); // select distinct // ids - sb.and("id", sb.entity().getId(), SearchCriteria.Op.EQ); - sb.and("name", sb.entity().getName(), SearchCriteria.Op.EQ); - sb.and("type", sb.entity().getType(), SearchCriteria.Op.LIKE); - sb.and("status", sb.entity().getStatus(), SearchCriteria.Op.EQ); - sb.and("dataCenterId", sb.entity().getZoneId(), SearchCriteria.Op.EQ); - sb.and("podId", sb.entity().getPodId(), SearchCriteria.Op.EQ); - sb.and("clusterId", sb.entity().getClusterId(), SearchCriteria.Op.EQ); - sb.and("oobmEnabled", sb.entity().isOutOfBandManagementEnabled(), SearchCriteria.Op.EQ); - sb.and("powerState", sb.entity().getOutOfBandManagementPowerState(), SearchCriteria.Op.EQ); - sb.and("resourceState", sb.entity().getResourceState(), SearchCriteria.Op.EQ); - sb.and("hypervisor_type", sb.entity().getHypervisorType(), SearchCriteria.Op.EQ); + hostSearchBuilder.and("id", hostSearchBuilder.entity().getId(), SearchCriteria.Op.EQ); + hostSearchBuilder.and("name", hostSearchBuilder.entity().getName(), SearchCriteria.Op.EQ); + hostSearchBuilder.and("type", hostSearchBuilder.entity().getType(), SearchCriteria.Op.LIKE); + hostSearchBuilder.and("status", hostSearchBuilder.entity().getStatus(), SearchCriteria.Op.EQ); + hostSearchBuilder.and("dataCenterId", hostSearchBuilder.entity().getDataCenterId(), SearchCriteria.Op.EQ); + hostSearchBuilder.and("podId", hostSearchBuilder.entity().getPodId(), SearchCriteria.Op.EQ); + hostSearchBuilder.and("clusterId", hostSearchBuilder.entity().getClusterId(), SearchCriteria.Op.EQ); + hostSearchBuilder.and("resourceState", hostSearchBuilder.entity().getResourceState(), SearchCriteria.Op.EQ); + hostSearchBuilder.and("hypervisor_type", hostSearchBuilder.entity().getHypervisorType(), SearchCriteria.Op.EQ); + + if (keyword != null) { + hostSearchBuilder.and().op("keywordName", hostSearchBuilder.entity().getName(), SearchCriteria.Op.LIKE); + hostSearchBuilder.or("keywordStatus", hostSearchBuilder.entity().getStatus(), SearchCriteria.Op.LIKE); + hostSearchBuilder.or("keywordType", hostSearchBuilder.entity().getType(), SearchCriteria.Op.LIKE); + hostSearchBuilder.cp(); + } + + if (outOfBandManagementEnabled != null || powerState != null) { + SearchBuilder oobmSearch = outOfBandManagementDao.createSearchBuilder(); + oobmSearch.and("oobmEnabled", oobmSearch.entity().isEnabled(), SearchCriteria.Op.EQ); + oobmSearch.and("powerState", oobmSearch.entity().getPowerState(), SearchCriteria.Op.EQ); + + hostSearchBuilder.join("oobmSearch", oobmSearch, hostSearchBuilder.entity().getId(), oobmSearch.entity().getHostId(), JoinBuilder.JoinType.INNER); + } String haTag = _haMgr.getHaTag(); if (haHosts != null && haTag != null && !haTag.isEmpty()) { + SearchBuilder hostTagSearchBuilder = _hostTagDao.createSearchBuilder(); if ((Boolean)haHosts) { - sb.and("tag", sb.entity().getTag(), SearchCriteria.Op.EQ); + hostTagSearchBuilder.and("tag", hostTagSearchBuilder.entity().getName(), SearchCriteria.Op.EQ); } else { - sb.and().op("tag", sb.entity().getTag(), SearchCriteria.Op.NEQ); - sb.or("tagNull", sb.entity().getTag(), SearchCriteria.Op.NULL); - sb.cp(); + hostTagSearchBuilder.and().op("tag", hostTagSearchBuilder.entity().getName(), Op.NEQ); + hostTagSearchBuilder.or("tagNull", hostTagSearchBuilder.entity().getName(), Op.NULL); + hostTagSearchBuilder.cp(); } - + hostSearchBuilder.join("hostTagSearch", hostTagSearchBuilder, hostSearchBuilder.entity().getId(), hostTagSearchBuilder.entity().getHostId(), JoinBuilder.JoinType.LEFT); } - SearchCriteria sc = sb.create(); + SearchCriteria sc = hostSearchBuilder.create(); if (keyword != null) { - SearchCriteria ssc = hostJoinDao.createSearchCriteria(); - ssc.addOr("name", SearchCriteria.Op.LIKE, "%" + keyword + "%"); - ssc.addOr("status", SearchCriteria.Op.LIKE, "%" + keyword + "%"); - ssc.addOr("type", SearchCriteria.Op.LIKE, "%" + keyword + "%"); - - sc.addAnd("name", SearchCriteria.Op.SC, ssc); + sc.setParameters("keywordName", "%" + keyword + "%"); + sc.setParameters("keywordStatus", "%" + keyword + "%"); + sc.setParameters("keywordType", "%" + keyword + "%"); } if (id != null) { @@ -2152,11 +2214,11 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q } if (outOfBandManagementEnabled != null) { - sc.setParameters("oobmEnabled", outOfBandManagementEnabled); + sc.setJoinParameters("oobmSearch", "oobmEnabled", outOfBandManagementEnabled); } if (powerState != null) { - sc.setParameters("powerState", powerState); + sc.setJoinParameters("oobmSearch", "powerState", powerState); } if (resourceState != null) { @@ -2164,28 +2226,17 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q } if (haHosts != null && haTag != null && !haTag.isEmpty()) { - sc.setParameters("tag", haTag); + sc.setJoinParameters("hostTagSearch", "tag", haTag); } if (hypervisorType != HypervisorType.None && hypervisorType != HypervisorType.Any) { sc.setParameters("hypervisor_type", hypervisorType); } - // search host details by ids - Pair, Integer> uniqueHostPair = hostJoinDao.searchAndCount(sc, searchFilter); - Integer count = uniqueHostPair.second(); - if (count.intValue() == 0) { - // handle empty result cases - return uniqueHostPair; - } - List uniqueHosts = uniqueHostPair.first(); - Long[] hostIds = new Long[uniqueHosts.size()]; - int i = 0; - for (HostJoinVO v : uniqueHosts) { - hostIds[i++] = v.getId(); - } - List hosts = hostJoinDao.searchByIds(hostIds); - return new Pair, Integer>(hosts, count); + Pair, Integer> uniqueHostPair = hostDao.searchAndCount(sc, searchFilter); + Integer count = uniqueHostPair.second(); + List hostIds = uniqueHostPair.first().stream().map(HostVO::getId).collect(Collectors.toList()); + return new Pair<>(hostIds, count); } @Override @@ -2231,6 +2282,19 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q } private Pair, Integer> searchForVolumesInternal(ListVolumesCmd cmd) { + Pair, Integer> volumeIdPage = searchForVolumeIdsAndCount(cmd); + + Integer count = volumeIdPage.second(); + Long[] idArray = volumeIdPage.first().toArray(new Long[0]); + + if (count == 0) { + return new Pair<>(new ArrayList<>(), count); + } + + List vms = _volumeJoinDao.searchByIds(idArray); + return new Pair<>(vms, count); + } + private Pair, Integer> searchForVolumeIdsAndCount(ListVolumesCmd cmd) { Account caller = CallContext.current().getCallingAccount(); List permittedAccounts = new ArrayList(); @@ -2258,61 +2322,97 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q Long domainId = domainIdRecursiveListProject.first(); Boolean isRecursive = domainIdRecursiveListProject.second(); ListProjectResourcesCriteria listProjectResourcesCriteria = domainIdRecursiveListProject.third(); - Filter searchFilter = new Filter(VolumeJoinVO.class, "created", false, cmd.getStartIndex(), cmd.getPageSizeVal()); + Filter searchFilter = new Filter(VolumeVO.class, "created", false, cmd.getStartIndex(), cmd.getPageSizeVal()); - // hack for now, this should be done better but due to needing a join I - // opted to - // do this quickly and worry about making it pretty later - SearchBuilder sb = _volumeJoinDao.createSearchBuilder(); - sb.select(null, Func.DISTINCT, sb.entity().getId()); // select distinct - // ids to get - // number of - // records with - // pagination - accountMgr.buildACLViewSearchBuilder(sb, domainId, isRecursive, permittedAccounts, listProjectResourcesCriteria); + SearchBuilder volumeSearchBuilder = volumeDao.createSearchBuilder(); + volumeSearchBuilder.select(null, Func.DISTINCT, volumeSearchBuilder.entity().getId()); // select distinct + accountMgr.buildACLSearchBuilder(volumeSearchBuilder, domainId, isRecursive, permittedAccounts, listProjectResourcesCriteria); - sb.and("name", sb.entity().getName(), SearchCriteria.Op.EQ); - sb.and("id", sb.entity().getId(), SearchCriteria.Op.EQ); - sb.and("idIN", sb.entity().getId(), SearchCriteria.Op.IN); - sb.and("volumeType", sb.entity().getVolumeType(), SearchCriteria.Op.LIKE); - sb.and("uuid", sb.entity().getUuid(), SearchCriteria.Op.NNULL); - sb.and("instanceId", sb.entity().getVmId(), SearchCriteria.Op.EQ); - sb.and("dataCenterId", sb.entity().getDataCenterId(), SearchCriteria.Op.EQ); - sb.and("podId", sb.entity().getPodId(), SearchCriteria.Op.EQ); + if (CollectionUtils.isNotEmpty(ids)) { + volumeSearchBuilder.and("idIN", volumeSearchBuilder.entity().getId(), SearchCriteria.Op.IN); + } + + volumeSearchBuilder.and("name", volumeSearchBuilder.entity().getName(), SearchCriteria.Op.EQ); + volumeSearchBuilder.and("volumeType", volumeSearchBuilder.entity().getVolumeType(), SearchCriteria.Op.LIKE); + volumeSearchBuilder.and("uuid", volumeSearchBuilder.entity().getUuid(), SearchCriteria.Op.NNULL); + volumeSearchBuilder.and("instanceId", volumeSearchBuilder.entity().getInstanceId(), SearchCriteria.Op.EQ); + volumeSearchBuilder.and("dataCenterId", volumeSearchBuilder.entity().getDataCenterId(), SearchCriteria.Op.EQ); + + if (keyword != null) { + volumeSearchBuilder.and().op("keywordName", volumeSearchBuilder.entity().getName(), SearchCriteria.Op.LIKE); + volumeSearchBuilder.or("keywordVolumeType", volumeSearchBuilder.entity().getVolumeType(), SearchCriteria.Op.LIKE); + volumeSearchBuilder.or("keywordState", volumeSearchBuilder.entity().getState(), SearchCriteria.Op.LIKE); + volumeSearchBuilder.cp(); + } + + StoragePoolVO poolVO = null; if (storageId != null) { - StoragePoolVO poolVO = storagePoolDao.findByUuid(storageId); - if (poolVO.getPoolType() == Storage.StoragePoolType.DatastoreCluster) { - sb.and("storageId", sb.entity().getPoolUuid(), SearchCriteria.Op.IN); + poolVO = storagePoolDao.findByUuid(storageId); + if (poolVO == null) { + throw new InvalidParameterValueException("Unable to find storage pool by uuid " + storageId); + } else if (poolVO.getPoolType() == Storage.StoragePoolType.DatastoreCluster) { + volumeSearchBuilder.and("storageId", volumeSearchBuilder.entity().getPoolId(), SearchCriteria.Op.IN); } else { - sb.and("storageId", sb.entity().getPoolUuid(), SearchCriteria.Op.EQ); + volumeSearchBuilder.and("storageId", volumeSearchBuilder.entity().getPoolId(), SearchCriteria.Op.EQ); } } - sb.and("diskOfferingId", sb.entity().getDiskOfferingId(), SearchCriteria.Op.EQ); - sb.and("display", sb.entity().isDisplayVolume(), SearchCriteria.Op.EQ); - sb.and("state", sb.entity().getState(), SearchCriteria.Op.EQ); - sb.and("stateNEQ", sb.entity().getState(), SearchCriteria.Op.NEQ); + if (clusterId != null || podId != null) { + SearchBuilder storagePoolSearch = storagePoolDao.createSearchBuilder(); + storagePoolSearch.and("clusterId", storagePoolSearch.entity().getClusterId(), SearchCriteria.Op.EQ); + storagePoolSearch.and("podId", storagePoolSearch.entity().getPodId(), SearchCriteria.Op.EQ); + volumeSearchBuilder.join("storagePoolSearch", storagePoolSearch, storagePoolSearch.entity().getId(), volumeSearchBuilder.entity().getPoolId(), JoinBuilder.JoinType.INNER); + } + + volumeSearchBuilder.and("diskOfferingId", volumeSearchBuilder.entity().getDiskOfferingId(), SearchCriteria.Op.EQ); + volumeSearchBuilder.and("display", volumeSearchBuilder.entity().isDisplayVolume(), SearchCriteria.Op.EQ); + volumeSearchBuilder.and("state", volumeSearchBuilder.entity().getState(), SearchCriteria.Op.EQ); + volumeSearchBuilder.and("stateNEQ", volumeSearchBuilder.entity().getState(), SearchCriteria.Op.NEQ); + + // Need to test thoroughly if (!shouldListSystemVms) { - sb.and().op("systemUse", sb.entity().isSystemUse(), SearchCriteria.Op.NEQ); - sb.or("nulltype", sb.entity().isSystemUse(), SearchCriteria.Op.NULL); - sb.cp(); + SearchBuilder vmSearch = _vmInstanceDao.createSearchBuilder(); + SearchBuilder serviceOfferingSearch = _srvOfferingDao.createSearchBuilder(); + vmSearch.and().op("svmType", vmSearch.entity().getType(), SearchCriteria.Op.NIN); + vmSearch.or("vmSearchNulltype", vmSearch.entity().getType(), SearchCriteria.Op.NULL); + vmSearch.cp(); - sb.and().op("type", sb.entity().getVmType(), SearchCriteria.Op.NIN); - sb.or("nulltype", sb.entity().getVmType(), SearchCriteria.Op.NULL); - sb.cp(); + serviceOfferingSearch.and().op("systemUse", serviceOfferingSearch.entity().isSystemUse(), SearchCriteria.Op.NEQ); + serviceOfferingSearch.or("serviceOfferingSearchNulltype", serviceOfferingSearch.entity().isSystemUse(), SearchCriteria.Op.NULL); + serviceOfferingSearch.cp(); + + vmSearch.join("serviceOfferingSearch", serviceOfferingSearch, serviceOfferingSearch.entity().getId(), vmSearch.entity().getServiceOfferingId(), JoinBuilder.JoinType.LEFT); + + volumeSearchBuilder.join("vmSearch", vmSearch, vmSearch.entity().getId(), volumeSearchBuilder.entity().getInstanceId(), JoinBuilder.JoinType.LEFT); + + } + + if (MapUtils.isNotEmpty(tags)) { + SearchBuilder resourceTagSearch = resourceTagDao.createSearchBuilder(); + resourceTagSearch.and("resourceType", resourceTagSearch.entity().getResourceType(), Op.EQ); + resourceTagSearch.and().op(); + for (int count = 0; count < tags.size(); count++) { + if (count == 0) { + resourceTagSearch.op("tagKey" + count, resourceTagSearch.entity().getKey(), Op.EQ); + } else { + resourceTagSearch.or().op("tagKey" + count, resourceTagSearch.entity().getKey(), Op.EQ); + } + resourceTagSearch.and("tagValue" + count, resourceTagSearch.entity().getValue(), Op.EQ); + resourceTagSearch.cp(); + } + resourceTagSearch.cp(); + + volumeSearchBuilder.join("tags", resourceTagSearch, resourceTagSearch.entity().getResourceId(), volumeSearchBuilder.entity().getId(), JoinBuilder.JoinType.INNER); } // now set the SC criteria... - SearchCriteria sc = sb.create(); - accountMgr.buildACLViewSearchCriteria(sc, domainId, isRecursive, permittedAccounts, listProjectResourcesCriteria); + SearchCriteria sc = volumeSearchBuilder.create(); + accountMgr.buildACLSearchCriteria(sc, domainId, isRecursive, permittedAccounts, listProjectResourcesCriteria); if (keyword != null) { - SearchCriteria ssc = _volumeJoinDao.createSearchCriteria(); - ssc.addOr("name", SearchCriteria.Op.LIKE, "%" + keyword + "%"); - ssc.addOr("volumeType", SearchCriteria.Op.LIKE, "%" + keyword + "%"); - ssc.addOr("state", SearchCriteria.Op.LIKE, "%" + keyword + "%"); - - sc.addAnd("name", SearchCriteria.Op.SC, ssc); + sc.setParameters("keywordName", "%" + keyword + "%"); + sc.setParameters("keywordVolumeType", "%" + keyword + "%"); + sc.setParameters("keywordState", "%" + keyword + "%"); } if (name != null) { @@ -2326,19 +2426,18 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q setIdsListToSearchCriteria(sc, ids); if (!shouldListSystemVms) { - sc.setParameters("systemUse", 1); - sc.setParameters("type", VirtualMachine.Type.ConsoleProxy, VirtualMachine.Type.SecondaryStorageVm, VirtualMachine.Type.DomainRouter); + sc.setJoinParameters("vmSearch", "svmType", VirtualMachine.Type.ConsoleProxy, VirtualMachine.Type.SecondaryStorageVm, VirtualMachine.Type.DomainRouter); + sc.getJoin("vmSearch").setJoinParameters("serviceOfferingSearch", "systemUse", 1); } - if (tags != null && !tags.isEmpty()) { - SearchCriteria tagSc = _volumeJoinDao.createSearchCriteria(); - for (String key : tags.keySet()) { - SearchCriteria tsc = _volumeJoinDao.createSearchCriteria(); - tsc.addAnd("tagKey", SearchCriteria.Op.EQ, key); - tsc.addAnd("tagValue", SearchCriteria.Op.EQ, tags.get(key)); - tagSc.addOr("tagKey", SearchCriteria.Op.SC, tsc); + if (MapUtils.isNotEmpty(tags)) { + int count = 0; + sc.setJoinParameters("tags", "resourceType", ResourceObjectType.Volume); + for (Map.Entry entry : tags.entrySet()) { + sc.setJoinParameters("tags", "tagKey" + count, entry.getKey()); + sc.setJoinParameters("tags", "tagValue" + count, entry.getValue()); + count++; } - sc.addAnd("tagKey", SearchCriteria.Op.SC, tagSc); } if (diskOffId != null) { @@ -2358,23 +2457,21 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q if (zoneId != null) { sc.setParameters("dataCenterId", zoneId); } - if (podId != null) { - sc.setParameters("podId", podId); - } if (storageId != null) { - StoragePoolVO poolVO = storagePoolDao.findByUuid(storageId); if (poolVO.getPoolType() == Storage.StoragePoolType.DatastoreCluster) { - List childDatastores = storagePoolDao.listChildStoragePoolsInDatastoreCluster(poolVO.getId()); - List childDatastoreIds = childDatastores.stream().map(mo -> mo.getUuid()).collect(Collectors.toList()); - sc.setParameters("storageId", childDatastoreIds.toArray()); + List childDataStores = storagePoolDao.listChildStoragePoolsInDatastoreCluster(poolVO.getId()); + sc.setParameters("storageId", childDataStores.stream().map(StoragePoolVO::getId).toArray()); } else { - sc.setParameters("storageId", storageId); + sc.setParameters("storageId", poolVO.getId()); } } if (clusterId != null) { - sc.setParameters("clusterId", clusterId); + sc.setJoinParameters("storagePoolSearch", "clusterId", clusterId); + } + if (podId != null) { + sc.setJoinParameters("storagePoolSearch", "podId", podId); } if (state != null) { @@ -2384,20 +2481,10 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q } // search Volume details by ids - Pair, Integer> uniqueVolPair = _volumeJoinDao.searchAndCount(sc, searchFilter); + Pair, Integer> uniqueVolPair = volumeDao.searchAndCount(sc, searchFilter); Integer count = uniqueVolPair.second(); - if (count.intValue() == 0) { - // empty result - return uniqueVolPair; - } - List uniqueVols = uniqueVolPair.first(); - Long[] vrIds = new Long[uniqueVols.size()]; - int i = 0; - for (VolumeJoinVO v : uniqueVols) { - vrIds[i++] = v.getId(); - } - List vrs = _volumeJoinDao.searchByIds(vrIds); - return new Pair, Integer>(vrs, count); + List vmIds = uniqueVolPair.first().stream().map(VolumeVO::getId).collect(Collectors.toList()); + return new Pair<>(vmIds, count); } private boolean shouldListSystemVms(ListVolumesCmd cmd, Long callerId) { @@ -2420,6 +2507,20 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q } private Pair, Integer> searchForDomainsInternal(ListDomainsCmd cmd) { + Pair, Integer> domainIdPage = searchForDomainIdsAndCount(cmd); + + Integer count = domainIdPage.second(); + Long[] idArray = domainIdPage.first().toArray(new Long[0]); + + if (count == 0) { + return new Pair<>(new ArrayList<>(), count); + } + + List domains = _domainJoinDao.searchByIds(idArray); + return new Pair<>(domains, count); + } + + private Pair, Integer> searchForDomainIdsAndCount(ListDomainsCmd cmd) { Account caller = CallContext.current().getCallingAccount(); Long domainId = cmd.getId(); boolean listAll = cmd.listAll(); @@ -2441,24 +2542,27 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q } } - Filter searchFilter = new Filter(DomainJoinVO.class, "id", true, cmd.getStartIndex(), cmd.getPageSizeVal()); + Filter searchFilter = new Filter(DomainVO.class, "id", true, cmd.getStartIndex(), cmd.getPageSizeVal()); String domainName = cmd.getDomainName(); Integer level = cmd.getLevel(); Object keyword = cmd.getKeyword(); - SearchBuilder sb = _domainJoinDao.createSearchBuilder(); - sb.and("id", sb.entity().getId(), SearchCriteria.Op.EQ); - sb.and("name", sb.entity().getName(), SearchCriteria.Op.EQ); - sb.and("level", sb.entity().getLevel(), SearchCriteria.Op.EQ); - sb.and("path", sb.entity().getPath(), SearchCriteria.Op.LIKE); - sb.and("state", sb.entity().getState(), SearchCriteria.Op.EQ); - - SearchCriteria sc = sb.create(); + SearchBuilder domainSearchBuilder = _domainDao.createSearchBuilder(); + domainSearchBuilder.select(null, Func.DISTINCT, domainSearchBuilder.entity().getId()); // select distinct + domainSearchBuilder.and("id", domainSearchBuilder.entity().getId(), SearchCriteria.Op.EQ); + domainSearchBuilder.and("name", domainSearchBuilder.entity().getName(), SearchCriteria.Op.EQ); + domainSearchBuilder.and("level", domainSearchBuilder.entity().getLevel(), SearchCriteria.Op.EQ); + domainSearchBuilder.and("path", domainSearchBuilder.entity().getPath(), SearchCriteria.Op.LIKE); + domainSearchBuilder.and("state", domainSearchBuilder.entity().getState(), SearchCriteria.Op.EQ); if (keyword != null) { - SearchCriteria ssc = _domainJoinDao.createSearchCriteria(); - ssc.addOr("name", SearchCriteria.Op.LIKE, "%" + keyword + "%"); - sc.addAnd("name", SearchCriteria.Op.SC, ssc); + domainSearchBuilder.and("keywordName", domainSearchBuilder.entity().getName(), SearchCriteria.Op.LIKE); + } + + SearchCriteria sc = domainSearchBuilder.create(); + + if (keyword != null) { + sc.setParameters("keywordName", "%" + keyword + "%"); } if (domainName != null) { @@ -2483,7 +2587,10 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q // return only Active domains to the API sc.setParameters("state", Domain.State.Active); - return _domainJoinDao.searchAndCount(sc, searchFilter); + Pair, Integer> uniqueDomainPair = _domainDao.searchAndCount(sc, searchFilter); + Integer count = uniqueDomainPair.second(); + List domainIds = uniqueDomainPair.first().stream().map(DomainVO::getId).collect(Collectors.toList()); + return new Pair<>(domainIds, count); } @Override @@ -2502,6 +2609,21 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q } private Pair, Integer> searchForAccountsInternal(ListAccountsCmd cmd) { + + Pair, Integer> accountIdPage = searchForAccountIdsAndCount(cmd); + + Integer count = accountIdPage.second(); + Long[] idArray = accountIdPage.first().toArray(new Long[0]); + + if (count == 0) { + return new Pair<>(new ArrayList<>(), count); + } + + List accounts = _accountJoinDao.searchByIds(idArray); + return new Pair<>(accounts, count); + } + + private Pair, Integer> searchForAccountIdsAndCount(ListAccountsCmd cmd) { Account caller = CallContext.current().getCallingAccount(); Long domainId = cmd.getDomainId(); Long accountId = cmd.getId(); @@ -2557,29 +2679,38 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q accountMgr.checkAccess(caller, null, true, account); } - Filter searchFilter = new Filter(AccountJoinVO.class, "id", true, cmd.getStartIndex(), cmd.getPageSizeVal()); + Filter searchFilter = new Filter(AccountVO.class, "id", true, cmd.getStartIndex(), cmd.getPageSizeVal()); Object type = cmd.getAccountType(); Object state = cmd.getState(); Object isCleanupRequired = cmd.isCleanupRequired(); Object keyword = cmd.getKeyword(); - SearchBuilder sb = _accountJoinDao.createSearchBuilder(); - sb.and("accountName", sb.entity().getAccountName(), SearchCriteria.Op.EQ); - sb.and("domainId", sb.entity().getDomainId(), SearchCriteria.Op.EQ); - sb.and("id", sb.entity().getId(), SearchCriteria.Op.EQ); - sb.and("type", sb.entity().getType(), SearchCriteria.Op.EQ); - sb.and("state", sb.entity().getState(), SearchCriteria.Op.EQ); - sb.and("needsCleanup", sb.entity().isNeedsCleanup(), SearchCriteria.Op.EQ); - sb.and("typeNEQ", sb.entity().getType(), SearchCriteria.Op.NEQ); - sb.and("idNEQ", sb.entity().getId(), SearchCriteria.Op.NEQ); - sb.and("type2NEQ", sb.entity().getType(), SearchCriteria.Op.NEQ); + SearchBuilder accountSearchBuilder = _accountDao.createSearchBuilder(); + accountSearchBuilder.select(null, Func.DISTINCT, accountSearchBuilder.entity().getId()); // select distinct + accountSearchBuilder.and("accountName", accountSearchBuilder.entity().getAccountName(), SearchCriteria.Op.EQ); + accountSearchBuilder.and("domainId", accountSearchBuilder.entity().getDomainId(), SearchCriteria.Op.EQ); + accountSearchBuilder.and("id", accountSearchBuilder.entity().getId(), SearchCriteria.Op.EQ); + accountSearchBuilder.and("type", accountSearchBuilder.entity().getType(), SearchCriteria.Op.EQ); + accountSearchBuilder.and("state", accountSearchBuilder.entity().getState(), SearchCriteria.Op.EQ); + accountSearchBuilder.and("needsCleanup", accountSearchBuilder.entity().getNeedsCleanup(), SearchCriteria.Op.EQ); + accountSearchBuilder.and("typeNEQ", accountSearchBuilder.entity().getType(), SearchCriteria.Op.NEQ); + accountSearchBuilder.and("idNEQ", accountSearchBuilder.entity().getId(), SearchCriteria.Op.NEQ); + accountSearchBuilder.and("type2NEQ", accountSearchBuilder.entity().getType(), SearchCriteria.Op.NEQ); if (domainId != null && isRecursive) { - sb.and("path", sb.entity().getDomainPath(), SearchCriteria.Op.LIKE); + SearchBuilder domainSearch = _domainDao.createSearchBuilder(); + domainSearch.and("path", domainSearch.entity().getPath(), SearchCriteria.Op.LIKE); + accountSearchBuilder.join("domainSearch", domainSearch, domainSearch.entity().getId(), accountSearchBuilder.entity().getDomainId(), JoinBuilder.JoinType.INNER); } - SearchCriteria sc = sb.create(); + if (keyword != null) { + accountSearchBuilder.and().op("keywordAccountName", accountSearchBuilder.entity().getAccountName(), SearchCriteria.Op.LIKE); + accountSearchBuilder.or("keywordState", accountSearchBuilder.entity().getState(), SearchCriteria.Op.LIKE); + accountSearchBuilder.cp(); + } + + SearchCriteria sc = accountSearchBuilder.create(); // don't return account of type project to the end user sc.setParameters("typeNEQ", Account.Type.PROJECT); @@ -2593,10 +2724,8 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q } if (keyword != null) { - SearchCriteria ssc = _accountJoinDao.createSearchCriteria(); - ssc.addOr("accountName", SearchCriteria.Op.LIKE, "%" + keyword + "%"); - ssc.addOr("state", SearchCriteria.Op.LIKE, "%" + keyword + "%"); - sc.addAnd("accountName", SearchCriteria.Op.SC, ssc); + sc.setParameters("keywordAccountName", "%" + keyword + "%"); + sc.setParameters("keywordState", "%" + keyword + "%"); } if (type != null) { @@ -2625,13 +2754,16 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q if (domain == null) { domain = _domainDao.findById(domainId); } - sc.setParameters("path", domain.getPath() + "%"); + sc.setJoinParameters("domainSearch", "path", domain.getPath() + "%"); } else { sc.setParameters("domainId", domainId); } } - return _accountJoinDao.searchAndCount(sc, searchFilter); + Pair, Integer> uniqueAccountPair = _accountDao.searchAndCount(sc, searchFilter); + Integer count = uniqueAccountPair.second(); + List accountIds = uniqueAccountPair.first().stream().map(AccountVO::getId).collect(Collectors.toList()); + return new Pair<>(accountIds, count); } @Override @@ -2714,9 +2846,10 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q @Override public ListResponse searchForStoragePools(ListStoragePoolsCmd cmd) { Pair, Integer> result = searchForStoragePoolsInternal(cmd); - ListResponse response = new ListResponse(); + ListResponse response = new ListResponse<>(); List poolResponses = ViewResponseHelper.createStoragePoolResponse(result.first().toArray(new StoragePoolJoinVO[result.first().size()])); + Map poolUuidToIdMap = result.first().stream().collect(Collectors.toMap(StoragePoolJoinVO::getUuid, StoragePoolJoinVO::getId)); for (StoragePoolResponse poolResponse : poolResponses) { DataStore store = dataStoreManager.getPrimaryDataStore(poolResponse.getId()); if (store != null) { @@ -2725,8 +2858,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q Map caps = driver.getCapabilities(); if (Storage.StoragePoolType.NetworkFilesystem.toString().equals(poolResponse.getType()) && HypervisorType.VMware.toString().equals(poolResponse.getHypervisor())) { - StoragePoolVO pool = storagePoolDao.findPoolByUUID(poolResponse.getId()); - StoragePoolDetailVO detail = _storagePoolDetailsDao.findDetail(pool.getId(), Storage.Capability.HARDWARE_ACCELERATION.toString()); + StoragePoolDetailVO detail = _storagePoolDetailsDao.findDetail(poolUuidToIdMap.get(poolResponse.getId()), Storage.Capability.HARDWARE_ACCELERATION.toString()); if (detail != null) { caps.put(Storage.Capability.HARDWARE_ACCELERATION.toString(), detail.getValue()); } @@ -2755,26 +2887,14 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q Long startIndex = cmd.getStartIndex(); Long pageSize = cmd.getPageSizeVal(); - Filter searchFilter = new Filter(StoragePoolJoinVO.class, "id", Boolean.TRUE, startIndex, pageSize); + Filter searchFilter = new Filter(StoragePoolVO.class, "id", Boolean.TRUE, startIndex, pageSize); - // search & count Pool details by ids - Pair, Integer> uniquePoolPair = _poolJoinDao.searchAndCount(id, name, zoneId, path, pod, + Pair, Integer> uniquePoolPair = storagePoolDao.searchForIdsAndCount(id, name, zoneId, path, pod, cluster, address, scopeType, status, keyword, searchFilter); - Integer count = uniquePoolPair.second(); - if (count.intValue() == 0) { - // empty result - return uniquePoolPair; - } - List uniquePools = uniquePoolPair.first(); - Long[] vrIds = new Long[uniquePools.size()]; - int i = 0; - for (StoragePoolJoinVO v : uniquePools) { - vrIds[i++] = v.getId(); - } - List vrs = _poolJoinDao.searchByIds(vrIds); - return new Pair, Integer>(vrs, count); + List storagePools = _poolJoinDao.searchByIds(uniquePoolPair.first().toArray(new Long[0])); + return new Pair<>(storagePools, uniquePoolPair.second()); } @Override @@ -3032,6 +3152,33 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q } private Pair, Integer> searchForDiskOfferingsInternal(ListDiskOfferingsCmd cmd) { + Ternary, Integer, String[]> diskOfferingIdPage = searchForDiskOfferingsIdsAndCount(cmd); + + Integer count = diskOfferingIdPage.second(); + Long[] idArray = diskOfferingIdPage.first().toArray(new Long[0]); + String[] requiredTagsArray = diskOfferingIdPage.third(); + + if (count == 0) { + return new Pair<>(new ArrayList<>(), count); + } + + List diskOfferings = _diskOfferingJoinDao.searchByIds(idArray); + + if (requiredTagsArray.length != 0) { + ListIterator iteratorForTagsChecking = diskOfferings.listIterator(); + while (iteratorForTagsChecking.hasNext()) { + DiskOfferingJoinVO offering = iteratorForTagsChecking.next(); + String offeringTags = offering.getTags(); + String[] offeringTagsArray = (offeringTags == null || offeringTags.isEmpty()) ? new String[0] : offeringTags.split(","); + if (!CollectionUtils.isSubCollection(Arrays.asList(requiredTagsArray), Arrays.asList(offeringTagsArray))) { + iteratorForTagsChecking.remove(); + } + } + } + return new Pair<>(diskOfferings, count); + } + + private Ternary, Integer, String[]> searchForDiskOfferingsIdsAndCount(ListDiskOfferingsCmd cmd) { // Note // The list method for offerings is being modified in accordance with // discussion with Will/Kevin @@ -3042,11 +3189,6 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q // till // root - Filter searchFilter = new Filter(DiskOfferingJoinVO.class, "sortKey", SortKeyAscending.value(), cmd.getStartIndex(), cmd.getPageSizeVal()); - searchFilter.addOrderBy(DiskOfferingJoinVO.class, "id", true); - SearchCriteria sc = _diskOfferingJoinDao.createSearchCriteria(); - sc.addAnd("computeOnly", Op.EQ, false); - Account account = CallContext.current().getCallingAccount(); Object name = cmd.getDiskOfferingName(); Object id = cmd.getId(); @@ -3059,6 +3201,15 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q Long storagePoolId = cmd.getStoragePoolId(); Boolean encrypt = cmd.getEncrypt(); final Long vmId = cmd.getVirtualMachineId(); + + Filter searchFilter = new Filter(DiskOfferingVO.class, "sortKey", SortKeyAscending.value(), cmd.getStartIndex(), cmd.getPageSizeVal()); + searchFilter.addOrderBy(DiskOfferingVO.class, "id", true); + SearchBuilder diskOfferingSearch = _diskOfferingDao.createSearchBuilder(); + diskOfferingSearch.select(null, Func.DISTINCT, diskOfferingSearch.entity().getId()); // select distinct + + diskOfferingSearch.and("computeOnly", diskOfferingSearch.entity().isComputeOnly(), Op.EQ); + diskOfferingSearch.and("activeState", diskOfferingSearch.entity().getState(), Op.EQ); + // Keeping this logic consistent with domain specific zones // if a domainId is provided, we just return the disk offering // associated with this domain @@ -3066,11 +3217,26 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q if (accountMgr.isRootAdmin(account.getId()) || isPermissible(account.getDomainId(), domainId)) { // check if the user's domain == do's domain || user's domain is // a child of so's domain for non-root users - sc.addAnd("domainId", Op.FIND_IN_SET, String.valueOf(domainId)); + SearchBuilder domainDetailsSearch = _diskOfferingDetailsDao.createSearchBuilder(); + domainDetailsSearch.and("domainId", domainDetailsSearch.entity().getValue(), Op.EQ); + + diskOfferingSearch.join("domainDetailsSearch", domainDetailsSearch, JoinBuilder.JoinType.LEFT, JoinBuilder.JoinCondition.AND, + diskOfferingSearch.entity().getId(), domainDetailsSearch.entity().getResourceId(), + domainDetailsSearch.entity().getName(), diskOfferingSearch.entity().setString(ApiConstants.DOMAIN_ID)); + if (!isRootAdmin) { - sc.addAnd("displayOffering", SearchCriteria.Op.EQ, 1); + diskOfferingSearch.and("displayOffering", diskOfferingSearch.entity().getDisplayOffering(), Op.EQ); } - return _diskOfferingJoinDao.searchAndCount(sc, searchFilter); + + SearchCriteria sc = diskOfferingSearch.create(); + sc.setParameters("computeOnly", false); + sc.setParameters("activeState", DiskOffering.State.Active); + + sc.setJoinParameters("domainDetailsSearch", "domainId", domainId); + + Pair, Integer> uniquePairs = _diskOfferingDao.searchAndCount(sc, searchFilter); + List idsArray = uniquePairs.first().stream().map(DiskOfferingVO::getId).collect(Collectors.toList()); + return new Ternary<>(idsArray, uniquePairs.second(), new String[0]); } else { throw new PermissionDeniedException("The account:" + account.getAccountName() + " does not fall in the same domain hierarchy as the disk offering"); } @@ -3091,76 +3257,112 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q } if (keyword != null) { - SearchCriteria ssc = _diskOfferingJoinDao.createSearchCriteria(); - ssc.addOr("displayText", SearchCriteria.Op.LIKE, "%" + keyword + "%"); - ssc.addOr("name", SearchCriteria.Op.LIKE, "%" + keyword + "%"); - - sc.addAnd("name", SearchCriteria.Op.SC, ssc); + diskOfferingSearch.and().op("keywordDisplayText", diskOfferingSearch.entity().getDisplayText(), Op.LIKE); + diskOfferingSearch.or("keywordName", diskOfferingSearch.entity().getName(), Op.LIKE); + diskOfferingSearch.cp(); } if (id != null) { - sc.addAnd("id", SearchCriteria.Op.EQ, id); + diskOfferingSearch.and("id", diskOfferingSearch.entity().getId(), Op.EQ); } if (name != null) { - sc.addAnd("name", SearchCriteria.Op.EQ, name); + diskOfferingSearch.and("name", diskOfferingSearch.entity().getName(), Op.EQ); } if (encrypt != null) { - sc.addAnd("encrypt", SearchCriteria.Op.EQ, encrypt); + diskOfferingSearch.and("encrypt", diskOfferingSearch.entity().getEncrypt(), Op.EQ); } if (zoneId != null) { - SearchBuilder sb = _diskOfferingJoinDao.createSearchBuilder(); - sb.and("zoneId", sb.entity().getZoneId(), Op.FIND_IN_SET); - sb.or("zId", sb.entity().getZoneId(), Op.NULL); - sb.done(); - SearchCriteria zoneSC = sb.create(); - zoneSC.setParameters("zoneId", String.valueOf(zoneId)); - sc.addAnd("zoneId", SearchCriteria.Op.SC, zoneSC); - DataCenterJoinVO zone = _dcJoinDao.findById(zoneId); - if (DataCenter.Type.Edge.equals(zone.getType())) { - sc.addAnd("useLocalStorage", Op.EQ, true); - } + diskOfferingSearch.and("useLocalStorage", diskOfferingSearch.entity().isUseLocalStorage(), Op.EQ); + + SearchBuilder zoneDetailSearch = _diskOfferingDetailsDao.createSearchBuilder(); + + zoneDetailSearch.and().op("zoneId", zoneDetailSearch.entity().getValue(), Op.EQ); + zoneDetailSearch.or("zoneIdNull", zoneDetailSearch.entity().getId(), Op.NULL); + zoneDetailSearch.cp(); + + diskOfferingSearch.join("zoneDetailSearch", zoneDetailSearch, JoinBuilder.JoinType.LEFT, JoinBuilder.JoinCondition.AND, + diskOfferingSearch.entity().getId(), zoneDetailSearch.entity().getResourceId(), + zoneDetailSearch.entity().getName(), diskOfferingSearch.entity().setString(ApiConstants.ZONE_ID)); } DiskOffering currentDiskOffering = null; + Volume volume = null; if (volumeId != null) { - Volume volume = volumeDao.findById(volumeId); + volume = volumeDao.findById(volumeId); if (volume == null) { throw new InvalidParameterValueException(String.format("Unable to find a volume with specified id %s", volumeId)); } currentDiskOffering = _diskOfferingDao.findByIdIncludingRemoved(volume.getDiskOfferingId()); if (!currentDiskOffering.isComputeOnly() && currentDiskOffering.getDiskSizeStrictness()) { - SearchCriteria ssc = _diskOfferingJoinDao.createSearchCriteria(); - ssc.addOr("diskSize", Op.EQ, volume.getSize()); - ssc.addOr("customized", SearchCriteria.Op.EQ, true); - sc.addAnd("diskSizeOrCustomized", SearchCriteria.Op.SC, ssc); + diskOfferingSearch.and().op("diskSize", diskOfferingSearch.entity().getDiskSize(), Op.EQ); + diskOfferingSearch.or("customized", diskOfferingSearch.entity().isCustomized(), Op.EQ); + diskOfferingSearch.cp(); } - sc.addAnd("id", SearchCriteria.Op.NEQ, currentDiskOffering.getId()); - sc.addAnd("diskSizeStrictness", Op.EQ, currentDiskOffering.getDiskSizeStrictness()); + diskOfferingSearch.and("idNEQ", diskOfferingSearch.entity().getId(), Op.NEQ); + diskOfferingSearch.and("diskSizeStrictness", diskOfferingSearch.entity().getDiskSizeStrictness(), Op.EQ); + } + + Account caller = CallContext.current().getCallingAccount(); + if (!Account.Type.ADMIN.equals(caller.getType())) { + SearchBuilder domainDetailsSearch = _diskOfferingDetailsDao.createSearchBuilder(); + domainDetailsSearch.and().op("domainIdIN", domainDetailsSearch.entity().getValue(), Op.IN); + domainDetailsSearch.or("domainIdNull", domainDetailsSearch.entity().getId(), Op.NULL); + domainDetailsSearch.cp(); + + diskOfferingSearch.join("domainDetailsSearch", domainDetailsSearch, JoinBuilder.JoinType.LEFT, JoinBuilder.JoinCondition.AND, + diskOfferingSearch.entity().getId(), domainDetailsSearch.entity().getResourceId(), + domainDetailsSearch.entity().getName(), diskOfferingSearch.entity().setString(ApiConstants.DOMAIN_ID)); + } + + SearchCriteria sc = diskOfferingSearch.create(); + + sc.setParameters("computeOnly", false); + sc.setParameters("activeState", DiskOffering.State.Active); + + if (keyword != null) { + sc.setParameters("keywordDisplayText", "%" + keyword + "%"); + sc.setParameters("keywordName", "%" + keyword + "%"); + } + + if (id != null) { + sc.setParameters("id", id); + } + + if (name != null) { + sc.setParameters("name", name); + } + + if (encrypt != null) { + sc.setParameters("encrypt", encrypt); + } + + if (zoneId != null) { + sc.setJoinParameters("zoneDetailSearch", "zoneId", zoneId); + + DataCenterJoinVO zone = _dcJoinDao.findById(zoneId); + if (DataCenter.Type.Edge.equals(zone.getType())) { + sc.setParameters("useLocalStorage", true); + } + } + + if (volumeId != null) { + if (!currentDiskOffering.isComputeOnly() && currentDiskOffering.getDiskSizeStrictness()) { + sc.setParameters("diskSize", volume.getSize()); + sc.setParameters("customized", true); + } + sc.setParameters("idNEQ", currentDiskOffering.getId()); + sc.setParameters("diskSizeStrictness", currentDiskOffering.getDiskSizeStrictness()); } // Filter offerings that are not associated with caller's domain - // Fetch the offering ids from the details table since theres no smart way to filter them in the join ... yet! - Account caller = CallContext.current().getCallingAccount(); - if (caller.getType() != Account.Type.ADMIN) { + if (!Account.Type.ADMIN.equals(caller.getType())) { Domain callerDomain = _domainDao.findById(caller.getDomainId()); List domainIds = findRelatedDomainIds(callerDomain, isRecursive); - List ids = _diskOfferingDetailsDao.findOfferingIdsByDomainIds(domainIds); - SearchBuilder sb = _diskOfferingJoinDao.createSearchBuilder(); - if (ids != null && !ids.isEmpty()) { - sb.and("id", sb.entity().getId(), Op.IN); - } - sb.or("domainId", sb.entity().getDomainId(), Op.NULL); - sb.done(); - - SearchCriteria scc = sb.create(); - if (ids != null && !ids.isEmpty()) { - scc.setParameters("id", ids.toArray()); - } - sc.addAnd("domainId", SearchCriteria.Op.SC, scc); + sc.setJoinParameters("domainDetailsSearch", "domainIdIN", domainIds.toArray()); } if (vmId != null) { @@ -3173,30 +3375,18 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q } } - Pair, Integer> result = _diskOfferingJoinDao.searchAndCount(sc, searchFilter); + Pair, Integer> uniquePairs = _diskOfferingDao.searchAndCount(sc, searchFilter); String[] requiredTagsArray = new String[0]; - if (CollectionUtils.isNotEmpty(result.first()) && VolumeApiServiceImpl.MatchStoragePoolTagsWithDiskOffering.valueIn(zoneId)) { + if (CollectionUtils.isNotEmpty(uniquePairs.first()) && VolumeApiServiceImpl.MatchStoragePoolTagsWithDiskOffering.valueIn(zoneId)) { if (volumeId != null) { - Volume volume = volumeDao.findById(volumeId); - currentDiskOffering = _diskOfferingDao.findByIdIncludingRemoved(volume.getDiskOfferingId()); requiredTagsArray = currentDiskOffering.getTagsArray(); } else if (storagePoolId != null) { requiredTagsArray = _storageTagDao.getStoragePoolTags(storagePoolId).toArray(new String[0]); } } - if (requiredTagsArray.length != 0) { - ListIterator iteratorForTagsChecking = result.first().listIterator(); - while (iteratorForTagsChecking.hasNext()) { - DiskOfferingJoinVO offering = iteratorForTagsChecking.next(); - String offeringTags = offering.getTags(); - String[] offeringTagsArray = (offeringTags == null || offeringTags.isEmpty()) ? new String[0] : offeringTags.split(","); - if (!CollectionUtils.isSubCollection(Arrays.asList(requiredTagsArray), Arrays.asList(offeringTagsArray))) { - iteratorForTagsChecking.remove(); - } - } - } + List idsArray = uniquePairs.first().stream().map(DiskOfferingVO::getId).collect(Collectors.toList()); - return new Pair<>(result.first(), result.second()); + return new Ternary<>(idsArray, uniquePairs.second(), requiredTagsArray); } private List findRelatedDomainIds(Domain domain, boolean isRecursive) { @@ -3239,6 +3429,20 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q } private Pair, Integer> searchForServiceOfferingsInternal(ListServiceOfferingsCmd cmd) { + Pair, Integer> offeringIdPage = searchForServiceOfferingIdsAndCount(cmd); + + Integer count = offeringIdPage.second(); + Long[] idArray = offeringIdPage.first().toArray(new Long[0]); + + if (count == 0) { + return new Pair<>(new ArrayList<>(), count); + } + + List srvOfferings = _srvOfferingJoinDao.searchByIds(idArray); + return new Pair<>(srvOfferings, count); + } + + private Pair, Integer> searchForServiceOfferingIdsAndCount(ListServiceOfferingsCmd cmd) { // Note // The filteredOfferings method for offerings is being modified in accordance with // discussion with Will/Kevin @@ -3248,9 +3452,6 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q // their domains+parent domains ... all the way // till // root - Filter searchFilter = new Filter(ServiceOfferingJoinVO.class, "sortKey", SortKeyAscending.value(), cmd.getStartIndex(), cmd.getPageSizeVal()); - searchFilter.addOrderBy(ServiceOfferingJoinVO.class, "id", true); - Account caller = CallContext.current().getCallingAccount(); Object name = cmd.getServiceOfferingName(); Object id = cmd.getId(); @@ -3260,6 +3461,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q Boolean isSystem = cmd.getIsSystem(); String vmTypeStr = cmd.getSystemVmType(); ServiceOfferingVO currentVmOffering = null; + DiskOfferingVO diskOffering = null; Boolean isRecursive = cmd.isRecursive(); Long zoneId = cmd.getZoneId(); Integer cpuNumber = cmd.getCpuNumber(); @@ -3268,9 +3470,8 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q Boolean encryptRoot = cmd.getEncryptRoot(); final Long templateId = cmd.getTemplateId(); - SearchCriteria sc = _srvOfferingJoinDao.createSearchCriteria(); if (!accountMgr.isRootAdmin(caller.getId()) && isSystem) { - throw new InvalidParameterValueException("Only ROOT admins can access system's offering"); + throw new InvalidParameterValueException("Only ROOT admins can access system offerings."); } // Keeping this logic consistent with domain specific zones @@ -3284,32 +3485,319 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q } } + VMInstanceVO vmInstance = null; if (vmId != null) { - VMInstanceVO vmInstance = _vmInstanceDao.findById(vmId); + vmInstance = _vmInstanceDao.findById(vmId); if ((vmInstance == null) || (vmInstance.getRemoved() != null)) { InvalidParameterValueException ex = new InvalidParameterValueException("unable to find a virtual machine with specified id"); ex.addProxyObject(vmId.toString(), "vmId"); throw ex; } - accountMgr.checkAccess(caller, null, true, vmInstance); + } + Filter searchFilter = new Filter(ServiceOfferingVO.class, "sortKey", SortKeyAscending.value(), cmd.getStartIndex(), cmd.getPageSizeVal()); + searchFilter.addOrderBy(ServiceOfferingVO.class, "id", true); + + SearchBuilder serviceOfferingSearch = _srvOfferingDao.createSearchBuilder(); + serviceOfferingSearch.select(null, Func.DISTINCT, serviceOfferingSearch.entity().getId()); // select distinct + serviceOfferingSearch.and("activeState", serviceOfferingSearch.entity().getState(), Op.EQ); + + if (vmId != null) { currentVmOffering = _srvOfferingDao.findByIdIncludingRemoved(vmInstance.getId(), vmInstance.getServiceOfferingId()); - if (! currentVmOffering.isDynamic()) { - sc.addAnd("id", SearchCriteria.Op.NEQ, currentVmOffering.getId()); + diskOffering = _diskOfferingDao.findByIdIncludingRemoved(currentVmOffering.getDiskOfferingId()); + if (!currentVmOffering.isDynamic()) { + serviceOfferingSearch.and("idNEQ", serviceOfferingSearch.entity().getId(), SearchCriteria.Op.NEQ); } if (currentVmOffering.getDiskOfferingStrictness()) { - sc.addAnd("diskOfferingId", Op.EQ, currentVmOffering.getDiskOfferingId()); - sc.addAnd("diskOfferingStrictness", Op.EQ, true); + serviceOfferingSearch.and("diskOfferingId", serviceOfferingSearch.entity().getDiskOfferingId(), SearchCriteria.Op.EQ); + } + serviceOfferingSearch.and("diskOfferingStrictness", serviceOfferingSearch.entity().getDiskOfferingStrictness(), SearchCriteria.Op.EQ); + + // In case vm is running return only offerings greater than equal to current offering compute and offering's dynamic scalability should match + if (vmInstance.getState() == VirtualMachine.State.Running) { + Integer vmCpu = currentVmOffering.getCpu(); + Integer vmMemory = currentVmOffering.getRamSize(); + Integer vmSpeed = currentVmOffering.getSpeed(); + if ((vmCpu == null || vmMemory == null || vmSpeed == null) && VirtualMachine.Type.User.equals(vmInstance.getType())) { + UserVmVO userVmVO = userVmDao.findById(vmId); + userVmDao.loadDetails(userVmVO); + Map details = userVmVO.getDetails(); + vmCpu = NumbersUtil.parseInt(details.get(ApiConstants.CPU_NUMBER), 0); + if (vmSpeed == null) { + vmSpeed = NumbersUtil.parseInt(details.get(ApiConstants.CPU_SPEED), 0); + } + vmMemory = NumbersUtil.parseInt(details.get(ApiConstants.MEMORY), 0); + } + if (vmCpu != null && vmCpu > 0) { + /* + (service_offering.cpu >= ?) + OR ( + service_offering.cpu IS NULL + AND (maxComputeDetailsSearch.value IS NULL OR maxComputeDetailsSearch.value >= ?) + ) + */ + SearchBuilder maxComputeDetailsSearch = _srvOfferingDetailsDao.createSearchBuilder(); + + serviceOfferingSearch.join("maxComputeDetailsSearch", maxComputeDetailsSearch, JoinBuilder.JoinType.LEFT, JoinBuilder.JoinCondition.AND, + serviceOfferingSearch.entity().getId(), maxComputeDetailsSearch.entity().getResourceId(), + maxComputeDetailsSearch.entity().getName(), serviceOfferingSearch.entity().setString(ApiConstants.MAX_CPU_NUMBER)); + + serviceOfferingSearch.and().op("vmCpu", serviceOfferingSearch.entity().getCpu(), Op.GTEQ); + serviceOfferingSearch.or().op("vmCpuNull", serviceOfferingSearch.entity().getCpu(), Op.NULL); + serviceOfferingSearch.and().op("maxComputeDetailsSearch", "vmMaxComputeNull", maxComputeDetailsSearch.entity().getValue(), Op.NULL); + serviceOfferingSearch.or("maxComputeDetailsSearch", "vmMaxComputeGTEQ", maxComputeDetailsSearch.entity().getValue(), Op.GTEQ).cp(); + + serviceOfferingSearch.cp().cp(); + + } + if (vmSpeed != null && vmSpeed > 0) { + serviceOfferingSearch.and().op("speedNULL", serviceOfferingSearch.entity().getSpeed(), Op.NULL); + serviceOfferingSearch.or("speedGTEQ", serviceOfferingSearch.entity().getSpeed(), Op.GTEQ); + serviceOfferingSearch.cp(); + } + if (vmMemory != null && vmMemory > 0) { + /* + (service_offering.ram_size >= ?) + OR ( + service_offering.ram_size IS NULL + AND (max_memory_details.value IS NULL OR max_memory_details.value >= ?) + ) + */ + SearchBuilder maxMemoryDetailsSearch = _srvOfferingDetailsDao.createSearchBuilder(); + + serviceOfferingSearch.join("maxMemoryDetailsSearch", maxMemoryDetailsSearch, JoinBuilder.JoinType.LEFT, JoinBuilder.JoinCondition.AND, + serviceOfferingSearch.entity().getId(), maxMemoryDetailsSearch.entity().getResourceId(), + maxMemoryDetailsSearch.entity().getName(), serviceOfferingSearch.entity().setString("maxmemory")); + + serviceOfferingSearch.and().op("vmMemory", serviceOfferingSearch.entity().getRamSize(), Op.GTEQ); + serviceOfferingSearch.or().op("vmMemoryNull", serviceOfferingSearch.entity().getRamSize(), Op.NULL); + serviceOfferingSearch.and().op("maxMemoryDetailsSearch", "vmMaxMemoryNull", maxMemoryDetailsSearch.entity().getValue(), Op.NULL); + serviceOfferingSearch.and("maxMemoryDetailsSearch", "vmMaxMemoryGTEQ", maxMemoryDetailsSearch.entity().getValue(), Op.GTEQ).cp(); + + serviceOfferingSearch.cp().cp(); + } + serviceOfferingSearch.and("dynamicScalingEnabled", serviceOfferingSearch.entity().isDynamicScalingEnabled(), SearchCriteria.Op.EQ); + } + } + + if ((accountMgr.isNormalUser(caller.getId()) || accountMgr.isDomainAdmin(caller.getId())) || caller.getType() == Account.Type.RESOURCE_DOMAIN_ADMIN) { + // For non-root users. + if (isSystem) { + throw new InvalidParameterValueException("Only root admins can access system's offering"); + } + if (isRecursive) { // domain + all sub-domains + if (caller.getType() == Account.Type.NORMAL) { + throw new InvalidParameterValueException("Only ROOT admins and Domain admins can list service offerings with isrecursive=true"); + } + } + } else { + // for root users + if (caller.getDomainId() != 1 && isSystem) { // NON ROOT admin + throw new InvalidParameterValueException("Non ROOT admins cannot access system's offering"); + } + if (domainId != null) { + SearchBuilder srvOffrDomainDetailSearch = _srvOfferingDetailsDao.createSearchBuilder(); + srvOffrDomainDetailSearch.and("domainId", srvOffrDomainDetailSearch.entity().getValue(), Op.EQ); + serviceOfferingSearch.join("domainDetailSearch", srvOffrDomainDetailSearch, JoinBuilder.JoinType.LEFT, JoinBuilder.JoinCondition.AND, + serviceOfferingSearch.entity().getId(), srvOffrDomainDetailSearch.entity().getResourceId(), + srvOffrDomainDetailSearch.entity().getName(), serviceOfferingSearch.entity().setString(ApiConstants.DOMAIN_ID)); + } + } + + if (keyword != null) { + serviceOfferingSearch.and().op("keywordName", serviceOfferingSearch.entity().getName(), SearchCriteria.Op.LIKE); + serviceOfferingSearch.or("keywordDisplayText", serviceOfferingSearch.entity().getDisplayText(), SearchCriteria.Op.LIKE); + serviceOfferingSearch.cp(); + } + + if (id != null) { + serviceOfferingSearch.and("id", serviceOfferingSearch.entity().getId(), SearchCriteria.Op.EQ); + } + + if (isSystem != null) { + // note that for non-root users, isSystem is always false when + // control comes to here + serviceOfferingSearch.and("systemUse", serviceOfferingSearch.entity().isSystemUse(), SearchCriteria.Op.EQ); + } + + if (name != null) { + serviceOfferingSearch.and("name", serviceOfferingSearch.entity().getName(), SearchCriteria.Op.EQ); + } + + if (vmTypeStr != null) { + serviceOfferingSearch.and("svmType", serviceOfferingSearch.entity().getVmType(), SearchCriteria.Op.EQ); + } + DataCenterJoinVO zone = null; + if (zoneId != null) { + SearchBuilder srvOffrZoneDetailSearch = _srvOfferingDetailsDao.createSearchBuilder(); + srvOffrZoneDetailSearch.and().op("zoneId", srvOffrZoneDetailSearch.entity().getValue(), Op.EQ); + srvOffrZoneDetailSearch.or("idNull", srvOffrZoneDetailSearch.entity().getId(), Op.NULL); + srvOffrZoneDetailSearch.cp(); + + serviceOfferingSearch.join("ZoneDetailSearch", srvOffrZoneDetailSearch, JoinBuilder.JoinType.LEFT, JoinBuilder.JoinCondition.AND, + serviceOfferingSearch.entity().getId(), srvOffrZoneDetailSearch.entity().getResourceId(), + srvOffrZoneDetailSearch.entity().getName(), serviceOfferingSearch.entity().setString(ApiConstants.ZONE_ID)); + zone = _dcJoinDao.findById(zoneId); + } + + if (encryptRoot != null || vmId != null || (zone != null && DataCenter.Type.Edge.equals(zone.getType()))) { + SearchBuilder diskOfferingSearch = _diskOfferingDao.createSearchBuilder(); + diskOfferingSearch.and("useLocalStorage", diskOfferingSearch.entity().isUseLocalStorage(), SearchCriteria.Op.EQ); + diskOfferingSearch.and("encrypt", diskOfferingSearch.entity().getEncrypt(), SearchCriteria.Op.EQ); + + if (diskOffering != null) { + List storageTags = com.cloud.utils.StringUtils.csvTagsToList(diskOffering.getTags()); + if (!storageTags.isEmpty() && VolumeApiServiceImpl.MatchStoragePoolTagsWithDiskOffering.value()) { + for (String tag : storageTags) { + diskOfferingSearch.and(tag, diskOfferingSearch.entity().getTags(), Op.EQ); + } + diskOfferingSearch.done(); + } + } + + serviceOfferingSearch.join("diskOfferingSearch", diskOfferingSearch, JoinBuilder.JoinType.INNER, JoinBuilder.JoinCondition.AND, + serviceOfferingSearch.entity().getDiskOfferingId(), diskOfferingSearch.entity().getId(), + serviceOfferingSearch.entity().setString("Active"), diskOfferingSearch.entity().getState()); + } + + if (cpuNumber != null) { + SearchBuilder maxComputeDetailsSearch = (SearchBuilder) serviceOfferingSearch.getJoinSB("maxComputeDetailsSearch"); + if (maxComputeDetailsSearch == null) { + maxComputeDetailsSearch = _srvOfferingDetailsDao.createSearchBuilder(); + serviceOfferingSearch.join("maxComputeDetailsSearch", maxComputeDetailsSearch, JoinBuilder.JoinType.LEFT, JoinBuilder.JoinCondition.AND, + serviceOfferingSearch.entity().getId(), maxComputeDetailsSearch.entity().getResourceId(), + maxComputeDetailsSearch.entity().getName(), serviceOfferingSearch.entity().setString(ApiConstants.MAX_CPU_NUMBER)); + } + + SearchBuilder minComputeDetailsSearch = _srvOfferingDetailsDao.createSearchBuilder(); + + serviceOfferingSearch.join("minComputeDetailsSearch", minComputeDetailsSearch, JoinBuilder.JoinType.LEFT, JoinBuilder.JoinCondition.AND, + serviceOfferingSearch.entity().getId(), minComputeDetailsSearch.entity().getResourceId(), + minComputeDetailsSearch.entity().getName(), serviceOfferingSearch.entity().setString(ApiConstants.MIN_CPU_NUMBER)); + + /* + (min_cpu IS NULL AND cpu IS NULL AND max_cpu IS NULL) + OR (cpu = X) + OR (min_cpu <= X AND max_cpu >= X) + + AND ( + (min_compute_details.value is NULL AND cpu is NULL) + OR (min_compute_details.value is NULL AND cpu >= X) + OR min_compute_details.value >= X + OR ( + ((min_compute_details.value is NULL AND cpu <= X) OR min_compute_details.value <= X) + AND ((max_compute_details.value is NULL AND cpu >= X) OR max_compute_details.value >= X) + ) + ) + */ + serviceOfferingSearch.and().op().op("minComputeDetailsSearch", "cpuConstraintMinComputeNull", minComputeDetailsSearch.entity().getValue(), Op.NULL); + serviceOfferingSearch.and("cpuConstraintNull", serviceOfferingSearch.entity().getCpu(), Op.NULL).cp(); + + serviceOfferingSearch.or().op("minComputeDetailsSearch", "cpuConstraintMinComputeNull", minComputeDetailsSearch.entity().getValue(), Op.NULL); + serviceOfferingSearch.and("cpuNumber", serviceOfferingSearch.entity().getCpu(), Op.GTEQ).cp(); + serviceOfferingSearch.or("cpuNumber", serviceOfferingSearch.entity().getCpu(), Op.GTEQ); + + serviceOfferingSearch.or().op().op(); + serviceOfferingSearch.op("minComputeDetailsSearch", "cpuConstraintMinComputeNull", minComputeDetailsSearch.entity().getValue(), Op.NULL); + serviceOfferingSearch.and("cpuNumber", serviceOfferingSearch.entity().getCpu(), Op.LTEQ).cp(); + serviceOfferingSearch.or("minComputeDetailsSearch", "cpuNumber", minComputeDetailsSearch.entity().getValue(), Op.LTEQ).cp(); + serviceOfferingSearch.and().op().op("maxComputeDetailsSearch", "cpuConstraintMaxComputeNull", maxComputeDetailsSearch.entity().getValue(), Op.NULL); + serviceOfferingSearch.and("cpuNumber", serviceOfferingSearch.entity().getCpu(), Op.GTEQ).cp(); + serviceOfferingSearch.or("maxComputeDetailsSearch", "cpuNumber", maxComputeDetailsSearch.entity().getValue(), Op.GTEQ).cp(); + serviceOfferingSearch.cp().cp(); + } + + if (memory != null) { + SearchBuilder maxMemoryDetailsSearch = (SearchBuilder) serviceOfferingSearch.getJoinSB("maxMemoryDetailsSearch"); + if (maxMemoryDetailsSearch == null) { + maxMemoryDetailsSearch = _srvOfferingDetailsDao.createSearchBuilder(); + serviceOfferingSearch.join("maxMemoryDetailsSearch", maxMemoryDetailsSearch, JoinBuilder.JoinType.LEFT, JoinBuilder.JoinCondition.AND, + serviceOfferingSearch.entity().getId(), maxMemoryDetailsSearch.entity().getResourceId(), + maxMemoryDetailsSearch.entity().getName(), serviceOfferingSearch.entity().setString("maxmemory")); + } + + SearchBuilder minMemoryDetailsSearch = _srvOfferingDetailsDao.createSearchBuilder(); + + serviceOfferingSearch.join("minMemoryDetailsSearch", minMemoryDetailsSearch, JoinBuilder.JoinType.LEFT, JoinBuilder.JoinCondition.AND, + serviceOfferingSearch.entity().getId(), minMemoryDetailsSearch.entity().getResourceId(), + minMemoryDetailsSearch.entity().getName(), serviceOfferingSearch.entity().setString("minmemory")); + + /* + (min_ram_size IS NULL AND ram_size IS NULL AND max_ram_size IS NULL) + OR (ram_size = X) + OR (min_ram_size <= X AND max_ram_size >= X) + */ + + serviceOfferingSearch.and().op().op("minMemoryDetailsSearch", "memoryConstraintMinMemoryNull", minMemoryDetailsSearch.entity().getValue(), Op.NULL); + serviceOfferingSearch.and("memoryConstraintNull", serviceOfferingSearch.entity().getRamSize(), Op.NULL).cp(); + + serviceOfferingSearch.or().op("minMemoryDetailsSearch", "memoryConstraintMinMemoryNull", minMemoryDetailsSearch.entity().getValue(), Op.NULL); + serviceOfferingSearch.and("memory", serviceOfferingSearch.entity().getRamSize(), Op.GTEQ).cp(); + serviceOfferingSearch.or("memory", serviceOfferingSearch.entity().getRamSize(), Op.GTEQ); + + serviceOfferingSearch.or().op().op(); + serviceOfferingSearch.op("minMemoryDetailsSearch", "memoryConstraintMinMemoryNull", minMemoryDetailsSearch.entity().getValue(), Op.NULL); + serviceOfferingSearch.and("memory", serviceOfferingSearch.entity().getRamSize(), Op.LTEQ).cp(); + serviceOfferingSearch.or("minMemoryDetailsSearch", "memory", minMemoryDetailsSearch.entity().getValue(), Op.LTEQ).cp(); + serviceOfferingSearch.and().op().op("maxMemoryDetailsSearch", "memoryConstraintMaxMemoryNull", maxMemoryDetailsSearch.entity().getValue(), Op.NULL); + serviceOfferingSearch.and("memory", serviceOfferingSearch.entity().getRamSize(), Op.GTEQ).cp(); + serviceOfferingSearch.or("maxMemoryDetailsSearch", "memory", maxMemoryDetailsSearch.entity().getValue(), Op.GTEQ).cp(); + serviceOfferingSearch.cp().cp(); + } + + if (cpuSpeed != null) { + serviceOfferingSearch.and().op("speedNull", serviceOfferingSearch.entity().getSpeed(), Op.NULL); + serviceOfferingSearch.or("speedGTEQ", serviceOfferingSearch.entity().getSpeed(), Op.GTEQ); + serviceOfferingSearch.cp(); + } + + // Filter offerings that are not associated with caller's domain + // Fetch the offering ids from the details table since theres no smart way to filter them in the join ... yet! + if (caller.getType() != Account.Type.ADMIN) { + SearchBuilder srvOffrDomainDetailSearch = _srvOfferingDetailsDao.createSearchBuilder(); + srvOffrDomainDetailSearch.and().op("domainIdIN", srvOffrDomainDetailSearch.entity().getValue(), Op.IN); + srvOffrDomainDetailSearch.or("idNull", srvOffrDomainDetailSearch.entity().getValue(), Op.NULL); + srvOffrDomainDetailSearch.cp(); + serviceOfferingSearch.join("domainDetailSearchNormalUser", srvOffrDomainDetailSearch, JoinBuilder.JoinType.LEFT, JoinBuilder.JoinCondition.AND, + serviceOfferingSearch.entity().getId(), srvOffrDomainDetailSearch.entity().getResourceId(), + srvOffrDomainDetailSearch.entity().getName(), serviceOfferingSearch.entity().setString(ApiConstants.DOMAIN_ID)); + } + + if (currentVmOffering != null) { + List hostTags = com.cloud.utils.StringUtils.csvTagsToList(currentVmOffering.getHostTag()); + if (!hostTags.isEmpty()) { + + serviceOfferingSearch.and().op("hostTag", serviceOfferingSearch.entity().getHostTag(), Op.NULL); + serviceOfferingSearch.or().op(); + + for(String tag : hostTags) { + serviceOfferingSearch.and(tag, serviceOfferingSearch.entity().getHostTag(), Op.EQ); + } + serviceOfferingSearch.cp().cp().done(); + } + } + + SearchCriteria sc = serviceOfferingSearch.create(); + sc.setParameters("activeState", ServiceOffering.State.Active); + + if (vmId != null) { + if (!currentVmOffering.isDynamic()) { + sc.setParameters("idNEQ", currentVmOffering.getId()); + } + + if (currentVmOffering.getDiskOfferingStrictness()) { + sc.setParameters("diskOfferingId", currentVmOffering.getDiskOfferingId()); + sc.setParameters("diskOfferingStrictness", true); } else { - sc.addAnd("diskOfferingStrictness", Op.EQ, false); + sc.setParameters("diskOfferingStrictness", false); } boolean isRootVolumeUsingLocalStorage = virtualMachineManager.isRootVolumeOnLocalStorage(vmId); // 1. Only return offerings with the same storage type than the storage pool where the VM's root volume is allocated - sc.addAnd("useLocalStorage", SearchCriteria.Op.EQ, isRootVolumeUsingLocalStorage); + sc.setJoinParameters("diskOfferingSearch", "useLocalStorage", isRootVolumeUsingLocalStorage); // 2.In case vm is running return only offerings greater than equal to current offering compute and offering's dynamic scalability should match if (vmInstance.getState() == VirtualMachine.State.Running) { @@ -3327,154 +3815,93 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q vmMemory = NumbersUtil.parseInt(details.get(ApiConstants.MEMORY), 0); } if (vmCpu != null && vmCpu > 0) { - sc.addAnd("cpu", Op.SC, getMinimumCpuServiceOfferingJoinSearchCriteria(vmCpu)); + sc.setParameters("vmCpu", vmCpu); + sc.setParameters("vmMaxComputeGTEQ", vmCpu); } if (vmSpeed != null && vmSpeed > 0) { - sc.addAnd("speed", Op.SC, getMinimumCpuSpeedServiceOfferingJoinSearchCriteria(vmSpeed)); + sc.setParameters("speedGTEQ", vmSpeed); } if (vmMemory != null && vmMemory > 0) { - sc.addAnd("ramSize", Op.SC, getMinimumMemoryServiceOfferingJoinSearchCriteria(vmMemory)); + sc.setParameters("vmMemory", vmMemory); + sc.setParameters("vmMaxMemoryGTEQ", vmMemory); } - sc.addAnd("dynamicScalingEnabled", Op.EQ, currentVmOffering.isDynamicScalingEnabled()); + sc.setParameters("dynamicScalingEnabled", currentVmOffering.isDynamicScalingEnabled()); } } - // boolean includePublicOfferings = false; - if ((accountMgr.isNormalUser(caller.getId()) || accountMgr.isDomainAdmin(caller.getId())) || caller.getType() == Account.Type.RESOURCE_DOMAIN_ADMIN) { - // For non-root users. - if (isSystem) { - throw new InvalidParameterValueException("Only root admins can access system's offering"); - } - if (isRecursive) { // domain + all sub-domains - if (caller.getType() == Account.Type.NORMAL) { - throw new InvalidParameterValueException("Only ROOT admins and Domain admins can list service offerings with isrecursive=true"); - } - } - } else { - // for root users - if (caller.getDomainId() != 1 && isSystem) { // NON ROOT admin - throw new InvalidParameterValueException("Non ROOT admins cannot access system's offering"); - } + if ((!accountMgr.isNormalUser(caller.getId()) && !accountMgr.isDomainAdmin(caller.getId())) && caller.getType() != Account.Type.RESOURCE_DOMAIN_ADMIN) { if (domainId != null) { - sc.addAnd("domainId", Op.FIND_IN_SET, String.valueOf(domainId)); + sc.setJoinParameters("domainDetailSearch", "domainId", domainId); } } if (keyword != null) { - SearchCriteria ssc = _srvOfferingJoinDao.createSearchCriteria(); - ssc.addOr("displayText", SearchCriteria.Op.LIKE, "%" + keyword + "%"); - ssc.addOr("name", SearchCriteria.Op.LIKE, "%" + keyword + "%"); - - sc.addAnd("name", SearchCriteria.Op.SC, ssc); + sc.setParameters("keywordName", "%" + keyword + "%"); + sc.setParameters("keywordDisplayText", "%" + keyword + "%"); } if (id != null) { - sc.addAnd("id", SearchCriteria.Op.EQ, id); + sc.setParameters("id", id); } if (isSystem != null) { // note that for non-root users, isSystem is always false when // control comes to here - sc.addAnd("systemUse", SearchCriteria.Op.EQ, isSystem); + sc.setParameters("systemUse", isSystem); } if (encryptRoot != null) { - sc.addAnd("encryptRoot", SearchCriteria.Op.EQ, encryptRoot); + sc.setJoinParameters("diskOfferingSearch", "encrypt", encryptRoot); } if (name != null) { - sc.addAnd("name", SearchCriteria.Op.EQ, name); + sc.setParameters("name", name); } if (vmTypeStr != null) { - sc.addAnd("vmType", SearchCriteria.Op.EQ, vmTypeStr); + sc.setParameters("svmType", vmTypeStr); } if (zoneId != null) { - SearchBuilder sb = _srvOfferingJoinDao.createSearchBuilder(); - sb.and("zoneId", sb.entity().getZoneId(), Op.FIND_IN_SET); - sb.or("zId", sb.entity().getZoneId(), Op.NULL); - sb.done(); - SearchCriteria zoneSC = sb.create(); - zoneSC.setParameters("zoneId", String.valueOf(zoneId)); - sc.addAnd("zoneId", SearchCriteria.Op.SC, zoneSC); - DataCenterJoinVO zone = _dcJoinDao.findById(zoneId); + sc.setJoinParameters("ZoneDetailSearch", "zoneId", zoneId); + if (DataCenter.Type.Edge.equals(zone.getType())) { - sc.addAnd("useLocalStorage", Op.EQ, true); + sc.setJoinParameters("diskOfferingSearch", "useLocalStorage", true); } } if (cpuNumber != null) { - SearchCriteria cpuConstraintSearchCriteria = _srvOfferingJoinDao.createSearchCriteria(); - cpuConstraintSearchCriteria.addAnd("minCpu", Op.LTEQ, cpuNumber); - cpuConstraintSearchCriteria.addAnd("maxCpu", Op.GTEQ, cpuNumber); - - SearchCriteria cpuSearchCriteria = _srvOfferingJoinDao.createSearchCriteria(); - cpuSearchCriteria.addOr("minCpu", Op.NULL); - cpuSearchCriteria.addOr("constraints", Op.SC, cpuConstraintSearchCriteria); - cpuSearchCriteria.addOr("minCpu", Op.GTEQ, cpuNumber); - - sc.addAnd("cpuConstraints", SearchCriteria.Op.SC, cpuSearchCriteria); + sc.setParameters("cpuNumber", cpuNumber); } if (memory != null) { - SearchCriteria memoryConstraintSearchCriteria = _srvOfferingJoinDao.createSearchCriteria(); - memoryConstraintSearchCriteria.addAnd("minMemory", Op.LTEQ, memory); - memoryConstraintSearchCriteria.addAnd("maxMemory", Op.GTEQ, memory); - - SearchCriteria memSearchCriteria = _srvOfferingJoinDao.createSearchCriteria(); - memSearchCriteria.addOr("minMemory", Op.NULL); - memSearchCriteria.addOr("memconstraints", Op.SC, memoryConstraintSearchCriteria); - memSearchCriteria.addOr("minMemory", Op.GTEQ, memory); - - sc.addAnd("memoryConstraints", SearchCriteria.Op.SC, memSearchCriteria); + sc.setParameters("memory", memory); } if (cpuSpeed != null) { - SearchCriteria cpuSpeedSearchCriteria = _srvOfferingJoinDao.createSearchCriteria(); - cpuSpeedSearchCriteria.addOr("speed", Op.NULL); - cpuSpeedSearchCriteria.addOr("speed", Op.GTEQ, cpuSpeed); - sc.addAnd("cpuspeedconstraints", SearchCriteria.Op.SC, cpuSpeedSearchCriteria); + sc.setParameters("speedGTEQ", cpuSpeed); } - // Filter offerings that are not associated with caller's domain - // Fetch the offering ids from the details table since theres no smart way to filter them in the join ... yet! if (caller.getType() != Account.Type.ADMIN) { Domain callerDomain = _domainDao.findById(caller.getDomainId()); List domainIds = findRelatedDomainIds(callerDomain, isRecursive); List ids = _srvOfferingDetailsDao.findOfferingIdsByDomainIds(domainIds); - SearchBuilder sb = _srvOfferingJoinDao.createSearchBuilder(); - if (ids != null && !ids.isEmpty()) { - sb.and("id", sb.entity().getId(), Op.IN); - } - sb.or("domainId", sb.entity().getDomainId(), Op.NULL); - sb.done(); - SearchCriteria scc = sb.create(); - if (ids != null && !ids.isEmpty()) { - scc.setParameters("id", ids.toArray()); - } - sc.addAnd("domainId", SearchCriteria.Op.SC, scc); + sc.setJoinParameters("domainDetailSearchNormalUser", "domainIdIN", domainIds.toArray()); } List hostTags = getHostTagsFromTemplateForServiceOfferingsListing(caller, templateId); if (currentVmOffering != null) { - DiskOfferingVO diskOffering = _diskOfferingDao.findByIdIncludingRemoved(currentVmOffering.getDiskOfferingId()); - List storageTags = com.cloud.utils.StringUtils.csvTagsToList(diskOffering.getTags()); - if (!storageTags.isEmpty() && VolumeApiServiceImpl.MatchStoragePoolTagsWithDiskOffering.value()) { - SearchBuilder sb = _srvOfferingJoinDao.createSearchBuilder(); - for(String tag : storageTags) { - sb.and(tag, sb.entity().getTags(), Op.FIND_IN_SET); - } - sb.done(); - SearchCriteria scc = sb.create(); - for(String tag : storageTags) { - scc.setParameters(tag, tag); + if (diskOffering != null) { + List storageTags = com.cloud.utils.StringUtils.csvTagsToList(diskOffering.getTags()); + if (!storageTags.isEmpty() && VolumeApiServiceImpl.MatchStoragePoolTagsWithDiskOffering.value()) { + for(String tag : storageTags) { + sc.setJoinParameters("diskOfferingSearch", tag, tag); + } } - sc.addAnd("storageTags", SearchCriteria.Op.SC, scc); } List offeringHostTags = com.cloud.utils.StringUtils.csvTagsToList(currentVmOffering.getHostTag()); @@ -3501,7 +3928,10 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q sc.addAnd("hostTagsConstraint", SearchCriteria.Op.SC, finalHostTagsSearchCriteria); } - return _srvOfferingJoinDao.searchAndCount(sc, searchFilter); + Pair, Integer> uniquePair = _srvOfferingDao.searchAndCount(sc, searchFilter); + Integer count = uniquePair.second(); + List offeringIds = uniquePair.first().stream().map(ServiceOfferingVO::getId).collect(Collectors.toList()); + return new Pair<>(offeringIds, count); } @Override diff --git a/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java b/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java index 5606e37041a..e30d21ab8ec 100644 --- a/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java +++ b/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java @@ -536,11 +536,7 @@ public class ViewResponseHelper { } public static List createAccountResponse(ResponseView view, EnumSet details, AccountJoinVO... accounts) { - List respList = new ArrayList(); - for (AccountJoinVO vt : accounts){ - respList.add(ApiDBUtils.newAccountResponse(view, details, vt)); - } - return respList; + return ApiDBUtils.newAccountResponses(view, details, accounts); } public static List createAsyncJobResponse(AsyncJobJoinVO... jobs) { diff --git a/server/src/main/java/com/cloud/api/query/dao/AccountJoinDao.java b/server/src/main/java/com/cloud/api/query/dao/AccountJoinDao.java index 42842f568b4..16bda5b4aa9 100644 --- a/server/src/main/java/com/cloud/api/query/dao/AccountJoinDao.java +++ b/server/src/main/java/com/cloud/api/query/dao/AccountJoinDao.java @@ -17,6 +17,7 @@ package com.cloud.api.query.dao; import java.util.EnumSet; +import java.util.List; import org.apache.cloudstack.api.ApiConstants.DomainDetails; import org.apache.cloudstack.api.ResponseObject.ResponseView; @@ -35,4 +36,5 @@ public interface AccountJoinDao extends GenericDao { void setResourceLimits(AccountJoinVO account, boolean accountIsAdmin, ResourceLimitAndCountResponse response); + List searchByIds(Long... ids); } diff --git a/server/src/main/java/com/cloud/api/query/dao/AccountJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/AccountJoinDaoImpl.java index 790758c627f..2daa411e4fb 100644 --- a/server/src/main/java/com/cloud/api/query/dao/AccountJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/AccountJoinDaoImpl.java @@ -16,11 +16,13 @@ // under the License. package com.cloud.api.query.dao; +import java.util.ArrayList; import java.util.EnumSet; import java.util.List; import javax.inject.Inject; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -46,12 +48,19 @@ import com.cloud.utils.db.SearchCriteria; public class AccountJoinDaoImpl extends GenericDaoBase implements AccountJoinDao { public static final Logger s_logger = Logger.getLogger(AccountJoinDaoImpl.class); + @Inject + private ConfigurationDao configDao; private final SearchBuilder acctIdSearch; + private final SearchBuilder domainSearch; @Inject AccountManager _acctMgr; protected AccountJoinDaoImpl() { + domainSearch = createSearchBuilder(); + domainSearch.and("idIN", domainSearch.entity().getId(), SearchCriteria.Op.IN); + domainSearch.done(); + acctIdSearch = createSearchBuilder(); acctIdSearch.and("id", acctIdSearch.entity().getId(), SearchCriteria.Op.EQ); acctIdSearch.done(); @@ -232,6 +241,50 @@ public class AccountJoinDaoImpl extends GenericDaoBase impl response.setSecondaryStorageAvailable(secondaryStorageAvail); } + @Override + public List searchByIds(Long... accountIds) { + // set detail batch query size + int DETAILS_BATCH_SIZE = 2000; + String batchCfg = configDao.getValue("detail.batch.query.size"); + if (batchCfg != null) { + DETAILS_BATCH_SIZE = Integer.parseInt(batchCfg); + } + + List uvList = new ArrayList<>(); + // query details by batches + int curr_index = 0; + if (accountIds.length > DETAILS_BATCH_SIZE) { + while ((curr_index + DETAILS_BATCH_SIZE) <= accountIds.length) { + Long[] ids = new Long[DETAILS_BATCH_SIZE]; + for (int k = 0, j = curr_index; j < curr_index + DETAILS_BATCH_SIZE; j++, k++) { + ids[k] = accountIds[j]; + } + SearchCriteria sc = domainSearch.create(); + sc.setParameters("idIN", ids); + List accounts = searchIncludingRemoved(sc, null, null, false); + if (accounts != null) { + uvList.addAll(accounts); + } + curr_index += DETAILS_BATCH_SIZE; + } + } + if (curr_index < accountIds.length) { + int batch_size = (accountIds.length - curr_index); + // set the ids value + Long[] ids = new Long[batch_size]; + for (int k = 0, j = curr_index; j < curr_index + batch_size; j++, k++) { + ids[k] = accountIds[j]; + } + SearchCriteria sc = domainSearch.create(); + sc.setParameters("idIN", ids); + List accounts = searchIncludingRemoved(sc, null, null, false); + if (accounts != null) { + uvList.addAll(accounts); + } + } + return uvList; + } + @Override public AccountJoinVO newAccountView(Account acct) { SearchCriteria sc = acctIdSearch.create(); diff --git a/server/src/main/java/com/cloud/api/query/dao/DiskOfferingJoinDao.java b/server/src/main/java/com/cloud/api/query/dao/DiskOfferingJoinDao.java index 3d612c63d38..3b38a562d58 100644 --- a/server/src/main/java/com/cloud/api/query/dao/DiskOfferingJoinDao.java +++ b/server/src/main/java/com/cloud/api/query/dao/DiskOfferingJoinDao.java @@ -33,4 +33,6 @@ public interface DiskOfferingJoinDao extends GenericDao searchByIds(Long... idArray); } diff --git a/server/src/main/java/com/cloud/api/query/dao/DiskOfferingJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/DiskOfferingJoinDaoImpl.java index 9592986151f..5341b3b56d7 100644 --- a/server/src/main/java/com/cloud/api/query/dao/DiskOfferingJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/DiskOfferingJoinDaoImpl.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.api.query.dao; +import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -26,6 +27,7 @@ import org.apache.cloudstack.annotation.dao.AnnotationDao; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.response.DiskOfferingResponse; import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -51,9 +53,12 @@ public class DiskOfferingJoinDaoImpl extends GenericDaoBase dofIdSearch; + private SearchBuilder diskOfferingSearch; private final Attribute _typeAttr; protected DiskOfferingJoinDaoImpl() { @@ -62,6 +67,11 @@ public class DiskOfferingJoinDaoImpl extends GenericDaoBase searchByIds(Long... offeringIds) { + // set detail batch query size + int DETAILS_BATCH_SIZE = 2000; + String batchCfg = configDao.getValue("detail.batch.query.size"); + if (batchCfg != null) { + DETAILS_BATCH_SIZE = Integer.parseInt(batchCfg); + } + + List uvList = new ArrayList<>(); + // query details by batches + int curr_index = 0; + if (offeringIds.length > DETAILS_BATCH_SIZE) { + while ((curr_index + DETAILS_BATCH_SIZE) <= offeringIds.length) { + Long[] ids = new Long[DETAILS_BATCH_SIZE]; + for (int k = 0, j = curr_index; j < curr_index + DETAILS_BATCH_SIZE; j++, k++) { + ids[k] = offeringIds[j]; + } + SearchCriteria sc = diskOfferingSearch.create(); + sc.setParameters("idIN", ids); + List accounts = searchIncludingRemoved(sc, null, null, false); + if (accounts != null) { + uvList.addAll(accounts); + } + curr_index += DETAILS_BATCH_SIZE; + } + } + if (curr_index < offeringIds.length) { + int batch_size = (offeringIds.length - curr_index); + // set the ids value + Long[] ids = new Long[batch_size]; + for (int k = 0, j = curr_index; j < curr_index + batch_size; j++, k++) { + ids[k] = offeringIds[j]; + } + SearchCriteria sc = diskOfferingSearch.create(); + sc.setParameters("idIN", ids); + List accounts = searchIncludingRemoved(sc, null, null, false); + if (accounts != null) { + uvList.addAll(accounts); + } + } + return uvList; + } } diff --git a/server/src/main/java/com/cloud/api/query/dao/DomainJoinDao.java b/server/src/main/java/com/cloud/api/query/dao/DomainJoinDao.java index 94efc28d2d4..c7960fd1110 100644 --- a/server/src/main/java/com/cloud/api/query/dao/DomainJoinDao.java +++ b/server/src/main/java/com/cloud/api/query/dao/DomainJoinDao.java @@ -17,6 +17,7 @@ package com.cloud.api.query.dao; import java.util.EnumSet; +import java.util.List; import org.apache.cloudstack.api.ApiConstants.DomainDetails; import org.apache.cloudstack.api.ResponseObject.ResponseView; @@ -35,4 +36,5 @@ public interface DomainJoinDao extends GenericDao { void setResourceLimits(DomainJoinVO domain, boolean isRootDomain, ResourceLimitAndCountResponse response); + List searchByIds(Long... domainIds); } diff --git a/server/src/main/java/com/cloud/api/query/dao/DomainJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/DomainJoinDaoImpl.java index 56f5417da3f..24200fa84f9 100644 --- a/server/src/main/java/com/cloud/api/query/dao/DomainJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/DomainJoinDaoImpl.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.api.query.dao; +import java.util.ArrayList; import java.util.EnumSet; import java.util.List; @@ -29,6 +30,7 @@ import org.apache.cloudstack.api.ResponseObject.ResponseView; import org.apache.cloudstack.api.response.DomainResponse; import org.apache.cloudstack.api.response.ResourceLimitAndCountResponse; import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -47,10 +49,13 @@ public class DomainJoinDaoImpl extends GenericDaoBase implem public static final Logger s_logger = Logger.getLogger(DomainJoinDaoImpl.class); private SearchBuilder domainIdSearch; + private SearchBuilder domainSearch; @Inject private AnnotationDao annotationDao; @Inject + private ConfigurationDao configDao; + @Inject private AccountManager accountManager; protected DomainJoinDaoImpl() { @@ -59,6 +64,10 @@ public class DomainJoinDaoImpl extends GenericDaoBase implem domainIdSearch.and("id", domainIdSearch.entity().getId(), SearchCriteria.Op.EQ); domainIdSearch.done(); + domainSearch = createSearchBuilder(); + domainSearch.and("idIN", domainSearch.entity().getId(), SearchCriteria.Op.IN); + domainSearch.done(); + this._count = "select count(distinct id) from domain_view WHERE "; } @@ -207,6 +216,50 @@ public class DomainJoinDaoImpl extends GenericDaoBase implem response.setSecondaryStorageAvailable(secondaryStorageAvail); } + @Override + public List searchByIds(Long... domainIds) { + // set detail batch query size + int DETAILS_BATCH_SIZE = 2000; + String batchCfg = configDao.getValue("detail.batch.query.size"); + if (batchCfg != null) { + DETAILS_BATCH_SIZE = Integer.parseInt(batchCfg); + } + + List uvList = new ArrayList<>(); + // query details by batches + int curr_index = 0; + if (domainIds.length > DETAILS_BATCH_SIZE) { + while ((curr_index + DETAILS_BATCH_SIZE) <= domainIds.length) { + Long[] ids = new Long[DETAILS_BATCH_SIZE]; + for (int k = 0, j = curr_index; j < curr_index + DETAILS_BATCH_SIZE; j++, k++) { + ids[k] = domainIds[j]; + } + SearchCriteria sc = domainSearch.create(); + sc.setParameters("idIN", ids); + List domains = searchIncludingRemoved(sc, null, null, false); + if (domains != null) { + uvList.addAll(domains); + } + curr_index += DETAILS_BATCH_SIZE; + } + } + if (curr_index < domainIds.length) { + int batch_size = (domainIds.length - curr_index); + // set the ids value + Long[] ids = new Long[batch_size]; + for (int k = 0, j = curr_index; j < curr_index + batch_size; j++, k++) { + ids[k] = domainIds[j]; + } + SearchCriteria sc = domainSearch.create(); + sc.setParameters("idIN", ids); + List domains = searchIncludingRemoved(sc, null, null, false); + if (domains != null) { + uvList.addAll(domains); + } + } + return uvList; + } + @Override public DomainJoinVO newDomainView(Domain domain) { SearchCriteria sc = domainIdSearch.create(); diff --git a/server/src/main/java/com/cloud/api/query/dao/ServiceOfferingJoinDao.java b/server/src/main/java/com/cloud/api/query/dao/ServiceOfferingJoinDao.java index 94cc943a812..4f535a25184 100644 --- a/server/src/main/java/com/cloud/api/query/dao/ServiceOfferingJoinDao.java +++ b/server/src/main/java/com/cloud/api/query/dao/ServiceOfferingJoinDao.java @@ -31,4 +31,6 @@ public interface ServiceOfferingJoinDao extends GenericDao searchByIds(Long... id); } diff --git a/server/src/main/java/com/cloud/api/query/dao/ServiceOfferingJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/ServiceOfferingJoinDaoImpl.java index 1cad19e40a2..bac7ff8b934 100644 --- a/server/src/main/java/com/cloud/api/query/dao/ServiceOfferingJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/ServiceOfferingJoinDaoImpl.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.api.query.dao; +import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -28,6 +29,7 @@ import com.cloud.storage.DiskOfferingVO; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.response.ServiceOfferingResponse; import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -50,10 +52,14 @@ public class ServiceOfferingJoinDaoImpl extends GenericDaoBase sofIdSearch; + private SearchBuilder srvOfferingSearch; + /** * Constant used to convert GB into Bytes (or the other way around). * GB * MB * KB = Bytes // @@ -67,6 +73,10 @@ public class ServiceOfferingJoinDaoImpl extends GenericDaoBase searchByIds(Long... offeringIds) { + // set detail batch query size + int DETAILS_BATCH_SIZE = 2000; + String batchCfg = configDao.getValue("detail.batch.query.size"); + if (batchCfg != null) { + DETAILS_BATCH_SIZE = Integer.parseInt(batchCfg); + } + + List uvList = new ArrayList<>(); + // query details by batches + int curr_index = 0; + if (offeringIds.length > DETAILS_BATCH_SIZE) { + while ((curr_index + DETAILS_BATCH_SIZE) <= offeringIds.length) { + Long[] ids = new Long[DETAILS_BATCH_SIZE]; + for (int k = 0, j = curr_index; j < curr_index + DETAILS_BATCH_SIZE; j++, k++) { + ids[k] = offeringIds[j]; + } + SearchCriteria sc = srvOfferingSearch.create(); + sc.setParameters("idIN", ids); + List accounts = searchIncludingRemoved(sc, null, null, false); + if (accounts != null) { + uvList.addAll(accounts); + } + curr_index += DETAILS_BATCH_SIZE; + } + } + if (curr_index < offeringIds.length) { + int batch_size = (offeringIds.length - curr_index); + // set the ids value + Long[] ids = new Long[batch_size]; + for (int k = 0, j = curr_index; j < curr_index + batch_size; j++, k++) { + ids[k] = offeringIds[j]; + } + SearchCriteria sc = srvOfferingSearch.create(); + sc.setParameters("idIN", ids); + List accounts = searchIncludingRemoved(sc, null, null, false); + if (accounts != null) { + uvList.addAll(accounts); + } + } + return uvList; + } } diff --git a/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDao.java b/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDao.java index 87659210ad7..55c028044f7 100644 --- a/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDao.java +++ b/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDao.java @@ -18,10 +18,6 @@ package com.cloud.api.query.dao; import java.util.List; -import com.cloud.storage.ScopeType; -import com.cloud.storage.StoragePoolStatus; -import com.cloud.utils.Pair; -import com.cloud.utils.db.Filter; import org.apache.cloudstack.api.response.StoragePoolResponse; import com.cloud.api.query.vo.StoragePoolJoinVO; @@ -42,6 +38,4 @@ public interface StoragePoolJoinDao extends GenericDao List searchByIds(Long... spIds); - Pair, Integer> searchAndCount(Long storagePoolId, String storagePoolName, Long zoneId, String path, Long podId, Long clusterId, String address, ScopeType scopeType, StoragePoolStatus status, String keyword, Filter searchFilter); - } diff --git a/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java index 6eb635039ba..b24723021af 100644 --- a/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java @@ -40,15 +40,11 @@ import com.cloud.api.ApiDBUtils; import com.cloud.api.query.vo.StoragePoolJoinVO; import com.cloud.capacity.CapacityManager; import com.cloud.storage.DataStoreRole; -import com.cloud.storage.ScopeType; import com.cloud.storage.Storage; import com.cloud.storage.StoragePool; -import com.cloud.storage.StoragePoolStatus; import com.cloud.storage.StorageStats; import com.cloud.user.AccountManager; -import com.cloud.utils.Pair; import com.cloud.utils.StringUtils; -import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; @@ -296,76 +292,4 @@ public class StoragePoolJoinDaoImpl extends GenericDaoBase, Integer> searchAndCount(Long storagePoolId, String storagePoolName, Long zoneId, String path, Long podId, Long clusterId, String address, ScopeType scopeType, StoragePoolStatus status, String keyword, Filter searchFilter) { - SearchCriteria sc = createStoragePoolSearchCriteria(storagePoolId, storagePoolName, zoneId, path, podId, clusterId, address, scopeType, status, keyword); - return searchAndCount(sc, searchFilter); - } - - private SearchCriteria createStoragePoolSearchCriteria(Long storagePoolId, String storagePoolName, Long zoneId, String path, Long podId, Long clusterId, String address, ScopeType scopeType, StoragePoolStatus status, String keyword) { - SearchBuilder sb = createSearchBuilder(); - sb.select(null, SearchCriteria.Func.DISTINCT, sb.entity().getId()); // select distinct - // ids - sb.and("id", sb.entity().getId(), SearchCriteria.Op.EQ); - sb.and("name", sb.entity().getName(), SearchCriteria.Op.EQ); - sb.and("path", sb.entity().getPath(), SearchCriteria.Op.EQ); - sb.and("dataCenterId", sb.entity().getZoneId(), SearchCriteria.Op.EQ); - sb.and("podId", sb.entity().getPodId(), SearchCriteria.Op.EQ); - sb.and("clusterId", sb.entity().getClusterId(), SearchCriteria.Op.EQ); - sb.and("hostAddress", sb.entity().getHostAddress(), SearchCriteria.Op.EQ); - sb.and("scope", sb.entity().getScope(), SearchCriteria.Op.EQ); - sb.and("status", sb.entity().getStatus(), SearchCriteria.Op.EQ); - sb.and("parent", sb.entity().getParent(), SearchCriteria.Op.EQ); - - SearchCriteria sc = sb.create(); - - if (keyword != null) { - SearchCriteria ssc = createSearchCriteria(); - ssc.addOr("name", SearchCriteria.Op.LIKE, "%" + keyword + "%"); - ssc.addOr("poolType", SearchCriteria.Op.LIKE, new Storage.StoragePoolType("%" + keyword + "%")); - - sc.addAnd("name", SearchCriteria.Op.SC, ssc); - } - - if (storagePoolId != null) { - sc.setParameters("id", storagePoolId); - } - - if (storagePoolName != null) { - sc.setParameters("name", storagePoolName); - } - - if (path != null) { - sc.setParameters("path", path); - } - if (zoneId != null) { - sc.setParameters("dataCenterId", zoneId); - } - if (podId != null) { - SearchCriteria ssc = createSearchCriteria(); - ssc.addOr("podId", SearchCriteria.Op.EQ, podId); - ssc.addOr("podId", SearchCriteria.Op.NULL); - - sc.addAnd("podId", SearchCriteria.Op.SC, ssc); - } - if (address != null) { - sc.setParameters("hostAddress", address); - } - if (clusterId != null) { - SearchCriteria ssc = createSearchCriteria(); - ssc.addOr("clusterId", SearchCriteria.Op.EQ, clusterId); - ssc.addOr("clusterId", SearchCriteria.Op.NULL); - - sc.addAnd("clusterId", SearchCriteria.Op.SC, ssc); - } - if (scopeType != null) { - sc.setParameters("scope", scopeType.toString()); - } - if (status != null) { - sc.setParameters("status", status.toString()); - } - sc.setParameters("parent", 0); - return sc; - } } diff --git a/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDaoImpl.java index f20a9aa2e13..649b20010d3 100644 --- a/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDaoImpl.java @@ -23,6 +23,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import javax.inject.Inject; @@ -171,13 +172,17 @@ public class TemplateJoinDaoImpl extends GenericDaoBaseWithTagInformation storesInZone = dataStoreDao.listStoresByZoneId(template.getDataCenterId()); Long[] storeIds = storesInZone.stream().map(ImageStoreVO::getId).toArray(Long[]::new); List templatesInStore = _templateStoreDao.listByTemplateNotBypassed(template.getId(), storeIds); + + List dataStoreIdList = templatesInStore.stream().map(TemplateDataStoreVO::getDataStoreId).collect(Collectors.toList()); + Map imageStoreMap = dataStoreDao.listByIds(dataStoreIdList).stream().collect(Collectors.toMap(ImageStoreVO::getId, imageStore -> imageStore)); + List> downloadProgressDetails = new ArrayList<>(); HashMap downloadDetailInImageStores = null; for (TemplateDataStoreVO templateInStore : templatesInStore) { downloadDetailInImageStores = new HashMap<>(); - ImageStoreVO datastore = dataStoreDao.findById(templateInStore.getDataStoreId()); - if (datastore != null) { - downloadDetailInImageStores.put("datastore", datastore.getName()); + ImageStoreVO imageStore = imageStoreMap.get(templateInStore.getDataStoreId()); + if (imageStore != null) { + downloadDetailInImageStores.put("datastore", imageStore.getName()); downloadDetailInImageStores.put("downloadPercent", Integer.toString(templateInStore.getDownloadPercent())); downloadDetailInImageStores.put("downloadState", (templateInStore.getDownloadState() != null ? templateInStore.getDownloadState().toString() : "")); downloadProgressDetails.add(downloadDetailInImageStores); diff --git a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java index e4a38ac5165..d6a0ad8c834 100644 --- a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java +++ b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java @@ -7387,7 +7387,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati networkRate = offering.getRateMbps(); } else { // for domain router service offering, get network rate from - if (offering.getSystemVmType() != null && offering.getSystemVmType().equalsIgnoreCase(VirtualMachine.Type.DomainRouter.toString())) { + if (offering.getVmType() != null && offering.getVmType().equalsIgnoreCase(VirtualMachine.Type.DomainRouter.toString())) { networkRate = NetworkOrchestrationService.NetworkThrottlingRate.valueIn(dataCenterId); } else { networkRate = Integer.parseInt(_configDao.getValue(Config.VmNetworkThrottlingRate.key())); diff --git a/server/src/main/java/com/cloud/network/NetworkServiceImpl.java b/server/src/main/java/com/cloud/network/NetworkServiceImpl.java index 896292d6b7d..33670c63b1e 100644 --- a/server/src/main/java/com/cloud/network/NetworkServiceImpl.java +++ b/server/src/main/java/com/cloud/network/NetworkServiceImpl.java @@ -4153,9 +4153,9 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C } final String virtualMachineDomainRouterType = VirtualMachine.Type.DomainRouter.toString(); - if (!virtualMachineDomainRouterType.equalsIgnoreCase(serviceOffering.getSystemVmType())) { + if (!virtualMachineDomainRouterType.equalsIgnoreCase(serviceOffering.getVmType())) { throw new InvalidParameterValueException(String.format("The specified service offering [%s] is of type [%s]. Virtual routers can only be created with service offering " - + "of type [%s].", serviceOffering, serviceOffering.getSystemVmType(), virtualMachineDomainRouterType.toLowerCase())); + + "of type [%s].", serviceOffering, serviceOffering.getVmType(), virtualMachineDomainRouterType.toLowerCase())); } } diff --git a/server/src/main/java/org/apache/cloudstack/acl/RoleManagerImpl.java b/server/src/main/java/org/apache/cloudstack/acl/RoleManagerImpl.java index f2f4a6fe06a..4ff96885a0d 100644 --- a/server/src/main/java/org/apache/cloudstack/acl/RoleManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/acl/RoleManagerImpl.java @@ -94,6 +94,33 @@ public class RoleManagerImpl extends ManagerBase implements RoleService, Configu return RoleService.EnableDynamicApiChecker.value(); } + private boolean isCallerRootAdmin() { + return accountManager.isRootAdmin(getCurrentAccount().getId()); + } + + @Override + public List findRoles(List ids, boolean ignorePrivateRoles) { + List result = new ArrayList<>(); + if (CollectionUtils.isEmpty(ids)) { + logger.trace(String.format("Role IDs are invalid [%s]", ids)); + return result; + } + + List roles = roleDao.searchByIds(ids.toArray(new Long[0])); + if (CollectionUtils.isEmpty(roles)) { + logger.trace(String.format("Roles not found [ids=%s]", ids)); + return result; + } + for (Role role : roles) { + if (!isCallerRootAdmin() && (RoleType.Admin == role.getRoleType() || ignorePrivateRoles)) { + logger.debug(String.format("Role [id=%s, name=%s] is either of 'Admin' type or is private and is only visible to 'Root admins'.", role.getId(), role.getName())); + continue; + } + result.add(role); + } + return result; + } + @Override public Role findRole(Long id) { if (id == null || id < 1L) { diff --git a/server/src/test/java/com/cloud/api/query/QueryManagerImplTest.java b/server/src/test/java/com/cloud/api/query/QueryManagerImplTest.java index a6face0efcc..d739cce419b 100644 --- a/server/src/test/java/com/cloud/api/query/QueryManagerImplTest.java +++ b/server/src/test/java/com/cloud/api/query/QueryManagerImplTest.java @@ -23,6 +23,8 @@ import java.util.ArrayList; import java.util.List; import java.util.UUID; +import com.cloud.event.EventVO; +import com.cloud.event.dao.EventDao; import org.apache.cloudstack.acl.SecurityChecker; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.command.user.event.ListEventsCmd; @@ -73,6 +75,10 @@ public class QueryManagerImplTest { EntityManager entityManager; @Mock AccountManager accountManager; + + @Mock + EventDao eventDao; + @Mock EventJoinDao eventJoinDao; @Mock @@ -102,13 +108,12 @@ public class QueryManagerImplTest { Mockito.any(Project.ListProjectResourcesCriteria.class)); Mockito.doNothing().when(accountManager).buildACLViewSearchCriteria(Mockito.any(), Mockito.anyLong(), Mockito.anyBoolean(), Mockito.anyList(), Mockito.any(Project.ListProjectResourcesCriteria.class)); - final SearchBuilder searchBuilder = Mockito.mock(SearchBuilder.class); - final SearchCriteria searchCriteria = Mockito.mock(SearchCriteria.class); - final EventJoinVO eventJoinVO = Mockito.mock(EventJoinVO.class); - when(searchBuilder.entity()).thenReturn(eventJoinVO); - when(searchBuilder.create()).thenReturn(searchCriteria); - Mockito.when(eventJoinDao.createSearchBuilder()).thenReturn(searchBuilder); - Mockito.when(eventJoinDao.createSearchCriteria()).thenReturn(searchCriteria); + final SearchBuilder eventSearchBuilder = Mockito.mock(SearchBuilder.class); + final SearchCriteria eventSearchCriteria = Mockito.mock(SearchCriteria.class); + final EventVO eventVO = Mockito.mock(EventVO.class); + when(eventSearchBuilder.entity()).thenReturn(eventVO); + when(eventSearchBuilder.create()).thenReturn(eventSearchCriteria); + Mockito.when(eventDao.createSearchBuilder()).thenReturn(eventSearchBuilder); } private ListEventsCmd setupMockListEventsCmd() { @@ -124,19 +129,24 @@ public class QueryManagerImplTest { String uuid = UUID.randomUUID().toString(); Mockito.when(cmd.getResourceId()).thenReturn(uuid); Mockito.when(cmd.getResourceType()).thenReturn(ApiCommandResourceType.Network.toString()); - List events = new ArrayList<>(); - events.add(Mockito.mock(EventJoinVO.class)); - events.add(Mockito.mock(EventJoinVO.class)); - events.add(Mockito.mock(EventJoinVO.class)); - Pair, Integer> pair = new Pair<>(events, events.size()); + List events = new ArrayList<>(); + events.add(Mockito.mock(EventVO.class)); + events.add(Mockito.mock(EventVO.class)); + events.add(Mockito.mock(EventVO.class)); + Pair, Integer> pair = new Pair<>(events, events.size()); + + List eventJoins = new ArrayList<>(); + eventJoins.add(Mockito.mock(EventJoinVO.class)); + eventJoins.add(Mockito.mock(EventJoinVO.class)); + eventJoins.add(Mockito.mock(EventJoinVO.class)); NetworkVO network = Mockito.mock(NetworkVO.class); Mockito.when(network.getId()).thenReturn(1L); Mockito.when(network.getAccountId()).thenReturn(account.getId()); Mockito.when(entityManager.findByUuidIncludingRemoved(Network.class, uuid)).thenReturn(network); Mockito.doNothing().when(accountManager).checkAccess(account, SecurityChecker.AccessType.ListEntry, true, network); - Mockito.when(eventJoinDao.searchAndCount(Mockito.any(), Mockito.any(Filter.class))).thenReturn(pair); + Mockito.when(eventDao.searchAndCount(Mockito.any(), Mockito.any(Filter.class))).thenReturn(pair); List respList = new ArrayList(); - for (EventJoinVO vt : events) { + for (EventJoinVO vt : eventJoins) { respList.add(eventJoinDao.newEventResponse(vt)); } PowerMockito.mockStatic(ViewResponseHelper.class); diff --git a/server/src/test/java/com/cloud/network/NetworkServiceImplTest.java b/server/src/test/java/com/cloud/network/NetworkServiceImplTest.java index fcf7ad3665a..0f4be6579bb 100644 --- a/server/src/test/java/com/cloud/network/NetworkServiceImplTest.java +++ b/server/src/test/java/com/cloud/network/NetworkServiceImplTest.java @@ -753,10 +753,10 @@ public class NetworkServiceImplTest { public void validateIfServiceOfferingIsActiveAndSystemVmTypeIsDomainRouterTestMustThrowInvalidParameterValueExceptionWhenSystemVmTypeIsNotDomainRouter() { doReturn(serviceOfferingVoMock).when(serviceOfferingDaoMock).findById(anyLong()); doReturn(ServiceOffering.State.Active).when(serviceOfferingVoMock).getState(); - doReturn(VirtualMachine.Type.ElasticLoadBalancerVm.toString()).when(serviceOfferingVoMock).getSystemVmType(); + doReturn(VirtualMachine.Type.ElasticLoadBalancerVm.toString()).when(serviceOfferingVoMock).getVmType(); String expectedMessage = String.format("The specified service offering [%s] is of type [%s]. Virtual routers can only be created with service offering of type [%s].", - serviceOfferingVoMock, serviceOfferingVoMock.getSystemVmType(), VirtualMachine.Type.DomainRouter.toString().toLowerCase()); + serviceOfferingVoMock, serviceOfferingVoMock.getVmType(), VirtualMachine.Type.DomainRouter.toString().toLowerCase()); InvalidParameterValueException assertThrows = Assert.assertThrows(expectedException, () -> { service.validateIfServiceOfferingIsActiveAndSystemVmTypeIsDomainRouter(1l); }); diff --git a/server/src/test/java/com/cloud/user/MockUsageEventDao.java b/server/src/test/java/com/cloud/user/MockUsageEventDao.java index 029625d3a8e..c393291476f 100644 --- a/server/src/test/java/com/cloud/user/MockUsageEventDao.java +++ b/server/src/test/java/com/cloud/user/MockUsageEventDao.java @@ -293,6 +293,11 @@ public class MockUsageEventDao implements UsageEventDao{ return null; } + @Override + public List findByUuids(String... uuids) { + return null; + } + @Override public List listLatestEvents(Date endDate) { return null;