mirror of https://github.com/apache/cloudstack.git
Speed up resource count calculation (#425)
* Speed up resource count calculation * server: remove supportedOwner from Resource.ResourceType (#7416) * Refactor resource count calculation * Start transaction for updateCountByDeltaForIds --------- Co-authored-by: GaOrtiga <49285692+GaOrtiga@users.noreply.github.com>
This commit is contained in:
Vishesh
GitHub
parent
26c1741af5
commit
fd9325a86d
|
|
@ -22,29 +22,27 @@ public interface Resource {
|
|||
String UNLIMITED = "Unlimited";
|
||||
|
||||
enum ResourceType { // Primary and Secondary storage are allocated_storage and not the physical storage.
|
||||
user_vm("user_vm", 0, ResourceOwnerType.Account, ResourceOwnerType.Domain),
|
||||
public_ip("public_ip", 1, ResourceOwnerType.Account, ResourceOwnerType.Domain),
|
||||
volume("volume", 2, ResourceOwnerType.Account, ResourceOwnerType.Domain),
|
||||
snapshot("snapshot", 3, ResourceOwnerType.Account, ResourceOwnerType.Domain),
|
||||
template("template", 4, ResourceOwnerType.Account, ResourceOwnerType.Domain),
|
||||
project("project", 5, ResourceOwnerType.Account, ResourceOwnerType.Domain),
|
||||
network("network", 6, ResourceOwnerType.Account, ResourceOwnerType.Domain),
|
||||
vpc("vpc", 7, ResourceOwnerType.Account, ResourceOwnerType.Domain),
|
||||
cpu("cpu", 8, ResourceOwnerType.Account, ResourceOwnerType.Domain),
|
||||
memory("memory", 9, ResourceOwnerType.Account, ResourceOwnerType.Domain),
|
||||
primary_storage("primary_storage", 10, ResourceOwnerType.Account, ResourceOwnerType.Domain),
|
||||
secondary_storage("secondary_storage", 11, ResourceOwnerType.Account, ResourceOwnerType.Domain);
|
||||
user_vm("user_vm", 0),
|
||||
public_ip("public_ip", 1),
|
||||
volume("volume", 2),
|
||||
snapshot("snapshot", 3),
|
||||
template("template", 4),
|
||||
project("project", 5),
|
||||
network("network", 6),
|
||||
vpc("vpc", 7),
|
||||
cpu("cpu", 8),
|
||||
memory("memory", 9),
|
||||
primary_storage("primary_storage", 10),
|
||||
secondary_storage("secondary_storage", 11);
|
||||
|
||||
private String name;
|
||||
private ResourceOwnerType[] supportedOwners;
|
||||
private int ordinal;
|
||||
public static final long bytesToKiB = 1024;
|
||||
public static final long bytesToMiB = bytesToKiB * 1024;
|
||||
public static final long bytesToGiB = bytesToMiB * 1024;
|
||||
|
||||
ResourceType(String name, int ordinal, ResourceOwnerType... supportedOwners) {
|
||||
ResourceType(String name, int ordinal) {
|
||||
this.name = name;
|
||||
this.supportedOwners = supportedOwners;
|
||||
this.ordinal = ordinal;
|
||||
}
|
||||
|
||||
|
|
@ -52,25 +50,6 @@ public interface Resource {
|
|||
return name;
|
||||
}
|
||||
|
||||
public ResourceOwnerType[] getSupportedOwners() {
|
||||
return supportedOwners;
|
||||
}
|
||||
|
||||
public boolean supportsOwner(ResourceOwnerType ownerType) {
|
||||
boolean success = false;
|
||||
if (supportedOwners != null) {
|
||||
int length = supportedOwners.length;
|
||||
for (int i = 0; i < length; i++) {
|
||||
if (supportedOwners[i].getName().equalsIgnoreCase(ownerType.getName())) {
|
||||
success = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return success;
|
||||
}
|
||||
|
||||
public int getOrdinal() {
|
||||
return ordinal;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -49,10 +49,15 @@ public interface ResourceCountDao extends GenericDao<ResourceCountVO, Long> {
|
|||
|
||||
ResourceCountVO findByOwnerAndTypeAndTag(long ownerId, ResourceOwnerType ownerType, ResourceType type, String tag);
|
||||
|
||||
List<ResourceCountVO> findByOwnersAndTypeAndTag(List<Long> ownerIdList, ResourceOwnerType ownerType,
|
||||
ResourceType type, String tag);
|
||||
|
||||
List<ResourceCountVO> listResourceCountByOwnerType(ResourceOwnerType ownerType);
|
||||
|
||||
Set<Long> listAllRowsToUpdate(long ownerId, ResourceOwnerType ownerType, ResourceType type, String tag);
|
||||
|
||||
boolean updateCountByDeltaForIds(List<Long> ids, boolean increment, long delta);
|
||||
|
||||
Set<Long> listRowsToUpdateForDomain(long domainId, ResourceType type, String tag);
|
||||
|
||||
long removeEntriesByOwner(long ownerId, ResourceOwnerType ownerType);
|
||||
|
|
@ -72,4 +77,6 @@ public interface ResourceCountDao extends GenericDao<ResourceCountVO, Long> {
|
|||
long countMemoryAllocatedToAccount(long accountId);
|
||||
|
||||
void removeResourceCountsForNonMatchingTags(Long ownerId, ResourceOwnerType ownerType, List<ResourceType> types, List<String> tags);
|
||||
|
||||
List<ResourceCountVO> lockRows(Set<Long> ids);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ import java.util.Arrays;
|
|||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import javax.inject.Inject;
|
||||
|
|
@ -40,7 +41,6 @@ import com.cloud.configuration.ResourceCountVO;
|
|||
import com.cloud.configuration.ResourceLimit;
|
||||
import com.cloud.domain.DomainVO;
|
||||
import com.cloud.domain.dao.DomainDao;
|
||||
import com.cloud.exception.UnsupportedServiceException;
|
||||
import com.cloud.user.AccountVO;
|
||||
import com.cloud.user.ResourceLimitService;
|
||||
import com.cloud.user.dao.AccountDao;
|
||||
|
|
@ -57,27 +57,30 @@ public class ResourceCountDaoImpl extends GenericDaoBase<ResourceCountVO, Long>
|
|||
private final SearchBuilder<ResourceCountVO> TypeSearch;
|
||||
private final SearchBuilder<ResourceCountVO> TypeNullTagSearch;
|
||||
private final SearchBuilder<ResourceCountVO> NonMatchingTagsSearch;
|
||||
|
||||
private final SearchBuilder<ResourceCountVO> AccountSearch;
|
||||
private final SearchBuilder<ResourceCountVO> DomainSearch;
|
||||
private final SearchBuilder<ResourceCountVO> IdsSearch;
|
||||
|
||||
@Inject
|
||||
private DomainDao _domainDao;
|
||||
@Inject
|
||||
private AccountDao _accountDao;
|
||||
|
||||
protected static final String INCREMENT_COUNT_BY_IDS_SQL = "UPDATE `cloud`.`resource_count` SET `count` = `count` + ? WHERE `id` IN (?)";
|
||||
protected static final String DECREMENT_COUNT_BY_IDS_SQL = "UPDATE `cloud`.`resource_count` SET `count` = `count` - ? WHERE `id` IN (?)";
|
||||
|
||||
public ResourceCountDaoImpl() {
|
||||
TypeSearch = createSearchBuilder();
|
||||
TypeSearch.and("type", TypeSearch.entity().getType(), SearchCriteria.Op.EQ);
|
||||
TypeSearch.and("accountId", TypeSearch.entity().getAccountId(), SearchCriteria.Op.EQ);
|
||||
TypeSearch.and("domainId", TypeSearch.entity().getDomainId(), SearchCriteria.Op.EQ);
|
||||
TypeSearch.and("accountId", TypeSearch.entity().getAccountId(), SearchCriteria.Op.IN);
|
||||
TypeSearch.and("domainId", TypeSearch.entity().getDomainId(), SearchCriteria.Op.IN);
|
||||
TypeSearch.and("tag", TypeSearch.entity().getTag(), SearchCriteria.Op.EQ);
|
||||
TypeSearch.done();
|
||||
|
||||
TypeNullTagSearch = createSearchBuilder();
|
||||
TypeNullTagSearch.and("type", TypeNullTagSearch.entity().getType(), SearchCriteria.Op.EQ);
|
||||
TypeNullTagSearch.and("accountId", TypeNullTagSearch.entity().getAccountId(), SearchCriteria.Op.EQ);
|
||||
TypeNullTagSearch.and("domainId", TypeNullTagSearch.entity().getDomainId(), SearchCriteria.Op.EQ);
|
||||
TypeNullTagSearch.and("accountId", TypeNullTagSearch.entity().getAccountId(), SearchCriteria.Op.IN);
|
||||
TypeNullTagSearch.and("domainId", TypeNullTagSearch.entity().getDomainId(), SearchCriteria.Op.IN);
|
||||
TypeNullTagSearch.and("tag", TypeNullTagSearch.entity().getTag(), SearchCriteria.Op.NULL);
|
||||
TypeNullTagSearch.done();
|
||||
|
||||
|
|
@ -91,6 +94,10 @@ public class ResourceCountDaoImpl extends GenericDaoBase<ResourceCountVO, Long>
|
|||
|
||||
AccountSearch = createSearchBuilder();
|
||||
DomainSearch = createSearchBuilder();
|
||||
|
||||
IdsSearch = createSearchBuilder();
|
||||
IdsSearch.and("id", IdsSearch.entity().getId(), SearchCriteria.Op.IN);
|
||||
IdsSearch.done();
|
||||
}
|
||||
|
||||
@PostConstruct
|
||||
|
|
@ -110,6 +117,19 @@ public class ResourceCountDaoImpl extends GenericDaoBase<ResourceCountVO, Long>
|
|||
|
||||
@Override
|
||||
public ResourceCountVO findByOwnerAndTypeAndTag(long ownerId, ResourceOwnerType ownerType, ResourceType type, String tag) {
|
||||
List<ResourceCountVO> resourceCounts = findByOwnersAndTypeAndTag(List.of(ownerId), ownerType, type, tag);
|
||||
if (CollectionUtils.isNotEmpty(resourceCounts)) {
|
||||
return resourceCounts.get(0);
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ResourceCountVO> findByOwnersAndTypeAndTag(List<Long> ownerIdList, ResourceOwnerType ownerType, ResourceType type, String tag) {
|
||||
if (CollectionUtils.isEmpty(ownerIdList)) {
|
||||
return new ArrayList<>();
|
||||
}
|
||||
SearchCriteria<ResourceCountVO> sc = tag != null ? TypeSearch.create() : TypeNullTagSearch.create();
|
||||
sc.setParameters("type", type);
|
||||
if (tag != null) {
|
||||
|
|
@ -117,13 +137,13 @@ public class ResourceCountDaoImpl extends GenericDaoBase<ResourceCountVO, Long>
|
|||
}
|
||||
|
||||
if (ownerType == ResourceOwnerType.Account) {
|
||||
sc.setParameters("accountId", ownerId);
|
||||
return findOneIncludingRemovedBy(sc);
|
||||
sc.setParameters("accountId", ownerIdList.toArray());
|
||||
return listIncludingRemovedBy(sc);
|
||||
} else if (ownerType == ResourceOwnerType.Domain) {
|
||||
sc.setParameters("domainId", ownerId);
|
||||
return findOneIncludingRemovedBy(sc);
|
||||
sc.setParameters("domainId", ownerIdList.toArray());
|
||||
return listIncludingRemovedBy(sc);
|
||||
} else {
|
||||
return null;
|
||||
return new ArrayList<>();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -155,6 +175,26 @@ public class ResourceCountDaoImpl extends GenericDaoBase<ResourceCountVO, Long>
|
|||
return update(resourceCountVO.getId(), resourceCountVO);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean updateCountByDeltaForIds(List<Long> ids, boolean increment, long delta) {
|
||||
if (CollectionUtils.isEmpty(ids)) {
|
||||
return false;
|
||||
}
|
||||
String updateSql = increment ? INCREMENT_COUNT_BY_IDS_SQL : DECREMENT_COUNT_BY_IDS_SQL;
|
||||
|
||||
String poolIdsInStr = ids.stream().map(String::valueOf).collect(Collectors.joining(",", "(", ")"));
|
||||
String sql = updateSql.replace("(?)", poolIdsInStr);
|
||||
|
||||
final TransactionLegacy txn = TransactionLegacy.currentTxn();
|
||||
try(PreparedStatement pstmt = txn.prepareStatement(sql);) {
|
||||
pstmt.setLong(1, delta);
|
||||
pstmt.executeUpdate();
|
||||
return true;
|
||||
} catch (SQLException e) {
|
||||
throw new CloudRuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<Long> listRowsToUpdateForDomain(long domainId, ResourceType type, String tag) {
|
||||
Set<Long> rowIds = new HashSet<Long>();
|
||||
|
|
@ -226,9 +266,6 @@ public class ResourceCountDaoImpl extends GenericDaoBase<ResourceCountVO, Long>
|
|||
storageTags = Arrays.asList(ResourceLimitService.ResourceLimitStorageTags.value().split(","));
|
||||
}
|
||||
for (ResourceType resourceType : resourceTypes) {
|
||||
if (!resourceType.supportsOwner(ownerType)) {
|
||||
continue;
|
||||
}
|
||||
ResourceCountVO resourceCountVO = new ResourceCountVO(resourceType, 0, ownerId, ownerType);
|
||||
persist(resourceCountVO);
|
||||
if (ResourceLimitService.HostTagsSupportingTypes.contains(resourceType)) {
|
||||
|
|
@ -278,17 +315,6 @@ public class ResourceCountDaoImpl extends GenericDaoBase<ResourceCountVO, Long>
|
|||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public ResourceCountVO persist(ResourceCountVO resourceCountVO) {
|
||||
ResourceOwnerType ownerType = resourceCountVO.getResourceOwnerType();
|
||||
ResourceType resourceType = resourceCountVO.getType();
|
||||
if (!resourceType.supportsOwner(ownerType)) {
|
||||
throw new UnsupportedServiceException("Resource type " + resourceType + " is not supported for owner of type " + ownerType.getName());
|
||||
}
|
||||
|
||||
return super.persist(resourceCountVO);
|
||||
}
|
||||
|
||||
@Override
|
||||
public long removeEntriesByOwner(long ownerId, ResourceOwnerType ownerType) {
|
||||
SearchCriteria<ResourceCountVO> sc = TypeSearch.create();
|
||||
|
|
@ -360,4 +386,14 @@ public class ResourceCountDaoImpl extends GenericDaoBase<ResourceCountVO, Long>
|
|||
}
|
||||
remove(sc);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ResourceCountVO> lockRows(Set<Long> ids) {
|
||||
if (CollectionUtils.isEmpty(ids)) {
|
||||
return new ArrayList<>();
|
||||
}
|
||||
SearchCriteria<ResourceCountVO> sc = IdsSearch.create();
|
||||
sc.setParameters("id", ids.toArray());
|
||||
return lockRows(sc, null, true);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -31,4 +31,5 @@ public interface ReservationDao extends GenericDao<ReservationVO, Long> {
|
|||
void setResourceId(Resource.ResourceType type, Long resourceId);
|
||||
List<Long> getResourceIds(long accountId, Resource.ResourceType type);
|
||||
List<ReservationVO> getReservationsForAccount(long accountId, Resource.ResourceType type, String tag);
|
||||
void removeByIds(List<Long> reservationIds);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ import com.cloud.utils.db.GenericDaoBase;
|
|||
import com.cloud.utils.db.SearchBuilder;
|
||||
import com.cloud.utils.db.SearchCriteria;
|
||||
import org.apache.cloudstack.user.ResourceReservation;
|
||||
import org.apache.commons.collections.CollectionUtils;
|
||||
import org.apache.log4j.Logger;
|
||||
|
||||
public class ReservationDaoImpl extends GenericDaoBase<ReservationVO, Long> implements ReservationDao {
|
||||
|
|
@ -39,6 +40,7 @@ public class ReservationDaoImpl extends GenericDaoBase<ReservationVO, Long> impl
|
|||
private static final String RESOURCE_ID = "resourceId";
|
||||
private static final String ACCOUNT_ID = "accountId";
|
||||
private static final String DOMAIN_ID = "domainId";
|
||||
private static final String IDS = "ids";
|
||||
private final SearchBuilder<ReservationVO> listResourceByAccountAndTypeSearch;
|
||||
private final SearchBuilder<ReservationVO> listAccountAndTypeSearch;
|
||||
private final SearchBuilder<ReservationVO> listAccountAndTypeAndNoTagSearch;
|
||||
|
|
@ -46,6 +48,7 @@ public class ReservationDaoImpl extends GenericDaoBase<ReservationVO, Long> impl
|
|||
private final SearchBuilder<ReservationVO> listDomainAndTypeSearch;
|
||||
private final SearchBuilder<ReservationVO> listDomainAndTypeAndNoTagSearch;
|
||||
private final SearchBuilder<ReservationVO> listResourceByAccountAndTypeAndNoTagSearch;
|
||||
private final SearchBuilder<ReservationVO> listIdsSearch;
|
||||
|
||||
public ReservationDaoImpl() {
|
||||
|
||||
|
|
@ -86,6 +89,10 @@ public class ReservationDaoImpl extends GenericDaoBase<ReservationVO, Long> impl
|
|||
listDomainAndTypeAndNoTagSearch.and(RESOURCE_TYPE, listDomainAndTypeAndNoTagSearch.entity().getResourceType(), SearchCriteria.Op.EQ);
|
||||
listDomainAndTypeAndNoTagSearch.and(RESOURCE_TAG, listDomainAndTypeAndNoTagSearch.entity().getTag(), SearchCriteria.Op.NULL);
|
||||
listDomainAndTypeAndNoTagSearch.done();
|
||||
|
||||
listIdsSearch = createSearchBuilder();
|
||||
listIdsSearch.and(IDS, listIdsSearch.entity().getId(), SearchCriteria.Op.IN);
|
||||
listIdsSearch.done();
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -160,4 +167,13 @@ public class ReservationDaoImpl extends GenericDaoBase<ReservationVO, Long> impl
|
|||
}
|
||||
return listBy(sc);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void removeByIds(List<Long> reservationIds) {
|
||||
if (CollectionUtils.isNotEmpty(reservationIds)) {
|
||||
SearchCriteria<ReservationVO> sc = listIdsSearch.create();
|
||||
sc.setParameters(IDS, reservationIds.toArray());
|
||||
remove(sc);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ import static com.cloud.utils.NumbersUtil.toHumanReadableSize;
|
|||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
|
@ -205,17 +206,12 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim
|
|||
|
||||
@SuppressWarnings("unchecked")
|
||||
protected void removeResourceReservationIfNeededAndIncrementResourceCount(final long accountId, final ResourceType type, String tag, final long numToIncrement) {
|
||||
Object obj = CallContext.current().getContextParameter(CheckedReservation.getResourceReservationContextParameterKey(type));
|
||||
List<Long> reservationIds = (List<Long>)obj; // This complains an unchecked casting warning
|
||||
Transaction.execute(new TransactionCallbackWithExceptionNoReturn<CloudRuntimeException>() {
|
||||
@Override
|
||||
public void doInTransactionWithoutResult(TransactionStatus status) throws CloudRuntimeException {
|
||||
|
||||
Object obj = CallContext.current().getContextParameter(CheckedReservation.getResourceReservationContextParameterKey(type));
|
||||
if (obj instanceof List) {
|
||||
List<Long> reservationIds = (List<Long>)obj; // This complains an unchecked casting warning
|
||||
for (Long reservationId : reservationIds) {
|
||||
reservationDao.remove(reservationId);
|
||||
}
|
||||
}
|
||||
reservationDao.removeByIds(reservationIds);
|
||||
if (!updateResourceCountForAccount(accountId, type, tag, true, numToIncrement)) {
|
||||
// we should fail the operation (resource creation) when failed to update the resource count
|
||||
throw new CloudRuntimeException("Failed to increment resource count of type " + type + " for account id=" + accountId);
|
||||
|
|
@ -581,13 +577,6 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim
|
|||
return _resourceCountDao.lockRows(sc, null, true);
|
||||
}
|
||||
|
||||
private List<ResourceCountVO> lockDomainRows(long domainId, final ResourceType type, String tag) {
|
||||
Set<Long> rowIdsToLock = _resourceCountDao.listAllRowsToUpdate(domainId, ResourceOwnerType.Domain, type, tag);
|
||||
SearchCriteria<ResourceCountVO> sc = ResourceCountSearch.create();
|
||||
sc.setParameters("id", rowIdsToLock.toArray());
|
||||
return _resourceCountDao.lockRows(sc, null, true);
|
||||
}
|
||||
|
||||
@Override
|
||||
public long findDefaultResourceLimitForDomain(ResourceType resourceType) {
|
||||
Long resourceLimit = null;
|
||||
|
|
@ -615,13 +604,11 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim
|
|||
}
|
||||
|
||||
@Override
|
||||
@DB
|
||||
public void checkResourceLimit(final Account account, final ResourceType type, long... count) throws ResourceAllocationException {
|
||||
checkResourceLimitWithTag(account, type, null, count);
|
||||
}
|
||||
|
||||
@Override
|
||||
@DB
|
||||
public void checkResourceLimitWithTag(final Account account, final ResourceType type, String tag, long... count) throws ResourceAllocationException {
|
||||
final long numResources = ((count.length == 0) ? 1 : count[0]);
|
||||
Project project = null;
|
||||
|
|
@ -802,7 +789,7 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim
|
|||
if (isAccount) {
|
||||
if (accountLimitStr.size() < resourceTypes.length) {
|
||||
for (ResourceType rt : resourceTypes) {
|
||||
if (!accountLimitStr.contains(rt.toString()) && rt.supportsOwner(ResourceOwnerType.Account)) {
|
||||
if (!accountLimitStr.contains(rt.toString())) {
|
||||
limits.add(new ResourceLimitVO(rt, findCorrectResourceLimitForAccount(_accountMgr.getAccount(accountId), rt, null), accountId, ResourceOwnerType.Account));
|
||||
}
|
||||
}
|
||||
|
|
@ -810,7 +797,7 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim
|
|||
} else {
|
||||
if (domainLimitStr.size() < resourceTypes.length) {
|
||||
for (ResourceType rt : resourceTypes) {
|
||||
if (!domainLimitStr.contains(rt.toString()) && rt.supportsOwner(ResourceOwnerType.Domain)) {
|
||||
if (!domainLimitStr.contains(rt.toString())) {
|
||||
limits.add(new ResourceLimitVO(rt, findCorrectResourceLimitForDomain(_domainDao.findById(domainId), rt, null), domainId, ResourceOwnerType.Domain));
|
||||
}
|
||||
}
|
||||
|
|
@ -1104,20 +1091,16 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim
|
|||
accountId != null ? ResourceOwnerType.Account : ResourceOwnerType.Domain, hostTags, storageTags);
|
||||
for (ResourceType type : resourceTypes) {
|
||||
if (accountId != null) {
|
||||
if (type.supportsOwner(ResourceOwnerType.Account)) {
|
||||
count = recalculateAccountResourceCount(accountId, type, tag);
|
||||
counts.add(new ResourceCountVO(type, count, accountId, ResourceOwnerType.Account));
|
||||
if (StringUtils.isEmpty(tag)) {
|
||||
counts.addAll(recalculateAccountTaggedResourceCount(accountId, type, hostTags, storageTags));
|
||||
}
|
||||
count = recalculateAccountResourceCount(accountId, type, tag);
|
||||
counts.add(new ResourceCountVO(type, count, accountId, ResourceOwnerType.Account));
|
||||
if (StringUtils.isEmpty(tag)) {
|
||||
counts.addAll(recalculateAccountTaggedResourceCount(accountId, type, hostTags, storageTags));
|
||||
}
|
||||
} else {
|
||||
if (type.supportsOwner(ResourceOwnerType.Domain)) {
|
||||
count = recalculateDomainResourceCount(domainId, type, tag);
|
||||
counts.add(new ResourceCountVO(type, count, domainId, ResourceOwnerType.Domain));
|
||||
if (StringUtils.isEmpty(tag)) {
|
||||
counts.addAll(recalculateDomainTaggedResourceCount(domainId, type, hostTags, storageTags));
|
||||
}
|
||||
count = recalculateDomainResourceCount(domainId, type, tag);
|
||||
counts.add(new ResourceCountVO(type, count, domainId, ResourceOwnerType.Domain));
|
||||
if (StringUtils.isEmpty(tag)) {
|
||||
counts.addAll(recalculateDomainTaggedResourceCount(domainId, type, hostTags, storageTags));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1130,7 +1113,6 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim
|
|||
return recalculateResourceCount(accountId, domainId, typeId, null);
|
||||
}
|
||||
|
||||
@DB
|
||||
protected boolean updateResourceCountForAccount(final long accountId, final ResourceType type, String tag, final boolean increment, final long delta) {
|
||||
if (s_logger.isDebugEnabled()) {
|
||||
String convertedDelta = String.valueOf(delta);
|
||||
|
|
@ -1140,25 +1122,9 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim
|
|||
String typeStr = StringUtils.isNotEmpty(tag) ? String.format("%s (tag: %s)", type, tag) : type.getName();
|
||||
s_logger.debug("Updating resource Type = " + typeStr + " count for Account = " + accountId + " Operation = " + (increment ? "increasing" : "decreasing") + " Amount = " + convertedDelta);
|
||||
}
|
||||
try {
|
||||
return Transaction.execute(new TransactionCallback<Boolean>() {
|
||||
@Override
|
||||
public Boolean doInTransaction(TransactionStatus status) {
|
||||
boolean result = true;
|
||||
List<ResourceCountVO> rowsToUpdate = lockAccountAndOwnerDomainRows(accountId, type, tag);
|
||||
for (ResourceCountVO rowToUpdate : rowsToUpdate) {
|
||||
if (!_resourceCountDao.updateById(rowToUpdate.getId(), increment, delta)) {
|
||||
s_logger.trace("Unable to update resource count for the row " + rowToUpdate);
|
||||
result = false;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
});
|
||||
} catch (Exception ex) {
|
||||
s_logger.error("Failed to update resource count for account id=" + accountId);
|
||||
return false;
|
||||
}
|
||||
|
||||
Set<Long> rowIdsToUpdate = _resourceCountDao.listAllRowsToUpdate(accountId, ResourceOwnerType.Account, type, tag);
|
||||
return _resourceCountDao.updateCountByDeltaForIds(new ArrayList<>(rowIdsToUpdate), increment, delta);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -1169,47 +1135,65 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim
|
|||
* @param type the resource type to do the recalculation for
|
||||
* @return the resulting new resource count
|
||||
*/
|
||||
@DB
|
||||
protected long recalculateDomainResourceCount(final long domainId, final ResourceType type, String tag) {
|
||||
return Transaction.execute(new TransactionCallback<Long>() {
|
||||
@Override
|
||||
public Long doInTransaction(TransactionStatus status) {
|
||||
long newResourceCount = 0;
|
||||
lockDomainRows(domainId, type, tag);
|
||||
ResourceCountVO domainRC = _resourceCountDao.findByOwnerAndTypeAndTag(domainId, ResourceOwnerType.Domain, type, tag);
|
||||
long oldResourceCount = domainRC.getCount();
|
||||
List<AccountVO> accounts = _accountDao.findActiveAccountsForDomain(domainId);
|
||||
List<DomainVO> childDomains = _domainDao.findImmediateChildrenForParent(domainId);
|
||||
|
||||
List<DomainVO> domainChildren = _domainDao.findImmediateChildrenForParent(domainId);
|
||||
// for each child domain update the resource count
|
||||
if (type.supportsOwner(ResourceOwnerType.Domain)) {
|
||||
|
||||
// calculate project count here
|
||||
if (type == ResourceType.project) {
|
||||
newResourceCount += _projectDao.countProjectsForDomain(domainId);
|
||||
}
|
||||
|
||||
for (DomainVO childDomain : domainChildren) {
|
||||
long childDomainResourceCount = recalculateDomainResourceCount(childDomain.getId(), type, tag);
|
||||
newResourceCount += childDomainResourceCount; // add the child domain count to parent domain count
|
||||
}
|
||||
}
|
||||
|
||||
if (type.supportsOwner(ResourceOwnerType.Account)) {
|
||||
List<AccountVO> accounts = _accountDao.findActiveAccountsForDomain(domainId);
|
||||
for (AccountVO account : accounts) {
|
||||
long accountResourceCount = recalculateAccountResourceCount(account.getId(), type, tag);
|
||||
newResourceCount += accountResourceCount; // add account's resource count to parent domain count
|
||||
}
|
||||
}
|
||||
_resourceCountDao.setResourceCount(domainId, ResourceOwnerType.Domain, type, tag, newResourceCount);
|
||||
|
||||
if (oldResourceCount != newResourceCount) {
|
||||
s_logger.warn("Discrepency in the resource count has been detected " + "(original count = " + oldResourceCount + " correct count = " + newResourceCount + ") for Type = " + type
|
||||
+ " for Domain ID = " + domainId + " is fixed during resource count recalculation.");
|
||||
}
|
||||
|
||||
return newResourceCount;
|
||||
if (CollectionUtils.isNotEmpty(childDomains)) {
|
||||
for (DomainVO childDomain : childDomains) {
|
||||
recalculateDomainResourceCount(childDomain.getId(), type, tag);
|
||||
}
|
||||
}
|
||||
if (CollectionUtils.isNotEmpty(accounts)) {
|
||||
for (AccountVO account : accounts) {
|
||||
recalculateAccountResourceCount(account.getId(), type, tag);
|
||||
}
|
||||
}
|
||||
|
||||
return Transaction.execute((TransactionCallback<Long>) status -> {
|
||||
long newResourceCount = 0L;
|
||||
List<Long> domainIdList = childDomains.stream().map(DomainVO::getId).collect(Collectors.toList());
|
||||
domainIdList.add(domainId);
|
||||
List<Long> accountIdList = accounts.stream().map(AccountVO::getId).collect(Collectors.toList());
|
||||
List<ResourceCountVO> domainRCList = _resourceCountDao.findByOwnersAndTypeAndTag(domainIdList, ResourceOwnerType.Domain, type, tag);
|
||||
List<ResourceCountVO> accountRCList = _resourceCountDao.findByOwnersAndTypeAndTag(accountIdList, ResourceOwnerType.Account, type, tag);
|
||||
|
||||
Set<Long> rowIdsToLock = new HashSet<>();
|
||||
if (domainRCList != null) {
|
||||
rowIdsToLock.addAll(domainRCList.stream().map(ResourceCountVO::getId).collect(Collectors.toList()));
|
||||
}
|
||||
if (accountRCList != null) {
|
||||
rowIdsToLock.addAll(accountRCList.stream().map(ResourceCountVO::getId).collect(Collectors.toList()));
|
||||
}
|
||||
// lock the resource count rows for current domain, immediate child domain & accounts
|
||||
List<ResourceCountVO> resourceCounts = _resourceCountDao.lockRows(rowIdsToLock);
|
||||
|
||||
long oldResourceCount = 0L;
|
||||
ResourceCountVO domainRC = null;
|
||||
|
||||
// calculate project count here
|
||||
if (type == ResourceType.project) {
|
||||
newResourceCount += _projectDao.countProjectsForDomain(domainId);
|
||||
}
|
||||
|
||||
for (ResourceCountVO resourceCount : resourceCounts) {
|
||||
if (resourceCount.getResourceOwnerType() == ResourceOwnerType.Domain && resourceCount.getDomainId() == domainId) {
|
||||
oldResourceCount = resourceCount.getCount();
|
||||
domainRC = resourceCount;
|
||||
} else {
|
||||
newResourceCount += resourceCount.getCount();
|
||||
}
|
||||
}
|
||||
|
||||
if (oldResourceCount != newResourceCount) {
|
||||
domainRC.setCount(newResourceCount);
|
||||
_resourceCountDao.update(domainRC.getId(), domainRC);
|
||||
s_logger.warn("Discrepency in the resource count has been detected "
|
||||
+ "(original count = " + oldResourceCount
|
||||
+ " correct count = "+ newResourceCount + ") for Type = " + type
|
||||
+ " for Domain ID = " + domainId + " is fixed during resource count recalculation.");
|
||||
}
|
||||
return newResourceCount;
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -1248,16 +1232,10 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim
|
|||
final ResourceCountVO accountRC = _resourceCountDao.findByOwnerAndTypeAndTag(accountId, ResourceOwnerType.Account, type, tag);
|
||||
if (accountRC != null) {
|
||||
oldCount = accountRC.getCount();
|
||||
}
|
||||
|
||||
if (newCount == null || !newCount.equals(oldCount)) {
|
||||
Transaction.execute(new TransactionCallbackNoReturn() {
|
||||
@Override
|
||||
public void doInTransactionWithoutResult(TransactionStatus status) {
|
||||
lockAccountAndOwnerDomainRows(accountId, type, tag);
|
||||
_resourceCountDao.setResourceCount(accountId, ResourceOwnerType.Account, type, tag, (newCount == null) ? 0 : newCount);
|
||||
}
|
||||
});
|
||||
if (newCount == null || !newCount.equals(oldCount)) {
|
||||
accountRC.setCount((newCount == null) ? 0 : newCount);
|
||||
_resourceCountDao.update(accountRC.getId(), accountRC);
|
||||
}
|
||||
}
|
||||
|
||||
// No need to log message for primary and secondary storage because both are recalculating the
|
||||
|
|
@ -1933,24 +1911,20 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim
|
|||
}
|
||||
|
||||
for (ResourceType type : ResourceType.values()) {
|
||||
if (type.supportsOwner(ResourceOwnerType.Domain)) {
|
||||
if (CollectionUtils.isEmpty(domains)) {
|
||||
recalculateDomainResourceCount(Domain.ROOT_DOMAIN, type, null);
|
||||
recalculateDomainTaggedResourceCount(Domain.ROOT_DOMAIN, type, getResourceLimitHostTags(), getResourceLimitStorageTags());
|
||||
} else {
|
||||
for (Domain domain : domains) {
|
||||
recalculateDomainResourceCount(domain.getId(), type, null);
|
||||
recalculateDomainTaggedResourceCount(domain.getId(), type, getResourceLimitHostTags(), getResourceLimitStorageTags());
|
||||
}
|
||||
if (CollectionUtils.isEmpty(domains)) {
|
||||
recalculateDomainResourceCount(Domain.ROOT_DOMAIN, type, null);
|
||||
recalculateDomainTaggedResourceCount(Domain.ROOT_DOMAIN, type, getResourceLimitHostTags(), getResourceLimitStorageTags());
|
||||
} else {
|
||||
for (Domain domain : domains) {
|
||||
recalculateDomainResourceCount(domain.getId(), type, null);
|
||||
recalculateDomainTaggedResourceCount(domain.getId(), type, getResourceLimitHostTags(), getResourceLimitStorageTags());
|
||||
}
|
||||
}
|
||||
|
||||
if (type.supportsOwner(ResourceOwnerType.Account)) {
|
||||
// run through the accounts in the root domain
|
||||
for (AccountVO account : accounts) {
|
||||
recalculateAccountResourceCount(account.getId(), type, null);
|
||||
recalculateAccountTaggedResourceCount(account.getId(), type, getResourceLimitHostTags(), getResourceLimitStorageTags());
|
||||
}
|
||||
// run through the accounts in the root domain
|
||||
for (AccountVO account : accounts) {
|
||||
recalculateAccountResourceCount(account.getId(), type, null);
|
||||
recalculateAccountTaggedResourceCount(account.getId(), type, getResourceLimitHostTags(), getResourceLimitStorageTags());
|
||||
}
|
||||
}
|
||||
s_logger.info("Finished resource counters recalculation periodic task.");
|
||||
|
|
|
|||
|
|
@ -1315,22 +1315,9 @@ public class ConfigurationServerImpl extends ManagerBase implements Configuratio
|
|||
List<ResourceCountVO> domainResourceCount = _resourceCountDao.listResourceCountByOwnerType(ResourceOwnerType.Domain);
|
||||
List<ResourceCountVO> accountResourceCount = _resourceCountDao.listResourceCountByOwnerType(ResourceOwnerType.Account);
|
||||
|
||||
final List<ResourceType> accountSupportedResourceTypes = new ArrayList<ResourceType>();
|
||||
final List<ResourceType> domainSupportedResourceTypes = new ArrayList<ResourceType>();
|
||||
final int expectedCount = resourceTypes.length;
|
||||
|
||||
for (ResourceType resourceType : resourceTypes) {
|
||||
if (resourceType.supportsOwner(ResourceOwnerType.Account)) {
|
||||
accountSupportedResourceTypes.add(resourceType);
|
||||
}
|
||||
if (resourceType.supportsOwner(ResourceOwnerType.Domain)) {
|
||||
domainSupportedResourceTypes.add(resourceType);
|
||||
}
|
||||
}
|
||||
|
||||
final int accountExpectedCount = accountSupportedResourceTypes.size();
|
||||
final int domainExpectedCount = domainSupportedResourceTypes.size();
|
||||
|
||||
if ((domainResourceCount.size() < domainExpectedCount * domains.size())) {
|
||||
if ((domainResourceCount.size() < expectedCount * domains.size())) {
|
||||
s_logger.debug("resource_count table has records missing for some domains...going to insert them");
|
||||
for (final DomainVO domain : domains) {
|
||||
// Lock domain
|
||||
|
|
@ -1344,8 +1331,8 @@ public class ConfigurationServerImpl extends ManagerBase implements Configuratio
|
|||
domainCountStr.add(domainCount.getType().toString());
|
||||
}
|
||||
|
||||
if (domainCountStr.size() < domainExpectedCount) {
|
||||
for (ResourceType resourceType : domainSupportedResourceTypes) {
|
||||
if (domainCountStr.size() < expectedCount) {
|
||||
for (ResourceType resourceType : resourceTypes) {
|
||||
if (!domainCountStr.contains(resourceType.toString())) {
|
||||
ResourceCountVO resourceCountVO = new ResourceCountVO(resourceType, 0, domain.getId(), ResourceOwnerType.Domain);
|
||||
s_logger.debug("Inserting resource count of type " + resourceType + " for domain id=" + domain.getId());
|
||||
|
|
@ -1359,7 +1346,7 @@ public class ConfigurationServerImpl extends ManagerBase implements Configuratio
|
|||
}
|
||||
}
|
||||
|
||||
if ((accountResourceCount.size() < accountExpectedCount * accounts.size())) {
|
||||
if ((accountResourceCount.size() < expectedCount * accounts.size())) {
|
||||
s_logger.debug("resource_count table has records missing for some accounts...going to insert them");
|
||||
for (final AccountVO account : accounts) {
|
||||
// lock account
|
||||
|
|
@ -1373,8 +1360,8 @@ public class ConfigurationServerImpl extends ManagerBase implements Configuratio
|
|||
accountCountStr.add(accountCount.getType().toString());
|
||||
}
|
||||
|
||||
if (accountCountStr.size() < accountExpectedCount) {
|
||||
for (ResourceType resourceType : accountSupportedResourceTypes) {
|
||||
if (accountCountStr.size() < expectedCount) {
|
||||
for (ResourceType resourceType : resourceTypes) {
|
||||
if (!accountCountStr.contains(resourceType.toString())) {
|
||||
ResourceCountVO resourceCountVO = new ResourceCountVO(resourceType, 0, account.getId(), ResourceOwnerType.Account);
|
||||
s_logger.debug("Inserting resource count of type " + resourceType + " for account id=" + account.getId());
|
||||
|
|
|
|||
Loading…
Reference in New Issue