etan39
/
cloudstack
mirror of https://github.com/apache/cloudstack.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #478 from shapeblue/scalability-improvements-fixes 

Following changes and improvements:
- Improvements in handling of PingRoutingCommand
   
    1. Added global config - vm.sync.power.state.transitioning, default value: false, to control syncing of power states for transitioning VMs
    2. Improved VirtualMachinePowerStateSync to allow power state sync for host VMs in a batch
    3. Optimized scanning stalled VMs
- Added caching for account/use role API access with expiration after write set to 60 seconds.
- Added caching for some recurring DB retrievals
   
    1. CapacityManager updateCapacityForHost - beneficial for calculation for host capacities by CapacityChecker, host joins
    2. CapacityManager - listing service offerings - beneficial in host capacity calculation
    3. LibvirtServerDiscoverer existing host for the cluster - beneficial for host joins
    4. DownloadListener - hypervisors for zone - beneficial for host joins
    5. VirtualMachineManagerImpl - VMs in progress- beneficial for processing stalled VMs during PingRoutingCommands
- Optimized MS list retrieval for agent connect 
- Optimize finding ready systemvm template for zone
- Database retrieval optimisations - fix and refactor for cases where only IDs or counts are used mainly for hosts and other infra entities. Also similar cases for VMs and other entities related to host concerning background tasks
- Changes in agent-manager connection with NIO classes
   
    1. Optimize use of executor service
    2. Do SSL handshakes within worker threads
    3. Added global configs to finely control the behaviour depending on the infra
    4. On agent side backoff and sslhandshake timeout can be controlled by agent properties
- Improvements in StatsCollection - minimize DB retrievals.
- Improvements in DeploymentPlanner to retrieve only desired host fields and lesser retrievals.
- Improvements in resource calculation
This commit is contained in:
Abhishek Kumar 2024-10-11 13:33:38 +05:30 committed by GitHub
parent 1d0b90f984 42111d8e83
commit af32bcf4ca
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
84 changed files with 1915 additions and 1140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
agent/conf/agent.properties
View File

@ -418,3 +418,9 @@ iscsi.session.cleanup.enabled=false
# Implicit host tags managed by agent.properties
# host.tags=
# Timeout(in seconds) for SSL handshake when agent connects to server
#ssl.handshake.timeout=
# Wait(in seconds) during agent reconnections
#backoff.seconds=

8
agent/src/main/java/com/cloud/agent/Agent.java
View File

@ -159,7 +159,7 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater
_shell = shell;
_link = null;
_connection = new NioClient("Agent", _shell.getNextHost(), _shell.getPort(), _shell.getWorkers(), this);
_connection = new NioClient("Agent", _shell.getNextHost(), _shell.getPort(), _shell.getWorkers(), _shell.getSslHandshakeTimeout(), this);
Runtime.getRuntime().addShutdownHook(_shutdownThread);
@ -195,7 +195,7 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater
}
final String host = _shell.getNextHost();
_connection = new NioClient("Agent", host, _shell.getPort(), _shell.getWorkers(), this);
_connection = new NioClient("Agent", host, _shell.getPort(), _shell.getWorkers(), _shell.getSslHandshakeTimeout(), this);
// ((NioClient)_connection).setBindAddress(_shell.getPrivateIp());
@ -291,7 +291,7 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater
while (!_connection.isStartup()) {
final String host = _shell.getNextHost();
_shell.getBackoffAlgorithm().waitBeforeRetry();
_connection = new NioClient("Agent", host, _shell.getPort(), _shell.getWorkers(), this);
_connection = new NioClient("Agent", host, _shell.getPort(), _shell.getWorkers(), _shell.getSslHandshakeTimeout(), this);
s_logger.info("Connecting to host:" + host);
try {
_connection.start();
@ -552,7 +552,7 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater
do {
final String host = _shell.getNextHost();
_connection = new NioClient("Agent", host, _shell.getPort(), _shell.getWorkers(), this);
_connection = new NioClient("Agent", host, _shell.getPort(), _shell.getWorkers(), _shell.getSslHandshakeTimeout(), this);
s_logger.info("Reconnecting to host:" + host);
try {
_connection.start();

55
agent/src/main/java/com/cloud/agent/AgentShell.java
View File

@ -16,28 +16,6 @@
// under the License.
package com.cloud.agent;
import com.cloud.agent.Agent.ExitStatus;
import com.cloud.agent.dao.StorageComponent;
import com.cloud.agent.dao.impl.PropertiesStorage;
import com.cloud.agent.properties.AgentProperties;
import com.cloud.agent.properties.AgentPropertiesFileHandler;
import com.cloud.resource.ServerResource;
import com.cloud.utils.LogUtils;
import com.cloud.utils.ProcessUtil;
import com.cloud.utils.PropertiesUtil;
import com.cloud.utils.backoff.BackoffAlgorithm;
import com.cloud.utils.backoff.impl.ConstantTimeBackoff;
import com.cloud.utils.exception.CloudRuntimeException;
import org.apache.commons.daemon.Daemon;
import org.apache.commons.daemon.DaemonContext;
import org.apache.commons.daemon.DaemonInitException;
import org.apache.commons.lang.math.NumberUtils;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
import org.apache.log4j.xml.DOMConfigurator;
import javax.naming.ConfigurationException;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
@ -52,6 +30,30 @@ import java.util.Map;
import java.util.Properties;
import java.util.UUID;
import javax.naming.ConfigurationException;
import org.apache.commons.daemon.Daemon;
import org.apache.commons.daemon.DaemonContext;
import org.apache.commons.daemon.DaemonInitException;
import org.apache.commons.lang.math.NumberUtils;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
import org.apache.log4j.xml.DOMConfigurator;
import com.cloud.agent.Agent.ExitStatus;
import com.cloud.agent.dao.StorageComponent;
import com.cloud.agent.dao.impl.PropertiesStorage;
import com.cloud.agent.properties.AgentProperties;
import com.cloud.agent.properties.AgentPropertiesFileHandler;
import com.cloud.resource.ServerResource;
import com.cloud.utils.LogUtils;
import com.cloud.utils.ProcessUtil;
import com.cloud.utils.PropertiesUtil;
import com.cloud.utils.backoff.BackoffAlgorithm;
import com.cloud.utils.backoff.impl.ConstantTimeBackoff;
import com.cloud.utils.exception.CloudRuntimeException;
public class AgentShell implements IAgentShell, Daemon {
private static final Logger s_logger = Logger.getLogger(AgentShell.class.getName());
@ -405,7 +407,9 @@ public class AgentShell implements IAgentShell, Daemon {
s_logger.info("Defaulting to the constant time backoff algorithm");
_backoff = new ConstantTimeBackoff();
_backoff.configure("ConstantTimeBackoff", new HashMap<String, Object>());
Map<String, Object> map = new HashMap<>();
map.put("seconds", _properties.getProperty("backoff.seconds"));
_backoff.configure("ConstantTimeBackoff", map);
}
private void launchAgent() throws ConfigurationException {
@ -454,6 +458,11 @@ public class AgentShell implements IAgentShell, Daemon {
agent.start();
}
@Override
public Integer getSslHandshakeTimeout() {
return AgentPropertiesFileHandler.getPropertyValue(AgentProperties.SSL_HANDSHAKE_TIMEOUT);
}
public synchronized int getNextAgentId() {
return _nextAgentId++;
}

2
agent/src/main/java/com/cloud/agent/IAgentShell.java
View File

@ -70,4 +70,6 @@ public interface IAgentShell {
String getConnectedHost();
void launchNewAgent(ServerResource resource) throws ConfigurationException;
Integer getSslHandshakeTimeout();
}

7
agent/src/main/java/com/cloud/agent/properties/AgentProperties.java
View File

@ -762,6 +762,13 @@ public class AgentProperties{
*/
public static final Property<String> HOST_TAGS = new Property<>("host.tags", null, String.class);
/**
* Timeout for SSL handshake in seconds
* Data type: Integer.<br>
* Default value: <code>null</code>
*/
public static final Property<Integer> SSL_HANDSHAKE_TIMEOUT = new Property<>("ssl.handshake.timeout", null, Integer.class);
public static class Property <T>{
private String name;
private T defaultValue;

7
api/src/main/java/org/apache/cloudstack/api/command/admin/domain/ListDomainsCmd.java
View File

@ -103,7 +103,7 @@ public class ListDomainsCmd extends BaseListCmd implements UserCmd {
dv = EnumSet.of(DomainDetails.all);
} else {
try {
ArrayList<DomainDetails> dc = new ArrayList<DomainDetails>();
ArrayList<DomainDetails> dc = new ArrayList<>();
for (String detail : viewDetails) {
dc.add(DomainDetails.valueOf(detail));
}
@ -145,7 +145,10 @@ public class ListDomainsCmd extends BaseListCmd implements UserCmd {
if (CollectionUtils.isEmpty(response)) {
return;
}
_resourceLimitService.updateTaggedResourceLimitsAndCountsForDomains(response, getTag());
EnumSet<DomainDetails> details = getDetails();
if (details.contains(DomainDetails.all) || details.contains(DomainDetails.resource)) {
_resourceLimitService.updateTaggedResourceLimitsAndCountsForDomains(response, getTag());
}
if (!getShowIcon()) {
return;
}

5
api/src/main/java/org/apache/cloudstack/api/command/user/account/ListAccountsCmd.java
View File

@ -151,7 +151,10 @@ public class ListAccountsCmd extends BaseListDomainResourcesCmd implements UserC
if (CollectionUtils.isEmpty(response)) {
return;
}
_resourceLimitService.updateTaggedResourceLimitsAndCountsForAccounts(response, getTag());
EnumSet<DomainDetails> details = getDetails();
if (details.contains(DomainDetails.all) || details.contains(DomainDetails.resource)) {
_resourceLimitService.updateTaggedResourceLimitsAndCountsForAccounts(response, getTag());
}
if (!getShowIcon()) {
return;
}

2
api/src/main/java/org/apache/cloudstack/outofbandmanagement/OutOfBandManagementService.java
View File

@ -39,7 +39,7 @@ public interface OutOfBandManagementService {
long getId();
boolean isOutOfBandManagementEnabled(Host host);
void submitBackgroundPowerSyncTask(Host host);
boolean transitionPowerStateToDisabled(List<? extends Host> hosts);
boolean transitionPowerStateToDisabled(List<Long> hostIds);
OutOfBandManagementResponse enableOutOfBandManagement(DataCenter zone);
OutOfBandManagementResponse enableOutOfBandManagement(Cluster cluster);

1
core/src/main/java/com/cloud/agent/api/CheckNetworkCommand.java
View File

@ -29,6 +29,7 @@ public class CheckNetworkCommand extends Command {
public CheckNetworkCommand(List<PhysicalNetworkSetupInfo> networkInfoList) {
this.networkInfoList = networkInfoList;
setWait(120);
}
public List<PhysicalNetworkSetupInfo> getPhysicalNetworkInfoList() {

19
engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java
View File

@ -87,6 +87,9 @@ public interface VirtualMachineManager extends Manager {
ConfigKey<String> MetadataCustomCloudName = new ConfigKey<>("Advanced", String.class, "metadata.custom.cloud.name", "",
"If provided, a custom cloud-name in cloud-init metadata", true, ConfigKey.Scope.Zone);
ConfigKey<Boolean> VmSyncPowerStateTransitioning = new ConfigKey<>("Advanced", Boolean.class, "vm.sync.power.state.transitioning", "false",
"Whether to sync power states of the transitioning and stalled VMs while processing VM power reports.", false);
interface Topics {
String VM_POWER_STATE = "vm.powerstate";
}
@ -272,24 +275,22 @@ public interface VirtualMachineManager extends Manager {
/**
* Obtains statistics for a list of VMs; CPU and network utilization
* @param hostId ID of the host
* @param hostName name of the host
* @param host host
* @param vmIds list of VM IDs
* @return map of VM ID and stats entry for the VM
*/
HashMap<Long, ? extends VmStats> getVirtualMachineStatistics(long hostId, String hostName, List<Long> vmIds);
HashMap<Long, ? extends VmStats> getVirtualMachineStatistics(Host host, List<Long> vmIds);
/**
* Obtains statistics for a list of VMs; CPU and network utilization
* @param hostId ID of the host
* @param hostName name of the host
* @param vmMap map of VM IDs and the corresponding VirtualMachine object
* @param host host
* @param vmMap map of VM instanceName and its ID
* @return map of VM ID and stats entry for the VM
*/
HashMap<Long, ? extends VmStats> getVirtualMachineStatistics(long hostId, String hostName, Map<Long, ? extends VirtualMachine> vmMap);
HashMap<Long, ? extends VmStats> getVirtualMachineStatistics(Host host, Map<String, Long> vmMap);
HashMap<Long, List<? extends VmDiskStats>> getVmDiskStatistics(long hostId, String hostName, Map<Long, ? extends VirtualMachine> vmMap);
HashMap<Long, List<? extends VmDiskStats>> getVmDiskStatistics(Host host, Map<String, Long> vmInstanceNameIdMap);
HashMap<Long, List<? extends VmNetworkStats>> getVmNetworkStatistics(long hostId, String hostName, Map<Long, ? extends VirtualMachine> vmMap);
HashMap<Long, List<? extends VmNetworkStats>> getVmNetworkStatistics(Host host, Map<String, Long> vmInstanceNameIdMap);
Map<Long, Boolean> getDiskOfferingSuitabilityForVm(long vmId, List<Long> diskOfferingIds);

9
engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java
View File

@ -16,14 +16,11 @@
// under the License.
package com.cloud.capacity;
import java.util.Map;
import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
import com.cloud.host.Host;
import com.cloud.offering.ServiceOffering;
import com.cloud.service.ServiceOfferingVO;
import com.cloud.storage.VMTemplateVO;
import com.cloud.utils.Pair;
import com.cloud.vm.VirtualMachine;
@ -133,8 +130,6 @@ public interface CapacityManager {
void updateCapacityForHost(Host host);
void updateCapacityForHost(Host host, Map<Long, ServiceOfferingVO> offeringsMap);
/**
* @param pool storage pool
* @param templateForVmCreation template that will be used for vm creation
@ -151,12 +146,12 @@ public interface CapacityManager {
/**
* Check if specified host has capability to support cpu cores and speed freq
* @param hostId the host to be checked
* @param host the host to be checked
* @param cpuNum cpu number to check
* @param cpuSpeed cpu Speed to check
* @return true if the count of host's running VMs >= hypervisor limit
*/
boolean checkIfHostHasCpuCapability(long hostId, Integer cpuNum, Integer cpuSpeed);
boolean checkIfHostHasCpuCapability(Host host, Integer cpuNum, Integer cpuSpeed);
/**
* Check if cluster will cross threshold if the cpu/memory requested are accommodated

2
engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java
View File

@ -138,7 +138,7 @@ public interface ResourceManager extends ResourceService, Configurable {
public HostVO findHostByName(String name);
HostStats getHostStatistics(long hostId);
HostStats getHostStatistics(Host host);
Long getGuestOSCategoryId(long hostId);

36
engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java
View File

@ -38,9 +38,6 @@ import java.util.concurrent.locks.ReentrantLock;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
import com.cloud.configuration.Config;
import com.cloud.utils.NumbersUtil;
import com.cloud.utils.db.GlobalLock;
import org.apache.cloudstack.agent.lb.IndirectAgentLB;
import org.apache.cloudstack.ca.CAManager;
import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
@ -53,6 +50,7 @@ import org.apache.cloudstack.managed.context.ManagedContextRunnable;
import org.apache.cloudstack.outofbandmanagement.dao.OutOfBandManagementDao;
import org.apache.cloudstack.utils.identity.ManagementServerNode;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
import org.apache.log4j.MDC;
@ -81,6 +79,7 @@ import com.cloud.agent.api.UnsupportedAnswer;
import com.cloud.agent.transport.Request;
import com.cloud.agent.transport.Response;
import com.cloud.alert.AlertManager;
import com.cloud.configuration.Config;
import com.cloud.configuration.ManagementServiceConfiguration;
import com.cloud.dc.ClusterVO;
import com.cloud.dc.DataCenterVO;
@ -104,11 +103,13 @@ import com.cloud.resource.Discoverer;
import com.cloud.resource.ResourceManager;
import com.cloud.resource.ResourceState;
import com.cloud.resource.ServerResource;
import com.cloud.utils.NumbersUtil;
import com.cloud.utils.Pair;
import com.cloud.utils.component.ManagerBase;
import com.cloud.utils.concurrency.NamedThreadFactory;
import com.cloud.utils.db.DB;
import com.cloud.utils.db.EntityManager;
import com.cloud.utils.db.GlobalLock;
import com.cloud.utils.db.QueryBuilder;
import com.cloud.utils.db.SearchCriteria.Op;
import com.cloud.utils.db.TransactionLegacy;
@ -123,7 +124,6 @@ import com.cloud.utils.nio.Link;
import com.cloud.utils.nio.NioServer;
import com.cloud.utils.nio.Task;
import com.cloud.utils.time.InaccurateClock;
import org.apache.commons.lang3.StringUtils;
/**
* Implementation of the Agent Manager. This class controls the connection to the agents.
@ -180,7 +180,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
protected ExecutorService _executor;
protected ThreadPoolExecutor _connectExecutor;
protected ScheduledExecutorService _directAgentExecutor;
protected ThreadPoolExecutor _directAgentExecutor;
protected ScheduledExecutorService _cronJobExecutor;
protected ScheduledExecutorService _monitorExecutor;
@ -197,6 +197,15 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
protected final ConfigKey<Integer> Workers = new ConfigKey<Integer>("Advanced", Integer.class, "workers", "5",
"Number of worker threads handling remote agent connections.", false);
protected final ConfigKey<Integer> Port = new ConfigKey<Integer>("Advanced", Integer.class, "port", "8250", "Port to listen on for remote agent connections.", false);
protected final ConfigKey<Integer> RemoteAgentSslHandshakeTimeout = new ConfigKey<>("Advanced",
Integer.class, "agent.ssl.handshake.timeout", "30",
"Seconds after which SSL handshake times out during remote agent connections.", false);
protected final ConfigKey<Integer> RemoteAgentSslHandshakeMinWorkers = new ConfigKey<>("Advanced",
Integer.class, "agent.ssl.handshake.min.workers", "5",
"Number of minimum worker threads handling SSL handshake with remote agents.", false);
protected final ConfigKey<Integer> RemoteAgentSslHandshakeMaxWorkers = new ConfigKey<>("Advanced",
Integer.class, "agent.ssl.handshake.max.workers", "50",
"Number of maximum worker threads handling SSL handshake with remote agents.", false);
protected final ConfigKey<Integer> AlertWait = new ConfigKey<Integer>("Advanced", Integer.class, "alert.wait", "1800",
"Seconds to wait before alerting on a disconnected agent", true);
protected final ConfigKey<Integer> DirectAgentLoadSize = new ConfigKey<Integer>("Advanced", Integer.class, "direct.agent.load.size", "16",
@ -213,8 +222,6 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
s_logger.info("Ping Timeout is " + mgmtServiceConf.getPingTimeout());
final int threads = DirectAgentLoadSize.value();
_nodeId = ManagementServerNode.getManagementServerId();
s_logger.info("Configuring AgentManagerImpl. management server node id(msid): " + _nodeId);
@ -225,17 +232,20 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
registerForHostEvents(new SetHostParamsListener(), true, true, false);
_executor = new ThreadPoolExecutor(threads, threads, 60l, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(), new NamedThreadFactory("AgentTaskPool"));
final int agentTaskThreads = DirectAgentLoadSize.value();
_executor = new ThreadPoolExecutor(Math.max(agentTaskThreads/10, 1), agentTaskThreads, 60L, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(), new NamedThreadFactory("AgentTaskPool"));
_connectExecutor = new ThreadPoolExecutor(100, 500, 60l, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(), new NamedThreadFactory("AgentConnectTaskPool"));
// allow core threads to time out even when there are no items in the queue
_connectExecutor.allowCoreThreadTimeOut(true);
_connection = new NioServer("AgentManager", Port.value(), Workers.value() + 10, this, caService);
_connection = new NioServer("AgentManager", Port.value(), Workers.value() + 10,
RemoteAgentSslHandshakeMinWorkers.value(), RemoteAgentSslHandshakeMaxWorkers.value(), this,
caService, RemoteAgentSslHandshakeTimeout.value());
s_logger.info("Listening on " + Port.value() + " with " + Workers.value() + " workers");
// executes all agent commands other than cron and ping
_directAgentExecutor = new ScheduledThreadPoolExecutor(DirectAgentPoolSize.value(), new NamedThreadFactory("DirectAgent"));
_directAgentExecutor = new ThreadPoolExecutor(Math.max(agentTaskThreads/10, 1), DirectAgentPoolSize.value(), 120L, TimeUnit.SECONDS, new LinkedBlockingQueue<>(), new NamedThreadFactory("DirectAgent"));
// executes cron and ping agent commands
_cronJobExecutor = new ScheduledThreadPoolExecutor(DirectAgentPoolSize.value(), new NamedThreadFactory("DirectAgentCronJob"));
s_logger.debug("Created DirectAgentAttache pool with size: " + DirectAgentPoolSize.value());
@ -1667,7 +1677,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
}
}
public ScheduledExecutorService getDirectAgentPool() {
public ThreadPoolExecutor getDirectAgentPool() {
return _directAgentExecutor;
}
@ -1838,7 +1848,9 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
@Override
public ConfigKey<?>[] getConfigKeys() {
return new ConfigKey<?>[] { CheckTxnBeforeSending, Workers, Port, Wait, AlertWait, DirectAgentLoadSize,
DirectAgentPoolSize, DirectAgentThreadCap, EnableKVMAutoEnableDisable, ReadyCommandWait };
DirectAgentPoolSize, DirectAgentThreadCap, EnableKVMAutoEnableDisable, ReadyCommandWait,
RemoteAgentSslHandshakeTimeout, RemoteAgentSslHandshakeMinWorkers, RemoteAgentSslHandshakeMaxWorkers
};
}
protected class SetHostParamsListener implements Listener {

17
engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java
View File

@ -250,14 +250,13 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
return new ClusteredAgentHandler(type, link, data);
}
protected AgentAttache createAttache(final long id) {
s_logger.debug("create forwarding ClusteredAgentAttache for " + id);
final HostVO host = _hostDao.findById(id);
final AgentAttache attache = new ClusteredAgentAttache(this, id, host.getName());
protected AgentAttache createAttache(final Host host) {
s_logger.debug("create forwarding ClusteredAgentAttache for " + host.getId());
final AgentAttache attache = new ClusteredAgentAttache(this, host.getId(), host.getName());
AgentAttache old = null;
synchronized (_agents) {
old = _agents.get(id);
_agents.put(id, attache);
old = _agents.get(host.getId());
_agents.put(host.getId(), attache);
}
if (old != null) {
if (s_logger.isDebugEnabled()) {
@ -567,12 +566,12 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
if (s_logger.isDebugEnabled()) {
s_logger.debug("Host " + hostId + " has switched to another management server, need to update agent map with a forwarding agent attache");
}
agent = createAttache(hostId);
agent = createAttache(host);
}
}
if (agent == null) {
final AgentUnavailableException ex = new AgentUnavailableException("Host with specified id is not in the right state: " + host.getStatus(), hostId);
ex.addProxyObject(_entityMgr.findById(Host.class, hostId).getUuid());
ex.addProxyObject(host.getUuid());
throw ex;
}
@ -1169,7 +1168,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
final ClusteredDirectAgentAttache attache = (ClusteredDirectAgentAttache)_agents.get(hostId);
if (attache != null && attache.getQueueSize() == 0 && attache.getNonRecurringListenersSize() == 0) {
handleDisconnectWithoutInvestigation(attache, Event.StartAgentRebalance, true, true);
final ClusteredAgentAttache forwardAttache = (ClusteredAgentAttache)createAttache(hostId);
final ClusteredAgentAttache forwardAttache = (ClusteredAgentAttache)createAttache(host);
if (forwardAttache == null) {
s_logger.warn("Unable to create a forward attache for the host " + hostId + " as a part of rebalance process");
return false;

177
engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java
View File

@ -48,10 +48,6 @@ import javax.inject.Inject;
import javax.naming.ConfigurationException;
import javax.persistence.EntityExistsException;
import com.cloud.configuration.Resource;
import com.cloud.event.ActionEventUtils;
import com.cloud.exception.ResourceAllocationException;
import com.google.gson.Gson;
import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao;
import org.apache.cloudstack.annotation.AnnotationService;
import org.apache.cloudstack.annotation.dao.AnnotationDao;
@ -88,6 +84,7 @@ import org.apache.cloudstack.resource.ResourceCleanupService;
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
import org.apache.cloudstack.storage.to.VolumeObjectTO;
import org.apache.cloudstack.utils.cache.SingleCache;
import org.apache.cloudstack.utils.identity.ManagementServerNode;
import org.apache.cloudstack.vm.UnmanagedVMsManager;
import org.apache.commons.collections.CollectionUtils;
@ -154,6 +151,7 @@ import com.cloud.api.query.dao.UserVmJoinDao;
import com.cloud.api.query.vo.DomainRouterJoinVO;
import com.cloud.api.query.vo.UserVmJoinVO;
import com.cloud.capacity.CapacityManager;
import com.cloud.configuration.Resource;
import com.cloud.dc.ClusterDetailsDao;
import com.cloud.dc.ClusterDetailsVO;
import com.cloud.dc.ClusterVO;
@ -171,6 +169,7 @@ import com.cloud.deploy.DeploymentPlanner;
import com.cloud.deploy.DeploymentPlanner.ExcludeList;
import com.cloud.deploy.DeploymentPlanningManager;
import com.cloud.deployasis.dao.UserVmDeployAsIsDetailsDao;
import com.cloud.event.ActionEventUtils;
import com.cloud.event.EventTypes;
import com.cloud.event.UsageEventUtils;
import com.cloud.event.UsageEventVO;
@ -182,6 +181,7 @@ import com.cloud.exception.InsufficientCapacityException;
import com.cloud.exception.InsufficientServerCapacityException;
import com.cloud.exception.InvalidParameterValueException;
import com.cloud.exception.OperationTimedoutException;
import com.cloud.exception.ResourceAllocationException;
import com.cloud.exception.ResourceUnavailableException;
import com.cloud.exception.StorageAccessException;
import com.cloud.exception.StorageUnavailableException;
@ -273,6 +273,7 @@ import com.cloud.vm.dao.VMInstanceDao;
import com.cloud.vm.snapshot.VMSnapshotManager;
import com.cloud.vm.snapshot.VMSnapshotVO;
import com.cloud.vm.snapshot.dao.VMSnapshotDao;
import com.google.gson.Gson;
public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMachineManager, VmWorkJobHandler, Listener, Configurable {
private static final Logger s_logger = Logger.getLogger(VirtualMachineManagerImpl.class);
@ -393,6 +394,10 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
private AnnotationDao annotationDao;
@Inject
ResourceCleanupService resourceCleanupService;
@Inject
VmWorkJobDao vmWorkJobDao;
private SingleCache<List<Long>> vmIdsInProgressCache;
VmWorkJobHandlerProxy _jobHandlerProxy = new VmWorkJobHandlerProxy(this);
@ -437,6 +442,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
Long.class, "systemvm.root.disk.size", "-1",
"Size of root volume (in GB) of system VMs and virtual routers", true);
private boolean syncTransitioningVmPowerState;
ScheduledExecutorService _executor = null;
private long _nodeId;
@ -805,6 +812,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
@Override
public boolean start() {
vmIdsInProgressCache = new SingleCache<>(10, vmWorkJobDao::listVmIdsWithPendingJob);
_executor.scheduleAtFixedRate(new CleanupTask(), 5, VmJobStateReportInterval.value(), TimeUnit.SECONDS);
_executor.scheduleAtFixedRate(new TransitionTask(), VmOpCleanupInterval.value(), VmOpCleanupInterval.value(), TimeUnit.SECONDS);
cancelWorkItems(_nodeId);
@ -832,6 +840,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
_messageBus.subscribe(VirtualMachineManager.Topics.VM_POWER_STATE, MessageDispatcher.getDispatcher(this));
syncTransitioningVmPowerState = Boolean.TRUE.equals(VmSyncPowerStateTransitioning.value());
return true;
}
@ -3520,7 +3530,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
if (MIGRATE_VM_ACROSS_CLUSTERS.valueIn(host.getDataCenterId()) &&
(HypervisorType.VMware.equals(host.getHypervisorType()) || !checkIfVmHasClusterWideVolumes(vm.getId()))) {
s_logger.info("Searching for hosts in the zone for vm migration");
List<Long> clustersToExclude = _clusterDao.listAllClusters(host.getDataCenterId());
List<Long> clustersToExclude = _clusterDao.listAllClusterIds(host.getDataCenterId());
List<ClusterVO> clusterList = _clusterDao.listByDcHyType(host.getDataCenterId(), host.getHypervisorType().toString());
for (ClusterVO cluster : clusterList) {
clustersToExclude.remove(cluster.getId());
@ -3821,11 +3831,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
if (ping.getHostVmStateReport() != null) {
_syncMgr.processHostVmStatePingReport(agentId, ping.getHostVmStateReport(), ping.getOutOfBand());
}
// CPU and DB hotspot
// FIXME: CPU & DB hotspot: listStalledVMInTransitionStateOnUpHost
// FIXME: CPU & DB hotspot: listVMInTransitionStateWithRecentReportOnUpHost
// scanStalledVMInTransitionStateOnUpHost(agentId);
scanStalledVMInTransitionStateOnUpHost(agentId);
processed = true;
}
}
@ -4795,7 +4801,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
VmOpLockStateRetry, VmOpWaitInterval, ExecuteInSequence, VmJobCheckInterval, VmJobTimeout, VmJobStateReportInterval,
VmConfigDriveLabel, VmConfigDriveOnPrimaryPool, VmConfigDriveForceHostCacheUse, VmConfigDriveUseHostCacheOnUnsupportedPool,
HaVmRestartHostUp, ResourceCountRunningVMsonly, AllowExposeHypervisorHostname, AllowExposeHypervisorHostnameAccountLevel, SystemVmRootDiskSize,
AllowExposeDomainInMetadata, MetadataCustomCloudName
AllowExposeDomainInMetadata, MetadataCustomCloudName, VmSyncPowerStateTransitioning
};
}
@ -4987,6 +4993,9 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
}
private void scanStalledVMInTransitionStateOnUpHost(final long hostId) {
if (!syncTransitioningVmPowerState) {
return;
}
// Check VM that is stuck in Starting, Stopping, Migrating states, we won't check
// VMs in expunging state (this need to be handled specially)
//
@ -5002,20 +5011,18 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
// and a VM stalls for status update, we will consider them to be powered off
// (which is relatively safe to do so)
final long stallThresholdInMs = VmJobStateReportInterval.value() * 2;
final Date cutTime = new Date(DateUtil.currentGMTTime().getTime() - stallThresholdInMs);
// FIXME: CPU & DB hotspot: listStalledVMInTransitionStateOnUpHost
final List<Long> mostLikelyStoppedVMs = listStalledVMInTransitionStateOnUpHost(hostId, cutTime);
for (final Long vmId : mostLikelyStoppedVMs) {
final VMInstanceVO vm = _vmDao.findById(vmId);
assert vm != null;
final long cutTime = new Date(DateUtil.currentGMTTime().getTime() - stallThresholdInMs).getTime();
if (!_hostDao.isHostUp(hostId)) {
return;
}
final List<VMInstanceVO> hostTransitionVms = _vmDao.listByHostAndState(hostId, State.Starting, State.Stopping, State.Migrating);
final List<VMInstanceVO> mostLikelyStoppedVMs = listStalledVMInTransitionStateOnUpHost(hostTransitionVms, cutTime);
for (final VMInstanceVO vm : mostLikelyStoppedVMs) {
handlePowerOffReportWithNoPendingJobsOnVM(vm);
}
// FIXME: CPU & DB hotspot: listVMInTransitionStateWithRecentReportOnUpHost
final List<Long> vmsWithRecentReport = listVMInTransitionStateWithRecentReportOnUpHost(hostId, cutTime);
for (final Long vmId : vmsWithRecentReport) {
final VMInstanceVO vm = _vmDao.findById(vmId);
assert vm != null;
final List<VMInstanceVO> vmsWithRecentReport = listVMInTransitionStateWithRecentReportOnUpHost(hostTransitionVms, cutTime);
for (final VMInstanceVO vm : vmsWithRecentReport) {
if (vm.getPowerState() == PowerState.PowerOn) {
handlePowerOnReportWithNoPendingJobsOnVM(vm);
} else {
@ -5036,65 +5043,39 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
}
}
private List<Long> listStalledVMInTransitionStateOnUpHost(final long hostId, final Date cutTime) {
final String sql = "SELECT i.id FROM vm_instance as i, host as h WHERE h.status = 'UP' " +
"AND h.id = ? AND i.power_state_update_time < ? AND i.host_id = h.id " +
"AND (i.state ='Starting' OR i.state='Stopping' OR i.state='Migrating') " +
"AND i.id NOT IN (SELECT w.vm_instance_id FROM vm_work_job AS w JOIN async_job AS j ON w.id = j.id WHERE j.job_status = ?)" +
"AND i.removed IS NULL";
final List<Long> l = new ArrayList<>();
try (TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.CLOUD_DB)) {
String cutTimeStr = DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutTime);
try (PreparedStatement pstmt = txn.prepareAutoCloseStatement(sql)) {
pstmt.setLong(1, hostId);
pstmt.setString(2, cutTimeStr);
pstmt.setInt(3, JobInfo.Status.IN_PROGRESS.ordinal());
final ResultSet rs = pstmt.executeQuery();
while (rs.next()) {
l.add(rs.getLong(1));
}
} catch (SQLException e) {
s_logger.error(String.format("Unable to execute SQL [%s] with params {\"h.id\": %s, \"i.power_state_update_time\": \"%s\"} due to [%s].", sql, hostId, cutTimeStr, e.getMessage()), e);
}
private List<VMInstanceVO> listStalledVMInTransitionStateOnUpHost(
final List<VMInstanceVO> transitioningVms, final long cutTime) {
if (CollectionUtils.isEmpty(transitioningVms)) {
return transitioningVms;
}
return l;
List<Long> vmIdsInProgress = vmIdsInProgressCache.get();
return transitioningVms.stream()
.filter(v -> v.getPowerStateUpdateTime().getTime() < cutTime && !vmIdsInProgress.contains(v.getId()))
.collect(Collectors.toList());
}
private List<Long> listVMInTransitionStateWithRecentReportOnUpHost(final long hostId, final Date cutTime) {
final String sql = "SELECT i.id FROM vm_instance as i, host as h WHERE h.status = 'UP' " +
"AND h.id = ? AND i.power_state_update_time > ? AND i.host_id = h.id " +
"AND (i.state ='Starting' OR i.state='Stopping' OR i.state='Migrating') " +
"AND i.id NOT IN (SELECT w.vm_instance_id FROM vm_work_job AS w JOIN async_job AS j ON w.id = j.id WHERE j.job_status = ?)" +
"AND i.removed IS NULL";
final List<Long> l = new ArrayList<>();
try (TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.CLOUD_DB)) {
String cutTimeStr = DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutTime);
int jobStatusInProgress = JobInfo.Status.IN_PROGRESS.ordinal();
try (PreparedStatement pstmt = txn.prepareAutoCloseStatement(sql)) {
pstmt.setLong(1, hostId);
pstmt.setString(2, cutTimeStr);
pstmt.setInt(3, jobStatusInProgress);
final ResultSet rs = pstmt.executeQuery();
while (rs.next()) {
l.add(rs.getLong(1));
}
} catch (final SQLException e) {
s_logger.error(String.format("Unable to execute SQL [%s] with params {\"h.id\": %s, \"i.power_state_update_time\": \"%s\", \"j.job_status\": %s} due to [%s].", sql, hostId, cutTimeStr, jobStatusInProgress, e.getMessage()), e);
}
return l;
private List<VMInstanceVO> listVMInTransitionStateWithRecentReportOnUpHost(
final List<VMInstanceVO> transitioningVms, final long cutTime) {
if (CollectionUtils.isEmpty(transitioningVms)) {
return transitioningVms;
}
List<Long> vmIdsInProgress = vmIdsInProgressCache.get();
return transitioningVms.stream()
.filter(v -> v.getPowerStateUpdateTime().getTime() > cutTime && !vmIdsInProgress.contains(v.getId()))
.collect(Collectors.toList());
}
private List<Long> listStalledVMInTransitionStateOnDisconnectedHosts(final Date cutTime) {
final String sql = "SELECT i.* FROM vm_instance as i, host as h WHERE h.status != 'UP' " +
"AND i.power_state_update_time < ? AND i.host_id = h.id " +
"AND (i.state ='Starting' OR i.state='Stopping' OR i.state='Migrating') " +
"AND i.id NOT IN (SELECT w.vm_instance_id FROM vm_work_job AS w JOIN async_job AS j ON w.id = j.id WHERE j.job_status = ?)" +
"AND i.removed IS NULL";
final String sql = "SELECT i.*\n" +
"FROM vm_instance AS i\n" +
"INNER JOIN host AS h ON i.host_id = h.id\n" +
"WHERE h.status != 'UP' \n" +
" AND i.power_state_update_time < ?\n" +
" AND i.state IN ('Starting', 'Stopping', 'Migrating')\n" +
" AND i.id NOT IN (SELECT vm_instance_id FROM vm_work_job AS w\n" +
" INNER JOIN async_job AS j ON w.id = j.id\n" +
" WHERE j.job_status = ?)\n" +
" AND i.removed IS NULL";
final List<Long> l = new ArrayList<>();
try (TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.CLOUD_DB)) {
@ -6000,29 +5981,23 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
}
@Override
public HashMap<Long, ? extends VmStats> getVirtualMachineStatistics(long hostId, String hostName, List<Long> vmIds) {
public HashMap<Long, ? extends VmStats> getVirtualMachineStatistics(Host host, List<Long> vmIds) {
HashMap<Long, VmStatsEntry> vmStatsById = new HashMap<>();
if (CollectionUtils.isEmpty(vmIds)) {
return vmStatsById;
}
Map<Long, VMInstanceVO> vmMap = new HashMap<>();
for (Long vmId : vmIds) {
vmMap.put(vmId, _vmDao.findById(vmId));
}
return getVirtualMachineStatistics(hostId, hostName, vmMap);
Map<String, Long> vmMap = _vmDao.getNameIdMapForVmIds(vmIds);
return getVirtualMachineStatistics(host, vmMap);
}
@Override
public HashMap<Long, ? extends VmStats> getVirtualMachineStatistics(long hostId, String hostName, Map<Long, ? extends VirtualMachine> vmMap) {
public HashMap<Long, ? extends VmStats> getVirtualMachineStatistics(Host host, Map<String, Long> vmInstanceNameIdMap) {
HashMap<Long, VmStatsEntry> vmStatsById = new HashMap<>();
if (MapUtils.isEmpty(vmMap)) {
if (MapUtils.isEmpty(vmInstanceNameIdMap)) {
return vmStatsById;
}
Map<String, Long> vmNames = new HashMap<>();
for (Map.Entry<Long, ? extends VirtualMachine> vmEntry : vmMap.entrySet()) {
vmNames.put(vmEntry.getValue().getInstanceName(), vmEntry.getKey());
}
Answer answer = _agentMgr.easySend(hostId, new GetVmStatsCommand(new ArrayList<>(vmNames.keySet()), _hostDao.findById(hostId).getGuid(), hostName));
Answer answer = _agentMgr.easySend(host.getId(), new GetVmStatsCommand(
new ArrayList<>(vmInstanceNameIdMap.keySet()), host.getGuid(), host.getName()));
if (answer == null || !answer.getResult()) {
s_logger.warn("Unable to obtain VM statistics.");
return vmStatsById;
@ -6033,23 +6008,20 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
return vmStatsById;
}
for (Map.Entry<String, VmStatsEntry> entry : vmStatsByName.entrySet()) {
vmStatsById.put(vmNames.get(entry.getKey()), entry.getValue());
vmStatsById.put(vmInstanceNameIdMap.get(entry.getKey()), entry.getValue());
}
}
return vmStatsById;
}
@Override
public HashMap<Long, List<? extends VmDiskStats>> getVmDiskStatistics(long hostId, String hostName, Map<Long, ? extends VirtualMachine> vmMap) {
public HashMap<Long, List<? extends VmDiskStats>> getVmDiskStatistics(Host host, Map<String, Long> vmInstanceNameIdMap) {
HashMap<Long, List<? extends VmDiskStats>> vmDiskStatsById = new HashMap<>();
if (MapUtils.isEmpty(vmMap)) {
if (MapUtils.isEmpty(vmInstanceNameIdMap)) {
return vmDiskStatsById;
}
Map<String, Long> vmNames = new HashMap<>();
for (Map.Entry<Long, ? extends VirtualMachine> vmEntry : vmMap.entrySet()) {
vmNames.put(vmEntry.getValue().getInstanceName(), vmEntry.getKey());
}
Answer answer = _agentMgr.easySend(hostId, new GetVmDiskStatsCommand(new ArrayList<>(vmNames.keySet()), _hostDao.findById(hostId).getGuid(), hostName));
Answer answer = _agentMgr.easySend(host.getId(), new GetVmDiskStatsCommand(
new ArrayList<>(vmInstanceNameIdMap.keySet()), host.getGuid(), host.getName()));
if (answer == null || !answer.getResult()) {
s_logger.warn("Unable to obtain VM disk statistics.");
return vmDiskStatsById;
@ -6060,23 +6032,20 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
return vmDiskStatsById;
}
for (Map.Entry<String, List<VmDiskStatsEntry>> entry: vmDiskStatsByName.entrySet()) {
vmDiskStatsById.put(vmNames.get(entry.getKey()), entry.getValue());
vmDiskStatsById.put(vmInstanceNameIdMap.get(entry.getKey()), entry.getValue());
}
}
return vmDiskStatsById;
}
@Override
public HashMap<Long, List<? extends VmNetworkStats>> getVmNetworkStatistics(long hostId, String hostName, Map<Long, ? extends VirtualMachine> vmMap) {
public HashMap<Long, List<? extends VmNetworkStats>> getVmNetworkStatistics(Host host, Map<String, Long> vmInstanceNameIdMap) {
HashMap<Long, List<? extends VmNetworkStats>> vmNetworkStatsById = new HashMap<>();
if (MapUtils.isEmpty(vmMap)) {
if (MapUtils.isEmpty(vmInstanceNameIdMap)) {
return vmNetworkStatsById;
}
Map<String, Long> vmNames = new HashMap<>();
for (Map.Entry<Long, ? extends VirtualMachine> vmEntry : vmMap.entrySet()) {
vmNames.put(vmEntry.getValue().getInstanceName(), vmEntry.getKey());
}
Answer answer = _agentMgr.easySend(hostId, new GetVmNetworkStatsCommand(new ArrayList<>(vmNames.keySet()), _hostDao.findById(hostId).getGuid(), hostName));
Answer answer = _agentMgr.easySend(host.getId(), new GetVmNetworkStatsCommand(
new ArrayList<>(vmInstanceNameIdMap.keySet()), host.getGuid(), host.getName()));
if (answer == null || !answer.getResult()) {
s_logger.warn("Unable to obtain VM network statistics.");
return vmNetworkStatsById;
@ -6087,7 +6056,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
return vmNetworkStatsById;
}
for (Map.Entry<String, List<VmNetworkStatsEntry>> entry: vmNetworkStatsByName.entrySet()) {
vmNetworkStatsById.put(vmNames.get(entry.getKey()), entry.getValue());
vmNetworkStatsById.put(vmInstanceNameIdMap.get(entry.getKey()), entry.getValue());
}
}
return vmNetworkStatsById;

189
engine/orchestration/src/main/java/com/cloud/vm/VirtualMachinePowerStateSyncImpl.java
View File

@ -16,23 +16,23 @@
// under the License.
package com.cloud.vm;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import javax.inject.Inject;
import org.apache.cloudstack.framework.messagebus.MessageBus;
import org.apache.cloudstack.framework.messagebus.PublishScope;
import org.apache.commons.collections.MapUtils;
import org.apache.log4j.Logger;
import com.cloud.agent.api.HostVmStateReportEntry;
import com.cloud.configuration.ManagementServiceConfiguration;
import com.cloud.utils.DateUtil;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.vm.dao.VMInstanceDao;
public class VirtualMachinePowerStateSyncImpl implements VirtualMachinePowerStateSync {
@ -68,106 +68,108 @@ public class VirtualMachinePowerStateSyncImpl implements VirtualMachinePowerStat
processReport(hostId, translatedInfo, force);
}
private void processReport(long hostId, Map<Long, VirtualMachine.PowerState> translatedInfo, boolean force) {
if (s_logger.isDebugEnabled()) {
s_logger.debug("Process VM state report. host: " + hostId + ", number of records in report: " + translatedInfo.size());
private void updateAndPublishVmPowerStates(long hostId, Map<Long, VirtualMachine.PowerState> instancePowerStates,
Date updateTime) {
if (instancePowerStates.isEmpty()) {
return;
}
for (Map.Entry<Long, VirtualMachine.PowerState> entry : translatedInfo.entrySet()) {
if (s_logger.isDebugEnabled())
s_logger.debug("VM state report. host: " + hostId + ", vm id: " + entry.getKey() + ", power state: " + entry.getValue());
if (_instanceDao.updatePowerState(entry.getKey(), hostId, entry.getValue(), DateUtil.currentGMTTime())) {
if (s_logger.isInfoEnabled()) {
s_logger.debug("VM state report is updated. host: " + hostId + ", vm id: " + entry.getKey() + ", power state: " + entry.getValue());
Set<Long> vmIds = instancePowerStates.keySet();
Map<Long, VirtualMachine.PowerState> notUpdated = _instanceDao.updatePowerState(instancePowerStates, hostId,
updateTime);
if (notUpdated.size() <= vmIds.size()) {
for (Long vmId : vmIds) {
if (!notUpdated.isEmpty() && !notUpdated.containsKey(vmId)) {
if (s_logger.isDebugEnabled()) {
s_logger.debug(String.format("VM state report is updated. host: %d, vm id: %d, power state: %s",
hostId, vmId, instancePowerStates.get(vmId)));
}
_messageBus.publish(null, VirtualMachineManager.Topics.VM_POWER_STATE,
PublishScope.GLOBAL, vmId);
continue;
}
_messageBus.publish(null, VirtualMachineManager.Topics.VM_POWER_STATE, PublishScope.GLOBAL, entry.getKey());
} else {
if (s_logger.isTraceEnabled()) {
s_logger.trace("VM power state does not change, skip DB writing. vm id: " + entry.getKey());
s_logger.trace(String.format("VM power state does not change, skip DB writing. vm id: %d", vmId));
}
}
}
}
private void processMissingVmReport(long hostId, Set<Long> vmIds, boolean force) {
// any state outdates should be checked against the time before this list was retrieved
Date startTime = DateUtil.currentGMTTime();
// for all running/stopping VMs, we provide monitoring of missing report
// FIXME: CPU & DB hotspot findByHostInStatesExcluding
List<VMInstanceVO> vmsThatAreMissingReport = _instanceDao.findByHostInStatesExcluding(hostId, new ArrayList<>(translatedInfo.keySet()),
List<VMInstanceVO> vmsThatAreMissingReport = _instanceDao.findByHostInStatesExcluding(hostId, vmIds,
VirtualMachine.State.Running, VirtualMachine.State.Stopping, VirtualMachine.State.Starting);
// here we need to be wary of out of band migration as opposed to other, more unexpected state changes
if (vmsThatAreMissingReport.size() > 0) {
Date currentTime = DateUtil.currentGMTTime();
if (s_logger.isDebugEnabled()) {
s_logger.debug("Run missing VM report. current time: " + currentTime.getTime());
}
if (vmsThatAreMissingReport.isEmpty()) {
return;
}
Date currentTime = DateUtil.currentGMTTime();
if (s_logger.isDebugEnabled()) {
s_logger.debug("Run missing VM report. current time: " + currentTime.getTime());
}
if (!force) {
List<Long> outdatedVms = vmsThatAreMissingReport.stream()
.filter(v -> !_instanceDao.isPowerStateUpToDate(v))
.map(VMInstanceVO::getId)
.collect(Collectors.toList());
_instanceDao.resetVmPowerStateTracking(outdatedVms);
vmsThatAreMissingReport = vmsThatAreMissingReport.stream()
.filter(v -> !outdatedVms.contains(v.getId()))
.collect(Collectors.toList());
}
// 2 times of sync-update interval for graceful period
long milliSecondsGracefullPeriod = mgmtServiceConf.getPingInterval() * 2000L;
for (VMInstanceVO instance : vmsThatAreMissingReport) {
// Make sure powerState is up to date for missing VMs
try {
if (!force && !_instanceDao.isPowerStateUpToDate(instance.getId())) {
s_logger.warn("Detected missing VM but power state is outdated, wait for another process report run for VM id: " + instance.getId());
_instanceDao.resetVmPowerStateTracking(instance.getId());
continue;
}
} catch (CloudRuntimeException e) {
s_logger.warn("Checked for missing powerstate of a none existing vm", e);
continue;
}
Date vmStateUpdateTime = instance.getPowerStateUpdateTime();
// 2 times of sync-update interval for graceful period
long milliSecondsGracefulPeriod = mgmtServiceConf.getPingInterval() * 2000L;
Map<Long, VirtualMachine.PowerState> instancePowerStates = new HashMap<>();
for (VMInstanceVO instance : vmsThatAreMissingReport) {
Date vmStateUpdateTime = instance.getPowerStateUpdateTime();
if (vmStateUpdateTime == null) {
s_logger.warn("VM power state update time is null, falling back to update time for vm id: " + instance.getId());
vmStateUpdateTime = instance.getUpdateTime();
if (vmStateUpdateTime == null) {
s_logger.warn("VM power state update time is null, falling back to update time for vm id: " + instance.getId());
vmStateUpdateTime = instance.getUpdateTime();
if (vmStateUpdateTime == null) {
s_logger.warn("VM update time is null, falling back to creation time for vm id: " + instance.getId());
vmStateUpdateTime = instance.getCreated();
}
s_logger.warn("VM update time is null, falling back to creation time for vm id: " + instance.getId());
vmStateUpdateTime = instance.getCreated();
}
if (s_logger.isInfoEnabled()) {
String lastTime = new SimpleDateFormat("yyyy/MM/dd'T'HH:mm:ss.SSS'Z'").format(vmStateUpdateTime);
s_logger.debug(
String.format("Detected missing VM. host: %d, vm id: %d(%s), power state: %s, last state update: %s"
, hostId
, instance.getId()
, instance.getUuid()
, VirtualMachine.PowerState.PowerReportMissing
, lastTime));
}
long milliSecondsSinceLastStateUpdate = currentTime.getTime() - vmStateUpdateTime.getTime();
if (force || milliSecondsSinceLastStateUpdate > milliSecondsGracefullPeriod) {
s_logger.debug("vm id: " + instance.getId() + " - time since last state update(" + milliSecondsSinceLastStateUpdate + "ms) has passed graceful period");
// this is were a race condition might have happened if we don't re-fetch the instance;
// between the startime of this job and the currentTime of this missing-branch
// an update might have occurred that we should not override in case of out of band migration
if (_instanceDao.updatePowerState(instance.getId(), hostId, VirtualMachine.PowerState.PowerReportMissing, startTime)) {
if (s_logger.isDebugEnabled()) {
s_logger.debug("VM state report is updated. host: " + hostId + ", vm id: " + instance.getId() + ", power state: PowerReportMissing ");
}
_messageBus.publish(null, VirtualMachineManager.Topics.VM_POWER_STATE, PublishScope.GLOBAL, instance.getId());
} else {
if (s_logger.isDebugEnabled()) {
s_logger.debug("VM power state does not change, skip DB writing. vm id: " + instance.getId());
}
}
} else {
s_logger.debug("vm id: " + instance.getId() + " - time since last state update(" + milliSecondsSinceLastStateUpdate + "ms) has not passed graceful period yet");
}
if (s_logger.isDebugEnabled()) {
s_logger.debug(
String.format("Detected missing VM. host: %d, vm id: %d(%s), power state: %s, last state update: %s"
, hostId
, instance.getId()
, instance.getUuid()
, VirtualMachine.PowerState.PowerReportMissing
, DateUtil.getOutputString(vmStateUpdateTime)));
}
long milliSecondsSinceLastStateUpdate = currentTime.getTime() - vmStateUpdateTime.getTime();
if (force || (milliSecondsSinceLastStateUpdate > milliSecondsGracefulPeriod)) {
if (s_logger.isDebugEnabled()) {
s_logger.debug(String.format("vm id: %d - time since last state update(%d ms) has passed graceful period",
instance.getId(), milliSecondsSinceLastStateUpdate));
}
// this is where a race condition might have happened if we don't re-fetch the instance;
// between the startime of this job and the currentTime of this missing-branch
// an update might have occurred that we should not override in case of out of band migration
instancePowerStates.put(instance.getId(), VirtualMachine.PowerState.PowerReportMissing);
} else {
s_logger.debug(String.format("vm id: %d - time since last state update(%d ms) has not passed graceful period yet",
instance.getId(), milliSecondsSinceLastStateUpdate));
}
}
updateAndPublishVmPowerStates(hostId, instancePowerStates, startTime);
}
private void processReport(long hostId, Map<Long, VirtualMachine.PowerState> translatedInfo, boolean force) {
if (s_logger.isDebugEnabled()) {
s_logger.debug(String.format("Process VM state report. Host: %d, number of records in report: %d. VMs: [%s]",
hostId,
translatedInfo.size(),
translatedInfo.entrySet().stream().map(entry -> entry.getKey() + ":" + entry.getValue())
.collect(Collectors.joining(", ")) + "]"));
}
updateAndPublishVmPowerStates(hostId, translatedInfo, DateUtil.currentGMTTime());
processMissingVmReport(hostId, translatedInfo.keySet(), force);
if (s_logger.isDebugEnabled())
s_logger.debug("Done with process of VM state report. host: " + hostId);
@ -175,24 +177,19 @@ public class VirtualMachinePowerStateSyncImpl implements VirtualMachinePowerStat
@Override
public Map<Long, VirtualMachine.PowerState> convertVmStateReport(Map<String, HostVmStateReportEntry> states) {
final HashMap<Long, VirtualMachine.PowerState> map = new HashMap<Long, VirtualMachine.PowerState>();
if (states == null) {
final HashMap<Long, VirtualMachine.PowerState> map = new HashMap<>();
if (MapUtils.isEmpty(states)) {
return map;
}
Map<String, Long> nameIdMap = _instanceDao.getNameIdMapForVmInstanceNames(states.keySet());
for (Map.Entry<String, HostVmStateReportEntry> entry : states.entrySet()) {
VMInstanceVO vm = findVM(entry.getKey());
if (vm != null) {
map.put(vm.getId(), entry.getValue().getState());
Long id = nameIdMap.get(entry.getKey());
if (id != null) {
map.put(id, entry.getValue().getState());
} else {
s_logger.debug("Unable to find matched VM in CloudStack DB. name: " + entry.getKey());
}
}
return map;
}
private VMInstanceVO findVM(String vmName) {
return _instanceDao.findVMByInstanceName(vmName);
}
}

2
engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDao.java
View File

@ -28,6 +28,8 @@ import com.cloud.utils.db.GenericDao;
public interface CapacityDao extends GenericDao<CapacityVO, Long> {
CapacityVO findByHostIdType(Long hostId, short capacityType);
List<CapacityVO> listByHostIdTypes(Long hostId, List<Short> capacityTypes);
List<Long> listClustersInZoneOrPodByHostCapacities(long id, long vmId, int requiredCpu, long requiredRam, short capacityTypeForOrdering, boolean isZone);
List<Long> listHostsWithEnoughCapacity(int requiredCpu, long requiredRam, Long clusterId, String hostType);

12
engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDaoImpl.java
View File

@ -674,6 +674,18 @@ public class CapacityDaoImpl extends GenericDaoBase<CapacityVO, Long> implements
return findOneBy(sc);
}
@Override
public List<CapacityVO> listByHostIdTypes(Long hostId, List<Short> capacityTypes) {
SearchBuilder<CapacityVO> sb = createSearchBuilder();
sb.and("hostId", sb.entity().getHostOrPoolId(), SearchCriteria.Op.EQ);
sb.and("type", sb.entity().getCapacityType(), SearchCriteria.Op.IN);
sb.done();
SearchCriteria<CapacityVO> sc = sb.create();
sc.setParameters("hostId", hostId);
sc.setParameters("type", capacityTypes.toArray());
return listBy(sc);
}
@Override
public List<Long> listClustersInZoneOrPodByHostCapacities(long id, long vmId, int requiredCpu, long requiredRam, short capacityTypeForOrdering, boolean isZone) {
TransactionLegacy txn = TransactionLegacy.currentTxn();

3
engine/schema/src/main/java/com/cloud/dc/ClusterDetailsDao.java
View File

@ -16,6 +16,7 @@
// under the License.
package com.cloud.dc;
import java.util.Collection;
import java.util.Map;
import com.cloud.utils.db.GenericDao;
@ -29,6 +30,8 @@ public interface ClusterDetailsDao extends GenericDao<ClusterDetailsVO, Long> {
ClusterDetailsVO findDetail(long clusterId, String name);
Map<String, String> findDetails(long clusterId, Collection<String> names);
void deleteDetails(long clusterId);
String getVmwareDcName(Long clusterId);

20
engine/schema/src/main/java/com/cloud/dc/ClusterDetailsDaoImpl.java
View File

@ -16,13 +16,16 @@
// under the License.
package com.cloud.dc;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.framework.config.ConfigKey.Scope;
import org.apache.cloudstack.framework.config.ScopedConfigStorage;
import org.apache.commons.collections.CollectionUtils;
import com.cloud.utils.crypt.DBEncryptionUtil;
import com.cloud.utils.db.GenericDaoBase;
@ -82,6 +85,23 @@ public class ClusterDetailsDaoImpl extends GenericDaoBase<ClusterDetailsVO, Long
return details;
}
@Override
public Map<String, String> findDetails(long clusterId, Collection<String> names) {
if (CollectionUtils.isEmpty(names)) {
return new HashMap<>();
}
SearchBuilder<ClusterDetailsVO> sb = createSearchBuilder();
sb.and("clusterId", sb.entity().getClusterId(), SearchCriteria.Op.EQ);
sb.and("name", sb.entity().getName(), SearchCriteria.Op.IN);
sb.done();
SearchCriteria<ClusterDetailsVO> sc = sb.create();
sc.setParameters("clusterId", clusterId);
sc.setParameters("name", names.toArray());
List<ClusterDetailsVO> results = search(sc, null);
return results.stream()
.collect(Collectors.toMap(ClusterDetailsVO::getName, ClusterDetailsVO::getValue));
}
@Override
public void deleteDetails(long clusterId) {
SearchCriteria<ClusterDetailsVO> sc = ClusterSearch.create();

12
engine/schema/src/main/java/com/cloud/dc/dao/ClusterDao.java
View File

@ -16,14 +16,14 @@
// under the License.
package com.cloud.dc.dao;
import com.cloud.dc.ClusterVO;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.utils.db.GenericDao;
import java.util.List;
import java.util.Map;
import java.util.Set;
import com.cloud.dc.ClusterVO;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.utils.db.GenericDao;
public interface ClusterDao extends GenericDao<ClusterVO, Long> {
List<ClusterVO> listByPodId(long podId);
@ -51,7 +51,9 @@ public interface ClusterDao extends GenericDao<ClusterVO, Long> {
List<ClusterVO> listClustersByDcId(long zoneId);
List<Long> listAllClusters(Long zoneId);
List<Long> listAllClusterIds(Long zoneId);
boolean getSupportsResigning(long clusterId);
List<Long> listAllIds();
}

36
engine/schema/src/main/java/com/cloud/dc/dao/ClusterDaoImpl.java
View File

@ -16,6 +16,20 @@
// under the License.
package com.cloud.dc.dao;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import javax.inject.Inject;
import org.springframework.stereotype.Component;
import com.cloud.dc.ClusterDetailsDao;
import com.cloud.dc.ClusterDetailsVO;
import com.cloud.dc.ClusterVO;
@ -32,18 +46,6 @@ import com.cloud.utils.db.SearchCriteria.Func;
import com.cloud.utils.db.SearchCriteria.Op;
import com.cloud.utils.db.TransactionLegacy;
import com.cloud.utils.exception.CloudRuntimeException;
import org.springframework.stereotype.Component;
import javax.inject.Inject;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@Component
public class ClusterDaoImpl extends GenericDaoBase<ClusterVO, Long> implements ClusterDao {
@ -283,7 +285,7 @@ public class ClusterDaoImpl extends GenericDaoBase<ClusterVO, Long> implements C
}
@Override
public List<Long> listAllClusters(Long zoneId) {
public List<Long> listAllClusterIds(Long zoneId) {
SearchCriteria<Long> sc = ClusterIdSearch.create();
if (zoneId != null) {
sc.setParameters("dataCenterId", zoneId);
@ -309,4 +311,12 @@ public class ClusterDaoImpl extends GenericDaoBase<ClusterVO, Long> implements C
return false;
}
@Override
public List<Long> listAllIds() {
GenericSearchBuilder<ClusterVO, Long> sb = createSearchBuilder(Long.class);
sb.selectFields(sb.entity().getId());
sb.done();
return customSearch(sb.create(), null);
}
}

28
engine/schema/src/main/java/com/cloud/host/dao/HostDao.java
View File

@ -81,32 +81,35 @@ public interface HostDao extends GenericDao<HostVO, Long>, StateDao<Status, Stat
List<HostVO> findHypervisorHostInCluster(long clusterId);
HostVO findAnyStateHypervisorHostInCluster(long clusterId);
HostVO findOldestExistentHypervisorHostInCluster(long clusterId);
List<HostVO> listAllUpAndEnabledNonHAHosts(Type type, Long clusterId, Long podId, long dcId, String haTag);
List<HostVO> findByDataCenterId(Long zoneId);
List<Long> listIdsByDataCenterId(Long zoneId);
List<HostVO> findByPodId(Long podId);
List<Long> listIdsByPodId(Long podId);
List<HostVO> findByClusterId(Long clusterId);
List<Long> listIdsByClusterId(Long clusterId);
List<HostVO> findByClusterIdAndEncryptionSupport(Long clusterId);
/**
* Returns hosts that are 'Up' and 'Enabled' from the given Data Center/Zone
*/
List<HostVO> listByDataCenterId(long id);
/**
* Returns hosts that are from the given Data Center/Zone and at a given state (e.g. Creating, Enabled, Disabled, etc).
*/
List<HostVO> listByDataCenterIdAndState(long id, ResourceState state);
List<Long> listEnabledIdsByDataCenterId(long id);
/**
* Returns hosts that are 'Up' and 'Disabled' from the given Data Center/Zone
*/
List<HostVO> listDisabledByDataCenterId(long id);
List<Long> listDisabledIdsByDataCenterId(long id);
List<HostVO> listByDataCenterIdAndHypervisorType(long zoneId, Hypervisor.HypervisorType hypervisorType);
@ -165,4 +168,15 @@ public interface HostDao extends GenericDao<HostVO, Long>, StateDao<Status, Stat
* @return ordered list of hypervisor versions
*/
List<String> listOrderedHostsHypervisorVersionsInDatacenter(long datacenterId, HypervisorType hypervisorType);
boolean isHostUp(long hostId);
List<Long> findHostIdsByZoneClusterResourceStateAndType(final Long zoneId, final Long clusterId,
final List<ResourceState> resourceStates, final List<Type> types);
List<Long> listAllIds();
List<HypervisorType> listDistinctHypervisorTypes(final Long zoneId);
List<HostVO> listByIds(final List<Long> ids);
}

152
engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java
View File

@ -20,6 +20,7 @@ import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
@ -32,6 +33,7 @@ import javax.annotation.PostConstruct;
import javax.inject.Inject;
import javax.persistence.TableGenerator;
import org.apache.commons.collections.CollectionUtils;
import org.apache.log4j.Logger;
import com.cloud.agent.api.VgpuTypesInfo;
@ -42,8 +44,8 @@ import com.cloud.dc.ClusterVO;
import com.cloud.dc.dao.ClusterDao;
import com.cloud.gpu.dao.HostGpuGroupsDao;
import com.cloud.gpu.dao.VGPUTypesDao;
import com.cloud.host.Host;
import com.cloud.host.DetailVO;
import com.cloud.host.Host;
import com.cloud.host.Host.Type;
import com.cloud.host.HostTagVO;
import com.cloud.host.HostVO;
@ -70,7 +72,6 @@ import com.cloud.utils.db.SearchCriteria.Op;
import com.cloud.utils.db.TransactionLegacy;
import com.cloud.utils.db.UpdateBuilder;
import com.cloud.utils.exception.CloudRuntimeException;
import java.util.Arrays;
@DB
@TableGenerator(name = "host_req_sq", table = "op_host", pkColumnName = "id", valueColumnName = "sequence", allocationSize = 1)
@ -97,6 +98,7 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
protected SearchBuilder<HostVO> TypePodDcStatusSearch;
protected SearchBuilder<HostVO> IdsSearch;
protected SearchBuilder<HostVO> IdStatusSearch;
protected SearchBuilder<HostVO> TypeDcSearch;
protected SearchBuilder<HostVO> TypeDcStatusSearch;
@ -109,6 +111,7 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
protected SearchBuilder<HostVO> GuidSearch;
protected SearchBuilder<HostVO> DcSearch;
protected GenericSearchBuilder<HostVO, Long> IdsDcSearch;
protected SearchBuilder<HostVO> PodSearch;
protected SearchBuilder<HostVO> ClusterSearch;
protected SearchBuilder<HostVO> TypeSearch;
@ -182,6 +185,8 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
HostTypeCountSearch = createSearchBuilder();
HostTypeCountSearch.and("type", HostTypeCountSearch.entity().getType(), SearchCriteria.Op.EQ);
HostTypeCountSearch.and("zoneId", HostTypeCountSearch.entity().getDataCenterId(), SearchCriteria.Op.EQ);
HostTypeCountSearch.and("resourceState", HostTypeCountSearch.entity().getResourceState(), SearchCriteria.Op.EQ);
HostTypeCountSearch.done();
ResponsibleMsCountSearch = createSearchBuilder();
@ -242,6 +247,10 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
TypeClusterStatusSearch.and("resourceState", TypeClusterStatusSearch.entity().getResourceState(), SearchCriteria.Op.EQ);
TypeClusterStatusSearch.done();
IdsSearch = createSearchBuilder();
IdsSearch.and("id", IdsSearch.entity().getId(), SearchCriteria.Op.IN);
IdsSearch.done();
IdStatusSearch = createSearchBuilder();
IdStatusSearch.and("id", IdStatusSearch.entity().getId(), SearchCriteria.Op.EQ);
IdStatusSearch.and("states", IdStatusSearch.entity().getStatus(), SearchCriteria.Op.IN);
@ -280,6 +289,13 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
DcSearch.and("resourceState", DcSearch.entity().getResourceState(), Op.EQ);
DcSearch.done();
IdsDcSearch = createSearchBuilder(Long.class);
IdsDcSearch.and("zoneId", IdsDcSearch.entity().getDataCenterId(), SearchCriteria.Op.EQ);
IdsDcSearch.and("resourceState", IdsDcSearch.entity().getResourceState(), SearchCriteria.Op.EQ);
IdsDcSearch.and("status", IdsDcSearch.entity().getStatus(), Op.EQ);
IdsDcSearch.and("type", IdsDcSearch.entity().getType(), SearchCriteria.Op.EQ);
IdsDcSearch.done();
ClusterStatusSearch = createSearchBuilder();
ClusterStatusSearch.and("cluster", ClusterStatusSearch.entity().getClusterId(), SearchCriteria.Op.EQ);
ClusterStatusSearch.and("status", ClusterStatusSearch.entity().getStatus(), SearchCriteria.Op.EQ);
@ -453,6 +469,7 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
HostIdSearch = createSearchBuilder(Long.class);
HostIdSearch.selectFields(HostIdSearch.entity().getId());
HostIdSearch.and("dataCenterId", HostIdSearch.entity().getDataCenterId(), Op.EQ);
HostIdSearch.and("clusterId", HostIdSearch.entity().getClusterId(), Op.EQ);
HostIdSearch.done();
_statusAttr = _allAttributes.get("status");
@ -499,7 +516,7 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
public Integer countAllByTypeInZone(long zoneId, Type type) {
SearchCriteria<HostVO> sc = HostTypeCountSearch.create();
sc.setParameters("type", type);
sc.setParameters("dc", zoneId);
sc.setParameters("zoneId", zoneId);
return getCount(sc);
}
@ -508,7 +525,7 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
SearchCriteria<HostVO> sc = HostTypeCountSearch.create();
sc.setParameters("type", Type.Routing);
sc.setParameters("resourceState", ResourceState.Enabled);
sc.setParameters("dc", zoneId);
sc.setParameters("zoneId", zoneId);
return getCount(sc);
}
@ -529,29 +546,23 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
return cpuSockets;
}
@Override
public List<HostVO> listByDataCenterId(long id) {
return listByDataCenterIdAndState(id, ResourceState.Enabled);
}
@Override
public List<HostVO> listByDataCenterIdAndState(long id, ResourceState state) {
SearchCriteria<HostVO> sc = scHostsFromZoneUpRouting(id);
private List<Long> listIdsByDataCenterIdAndResourceState(long id, ResourceState state) {
SearchCriteria<Long> sc = IdsDcSearch.create();
sc.setParameters("zoneId", id);
sc.setParameters("resourceState", state);
return listBy(sc);
sc.setParameters("status", Status.Up);
sc.setParameters("type", Type.Routing);
return customSearch(sc, null);
}
@Override
public List<HostVO> listDisabledByDataCenterId(long id) {
return listByDataCenterIdAndState(id, ResourceState.Disabled);
public List<Long> listEnabledIdsByDataCenterId(long id) {
return listIdsByDataCenterIdAndResourceState(id, ResourceState.Enabled);
}
private SearchCriteria<HostVO> scHostsFromZoneUpRouting(long id) {
SearchCriteria<HostVO> sc = DcSearch.create();
sc.setParameters("dc", id);
sc.setParameters("status", Status.Up);
sc.setParameters("type", Host.Type.Routing);
return sc;
@Override
public List<Long> listDisabledIdsByDataCenterId(long id) {
return listIdsByDataCenterIdAndResourceState(id, ResourceState.Disabled);
}
@Override
@ -1186,6 +1197,14 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
return listBy(sc);
}
@Override
public List<Long> listIdsByDataCenterId(Long zoneId) {
SearchCriteria<Long> sc = IdsDcSearch.create();
sc.setParameters("dc", zoneId);
sc.setParameters("type", Type.Routing);
return customSearch(sc, null);
}
@Override
public List<HostVO> findByPodId(Long podId) {
SearchCriteria<HostVO> sc = PodSearch.create();
@ -1193,6 +1212,16 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
return listBy(sc);
}
@Override
public List<Long> listIdsByPodId(Long podId) {
GenericSearchBuilder<HostVO, Long> sb = createSearchBuilder(Long.class);
sb.and("podId", sb.entity().getPodId(), SearchCriteria.Op.EQ);
sb.done();
SearchCriteria<Long> sc = sb.create();
sc.setParameters("podId", podId);
return customSearch(sc, null);
}
@Override
public List<HostVO> findByClusterId(Long clusterId) {
SearchCriteria<HostVO> sc = ClusterSearch.create();
@ -1200,6 +1229,16 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
return listBy(sc);
}
@Override
public List<Long> listIdsByClusterId(Long clusterId) {
GenericSearchBuilder<HostVO, Long> sb = createSearchBuilder(Long.class);
sb.and("clusterId", sb.entity().getClusterId(), SearchCriteria.Op.EQ);
sb.done();
SearchCriteria<Long> sc = sb.create();
sc.setParameters("clusterId", clusterId);
return customSearch(sc, null);
}
@Override
public List<HostVO> findByClusterIdAndEncryptionSupport(Long clusterId) {
SearchBuilder<DetailVO> hostCapabilitySearch = _detailsDao.createSearchBuilder();
@ -1252,6 +1291,15 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
return listBy(sc);
}
@Override
public HostVO findAnyStateHypervisorHostInCluster(long clusterId) {
SearchCriteria<HostVO> sc = TypeClusterStatusSearch.create();
sc.setParameters("type", Host.Type.Routing);
sc.setParameters("cluster", clusterId);
List<HostVO> list = listBy(sc, new Filter(1));
return list.isEmpty() ? null : list.get(0);
}
@Override
public HostVO findOldestExistentHypervisorHostInCluster(long clusterId) {
SearchCriteria<HostVO> sc = TypeClusterStatusSearch.create();
@ -1510,4 +1558,66 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
}
return String.format(sqlFindHostInZoneToExecuteCommand, hostResourceStatus);
}
@Override
public boolean isHostUp(long hostId) {
GenericSearchBuilder<HostVO, Status> sb = createSearchBuilder(Status.class);
sb.and("id", sb.entity().getId(), Op.EQ);
sb.selectFields(sb.entity().getStatus());
SearchCriteria<Status> sc = sb.create();
List<Status> statuses = customSearch(sc, null);
return CollectionUtils.isNotEmpty(statuses) && Status.Up.equals(statuses.get(0));
}
@Override
public List<Long> findHostIdsByZoneClusterResourceStateAndType(final Long zoneId, final Long clusterId,
final List<ResourceState> resourceStates, final List<Type> types) {
GenericSearchBuilder<HostVO, Long> sb = createSearchBuilder(Long.class);
sb.selectFields(sb.entity().getId());
sb.and("zoneId", sb.entity().getDataCenterId(), SearchCriteria.Op.EQ);
sb.and("clusterId", sb.entity().getClusterId(), SearchCriteria.Op.EQ);
sb.and("resourceState", sb.entity().getResourceState(), SearchCriteria.Op.IN);
sb.and("type", sb.entity().getType(), SearchCriteria.Op.IN);
sb.done();
SearchCriteria<Long> sc = sb.create();
if (zoneId != null) {
sc.setParameters("zoneId", zoneId);
}
if (clusterId != null) {
sc.setParameters("clusterId", clusterId);
}
sc.setParameters("resourceState", resourceStates.toArray());
sc.setParameters("type", types.toArray());
return customSearch(sc, null);
}
@Override
public List<Long> listAllIds() {
return customSearch(HostIdSearch.create(), null);
}
@Override
public List<HypervisorType> listDistinctHypervisorTypes(final Long zoneId) {
GenericSearchBuilder<HostVO, HypervisorType> sb = createSearchBuilder(HypervisorType.class);
sb.and("zoneId", sb.entity().getDataCenterId(), SearchCriteria.Op.EQ);
sb.and("type", sb.entity().getType(), SearchCriteria.Op.EQ);
sb.select(null, Func.DISTINCT, sb.entity().getHypervisorType());
sb.done();
SearchCriteria<HypervisorType> sc = sb.create();
if (zoneId != null) {
sc.setParameters("zoneId", zoneId);
}
sc.setParameters("type", Type.Routing);
return customSearch(sc, null);
}
@Override
public List<HostVO> listByIds(List<Long> ids) {
if (CollectionUtils.isEmpty(ids)) {
return new ArrayList<>();
}
SearchCriteria<HostVO> sc = IdsSearch.create();
sc.setParameters("id", ids.toArray());
return search(sc, null);
}
}

2
engine/schema/src/main/java/com/cloud/service/dao/ServiceOfferingDao.java
View File

@ -56,5 +56,5 @@ public interface ServiceOfferingDao extends GenericDao<ServiceOfferingVO, Long>
ServiceOfferingVO findServiceOfferingByComputeOnlyDiskOffering(long diskOfferingId);
List<ServiceOfferingVO> listByHostTag(String tag);
List<Long> listIdsByHostTag(String tag);
}

11
engine/schema/src/main/java/com/cloud/service/dao/ServiceOfferingDaoImpl.java
View File

@ -35,6 +35,7 @@ import com.cloud.service.ServiceOfferingVO;
import com.cloud.storage.Storage.ProvisioningType;
import com.cloud.utils.db.DB;
import com.cloud.utils.db.GenericDaoBase;
import com.cloud.utils.db.GenericSearchBuilder;
import com.cloud.utils.db.SearchBuilder;
import com.cloud.utils.db.SearchCriteria;
import com.cloud.utils.exception.CloudRuntimeException;
@ -295,8 +296,9 @@ public class ServiceOfferingDaoImpl extends GenericDaoBase<ServiceOfferingVO, Lo
}
@Override
public List<ServiceOfferingVO> listByHostTag(String tag) {
SearchBuilder<ServiceOfferingVO> sb = createSearchBuilder();
public List<Long> listIdsByHostTag(String tag) {
GenericSearchBuilder<ServiceOfferingVO, Long> sb = createSearchBuilder(Long.class);
sb.selectFields(sb.entity().getId());
sb.and("tagNotNull", sb.entity().getHostTag(), SearchCriteria.Op.NNULL);
sb.and().op("tagEq", sb.entity().getHostTag(), SearchCriteria.Op.EQ);
sb.or("tagStartLike", sb.entity().getHostTag(), SearchCriteria.Op.LIKE);
@ -304,11 +306,12 @@ public class ServiceOfferingDaoImpl extends GenericDaoBase<ServiceOfferingVO, Lo
sb.or("tagEndLike", sb.entity().getHostTag(), SearchCriteria.Op.LIKE);
sb.cp();
sb.done();
SearchCriteria<ServiceOfferingVO> sc = sb.create();
SearchCriteria<Long> sc = sb.create();
sc.setParameters("tagEq", tag);
sc.setParameters("tagStartLike", tag + ",%");
sc.setParameters("tagMidLike", "%," + tag + ",%");
sc.setParameters("tagEndLike", "%," + tag);
return listBy(sc);
return customSearch(sc, null);
}
}

2
engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java
View File

@ -91,5 +91,5 @@ public interface VMTemplateDao extends GenericDao<VMTemplateVO, Long>, StateDao<
List<VMTemplateVO> findTemplatesLinkedToUserdata(long userdataId);
List<VMTemplateVO> listByTemplateTag(String tag);
List<Long> listIdsByTemplateTag(String tag);
}

55
engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java
View File

@ -17,6 +17,7 @@
package com.cloud.storage.dao;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Map;
@ -26,6 +27,7 @@ import javax.naming.ConfigurationException;
import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO;
import org.apache.commons.collections.CollectionUtils;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;
@ -343,19 +345,11 @@ public class VMTemplateDaoImpl extends GenericDaoBase<VMTemplateVO, Long> implem
readySystemTemplateSearch = createSearchBuilder();
readySystemTemplateSearch.and("state", readySystemTemplateSearch.entity().getState(), SearchCriteria.Op.EQ);
readySystemTemplateSearch.and("templateType", readySystemTemplateSearch.entity().getTemplateType(), SearchCriteria.Op.EQ);
readySystemTemplateSearch.and("hypervisorType", readySystemTemplateSearch.entity().getHypervisorType(), SearchCriteria.Op.IN);
SearchBuilder<TemplateDataStoreVO> templateDownloadSearch = _templateDataStoreDao.createSearchBuilder();
templateDownloadSearch.and("downloadState", templateDownloadSearch.entity().getDownloadState(), SearchCriteria.Op.IN);
readySystemTemplateSearch.join("vmTemplateJoinTemplateStoreRef", templateDownloadSearch, templateDownloadSearch.entity().getTemplateId(),
readySystemTemplateSearch.entity().getId(), JoinBuilder.JoinType.INNER);
SearchBuilder<HostVO> hostHyperSearch2 = _hostDao.createSearchBuilder();
hostHyperSearch2.and("type", hostHyperSearch2.entity().getType(), SearchCriteria.Op.EQ);
hostHyperSearch2.and("zoneId", hostHyperSearch2.entity().getDataCenterId(), SearchCriteria.Op.EQ);
hostHyperSearch2.and("removed", hostHyperSearch2.entity().getRemoved(), SearchCriteria.Op.NULL);
hostHyperSearch2.groupBy(hostHyperSearch2.entity().getHypervisorType());
readySystemTemplateSearch.join("tmplHyper", hostHyperSearch2, hostHyperSearch2.entity().getHypervisorType(), readySystemTemplateSearch.entity()
.getHypervisorType(), JoinBuilder.JoinType.INNER);
hostHyperSearch2.done();
readySystemTemplateSearch.groupBy(readySystemTemplateSearch.entity().getId());
readySystemTemplateSearch.done();
@ -552,32 +546,34 @@ public class VMTemplateDaoImpl extends GenericDaoBase<VMTemplateVO, Long> implem
@Override
public List<VMTemplateVO> listAllReadySystemVMTemplates(Long zoneId) {
List<HypervisorType> availableHypervisors = _hostDao.listDistinctHypervisorTypes(zoneId);
if (CollectionUtils.isEmpty(availableHypervisors)) {
return Collections.emptyList();
}
SearchCriteria<VMTemplateVO> sc = readySystemTemplateSearch.create();
sc.setParameters("templateType", Storage.TemplateType.SYSTEM);
sc.setParameters("state", VirtualMachineTemplate.State.Active);
sc.setJoinParameters("tmplHyper", "type", Host.Type.Routing);
if (zoneId != null) {
sc.setJoinParameters("tmplHyper", "zoneId", zoneId);
}
sc.setJoinParameters("vmTemplateJoinTemplateStoreRef", "downloadState", new VMTemplateStorageResourceAssoc.Status[] {VMTemplateStorageResourceAssoc.Status.DOWNLOADED, VMTemplateStorageResourceAssoc.Status.BYPASSED});
sc.setParameters("hypervisorType", availableHypervisors.toArray());
sc.setJoinParameters("vmTemplateJoinTemplateStoreRef", "downloadState",
List.of(VMTemplateStorageResourceAssoc.Status.DOWNLOADED,
VMTemplateStorageResourceAssoc.Status.BYPASSED).toArray());
// order by descending order of id
return listBy(sc, new Filter(VMTemplateVO.class, "id", false, null, null));
}
@Override
public VMTemplateVO findSystemVMReadyTemplate(long zoneId, HypervisorType hypervisorType) {
List<VMTemplateVO> tmplts = listAllReadySystemVMTemplates(zoneId);
if (tmplts.size() > 0) {
if (hypervisorType == HypervisorType.Any) {
return tmplts.get(0);
}
for (VMTemplateVO tmplt : tmplts) {
if (tmplt.getHypervisorType() == hypervisorType) {
return tmplt;
}
}
List<VMTemplateVO> templates = listAllReadySystemVMTemplates(zoneId);
if (CollectionUtils.isEmpty(templates)) {
return null;
}
return null;
if (hypervisorType == HypervisorType.Any) {
return templates.get(0);
}
return templates.stream()
.filter(t -> t.getHypervisorType() == hypervisorType)
.findFirst()
.orElse(null);
}
@Override
@ -676,13 +672,14 @@ public class VMTemplateDaoImpl extends GenericDaoBase<VMTemplateVO, Long> implem
}
@Override
public List<VMTemplateVO> listByTemplateTag(String tag) {
SearchBuilder<VMTemplateVO> sb = createSearchBuilder();
public List<Long> listIdsByTemplateTag(String tag) {
GenericSearchBuilder<VMTemplateVO, Long> sb = createSearchBuilder(Long.class);
sb.selectFields(sb.entity().getId());
sb.and("tag", sb.entity().getTemplateTag(), SearchCriteria.Op.EQ);
sb.done();
SearchCriteria<VMTemplateVO> sc = sb.create();
SearchCriteria<Long> sc = sb.create();
sc.setParameters("tag", tag);
return listIncludingRemovedBy(sc);
return customSearchIncludingRemoved(sc, null);
}
@Override

22
engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java
View File

@ -16,6 +16,7 @@
// under the License.
package com.cloud.vm.dao;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
@ -81,7 +82,7 @@ public interface VMInstanceDao extends GenericDao<VMInstanceVO, Long>, StateDao<
List<VMInstanceVO> listByHostAndState(long hostId, State... states);
List<VMInstanceVO> listByTypes(VirtualMachine.Type... types);
int countByTypes(VirtualMachine.Type... types);
VMInstanceVO findByIdTypes(long id, VirtualMachine.Type... types);
@ -144,7 +145,7 @@ public interface VMInstanceDao extends GenericDao<VMInstanceVO, Long>, StateDao<
*/
List<String> listDistinctHostNames(long networkId, VirtualMachine.Type... types);
List<VMInstanceVO> findByHostInStatesExcluding(Long hostId, List<Long> excludingIds, State... states);
List<VMInstanceVO> findByHostInStatesExcluding(Long hostId, Collection<Long> excludingIds, State... states);
List<VMInstanceVO> findByHostInStates(Long hostId, State... states);
@ -152,15 +153,20 @@ public interface VMInstanceDao extends GenericDao<VMInstanceVO, Long>, StateDao<
boolean updatePowerState(long instanceId, long powerHostId, VirtualMachine.PowerState powerState, Date wisdomEra);
Map<Long, VirtualMachine.PowerState> updatePowerState(Map<Long, VirtualMachine.PowerState> instancePowerStates,
long powerHostId, Date wisdomEra);
void resetVmPowerStateTracking(long instanceId);
void resetVmPowerStateTracking(List<Long> instanceId);
void resetHostPowerStateTracking(long hostId);
HashMap<String, Long> countVgpuVMs(Long dcId, Long podId, Long clusterId);
VMInstanceVO findVMByHostNameInZone(String hostName, long zoneId);
boolean isPowerStateUpToDate(long instanceId);
boolean isPowerStateUpToDate(VMInstanceVO instance);
List<VMInstanceVO> listNonMigratingVmsByHostEqualsLastHost(long hostId);
@ -170,4 +176,14 @@ public interface VMInstanceDao extends GenericDao<VMInstanceVO, Long>, StateDao<
List<VMInstanceVO> searchRemovedByRemoveDate(final Date startDate, final Date endDate, final Long batchSize,
List<Long> skippedVmIds);
List<VMInstanceVO> listIdServiceOfferingForUpVmsByHostId(Long hostId);
List<VMInstanceVO> listIdServiceOfferingForVmsMigratingFromHost(Long hostId);
List<VMInstanceVO> listIdServiceOfferingForVmsByLastHostId(Long hostId);
Map<String, Long> getNameIdMapForVmInstanceNames(Collection<String> names);
Map<String, Long> getNameIdMapForVmIds(Collection<Long> ids);
}

222
engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java
View File

@ -20,10 +20,12 @@ import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import javax.annotation.PostConstruct;
import javax.inject.Inject;
@ -74,6 +76,7 @@ public class VMInstanceDaoImpl extends GenericDaoBase<VMInstanceVO, Long> implem
protected SearchBuilder<VMInstanceVO> LHVMClusterSearch;
protected SearchBuilder<VMInstanceVO> IdStatesSearch;
protected SearchBuilder<VMInstanceVO> AllFieldsSearch;
protected SearchBuilder<VMInstanceVO> IdServiceOfferingIdSelectSearch;
protected SearchBuilder<VMInstanceVO> ZoneTemplateNonExpungedSearch;
protected SearchBuilder<VMInstanceVO> TemplateNonExpungedSearch;
protected SearchBuilder<VMInstanceVO> NameLikeSearch;
@ -99,6 +102,7 @@ public class VMInstanceDaoImpl extends GenericDaoBase<VMInstanceVO, Long> implem
protected SearchBuilder<VMInstanceVO> NotMigratingSearch;
protected SearchBuilder<VMInstanceVO> BackupSearch;
protected SearchBuilder<VMInstanceVO> LastHostAndStatesSearch;
protected SearchBuilder<VMInstanceVO> IdsPowerStateSelectSearch;
@Inject
ResourceTagDao _tagsDao;
@ -171,6 +175,14 @@ public class VMInstanceDaoImpl extends GenericDaoBase<VMInstanceVO, Long> implem
AllFieldsSearch.and("account", AllFieldsSearch.entity().getAccountId(), Op.EQ);
AllFieldsSearch.done();
IdServiceOfferingIdSelectSearch = createSearchBuilder();
IdServiceOfferingIdSelectSearch.and("host", IdServiceOfferingIdSelectSearch.entity().getHostId(), Op.EQ);
IdServiceOfferingIdSelectSearch.and("lastHost", IdServiceOfferingIdSelectSearch.entity().getLastHostId(), Op.EQ);
IdServiceOfferingIdSelectSearch.and("state", IdServiceOfferingIdSelectSearch.entity().getState(), Op.EQ);
IdServiceOfferingIdSelectSearch.and("states", IdServiceOfferingIdSelectSearch.entity().getState(), Op.IN);
IdServiceOfferingIdSelectSearch.selectFields(IdServiceOfferingIdSelectSearch.entity().getId(), IdServiceOfferingIdSelectSearch.entity().getServiceOfferingId());
IdServiceOfferingIdSelectSearch.done();
ZoneTemplateNonExpungedSearch = createSearchBuilder();
ZoneTemplateNonExpungedSearch.and("zone", ZoneTemplateNonExpungedSearch.entity().getDataCenterId(), Op.EQ);
ZoneTemplateNonExpungedSearch.and("template", ZoneTemplateNonExpungedSearch.entity().getTemplateId(), Op.EQ);
@ -310,6 +322,15 @@ public class VMInstanceDaoImpl extends GenericDaoBase<VMInstanceVO, Long> implem
LastHostAndStatesSearch.and("states", LastHostAndStatesSearch.entity().getState(), Op.IN);
LastHostAndStatesSearch.done();
IdsPowerStateSelectSearch = createSearchBuilder();
IdsPowerStateSelectSearch.and("id", IdsPowerStateSelectSearch.entity().getId(), Op.IN);
IdsPowerStateSelectSearch.selectFields(IdsPowerStateSelectSearch.entity().getId(),
IdsPowerStateSelectSearch.entity().getPowerHostId(),
IdsPowerStateSelectSearch.entity().getPowerState(),
IdsPowerStateSelectSearch.entity().getPowerStateUpdateCount(),
IdsPowerStateSelectSearch.entity().getPowerStateUpdateTime());
IdsPowerStateSelectSearch.done();
}
@Override
@ -445,10 +466,10 @@ public class VMInstanceDaoImpl extends GenericDaoBase<VMInstanceVO, Long> implem
}
@Override
public List<VMInstanceVO> listByTypes(Type... types) {
public int countByTypes(Type... types) {
SearchCriteria<VMInstanceVO> sc = TypesSearch.create();
sc.setParameters("types", (Object[])types);
return listBy(sc);
return getCount(sc);
}
@Override
@ -884,7 +905,7 @@ public class VMInstanceDaoImpl extends GenericDaoBase<VMInstanceVO, Long> implem
}
@Override
public List<VMInstanceVO> findByHostInStatesExcluding(Long hostId, List<Long> excludingIds, State... states) {
public List<VMInstanceVO> findByHostInStatesExcluding(Long hostId, Collection<Long> excludingIds, State... states) {
SearchCriteria<VMInstanceVO> sc = HostAndStateSearch.create();
sc.setParameters("host", hostId);
if (excludingIds != null && !excludingIds.isEmpty()) {
@ -909,42 +930,107 @@ public class VMInstanceDaoImpl extends GenericDaoBase<VMInstanceVO, Long> implem
return listBy(sc);
}
protected List<VMInstanceVO> listSelectPowerStateByIds(final List<Long> ids) {
if (CollectionUtils.isEmpty(ids)) {
return new ArrayList<>();
}
SearchCriteria<VMInstanceVO> sc = IdsPowerStateSelectSearch.create();
sc.setParameters("id", ids.toArray());
return customSearch(sc, null);
}
protected Integer getPowerUpdateCount(final VMInstanceVO instance, final long powerHostId, final VirtualMachine.PowerState powerState, Date wisdomEra) {
if (instance.getPowerStateUpdateTime() == null || instance.getPowerStateUpdateTime().before(wisdomEra)) {
Long savedPowerHostId = instance.getPowerHostId();
boolean isStateMismatch = instance.getPowerState() != powerState
|| savedPowerHostId == null
|| !savedPowerHostId.equals(powerHostId)
|| !isPowerStateInSyncWithInstanceState(powerState, powerHostId, instance);
if (isStateMismatch) {
return 1;
} else if (instance.getPowerStateUpdateCount() < MAX_CONSECUTIVE_SAME_STATE_UPDATE_COUNT) {
return instance.getPowerStateUpdateCount() + 1;
}
}
return null;
}
@Override
public boolean updatePowerState(final long instanceId, final long powerHostId, final VirtualMachine.PowerState powerState, Date wisdomEra) {
return Transaction.execute(new TransactionCallback<>() {
@Override
public Boolean doInTransaction(TransactionStatus status) {
boolean needToUpdate = false;
VMInstanceVO instance = findById(instanceId);
if (instance != null
&& (null == instance.getPowerStateUpdateTime()
|| instance.getPowerStateUpdateTime().before(wisdomEra))) {
Long savedPowerHostId = instance.getPowerHostId();
if (instance.getPowerState() != powerState
|| savedPowerHostId == null
|| savedPowerHostId != powerHostId
|| !isPowerStateInSyncWithInstanceState(powerState, powerHostId, instance)) {
instance.setPowerState(powerState);
instance.setPowerHostId(powerHostId);
instance.setPowerStateUpdateCount(1);
instance.setPowerStateUpdateTime(DateUtil.currentGMTTime());
needToUpdate = true;
update(instanceId, instance);
} else {
// to reduce DB updates, consecutive same state update for more than 3 times
if (instance.getPowerStateUpdateCount() < MAX_CONSECUTIVE_SAME_STATE_UPDATE_COUNT) {
instance.setPowerStateUpdateCount(instance.getPowerStateUpdateCount() + 1);
instance.setPowerStateUpdateTime(DateUtil.currentGMTTime());
needToUpdate = true;
update(instanceId, instance);
}
}
}
return needToUpdate;
return Transaction.execute((TransactionCallback<Boolean>) status -> {
VMInstanceVO instance = findById(instanceId);
if (instance == null) {
return false;
}
// Check if we need to update based on powerStateUpdateTime
if (instance.getPowerStateUpdateTime() == null || instance.getPowerStateUpdateTime().before(wisdomEra)) {
Long savedPowerHostId = instance.getPowerHostId();
boolean isStateMismatch = instance.getPowerState() != powerState
|| savedPowerHostId == null
|| !savedPowerHostId.equals(powerHostId)
|| !isPowerStateInSyncWithInstanceState(powerState, powerHostId, instance);
if (isStateMismatch) {
instance.setPowerState(powerState);
instance.setPowerHostId(powerHostId);
instance.setPowerStateUpdateCount(1);
} else if (instance.getPowerStateUpdateCount() < MAX_CONSECUTIVE_SAME_STATE_UPDATE_COUNT) {
instance.setPowerStateUpdateCount(instance.getPowerStateUpdateCount() + 1);
} else {
// No need to update if power state is already in sync and count exceeded
return false;
}
instance.setPowerStateUpdateTime(DateUtil.currentGMTTime());
update(instanceId, instance);
return true; // Return true since an update occurred
}
return false;
});
}
@Override
public Map<Long, VirtualMachine.PowerState> updatePowerState(Map<Long, VirtualMachine.PowerState> instancePowerStates, long powerHostId, Date wisdomEra) {
Map<Long, VirtualMachine.PowerState> notUpdated = new HashMap<>();
List<VMInstanceVO> instances = listSelectPowerStateByIds(new ArrayList<>(instancePowerStates.keySet()));
Map<Long, Integer> updateCounts = new HashMap<>();
for (VMInstanceVO instance : instances) {
VirtualMachine.PowerState powerState = instancePowerStates.get(instance.getId());
Integer count = getPowerUpdateCount(instance, powerHostId, powerState, wisdomEra);
if (count != null) {
updateCounts.put(instance.getId(), count);
} else {
notUpdated.put(instance.getId(), powerState);
}
}
if (updateCounts.isEmpty()) {
return notUpdated;
}
StringBuilder sql = new StringBuilder("UPDATE vm_instance SET " +
"power_host = ?, power_state_update_time = now(), power_state = CASE ");
updateCounts.keySet().forEach(key -> {
sql.append("WHEN id = ").append(key).append(" THEN '").append(instancePowerStates.get(key)).append("' ");
});
sql.append("END, power_state_update_count = CASE ");
StringBuilder idList = new StringBuilder();
updateCounts.forEach((key, value) -> {
sql.append("WHEN id = ").append(key).append(" THEN ").append(value).append(" ");
idList.append(key).append(",");
});
idList.setLength(idList.length() - 1);
sql.append("END WHERE id IN (").append(idList).append(")");
try (TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.CLOUD_DB)) {
try (PreparedStatement pstmt = txn.prepareAutoCloseStatement(sql.toString())) {
pstmt.setLong(1, powerHostId);
pstmt.executeUpdate();
} catch (SQLException e) {
s_logger.error(String.format("Unable to execute update power states SQL from VMs %s due to: %s",
idList, e.getMessage()), e);
return instancePowerStates;
}
}
return notUpdated;
}
private boolean isPowerStateInSyncWithInstanceState(final VirtualMachine.PowerState powerState, final long powerHostId, final VMInstanceVO instance) {
State instanceState = instance.getState();
if ((powerState == VirtualMachine.PowerState.PowerOff && instanceState == State.Running)
@ -957,11 +1043,7 @@ public class VMInstanceDaoImpl extends GenericDaoBase<VMInstanceVO, Long> implem
}
@Override
public boolean isPowerStateUpToDate(final long instanceId) {
VMInstanceVO instance = findById(instanceId);
if(instance == null) {
throw new CloudRuntimeException("checking power state update count on non existing instance " + instanceId);
}
public boolean isPowerStateUpToDate(final VMInstanceVO instance) {
return instance.getPowerStateUpdateCount() < MAX_CONSECUTIVE_SAME_STATE_UPDATE_COUNT;
}
@ -980,6 +1062,22 @@ public class VMInstanceDaoImpl extends GenericDaoBase<VMInstanceVO, Long> implem
});
}
@Override
public void resetVmPowerStateTracking(List<Long> instanceIds) {
Transaction.execute(new TransactionCallbackNoReturn() {
@Override
public void doInTransactionWithoutResult(TransactionStatus status) {
SearchCriteria<VMInstanceVO> sc = IdsPowerStateSelectSearch.create();
sc.setParameters("id", instanceIds.toArray());
VMInstanceVO vm = createForUpdate();
vm.setPowerStateUpdateCount(0);
vm.setPowerStateUpdateTime(DateUtil.currentGMTTime());
UpdateBuilder ub = getUpdateBuilder(vm);
update(ub, sc, null);
}
});
}
@Override @DB
public void resetHostPowerStateTracking(final long hostId) {
Transaction.execute(new TransactionCallbackNoReturn() {
@ -1054,4 +1152,52 @@ public class VMInstanceDaoImpl extends GenericDaoBase<VMInstanceVO, Long> implem
Filter filter = new Filter(VMInstanceVO.class, "id", true, 0L, batchSize);
return searchIncludingRemoved(sc, filter, null, false);
}
@Override
public List<VMInstanceVO> listIdServiceOfferingForUpVmsByHostId(Long hostId) {
SearchCriteria<VMInstanceVO> sc = IdServiceOfferingIdSelectSearch.create();
sc.setParameters("host", hostId);
sc.setParameters("states", new Object[] {State.Starting, State.Running, State.Stopping, State.Migrating});
return customSearch(sc, null);
}
@Override
public List<VMInstanceVO> listIdServiceOfferingForVmsMigratingFromHost(Long hostId) {
SearchCriteria<VMInstanceVO> sc = IdServiceOfferingIdSelectSearch.create();
sc.setParameters("lastHost", hostId);
sc.setParameters("state", State.Migrating);
return customSearch(sc, null);
}
@Override
public List<VMInstanceVO> listIdServiceOfferingForVmsByLastHostId(Long hostId) {
SearchCriteria<VMInstanceVO> sc = IdServiceOfferingIdSelectSearch.create();
sc.setParameters("lastHost", hostId);
sc.setParameters("state", State.Stopped);
return customSearch(sc, null);
}
@Override
public Map<String, Long> getNameIdMapForVmInstanceNames(Collection<String> names) {
SearchBuilder<VMInstanceVO> sb = createSearchBuilder();
sb.and("name", sb.entity().getInstanceName(), Op.IN);
sb.selectFields(sb.entity().getId(), sb.entity().getInstanceName());
SearchCriteria<VMInstanceVO> sc = sb.create();
sc.setParameters("name", names.toArray());
List<VMInstanceVO> vms = customSearch(sc, null);
return vms.stream()
.collect(Collectors.toMap(VMInstanceVO::getInstanceName, VMInstanceVO::getId));
}
@Override
public Map<String, Long> getNameIdMapForVmIds(Collection<Long> ids) {
SearchBuilder<VMInstanceVO> sb = createSearchBuilder();
sb.and("id", sb.entity().getId(), Op.IN);
sb.selectFields(sb.entity().getId(), sb.entity().getInstanceName());
SearchCriteria<VMInstanceVO> sc = sb.create();
sc.setParameters("id", ids.toArray());
List<VMInstanceVO> vms = customSearch(sc, null);
return vms.stream()
.collect(Collectors.toMap(VMInstanceVO::getInstanceName, VMInstanceVO::getId));
}
}

2
engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/ResourceDetailsDao.java
View File

@ -88,6 +88,8 @@ public interface ResourceDetailsDao<R extends ResourceDetail> extends GenericDao
public Map<String, String> listDetailsKeyPairs(long resourceId);
Map<String, String> listDetailsKeyPairs(long resourceId, List<String> keys);
public Map<String, String> listDetailsKeyPairs(long resourceId, boolean forDisplay);
Map<String, Boolean> listDetailsVisibility(long resourceId);

15
engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/ResourceDetailsDaoBase.java
View File

@ -19,6 +19,7 @@ package org.apache.cloudstack.resourcedetail;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.apache.cloudstack.api.ResourceDetail;
import org.apache.commons.collections.CollectionUtils;
@ -91,6 +92,20 @@ public abstract class ResourceDetailsDaoBase<R extends ResourceDetail> extends G
return details;
}
@Override
public Map<String, String> listDetailsKeyPairs(long resourceId, List<String> keys) {
SearchBuilder<R> sb = createSearchBuilder();
sb.and("resourceId", sb.entity().getResourceId(), SearchCriteria.Op.EQ);
sb.and("name", sb.entity().getName(), SearchCriteria.Op.IN);
sb.done();
SearchCriteria<R> sc = sb.create();
sc.setParameters("resourceId", resourceId);
sc.setParameters("name", keys.toArray());
List<R> results = search(sc, null);
return results.stream().collect(Collectors.toMap(R::getName, R::getValue));
}
public Map<String, Boolean> listDetailsVisibility(long resourceId) {
SearchCriteria<R> sc = AllFieldsSearch.create();
sc.setParameters("resourceId", resourceId);

4
framework/agent-lb/src/main/java/org/apache/cloudstack/agent/lb/IndirectAgentLBAlgorithm.java
View File

@ -42,4 +42,8 @@ public interface IndirectAgentLBAlgorithm {
* @return true if the lists are equal, false if not
*/
boolean compare(final List<String> msList, final List<String> receivedMsList);
default boolean isHostListNeeded() {
return false;
}
}

14
framework/config/src/main/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImpl.java
View File

@ -23,7 +23,6 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import javax.annotation.PostConstruct;
import javax.inject.Inject;
@ -36,6 +35,7 @@ import org.apache.cloudstack.framework.config.ScopedConfigStorage;
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.apache.cloudstack.framework.config.dao.ConfigurationGroupDao;
import org.apache.cloudstack.framework.config.dao.ConfigurationSubGroupDao;
import org.apache.cloudstack.utils.cache.LazyCache;
import org.apache.commons.lang.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
@ -43,8 +43,6 @@ import org.apache.log4j.Logger;
import com.cloud.utils.Pair;
import com.cloud.utils.Ternary;
import com.cloud.utils.exception.CloudRuntimeException;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
/**
* ConfigDepotImpl implements the ConfigDepot and ConfigDepotAdmin interface.
@ -86,17 +84,15 @@ public class ConfigDepotImpl implements ConfigDepot, ConfigDepotAdmin {
List<ScopedConfigStorage> _scopedStorages;
Set<Configurable> _configured = Collections.synchronizedSet(new HashSet<Configurable>());
Set<String> newConfigs = Collections.synchronizedSet(new HashSet<>());
Cache<String, String> configCache;
LazyCache<String, String> configCache;
private HashMap<String, Pair<String, ConfigKey<?>>> _allKeys = new HashMap<String, Pair<String, ConfigKey<?>>>(1007);
HashMap<ConfigKey.Scope, Set<ConfigKey<?>>> _scopeLevelConfigsMap = new HashMap<ConfigKey.Scope, Set<ConfigKey<?>>>();
public ConfigDepotImpl() {
configCache = Caffeine.newBuilder()
.maximumSize(512)
.expireAfterWrite(CONFIG_CACHE_EXPIRE_SECONDS, TimeUnit.SECONDS)
.build();
configCache = new LazyCache<>(512,
CONFIG_CACHE_EXPIRE_SECONDS, this::getConfigStringValueInternal);
ConfigKey.init(this);
createEmptyScopeLevelMappings();
}
@ -310,7 +306,7 @@ public class ConfigDepotImpl implements ConfigDepot, ConfigDepotAdmin {
@Override
public String getConfigStringValue(String key, ConfigKey.Scope scope, Long scopeId) {
return configCache.get(getConfigCacheKey(key, scope, scopeId), this::getConfigStringValueInternal);
return configCache.get(getConfigCacheKey(key, scope, scopeId));
}
@Override

1
framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/VmWorkJobDao.java
View File

@ -40,4 +40,5 @@ public interface VmWorkJobDao extends GenericDao<VmWorkJobVO, Long> {
void expungeLeftoverWorkJobs(long msid);
int expungeByVmList(List<Long> vmIds, Long batchSize);
List<Long> listVmIdsWithPendingJob();
}

16
framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/VmWorkJobDaoImpl.java
View File

@ -24,6 +24,7 @@ import java.util.List;
import javax.annotation.PostConstruct;
import javax.inject.Inject;
import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO;
import org.apache.cloudstack.framework.jobs.impl.VmWorkJobVO;
import org.apache.cloudstack.framework.jobs.impl.VmWorkJobVO.Step;
import org.apache.cloudstack.jobs.JobInfo;
@ -33,6 +34,8 @@ import org.apache.log4j.Logger;
import com.cloud.utils.DateUtil;
import com.cloud.utils.db.Filter;
import com.cloud.utils.db.GenericDaoBase;
import com.cloud.utils.db.GenericSearchBuilder;
import com.cloud.utils.db.JoinBuilder;
import com.cloud.utils.db.SearchBuilder;
import com.cloud.utils.db.SearchCriteria;
import com.cloud.utils.db.SearchCriteria.Op;
@ -226,4 +229,17 @@ public class VmWorkJobDaoImpl extends GenericDaoBase<VmWorkJobVO, Long> implemen
sc.setParameters("vmIds", vmIds.toArray());
return batchExpunge(sc, batchSize);
}
@Override
public List<Long> listVmIdsWithPendingJob() {
GenericSearchBuilder<VmWorkJobVO, Long> sb = createSearchBuilder(Long.class);
SearchBuilder<AsyncJobVO> asyncJobSearch = _baseJobDao.createSearchBuilder();
asyncJobSearch.and("status", asyncJobSearch.entity().getStatus(), SearchCriteria.Op.EQ);
sb.join("asyncJobSearch", asyncJobSearch, sb.entity().getId(), asyncJobSearch.entity().getId(), JoinBuilder.JoinType.INNER);
sb.and("removed", sb.entity().getRemoved(), Op.NULL);
sb.selectFields(sb.entity().getVmInstanceId());
SearchCriteria<Long> sc = sb.create();
sc.setJoinParameters("asyncJobSearch", "status", JobInfo.Status.IN_PROGRESS);
return customSearch(sc, null);
}
}

53
plugins/acl/dynamic-role-based/src/main/java/org/apache/cloudstack/acl/DynamicRoleBasedAPIAccessChecker.java
View File

@ -27,6 +27,7 @@ import javax.inject.Inject;
import javax.naming.ConfigurationException;
import org.apache.cloudstack.api.APICommand;
import org.apache.cloudstack.utils.cache.LazyCache;
import org.apache.log4j.Logger;
import org.apache.cloudstack.acl.RolePermissionEntity.Permission;
@ -35,6 +36,7 @@ import com.cloud.exception.UnavailableCommandException;
import com.cloud.user.Account;
import com.cloud.user.AccountService;
import com.cloud.user.User;
import com.cloud.utils.Pair;
import com.cloud.utils.component.AdapterBase;
import com.cloud.utils.component.PluggableService;
import org.apache.commons.lang3.StringUtils;
@ -49,6 +51,9 @@ public class DynamicRoleBasedAPIAccessChecker extends AdapterBase implements API
private List<PluggableService> services;
private Map<RoleType, Set<String>> annotationRoleBasedApisMap = new HashMap<RoleType, Set<String>>();
final private LazyCache<Long, Account> accountCache;
final private LazyCache<Long, Pair<Role, List<RolePermission>>> rolePermissionsCache;
private static final Logger LOGGER = Logger.getLogger(DynamicRoleBasedAPIAccessChecker.class.getName());
protected DynamicRoleBasedAPIAccessChecker() {
@ -56,6 +61,10 @@ public class DynamicRoleBasedAPIAccessChecker extends AdapterBase implements API
for (RoleType roleType : RoleType.values()) {
annotationRoleBasedApisMap.put(roleType, new HashSet<String>());
}
accountCache = new LazyCache<>(32, 60,
this::getAccountFromId);
rolePermissionsCache = new LazyCache<>(32, 60,
this::getRolePermissions);
}
@Override
@ -101,24 +110,52 @@ public class DynamicRoleBasedAPIAccessChecker extends AdapterBase implements API
annotationRoleBasedApisMap.get(role.getRoleType()).contains(apiName);
}
protected Account getAccountFromId(long accountId) {
return accountService.getAccount(accountId);
}
protected Pair<Role, List<RolePermission>> getRolePermissions(long roleId) {
final Role accountRole = roleService.findRole(roleId);
if (accountRole == null || accountRole.getId() < 1L) {
return new Pair<>(null, null);
}
if (accountRole.getRoleType() == RoleType.Admin && accountRole.getId() == RoleType.Admin.getId()) {
return new Pair<>(accountRole, null);
}
return new Pair<>(accountRole, roleService.findAllPermissionsBy(accountRole.getId()));
}
@Override
public boolean checkAccess(User user, String commandName) throws PermissionDeniedException {
if (!isEnabled()) {
return true;
}
// FIXME: potential hotspot, could be cached?
Account account = accountService.getAccount(user.getAccountId());
Account account = accountCache.get(user.getAccountId());
if (account == null) {
throw new PermissionDeniedException(String.format("The account id [%s] for user id [%s] is null.", user.getAccountId(), user.getUuid()));
throw new PermissionDeniedException(String.format("Account for user id [%s] cannot be found", user.getUuid()));
}
return checkAccess(account, commandName);
Pair<Role, List<RolePermission>> roleAndPermissions = rolePermissionsCache.get(account.getRoleId());
final Role accountRole = roleAndPermissions.first();
if (accountRole == null) {
throw new PermissionDeniedException(String.format("Account role for user id [%s] cannot be found.", user.getUuid()));
}
if (accountRole.getRoleType() == RoleType.Admin && accountRole.getId() == RoleType.Admin.getId()) {
LOGGER.info(String.format("Account for user id [%s] is Root Admin or Domain Admin, all APIs are allowed.", user.getUuid()));
return true;
}
List<RolePermission> allPermissions = roleAndPermissions.second();
if (checkApiPermissionByRole(accountRole, commandName, allPermissions)) {
return true;
}
throw new UnavailableCommandException(String.format("The API [%s] does not exist or is not available for the account for user id [%s].", commandName, user.getUuid()));
}
public boolean checkAccess(Account account, String commandName) {
final Role accountRole = roleService.findRole(account.getRoleId());
if (accountRole == null || accountRole.getId() < 1L) {
Pair<Role, List<RolePermission>> roleAndPermissions = rolePermissionsCache.get(account.getRoleId());
final Role accountRole = roleAndPermissions.first();
if (accountRole == null) {
throw new PermissionDeniedException(String.format("The account [%s] has role null or unknown.", account));
}

18
plugins/affinity-group-processors/explicit-dedication/src/main/java/org/apache/cloudstack/affinity/ExplicitDedicationProcessor.java
View File

@ -323,13 +323,13 @@ public class ExplicitDedicationProcessor extends AffinityProcessorBase implement
}
}
//add all hosts inside this in includeList
List<HostVO> hostList = _hostDao.listByDataCenterId(dr.getDataCenterId());
for (HostVO host : hostList) {
DedicatedResourceVO dHost = _dedicatedDao.findByHostId(host.getId());
List<Long> hostList = _hostDao.listEnabledIdsByDataCenterId(dr.getDataCenterId());
for (Long hostId : hostList) {
DedicatedResourceVO dHost = _dedicatedDao.findByHostId(hostId);
if (dHost != null && !dedicatedResources.contains(dHost)) {
avoidList.addHost(host.getId());
avoidList.addHost(hostId);
} else {
includeList.addHost(host.getId());
includeList.addHost(hostId);
}
}
}
@ -339,7 +339,7 @@ public class ExplicitDedicationProcessor extends AffinityProcessorBase implement
List<HostPodVO> pods = _podDao.listByDataCenterId(dc.getId());
List<ClusterVO> clusters = _clusterDao.listClustersByDcId(dc.getId());
List<HostVO> hosts = _hostDao.listByDataCenterId(dc.getId());
List<Long> hostIds = _hostDao.listEnabledIdsByDataCenterId(dc.getId());
Set<Long> podsInIncludeList = includeList.getPodsToAvoid();
Set<Long> clustersInIncludeList = includeList.getClustersToAvoid();
Set<Long> hostsInIncludeList = includeList.getHostsToAvoid();
@ -359,9 +359,9 @@ public class ExplicitDedicationProcessor extends AffinityProcessorBase implement
}
}
for (HostVO host : hosts) {
if (hostsInIncludeList != null && !hostsInIncludeList.contains(host.getId())) {
avoidList.addHost(host.getId());
for (Long hostId : hostIds) {
if (hostsInIncludeList != null && !hostsInIncludeList.contains(hostId)) {
avoidList.addHost(hostId);
}
}
return avoidList;

41
plugins/dedicated-resources/src/main/java/org/apache/cloudstack/dedicated/DedicatedResourceManagerImpl.java
View File

@ -124,7 +124,7 @@ public class DedicatedResourceManagerImpl implements DedicatedService {
@ActionEvent(eventType = EventTypes.EVENT_DEDICATE_RESOURCE, eventDescription = "dedicating a Zone")
public List<DedicatedResourceVO> dedicateZone(final Long zoneId, final Long domainId, final String accountName) {
Long accountId = null;
List<HostVO> hosts = null;
List<Long> hostIds = null;
if (accountName != null) {
Account caller = CallContext.current().getCallingAccount();
Account owner = _accountMgr.finalizeOwner(caller, accountName, domainId, null);
@ -201,17 +201,19 @@ public class DedicatedResourceManagerImpl implements DedicatedService {
releaseDedicatedResource(null, null, dr.getClusterId(), null);
}
hosts = _hostDao.listByDataCenterId(dc.getId());
for (HostVO host : hosts) {
DedicatedResourceVO dHost = _dedicatedDao.findByHostId(host.getId());
hostIds = _hostDao.listEnabledIdsByDataCenterId(dc.getId());
for (Long hostId : hostIds) {
DedicatedResourceVO dHost = _dedicatedDao.findByHostId(hostId);
if (dHost != null) {
if (!(childDomainIds.contains(dHost.getDomainId()))) {
HostVO host = _hostDao.findById(hostId);
throw new CloudRuntimeException("Host " + host.getName() + " under this Zone " + dc.getName() + " is dedicated to different account/domain");
}
if (accountId != null) {
if (dHost.getAccountId().equals(accountId)) {
hostsToRelease.add(dHost);
} else {
HostVO host = _hostDao.findById(hostId);
s_logger.error("Host " + host.getName() + " under this Zone " + dc.getName() + " is dedicated to different account/domain");
throw new CloudRuntimeException("Host " + host.getName() + " under this Zone " + dc.getName() + " is dedicated to different account/domain");
}
@ -228,7 +230,7 @@ public class DedicatedResourceManagerImpl implements DedicatedService {
}
}
checkHostsSuitabilityForExplicitDedication(accountId, childDomainIds, hosts);
checkHostsSuitabilityForExplicitDedication(accountId, childDomainIds, hostIds);
final Long accountIdFinal = accountId;
return Transaction.execute(new TransactionCallback<List<DedicatedResourceVO>>() {
@ -282,7 +284,7 @@ public class DedicatedResourceManagerImpl implements DedicatedService {
childDomainIds.add(domainId);
checkAccountAndDomain(accountId, domainId);
HostPodVO pod = _podDao.findById(podId);
List<HostVO> hosts = null;
List<Long> hostIds = null;
if (pod == null) {
throw new InvalidParameterValueException("Unable to find pod by id " + podId);
} else {
@ -337,17 +339,19 @@ public class DedicatedResourceManagerImpl implements DedicatedService {
releaseDedicatedResource(null, null, dr.getClusterId(), null);
}
hosts = _hostDao.findByPodId(pod.getId());
for (HostVO host : hosts) {
DedicatedResourceVO dHost = _dedicatedDao.findByHostId(host.getId());
hostIds = _hostDao.listIdsByPodId(pod.getId());
for (Long hostId : hostIds) {
DedicatedResourceVO dHost = _dedicatedDao.findByHostId(hostId);
if (dHost != null) {
if (!(getDomainChildIds(domainId).contains(dHost.getDomainId()))) {
HostVO host = _hostDao.findById(hostId);
throw new CloudRuntimeException("Host " + host.getName() + " under this Pod " + pod.getName() + " is dedicated to different account/domain");
}
if (accountId != null) {
if (dHost.getAccountId().equals(accountId)) {
hostsToRelease.add(dHost);
} else {
HostVO host = _hostDao.findById(hostId);
s_logger.error("Host " + host.getName() + " under this Pod " + pod.getName() + " is dedicated to different account/domain");
throw new CloudRuntimeException("Host " + host.getName() + " under this Pod " + pod.getName() + " is dedicated to different account/domain");
}
@ -364,7 +368,7 @@ public class DedicatedResourceManagerImpl implements DedicatedService {
}
}
checkHostsSuitabilityForExplicitDedication(accountId, childDomainIds, hosts);
checkHostsSuitabilityForExplicitDedication(accountId, childDomainIds, hostIds);
final Long accountIdFinal = accountId;
return Transaction.execute(new TransactionCallback<List<DedicatedResourceVO>>() {
@ -400,7 +404,7 @@ public class DedicatedResourceManagerImpl implements DedicatedService {
@ActionEvent(eventType = EventTypes.EVENT_DEDICATE_RESOURCE, eventDescription = "dedicating a Cluster")
public List<DedicatedResourceVO> dedicateCluster(final Long clusterId, final Long domainId, final String accountName) {
Long accountId = null;
List<HostVO> hosts = null;
List<Long> hostIds = null;
if (accountName != null) {
Account caller = CallContext.current().getCallingAccount();
Account owner = _accountMgr.finalizeOwner(caller, accountName, domainId, null);
@ -446,12 +450,13 @@ public class DedicatedResourceManagerImpl implements DedicatedService {
}
//check if any resource under this cluster is dedicated to different account or sub-domain
hosts = _hostDao.findByClusterId(cluster.getId());
hostIds = _hostDao.listIdsByClusterId(cluster.getId());
List<DedicatedResourceVO> hostsToRelease = new ArrayList<DedicatedResourceVO>();
for (HostVO host : hosts) {
DedicatedResourceVO dHost = _dedicatedDao.findByHostId(host.getId());
for (Long hostId : hostIds) {
DedicatedResourceVO dHost = _dedicatedDao.findByHostId(hostId);
if (dHost != null) {
if (!(childDomainIds.contains(dHost.getDomainId()))) {
HostVO host = _hostDao.findById(hostId);
throw new CloudRuntimeException("Host " + host.getName() + " under this Cluster " + cluster.getName() +
" is dedicated to different account/domain");
}
@ -477,7 +482,7 @@ public class DedicatedResourceManagerImpl implements DedicatedService {
}
}
checkHostsSuitabilityForExplicitDedication(accountId, childDomainIds, hosts);
checkHostsSuitabilityForExplicitDedication(accountId, childDomainIds, hostIds);
final Long accountIdFinal = accountId;
return Transaction.execute(new TransactionCallback<List<DedicatedResourceVO>>() {
@ -682,10 +687,10 @@ public class DedicatedResourceManagerImpl implements DedicatedService {
return suitable;
}
private boolean checkHostsSuitabilityForExplicitDedication(Long accountId, List<Long> domainIds, List<HostVO> hosts) {
private boolean checkHostsSuitabilityForExplicitDedication(Long accountId, List<Long> domainIds, List<Long> hostIds) {
boolean suitable = true;
for (HostVO host : hosts) {
checkHostSuitabilityForExplicitDedication(accountId, domainIds, host.getId());
for (Long hostId : hostIds) {
checkHostSuitabilityForExplicitDedication(accountId, domainIds, hostId);
}
return suitable;
}

50
plugins/deployment-planners/implicit-dedication/src/main/java/com/cloud/deploy/ImplicitDedicationPlanner.java
View File

@ -21,15 +21,16 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
import org.apache.commons.collections.CollectionUtils;
import org.apache.log4j.Logger;
import com.cloud.configuration.Config;
import com.cloud.exception.InsufficientServerCapacityException;
import com.cloud.host.HostVO;
import com.cloud.resource.ResourceManager;
import com.cloud.service.ServiceOfferingVO;
import com.cloud.service.dao.ServiceOfferingDao;
@ -39,7 +40,6 @@ import com.cloud.utils.DateUtil;
import com.cloud.utils.NumbersUtil;
import com.cloud.vm.VMInstanceVO;
import com.cloud.vm.VirtualMachineProfile;
import org.springframework.util.CollectionUtils;
public class ImplicitDedicationPlanner extends FirstFitPlanner implements DeploymentClusterPlanner {
@ -75,12 +75,11 @@ public class ImplicitDedicationPlanner extends FirstFitPlanner implements Deploy
boolean preferred = isServiceOfferingUsingPlannerInPreferredMode(vmProfile.getServiceOfferingId());
// Get the list of all the hosts in the given clusters
List<Long> allHosts = new ArrayList<Long>();
for (Long cluster : clusterList) {
List<HostVO> hostsInCluster = resourceMgr.listAllHostsInCluster(cluster);
for (HostVO hostVO : hostsInCluster) {
allHosts.add(hostVO.getId());
}
List<Long> allHosts = new ArrayList<>();
if (CollectionUtils.isNotEmpty(clusterList)) {
allHosts = clusterList.stream()
.flatMap(cluster -> hostDao.listIdsByClusterId(cluster).stream())
.collect(Collectors.toList());
}
// Go over all the hosts in the cluster and get a list of
@ -223,20 +222,15 @@ public class ImplicitDedicationPlanner extends FirstFitPlanner implements Deploy
}
private List<Long> getUpdatedClusterList(List<Long> clusterList, Set<Long> hostsSet) {
List<Long> updatedClusterList = new ArrayList<Long>();
for (Long cluster : clusterList) {
List<HostVO> hosts = resourceMgr.listAllHostsInCluster(cluster);
Set<Long> hostsInClusterSet = new HashSet<Long>();
for (HostVO host : hosts) {
hostsInClusterSet.add(host.getId());
}
if (!hostsSet.containsAll(hostsInClusterSet)) {
updatedClusterList.add(cluster);
}
if (CollectionUtils.isEmpty(clusterList)) {
return new ArrayList<>();
}
return updatedClusterList;
return clusterList.stream()
.filter(cluster -> {
Set<Long> hostsInClusterSet = new HashSet<>(hostDao.listIdsByClusterId(cluster));
return !hostsSet.containsAll(hostsInClusterSet);
})
.collect(Collectors.toList());
}
@Override
@ -256,15 +250,11 @@ public class ImplicitDedicationPlanner extends FirstFitPlanner implements Deploy
Account account = vmProfile.getOwner();
// Get the list of all the hosts in the given clusters
List<Long> allHosts = new ArrayList<Long>();
if (!CollectionUtils.isEmpty(clusterList)) {
for (Long cluster : clusterList) {
List<HostVO> hostsInCluster = resourceMgr.listAllHostsInCluster(cluster);
for (HostVO hostVO : hostsInCluster) {
allHosts.add(hostVO.getId());
}
}
List<Long> allHosts = new ArrayList<>();
if (CollectionUtils.isNotEmpty(clusterList)) {
allHosts = clusterList.stream()
.flatMap(cluster -> hostDao.listIdsByClusterId(cluster).stream())
.collect(Collectors.toList());
}
// Go over all the hosts in the cluster and get a list of
// 1. All empty hosts, not running any vms.

38
plugins/deployment-planners/implicit-dedication/src/test/java/org/apache/cloudstack/implicitplanner/ImplicitPlannerTest.java
View File

@ -16,11 +16,11 @@
// under the License.
package org.apache.cloudstack.implicitplanner;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.everyItem;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.everyItem;
import static org.hamcrest.Matchers.equalTo;
import static org.mockito.Matchers.anyString;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@ -36,7 +36,11 @@ import java.util.UUID;
import javax.inject.Inject;
import com.cloud.user.User;
import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
import org.apache.cloudstack.test.utils.SpringUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@ -54,12 +58,6 @@ import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import org.springframework.test.context.support.AnnotationConfigContextLoader;
import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
import org.apache.cloudstack.test.utils.SpringUtils;
import com.cloud.capacity.Capacity;
import com.cloud.capacity.CapacityManager;
import com.cloud.capacity.dao.CapacityDao;
@ -73,7 +71,6 @@ import com.cloud.deploy.DeploymentPlanner.ExcludeList;
import com.cloud.deploy.ImplicitDedicationPlanner;
import com.cloud.exception.InsufficientServerCapacityException;
import com.cloud.gpu.dao.HostGpuGroupsDao;
import com.cloud.host.HostVO;
import com.cloud.host.dao.HostDao;
import com.cloud.host.dao.HostDetailsDao;
import com.cloud.host.dao.HostTagsDao;
@ -90,6 +87,7 @@ import com.cloud.storage.dao.VolumeDao;
import com.cloud.user.Account;
import com.cloud.user.AccountManager;
import com.cloud.user.AccountVO;
import com.cloud.user.User;
import com.cloud.user.UserVO;
import com.cloud.utils.Pair;
import com.cloud.utils.component.ComponentContext;
@ -387,21 +385,9 @@ public class ImplicitPlannerTest {
when(serviceOfferingDetailsDao.listDetailsKeyPairs(offeringId)).thenReturn(details);
// Initialize hosts in clusters
HostVO host1 = mock(HostVO.class);
when(host1.getId()).thenReturn(5L);
HostVO host2 = mock(HostVO.class);
when(host2.getId()).thenReturn(6L);
HostVO host3 = mock(HostVO.class);
when(host3.getId()).thenReturn(7L);
List<HostVO> hostsInCluster1 = new ArrayList<HostVO>();
List<HostVO> hostsInCluster2 = new ArrayList<HostVO>();
List<HostVO> hostsInCluster3 = new ArrayList<HostVO>();
hostsInCluster1.add(host1);
hostsInCluster2.add(host2);
hostsInCluster3.add(host3);
when(resourceMgr.listAllHostsInCluster(1)).thenReturn(hostsInCluster1);
when(resourceMgr.listAllHostsInCluster(2)).thenReturn(hostsInCluster2);
when(resourceMgr.listAllHostsInCluster(3)).thenReturn(hostsInCluster3);
when(hostDao.listIdsByClusterId(1L)).thenReturn(List.of(5L));
when(hostDao.listIdsByClusterId(2L)).thenReturn(List.of(6L));
when(hostDao.listIdsByClusterId(3L)).thenReturn(List.of(7L));
// Mock vms on each host.
long offeringIdForVmsOfThisAccount = 15L;

9
plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/discoverer/XcpServerDiscoverer.java
View File

@ -31,6 +31,7 @@ import javax.naming.ConfigurationException;
import javax.persistence.EntityExistsException;
import org.apache.cloudstack.hypervisor.xenserver.XenserverConfigs;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
import org.apache.maven.artifact.versioning.ComparableVersion;
@ -146,8 +147,8 @@ public class XcpServerDiscoverer extends DiscovererBase implements Discoverer, L
sc.and(sc.entity().getGuid(), Op.EQ, guid);
List<ClusterVO> clusters = sc.list();
ClusterVO clu = clusters.get(0);
List<HostVO> clusterHosts = _resourceMgr.listAllHostsInCluster(clu.getId());
if (clusterHosts == null || clusterHosts.size() == 0) {
List<Long> clusterHostIds = _hostDao.listIdsByClusterId(clu.getId());
if (CollectionUtils.isEmpty(clusterHostIds)) {
clu.setGuid(null);
_clusterDao.update(clu.getId(), clu);
_clusterDao.update(cluster.getId(), cluster);
@ -247,8 +248,8 @@ public class XcpServerDiscoverer extends DiscovererBase implements Discoverer, L
if (clu.getGuid() == null) {
setClusterGuid(clu, poolUuid);
} else {
List<HostVO> clusterHosts = _resourceMgr.listAllHostsInCluster(clusterId);
if (clusterHosts != null && clusterHosts.size() > 0) {
List<Long> clusterHostIds = _hostDao.listIdsByClusterId(clusterId);
if (CollectionUtils.isNotEmpty(clusterHostIds)) {
if (!clu.getGuid().equals(poolUuid)) {
String msg = "Please join the host " + hostIp + " to XS pool "
+ clu.getGuid() + " through XC/XS before adding it through CS UI";

4
plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java
View File

@ -263,8 +263,8 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp
metricsList.add(new ItemHostMemory(zoneName, zoneUuid, null, null, null, null, ALLOCATED, allocatedCapacityByTag.third(), 0, tag));
});
List<String> allHostTags = hostDao.listAll().stream()
.flatMap( h -> _hostTagsDao.getHostTags(h.getId()).stream())
List<String> allHostTags = hostDao.listAllIds().stream()
.flatMap( h -> _hostTagsDao.getHostTags(h).stream())
.distinct()
.collect(Collectors.toList());

2
plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsServiceImpl.java
View File

@ -558,7 +558,7 @@ public class MetricsServiceImpl extends MutualExclusiveIdsManagerBase implements
response.setHosts(hostDao.countAllByType(Host.Type.Routing));
response.setStoragePools(storagePoolDao.countAll());
response.setImageStores(imageStoreDao.countAllImageStores());
response.setSystemvms(vmInstanceDao.listByTypes(VirtualMachine.Type.ConsoleProxy, VirtualMachine.Type.SecondaryStorageVm).size());
response.setSystemvms(vmInstanceDao.countByTypes(VirtualMachine.Type.ConsoleProxy, VirtualMachine.Type.SecondaryStorageVm));
response.setRouters(domainRouterDao.countAllByRole(VirtualRouter.Role.VIRTUAL_ROUTER));
response.setInternalLbs(domainRouterDao.countAllByRole(VirtualRouter.Role.INTERNAL_LB_VM));
response.setAlerts(alertDao.countAll());

14
plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/util/StorPoolHelper.java
View File

@ -225,17 +225,17 @@ public class StorPoolHelper {
}
public static Long findClusterIdByGlobalId(String globalId, ClusterDao clusterDao) {
List<ClusterVO> clusterVo = clusterDao.listAll();
if (clusterVo.size() == 1) {
List<Long> clusterIds = clusterDao.listAllIds();
if (clusterIds.size() == 1) {
StorPoolUtil.spLog("There is only one cluster, sending backup to secondary command");
return null;
}
for (ClusterVO clusterVO2 : clusterVo) {
if (globalId != null && StorPoolConfigurationManager.StorPoolClusterId.valueIn(clusterVO2.getId()) != null
&& globalId.contains(StorPoolConfigurationManager.StorPoolClusterId.valueIn(clusterVO2.getId()).toString())) {
StorPoolUtil.spLog("Found cluster with id=%s for object with globalId=%s", clusterVO2.getId(),
for (Long clusterId : clusterIds) {
if (globalId != null && StorPoolConfigurationManager.StorPoolClusterId.valueIn(clusterId) != null
&& globalId.contains(StorPoolConfigurationManager.StorPoolClusterId.valueIn(clusterId))) {
StorPoolUtil.spLog("Found cluster with id=%s for object with globalId=%s", clusterId,
globalId);
return clusterVO2.getId();
return clusterId;
}
}
throw new CloudRuntimeException(

12
server/src/main/java/com/cloud/alert/AlertManagerImpl.java
View File

@ -75,8 +75,6 @@ import com.cloud.network.Ipv6Service;
import com.cloud.network.dao.IPAddressDao;
import com.cloud.org.Grouping.AllocationState;
import com.cloud.resource.ResourceManager;
import com.cloud.service.ServiceOfferingVO;
import com.cloud.service.dao.ServiceOfferingDao;
import com.cloud.storage.StorageManager;
import com.cloud.utils.Pair;
import com.cloud.utils.component.ManagerBase;
@ -120,8 +118,6 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager, Confi
@Inject
protected ConfigDepot _configDepot;
@Inject
ServiceOfferingDao _offeringsDao;
@Inject
Ipv6Service ipv6Service;
private Timer _timer = null;
@ -276,14 +272,8 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager, Confi
// get all hosts...even if they are not in 'UP' state
List<HostVO> hosts = _resourceMgr.listAllNotInMaintenanceHostsInOneZone(Host.Type.Routing, null);
if (hosts != null) {
// prepare the service offerings
List<ServiceOfferingVO> offerings = _offeringsDao.listAllIncludingRemoved();
Map<Long, ServiceOfferingVO> offeringsMap = new HashMap<Long, ServiceOfferingVO>();
for (ServiceOfferingVO offering : offerings) {
offeringsMap.put(offering.getId(), offering);
}
for (HostVO host : hosts) {
_capacityMgr.updateCapacityForHost(host, offeringsMap);
_capacityMgr.updateCapacityForHost(host);
}
}
if (logger.isDebugEnabled()) {

2
server/src/main/java/com/cloud/api/ApiServer.java
View File

@ -1236,8 +1236,6 @@ public class ApiServer extends ManagerBase implements HttpRequestHandler, ApiSer
throw new PermissionDeniedException("User is null for role based API access check for command" + commandName);
}
// FIXME: ConfigKey causes hotspot + connection leaks
// TODO: Implement LRU caching here...
final Account account = accountMgr.getAccount(user.getAccountId());
final String accessAllowedCidrs = ApiServiceConfiguration.ApiAllowedSourceCidrList.valueIn(account.getId()).replaceAll("\\s","");
final Boolean apiSourceCidrChecksEnabled = ApiServiceConfiguration.ApiSourceCidrChecksEnabled.value();

5
server/src/main/java/com/cloud/api/query/dao/UserVmJoinDao.java
View File

@ -42,6 +42,7 @@ public interface UserVmJoinDao extends GenericDao<UserVmJoinVO, Long> {
List<UserVmJoinVO> listActiveByIsoId(Long isoId);
List<UserVmJoinVO> listByAccountServiceOfferingTemplateAndNotInState(long accountId, List<VirtualMachine.State> states,
List<Long> offeringIds, List<Long> templateIds);
// TODO - We only need id, cpu and ram_size here. This could be done using JOIN on fly
List<UserVmJoinVO> listByAccountServiceOfferingTemplateAndNotInState(long accountId,
List<VirtualMachine.State> states, List<Long> offeringIds, List<Long> templateIds);
}

4
server/src/main/java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java
View File

@ -624,6 +624,8 @@ public class UserVmJoinDaoImpl extends GenericDaoBaseWithTagInformation<UserVmJo
public List<UserVmJoinVO> listByAccountServiceOfferingTemplateAndNotInState(long accountId, List<State> states,
List<Long> offeringIds, List<Long> templateIds) {
SearchBuilder<UserVmJoinVO> userVmSearch = createSearchBuilder();
userVmSearch.selectFields(userVmSearch.entity().getId(), userVmSearch.entity().getCpu(),
userVmSearch.entity().getRamSize());
userVmSearch.and("accountId", userVmSearch.entity().getAccountId(), Op.EQ);
userVmSearch.and("serviceOfferingId", userVmSearch.entity().getServiceOfferingId(), Op.IN);
userVmSearch.and("templateId", userVmSearch.entity().getTemplateId(), Op.IN);
@ -644,6 +646,6 @@ public class UserVmJoinDaoImpl extends GenericDaoBaseWithTagInformation<UserVmJo
sc.setParameters("state", states.toArray());
}
sc.setParameters("displayVm", 1);
return listBy(sc);
return customSearch(sc, null);
}
}

165
server/src/main/java/com/cloud/capacity/CapacityManagerImpl.java
View File

@ -23,6 +23,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
@ -37,6 +38,11 @@ import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.apache.cloudstack.framework.messagebus.MessageBus;
import org.apache.cloudstack.framework.messagebus.PublishScope;
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
import org.apache.cloudstack.utils.cache.LazyCache;
import org.apache.cloudstack.utils.cache.SingleCache;
import org.apache.cloudstack.utils.executor.QueuedExecutor;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.log4j.Logger;
import com.cloud.agent.AgentManager;
@ -51,7 +57,6 @@ import com.cloud.capacity.dao.CapacityDao;
import com.cloud.configuration.Config;
import com.cloud.dc.ClusterDetailsDao;
import com.cloud.dc.ClusterDetailsVO;
import com.cloud.dc.ClusterVO;
import com.cloud.dc.dao.ClusterDao;
import com.cloud.deploy.DeploymentClusterPlanner;
import com.cloud.event.UsageEventVO;
@ -143,6 +148,10 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager,
@Inject
MessageBus _messageBus;
private LazyCache<Long, Pair<String, String>> clusterValuesCache;
private SingleCache<Map<Long, ServiceOfferingVO>> serviceOfferingsCache;
private QueuedExecutor<Host> hostCapacityUpdateExecutor;
@Override
public boolean configure(String name, Map<String, Object> params) throws ConfigurationException {
_vmCapacityReleaseInterval = NumbersUtil.parseInt(_configDao.getValue(Config.CapacitySkipcountingHours.key()), 3600);
@ -151,6 +160,9 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager,
_agentManager.registerForHostEvents(new StorageCapacityListener(_capacityDao, _storageMgr), true, false, false);
_agentManager.registerForHostEvents(new ComputeCapacityListener(_capacityDao, this), true, false, false);
hostCapacityUpdateExecutor = new QueuedExecutor<>("HostCapacityUpdateExecutor", 10, 10,
1, s_logger, this::updateCapacityForHostInternal);
return true;
}
@ -158,11 +170,15 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager,
public boolean start() {
_resourceMgr.registerResourceEvent(ResourceListener.EVENT_PREPARE_MAINTENANCE_AFTER, this);
_resourceMgr.registerResourceEvent(ResourceListener.EVENT_CANCEL_MAINTENANCE_AFTER, this);
clusterValuesCache = new LazyCache<>(128, 60, this::getClusterValues);
serviceOfferingsCache = new SingleCache<>(60, this::getServiceOfferingsMap);
hostCapacityUpdateExecutor.startProcessing();
return true;
}
@Override
public boolean stop() {
hostCapacityUpdateExecutor.shutdown();
return true;
}
@ -211,8 +227,8 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager,
long reservedMem = capacityMemory.getReservedCapacity();
long reservedCpuCore = capacityCpuCore.getReservedCapacity();
long actualTotalCpu = capacityCpu.getTotalCapacity();
float cpuOvercommitRatio = Float.parseFloat(_clusterDetailsDao.findDetail(clusterIdFinal, "cpuOvercommitRatio").getValue());
float memoryOvercommitRatio = Float.parseFloat(_clusterDetailsDao.findDetail(clusterIdFinal, "memoryOvercommitRatio").getValue());
float cpuOvercommitRatio = Float.parseFloat(_clusterDetailsDao.findDetail(clusterIdFinal, VmDetailConstants.CPU_OVER_COMMIT_RATIO).getValue());
float memoryOvercommitRatio = Float.parseFloat(_clusterDetailsDao.findDetail(clusterIdFinal, VmDetailConstants.MEMORY_OVER_COMMIT_RATIO).getValue());
int vmCPU = svo.getCpu() * svo.getSpeed();
int vmCPUCore = svo.getCpu();
long vmMem = svo.getRamSize() * 1024L * 1024L;
@ -286,8 +302,8 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager,
final long hostId = vm.getHostId();
final HostVO host = _hostDao.findById(hostId);
final long clusterId = host.getClusterId();
final float cpuOvercommitRatio = Float.parseFloat(_clusterDetailsDao.findDetail(clusterId, "cpuOvercommitRatio").getValue());
final float memoryOvercommitRatio = Float.parseFloat(_clusterDetailsDao.findDetail(clusterId, "memoryOvercommitRatio").getValue());
final float cpuOvercommitRatio = Float.parseFloat(_clusterDetailsDao.findDetail(clusterId, VmDetailConstants.CPU_OVER_COMMIT_RATIO).getValue());
final float memoryOvercommitRatio = Float.parseFloat(_clusterDetailsDao.findDetail(clusterId, VmDetailConstants.MEMORY_OVER_COMMIT_RATIO).getValue());
final ServiceOfferingVO svo = _offeringsDao.findById(vm.getId(), vm.getServiceOfferingId());
@ -373,13 +389,13 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager,
",alloc_from_last:" + fromLastHost);
long cluster_id = host.getClusterId();
ClusterDetailsVO cluster_detail_cpu = _clusterDetailsDao.findDetail(cluster_id, "cpuOvercommitRatio");
ClusterDetailsVO cluster_detail_ram = _clusterDetailsDao.findDetail(cluster_id, "memoryOvercommitRatio");
ClusterDetailsVO cluster_detail_cpu = _clusterDetailsDao.findDetail(cluster_id, VmDetailConstants.CPU_OVER_COMMIT_RATIO);
ClusterDetailsVO cluster_detail_ram = _clusterDetailsDao.findDetail(cluster_id, VmDetailConstants.MEMORY_OVER_COMMIT_RATIO);
Float cpuOvercommitRatio = Float.parseFloat(cluster_detail_cpu.getValue());
Float memoryOvercommitRatio = Float.parseFloat(cluster_detail_ram.getValue());
boolean hostHasCpuCapability, hostHasCapacity = false;
hostHasCpuCapability = checkIfHostHasCpuCapability(host.getId(), cpucore, cpuspeed);
hostHasCpuCapability = checkIfHostHasCpuCapability(host, cpucore, cpuspeed);
if (hostHasCpuCapability) {
// first check from reserved capacity
@ -409,22 +425,20 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager,
}
@Override
public boolean checkIfHostHasCpuCapability(long hostId, Integer cpuNum, Integer cpuSpeed) {
public boolean checkIfHostHasCpuCapability(Host host, Integer cpuNum, Integer cpuSpeed) {
// Check host can support the Cpu Number and Speed.
Host host = _hostDao.findById(hostId);
boolean isCpuNumGood = host.getCpus().intValue() >= cpuNum;
boolean isCpuSpeedGood = host.getSpeed().intValue() >= cpuSpeed;
if (isCpuNumGood && isCpuSpeedGood) {
if (s_logger.isDebugEnabled()) {
s_logger.debug("Host: " + hostId + " has cpu capability (cpu:" + host.getCpus() + ", speed:" + host.getSpeed() +
") to support requested CPU: " + cpuNum + " and requested speed: " + cpuSpeed);
s_logger.debug("Host: " + host.getId() + " has cpu capability (cpu:" + host.getCpus() + ", speed:" + host.getSpeed() +
") to support requested CPU: " + cpuNum + " and requested speed: " + cpuSpeed);
}
return true;
} else {
if (s_logger.isDebugEnabled()) {
s_logger.debug("Host: " + hostId + " doesn't have cpu capability (cpu:" + host.getCpus() + ", speed:" + host.getSpeed() +
") to support requested CPU: " + cpuNum + " and requested speed: " + cpuSpeed);
s_logger.debug("Host: " + host.getId() + " doesn't have cpu capability (cpu:" + host.getCpus() + ", speed:" + host.getSpeed() +
") to support requested CPU: " + cpuNum + " and requested speed: " + cpuSpeed);
}
return false;
}
@ -635,18 +649,52 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager,
@DB
@Override
public void updateCapacityForHost(final Host host) {
// prepare the service offerings
List<ServiceOfferingVO> offerings = _offeringsDao.listAllIncludingRemoved();
Map<Long, ServiceOfferingVO> offeringsMap = new HashMap<Long, ServiceOfferingVO>();
for (ServiceOfferingVO offering : offerings) {
offeringsMap.put(offering.getId(), offering);
hostCapacityUpdateExecutor.queueRequest(host);
}
protected Pair<String, String> getClusterValues(long clusterId) {
Map<String, String> map = _clusterDetailsDao.findDetails(clusterId,
List.of(VmDetailConstants.CPU_OVER_COMMIT_RATIO, VmDetailConstants.CPU_OVER_COMMIT_RATIO));
return new Pair<>(map.get(VmDetailConstants.CPU_OVER_COMMIT_RATIO),
map.get(VmDetailConstants.CPU_OVER_COMMIT_RATIO));
}
protected Map<Long, ServiceOfferingVO> getServiceOfferingsMap() {
List<ServiceOfferingVO> serviceOfferings = _offeringsDao.listAllIncludingRemoved();
if (CollectionUtils.isEmpty(serviceOfferings)) {
return new HashMap<>();
}
updateCapacityForHost(host, offeringsMap);
return serviceOfferings.stream()
.collect(Collectors.toMap(
ServiceOfferingVO::getId,
offering -> offering
));
}
protected ServiceOfferingVO getServiceOffering(long id) {
Map <Long, ServiceOfferingVO> map = serviceOfferingsCache.get();
if (map.containsKey(id)) {
return map.get(id);
}
ServiceOfferingVO serviceOfferingVO = _offeringsDao.findByIdIncludingRemoved(id);
if (serviceOfferingVO != null) {
serviceOfferingsCache.invalidate();
}
return serviceOfferingVO;
}
protected Map<String, String> getVmDetailsForCapacityCalculation(long vmId) {
return _userVmDetailsDao.listDetailsKeyPairs(vmId,
List.of(VmDetailConstants.CPU_OVER_COMMIT_RATIO,
VmDetailConstants.MEMORY_OVER_COMMIT_RATIO,
UsageEventVO.DynamicParameters.memory.name(),
UsageEventVO.DynamicParameters.cpuNumber.name(),
UsageEventVO.DynamicParameters.cpuSpeed.name()));
}
@DB
@Override
public void updateCapacityForHost(final Host host, final Map<Long, ServiceOfferingVO> offeringsMap) {
protected void updateCapacityForHostInternal(final Host host) {
long usedCpuCore = 0;
long reservedCpuCore = 0;
long usedCpu = 0;
@ -655,32 +703,31 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager,
long reservedCpu = 0;
final CapacityState capacityState = (host.getResourceState() == ResourceState.Enabled) ? CapacityState.Enabled : CapacityState.Disabled;
List<VMInstanceVO> vms = _vmDao.listUpByHostId(host.getId());
List<VMInstanceVO> vms = _vmDao.listIdServiceOfferingForUpVmsByHostId(host.getId());
if (s_logger.isDebugEnabled()) {
s_logger.debug("Found " + vms.size() + " VMs on host " + host.getId());
}
final List<VMInstanceVO> vosMigrating = _vmDao.listVmsMigratingFromHost(host.getId());
final List<VMInstanceVO> vosMigrating = _vmDao.listIdServiceOfferingForVmsMigratingFromHost(host.getId());
if (s_logger.isDebugEnabled()) {
s_logger.debug("Found " + vosMigrating.size() + " VMs are Migrating from host " + host.getId());
}
vms.addAll(vosMigrating);
ClusterVO cluster = _clusterDao.findById(host.getClusterId());
ClusterDetailsVO clusterDetailCpu = _clusterDetailsDao.findDetail(cluster.getId(), "cpuOvercommitRatio");
ClusterDetailsVO clusterDetailRam = _clusterDetailsDao.findDetail(cluster.getId(), "memoryOvercommitRatio");
Float clusterCpuOvercommitRatio = Float.parseFloat(clusterDetailCpu.getValue());
Float clusterRamOvercommitRatio = Float.parseFloat(clusterDetailRam.getValue());
Pair<String, String> clusterValues =
clusterValuesCache.get(host.getClusterId());
Float clusterCpuOvercommitRatio = Float.parseFloat(clusterValues.first());
Float clusterRamOvercommitRatio = Float.parseFloat(clusterValues.second());
for (VMInstanceVO vm : vms) {
Float cpuOvercommitRatio = 1.0f;
Float ramOvercommitRatio = 1.0f;
Map<String, String> vmDetails = _userVmDetailsDao.listDetailsKeyPairs(vm.getId());
String vmDetailCpu = vmDetails.get("cpuOvercommitRatio");
String vmDetailRam = vmDetails.get("memoryOvercommitRatio");
Map<String, String> vmDetails = getVmDetailsForCapacityCalculation(vm.getId());
String vmDetailCpu = vmDetails.get(VmDetailConstants.CPU_OVER_COMMIT_RATIO);
String vmDetailRam = vmDetails.get(VmDetailConstants.MEMORY_OVER_COMMIT_RATIO);
// if vmDetailCpu or vmDetailRam is not null it means it is running in a overcommitted cluster.
cpuOvercommitRatio = (vmDetailCpu != null) ? Float.parseFloat(vmDetailCpu) : clusterCpuOvercommitRatio;
ramOvercommitRatio = (vmDetailRam != null) ? Float.parseFloat(vmDetailRam) : clusterRamOvercommitRatio;
ServiceOffering so = offeringsMap.get(vm.getServiceOfferingId());
ServiceOffering so = getServiceOffering(vm.getServiceOfferingId());
if (so == null) {
so = _offeringsDao.findByIdIncludingRemoved(vm.getServiceOfferingId());
}
@ -705,27 +752,28 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager,
}
}
List<VMInstanceVO> vmsByLastHostId = _vmDao.listByLastHostId(host.getId());
List<VMInstanceVO> vmsByLastHostId = _vmDao.listIdServiceOfferingForVmsByLastHostId(host.getId());
if (s_logger.isDebugEnabled()) {
s_logger.debug("Found " + vmsByLastHostId.size() + " VM, not running on host " + host.getId());
}
for (VMInstanceVO vm : vmsByLastHostId) {
Float cpuOvercommitRatio = 1.0f;
Float ramOvercommitRatio = 1.0f;
long lastModificationTime = Optional.ofNullable(vm.getUpdateTime()).orElse(vm.getCreated()).getTime();
long secondsSinceLastUpdate = (DateUtil.currentGMTTime().getTime() - lastModificationTime) / 1000;
if (secondsSinceLastUpdate < _vmCapacityReleaseInterval) {
UserVmDetailVO vmDetailCpu = _userVmDetailsDao.findDetail(vm.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO);
UserVmDetailVO vmDetailRam = _userVmDetailsDao.findDetail(vm.getId(), VmDetailConstants.MEMORY_OVER_COMMIT_RATIO);
Map<String, String> vmDetails = getVmDetailsForCapacityCalculation(vm.getId());
String vmDetailCpu = vmDetails.get(VmDetailConstants.CPU_OVER_COMMIT_RATIO);
String vmDetailRam = vmDetails.get(VmDetailConstants.MEMORY_OVER_COMMIT_RATIO);
if (vmDetailCpu != null) {
//if vmDetail_cpu is not null it means it is running in a overcommited cluster.
cpuOvercommitRatio = Float.parseFloat(vmDetailCpu.getValue());
cpuOvercommitRatio = Float.parseFloat(vmDetailCpu);
}
if (vmDetailRam != null) {
ramOvercommitRatio = Float.parseFloat(vmDetailRam.getValue());
ramOvercommitRatio = Float.parseFloat(vmDetailRam);
}
ServiceOffering so = offeringsMap.get(vm.getServiceOfferingId());
Map<String, String> vmDetails = _userVmDetailsDao.listDetailsKeyPairs(vm.getId());
ServiceOffering so = getServiceOffering(vm.getServiceOfferingId());
if (so == null) {
so = _offeringsDao.findByIdIncludingRemoved(vm.getServiceOfferingId());
}
@ -765,9 +813,24 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager,
}
}
CapacityVO cpuCap = _capacityDao.findByHostIdType(host.getId(), Capacity.CAPACITY_TYPE_CPU);
CapacityVO memCap = _capacityDao.findByHostIdType(host.getId(), Capacity.CAPACITY_TYPE_MEMORY);
CapacityVO cpuCoreCap = _capacityDao.findByHostIdType(host.getId(), CapacityVO.CAPACITY_TYPE_CPU_CORE);
List<CapacityVO> capacities = _capacityDao.listByHostIdTypes(host.getId(), List.of(Capacity.CAPACITY_TYPE_CPU,
Capacity.CAPACITY_TYPE_MEMORY,
CapacityVO.CAPACITY_TYPE_CPU_CORE));
CapacityVO cpuCap = null;
CapacityVO memCap = null;
CapacityVO cpuCoreCap = null;
for (CapacityVO c : capacities) {
if (c.getCapacityType() == Capacity.CAPACITY_TYPE_CPU) {
cpuCap = c;
} else if (c.getCapacityType() == Capacity.CAPACITY_TYPE_MEMORY) {
memCap = c;
} else if (c.getCapacityType() == Capacity.CAPACITY_TYPE_CPU_CORE) {
cpuCoreCap = c;
}
if (ObjectUtils.allNotNull(cpuCap, memCap, cpuCoreCap)) {
break;
}
}
if (cpuCoreCap != null) {
long hostTotalCpuCore = host.getCpus().longValue();
@ -1012,8 +1075,8 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager,
capacityCPU.addAnd("podId", SearchCriteria.Op.EQ, server.getPodId());
capacityCPU.addAnd("capacityType", SearchCriteria.Op.EQ, Capacity.CAPACITY_TYPE_CPU);
List<CapacityVO> capacityVOCpus = _capacityDao.search(capacitySC, null);
Float cpuovercommitratio = Float.parseFloat(_clusterDetailsDao.findDetail(server.getClusterId(), "cpuOvercommitRatio").getValue());
Float memoryOvercommitRatio = Float.parseFloat(_clusterDetailsDao.findDetail(server.getClusterId(), "memoryOvercommitRatio").getValue());
Float cpuovercommitratio = Float.parseFloat(_clusterDetailsDao.findDetail(server.getClusterId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO).getValue());
Float memoryOvercommitRatio = Float.parseFloat(_clusterDetailsDao.findDetail(server.getClusterId(), VmDetailConstants.MEMORY_OVER_COMMIT_RATIO).getValue());
if (capacityVOCpus != null && !capacityVOCpus.isEmpty()) {
CapacityVO CapacityVOCpu = capacityVOCpus.get(0);
@ -1070,9 +1133,9 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager,
String capacityOverProvisioningName = "";
if (capacityType == Capacity.CAPACITY_TYPE_CPU) {
capacityOverProvisioningName = "cpuOvercommitRatio";
capacityOverProvisioningName = VmDetailConstants.CPU_OVER_COMMIT_RATIO;
} else if (capacityType == Capacity.CAPACITY_TYPE_MEMORY) {
capacityOverProvisioningName = "memoryOvercommitRatio";
capacityOverProvisioningName = VmDetailConstants.MEMORY_OVER_COMMIT_RATIO;
} else {
throw new CloudRuntimeException("Invalid capacityType - " + capacityType);
}
@ -1114,12 +1177,12 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager,
int cpu_requested = offering.getCpu() * offering.getSpeed();
long ram_requested = offering.getRamSize() * 1024L * 1024L;
Cluster cluster = _clusterDao.findById(host.getClusterId());
ClusterDetailsVO clusterDetailsCpuOvercommit = _clusterDetailsDao.findDetail(cluster.getId(), "cpuOvercommitRatio");
ClusterDetailsVO clusterDetailsRamOvercommmt = _clusterDetailsDao.findDetail(cluster.getId(), "memoryOvercommitRatio");
ClusterDetailsVO clusterDetailsCpuOvercommit = _clusterDetailsDao.findDetail(cluster.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO);
ClusterDetailsVO clusterDetailsRamOvercommmt = _clusterDetailsDao.findDetail(cluster.getId(), VmDetailConstants.MEMORY_OVER_COMMIT_RATIO);
Float cpuOvercommitRatio = Float.parseFloat(clusterDetailsCpuOvercommit.getValue());
Float memoryOvercommitRatio = Float.parseFloat(clusterDetailsRamOvercommmt.getValue());
boolean hostHasCpuCapability = checkIfHostHasCpuCapability(host.getId(), offering.getCpu(), offering.getSpeed());
boolean hostHasCpuCapability = checkIfHostHasCpuCapability(host, offering.getCpu(), offering.getSpeed());
boolean hostHasCapacity = checkIfHostHasCapacity(host.getId(), cpu_requested, ram_requested, false, cpuOvercommitRatio, memoryOvercommitRatio,
considerReservedCapacity);

2
server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java
View File

@ -2350,7 +2350,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
// Check if there are any non-removed hosts in the zone.
if (!_hostDao.listByDataCenterId(zoneId).isEmpty()) {
if (!_hostDao.listEnabledIdsByDataCenterId(zoneId).isEmpty()) {
throw new CloudRuntimeException(errorMsg + "there are servers in this zone.");
}

28
server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java
View File

@ -27,6 +27,7 @@ import java.util.Optional;
import java.util.Set;
import java.util.Timer;
import java.util.TreeSet;
import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
@ -143,6 +144,7 @@ import com.cloud.vm.VirtualMachine;
import com.cloud.vm.VirtualMachine.Event;
import com.cloud.vm.VirtualMachine.State;
import com.cloud.vm.VirtualMachineProfile;
import com.cloud.vm.VmDetailConstants;
import com.cloud.vm.dao.UserVmDao;
import com.cloud.vm.dao.VMInstanceDao;
@ -443,14 +445,14 @@ StateListener<State, VirtualMachine.Event, VirtualMachine>, Configurable {
if (checkVmProfileAndHost(vmProfile, host)) {
long cluster_id = host.getClusterId();
ClusterDetailsVO cluster_detail_cpu = _clusterDetailsDao.findDetail(cluster_id,
"cpuOvercommitRatio");
VmDetailConstants.CPU_OVER_COMMIT_RATIO);
ClusterDetailsVO cluster_detail_ram = _clusterDetailsDao.findDetail(cluster_id,
"memoryOvercommitRatio");
VmDetailConstants.MEMORY_OVER_COMMIT_RATIO);
Float cpuOvercommitRatio = Float.parseFloat(cluster_detail_cpu.getValue());
Float memoryOvercommitRatio = Float.parseFloat(cluster_detail_ram.getValue());
boolean hostHasCpuCapability, hostHasCapacity = false;
hostHasCpuCapability = _capacityMgr.checkIfHostHasCpuCapability(host.getId(), offering.getCpu(), offering.getSpeed());
hostHasCpuCapability = _capacityMgr.checkIfHostHasCpuCapability(host, offering.getCpu(), offering.getSpeed());
if (hostHasCpuCapability) {
// first check from reserved capacity
@ -649,10 +651,8 @@ StateListener<State, VirtualMachine.Event, VirtualMachine>, Configurable {
* Adds disabled Hosts to the ExcludeList in order to avoid them at the deployment planner.
*/
protected void avoidDisabledHosts(DataCenter dc, ExcludeList avoids) {
List<HostVO> disabledHosts = _hostDao.listDisabledByDataCenterId(dc.getId());
for (HostVO host : disabledHosts) {
avoids.addHost(host.getId());
}
List<Long> disabledHostIds = _hostDao.listDisabledIdsByDataCenterId(dc.getId());
disabledHostIds.forEach(avoids::addHost);
}
/**
@ -766,7 +766,7 @@ StateListener<State, VirtualMachine.Event, VirtualMachine>, Configurable {
List<Long> allDedicatedPods = _dedicatedDao.listAllPods();
allPodsInDc.retainAll(allDedicatedPods);
List<Long> allClustersInDc = _clusterDao.listAllClusters(dc.getId());
List<Long> allClustersInDc = _clusterDao.listAllClusterIds(dc.getId());
List<Long> allDedicatedClusters = _dedicatedDao.listAllClusters();
allClustersInDc.retainAll(allDedicatedClusters);
@ -1065,11 +1065,13 @@ StateListener<State, VirtualMachine.Event, VirtualMachine>, Configurable {
private void checkHostReservations() {
List<PlannerHostReservationVO> reservedHosts = _plannerHostReserveDao.listAllReservedHosts();
for (PlannerHostReservationVO hostReservation : reservedHosts) {
HostVO host = _hostDao.findById(hostReservation.getHostId());
List<HostVO> hosts = _hostDao.listByIds(reservedHosts
.stream()
.map(PlannerHostReservationVO::getHostId)
.collect(Collectors.toList()));
for (HostVO host : hosts) {
if (host != null && host.getManagementServerId() != null && host.getManagementServerId() == _nodeId) {
checkHostReservationRelease(hostReservation.getHostId());
checkHostReservationRelease(host.getId());
}
}
@ -1256,7 +1258,7 @@ StateListener<State, VirtualMachine.Event, VirtualMachine>, Configurable {
Pair<Host, Map<Volume, StoragePool>> potentialResources = findPotentialDeploymentResources(suitableHosts, suitableVolumeStoragePools, avoid,
resourceUsageRequired, readyAndReusedVolumes, plan.getPreferredHosts(), vmProfile.getVirtualMachine());
if (potentialResources != null) {
Host host = _hostDao.findById(potentialResources.first().getId());
Host host = potentialResources.first();
Map<Volume, StoragePool> storageVolMap = potentialResources.second();
// remove the reused vol<->pool from destination, since
// we don't have to prepare this volume.

64
server/src/main/java/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java
View File

@ -16,6 +16,29 @@
// under the License.
package com.cloud.hypervisor.kvm.discoverer;
import static com.cloud.configuration.ConfigurationManagerImpl.ADD_HOST_ON_SERVICE_RESTART_KVM;
import java.net.InetAddress;
import java.net.URI;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
import org.apache.cloudstack.agent.lb.IndirectAgentLB;
import org.apache.cloudstack.ca.CAManager;
import org.apache.cloudstack.ca.SetupCertificateCommand;
import org.apache.cloudstack.direct.download.DirectDownloadManager;
import org.apache.cloudstack.framework.ca.Certificate;
import org.apache.cloudstack.utils.cache.LazyCache;
import org.apache.cloudstack.utils.security.KeyStoreUtils;
import org.apache.log4j.Logger;
import com.cloud.agent.AgentManager;
import com.cloud.agent.Listener;
import com.cloud.agent.api.AgentControlAnswer;
@ -48,26 +71,6 @@ import com.cloud.utils.StringUtils;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.ssh.SSHCmdHelper;
import com.trilead.ssh2.Connection;
import org.apache.cloudstack.agent.lb.IndirectAgentLB;
import org.apache.cloudstack.ca.CAManager;
import org.apache.cloudstack.ca.SetupCertificateCommand;
import org.apache.cloudstack.direct.download.DirectDownloadManager;
import org.apache.cloudstack.framework.ca.Certificate;
import org.apache.cloudstack.utils.security.KeyStoreUtils;
import org.apache.log4j.Logger;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
import java.net.InetAddress;
import java.net.URI;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import static com.cloud.configuration.ConfigurationManagerImpl.ADD_HOST_ON_SERVICE_RESTART_KVM;
public abstract class LibvirtServerDiscoverer extends DiscovererBase implements Discoverer, Listener, ResourceStateAdapter {
private static final Logger s_logger = Logger.getLogger(LibvirtServerDiscoverer.class);
@ -86,6 +89,16 @@ public abstract class LibvirtServerDiscoverer extends DiscovererBase implements
@Inject
private HostDao hostDao;
private LazyCache<Long, HostVO> clusterExistingHostCache;
private HostVO getExistingHostForCluster(long clusterId) {
HostVO existingHostInCluster = _hostDao.findAnyStateHypervisorHostInCluster(clusterId);
if (existingHostInCluster != null) {
_hostDao.loadDetails(existingHostInCluster);
}
return existingHostInCluster;
}
@Override
public abstract Hypervisor.HypervisorType getHypervisorType();
@ -422,6 +435,9 @@ public abstract class LibvirtServerDiscoverer extends DiscovererBase implements
_kvmGuestNic = _kvmPrivateNic;
}
clusterExistingHostCache = new LazyCache<>(32, 30,
this::getExistingHostForCluster);
agentMgr.registerForHostEvents(this, true, false, false);
_resourceMgr.registerResourceStateAdapter(this.getClass().getSimpleName(), this);
return true;
@ -464,11 +480,9 @@ public abstract class LibvirtServerDiscoverer extends DiscovererBase implements
throw new IllegalArgumentException("cannot add host, due to can't find cluster: " + host.getClusterId());
}
List<HostVO> hostsInCluster = _resourceMgr.listAllHostsInCluster(clusterVO.getId());
if (!hostsInCluster.isEmpty()) {
HostVO oneHost = hostsInCluster.get(0);
_hostDao.loadDetails(oneHost);
String hostOsInCluster = oneHost.getDetail("Host.OS");
HostVO existingHostInCluster = clusterExistingHostCache.get(clusterVO.getId());
if (existingHostInCluster != null) {
String hostOsInCluster = existingHostInCluster.getDetail("Host.OS");
String hostOs = ssCmd.getHostDetails().get("Host.OS");
if (!hostOsInCluster.equalsIgnoreCase(hostOs)) {
String msg = String.format("host: %s with hostOS, \"%s\"into a cluster, in which there are \"%s\" hosts added", firstCmd.getPrivateIpAddress(), hostOs, hostOsInCluster);

280
server/src/main/java/com/cloud/network/NetworkServiceImpl.java
View File

@ -36,7 +36,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
@ -76,6 +75,7 @@ import org.apache.cloudstack.network.NetworkPermissionVO;
import org.apache.cloudstack.network.dao.NetworkPermissionDao;
import org.apache.cloudstack.network.element.InternalLoadBalancerElementService;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.EnumUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
@ -2050,6 +2050,9 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C
Long associatedNetworkId = cmd.getAssociatedNetworkId();
String networkFilterStr = cmd.getNetworkFilter();
boolean applyManualPagination = CollectionUtils.isNotEmpty(supportedServicesStr) ||
Boolean.TRUE.equals(canUseForDeploy);
String vlanId = null;
if (cmd instanceof ListNetworksCmdByAdmin) {
vlanId = ((ListNetworksCmdByAdmin)cmd).getVlan();
@ -2135,7 +2138,13 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C
isRecursive = true;
}
Filter searchFilter = new Filter(NetworkVO.class, "id", false, null, null);
Long offset = cmd.getStartIndex();
Long limit = cmd.getPageSizeVal();
if (applyManualPagination) {
offset = null;
limit = null;
}
Filter searchFilter = new Filter(NetworkVO.class, "id", false, offset, limit);
SearchBuilder<NetworkVO> sb = _networksDao.createSearchBuilder();
if (forVpc != null) {
@ -2174,14 +2183,14 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C
if (permittedAccounts.isEmpty()) {
SearchBuilder<DomainVO> domainSearch = _domainDao.createSearchBuilder();
domainSearch.and("path", domainSearch.entity().getPath(), SearchCriteria.Op.LIKE);
sb.join("domainSearch", domainSearch, sb.entity().getDomainId(), domainSearch.entity().getId(), JoinBuilder.JoinType.INNER);
sb.join("domain", domainSearch, sb.entity().getDomainId(), domainSearch.entity().getId(), JoinBuilder.JoinType.INNER);
}
SearchBuilder<AccountVO> accountSearch = _accountDao.createSearchBuilder();
accountSearch.and("typeNEQ", accountSearch.entity().getType(), SearchCriteria.Op.NEQ);
accountSearch.and("typeEQ", accountSearch.entity().getType(), SearchCriteria.Op.EQ);
sb.join("accountSearch", accountSearch, sb.entity().getAccountId(), accountSearch.entity().getId(), JoinBuilder.JoinType.INNER);
sb.join("account", accountSearch, sb.entity().getAccountId(), accountSearch.entity().getId(), JoinBuilder.JoinType.INNER);
if (associatedNetworkId != null) {
SearchBuilder<NetworkDetailVO> associatedNetworkSearch = _networkDetailsDao.createSearchBuilder();
@ -2190,113 +2199,138 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C
sb.join("associatedNetworkSearch", associatedNetworkSearch, sb.entity().getId(), associatedNetworkSearch.entity().getResourceId(), JoinBuilder.JoinType.INNER);
}
List<NetworkVO> networksToReturn = new ArrayList<NetworkVO>();
if (isSystem == null || !isSystem) {
if (!permittedAccounts.isEmpty()) {
if (Arrays.asList(Network.NetworkFilter.Account, Network.NetworkFilter.AccountDomain, Network.NetworkFilter.All).contains(networkFilter)) {
//get account level networks
networksToReturn.addAll(listAccountSpecificNetworks(buildNetworkSearchCriteria(sb, keyword, id, isSystem, zoneId, guestIpType, trafficType, physicalNetworkId, networkOfferingId,
aclType, skipProjectNetworks, restartRequired, specifyIpRanges, vpcId, tags, display, vlanId, associatedNetworkId), searchFilter, permittedAccounts));
}
if (domainId != null && Arrays.asList(Network.NetworkFilter.Domain, Network.NetworkFilter.AccountDomain, Network.NetworkFilter.All).contains(networkFilter)) {
//get domain level networks
networksToReturn.addAll(listDomainLevelNetworks(buildNetworkSearchCriteria(sb, keyword, id, isSystem, zoneId, guestIpType, trafficType, physicalNetworkId, networkOfferingId,
aclType, true, restartRequired, specifyIpRanges, vpcId, tags, display, vlanId, associatedNetworkId), searchFilter, domainId, false));
}
if (Arrays.asList(Network.NetworkFilter.Shared, Network.NetworkFilter.All).contains(networkFilter)) {
// get shared networks
List<NetworkVO> sharedNetworks = listSharedNetworks(buildNetworkSearchCriteria(sb, keyword, id, isSystem, zoneId, guestIpType, trafficType, physicalNetworkId, networkOfferingId,
aclType, true, restartRequired, specifyIpRanges, vpcId, tags, display, vlanId, associatedNetworkId), searchFilter, permittedAccounts);
addNetworksToReturnIfNotExist(networksToReturn, sharedNetworks);
}
} else {
if (Arrays.asList(Network.NetworkFilter.Account, Network.NetworkFilter.AccountDomain, Network.NetworkFilter.All).contains(networkFilter)) {
//add account specific networks
networksToReturn.addAll(listAccountSpecificNetworksByDomainPath(buildNetworkSearchCriteria(sb, keyword, id, isSystem, zoneId, guestIpType, trafficType, physicalNetworkId, networkOfferingId,
aclType, skipProjectNetworks, restartRequired, specifyIpRanges, vpcId, tags, display, vlanId, associatedNetworkId), searchFilter, path, isRecursive));
}
if (Arrays.asList(Network.NetworkFilter.Domain, Network.NetworkFilter.AccountDomain, Network.NetworkFilter.All).contains(networkFilter)) {
//add domain specific networks of domain + parent domains
networksToReturn.addAll(listDomainSpecificNetworksByDomainPath(buildNetworkSearchCriteria(sb, keyword, id, isSystem, zoneId, guestIpType, trafficType, physicalNetworkId, networkOfferingId,
aclType, true, restartRequired, specifyIpRanges, vpcId, tags, display, vlanId, associatedNetworkId), searchFilter, path, isRecursive));
//add networks of subdomains
if (domainId == null) {
networksToReturn.addAll(listDomainLevelNetworks(buildNetworkSearchCriteria(sb, keyword, id, isSystem, zoneId, guestIpType, trafficType, physicalNetworkId, networkOfferingId,
aclType, true, restartRequired, specifyIpRanges, vpcId, tags, display, vlanId, associatedNetworkId), searchFilter, caller.getDomainId(), true));
}
}
if (Arrays.asList(Network.NetworkFilter.Shared, Network.NetworkFilter.All).contains(networkFilter)) {
// get shared networks
List<NetworkVO> sharedNetworks = listSharedNetworksByDomainPath(buildNetworkSearchCriteria(sb, keyword, id, isSystem, zoneId, guestIpType, trafficType, physicalNetworkId, networkOfferingId,
aclType, true, restartRequired, specifyIpRanges, vpcId, tags, display, vlanId, associatedNetworkId), searchFilter, path, isRecursive);
addNetworksToReturnIfNotExist(networksToReturn, sharedNetworks);
}
}
Pair<List<NetworkVO>, Integer> result = new Pair<>(new ArrayList<>(), 0);
if (BooleanUtils.isTrue(isSystem)) {
SearchCriteria<NetworkVO> sc = createNetworkSearchCriteria(sb, keyword, id, isSystem, zoneId, guestIpType, trafficType,
physicalNetworkId, networkOfferingId, null, restartRequired, specifyIpRanges,
vpcId, tags, display, vlanId, associatedNetworkId);
addProjectNetworksConditionToSearch(sc, true);
result = _networksDao.searchAndCount(sc, searchFilter);
} else {
networksToReturn = _networksDao.search(buildNetworkSearchCriteria(sb, keyword, id, isSystem, zoneId, guestIpType, trafficType, physicalNetworkId, networkOfferingId,
null, true, restartRequired, specifyIpRanges, vpcId, tags, display, vlanId, associatedNetworkId), searchFilter);
SearchCriteria<NetworkVO> additionalSC = _networksDao.createSearchCriteria();
addAccountSpecificNetworksToSearch(additionalSC, sb, networkFilter, skipProjectNetworks, permittedAccounts, path, isRecursive);
addDomainSpecificNetworksToSearch(additionalSC, sb, networkFilter, permittedAccounts, domainId, path, isRecursive);
addSharedNetworksToSearch(additionalSC, sb, networkFilter, permittedAccounts, path, isRecursive);
if (CollectionUtils.isNotEmpty(additionalSC.getValues())) {
SearchCriteria<NetworkVO> sc = createNetworkSearchCriteria(sb, keyword, id, isSystem, zoneId, guestIpType,
trafficType, physicalNetworkId, networkOfferingId, aclType, restartRequired, specifyIpRanges, vpcId,
tags, display, vlanId, associatedNetworkId);
sc.addAnd("id", SearchCriteria.Op.SC, additionalSC);
result = _networksDao.searchAndCount(sc, searchFilter);
}
}
List<NetworkVO> networksToReturn = result.first();
if (supportedServicesStr != null && !supportedServicesStr.isEmpty() && !networksToReturn.isEmpty()) {
List<NetworkVO> supportedNetworks = new ArrayList<NetworkVO>();
Service[] suppportedServices = new Service[supportedServicesStr.size()];
List<NetworkVO> supportedNetworks = new ArrayList<>();
Service[] supportedServices = new Service[supportedServicesStr.size()];
int i = 0;
for (String supportedServiceStr : supportedServicesStr) {
Service service = Service.getService(supportedServiceStr);
if (service == null) {
throw new InvalidParameterValueException("Invalid service specified " + supportedServiceStr);
} else {
suppportedServices[i] = service;
supportedServices[i] = service;
}
i++;
}
for (NetworkVO network : networksToReturn) {
if (areServicesSupportedInNetwork(network.getId(), suppportedServices)) {
if (areServicesSupportedInNetwork(network.getId(), supportedServices)) {
supportedNetworks.add(network);
}
}
networksToReturn = supportedNetworks;
}
if (canUseForDeploy != null) {
List<NetworkVO> networksForDeploy = new ArrayList<NetworkVO>();
List<NetworkVO> networksForDeploy = new ArrayList<>();
for (NetworkVO network : networksToReturn) {
if (_networkModel.canUseForDeploy(network) == canUseForDeploy) {
networksForDeploy.add(network);
}
}
networksToReturn = networksForDeploy;
}
//Now apply pagination
List<? extends Network> wPagination = com.cloud.utils.StringUtils.applyPagination(networksToReturn, cmd.getStartIndex(), cmd.getPageSizeVal());
if (wPagination != null) {
Pair<List<? extends Network>, Integer> listWPagination = new Pair<List<? extends Network>, Integer>(wPagination, networksToReturn.size());
return listWPagination;
if (applyManualPagination) {
//Now apply pagination
List<? extends Network> wPagination = com.cloud.utils.StringUtils.applyPagination(networksToReturn, cmd.getStartIndex(), cmd.getPageSizeVal());
if (wPagination != null) {
Pair<List<? extends Network>, Integer> listWPagination = new Pair<>(wPagination, networksToReturn.size());
return listWPagination;
}
return new Pair<>(networksToReturn, networksToReturn.size());
}
return new Pair<List<? extends Network>, Integer>(networksToReturn, networksToReturn.size());
return new Pair<>(result.first(), result.second());
}
private void addNetworksToReturnIfNotExist(final List<NetworkVO> networksToReturn, final List<NetworkVO> sharedNetworks) {
Set<Long> networkIds = networksToReturn.stream()
.map(NetworkVO::getId)
.collect(Collectors.toSet());
List<NetworkVO> sharedNetworksToReturn = sharedNetworks.stream()
.filter(network -> ! networkIds.contains(network.getId()))
.collect(Collectors.toList());
networksToReturn.addAll(sharedNetworksToReturn);
private void addAccountSpecificNetworksToSearch(SearchCriteria<NetworkVO> additionalSC, SearchBuilder<NetworkVO> sb,
Network.NetworkFilter networkFilter, boolean skipProjectNetworks,
List<Long> permittedAccounts, String path, boolean isRecursive) {
if (!Arrays.asList(Network.NetworkFilter.Account, Network.NetworkFilter.AccountDomain, Network.NetworkFilter.All).contains(networkFilter)) {
return;
}
SearchCriteria<NetworkVO> accountSC = sb.create();
accountSC.addAnd("aclType", SearchCriteria.Op.EQ, ACLType.Account.toString());
if (permittedAccounts.isEmpty()) {
if (path != null) {
// accountSC's WHERE clause gets OR-included in additionalSC's WHERE clause, which then gets AND-included
// in the main search criteria's WHERE clause. If we added the path filter as join parameters, it would not
// be present in the main search criteria. This is a way to add a condition that would normally go to accountSC's
// join parameters to its WHERE clause.
accountSC.getJoin("domain").addAnd("path", SearchCriteria.Op.LIKE, isRecursive ? path + "%" : path);
accountSC.addAnd("id", SearchCriteria.Op.SC, accountSC.getJoin("domain"));
}
} else {
accountSC.addAnd("accountId", SearchCriteria.Op.IN, permittedAccounts.toArray());
}
addProjectNetworksConditionToSearch(accountSC, skipProjectNetworks);
additionalSC.addOr("id", SearchCriteria.Op.SC, accountSC);
}
private SearchCriteria<NetworkVO> buildNetworkSearchCriteria(SearchBuilder<NetworkVO> sb, String keyword, Long id,
Boolean isSystem, Long zoneId, String guestIpType, String trafficType, Long physicalNetworkId,
Long networkOfferingId, String aclType, boolean skipProjectNetworks, Boolean restartRequired,
Boolean specifyIpRanges, Long vpcId, Map<String, String> tags, Boolean display, String vlanId, Long associatedNetworkId) {
private void addDomainSpecificNetworksToSearch(SearchCriteria<NetworkVO> additionalSC, SearchBuilder<NetworkVO> sb, Network.NetworkFilter networkFilter,
List<Long> permittedAccounts, Long domainId, String path, boolean isRecursive) {
if (!Arrays.asList(Network.NetworkFilter.Domain, Network.NetworkFilter.AccountDomain, Network.NetworkFilter.All).contains(networkFilter)) {
return;
}
if (permittedAccounts.isEmpty()) {
// Add domain specific networks of domain + parent domains
addDomainNetworksByDomainPathToSearch(additionalSC, sb, path, isRecursive);
if (domainId == null) {
// Add networks of subdomains
Account caller = CallContext.current().getCallingAccount();
addDomainLevelNetworksToSearch(additionalSC, sb, caller.getDomainId(), true);
}
} else {
if (domainId != null) {
// Add domain level networks
addDomainLevelNetworksToSearch(additionalSC, sb, domainId, false);
}
}
}
private void addSharedNetworksToSearch(SearchCriteria<NetworkVO> additionalSC, SearchBuilder<NetworkVO> sb, Network.NetworkFilter networkFilter,
List<Long> permittedAccounts, String path, boolean isRecursive) {
if (!Arrays.asList(Network.NetworkFilter.Shared, Network.NetworkFilter.All).contains(networkFilter)) {
return;
}
if (permittedAccounts.isEmpty()) {
addSharedNetworksByDomainPathToSearch(additionalSC, sb, path, isRecursive);
} else {
addSharedNetworksByAccountsToSearch(additionalSC, sb, permittedAccounts);
}
}
private SearchCriteria<NetworkVO> createNetworkSearchCriteria(SearchBuilder<NetworkVO> sb, String keyword, Long id,
Boolean isSystem, Long zoneId, String guestIpType, String trafficType, Long physicalNetworkId,
Long networkOfferingId, String aclType, Boolean restartRequired,
Boolean specifyIpRanges, Long vpcId, Map<String, String> tags, Boolean display, String vlanId, Long associatedNetworkId) {
SearchCriteria<NetworkVO> sc = sb.create();
@ -2338,12 +2372,6 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C
sc.addAnd("physicalNetworkId", SearchCriteria.Op.EQ, physicalNetworkId);
}
if (skipProjectNetworks) {
sc.setJoinParameters("accountSearch", "typeNEQ", Account.Type.PROJECT);
} else {
sc.setJoinParameters("accountSearch", "typeEQ", Account.Type.PROJECT);
}
if (restartRequired != null) {
sc.addAnd("restartRequired", SearchCriteria.Op.EQ, restartRequired);
}
@ -2384,8 +2412,9 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C
return sc;
}
private List<NetworkVO> listDomainLevelNetworks(SearchCriteria<NetworkVO> sc, Filter searchFilter, long domainId, boolean parentDomainsOnly) {
List<Long> networkIds = new ArrayList<Long>();
private void addDomainLevelNetworksToSearch(SearchCriteria<NetworkVO> additionalSC, SearchBuilder<NetworkVO> sb,
long domainId, boolean parentDomainsOnly) {
List<Long> networkIds = new ArrayList<>();
Set<Long> allowedDomains = _domainMgr.getDomainParentIds(domainId);
List<NetworkDomainVO> maps = _networkDomainDao.listDomainNetworkMapByDomain(allowedDomains.toArray());
@ -2400,48 +2429,17 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C
}
if (!networkIds.isEmpty()) {
SearchCriteria<NetworkVO> domainSC = _networksDao.createSearchCriteria();
SearchCriteria<NetworkVO> domainSC = sb.create();
domainSC.addAnd("id", SearchCriteria.Op.IN, networkIds.toArray());
domainSC.addAnd("aclType", SearchCriteria.Op.EQ, ACLType.Domain.toString());
sc.addAnd("id", SearchCriteria.Op.SC, domainSC);
return _networksDao.search(sc, searchFilter);
} else {
return new ArrayList<NetworkVO>();
addProjectNetworksConditionToSearch(domainSC, true);
additionalSC.addOr("id", SearchCriteria.Op.SC, domainSC);
}
}
private List<NetworkVO> listAccountSpecificNetworks(SearchCriteria<NetworkVO> sc, Filter searchFilter, List<Long> permittedAccounts) {
SearchCriteria<NetworkVO> accountSC = _networksDao.createSearchCriteria();
if (!permittedAccounts.isEmpty()) {
accountSC.addAnd("accountId", SearchCriteria.Op.IN, permittedAccounts.toArray());
}
accountSC.addAnd("aclType", SearchCriteria.Op.EQ, ACLType.Account.toString());
sc.addAnd("id", SearchCriteria.Op.SC, accountSC);
return _networksDao.search(sc, searchFilter);
}
private List<NetworkVO> listAccountSpecificNetworksByDomainPath(SearchCriteria<NetworkVO> sc, Filter searchFilter, String path, boolean isRecursive) {
SearchCriteria<NetworkVO> accountSC = _networksDao.createSearchCriteria();
accountSC.addAnd("aclType", SearchCriteria.Op.EQ, ACLType.Account.toString());
if (path != null) {
if (isRecursive) {
sc.setJoinParameters("domainSearch", "path", path + "%");
} else {
sc.setJoinParameters("domainSearch", "path", path);
}
}
sc.addAnd("id", SearchCriteria.Op.SC, accountSC);
return _networksDao.search(sc, searchFilter);
}
private List<NetworkVO> listDomainSpecificNetworksByDomainPath(SearchCriteria<NetworkVO> sc, Filter searchFilter, String path, boolean isRecursive) {
Set<Long> allowedDomains = new HashSet<Long>();
private void addDomainNetworksByDomainPathToSearch(SearchCriteria<NetworkVO> additionalSC, SearchBuilder<NetworkVO> sb,
String path, boolean isRecursive) {
Set<Long> allowedDomains = new HashSet<>();
if (path != null) {
if (isRecursive) {
allowedDomains = _domainMgr.getDomainChildrenIds(path);
@ -2451,39 +2449,40 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C
}
}
List<Long> networkIds = new ArrayList<Long>();
List<Long> networkIds = new ArrayList<>();
List<NetworkDomainVO> maps = _networkDomainDao.listDomainNetworkMapByDomain(allowedDomains.toArray());
for (NetworkDomainVO map : maps) {
networkIds.add(map.getNetworkId());
}
if (!networkIds.isEmpty()) {
SearchCriteria<NetworkVO> domainSC = _networksDao.createSearchCriteria();
SearchCriteria<NetworkVO> domainSC = sb.create();
domainSC.addAnd("id", SearchCriteria.Op.IN, networkIds.toArray());
domainSC.addAnd("aclType", SearchCriteria.Op.EQ, ACLType.Domain.toString());
sc.addAnd("id", SearchCriteria.Op.SC, domainSC);
return _networksDao.search(sc, searchFilter);
} else {
return new ArrayList<NetworkVO>();
addProjectNetworksConditionToSearch(domainSC, true);
additionalSC.addOr("id", SearchCriteria.Op.SC, domainSC);
}
}
private List<NetworkVO> listSharedNetworks(SearchCriteria<NetworkVO> sc, Filter searchFilter, List<Long> permittedAccounts) {
private void addProjectNetworksConditionToSearch(SearchCriteria<NetworkVO> sc, boolean skipProjectNetworks) {
sc.getJoin("account").addAnd("type", skipProjectNetworks ? Op.NEQ : Op.EQ, Account.Type.PROJECT);
sc.addAnd("id", Op.SC, sc.getJoin("account"));
}
private void addSharedNetworksByAccountsToSearch(SearchCriteria<NetworkVO> additionalSC, SearchBuilder<NetworkVO> sb,
List<Long> permittedAccounts) {
List<Long> sharedNetworkIds = _networkPermissionDao.listPermittedNetworkIdsByAccounts(permittedAccounts);
if (!sharedNetworkIds.isEmpty()) {
SearchCriteria<NetworkVO> ssc = _networksDao.createSearchCriteria();
SearchCriteria<NetworkVO> ssc = sb.create();
ssc.addAnd("id", SearchCriteria.Op.IN, sharedNetworkIds.toArray());
sc.addAnd("id", SearchCriteria.Op.SC, ssc);
return _networksDao.search(sc, searchFilter);
addProjectNetworksConditionToSearch(ssc, true);
additionalSC.addOr("id", SearchCriteria.Op.SC, ssc);
}
return new ArrayList<NetworkVO>();
}
private List<NetworkVO> listSharedNetworksByDomainPath(SearchCriteria<NetworkVO> sc, Filter searchFilter, String path, boolean isRecursive) {
Set<Long> allowedDomains = new HashSet<Long>();
private void addSharedNetworksByDomainPathToSearch(SearchCriteria<NetworkVO> additionalSC, SearchBuilder<NetworkVO> sb, String path, boolean isRecursive) {
Set<Long> allowedDomains = new HashSet<>();
if (path != null) {
if (isRecursive) {
allowedDomains = _domainMgr.getDomainChildrenIds(path);
@ -2492,7 +2491,7 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C
allowedDomains.add(domain.getId());
}
}
List<Long> allowedDomainsList = new ArrayList<Long>(allowedDomains);
List<Long> allowedDomainsList = new ArrayList<>(allowedDomains);
if (!allowedDomainsList.isEmpty()) {
GenericSearchBuilder<AccountVO, Long> accountIdSearch = _accountDao.createSearchBuilder(Long.class);
@ -2505,13 +2504,12 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C
List<Long> sharedNetworkIds = _networkPermissionDao.listPermittedNetworkIdsByAccounts(allowedAccountsList);
if (!sharedNetworkIds.isEmpty()) {
SearchCriteria<NetworkVO> ssc = _networksDao.createSearchCriteria();
SearchCriteria<NetworkVO> ssc = sb.create();
ssc.addAnd("id", SearchCriteria.Op.IN, sharedNetworkIds.toArray());
sc.addAnd("id", SearchCriteria.Op.SC, ssc);
return _networksDao.search(sc, searchFilter);
addProjectNetworksConditionToSearch(ssc, true);
additionalSC.addOr("id", SearchCriteria.Op.SC, ssc);
}
}
return new ArrayList<NetworkVO>();
}
@Override

2
server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java
View File

@ -2707,7 +2707,7 @@ public class AutoScaleManagerImpl extends ManagerBase implements AutoScaleManage
return vmStatsById;
}
try {
vmStatsById = virtualMachineManager.getVirtualMachineStatistics(host.getId(), host.getName(), vmIds);
vmStatsById = virtualMachineManager.getVirtualMachineStatistics(host, vmIds);
if (MapUtils.isEmpty(vmStatsById)) {
s_logger.warn("Got empty result for virtual machine statistics from host: " + host);
}

47
server/src/main/java/com/cloud/resource/ResourceManagerImpl.java
View File

@ -547,8 +547,8 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
details.put("ovm3pool", allParams.get("ovm3pool"));
details.put("ovm3cluster", allParams.get("ovm3cluster"));
}
details.put("cpuOvercommitRatio", CapacityManager.CpuOverprovisioningFactor.value().toString());
details.put("memoryOvercommitRatio", CapacityManager.MemOverprovisioningFactor.value().toString());
details.put(VmDetailConstants.CPU_OVER_COMMIT_RATIO, CapacityManager.CpuOverprovisioningFactor.value().toString());
details.put(VmDetailConstants.MEMORY_OVER_COMMIT_RATIO, CapacityManager.MemOverprovisioningFactor.value().toString());
_clusterDetailsDao.persist(cluster.getId(), details);
return result;
}
@ -558,8 +558,8 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
details.put("url", url);
details.put("username", StringUtils.defaultString(username));
details.put("password", StringUtils.defaultString(password));
details.put("cpuOvercommitRatio", CapacityManager.CpuOverprovisioningFactor.value().toString());
details.put("memoryOvercommitRatio", CapacityManager.MemOverprovisioningFactor.value().toString());
details.put(VmDetailConstants.CPU_OVER_COMMIT_RATIO, CapacityManager.CpuOverprovisioningFactor.value().toString());
details.put(VmDetailConstants.MEMORY_OVER_COMMIT_RATIO, CapacityManager.MemOverprovisioningFactor.value().toString());
_clusterDetailsDao.persist(cluster.getId(), details);
boolean success = false;
@ -643,8 +643,8 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
throw ex;
} else {
if (cluster.getGuid() == null) {
final List<HostVO> hosts = listAllHostsInCluster(clusterId);
if (!hosts.isEmpty()) {
final List<Long> hostIds = _hostDao.listIdsByClusterId(clusterId);
if (!hostIds.isEmpty()) {
final CloudRuntimeException ex =
new CloudRuntimeException("Guid is not updated for cluster with specified cluster id; need to wait for hosts in this cluster to come up");
ex.addProxyObject(cluster.getUuid(), "clusterId");
@ -778,9 +778,9 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
}
}
clusterId = cluster.getId();
if (_clusterDetailsDao.findDetail(clusterId, "cpuOvercommitRatio") == null) {
final ClusterDetailsVO cluster_cpu_detail = new ClusterDetailsVO(clusterId, "cpuOvercommitRatio", "1");
final ClusterDetailsVO cluster_memory_detail = new ClusterDetailsVO(clusterId, "memoryOvercommitRatio", "1");
if (_clusterDetailsDao.findDetail(clusterId, VmDetailConstants.CPU_OVER_COMMIT_RATIO) == null) {
final ClusterDetailsVO cluster_cpu_detail = new ClusterDetailsVO(clusterId, VmDetailConstants.CPU_OVER_COMMIT_RATIO, "1");
final ClusterDetailsVO cluster_memory_detail = new ClusterDetailsVO(clusterId, VmDetailConstants.MEMORY_OVER_COMMIT_RATIO, "1");
_clusterDetailsDao.persist(cluster_cpu_detail);
_clusterDetailsDao.persist(cluster_memory_detail);
}
@ -962,8 +962,8 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
Host hostRemoved = _hostDao.findById(hostId);
_hostDao.remove(hostId);
if (clusterId != null) {
final List<HostVO> hosts = listAllHostsInCluster(clusterId);
if (hosts.size() == 0) {
final List<Long> hostIds = _hostDao.listIdsByClusterId(clusterId);
if (CollectionUtils.isEmpty(hostIds)) {
final ClusterVO cluster = _clusterDao.findById(clusterId);
cluster.setGuid(null);
_clusterDao.update(clusterId, cluster);
@ -1087,8 +1087,8 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
final Hypervisor.HypervisorType hypervisorType = cluster.getHypervisorType();
final List<HostVO> hosts = listAllHostsInCluster(cmd.getId());
if (hosts.size() > 0) {
final List<Long> hostIds = _hostDao.listIdsByClusterId(cmd.getId());
if (!hostIds.isEmpty()) {
if (s_logger.isDebugEnabled()) {
s_logger.debug("Cluster: " + cmd.getId() + " still has hosts, can't remove");
}
@ -2425,10 +2425,10 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
boolean clusterSupportsResigning = true;
List<HostVO> hostVOs = _hostDao.findByClusterId(host.getClusterId());
List<Long> hostIds = _hostDao.listIdsByClusterId(host.getClusterId());
for (HostVO hostVO : hostVOs) {
DetailVO hostDetailVO = _hostDetailsDao.findDetail(hostVO.getId(), name);
for (Long hostId : hostIds) {
DetailVO hostDetailVO = _hostDetailsDao.findDetail(hostId, name);
if (hostDetailVO == null || Boolean.parseBoolean(hostDetailVO.getValue()) == false) {
clusterSupportsResigning = false;
@ -3034,10 +3034,10 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
public boolean updateClusterPassword(final UpdateHostPasswordCmd command) {
final boolean shouldUpdateHostPasswd = command.getUpdatePasswdOnHost();
// get agents for the cluster
final List<HostVO> hosts = listAllHostsInCluster(command.getClusterId());
for (final HostVO host : hosts) {
final List<Long> hostIds = _hostDao.listIdsByClusterId(command.getClusterId());
for (final Long hostId : hostIds) {
try {
final Boolean result = propagateResourceEvent(host.getId(), ResourceState.Event.UpdatePassword);
final Boolean result = propagateResourceEvent(hostId, ResourceState.Event.UpdatePassword);
if (result != null) {
return result;
}
@ -3046,8 +3046,9 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
}
if (shouldUpdateHostPasswd) {
final boolean isUpdated = doUpdateHostPassword(host.getId());
final boolean isUpdated = doUpdateHostPassword(hostId);
if (!isUpdated) {
HostVO host = _hostDao.findById(hostId);
throw new CloudRuntimeException(
String.format("CloudStack failed to update the password of %s. Please make sure you are still able to connect to your hosts.", host));
}
@ -3286,15 +3287,15 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
}
@Override
public HostStats getHostStatistics(final long hostId) {
final Answer answer = _agentMgr.easySend(hostId, new GetHostStatsCommand(_hostDao.findById(hostId).getGuid(), _hostDao.findById(hostId).getName(), hostId));
public HostStats getHostStatistics(final Host host) {
final Answer answer = _agentMgr.easySend(host.getId(), new GetHostStatsCommand(host.getGuid(), host.getName(), host.getId()));
if (answer != null && answer instanceof UnsupportedAnswer) {
return null;
}
if (answer == null || !answer.getResult()) {
final String msg = "Unable to obtain host " + hostId + " statistics. ";
final String msg = "Unable to obtain host " + host.getId() + " statistics. ";
s_logger.warn(msg);
return null;
} else {

2
server/src/main/java/com/cloud/resource/RollingMaintenanceManagerImpl.java
View File

@ -637,7 +637,7 @@ public class RollingMaintenanceManagerImpl extends ManagerBase implements Rollin
continue;
}
boolean maxGuestLimit = capacityManager.checkIfHostReachMaxGuestLimit(host);
boolean hostHasCPUCapacity = capacityManager.checkIfHostHasCpuCapability(hostInCluster.getId(), serviceOffering.getCpu(), serviceOffering.getSpeed());
boolean hostHasCPUCapacity = capacityManager.checkIfHostHasCpuCapability(hostInCluster, serviceOffering.getCpu(), serviceOffering.getSpeed());
int cpuRequested = serviceOffering.getCpu() * serviceOffering.getSpeed();
long ramRequested = serviceOffering.getRamSize() * 1024L * 1024L;
ClusterDetailsVO clusterDetailsCpuOvercommit = clusterDetailsDao.findDetail(cluster.getId(), "cpuOvercommitRatio");

11
server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java
View File

@ -94,7 +94,6 @@ import com.cloud.projects.Project;
import com.cloud.projects.ProjectAccount.Role;
import com.cloud.projects.dao.ProjectAccountDao;
import com.cloud.projects.dao.ProjectDao;
import com.cloud.service.ServiceOfferingVO;
import com.cloud.service.dao.ServiceOfferingDao;
import com.cloud.storage.DataStoreRole;
import com.cloud.storage.DiskOfferingVO;
@ -1294,16 +1293,14 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim
if (StringUtils.isEmpty(tag)) {
return _userVmJoinDao.listByAccountServiceOfferingTemplateAndNotInState(accountId, states, null, null);
}
List<ServiceOfferingVO> offerings = serviceOfferingDao.listByHostTag(tag);
List<VMTemplateVO> templates = _vmTemplateDao.listByTemplateTag(tag);
List<Long> offerings = serviceOfferingDao.listIdsByHostTag(tag);
List<Long> templates = _vmTemplateDao.listIdsByTemplateTag(tag);
if (CollectionUtils.isEmpty(offerings) && CollectionUtils.isEmpty(templates)) {
return new ArrayList<>();
}
return _userVmJoinDao.listByAccountServiceOfferingTemplateAndNotInState(accountId, states,
offerings.stream().map(ServiceOfferingVO::getId).collect(Collectors.toList()),
templates.stream().map(VMTemplateVO::getId).collect(Collectors.toList())
);
offerings, templates);
}
protected List<UserVmJoinVO> getVmsWithAccount(long accountId) {
@ -1321,7 +1318,7 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim
vrIds);
}
private long calculateReservedResources(List<UserVmJoinVO> vms,long accountId, ResourceType type, String tag) {
private long calculateReservedResources(List<UserVmJoinVO> vms, long accountId, ResourceType type, String tag) {
Set<Long> vmIds = vms.stream().map(UserVmJoinVO::getId).collect(Collectors.toSet());
List<ReservationVO> reservations = reservationDao.getReservationsForAccount(accountId, type, tag);
long reserved = 0;

14
server/src/main/java/com/cloud/server/ManagementServerImpl.java
View File

@ -4860,7 +4860,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
private boolean updateHostsInCluster(final UpdateHostPasswordCmd command) {
// get all the hosts in this cluster
final List<HostVO> hosts = _resourceMgr.listAllHostsInCluster(command.getClusterId());
final List<Long> hostIds = _hostDao.listIdsByClusterId(command.getClusterId());
String userNameWithoutSpaces = StringUtils.deleteWhitespace(command.getUsername());
if (StringUtils.isBlank(userNameWithoutSpaces)) {
@ -4870,19 +4870,19 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
Transaction.execute(new TransactionCallbackNoReturn() {
@Override
public void doInTransactionWithoutResult(final TransactionStatus status) {
for (final HostVO h : hosts) {
for (final Long hostId : hostIds) {
if (s_logger.isDebugEnabled()) {
s_logger.debug("Changing password for host name = " + h.getName());
s_logger.debug("Changing password for host ID: " + hostId);
}
// update password for this host
final DetailVO nv = _detailsDao.findDetail(h.getId(), ApiConstants.USERNAME);
final DetailVO nv = _detailsDao.findDetail(hostId, ApiConstants.USERNAME);
if (nv == null) {
final DetailVO nvu = new DetailVO(h.getId(), ApiConstants.USERNAME, userNameWithoutSpaces);
final DetailVO nvu = new DetailVO(hostId, ApiConstants.USERNAME, userNameWithoutSpaces);
_detailsDao.persist(nvu);
final DetailVO nvp = new DetailVO(h.getId(), ApiConstants.PASSWORD, DBEncryptionUtil.encrypt(command.getPassword()));
final DetailVO nvp = new DetailVO(hostId, ApiConstants.PASSWORD, DBEncryptionUtil.encrypt(command.getPassword()));
_detailsDao.persist(nvp);
} else if (nv.getValue().equals(userNameWithoutSpaces)) {
final DetailVO nvp = _detailsDao.findDetail(h.getId(), ApiConstants.PASSWORD);
final DetailVO nvp = _detailsDao.findDetail(hostId, ApiConstants.PASSWORD);
nvp.setValue(DBEncryptionUtil.encrypt(command.getPassword()));
_detailsDao.persist(nvp);
} else {

101
server/src/main/java/com/cloud/server/StatsCollector.java
View File

@ -41,6 +41,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import javax.inject.Inject;
@ -131,6 +132,7 @@ import com.cloud.user.dao.UserStatisticsDao;
import com.cloud.user.dao.VmDiskStatisticsDao;
import com.cloud.utils.LogUtils;
import com.cloud.utils.NumbersUtil;
import com.cloud.utils.Pair;
import com.cloud.utils.component.ComponentMethodInterceptable;
import com.cloud.utils.component.ManagerBase;
import com.cloud.utils.concurrency.NamedThreadFactory;
@ -147,7 +149,6 @@ import com.cloud.utils.net.MacAddress;
import com.cloud.utils.script.Script;
import com.cloud.vm.NicVO;
import com.cloud.vm.UserVmManager;
import com.cloud.vm.UserVmVO;
import com.cloud.vm.VMInstanceVO;
import com.cloud.vm.VirtualMachine;
import com.cloud.vm.VirtualMachineManager;
@ -627,17 +628,21 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc
externalStatsPrefix, externalStatsHost, externalStatsPort));
}
protected Map<Long, VMInstanceVO> getVmMapForStatsForHost(Host host) {
protected Pair<Map<Long, VMInstanceVO>, Map<String, Long>> getVmMapForStatsForHost(Host host) {
List<VMInstanceVO> vms = _vmInstance.listByHostAndState(host.getId(), VirtualMachine.State.Running);
boolean collectUserVMStatsOnly = Boolean.TRUE.equals(vmStatsCollectUserVMOnly.value());
Map<Long, VMInstanceVO> vmMap = new HashMap<>();
for (VMInstanceVO vm : vms) {
if (collectUserVMStatsOnly && !VirtualMachine.Type.User.equals(vm.getType())) {
continue;
}
vmMap.put(vm.getId(), vm);
if (collectUserVMStatsOnly) {
vms = vms.stream().filter(vm -> VirtualMachine.Type.User.equals(vm.getType())).collect(Collectors.toList());
}
return vmMap;
Map<Long, VMInstanceVO> idInstanceMap = new HashMap<>();
Map<String, Long> instanceNameIdMap = new HashMap<>();
vms.forEach(vm -> {
if (!collectUserVMStatsOnly || VirtualMachine.Type.User.equals(vm.getType())) {
idInstanceMap.put(vm.getId(), vm);
instanceNameIdMap.put(vm.getInstanceName(), vm.getId());
}
});
return new Pair<>(idInstanceMap, instanceNameIdMap);
}
class HostCollector extends AbstractStatsCollector {
@ -652,7 +657,7 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc
List<HostVO> hosts = _hostDao.search(sc, null);
for (HostVO host : hosts) {
HostStatsEntry hostStatsEntry = (HostStatsEntry) _resourceMgr.getHostStatistics(host.getId());
HostStatsEntry hostStatsEntry = (HostStatsEntry) _resourceMgr.getHostStatistics(host);
if (hostStatsEntry != null) {
hostStatsEntry.setHostVo(host);
metrics.put(hostStatsEntry.getHostId(), hostStatsEntry);
@ -1201,37 +1206,39 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc
for (HostVO host : hosts) {
Date timestamp = new Date();
Map<Long, VMInstanceVO> vmMap = getVmMapForStatsForHost(host);
Pair<Map<Long, VMInstanceVO>, Map<String, Long>> vmsAndMap = getVmMapForStatsForHost(host);
Map<Long, VMInstanceVO> vmMap = vmsAndMap.first();
try {
Map<Long, ? extends VmStats> vmStatsById = virtualMachineManager.getVirtualMachineStatistics(host.getId(), host.getName(), vmMap);
if (vmStatsById != null) {
Set<Long> vmIdSet = vmStatsById.keySet();
for (Long vmId : vmIdSet) {
VmStatsEntry statsForCurrentIteration = (VmStatsEntry)vmStatsById.get(vmId);
statsForCurrentIteration.setVmId(vmId);
VMInstanceVO vm = vmMap.get(vmId);
statsForCurrentIteration.setVmUuid(vm.getUuid());
persistVirtualMachineStats(statsForCurrentIteration, timestamp);
if (externalStatsType == ExternalStatsProtocol.GRAPHITE) {
prepareVmMetricsForGraphite(metrics, statsForCurrentIteration);
} else {
metrics.put(statsForCurrentIteration.getVmId(), statsForCurrentIteration);
}
}
if (!metrics.isEmpty()) {
if (externalStatsType == ExternalStatsProtocol.GRAPHITE) {
sendVmMetricsToGraphiteHost(metrics, host);
} else if (externalStatsType == ExternalStatsProtocol.INFLUXDB) {
sendMetricsToInfluxdb(metrics);
}
}
metrics.clear();
Map<Long, ? extends VmStats> vmStatsById = virtualMachineManager.getVirtualMachineStatistics(
host, vmsAndMap.second());
if (MapUtils.isEmpty(vmStatsById)) {
continue;
}
Set<Long> vmIdSet = vmStatsById.keySet();
for (Long vmId : vmIdSet) {
VmStatsEntry statsForCurrentIteration = (VmStatsEntry)vmStatsById.get(vmId);
statsForCurrentIteration.setVmId(vmId);
VMInstanceVO vm = vmMap.get(vmId);
statsForCurrentIteration.setVmUuid(vm.getUuid());
persistVirtualMachineStats(statsForCurrentIteration, timestamp);
if (externalStatsType == ExternalStatsProtocol.GRAPHITE) {
prepareVmMetricsForGraphite(metrics, statsForCurrentIteration);
} else {
metrics.put(statsForCurrentIteration.getVmId(), statsForCurrentIteration);
}
}
if (!metrics.isEmpty()) {
if (externalStatsType == ExternalStatsProtocol.GRAPHITE) {
sendVmMetricsToGraphiteHost(metrics, host);
} else if (externalStatsType == ExternalStatsProtocol.INFLUXDB) {
sendMetricsToInfluxdb(metrics);
}
}
metrics.clear();
} catch (Exception e) {
LOGGER.debug("Failed to get VM stats for host with ID: " + host.getId());
continue;
@ -1419,8 +1426,10 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc
Transaction.execute(new TransactionCallbackNoReturn() {
@Override
public void doInTransactionWithoutResult(TransactionStatus status) {
Map<Long, VMInstanceVO> vmMap = getVmMapForStatsForHost(host);
HashMap<Long, List<? extends VmDiskStats>> vmDiskStatsById = virtualMachineManager.getVmDiskStatistics(host.getId(), host.getName(), vmMap);
Pair<Map<Long, VMInstanceVO>, Map<String, Long>> vmsAndMap = getVmMapForStatsForHost(host);
Map<Long, VMInstanceVO> vmMap = vmsAndMap.first();
HashMap<Long, List<? extends VmDiskStats>> vmDiskStatsById =
virtualMachineManager.getVmDiskStatistics(host, vmsAndMap.second());
if (vmDiskStatsById == null)
return;
@ -1527,8 +1536,10 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc
Transaction.execute(new TransactionCallbackNoReturn() {
@Override
public void doInTransactionWithoutResult(TransactionStatus status) {
Map<Long, VMInstanceVO> vmMap = getVmMapForStatsForHost(host);
HashMap<Long, List<? extends VmNetworkStats>> vmNetworkStatsById = virtualMachineManager.getVmNetworkStatistics(host.getId(), host.getName(), vmMap);
Pair<Map<Long, VMInstanceVO>, Map<String, Long>> vmsAndMap = getVmMapForStatsForHost(host);
Map<Long, VMInstanceVO> vmMap = vmsAndMap.first();
HashMap<Long, List<? extends VmNetworkStats>> vmNetworkStatsById =
virtualMachineManager.getVmNetworkStatistics(host, vmsAndMap.second());
if (vmNetworkStatsById == null)
return;
@ -1537,8 +1548,8 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc
List<? extends VmNetworkStats> vmNetworkStats = vmNetworkStatsById.get(vmId);
if (CollectionUtils.isEmpty(vmNetworkStats))
continue;
UserVmVO userVm = _userVmDao.findById(vmId);
if (userVm == null) {
VMInstanceVO userVm = vmMap.get(vmId);
if (!VirtualMachine.Type.User.equals(userVm.getType())) {
LOGGER.debug("Cannot find uservm with id: " + vmId + " , continue");
continue;
}

41
server/src/main/java/com/cloud/storage/download/DownloadListener.java
View File

@ -16,6 +16,7 @@
// under the License.
package com.cloud.storage.download;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
@ -24,9 +25,6 @@ import java.util.Timer;
import javax.inject.Inject;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
@ -40,6 +38,9 @@ import org.apache.cloudstack.storage.command.DownloadCommand;
import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType;
import org.apache.cloudstack.storage.command.DownloadProgressCommand;
import org.apache.cloudstack.storage.command.DownloadProgressCommand.RequestType;
import org.apache.cloudstack.utils.cache.LazyCache;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import com.cloud.agent.Listener;
import com.cloud.agent.api.AgentControlAnswer;
@ -53,6 +54,7 @@ import com.cloud.agent.api.storage.DownloadAnswer;
import com.cloud.agent.api.to.DataObjectType;
import com.cloud.exception.ConnectionException;
import com.cloud.host.Host;
import com.cloud.hypervisor.Hypervisor;
import com.cloud.resource.ResourceManager;
import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
import com.cloud.storage.download.DownloadState.DownloadEvent;
@ -134,6 +136,20 @@ public class DownloadListener implements Listener {
@Inject
private VolumeService _volumeSrv;
private LazyCache<Long, List<Hypervisor.HypervisorType>> zoneHypervisorsCache;
private List<Hypervisor.HypervisorType> listAvailHypervisorInZone(long zoneId) {
if (_resourceMgr == null) {
return Collections.emptyList();
}
return _resourceMgr.listAvailHypervisorInZone(zoneId);
}
protected void initZoneHypervisorsCache() {
zoneHypervisorsCache =
new LazyCache<>(32, 30, this::listAvailHypervisorInZone);
}
// TODO: this constructor should be the one used for template only, remove other template constructor later
public DownloadListener(EndPoint ssAgent, DataStore store, DataObject object, Timer timer, DownloadMonitorImpl downloadMonitor, DownloadCommand cmd,
AsyncCompletionCallback<DownloadAnswer> callback) {
@ -149,6 +165,12 @@ public class DownloadListener implements Listener {
_callback = callback;
DownloadAnswer answer = new DownloadAnswer("", Status.NOT_DOWNLOADED);
callback(answer);
initZoneHypervisorsCache();
}
public DownloadListener(DownloadMonitorImpl monitor) {
_downloadMonitor = monitor;
initZoneHypervisorsCache();
}
public AsyncCompletionCallback<DownloadAnswer> getCallback() {
@ -206,10 +228,6 @@ public class DownloadListener implements Listener {
s_logger.log(level, message + ", " + object.getType() + ": " + object.getId() + " at host " + _ssAgent.getId());
}
public DownloadListener(DownloadMonitorImpl monitor) {
_downloadMonitor = monitor;
}
@Override
public boolean isRecurring() {
return false;
@ -274,16 +292,15 @@ public class DownloadListener implements Listener {
@Override
public void processConnect(Host agent, StartupCommand cmd, boolean forRebalance) throws ConnectionException {
if (cmd instanceof StartupRoutingCommand) {
/* FIXME: CPU and DB hotspot
List<HypervisorType> hypers = _resourceMgr.listAvailHypervisorInZone(agent.getDataCenterId());
HypervisorType hostHyper = agent.getHypervisorType();
if (hypers.contains(hostHyper)) {
List<Hypervisor.HypervisorType> hypervisors = zoneHypervisorsCache.get(agent.getDataCenterId());
Hypervisor.HypervisorType hostHyper = agent.getHypervisorType();
if (hypervisors.contains(hostHyper)) {
return;
}
_imageSrv.handleSysTemplateDownload(hostHyper, agent.getDataCenterId());
// update template_zone_ref for cross-zone templates
_imageSrv.associateCrosszoneTemplatesToZone(agent.getDataCenterId());
*/
}
/* This can be removed
else if ( cmd instanceof StartupStorageCommand) {

2
server/src/main/java/com/cloud/vm/UserVmManagerImpl.java
View File

@ -2010,7 +2010,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
// #1 Check existing host has capacity & and the correct tags
if (!excludes.shouldAvoid(ApiDBUtils.findHostById(vmInstance.getHostId()))) {
existingHostHasCapacity = _capacityMgr.checkIfHostHasCpuCapability(vmInstance.getHostId(), newCpu, newSpeed)
existingHostHasCapacity = _capacityMgr.checkIfHostHasCpuCapability(host, newCpu, newSpeed)
&& _capacityMgr.checkIfHostHasCapacity(vmInstance.getHostId(), cpuDiff, ByteScaleUtils.mebibytesToBytes(memoryDiff), false,
_capacityMgr.getClusterOverProvisioningFactor(host.getClusterId(), Capacity.CAPACITY_TYPE_CPU),
_capacityMgr.getClusterOverProvisioningFactor(host.getClusterId(), Capacity.CAPACITY_TYPE_MEMORY), false)

130
server/src/main/java/org/apache/cloudstack/agent/lb/IndirectAgentLBServiceImpl.java
View File

@ -18,9 +18,7 @@ package org.apache.cloudstack.agent.lb;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -34,18 +32,19 @@ import org.apache.cloudstack.agent.lb.algorithm.IndirectAgentLBStaticAlgorithm;
import org.apache.cloudstack.config.ApiServiceConfiguration;
import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.framework.config.Configurable;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
import com.cloud.agent.AgentManager;
import com.cloud.agent.api.Answer;
import com.cloud.dc.DataCenterVO;
import com.cloud.dc.dao.ClusterDao;
import com.cloud.dc.dao.DataCenterDao;
import com.cloud.host.Host;
import com.cloud.host.HostVO;
import com.cloud.host.dao.HostDao;
import com.cloud.hypervisor.Hypervisor;
import com.cloud.resource.ResourceState;
import com.cloud.utils.component.ComponentLifecycleBase;
import com.cloud.utils.exception.CloudRuntimeException;
import org.apache.commons.lang3.StringUtils;
public class IndirectAgentLBServiceImpl extends ComponentLifecycleBase implements IndirectAgentLB, Configurable {
public static final Logger LOG = Logger.getLogger(IndirectAgentLBServiceImpl.class);
@ -62,6 +61,10 @@ public class IndirectAgentLBServiceImpl extends ComponentLifecycleBase implement
private static Map<String, org.apache.cloudstack.agent.lb.IndirectAgentLBAlgorithm> algorithmMap = new HashMap<>();
@Inject
private DataCenterDao dataCenterDao;
@Inject
private ClusterDao clusterDao;
@Inject
private HostDao hostDao;
@Inject
@ -78,10 +81,15 @@ public class IndirectAgentLBServiceImpl extends ComponentLifecycleBase implement
throw new CloudRuntimeException(String.format("No management server addresses are defined in '%s' setting",
ApiServiceConfiguration.ManagementServerAddresses.key()));
}
final List<String> msList = Arrays.asList(msServerAddresses.replace(" ", "").split(","));
if (msList.size() == 1) {
return msList;
}
final org.apache.cloudstack.agent.lb.IndirectAgentLBAlgorithm algorithm = getAgentMSLBAlgorithm();
List<Long> hostIdList = orderedHostIdList;
if (hostIdList == null) {
hostIdList = getOrderedHostIdList(dcId);
hostIdList = algorithm.isHostListNeeded() ? getOrderedHostIdList(dcId) : new ArrayList<>();
}
// just in case we have a host in creating state make sure it is in the list:
@ -91,9 +99,6 @@ public class IndirectAgentLBServiceImpl extends ComponentLifecycleBase implement
}
hostIdList.add(hostId);
}
final org.apache.cloudstack.agent.lb.IndirectAgentLBAlgorithm algorithm = getAgentMSLBAlgorithm();
final List<String> msList = Arrays.asList(msServerAddresses.replace(" ", "").split(","));
return algorithm.sort(msList, hostIdList, hostId);
}
@ -121,76 +126,17 @@ public class IndirectAgentLBServiceImpl extends ComponentLifecycleBase implement
}
List<Long> getOrderedHostIdList(final Long dcId) {
final List<Long> hostIdList = new ArrayList<>();
for (final Host host : getAllAgentBasedHosts()) {
if (host.getDataCenterId() == dcId) {
hostIdList.add(host.getId());
}
}
Collections.sort(hostIdList, new Comparator<Long>() {
@Override
public int compare(Long x, Long y) {
return Long.compare(x,y);
}
});
final List<Long> hostIdList = getAllAgentBasedHostsFromDB(dcId, null);
hostIdList.sort(Comparator.comparingLong(x -> x));
return hostIdList;
}
private List<Host> getAllAgentBasedHosts() {
final List<HostVO> allHosts = hostDao.listAll();
if (allHosts == null) {
return new ArrayList<>();
}
final List <Host> agentBasedHosts = new ArrayList<>();
for (final Host host : allHosts) {
conditionallyAddHost(agentBasedHosts, host);
}
return agentBasedHosts;
}
private void conditionallyAddHost(List<Host> agentBasedHosts, Host host) {
if (host == null) {
if (LOG.isTraceEnabled()) {
LOG.trace("trying to add no host to a list");
}
return;
}
EnumSet<ResourceState> allowedStates = EnumSet.of(
ResourceState.Enabled,
ResourceState.Maintenance,
ResourceState.Disabled,
ResourceState.ErrorInMaintenance,
ResourceState.PrepareForMaintenance);
// so the remaining EnumSet<ResourceState> disallowedStates = EnumSet.complementOf(allowedStates)
// would be {ResourceState.Creating, ResourceState.Error};
if (!allowedStates.contains(host.getResourceState())) {
if (LOG.isTraceEnabled()) {
LOG.trace(String.format("host is in '%s' state, not adding to the host list, (id = %s)", host.getResourceState(), host.getUuid()));
}
return;
}
if (host.getType() != Host.Type.Routing
&& host.getType() != Host.Type.ConsoleProxy
&& host.getType() != Host.Type.SecondaryStorage
&& host.getType() != Host.Type.SecondaryStorageVM) {
if (LOG.isTraceEnabled()) {
LOG.trace(String.format("host is of wrong type, not adding to the host list, (id = %s, type = %s)", host.getUuid(), host.getType()));
}
return;
}
if (host.getHypervisorType() != null
&& ! (host.getHypervisorType() == Hypervisor.HypervisorType.KVM || host.getHypervisorType() == Hypervisor.HypervisorType.LXC)) {
if (LOG.isTraceEnabled()) {
LOG.trace(String.format("hypervisor is not the right type, not adding to the host list, (id = %s, hypervisortype = %s)", host.getUuid(), host.getHypervisorType()));
}
return;
}
agentBasedHosts.add(host);
private List<Long> getAllAgentBasedHostsFromDB(final Long zoneId, final Long clusterId) {
return hostDao.findHostIdsByZoneClusterResourceStateAndType(zoneId, clusterId,
List.of(ResourceState.Enabled, ResourceState.Maintenance, ResourceState.Disabled,
ResourceState.ErrorInMaintenance, ResourceState.PrepareForMaintenance),
List.of(Host.Type.Routing, Host.Type.ConsoleProxy,
Host.Type.SecondaryStorage, Host.Type.SecondaryStorageVM));
}
private org.apache.cloudstack.agent.lb.IndirectAgentLBAlgorithm getAgentMSLBAlgorithm() {
@ -210,18 +156,28 @@ public class IndirectAgentLBServiceImpl extends ComponentLifecycleBase implement
public void propagateMSListToAgents() {
LOG.debug("Propagating management server list update to agents");
final String lbAlgorithm = getLBAlgorithmName();
final Map<Long, List<Long>> dcOrderedHostsMap = new HashMap<>();
for (final Host host : getAllAgentBasedHosts()) {
final Long dcId = host.getDataCenterId();
if (!dcOrderedHostsMap.containsKey(dcId)) {
dcOrderedHostsMap.put(dcId, getOrderedHostIdList(dcId));
List<DataCenterVO> zones = dataCenterDao.listAll();
for (DataCenterVO zone : zones) {
List<Long> zoneHostIds = new ArrayList<>();
Map<Long, List<Long>> clusterHostIdsMap = new HashMap<>();
List<Long> clusterIds = clusterDao.listAllClusterIds(zone.getId());
for (Long clusterId : clusterIds) {
List<Long> hostIds = getAllAgentBasedHostsFromDB(zone.getId(), clusterId);
clusterHostIdsMap.put(clusterId, hostIds);
zoneHostIds.addAll(hostIds);
}
final List<String> msList = getManagementServerList(host.getId(), host.getDataCenterId(), dcOrderedHostsMap.get(dcId));
final Long lbCheckInterval = getLBPreferredHostCheckInterval(host.getClusterId());
final SetupMSListCommand cmd = new SetupMSListCommand(msList, lbAlgorithm, lbCheckInterval);
final Answer answer = agentManager.easySend(host.getId(), cmd);
if (answer == null || !answer.getResult()) {
LOG.warn(String.format("Failed to setup management servers list to the agent of %s", host));
zoneHostIds.sort(Comparator.comparingLong(x -> x));
for (Long clusterId : clusterIds) {
final Long lbCheckInterval = getLBPreferredHostCheckInterval(clusterId);
List<Long> hostIds = clusterHostIdsMap.get(clusterId);
for (Long hostId : hostIds) {
final List<String> msList = getManagementServerList(hostId, zone.getId(), zoneHostIds);
final SetupMSListCommand cmd = new SetupMSListCommand(msList, lbAlgorithm, lbCheckInterval);
final Answer answer = agentManager.easySend(hostId, cmd);
if (answer == null || !answer.getResult()) {
LOG.warn(String.format("Failed to setup management servers list to the agent of ID: %d", hostId));
}
}
}
}
}

5
server/src/main/java/org/apache/cloudstack/agent/lb/algorithm/IndirectAgentLBRoundRobinAlgorithm.java
View File

@ -56,4 +56,9 @@ public class IndirectAgentLBRoundRobinAlgorithm implements IndirectAgentLBAlgori
public boolean compare(final List<String> msList, final List<String> receivedMsList) {
return msList != null && receivedMsList != null && msList.equals(receivedMsList);
}
@Override
public boolean isHostListNeeded() {
return true;
}
}

16
server/src/main/java/org/apache/cloudstack/outofbandmanagement/OutOfBandManagementServiceImpl.java
View File

@ -285,11 +285,11 @@ public class OutOfBandManagementServiceImpl extends ManagerBase implements OutOf
&& isOutOfBandManagementEnabledForHost(host.getId());
}
public boolean transitionPowerStateToDisabled(List<? extends Host> hosts) {
public boolean transitionPowerStateToDisabled(List<Long> hostIds) {
boolean result = true;
for (Host host : hosts) {
for (Long hostId : hostIds) {
result = result && transitionPowerState(OutOfBandManagement.PowerState.Event.Disabled,
outOfBandManagementDao.findByHost(host.getId()));
outOfBandManagementDao.findByHost(hostId));
}
return result;
}
@ -318,7 +318,7 @@ public class OutOfBandManagementServiceImpl extends ManagerBase implements OutOf
@ActionEvent(eventType = EventTypes.EVENT_HOST_OUTOFBAND_MANAGEMENT_DISABLE, eventDescription = "disabling out-of-band management on a zone")
public OutOfBandManagementResponse disableOutOfBandManagement(final DataCenter zone) {
dataCenterDetailsDao.persist(zone.getId(), OOBM_ENABLED_DETAIL, String.valueOf(false));
transitionPowerStateToDisabled(hostDao.findByDataCenterId(zone.getId()));
transitionPowerStateToDisabled(hostDao.listIdsByDataCenterId(zone.getId()));
return buildEnableDisableResponse(false);
}
@ -334,7 +334,7 @@ public class OutOfBandManagementServiceImpl extends ManagerBase implements OutOf
@ActionEvent(eventType = EventTypes.EVENT_HOST_OUTOFBAND_MANAGEMENT_DISABLE, eventDescription = "disabling out-of-band management on a cluster")
public OutOfBandManagementResponse disableOutOfBandManagement(final Cluster cluster) {
clusterDetailsDao.persist(cluster.getId(), OOBM_ENABLED_DETAIL, String.valueOf(false));
transitionPowerStateToDisabled(hostDao.findByClusterId(cluster.getId()));
transitionPowerStateToDisabled(hostDao.listIdsByClusterId(cluster.getId()));
return buildEnableDisableResponse(false);
}
@ -354,7 +354,7 @@ public class OutOfBandManagementServiceImpl extends ManagerBase implements OutOf
outOfBandManagementConfig.setEnabled(true);
boolean updateResult = outOfBandManagementDao.update(outOfBandManagementConfig.getId(), (OutOfBandManagementVO) outOfBandManagementConfig);
if (updateResult) {
transitionPowerStateToDisabled(Collections.singletonList(host));
transitionPowerStateToDisabled(Collections.singletonList(host.getId()));
}
return buildEnableDisableResponse(true);
}
@ -367,7 +367,7 @@ public class OutOfBandManagementServiceImpl extends ManagerBase implements OutOf
outOfBandManagementConfig.setEnabled(false);
boolean updateResult = outOfBandManagementDao.update(outOfBandManagementConfig.getId(), (OutOfBandManagementVO) outOfBandManagementConfig);
if (updateResult) {
transitionPowerStateToDisabled(Collections.singletonList(host));
transitionPowerStateToDisabled(Collections.singletonList(host.getId()));
}
return buildEnableDisableResponse(false);
}
@ -578,7 +578,7 @@ public class OutOfBandManagementServiceImpl extends ManagerBase implements OutOf
if (isOutOfBandManagementEnabled(host)) {
submitBackgroundPowerSyncTask(host);
} else if (outOfBandManagementHost.getPowerState() != OutOfBandManagement.PowerState.Disabled) {
if (transitionPowerStateToDisabled(Collections.singletonList(host))) {
if (transitionPowerStateToDisabled(Collections.singletonList(host.getId()))) {
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("Out-of-band management was disabled in zone/cluster/host, disabled power state for %s", host));
}

20
server/src/test/java/com/cloud/configuration/ConfigurationManagerTest.java
View File

@ -733,7 +733,7 @@ public class ConfigurationManagerTest {
@Test
public void checkIfZoneIsDeletableSuccessTest() {
Mockito.when(_hostDao.listByDataCenterId(anyLong())).thenReturn(new ArrayList<HostVO>());
Mockito.when(_hostDao.listEnabledIdsByDataCenterId(anyLong())).thenReturn(new ArrayList<>());
Mockito.when(_podDao.listByDataCenterId(anyLong())).thenReturn(new ArrayList<HostPodVO>());
Mockito.when(_privateIpAddressDao.countIPs(anyLong(), anyBoolean())).thenReturn(0);
Mockito.when(_publicIpAddressDao.countIPs(anyLong(), anyBoolean())).thenReturn(0);
@ -747,11 +747,7 @@ public class ConfigurationManagerTest {
@Test(expected = CloudRuntimeException.class)
public void checkIfZoneIsDeletableFailureOnHostTest() {
HostVO hostVO = Mockito.mock(HostVO.class);
ArrayList<HostVO> arrayList = new ArrayList<HostVO>();
arrayList.add(hostVO);
Mockito.when(_hostDao.listByDataCenterId(anyLong())).thenReturn(arrayList);
Mockito.when(_hostDao.listEnabledIdsByDataCenterId(anyLong())).thenReturn(List.of(1L));
Mockito.when(_podDao.listByDataCenterId(anyLong())).thenReturn(new ArrayList<HostPodVO>());
Mockito.when(_privateIpAddressDao.countIPs(anyLong(), anyBoolean())).thenReturn(0);
Mockito.when(_publicIpAddressDao.countIPs(anyLong(), anyBoolean())).thenReturn(0);
@ -769,7 +765,7 @@ public class ConfigurationManagerTest {
ArrayList<HostPodVO> arrayList = new ArrayList<HostPodVO>();
arrayList.add(hostPodVO);
Mockito.when(_hostDao.listByDataCenterId(anyLong())).thenReturn(new ArrayList<HostVO>());
Mockito.when(_hostDao.listEnabledIdsByDataCenterId(anyLong())).thenReturn(new ArrayList<>());
Mockito.when(_podDao.listByDataCenterId(anyLong())).thenReturn(arrayList);
Mockito.when(_privateIpAddressDao.countIPs(anyLong(), anyBoolean())).thenReturn(0);
Mockito.when(_publicIpAddressDao.countIPs(anyLong(), anyBoolean())).thenReturn(0);
@ -783,7 +779,7 @@ public class ConfigurationManagerTest {
@Test(expected = CloudRuntimeException.class)
public void checkIfZoneIsDeletableFailureOnPrivateIpAddressTest() {
Mockito.when(_hostDao.listByDataCenterId(anyLong())).thenReturn(new ArrayList<HostVO>());
Mockito.when(_hostDao.listEnabledIdsByDataCenterId(anyLong())).thenReturn(new ArrayList<>());
Mockito.when(_podDao.listByDataCenterId(anyLong())).thenReturn(new ArrayList<HostPodVO>());
Mockito.when(_privateIpAddressDao.countIPs(anyLong(), anyBoolean())).thenReturn(1);
Mockito.when(_publicIpAddressDao.countIPs(anyLong(), anyBoolean())).thenReturn(0);
@ -797,7 +793,7 @@ public class ConfigurationManagerTest {
@Test(expected = CloudRuntimeException.class)
public void checkIfZoneIsDeletableFailureOnPublicIpAddressTest() {
Mockito.when(_hostDao.listByDataCenterId(anyLong())).thenReturn(new ArrayList<HostVO>());
Mockito.when(_hostDao.listEnabledIdsByDataCenterId(anyLong())).thenReturn(new ArrayList<>());
Mockito.when(_podDao.listByDataCenterId(anyLong())).thenReturn(new ArrayList<HostPodVO>());
Mockito.when(_privateIpAddressDao.countIPs(anyLong(), anyBoolean())).thenReturn(0);
Mockito.when(_publicIpAddressDao.countIPs(anyLong(), anyBoolean())).thenReturn(1);
@ -815,7 +811,7 @@ public class ConfigurationManagerTest {
ArrayList<VMInstanceVO> arrayList = new ArrayList<VMInstanceVO>();
arrayList.add(vMInstanceVO);
Mockito.when(_hostDao.listByDataCenterId(anyLong())).thenReturn(new ArrayList<HostVO>());
Mockito.when(_hostDao.listEnabledIdsByDataCenterId(anyLong())).thenReturn(new ArrayList<>());
Mockito.when(_podDao.listByDataCenterId(anyLong())).thenReturn(new ArrayList<HostPodVO>());
Mockito.when(_privateIpAddressDao.countIPs(anyLong(), anyBoolean())).thenReturn(0);
Mockito.when(_publicIpAddressDao.countIPs(anyLong(), anyBoolean())).thenReturn(0);
@ -833,7 +829,7 @@ public class ConfigurationManagerTest {
ArrayList<VolumeVO> arrayList = new ArrayList<VolumeVO>();
arrayList.add(volumeVO);
Mockito.when(_hostDao.listByDataCenterId(anyLong())).thenReturn(new ArrayList<HostVO>());
Mockito.when(_hostDao.listEnabledIdsByDataCenterId(anyLong())).thenReturn(new ArrayList<>());
Mockito.when(_podDao.listByDataCenterId(anyLong())).thenReturn(new ArrayList<HostPodVO>());
Mockito.when(_privateIpAddressDao.countIPs(anyLong(), anyBoolean())).thenReturn(0);
Mockito.when(_publicIpAddressDao.countIPs(anyLong(), anyBoolean())).thenReturn(0);
@ -851,7 +847,7 @@ public class ConfigurationManagerTest {
ArrayList<PhysicalNetworkVO> arrayList = new ArrayList<PhysicalNetworkVO>();
arrayList.add(physicalNetworkVO);
Mockito.when(_hostDao.listByDataCenterId(anyLong())).thenReturn(new ArrayList<HostVO>());
Mockito.when(_hostDao.listEnabledIdsByDataCenterId(anyLong())).thenReturn(new ArrayList<>());
Mockito.when(_podDao.listByDataCenterId(anyLong())).thenReturn(new ArrayList<HostPodVO>());
Mockito.when(_privateIpAddressDao.countIPs(anyLong(), anyBoolean())).thenReturn(0);
Mockito.when(_publicIpAddressDao.countIPs(anyLong(), anyBoolean())).thenReturn(0);

2
server/src/test/java/com/cloud/deploy/DeploymentPlanningManagerImplTest.java
View File

@ -864,7 +864,7 @@ public class DeploymentPlanningManagerImplTest {
Pair<Host, Map<Volume, StoragePool>> potentialResources = new Pair<>(host, suitableVolumeStoragePoolMap);
Mockito.when(capacityMgr.checkIfHostReachMaxGuestLimit(host)).thenReturn(false);
Mockito.when(capacityMgr.checkIfHostHasCpuCapability(ArgumentMatchers.anyLong(), ArgumentMatchers.anyInt(), ArgumentMatchers.anyInt())).thenReturn(true);
Mockito.when(capacityMgr.checkIfHostHasCpuCapability(ArgumentMatchers.any(Host.class), ArgumentMatchers.anyInt(), ArgumentMatchers.anyInt())).thenReturn(true);
Mockito.when(capacityMgr.checkIfHostHasCapacity(
ArgumentMatchers.anyLong(),
ArgumentMatchers.anyInt(),

9
server/src/test/java/com/cloud/network/as/AutoScaleManagerImplTest.java
View File

@ -100,6 +100,7 @@ import com.cloud.exception.OperationTimedoutException;
import com.cloud.exception.ResourceAllocationException;
import com.cloud.exception.ResourceInUseException;
import com.cloud.exception.ResourceUnavailableException;
import com.cloud.host.Host;
import com.cloud.host.HostVO;
import com.cloud.host.dao.HostDao;
import com.cloud.network.Network;
@ -2279,13 +2280,13 @@ public class AutoScaleManagerImplTest {
public void getVmStatsByIdFromHost() {
List<Long> vmIds = Mockito.mock(ArrayList.class);
Map<Long, VmStatsEntry> vmStatsById = Mockito.mock(HashMap.class);
Mockito.doReturn(vmStatsById).when(virtualMachineManager).getVirtualMachineStatistics(anyLong(), anyString(), anyList());
Mockito.doReturn(vmStatsById).when(virtualMachineManager).getVirtualMachineStatistics(Mockito.any(Host.class), anyList());
Map<Long, ? extends VmStats> result = autoScaleManagerImplSpy.getVmStatsByIdFromHost(-1L, vmIds);
Assert.assertEquals(0, result.size());
Mockito.verify(virtualMachineManager, never()).getVirtualMachineStatistics(anyLong(), anyString(), anyList());
Mockito.verify(virtualMachineManager, never()).getVirtualMachineStatistics(Mockito.any(Host.class), anyList());
}
@Test
@ -2297,13 +2298,13 @@ public class AutoScaleManagerImplTest {
when(hostDao.findById(hostId)).thenReturn(hostMock);
when(hostMock.getId()).thenReturn(hostId);
when(hostMock.getName()).thenReturn(hostName);
Mockito.doReturn(vmStatsById).when(virtualMachineManager).getVirtualMachineStatistics(anyLong(), anyString(), anyList());
Mockito.doReturn(vmStatsById).when(virtualMachineManager).getVirtualMachineStatistics(Mockito.any(Host.class), anyList());
Map<Long, ? extends VmStats> result = autoScaleManagerImplSpy.getVmStatsByIdFromHost(hostId, vmIds);
Assert.assertEquals(vmStatsById, result);
Mockito.verify(virtualMachineManager).getVirtualMachineStatistics(anyLong(), anyString(), anyList());
Mockito.verify(virtualMachineManager).getVirtualMachineStatistics(Mockito.any(Host.class), anyList());
}
@Test

2
server/src/test/java/com/cloud/resource/MockResourceManagerImpl.java
View File

@ -465,7 +465,7 @@ public class MockResourceManagerImpl extends ManagerBase implements ResourceMana
* @see com.cloud.resource.ResourceManager#getHostStatistics(long)
*/
@Override
public HostStats getHostStatistics(final long hostId) {
public HostStats getHostStatistics(final Host host) {
// TODO Auto-generated method stub
return null;
}

14
server/src/test/java/com/cloud/resourcelimit/ResourceLimitManagerImplTest.java
View File

@ -58,10 +58,8 @@ import com.cloud.offering.DiskOffering;
import com.cloud.offering.ServiceOffering;
import com.cloud.projects.ProjectVO;
import com.cloud.projects.dao.ProjectDao;
import com.cloud.service.ServiceOfferingVO;
import com.cloud.service.dao.ServiceOfferingDao;
import com.cloud.storage.DiskOfferingVO;
import com.cloud.storage.VMTemplateVO;
import com.cloud.storage.VolumeVO;
import com.cloud.storage.dao.DiskOfferingDao;
import com.cloud.storage.dao.VMTemplateDao;
@ -614,8 +612,8 @@ public class ResourceLimitManagerImplTest {
@Test
public void testGetVmsWithAccountAndTagNegative() {
String tag = hostTags.get(0);
Mockito.when(serviceOfferingDao.listByHostTag(tag)).thenReturn(null);
Mockito.when(vmTemplateDao.listByTemplateTag(tag)).thenReturn(null);
Mockito.when(serviceOfferingDao.listIdsByHostTag(tag)).thenReturn(null);
Mockito.when(vmTemplateDao.listIdsByTemplateTag(tag)).thenReturn(null);
List<UserVmJoinVO> result = resourceLimitManager.getVmsWithAccountAndTag(1L, hostTags.get(0));
Assert.assertTrue(CollectionUtils.isEmpty(result));
}
@ -624,12 +622,8 @@ public class ResourceLimitManagerImplTest {
public void testGetVmsWithAccountAndTag() throws NoSuchFieldException, IllegalAccessException {
overrideDefaultConfigValue(VirtualMachineManager.ResourceCountRunningVMsonly, "_defaultValue", "true");
String tag = hostTags.get(0);
ServiceOfferingVO serviceOfferingVO = Mockito.mock(ServiceOfferingVO.class);
Mockito.when(serviceOfferingVO.getId()).thenReturn(1L);
VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class);
Mockito.when(templateVO.getId()).thenReturn(1L);
Mockito.when(serviceOfferingDao.listByHostTag(tag)).thenReturn(List.of(serviceOfferingVO));
Mockito.when(vmTemplateDao.listByTemplateTag(tag)).thenReturn(List.of(templateVO));
Mockito.when(serviceOfferingDao.listIdsByHostTag(tag)).thenReturn(List.of(1L));
Mockito.when(vmTemplateDao.listIdsByTemplateTag(tag)).thenReturn(List.of(1L));
List<UserVmJoinVO> vmList = List.of(Mockito.mock(UserVmJoinVO.class));
Mockito.when(userVmJoinDao.listByAccountServiceOfferingTemplateAndNotInState(Mockito.anyLong(), Mockito.anyList(), Mockito.anyList(), Mockito.anyList())).thenReturn(vmList);
List<UserVmJoinVO> result = resourceLimitManager.getVmsWithAccountAndTag(1L, tag);

26
server/src/test/java/org/apache/cloudstack/agent/lb/IndirectAgentLBServiceImplTest.java
View File

@ -16,13 +16,16 @@
// under the License.
package org.apache.cloudstack.agent.lb;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.when;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.cloudstack.config.ApiServiceConfiguration;
import org.apache.cloudstack.framework.config.ConfigKey;
@ -32,6 +35,7 @@ import org.junit.Before;
import org.junit.Test;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.MockitoAnnotations;
import org.mockito.Spy;
@ -84,6 +88,7 @@ public class IndirectAgentLBServiceImplTest {
}
private void configureMocks() throws NoSuchFieldException, IllegalAccessException {
List<HostVO> hosts = Arrays.asList(host1, host2, host3, host4);
long id = 1;
for (HostVO h : Arrays.asList(host1, host2, host3, host4)) {
when(h.getId()).thenReturn(id);
@ -97,7 +102,9 @@ public class IndirectAgentLBServiceImplTest {
addField(agentMSLB, "hostDao", hostDao);
addField(agentMSLB, "agentManager", agentManager);
when(hostDao.listAll()).thenReturn(Arrays.asList(host4, host2, host1, host3));
List<Long> hostIds = hosts.stream().map(HostVO::getId).collect(Collectors.toList());
doReturn(hostIds).when(hostDao).findHostIdsByZoneClusterResourceStateAndType(Mockito.anyLong(), Mockito.eq(null),
Mockito.anyList(), Mockito.anyList());
}
@Before
@ -187,21 +194,18 @@ public class IndirectAgentLBServiceImplTest {
}
@Test
public void testGetOrderedRunningHostIdsNullList() {
when(hostDao.listAll()).thenReturn(null);
Assert.assertTrue(agentMSLB.getOrderedHostIdList(DC_1_ID).size() == 0);
public void testGetOrderedRunningHostIdsEmptyList() {
doReturn(Collections.emptyList()).when(hostDao).findHostIdsByZoneClusterResourceStateAndType(Mockito.eq(DC_1_ID), Mockito.eq(null),
Mockito.anyList(), Mockito.anyList());
Assert.assertTrue(agentMSLB.getOrderedHostIdList(DC_1_ID).isEmpty());
}
@Test
public void testGetOrderedRunningHostIdsOrderList() {
when(hostDao.listAll()).thenReturn(Arrays.asList(host4, host2, host1, host3));
doReturn(Arrays.asList(host4.getId(), host2.getId(), host1.getId(), host3.getId())).when(hostDao)
.findHostIdsByZoneClusterResourceStateAndType(Mockito.eq(DC_1_ID), Mockito.eq(null),
Mockito.anyList(), Mockito.anyList());
Assert.assertEquals(Arrays.asList(host1.getId(), host2.getId(), host3.getId(), host4.getId()),
agentMSLB.getOrderedHostIdList(DC_1_ID));
}
@Test
public void testGetHostsPerZoneNullHosts() {
when(hostDao.listAll()).thenReturn(null);
Assert.assertTrue(agentMSLB.getOrderedHostIdList(DC_2_ID).size() == 0);
}
}

48
server/src/test/java/org/apache/cloudstack/networkoffering/CreateNetworkOfferingTest.java
View File

@ -18,7 +18,6 @@
package org.apache.cloudstack.networkoffering;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.nullable;
import java.util.HashMap;
@ -26,22 +25,21 @@ import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import javax.inject.Inject;
import org.apache.cloudstack.annotation.dao.AnnotationDao;
import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.apache.cloudstack.framework.config.impl.ConfigurationVO;
import org.apache.cloudstack.resourcedetail.dao.UserIpAddressDetailsDao;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import org.mockito.junit.MockitoJUnitRunner;
import com.cloud.configuration.ConfigurationManager;
import com.cloud.configuration.ConfigurationManagerImpl;
import com.cloud.event.dao.UsageEventDao;
import com.cloud.event.dao.UsageEventDetailsDao;
import com.cloud.exception.InvalidParameterValueException;
@ -59,60 +57,52 @@ import com.cloud.offerings.dao.NetworkOfferingServiceMapDao;
import com.cloud.user.AccountManager;
import com.cloud.user.AccountVO;
import com.cloud.user.UserVO;
import com.cloud.utils.component.ComponentContext;
import com.cloud.vm.dao.UserVmDetailsDao;
import junit.framework.TestCase;
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration(locations = "classpath:/createNetworkOffering.xml")
@RunWith(MockitoJUnitRunner.class)
public class CreateNetworkOfferingTest extends TestCase {
@Inject
ConfigurationManager configMgr;
@Inject
@Mock
ConfigurationDao configDao;
@Inject
@Mock
NetworkOfferingDao offDao;
@Inject
@Mock
NetworkOfferingServiceMapDao mapDao;
@Inject
@Mock
AccountManager accountMgr;
@Inject
@Mock
VpcManager vpcMgr;
@Inject
@Mock
UserVmDetailsDao userVmDetailsDao;
@Inject
@Mock
UsageEventDao UsageEventDao;
@Inject
@Mock
UsageEventDetailsDao usageEventDetailsDao;
@Inject
@Mock
UserIpAddressDetailsDao userIpAddressDetailsDao;
@Inject
@Mock
LoadBalancerVMMapDao _loadBalancerVMMapDao;
@Inject
@Mock
AnnotationDao annotationDao;
@InjectMocks
ConfigurationManager configMgr = new ConfigurationManagerImpl();
@Override
@Before
public void setUp() {
ComponentContext.initComponentsLifeCycle();
ConfigurationVO configVO = new ConfigurationVO("200", "200", "200", "200", "200", "200");
Mockito.when(configDao.findByName(anyString())).thenReturn(configVO);
Mockito.when(offDao.persist(any(NetworkOfferingVO.class))).thenReturn(new NetworkOfferingVO());
Mockito.when(offDao.persist(any(NetworkOfferingVO.class), nullable(Map.class))).thenReturn(new NetworkOfferingVO());
Mockito.when(mapDao.persist(any(NetworkOfferingServiceMapVO.class))).thenReturn(new NetworkOfferingServiceMapVO());
Mockito.when(accountMgr.getSystemUser()).thenReturn(new UserVO(1));

76
server/src/test/resources/createNetworkOffering.xml
View File

@ -1,76 +0,0 @@
<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
license agreements. See the NOTICE file distributed with this work for additional
information regarding copyright ownership. The ASF licenses this file to
you under the Apache License, Version 2.0 (the "License"); you may not use
this file except in compliance with the License. You may obtain a copy of
the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
by applicable law or agreed to in writing, software distributed under the
License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
OF ANY KIND, either express or implied. See the License for the specific
language governing permissions and limitations under the License. -->
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:context="http://www.springframework.org/schema/context"
xmlns:tx="http://www.springframework.org/schema/tx" xmlns:aop="http://www.springframework.org/schema/aop"
xsi:schemaLocation="http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/tx
http://www.springframework.org/schema/tx/spring-tx.xsd
http://www.springframework.org/schema/aop
http://www.springframework.org/schema/aop/spring-aop.xsd
http://www.springframework.org/schema/context
http://www.springframework.org/schema/context/spring-context.xsd">
<context:annotation-config />
<!-- @DB support -->
<bean id="componentContext" class="com.cloud.utils.component.ComponentContext" />
<bean id="transactionContextBuilder" class="com.cloud.utils.db.TransactionContextBuilder" />
<bean id="actionEventInterceptor" class="com.cloud.event.ActionEventInterceptor" />
<bean id="instantiatePostProcessor" class="com.cloud.utils.component.ComponentInstantiationPostProcessor">
<property name="Interceptors">
<list>
<ref bean="transactionContextBuilder" />
<ref bean="actionEventInterceptor" />
</list>
</property>
</bean>
<bean id="nicIpAliasDaoImpl" class="com.cloud.vm.dao.NicIpAliasDaoImpl" />
<bean id="ConfigurationManager" class="com.cloud.configuration.ConfigurationManagerImpl">
<property name="name" value="ConfigurationManager"/>
</bean>
<bean id="dataCenterGuestIpv6PrefixDaoImpl" class="com.cloud.dc.dao.DataCenterGuestIpv6PrefixDaoImpl" />
<bean id="ipv6GuestPrefixSubnetNetworkMapDaoImpl" class="com.cloud.network.dao.Ipv6GuestPrefixSubnetNetworkMapDaoImpl" />
<bean id="networkDetailsDaoImpl" class="com.cloud.network.dao.NetworkDetailsDaoImpl" />
<bean id="domainRouterDaoImpl" class="com.cloud.vm.dao.DomainRouterDaoImpl" />
<bean id="routerNetworkDaoImpl" class="com.cloud.network.dao.RouterNetworkDaoImpl" />
<bean id="userStatisticsDaoImpl" class="com.cloud.user.dao.UserStatisticsDaoImpl" />
<bean id="ipv6ServiceImpl" class="com.cloud.network.Ipv6ServiceImpl" />
<bean id="managementServiceConfigurationImpl" class="com.cloud.configuration.ManagementServiceConfigurationImpl" />
<bean class="org.apache.cloudstack.networkoffering.ChildTestConfiguration" />
<bean id="UservmDetailsDaoImpl" class="com.cloud.vm.dao.UserVmDetailsDaoImpl" />
<bean id="hostGpuGroupsDaoImpl" class="com.cloud.gpu.dao.HostGpuGroupsDaoImpl" />
<bean id="vGPUTypesDaoImpl" class="com.cloud.gpu.dao.VGPUTypesDaoImpl" />
<bean id="usageEventDaoImpl" class="com.cloud.event.dao.UsageEventDaoImpl" />
<bean id="usageEventDetailsDaoImpl" class="com.cloud.event.dao.UsageEventDetailsDaoImpl" />
<bean id="storagePoolHostDaoImpl" class="com.cloud.storage.dao.StoragePoolHostDaoImpl" />
<bean id="storagePoolTagsDaoImpl" class="com.cloud.storage.dao.StoragePoolTagsDaoImpl" />
<bean id="primaryDataStoreDaoImpl" class="org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDaoImpl" />
<bean id="userIpAddressDetailsDao" class="org.apache.cloudstack.resourcedetail.dao.UserIpAddressDetailsDaoImpl" />
<bean id="loadBalancerVMMapDaoImpl" class="com.cloud.network.dao.LoadBalancerVMMapDaoImpl" />
<bean id="imageStoreDaoImpl" class="org.apache.cloudstack.storage.datastore.db.ImageStoreDaoImpl" />
<bean id="messageBus" class="org.apache.cloudstack.framework.messagebus.MessageBusBase" />
<bean id="DiskOfferingDetailsDaoImpl" class="org.apache.cloudstack.resourcedetail.dao.DiskOfferingDetailsDaoImpl" />
<bean id="networkOfferingJoinDaoImpl" class="com.cloud.api.query.dao.NetworkOfferingJoinDaoImpl" />
<bean id="networkOfferingDetailsDaoImpl" class="com.cloud.offerings.dao.NetworkOfferingDetailsDaoImpl" />
<bean id="vMTemplateZoneDaoImpl" class="com.cloud.storage.dao.VMTemplateZoneDaoImpl" />
<bean id="indirectAgentLBImpl" class="org.apache.cloudstack.agent.lb.IndirectAgentLBServiceImpl" />
<bean id="VsphereStoragePolicyDaoImpl" class="com.cloud.dc.dao.VsphereStoragePolicyDaoImpl" />
<bean id="annotationDaoImpl" class="org.apache.cloudstack.annotation.dao.AnnotationDaoImpl" />
<bean id="PassphraseDaoImpl" class="org.apache.cloudstack.secret.dao.PassphraseDaoImpl" />
<bean id="configurationGroupDaoImpl" class="org.apache.cloudstack.framework.config.dao.ConfigurationGroupDaoImpl" />
<bean id="configurationSubGroupDaoImpl" class="org.apache.cloudstack.framework.config.dao.ConfigurationSubGroupDaoImpl" />
<bean id="reservationDao" class="org.apache.cloudstack.reservation.dao.ReservationDaoImpl" />
</beans>

1
utils/src/main/java/com/cloud/utils/backoff/impl/ConstantTimeBackoff.java
View File

@ -56,7 +56,6 @@ public class ConstantTimeBackoff extends AdapterBase implements BackoffAlgorithm
} finally {
_asleep.remove(current.getName());
}
return;
}
@Override

77
utils/src/main/java/com/cloud/utils/backoff/impl/RangeTimeBackoff.java Normal file
View File

@ -0,0 +1,77 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package com.cloud.utils.backoff.impl;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ThreadLocalRandom;
import org.apache.log4j.Logger;
import com.cloud.utils.NumbersUtil;
import com.cloud.utils.backoff.BackoffAlgorithm;
import com.cloud.utils.component.AdapterBase;
/**
* An implementation of BackoffAlgorithm that waits for some random seconds
* within a given range.
* After the time the client can try to perform the operation again.
*
**/
public class RangeTimeBackoff extends AdapterBase implements BackoffAlgorithm {
protected static final int DEFAULT_MIN_TIME = 5;
private int minTime = DEFAULT_MIN_TIME;
private int maxTime = DEFAULT_MIN_TIME;
private final Map<String, Thread> asleep = new ConcurrentHashMap<>();
private static final Logger LOG = Logger.getLogger(RangeTimeBackoff.class.getName());
@Override
public void waitBeforeRetry() {
long time = Math.max(minTime, 0) * 1000L;
Thread current = Thread.currentThread();
try {
asleep.put(current.getName(), current);
if (maxTime > minTime) {
time = ThreadLocalRandom.current().nextInt(minTime, maxTime) * 1000L;
}
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("Waiting %s for %d milliseconds", current.getName(), time));
}
Thread.sleep(time);
} catch (InterruptedException e) {
// JMX or other threads may interrupt this thread, but let's log it
// anyway, no exception to log as this is not an error
LOG.info("Thread " + current.getName() + " interrupted while waiting for retry");
} finally {
asleep.remove(current.getName());
}
}
@Override
public void reset() {
}
@Override
public boolean configure(String name, Map<String, Object> params) {
minTime = NumbersUtil.parseInt((String)params.get("minSeconds"), DEFAULT_MIN_TIME);
maxTime = NumbersUtil.parseInt((String)params.get("maxSeconds"), minTime);
return true;
}
}

12
utils/src/main/java/com/cloud/utils/nio/Link.java
View File

@ -48,6 +48,7 @@ import javax.net.ssl.TrustManagerFactory;
import org.apache.cloudstack.framework.ca.CAService;
import org.apache.cloudstack.utils.security.KeyStoreUtils;
import org.apache.cloudstack.utils.security.SSLUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.log4j.Logger;
import com.cloud.utils.PropertiesUtil;
@ -591,6 +592,10 @@ public class Link {
}
public static boolean doHandshake(final SocketChannel socketChannel, final SSLEngine sslEngine) throws IOException {
return doHandshake(socketChannel, sslEngine, null);
}
public static boolean doHandshake(final SocketChannel socketChannel, final SSLEngine sslEngine, Integer timeout) throws IOException {
if (socketChannel == null || sslEngine == null) {
return false;
}
@ -605,12 +610,13 @@ public class Link {
final long startTimeMills = System.currentTimeMillis();
HandshakeStatus handshakeStatus = sslEngine.getHandshakeStatus();
long timeoutMillis = ObjectUtils.defaultIfNull(timeout, 30) * 1000L;
while (handshakeStatus != SSLEngineResult.HandshakeStatus.FINISHED
&& handshakeStatus != SSLEngineResult.HandshakeStatus.NOT_HANDSHAKING) {
final long timeTaken = System.currentTimeMillis() - startTimeMills;
if (timeTaken > 30000L) {
s_logger.warn("SSL Handshake has taken more than 30s to connect to: " + socketChannel.getRemoteAddress() +
". Please investigate this connection.");
if (timeTaken > timeoutMillis) {
s_logger.warn(String.format("SSL Handshake has taken more than 30s to connect to: %s" +
" while status: %s. Please investigate this connection.", socketChannel.getRemoteAddress(), handshakeStatus));
return false;
}
switch (handshakeStatus) {

7
utils/src/main/java/com/cloud/utils/nio/NioClient.java
View File

@ -38,8 +38,8 @@ public class NioClient extends NioConnection {
protected String _host;
protected SocketChannel _clientConnection;
public NioClient(final String name, final String host, final int port, final int workers, final HandlerFactory factory) {
super(name, port, workers, factory);
public NioClient(final String name, final String host, final int port, final int workers, final Integer sslHandshakeTimeout, final HandlerFactory factory) {
super(name, port, workers, 1, 2, factory);
_host = host;
}
@ -61,7 +61,7 @@ public class NioClient extends NioConnection {
sslEngine.setUseClientMode(true);
sslEngine.setEnabledProtocols(SSLUtils.getSupportedProtocols(sslEngine.getEnabledProtocols()));
sslEngine.beginHandshake();
if (!Link.doHandshake(_clientConnection, sslEngine)) {
if (!Link.doHandshake(_clientConnection, sslEngine, getSslHandshakeTimeout())) {
s_logger.error("SSL Handshake failed while connecting to host: " + _host + " port: " + _port);
_selector.close();
throw new IOException("SSL Handshake failed while connecting to host: " + _host + " port: " + _port);
@ -81,6 +81,7 @@ public class NioClient extends NioConnection {
_selector.close();
throw new IOException("Failed to initialise security", e);
} catch (final IOException e) {
s_logger.error("IOException", e);
_selector.close();
throw e;
}

118
utils/src/main/java/com/cloud/utils/nio/NioConnection.java
View File

@ -32,17 +32,22 @@ import java.nio.channels.SelectionKey;
import java.nio.channels.Selector;
import java.nio.channels.ServerSocketChannel;
import java.nio.channels.SocketChannel;
import java.security.GeneralSecurityException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import javax.net.ssl.SSLEngine;
@ -73,15 +78,27 @@ public abstract class NioConnection implements Callable<Boolean> {
protected ExecutorService _executor;
protected ExecutorService _sslHandshakeExecutor;
protected CAService caService;
protected Integer sslHandshakeTimeout = null;
private int sslHandshakeMaxWorkers;
private final AtomicInteger activeAcceptConnections = new AtomicInteger(0);
private final BlockingQueue<Runnable> workerQueue;
private final BlockingQueue<Runnable> sslHandshakeQueue;
public NioConnection(final String name, final int port, final int workers, final HandlerFactory factory) {
public NioConnection(final String name, final int port, final int workers, final int sslHandshakeMinWorkers,
final int sslHandshakeMaxWorkers, final HandlerFactory factory) {
_name = name;
_isRunning = false;
_selector = null;
_port = port;
_factory = factory;
_executor = new ThreadPoolExecutor(workers, 5 * workers, 1, TimeUnit.DAYS, new LinkedBlockingQueue<Runnable>(), new NamedThreadFactory(name + "-Handler"));
_sslHandshakeExecutor = Executors.newCachedThreadPool(new NamedThreadFactory(name + "-SSLHandshakeHandler"));
this.sslHandshakeMaxWorkers = sslHandshakeMaxWorkers;
workerQueue = new LinkedBlockingQueue<>(5 * workers);
_executor = new ThreadPoolExecutor(workers, 5 * workers, 1, TimeUnit.DAYS,
workerQueue, new NamedThreadFactory(name + "-Handler"), new ThreadPoolExecutor.AbortPolicy());
sslHandshakeQueue = new SynchronousQueue<>();
_sslHandshakeExecutor = new ThreadPoolExecutor(sslHandshakeMinWorkers, sslHandshakeMaxWorkers, 30,
TimeUnit.MINUTES, sslHandshakeQueue, new NamedThreadFactory(name + "-Handler"),
new ThreadPoolExecutor.AbortPolicy());
}
public void setCAService(final CAService caService) {
@ -94,7 +111,7 @@ public abstract class NioConnection implements Callable<Boolean> {
try {
init();
} catch (final ConnectException e) {
s_logger.warn("Unable to connect to remote: is there a server running on port " + _port);
s_logger.warn("Unable to connect to remote: is there a server running on port" + _port);
return;
} catch (final IOException e) {
s_logger.error("Unable to initialize the threads.", e);
@ -192,6 +209,15 @@ public abstract class NioConnection implements Callable<Boolean> {
protected void accept(final SelectionKey key) throws IOException {
final ServerSocketChannel serverSocketChannel = (ServerSocketChannel)key.channel();
final SocketChannel socketChannel = serverSocketChannel.accept();
if (activeAcceptConnections.get() >= sslHandshakeMaxWorkers) {
// Reject new connection if the server is busy
s_logger.warn("Server is busy. Rejecting new connection.");
if (socketChannel != null) {
socketChannel.close();
}
_selector.wakeup();
return;
}
socketChannel.configureBlocking(false);
final Socket socket = socketChannel.socket();
@ -201,48 +227,52 @@ public abstract class NioConnection implements Callable<Boolean> {
s_logger.trace("Connection accepted for " + socket);
}
final SSLEngine sslEngine;
try {
sslEngine = Link.initServerSSLEngine(caService, socketChannel.getRemoteAddress().toString());
sslEngine.setUseClientMode(false);
sslEngine.setEnabledProtocols(SSLUtils.getSupportedProtocols(sslEngine.getEnabledProtocols()));
final NioConnection nioConnection = this;
_sslHandshakeExecutor.submit(new Runnable() {
@Override
public void run() {
_selector.wakeup();
try {
sslEngine.beginHandshake();
if (!Link.doHandshake(socketChannel, sslEngine)) {
throw new IOException("SSL handshake timed out with " + socketChannel.getRemoteAddress());
}
if (s_logger.isTraceEnabled()) {
s_logger.trace("SSL: Handshake done");
}
final InetSocketAddress saddr = (InetSocketAddress)socket.getRemoteSocketAddress();
final Link link = new Link(saddr, nioConnection);
link.setSSLEngine(sslEngine);
link.setKey(socketChannel.register(key.selector(), SelectionKey.OP_READ, link));
final Task task = _factory.create(Task.Type.CONNECT, link, null);
registerLink(saddr, link);
_executor.submit(task);
} catch (IOException e) {
if (s_logger.isTraceEnabled()) {
s_logger.trace("Connection closed due to failure: " + e.getMessage());
}
closeAutoCloseable(socket, "accepting socket");
closeAutoCloseable(socketChannel, "accepting socketChannel");
} finally {
_selector.wakeup();
_sslHandshakeExecutor.submit(() -> {
final InetSocketAddress socketAddress = (InetSocketAddress)socket.getRemoteSocketAddress();
activeAcceptConnections.incrementAndGet();
long startTime = System.currentTimeMillis();
_selector.wakeup();
try {
final SSLEngine sslEngine = Link.initServerSSLEngine(caService, socketChannel.getRemoteAddress().toString());
sslEngine.setUseClientMode(false);
sslEngine.setEnabledProtocols(SSLUtils.getSupportedProtocols(sslEngine.getEnabledProtocols()));
sslEngine.beginHandshake();
if (!Link.doHandshake(socketChannel, sslEngine, getSslHandshakeTimeout())) {
throw new IOException("SSL handshake timed out with " + socketAddress);
}
if (s_logger.isTraceEnabled()) {
s_logger.trace("SSL: Handshake done");
}
final Link link = new Link(socketAddress, nioConnection);
link.setSSLEngine(sslEngine);
link.setKey(socketChannel.register(key.selector(), SelectionKey.OP_READ, link));
final Task task = _factory.create(Task.Type.CONNECT, link, null);
registerLink(socketAddress, link);
_executor.submit(task);
} catch (final GeneralSecurityException | IOException e) {
if (s_logger.isTraceEnabled()) {
s_logger.trace(socket.getRemoteSocketAddress()+ "Connection closed due to failure: " + e.getMessage());
}
closeAutoCloseable(socket, "accepting socket");
closeAutoCloseable(socketChannel, "accepting socketChannel");
} finally {
int connections = activeAcceptConnections.decrementAndGet();
if (s_logger.isTraceEnabled()) {
s_logger.trace(String.format("Accept task complete for %s - time taken: %d, " +
"active accept connections: %d",
socketAddress, (System.currentTimeMillis() - startTime), connections));
}
_selector.wakeup();
}
});
} catch (final Exception e) {
} catch (final RejectedExecutionException e) {
if (s_logger.isTraceEnabled()) {
s_logger.trace("Connection closed due to failure: " + e.getMessage());
s_logger.trace(socket.getRemoteSocketAddress()+ " Accept Task rejected: " + e.getMessage());
}
closeAutoCloseable(socket, "accepting socket");
closeAutoCloseable(socketChannel, "accepting socketChannel");
closeAutoCloseable(socket, "Rejecting connection - accepting socket");
closeAutoCloseable(socketChannel, "Rejecting connection - accepting socketChannel");
} finally {
_selector.wakeup();
}
@ -412,7 +442,7 @@ public abstract class NioConnection implements Callable<Boolean> {
try {
_executor.submit(task);
} catch (final Exception e) {
s_logger.warn("Exception occurred when submitting the task", e);
s_logger.warn(String.format("Exception occurred when submitting the task for connect: %s", socket), e);
}
} catch (final IOException e) {
logTrace(e, key, 2);
@ -512,4 +542,12 @@ public abstract class NioConnection implements Callable<Boolean> {
this.att = att;
}
}
public Integer getSslHandshakeTimeout() {
return sslHandshakeTimeout;
}
public void setSslHandshakeTimeout(Integer sslHandshakeTimeout) {
this.sslHandshakeTimeout = sslHandshakeTimeout;
}
}

7
utils/src/main/java/com/cloud/utils/nio/NioServer.java
View File

@ -38,9 +38,12 @@ public class NioServer extends NioConnection {
protected WeakHashMap<InetSocketAddress, Link> _links;
public NioServer(final String name, final int port, final int workers, final HandlerFactory factory, final CAService caService) {
super(name, port, workers, factory);
public NioServer(final String name, final int port, final int workers, final int sslHandshakeMinWorkers,
final int sslHandshakeMaxWorkers, final HandlerFactory factory, final CAService caService,
final Integer sslHandShakeTimeout) {
super(name, port, workers, sslHandshakeMinWorkers, sslHandshakeMaxWorkers, factory);
setCAService(caService);
setSslHandshakeTimeout(sslHandShakeTimeout);
_localAddr = null;
_links = new WeakHashMap<InetSocketAddress, Link>(1024);
}

48
utils/src/main/java/org/apache/cloudstack/utils/cache/LazyCache.java vendored Normal file
View File

@ -0,0 +1,48 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.utils.cache;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.github.benmanes.caffeine.cache.LoadingCache;
public class LazyCache<K, V> {
private final LoadingCache<K, V> cache;
public LazyCache(long maximumSize, long expireAfterWriteSeconds, Function<K, V> loader) {
this.cache = Caffeine.newBuilder()
.maximumSize(maximumSize)
.expireAfterWrite(expireAfterWriteSeconds, TimeUnit.SECONDS)
.build(loader::apply);
}
public V get(K key) {
return cache.get(key);
}
public void invalidate(K key) {
cache.invalidate(key);
}
public void clear() {
cache.invalidateAll();
}
}

48
utils/src/main/java/org/apache/cloudstack/utils/cache/SingleCache.java vendored Normal file
View File

@ -0,0 +1,48 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.utils.cache;
import java.util.concurrent.TimeUnit;
import java.util.function.Supplier;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.github.benmanes.caffeine.cache.LoadingCache;
public class SingleCache<V> {
private final LoadingCache<Integer, V> cache;
public SingleCache(long expireAfterWriteSeconds, Supplier<V> loader) {
this.cache = Caffeine.newBuilder()
.maximumSize(1)
.expireAfterWrite(expireAfterWriteSeconds, TimeUnit.SECONDS)
.build(key -> loader.get());
}
public V get() {
return cache.get(0);
}
public void invalidate() {
cache.invalidate(0);
}
public void clear() {
cache.invalidateAll();
}
}

84
utils/src/main/java/org/apache/cloudstack/utils/executor/QueuedExecutor.java Normal file
View File

@ -0,0 +1,84 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.utils.executor;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
import org.apache.log4j.Logger;
import com.cloud.utils.concurrency.NamedThreadFactory;
public class QueuedExecutor<K> {
private final String name;
private final int processingSize;
private final int processingInterval;
private final BlockingQueue<K> requestQueue;
private final ScheduledExecutorService executorService;
private final Logger logger;
private final Consumer<K> consumer;
public QueuedExecutor(String name, int processingSize, int processingInterval, int maxThreads, Logger logger,
Consumer<K> consumer) {
this.name = name;
this.processingSize = processingSize;
this.processingInterval = processingInterval;
this.logger = logger;
this.consumer = consumer;
requestQueue = new LinkedBlockingQueue<>(processingSize);
executorService = Executors.newScheduledThreadPool(maxThreads, new NamedThreadFactory(name));
}
public void queueRequest(K request) {
try {
requestQueue.put(request);
if (requestQueue.size() >= processingSize) {
processRequests();
}
} catch (InterruptedException e) {
logger.warn(String.format("Error queuing request for %s", name), e);
}
}
public void startProcessing() {
executorService.scheduleAtFixedRate(this::processRequests, 0,
processingInterval, TimeUnit.SECONDS);
}
private void processRequests() {
List<K> requestsToProcess = new ArrayList<>();
requestQueue.drainTo(requestsToProcess, processingSize);
if (!requestsToProcess.isEmpty()) {
for (K request : requestsToProcess) {
consumer.accept(request);
}
}
}
public void shutdown() {
executorService.shutdown();
}
}

6
utils/src/test/java/com/cloud/utils/testcase/NioTest.java
View File

@ -98,7 +98,7 @@ public class NioTest {
testBytes = new byte[1000000];
randomGenerator.nextBytes(testBytes);
server = new NioServer("NioTestServer", 0, 1, new NioTestServer(), null);
server = new NioServer("NioTestServer", 0, 1, 1, 2 * totalTestCount, new NioTestServer(), null, null);
try {
server.start();
} catch (final NioConnectionException e) {
@ -110,7 +110,7 @@ public class NioTest {
maliciousClients.add(maliciousClient);
maliciousExecutor.submit(new ThreadedNioClient(maliciousClient));
final NioClient client = new NioClient("NioTestClient-" + i, "127.0.0.1", server.getPort(), 1, new NioTestClient());
final NioClient client = new NioClient("NioTestClient-" + i, "127.0.0.1", server.getPort(), 1, null, new NioTestClient());
clients.add(client);
clientExecutor.submit(new ThreadedNioClient(client));
}
@ -179,7 +179,7 @@ public class NioTest {
public class NioMaliciousClient extends NioClient {
public NioMaliciousClient(String name, String host, int port, int workers, HandlerFactory factory) {
super(name, host, port, workers, factory);
super(name, host, port, workers, null, factory);
}
@Override

115
utils/src/test/java/org/apache/cloudstack/utils/cache/LazyCacheTest.java vendored Normal file
View File

@ -0,0 +1,115 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.utils.cache;
import static org.junit.Assert.assertEquals;
import java.util.function.Function;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
@RunWith(MockitoJUnitRunner.class)
public class LazyCacheTest {
private final long expireSeconds = 1;
private final String cacheValuePrefix = "ComputedValueFor:";
private LazyCache<String, String> cache;
private Function<String, String> mockLoader;
@Before
public void setUp() {
mockLoader = Mockito.mock(Function.class);
Mockito.when(mockLoader.apply(Mockito.anyString())).thenAnswer(invocation -> cacheValuePrefix + invocation.getArgument(0));
cache = new LazyCache<>(4, expireSeconds, mockLoader);
}
@Test
public void testCacheMissAndLoader() {
String key = "key1";
String value = cache.get(key);
assertEquals(cacheValuePrefix + key, value);
Mockito.verify(mockLoader).apply(key);
}
@Test
public void testLoaderNotCalledIfPresent() {
String key = "key2";
cache.get(key);
try {
Thread.sleep((long)(0.9 * expireSeconds * 1000));
} catch (InterruptedException ie) {
Assert.fail(String.format("Exception occurred: %s", ie.getMessage()));
}
cache.get(key);
Mockito.verify(mockLoader, Mockito.times(1)).apply(key);
}
@Test
public void testCacheExpiration() {
String key = "key3";
cache.get(key);
try {
Thread.sleep((long)(1.1 * expireSeconds * 1000));
} catch (InterruptedException ie) {
Assert.fail(String.format("Exception occurred: %s", ie.getMessage()));
}
cache.get(key);
Mockito.verify(mockLoader, Mockito.times(2)).apply(key);
}
@Test
public void testInvalidateKey() {
String key = "key4";
cache.get(key);
cache.invalidate(key);
cache.get(key);
Mockito.verify(mockLoader, Mockito.times(2)).apply(key);
}
@Test
public void testClearCache() {
String key1 = "key5";
String key2 = "key6";
cache.get(key1);
cache.get(key2);
cache.clear();
cache.get(key1);
Mockito.verify(mockLoader, Mockito.times(2)).apply(key1);
Mockito.verify(mockLoader, Mockito.times(1)).apply(key2);
}
@Test
public void testMaximumSize() {
String key = "key7";
cache.get(key);
for (int i = 0; i < 4; i++) {
cache.get(String.format("newkey-%d", i));
}
try {
Thread.sleep(100);
} catch (InterruptedException ie) {
Assert.fail(String.format("Exception occurred: %s", ie.getMessage()));
}
cache.get(key);
Mockito.verify(mockLoader, Mockito.times(2)).apply(key);
}
}