etan39
/
cloudstack
mirror of https://github.com/apache/cloudstack.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch '4.20'

This commit is contained in:
Daan Hoogland 2025-02-01 21:20:08 +01:00
parent 7abda3b963 085bd3bda5
commit 2654890e86
145 changed files with 4841 additions and 2413 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.python-version
View File

@ -1 +1 @@
3.6
3.10

7
agent/conf/agent.properties
View File

@ -434,3 +434,10 @@ iscsi.session.cleanup.enabled=false
# Implicit host tags managed by agent.properties
# host.tags=
# Timeout(in seconds) for SSL handshake when agent connects to server. When no value is set then default value of 30s
# will be used
#ssl.handshake.timeout=
# Wait(in seconds) during agent reconnections. When no value is set then default value of 5s will be used
#backoff.seconds=

829
agent/src/main/java/com/cloud/agent/Agent.java
View File

File diff suppressed because it is too large Load Diff

57
agent/src/main/java/com/cloud/agent/AgentShell.java
View File

@ -16,29 +16,6 @@
// under the License.
package com.cloud.agent;
import com.cloud.agent.Agent.ExitStatus;
import com.cloud.agent.dao.StorageComponent;
import com.cloud.agent.dao.impl.PropertiesStorage;
import com.cloud.agent.properties.AgentProperties;
import com.cloud.agent.properties.AgentPropertiesFileHandler;
import com.cloud.resource.ServerResource;
import com.cloud.utils.LogUtils;
import com.cloud.utils.ProcessUtil;
import com.cloud.utils.PropertiesUtil;
import com.cloud.utils.backoff.BackoffAlgorithm;
import com.cloud.utils.backoff.impl.ConstantTimeBackoff;
import com.cloud.utils.exception.CloudRuntimeException;
import org.apache.commons.daemon.Daemon;
import org.apache.commons.daemon.DaemonContext;
import org.apache.commons.daemon.DaemonInitException;
import org.apache.commons.lang.math.NumberUtils;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.core.config.Configurator;
import javax.naming.ConfigurationException;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
@ -53,6 +30,31 @@ import java.util.Map;
import java.util.Properties;
import java.util.UUID;
import javax.naming.ConfigurationException;
import org.apache.commons.daemon.Daemon;
import org.apache.commons.daemon.DaemonContext;
import org.apache.commons.daemon.DaemonInitException;
import org.apache.commons.lang.math.NumberUtils;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.core.config.Configurator;
import com.cloud.agent.Agent.ExitStatus;
import com.cloud.agent.dao.StorageComponent;
import com.cloud.agent.dao.impl.PropertiesStorage;
import com.cloud.agent.properties.AgentProperties;
import com.cloud.agent.properties.AgentPropertiesFileHandler;
import com.cloud.resource.ServerResource;
import com.cloud.utils.LogUtils;
import com.cloud.utils.ProcessUtil;
import com.cloud.utils.PropertiesUtil;
import com.cloud.utils.backoff.BackoffAlgorithm;
import com.cloud.utils.backoff.impl.ConstantTimeBackoff;
import com.cloud.utils.exception.CloudRuntimeException;
public class AgentShell implements IAgentShell, Daemon {
protected static Logger LOGGER = LogManager.getLogger(AgentShell.class);
@ -415,7 +417,9 @@ public class AgentShell implements IAgentShell, Daemon {
LOGGER.info("Defaulting to the constant time backoff algorithm");
_backoff = new ConstantTimeBackoff();
_backoff.configure("ConstantTimeBackoff", new HashMap<String, Object>());
Map<String, Object> map = new HashMap<>();
map.put("seconds", _properties.getProperty("backoff.seconds"));
_backoff.configure("ConstantTimeBackoff", map);
}
private void launchAgent() throws ConfigurationException {
@ -464,6 +468,11 @@ public class AgentShell implements IAgentShell, Daemon {
agent.start();
}
@Override
public Integer getSslHandshakeTimeout() {
return AgentPropertiesFileHandler.getPropertyValue(AgentProperties.SSL_HANDSHAKE_TIMEOUT);
}
public synchronized int getNextAgentId() {
return _nextAgentId++;
}

2
agent/src/main/java/com/cloud/agent/IAgentShell.java
View File

@ -74,4 +74,6 @@ public interface IAgentShell {
boolean isConnectionTransfer();
void setConnectionTransfer(boolean connectionTransfer);
Integer getSslHandshakeTimeout();
}

7
agent/src/main/java/com/cloud/agent/properties/AgentProperties.java
View File

@ -811,6 +811,13 @@ public class AgentProperties{
*/
public static final Property<String> HOST_TAGS = new Property<>("host.tags", null, String.class);
/**
* Timeout for SSL handshake in seconds
* Data type: Integer.<br>
* Default value: <code>null</code>
*/
public static final Property<Integer> SSL_HANDSHAKE_TIMEOUT = new Property<>("ssl.handshake.timeout", null, Integer.class);
public static class Property <T>{
private String name;
private T defaultValue;

7
agent/src/test/java/com/cloud/agent/AgentShellTest.java
View File

@ -362,4 +362,11 @@ public class AgentShellTest {
Assert.assertEquals(expected, shell.getConnectedHost());
}
@Test
public void testGetSslHandshakeTimeout() {
Integer expected = 1;
agentPropertiesFileHandlerMocked.when(() -> AgentPropertiesFileHandler.getPropertyValue(Mockito.eq(AgentProperties.SSL_HANDSHAKE_TIMEOUT))).thenReturn(expected);
Assert.assertEquals(expected, agentShellSpy.getSslHandshakeTimeout());
}
}

257
agent/src/test/java/com/cloud/agent/AgentTest.java Normal file
View File

@ -0,0 +1,257 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package com.cloud.agent;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.eq;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.IOException;
import java.net.InetSocketAddress;
import javax.naming.ConfigurationException;
import org.apache.logging.log4j.Logger;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.junit.MockitoJUnitRunner;
import org.springframework.test.util.ReflectionTestUtils;
import com.cloud.resource.ServerResource;
import com.cloud.utils.backoff.impl.ConstantTimeBackoff;
import com.cloud.utils.nio.Link;
import com.cloud.utils.nio.NioConnection;
@RunWith(MockitoJUnitRunner.class)
public class AgentTest {
Agent agent;
private AgentShell shell;
private ServerResource serverResource;
private Logger logger;
@Before
public void setUp() throws ConfigurationException {
shell = mock(AgentShell.class);
serverResource = mock(ServerResource.class);
doReturn(true).when(serverResource).configure(any(), any());
doReturn(1).when(shell).getWorkers();
doReturn(1).when(shell).getPingRetries();
agent = new Agent(shell, 1, serverResource);
logger = mock(Logger.class);
ReflectionTestUtils.setField(agent, "logger", logger);
}
@Test
public void testGetLinkLogNullLinkReturnsEmptyString() {
Link link = null;
String result = agent.getLinkLog(link);
assertEquals("", result);
}
@Test
public void testGetLinkLogLinkWithTraceEnabledReturnsLinkLogWithHashCode() {
Link link = mock(Link.class);
InetSocketAddress socketAddress = new InetSocketAddress("192.168.1.100", 1111);
when(link.getSocketAddress()).thenReturn(socketAddress);
when(logger.isTraceEnabled()).thenReturn(true);
String result = agent.getLinkLog(link);
System.out.println(result);
assertTrue(result.startsWith(System.identityHashCode(link) + "-"));
assertTrue(result.contains("192.168.1.100"));
}
@Test
public void testGetAgentNameWhenServerResourceIsNull() {
ReflectionTestUtils.setField(agent, "serverResource", null);
assertEquals("Agent", agent.getAgentName());
}
@Test
public void testGetAgentNameWhenAppendAgentNameIsTrue() {
when(serverResource.isAppendAgentNameToLogs()).thenReturn(true);
when(serverResource.getName()).thenReturn("TestAgent");
String agentName = agent.getAgentName();
assertEquals("TestAgent", agentName);
}
@Test
public void testGetAgentNameWhenAppendAgentNameIsFalse() {
when(serverResource.isAppendAgentNameToLogs()).thenReturn(false);
String agentName = agent.getAgentName();
assertEquals("Agent", agentName);
}
@Test
public void testAgentInitialization() {
Runtime.getRuntime().removeShutdownHook(agent.shutdownThread);
when(shell.getPingRetries()).thenReturn(3);
when(shell.getWorkers()).thenReturn(5);
agent.setupShutdownHookAndInitExecutors();
assertNotNull(agent.selfTaskExecutor);
assertNotNull(agent.outRequestHandler);
assertNotNull(agent.requestHandler);
}
@Test
public void testAgentShutdownHookAdded() {
Runtime.getRuntime().removeShutdownHook(agent.shutdownThread);
agent.setupShutdownHookAndInitExecutors();
verify(logger).trace("Adding shutdown hook");
}
@Test
public void testGetResourceGuidValidGuidAndResourceName() {
when(shell.getGuid()).thenReturn("12345");
String result = agent.getResourceGuid();
assertTrue(result.startsWith("12345-" + ServerResource.class.getSimpleName()));
}
@Test
public void testGetZoneReturnsValidZone() {
when(shell.getZone()).thenReturn("ZoneA");
String result = agent.getZone();
assertEquals("ZoneA", result);
}
@Test
public void testGetPodReturnsValidPod() {
when(shell.getPod()).thenReturn("PodA");
String result = agent.getPod();
assertEquals("PodA", result);
}
@Test
public void testSetLinkAssignsLink() {
Link mockLink = mock(Link.class);
agent.setLink(mockLink);
assertEquals(mockLink, agent.link);
}
@Test
public void testGetResourceReturnsServerResource() {
ServerResource mockResource = mock(ServerResource.class);
ReflectionTestUtils.setField(agent, "serverResource", mockResource);
ServerResource result = agent.getResource();
assertSame(mockResource, result);
}
@Test
public void testGetResourceName() {
String result = agent.getResourceName();
assertTrue(result.startsWith(ServerResource.class.getSimpleName()));
}
@Test
public void testUpdateLastPingResponseTimeUpdatesCurrentTime() {
long beforeUpdate = System.currentTimeMillis();
agent.updateLastPingResponseTime();
long updatedTime = agent.lastPingResponseTime.get();
assertTrue(updatedTime >= beforeUpdate);
assertTrue(updatedTime <= System.currentTimeMillis());
}
@Test
public void testGetNextSequenceIncrementsSequence() {
long initialSequence = agent.getNextSequence();
long nextSequence = agent.getNextSequence();
assertEquals(initialSequence + 1, nextSequence);
long thirdSequence = agent.getNextSequence();
assertEquals(nextSequence + 1, thirdSequence);
}
@Test
public void testRegisterControlListenerAddsListener() {
IAgentControlListener listener = mock(IAgentControlListener.class);
agent.registerControlListener(listener);
assertTrue(agent.controlListeners.contains(listener));
}
@Test
public void testUnregisterControlListenerRemovesListener() {
IAgentControlListener listener = mock(IAgentControlListener.class);
agent.registerControlListener(listener);
assertTrue(agent.controlListeners.contains(listener));
agent.unregisterControlListener(listener);
assertFalse(agent.controlListeners.contains(listener));
}
@Test
public void testCloseAndTerminateLinkLinkIsNullDoesNothing() {
agent.closeAndTerminateLink(null);
}
@Test
public void testCloseAndTerminateLinkValidLinkCallsCloseAndTerminate() {
Link mockLink = mock(Link.class);
agent.closeAndTerminateLink(mockLink);
verify(mockLink).close();
verify(mockLink).terminated();
}
@Test
public void testStopAndCleanupConnectionConnectionIsNullDoesNothing() {
agent.connection = null;
agent.stopAndCleanupConnection(false);
}
@Test
public void testStopAndCleanupConnectionValidConnectionNoWaitStopsAndCleansUp() throws IOException {
NioConnection mockConnection = mock(NioConnection.class);
agent.connection = mockConnection;
agent.stopAndCleanupConnection(false);
verify(mockConnection).stop();
verify(mockConnection).cleanUp();
}
@Test
public void testStopAndCleanupConnectionCleanupThrowsIOExceptionLogsWarning() throws IOException {
NioConnection mockConnection = mock(NioConnection.class);
agent.connection = mockConnection;
doThrow(new IOException("Cleanup failed")).when(mockConnection).cleanUp();
agent.stopAndCleanupConnection(false);
verify(mockConnection).stop();
verify(logger).warn(eq("Fail to clean up old connection. {}"), any(IOException.class));
}
@Test
public void testStopAndCleanupConnectionValidConnectionWaitForStopWaitsForStartupToStop() throws IOException {
NioConnection mockConnection = mock(NioConnection.class);
ConstantTimeBackoff mockBackoff = mock(ConstantTimeBackoff.class);
mockBackoff.setTimeToWait(0);
agent.connection = mockConnection;
when(shell.getBackoffAlgorithm()).thenReturn(mockBackoff);
when(mockConnection.isStartup()).thenReturn(true, true, false);
agent.stopAndCleanupConnection(true);
verify(mockConnection).stop();
verify(mockConnection).cleanUp();
verify(mockBackoff, times(3)).waitBeforeRetry();
}
}

5
api/src/main/java/org/apache/cloudstack/acl/RoleService.java
View File

@ -30,6 +30,11 @@ public interface RoleService {
ConfigKey<Boolean> EnableDynamicApiChecker = new ConfigKey<>("Advanced", Boolean.class, "dynamic.apichecker.enabled", "false",
"If set to true, this enables the dynamic role-based api access checker and disables the default static role-based api access checker.", true);
ConfigKey<Integer> DynamicApiCheckerCachePeriod = new ConfigKey<>("Advanced", Integer.class,
"dynamic.apichecker.cache.period", "0",
"Defines the expiration time in seconds for the Dynamic API Checker cache, determining how long cached data is retained before being refreshed. If set to zero then caching will be disabled",
false);
boolean isEnabled();
/**

7
api/src/main/java/org/apache/cloudstack/api/command/admin/domain/ListDomainsCmd.java
View File

@ -100,7 +100,7 @@ public class ListDomainsCmd extends BaseListCmd implements UserCmd {
dv = EnumSet.of(DomainDetails.all);
} else {
try {
ArrayList<DomainDetails> dc = new ArrayList<DomainDetails>();
ArrayList<DomainDetails> dc = new ArrayList<>();
for (String detail : viewDetails) {
dc.add(DomainDetails.valueOf(detail));
}
@ -142,7 +142,10 @@ public class ListDomainsCmd extends BaseListCmd implements UserCmd {
if (CollectionUtils.isEmpty(response)) {
return;
}
_resourceLimitService.updateTaggedResourceLimitsAndCountsForDomains(response, getTag());
EnumSet<DomainDetails> details = getDetails();
if (details.contains(DomainDetails.all) || details.contains(DomainDetails.resource)) {
_resourceLimitService.updateTaggedResourceLimitsAndCountsForDomains(response, getTag());
}
if (!getShowIcon()) {
return;
}

5
api/src/main/java/org/apache/cloudstack/api/command/user/account/ListAccountsCmd.java
View File

@ -157,7 +157,10 @@ public class ListAccountsCmd extends BaseListDomainResourcesCmd implements UserC
if (CollectionUtils.isEmpty(response)) {
return;
}
_resourceLimitService.updateTaggedResourceLimitsAndCountsForAccounts(response, getTag());
EnumSet<DomainDetails> details = getDetails();
if (details.contains(DomainDetails.all) || details.contains(DomainDetails.resource)) {
_resourceLimitService.updateTaggedResourceLimitsAndCountsForAccounts(response, getTag());
}
if (!getShowIcon()) {
return;
}

7
api/src/main/java/org/apache/cloudstack/api/command/user/firewall/CreateFirewallRuleCmd.java
View File

@ -19,6 +19,7 @@ package org.apache.cloudstack.api.command.user.firewall;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.collections.CollectionUtils;
import org.apache.cloudstack.acl.RoleType;
import org.apache.cloudstack.api.APICommand;
@ -40,6 +41,7 @@ import com.cloud.exception.ResourceUnavailableException;
import com.cloud.network.IpAddress;
import com.cloud.network.rules.FirewallRule;
import com.cloud.user.Account;
import com.cloud.utils.StringUtils;
import com.cloud.utils.net.NetUtils;
@APICommand(name = "createFirewallRule", description = "Creates a firewall rule for a given IP address", responseObject = FirewallResponse.class, entityType = {FirewallRule.class},
@ -125,14 +127,13 @@ public class CreateFirewallRuleCmd extends BaseAsyncCreateCmd implements Firewal
@Override
public List<String> getSourceCidrList() {
if (cidrlist != null) {
if (CollectionUtils.isNotEmpty(cidrlist) && !(cidrlist.size() == 1 && StringUtils.isBlank(cidrlist.get(0)))) {
return cidrlist;
} else {
List<String> oneCidrList = new ArrayList<String>();
List<String> oneCidrList = new ArrayList<>();
oneCidrList.add(NetUtils.ALL_IP4_CIDRS);
return oneCidrList;
}
}
// ///////////////////////////////////////////////////

2
api/src/main/java/org/apache/cloudstack/outofbandmanagement/OutOfBandManagementService.java
View File

@ -39,7 +39,7 @@ public interface OutOfBandManagementService {
long getId();
boolean isOutOfBandManagementEnabled(Host host);
void submitBackgroundPowerSyncTask(Host host);
boolean transitionPowerStateToDisabled(List<? extends Host> hosts);
boolean transitionPowerStateToDisabled(List<Long> hostIds);
OutOfBandManagementResponse enableOutOfBandManagement(DataCenter zone);
OutOfBandManagementResponse enableOutOfBandManagement(Cluster cluster);

13
api/src/test/java/org/apache/cloudstack/api/command/admin/domain/ListDomainsCmdTest.java
View File

@ -18,6 +18,7 @@ package org.apache.cloudstack.api.command.admin.domain;
import java.util.List;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.api.response.DomainResponse;
import org.junit.Assert;
import org.junit.Test;
@ -71,7 +72,17 @@ public class ListDomainsCmdTest {
cmd._resourceLimitService = resourceLimitService;
ReflectionTestUtils.setField(cmd, "tag", "abc");
cmd.updateDomainResponse(List.of(Mockito.mock(DomainResponse.class)));
Mockito.verify(resourceLimitService, Mockito.times(1)).updateTaggedResourceLimitsAndCountsForDomains(Mockito.any(), Mockito.any());
Mockito.verify(resourceLimitService).updateTaggedResourceLimitsAndCountsForDomains(Mockito.any(), Mockito.any());
}
@Test
public void testUpdateDomainResponseWithDomainsMinDetails() {
ListDomainsCmd cmd = new ListDomainsCmd();
ReflectionTestUtils.setField(cmd, "viewDetails", List.of(ApiConstants.DomainDetails.min.toString()));
cmd._resourceLimitService = resourceLimitService;
ReflectionTestUtils.setField(cmd, "tag", "abc");
cmd.updateDomainResponse(List.of(Mockito.mock(DomainResponse.class)));
Mockito.verify(resourceLimitService, Mockito.never()).updateTaggedResourceLimitsAndCountsForDomains(Mockito.any(), Mockito.any());
}
}

15
api/src/test/java/org/apache/cloudstack/api/command/user/account/ListAccountsCmdTest.java
View File

@ -18,6 +18,7 @@ package org.apache.cloudstack.api.command.user.account;
import java.util.List;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.api.response.AccountResponse;
import org.junit.Assert;
import org.junit.Test;
@ -58,7 +59,7 @@ public class ListAccountsCmdTest {
}
@Test
public void testUpdateDomainResponseNoDomains() {
public void testUpdateAccountResponseNoAccounts() {
ListAccountsCmd cmd = new ListAccountsCmd();
cmd._resourceLimitService = resourceLimitService;
cmd.updateAccountResponse(null);
@ -66,11 +67,21 @@ public class ListAccountsCmdTest {
}
@Test
public void testUpdateDomainResponseWithDomains() {
public void testUpdateDomainResponseWithAccounts() {
ListAccountsCmd cmd = new ListAccountsCmd();
cmd._resourceLimitService = resourceLimitService;
ReflectionTestUtils.setField(cmd, "tag", "abc");
cmd.updateAccountResponse(List.of(Mockito.mock(AccountResponse.class)));
Mockito.verify(resourceLimitService, Mockito.times(1)).updateTaggedResourceLimitsAndCountsForAccounts(Mockito.any(), Mockito.any());
}
@Test
public void testUpdateDomainResponseWithAccountsMinDetails() {
ListAccountsCmd cmd = new ListAccountsCmd();
ReflectionTestUtils.setField(cmd, "viewDetails", List.of(ApiConstants.DomainDetails.min.toString()));
cmd._resourceLimitService = resourceLimitService;
ReflectionTestUtils.setField(cmd, "tag", "abc");
cmd.updateAccountResponse(List.of(Mockito.mock(AccountResponse.class)));
Mockito.verify(resourceLimitService, Mockito.never()).updateTaggedResourceLimitsAndCountsForAccounts(Mockito.any(), Mockito.any());
}
}

91
api/src/test/java/org/apache/cloudstack/api/command/user/firewall/CreateFirewallRuleCmdTest.java Normal file
View File

@ -0,0 +1,91 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.api.command.user.firewall;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import org.apache.commons.collections.CollectionUtils;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.junit.MockitoJUnitRunner;
import org.springframework.test.util.ReflectionTestUtils;
import com.cloud.utils.net.NetUtils;
@RunWith(MockitoJUnitRunner.class)
public class CreateFirewallRuleCmdTest {
private void validateAllIp4Cidr(final CreateFirewallRuleCmd cmd) {
Assert.assertTrue(CollectionUtils.isNotEmpty(cmd.getSourceCidrList()));
Assert.assertEquals(1, cmd.getSourceCidrList().size());
Assert.assertEquals(NetUtils.ALL_IP4_CIDRS, cmd.getSourceCidrList().get(0));
}
@Test
public void testGetSourceCidrList_Null() {
final CreateFirewallRuleCmd cmd = new CreateFirewallRuleCmd();
ReflectionTestUtils.setField(cmd, "cidrlist", null);
validateAllIp4Cidr(cmd);
}
@Test
public void testGetSourceCidrList_Empty() {
final CreateFirewallRuleCmd cmd = new CreateFirewallRuleCmd();
ReflectionTestUtils.setField(cmd, "cidrlist", new ArrayList<>());
validateAllIp4Cidr(cmd);
}
@Test
public void testGetSourceCidrList_NullFirstElement() {
final CreateFirewallRuleCmd cmd = new CreateFirewallRuleCmd();
List<String> list = new ArrayList<>();
list.add(null);
ReflectionTestUtils.setField(cmd, "cidrlist", list);
validateAllIp4Cidr(cmd);
}
@Test
public void testGetSourceCidrList_EmptyFirstElement() {
final CreateFirewallRuleCmd cmd = new CreateFirewallRuleCmd();
ReflectionTestUtils.setField(cmd, "cidrlist", Collections.singletonList(" "));
validateAllIp4Cidr(cmd);
}
@Test
public void testGetSourceCidrList_Valid() {
final CreateFirewallRuleCmd cmd = new CreateFirewallRuleCmd();
String cidr = "10.1.1.1/22";
ReflectionTestUtils.setField(cmd, "cidrlist", Collections.singletonList(cidr));
Assert.assertTrue(CollectionUtils.isNotEmpty(cmd.getSourceCidrList()));
Assert.assertEquals(1, cmd.getSourceCidrList().size());
Assert.assertEquals(cidr, cmd.getSourceCidrList().get(0));
}
@Test
public void testGetSourceCidrList_EmptyFirstElementButMore() {
final CreateFirewallRuleCmd cmd = new CreateFirewallRuleCmd();
String cidr = "10.1.1.1/22";
ReflectionTestUtils.setField(cmd, "cidrlist", Arrays.asList(" ", cidr));
Assert.assertTrue(CollectionUtils.isNotEmpty(cmd.getSourceCidrList()));
Assert.assertEquals(2, cmd.getSourceCidrList().size());
Assert.assertEquals(cidr, cmd.getSourceCidrList().get(1));
}
}

8
core/src/main/java/com/cloud/resource/ServerResource.java
View File

@ -82,4 +82,12 @@ public interface ServerResource extends Manager {
void setAgentControl(IAgentControl agentControl);
default boolean isExitOnFailures() {
return true;
}
default boolean isAppendAgentNameToLogs() {
return false;
}
}

22
engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java
View File

@ -22,7 +22,6 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import com.cloud.exception.ResourceAllocationException;
import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.framework.config.ConfigKey;
@ -38,6 +37,7 @@ import com.cloud.exception.ConcurrentOperationException;
import com.cloud.exception.InsufficientCapacityException;
import com.cloud.exception.InsufficientServerCapacityException;
import com.cloud.exception.OperationTimedoutException;
import com.cloud.exception.ResourceAllocationException;
import com.cloud.exception.ResourceUnavailableException;
import com.cloud.host.Host;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
@ -101,6 +101,10 @@ public interface VirtualMachineManager extends Manager {
"refer documentation",
true, ConfigKey.Scope.Zone);
ConfigKey<Boolean> VmSyncPowerStateTransitioning = new ConfigKey<>("Advanced", Boolean.class, "vm.sync.power.state.transitioning", "true",
"Whether to sync power states of the transitioning and stalled VMs while processing VM power reports.", false);
interface Topics {
String VM_POWER_STATE = "vm.powerstate";
}
@ -286,24 +290,22 @@ public interface VirtualMachineManager extends Manager {
/**
* Obtains statistics for a list of VMs; CPU and network utilization
* @param hostId ID of the host
* @param hostName name of the host
* @param host host
* @param vmIds list of VM IDs
* @return map of VM ID and stats entry for the VM
*/
HashMap<Long, ? extends VmStats> getVirtualMachineStatistics(long hostId, String hostName, List<Long> vmIds);
HashMap<Long, ? extends VmStats> getVirtualMachineStatistics(Host host, List<Long> vmIds);
/**
* Obtains statistics for a list of VMs; CPU and network utilization
* @param hostId ID of the host
* @param hostName name of the host
* @param vmMap map of VM IDs and the corresponding VirtualMachine object
* @param host host
* @param vmMap map of VM instanceName and its ID
* @return map of VM ID and stats entry for the VM
*/
HashMap<Long, ? extends VmStats> getVirtualMachineStatistics(long hostId, String hostName, Map<Long, ? extends VirtualMachine> vmMap);
HashMap<Long, ? extends VmStats> getVirtualMachineStatistics(Host host, Map<String, Long> vmMap);
HashMap<Long, List<? extends VmDiskStats>> getVmDiskStatistics(long hostId, String hostName, Map<Long, ? extends VirtualMachine> vmMap);
HashMap<Long, List<? extends VmDiskStats>> getVmDiskStatistics(Host host, Map<String, Long> vmInstanceNameIdMap);
HashMap<Long, List<? extends VmNetworkStats>> getVmNetworkStatistics(long hostId, String hostName, Map<Long, ? extends VirtualMachine> vmMap);
HashMap<Long, List<? extends VmNetworkStats>> getVmNetworkStatistics(Host host, Map<String, Long> vmInstanceNameIdMap);
Map<Long, Boolean> getDiskOfferingSuitabilityForVm(long vmId, List<Long> diskOfferingIds);

3
engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java
View File

@ -82,6 +82,9 @@ public interface NetworkOrchestrationService {
ConfigKey<Integer> NetworkLockTimeout = new ConfigKey<Integer>(Integer.class, NetworkLockTimeoutCK, "Network", "600",
"Lock wait timeout (seconds) while implementing network", true, Scope.Global, null);
ConfigKey<String> DeniedRoutes = new ConfigKey<String>(String.class, "denied.routes", "Network", "",
"Routes that are denied, can not be used for Static Routes creation for the VPC Private Gateway", true, ConfigKey.Scope.Zone, null);
ConfigKey<String> GuestDomainSuffix = new ConfigKey<String>(String.class, GuestDomainSuffixCK, "Network", "cloud.internal",
"Default domain name for vms inside virtualized networks fronted by router", true, ConfigKey.Scope.Zone, null);

13
engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java
View File

@ -16,14 +16,11 @@
// under the License.
package com.cloud.capacity;
import java.util.Map;
import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
import com.cloud.host.Host;
import com.cloud.offering.ServiceOffering;
import com.cloud.service.ServiceOfferingVO;
import com.cloud.storage.VMTemplateVO;
import com.cloud.utils.Pair;
import com.cloud.vm.VirtualMachine;
@ -130,6 +127,10 @@ public interface CapacityManager {
true,
ConfigKey.Scope.Zone);
ConfigKey<Integer> CapacityCalculateWorkers = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, Integer.class,
"capacity.calculate.workers", "1",
"Number of worker threads to be used for capacities calculation", true);
public boolean releaseVmCapacity(VirtualMachine vm, boolean moveFromReserved, boolean moveToReservered, Long hostId);
void allocateVmCapacity(VirtualMachine vm, boolean fromLastHost);
@ -145,8 +146,6 @@ public interface CapacityManager {
void updateCapacityForHost(Host host);
void updateCapacityForHost(Host host, Map<Long, ServiceOfferingVO> offeringsMap);
/**
* @param pool storage pool
* @param templateForVmCreation template that will be used for vm creation
@ -163,12 +162,12 @@ public interface CapacityManager {
/**
* Check if specified host has capability to support cpu cores and speed freq
* @param hostId the host to be checked
* @param host the host to be checked
* @param cpuNum cpu number to check
* @param cpuSpeed cpu Speed to check
* @return true if the count of host's running VMs >= hypervisor limit
*/
boolean checkIfHostHasCpuCapability(long hostId, Integer cpuNum, Integer cpuSpeed);
boolean checkIfHostHasCpuCapability(Host host, Integer cpuNum, Integer cpuSpeed);
/**
* Check if cluster will cross threshold if the cpu/memory requested are accommodated

4
engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java
View File

@ -140,13 +140,13 @@ public interface ResourceManager extends ResourceService, Configurable {
public List<HostVO> listAllHostsInOneZoneNotInClusterByHypervisors(List<HypervisorType> types, long dcId, long clusterId);
public List<HypervisorType> listAvailHypervisorInZone(Long hostId, Long zoneId);
public List<HypervisorType> listAvailHypervisorInZone(Long zoneId);
public HostVO findHostByGuid(String guid);
public HostVO findHostByName(String name);
HostStats getHostStatistics(long hostId);
HostStats getHostStatistics(Host host);
Long getGuestOSCategoryId(long hostId);

9
engine/components-api/src/main/java/com/cloud/storage/StorageManager.java
View File

@ -22,6 +22,7 @@ import java.util.Map;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.HypervisorHostListener;
import org.apache.cloudstack.engine.subsystem.api.storage.Scope;
import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
@ -42,6 +43,7 @@ import com.cloud.offering.DiskOffering;
import com.cloud.offering.ServiceOffering;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.utils.Pair;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.vm.DiskProfile;
import com.cloud.vm.VMInstanceVO;
@ -214,6 +216,10 @@ public interface StorageManager extends StorageService {
"when resize a volume upto resize capacity disable threshold (pool.storage.allocated.resize.capacity.disablethreshold)",
true, ConfigKey.Scope.Zone);
ConfigKey<Integer> StoragePoolHostConnectWorkers = new ConfigKey<>("Storage", Integer.class,
"storage.pool.host.connect.workers", "1",
"Number of worker threads to be used to connect hosts to a primary storage", true);
/**
* should we execute in sequence not involving any storages?
* @return tru if commands should execute in sequence
@ -365,6 +371,9 @@ public interface StorageManager extends StorageService {
String getStoragePoolMountFailureReason(String error);
void connectHostsToPool(DataStore primaryStore, List<Long> hostIds, Scope scope,
boolean handleStorageConflictException, boolean errorOnNoUpHost) throws CloudRuntimeException;
boolean connectHostToSharedPool(Host host, long poolId) throws StorageUnavailableException, StorageConflictException;
void disconnectHostFromSharedPool(Host host, StoragePool pool) throws StorageUnavailableException, StorageConflictException;

302
engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java
View File

@ -19,6 +19,7 @@ package com.cloud.agent.manager;
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.net.SocketAddress;
import java.nio.channels.ClosedChannelException;
import java.util.ArrayList;
import java.util.Arrays;
@ -26,25 +27,20 @@ import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
import com.cloud.cluster.ManagementServerHostVO;
import com.cloud.cluster.dao.ManagementServerHostDao;
import com.cloud.configuration.Config;
import com.cloud.org.Cluster;
import com.cloud.utils.NumbersUtil;
import com.cloud.utils.db.GlobalLock;
import org.apache.cloudstack.agent.lb.IndirectAgentLB;
import org.apache.cloudstack.ca.CAManager;
import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
@ -62,6 +58,8 @@ import org.apache.cloudstack.utils.identity.ManagementServerNode;
import org.apache.commons.collections.MapUtils;
import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.ThreadContext;
import com.cloud.agent.AgentManager;
import com.cloud.agent.Listener;
@ -88,6 +86,9 @@ import com.cloud.agent.api.UnsupportedAnswer;
import com.cloud.agent.transport.Request;
import com.cloud.agent.transport.Response;
import com.cloud.alert.AlertManager;
import com.cloud.cluster.ManagementServerHostVO;
import com.cloud.cluster.dao.ManagementServerHostDao;
import com.cloud.configuration.Config;
import com.cloud.configuration.ManagementServiceConfiguration;
import com.cloud.dc.ClusterVO;
import com.cloud.dc.DataCenterVO;
@ -107,15 +108,18 @@ import com.cloud.host.Status.Event;
import com.cloud.host.dao.HostDao;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.hypervisor.HypervisorGuruManager;
import com.cloud.org.Cluster;
import com.cloud.resource.Discoverer;
import com.cloud.resource.ResourceManager;
import com.cloud.resource.ResourceState;
import com.cloud.resource.ServerResource;
import com.cloud.utils.NumbersUtil;
import com.cloud.utils.Pair;
import com.cloud.utils.component.ManagerBase;
import com.cloud.utils.concurrency.NamedThreadFactory;
import com.cloud.utils.db.DB;
import com.cloud.utils.db.EntityManager;
import com.cloud.utils.db.GlobalLock;
import com.cloud.utils.db.QueryBuilder;
import com.cloud.utils.db.SearchCriteria.Op;
import com.cloud.utils.db.TransactionLegacy;
@ -130,8 +134,6 @@ import com.cloud.utils.nio.Link;
import com.cloud.utils.nio.NioServer;
import com.cloud.utils.nio.Task;
import com.cloud.utils.time.InaccurateClock;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.ThreadContext;
/**
* Implementation of the Agent Manager. This class controls the connection to the agents.
@ -142,14 +144,13 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
* _agents is a ConcurrentHashMap, but it is used from within a synchronized block. This will be reported by findbugs as JLM_JSR166_UTILCONCURRENT_MONITORENTER. Maybe a
* ConcurrentHashMap is not the right thing to use here, but i'm not sure so i leave it alone.
*/
protected ConcurrentHashMap<Long, AgentAttache> _agents = new ConcurrentHashMap<Long, AgentAttache>(10007);
protected List<Pair<Integer, Listener>> _hostMonitors = new ArrayList<Pair<Integer, Listener>>(17);
protected List<Pair<Integer, Listener>> _cmdMonitors = new ArrayList<Pair<Integer, Listener>>(17);
protected List<Pair<Integer, StartupCommandProcessor>> _creationMonitors = new ArrayList<Pair<Integer, StartupCommandProcessor>>(17);
protected List<Long> _loadingAgents = new ArrayList<Long>();
protected ConcurrentHashMap<Long, AgentAttache> _agents = new ConcurrentHashMap<>(10007);
protected List<Pair<Integer, Listener>> _hostMonitors = new ArrayList<>(17);
protected List<Pair<Integer, Listener>> _cmdMonitors = new ArrayList<>(17);
protected List<Pair<Integer, StartupCommandProcessor>> _creationMonitors = new ArrayList<>(17);
protected List<Long> _loadingAgents = new ArrayList<>();
protected Map<String, Integer> _commandTimeouts = new HashMap<>();
private int _monitorId = 0;
private final Lock _agentStatusLock = new ReentrantLock();
@Inject
protected CAManager caService;
@ -201,25 +202,36 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
private List<String> lastAgents = null;
protected StateMachine2<Status, Status.Event, Host> _statusStateMachine = Status.getStateMachine();
private final ConcurrentHashMap<Long, Long> _pingMap = new ConcurrentHashMap<Long, Long>(10007);
private final ConcurrentHashMap<Long, Long> _pingMap = new ConcurrentHashMap<>(10007);
private int maxConcurrentNewAgentConnections;
private final ConcurrentHashMap<String, Long> newAgentConnections = new ConcurrentHashMap<>();
protected ScheduledExecutorService newAgentConnectionsMonitor;
@Inject
ResourceManager _resourceMgr;
@Inject
ManagementServiceConfiguration mgmtServiceConf;
protected final ConfigKey<Integer> Workers = new ConfigKey<Integer>("Advanced", Integer.class, "workers", "5",
protected final ConfigKey<Integer> Workers = new ConfigKey<>("Advanced", Integer.class, "workers", "5",
"Number of worker threads handling remote agent connections.", false);
protected final ConfigKey<Integer> Port = new ConfigKey<Integer>("Advanced", Integer.class, "port", "8250", "Port to listen on for remote agent connections.", false);
protected final ConfigKey<Integer> AlertWait = new ConfigKey<Integer>("Advanced", Integer.class, "alert.wait", "1800",
protected final ConfigKey<Integer> Port = new ConfigKey<>("Advanced", Integer.class, "port", "8250", "Port to listen on for remote agent connections.", false);
protected final ConfigKey<Integer> RemoteAgentSslHandshakeTimeout = new ConfigKey<>("Advanced",
Integer.class, "agent.ssl.handshake.timeout", "30",
"Seconds after which SSL handshake times out during remote agent connections.", false);
protected final ConfigKey<Integer> RemoteAgentMaxConcurrentNewConnections = new ConfigKey<>("Advanced",
Integer.class, "agent.max.concurrent.new.connections", "0",
"Number of maximum concurrent new connections server allows for remote agents. " +
"If set to zero (default value) then no limit will be enforced on concurrent new connections",
false);
protected final ConfigKey<Integer> AlertWait = new ConfigKey<>("Advanced", Integer.class, "alert.wait", "1800",
"Seconds to wait before alerting on a disconnected agent", true);
protected final ConfigKey<Integer> DirectAgentLoadSize = new ConfigKey<Integer>("Advanced", Integer.class, "direct.agent.load.size", "16",
protected final ConfigKey<Integer> DirectAgentLoadSize = new ConfigKey<>("Advanced", Integer.class, "direct.agent.load.size", "16",
"The number of direct agents to load each time", false);
protected final ConfigKey<Integer> DirectAgentPoolSize = new ConfigKey<Integer>("Advanced", Integer.class, "direct.agent.pool.size", "500",
protected final ConfigKey<Integer> DirectAgentPoolSize = new ConfigKey<>("Advanced", Integer.class, "direct.agent.pool.size", "500",
"Default size for DirectAgentPool", false);
protected final ConfigKey<Float> DirectAgentThreadCap = new ConfigKey<Float>("Advanced", Float.class, "direct.agent.thread.cap", "1",
protected final ConfigKey<Float> DirectAgentThreadCap = new ConfigKey<>("Advanced", Float.class, "direct.agent.thread.cap", "1",
"Percentage (as a value between 0 and 1) of direct.agent.pool.size to be used as upper thread cap for a single direct agent to process requests", false);
protected final ConfigKey<Boolean> CheckTxnBeforeSending = new ConfigKey<Boolean>("Developer", Boolean.class, "check.txn.before.sending.agent.commands", "false",
protected final ConfigKey<Boolean> CheckTxnBeforeSending = new ConfigKey<>("Developer", Boolean.class, "check.txn.before.sending.agent.commands", "false",
"This parameter allows developers to enable a check to see if a transaction wraps commands that are sent to the resource. This is not to be enabled on production systems.", true);
@Override
@ -227,8 +239,6 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
logger.info("Ping Timeout is {}.", mgmtServiceConf.getPingTimeout());
final int threads = DirectAgentLoadSize.value();
_nodeId = ManagementServerNode.getManagementServerId();
logger.info("Configuring AgentManagerImpl. management server node id(msid): {}.", _nodeId);
@ -241,24 +251,32 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
managementServerMaintenanceManager.registerListener(this);
_executor = new ThreadPoolExecutor(threads, threads, 60l, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(), new NamedThreadFactory("AgentTaskPool"));
final int agentTaskThreads = DirectAgentLoadSize.value();
_connectExecutor = new ThreadPoolExecutor(100, 500, 60l, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(), new NamedThreadFactory("AgentConnectTaskPool"));
_executor = new ThreadPoolExecutor(agentTaskThreads, agentTaskThreads, 60L, TimeUnit.SECONDS, new LinkedBlockingQueue<>(), new NamedThreadFactory("AgentTaskPool"));
_connectExecutor = new ThreadPoolExecutor(100, 500, 60L, TimeUnit.SECONDS, new LinkedBlockingQueue<>(), new NamedThreadFactory("AgentConnectTaskPool"));
// allow core threads to time out even when there are no items in the queue
_connectExecutor.allowCoreThreadTimeOut(true);
_connection = new NioServer("AgentManager", Port.value(), Workers.value() + 10, this, caService);
maxConcurrentNewAgentConnections = RemoteAgentMaxConcurrentNewConnections.value();
_connection = new NioServer("AgentManager", Port.value(), Workers.value() + 10,
this, caService, RemoteAgentSslHandshakeTimeout.value());
logger.info("Listening on {} with {} workers.", Port.value(), Workers.value());
final int directAgentPoolSize = DirectAgentPoolSize.value();
// executes all agent commands other than cron and ping
_directAgentExecutor = new ScheduledThreadPoolExecutor(DirectAgentPoolSize.value(), new NamedThreadFactory("DirectAgent"));
_directAgentExecutor = new ScheduledThreadPoolExecutor(directAgentPoolSize, new NamedThreadFactory("DirectAgent"));
// executes cron and ping agent commands
_cronJobExecutor = new ScheduledThreadPoolExecutor(DirectAgentPoolSize.value(), new NamedThreadFactory("DirectAgentCronJob"));
logger.debug("Created DirectAgentAttache pool with size: {}.", DirectAgentPoolSize.value());
_directAgentThreadCap = Math.round(DirectAgentPoolSize.value() * DirectAgentThreadCap.value()) + 1; // add 1 to always make the value > 0
_cronJobExecutor = new ScheduledThreadPoolExecutor(directAgentPoolSize, new NamedThreadFactory("DirectAgentCronJob"));
logger.debug("Created DirectAgentAttache pool with size: {}.", directAgentPoolSize);
_directAgentThreadCap = Math.round(directAgentPoolSize * DirectAgentThreadCap.value()) + 1; // add 1 to always make the value > 0
_monitorExecutor = new ScheduledThreadPoolExecutor(1, new NamedThreadFactory("AgentMonitor"));
newAgentConnectionsMonitor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("NewAgentConnectionsMonitor"));
initializeCommandTimeouts();
return true;
@ -269,22 +287,44 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
return new AgentHandler(type, link, data);
}
@Override
public int getMaxConcurrentNewConnectionsCount() {
return maxConcurrentNewAgentConnections;
}
@Override
public int getNewConnectionsCount() {
return newAgentConnections.size();
}
@Override
public void registerNewConnection(SocketAddress address) {
logger.trace("Adding new agent connection from {}", address.toString());
newAgentConnections.putIfAbsent(address.toString(), System.currentTimeMillis());
}
@Override
public void unregisterNewConnection(SocketAddress address) {
logger.trace("Removing new agent connection for {}", address.toString());
newAgentConnections.remove(address.toString());
}
@Override
public int registerForHostEvents(final Listener listener, final boolean connections, final boolean commands, final boolean priority) {
synchronized (_hostMonitors) {
_monitorId++;
if (connections) {
if (priority) {
_hostMonitors.add(0, new Pair<Integer, Listener>(_monitorId, listener));
_hostMonitors.add(0, new Pair<>(_monitorId, listener));
} else {
_hostMonitors.add(new Pair<Integer, Listener>(_monitorId, listener));
_hostMonitors.add(new Pair<>(_monitorId, listener));
}
}
if (commands) {
if (priority) {
_cmdMonitors.add(0, new Pair<Integer, Listener>(_monitorId, listener));
_cmdMonitors.add(0, new Pair<>(_monitorId, listener));
} else {
_cmdMonitors.add(new Pair<Integer, Listener>(_monitorId, listener));
_cmdMonitors.add(new Pair<>(_monitorId, listener));
}
}
logger.debug("Registering listener {} with id {}", listener.getClass().getSimpleName(), _monitorId);
@ -297,9 +337,9 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
synchronized (_hostMonitors) {
_monitorId++;
if (priority) {
_creationMonitors.add(0, new Pair<Integer, StartupCommandProcessor>(_monitorId, creator));
_creationMonitors.add(0, new Pair<>(_monitorId, creator));
} else {
_creationMonitors.add(new Pair<Integer, StartupCommandProcessor>(_monitorId, creator));
_creationMonitors.add(new Pair<>(_monitorId, creator));
}
return _monitorId;
}
@ -331,7 +371,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
public void onManagementServerCancelMaintenance() {
logger.debug("Management server maintenance disabled");
if (_connectExecutor.isShutdown()) {
_connectExecutor = new ThreadPoolExecutor(100, 500, 60l, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(), new NamedThreadFactory("AgentConnectTaskPool"));
_connectExecutor = new ThreadPoolExecutor(100, 500, 60L, TimeUnit.SECONDS, new LinkedBlockingQueue<>(), new NamedThreadFactory("AgentConnectTaskPool"));
_connectExecutor.allowCoreThreadTimeOut(true);
}
@ -351,7 +391,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
}
private AgentControlAnswer handleControlCommand(final AgentAttache attache, final AgentControlCommand cmd) {
AgentControlAnswer answer = null;
AgentControlAnswer answer;
for (final Pair<Integer, Listener> listener : _cmdMonitors) {
answer = listener.second().processControlCommand(attache.getId(), cmd);
@ -379,7 +419,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
}
public AgentAttache findAttache(final long hostId) {
AgentAttache attache = null;
AgentAttache attache;
synchronized (_agents) {
attache = _agents.get(hostId);
}
@ -431,12 +471,10 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
cmds.addCommand(cmd);
send(hostId, cmds, cmd.getWait());
final Answer[] answers = cmds.getAnswers();
if (answers != null && !(answers[0] instanceof UnsupportedAnswer)) {
return answers[0];
}
if (answers != null && answers[0] instanceof UnsupportedAnswer) {
logger.warn("Unsupported Command: {}", answers[0].getDetails());
if (answers != null) {
if (answers[0] instanceof UnsupportedAnswer) {
logger.warn("Unsupported Command: {}", answers[0].getDetails());
}
return answers[0];
}
@ -467,8 +505,8 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
}
/**
* @param commands
* @return
* @param commands object container of commands
* @return array of commands
*/
private Command[] checkForCommandsAndTag(final Commands commands) {
final Command[] cmds = commands.toCommands();
@ -484,8 +522,8 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
}
/**
* @param commands
* @param cmds
* @param commands object container of commands
* @param cmds array of commands
*/
private void setEmptyAnswers(final Commands commands, final Command[] cmds) {
if (cmds.length == 0) {
@ -524,7 +562,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
String commandWaits = GranularWaitTimeForCommands.value().trim();
if (StringUtils.isNotEmpty(commandWaits)) {
_commandTimeouts = getCommandTimeoutsMap(commandWaits);
logger.info(String.format("Timeouts for management server internal commands successfully initialized from global setting commands.timeout: %s", _commandTimeouts));
logger.info("Timeouts for management server internal commands successfully initialized from global setting commands.timeout: {}", _commandTimeouts);
}
}
@ -540,10 +578,10 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
int commandTimeout = Integer.parseInt(parts[1].trim());
commandTimeouts.put(commandName, commandTimeout);
} catch (NumberFormatException e) {
logger.error(String.format("Initialising the timeouts using commands.timeout: %s for management server internal commands failed with error %s", commandPair, e.getMessage()));
logger.error("Initialising the timeouts using commands.timeout: {} for management server internal commands failed with error {}", commandPair, e.getMessage());
}
} else {
logger.error(String.format("Error initialising the timeouts for management server internal commands. Invalid format in commands.timeout: %s", commandPair));
logger.error("Error initialising the timeouts for management server internal commands. Invalid format in commands.timeout: {}", commandPair);
}
}
return commandTimeouts;
@ -557,7 +595,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
}
int wait = getTimeout(commands, timeout);
logger.debug(String.format("Wait time setting on %s is %d seconds", commands, wait));
logger.debug("Wait time setting on {} is {} seconds", commands, wait);
for (Command cmd : commands) {
String simpleCommandName = cmd.getClass().getSimpleName();
Integer commandTimeout = _commandTimeouts.get(simpleCommandName);
@ -644,7 +682,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
}
final long hostId = attache.getId();
logger.debug("Remove Agent : {}", attache);
AgentAttache removed = null;
AgentAttache removed;
boolean conflict = false;
synchronized (_agents) {
removed = _agents.remove(hostId);
@ -697,16 +735,15 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
}
} catch (final HypervisorVersionChangedException hvce) {
handleDisconnectWithoutInvestigation(attache, Event.ShutdownRequested, true, true);
throw new CloudRuntimeException("Unable to connect " + (attache == null ? "<unknown agent>" : attache.getId()), hvce);
throw new CloudRuntimeException("Unable to connect " + attache.getId(), hvce);
} catch (final Exception e) {
logger.error("Monitor {} says there is an error in the connect process for {} due to {}", monitor.second().getClass().getSimpleName(), hostId, e.getMessage(), e);
handleDisconnectWithoutInvestigation(attache, Event.AgentDisconnected, true, true);
throw new CloudRuntimeException("Unable to connect " + (attache == null ? "<unknown agent>" : attache.getId()), e);
throw new CloudRuntimeException("Unable to connect " + attache.getId(), e);
}
}
}
final Long dcId = host.getDataCenterId();
final ReadyCommand ready = new ReadyCommand(host, NumbersUtil.enableHumanReadableSizes);
ready.setWait(ReadyCommandWait.value());
final Answer answer = easySend(hostId, ready);
@ -757,6 +794,10 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
_monitorExecutor.scheduleWithFixedDelay(new MonitorTask(), mgmtServiceConf.getPingInterval(), mgmtServiceConf.getPingInterval(), TimeUnit.SECONDS);
final int cleanupTime = Wait.value();
newAgentConnectionsMonitor.scheduleAtFixedRate(new AgentNewConnectionsMonitorTask(), cleanupTime,
cleanupTime, TimeUnit.MINUTES);
return true;
}
@ -775,25 +816,25 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
final Constructor<?> constructor = clazz.getConstructor();
resource = (ServerResource)constructor.newInstance();
} catch (final ClassNotFoundException e) {
logger.warn("Unable to find class " + host.getResource(), e);
logger.warn("Unable to find class {}", host.getResource(), e);
} catch (final InstantiationException e) {
logger.warn("Unable to instantiate class " + host.getResource(), e);
logger.warn("Unable to instantiate class {}", host.getResource(), e);
} catch (final IllegalAccessException e) {
logger.warn("Illegal access " + host.getResource(), e);
logger.warn("Illegal access {}", host.getResource(), e);
} catch (final SecurityException e) {
logger.warn("Security error on " + host.getResource(), e);
logger.warn("Security error on {}", host.getResource(), e);
} catch (final NoSuchMethodException e) {
logger.warn("NoSuchMethodException error on " + host.getResource(), e);
logger.warn("NoSuchMethodException error on {}", host.getResource(), e);
} catch (final IllegalArgumentException e) {
logger.warn("IllegalArgumentException error on " + host.getResource(), e);
logger.warn("IllegalArgumentException error on {}", host.getResource(), e);
} catch (final InvocationTargetException e) {
logger.warn("InvocationTargetException error on " + host.getResource(), e);
logger.warn("InvocationTargetException error on {}", host.getResource(), e);
}
if (resource != null) {
_hostDao.loadDetails(host);
final HashMap<String, Object> params = new HashMap<String, Object>(host.getDetails().size() + 5);
final HashMap<String, Object> params = new HashMap<>(host.getDetails().size() + 5);
params.putAll(host.getDetails());
params.put("guid", host.getGuid());
@ -803,7 +844,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
}
if (host.getClusterId() != null) {
params.put("cluster", Long.toString(host.getClusterId()));
String guid = null;
String guid;
final ClusterVO cluster = _clusterDao.findById(host.getClusterId());
if (cluster.getGuid() == null) {
guid = host.getDetail("pool");
@ -843,7 +884,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
protected boolean loadDirectlyConnectedHost(final HostVO host, final boolean forRebalance, final boolean isTransferredConnection) {
boolean initialized = false;
ServerResource resource = null;
ServerResource resource;
try {
// load the respective discoverer
final Discoverer discoverer = _resourceMgr.getMatchingDiscover(host.getHypervisorType());
@ -873,18 +914,18 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
final Host h = _resourceMgr.createHostAndAgent(host.getId(), resource, host.getDetails(), false, null, true, isTransferredConnection);
tapLoadingAgents(host.getId(), TapAgentsAction.Del);
return h == null ? false : true;
return h != null;
} else {
_executor.execute(new SimulateStartTask(host.getId(), host.getUuid(), host.getName(), resource, host.getDetails()));
return true;
}
}
protected AgentAttache createAttacheForDirectConnect(final Host host, final ServerResource resource) throws ConnectionException {
protected AgentAttache createAttacheForDirectConnect(final Host host, final ServerResource resource) {
logger.debug("create DirectAgentAttache for {}", host);
final DirectAgentAttache attache = new DirectAgentAttache(this, host.getId(), host.getUuid(), host.getName(), resource, host.isInMaintenanceStates());
AgentAttache old = null;
AgentAttache old;
synchronized (_agents) {
old = _agents.put(host.getId(), attache);
}
@ -918,6 +959,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
_connectExecutor.shutdownNow();
_monitorExecutor.shutdownNow();
newAgentConnectionsMonitor.shutdownNow();
return true;
}
@ -949,7 +991,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
try {
logger.info("Host {} is disconnecting with event {}",
attache, event);
Status nextStatus = null;
Status nextStatus;
final HostVO host = _hostDao.findById(hostId);
if (host == null) {
logger.warn("Can't find host with {} ({})", hostId, attache);
@ -1082,7 +1124,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
@Override
protected void runInContext() {
try {
if (_investigate == true) {
if (_investigate) {
handleDisconnectWithInvestigation(_attache, _event);
} else {
handleDisconnectWithoutInvestigation(_attache, _event, true, false);
@ -1134,8 +1176,8 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
public Answer[] send(final Long hostId, final Commands cmds) throws AgentUnavailableException, OperationTimedoutException {
int wait = 0;
if (cmds.size() > 1) {
logger.debug(String.format("Checking the wait time in seconds to be used for the following commands : %s. If there are multiple commands sent at once," +
"then max wait time of those will be used", cmds));
logger.debug("Checking the wait time in seconds to be used for the following commands : {}. If there are multiple commands sent at once," +
"then max wait time of those will be used", cmds);
}
for (final Command cmd : cmds) {
@ -1198,7 +1240,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
public boolean executeUserRequest(final long hostId, final Event event) throws AgentUnavailableException {
if (event == Event.AgentDisconnected) {
AgentAttache attache = null;
AgentAttache attache;
attache = findAttache(hostId);
logger.debug("Received agent disconnect event for host {} ({})", hostId, attache);
if (attache != null) {
@ -1224,12 +1266,12 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
return agentAttache != null;
}
protected AgentAttache createAttacheForConnect(final HostVO host, final Link link) throws ConnectionException {
protected AgentAttache createAttacheForConnect(final HostVO host, final Link link) {
logger.debug("create ConnectedAgentAttache for {}", host);
final AgentAttache attache = new ConnectedAgentAttache(this, host.getId(), host.getUuid(), host.getName(), link, host.isInMaintenanceStates());
link.attach(attache);
AgentAttache old = null;
AgentAttache old;
synchronized (_agents) {
old = _agents.put(host.getId(), attache);
}
@ -1254,7 +1296,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
}
}
ready.setArch(host.getArch().getType());
AgentAttache attache = null;
AgentAttache attache;
GlobalLock joinLock = getHostJoinLock(host.getId());
if (joinLock.lock(60)) {
try {
@ -1280,7 +1322,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
return attache;
}
private AgentAttache handleConnectedAgent(final Link link, final StartupCommand[] startup, final Request request) {
private AgentAttache handleConnectedAgent(final Link link, final StartupCommand[] startup) {
AgentAttache attache = null;
ReadyCommand ready = null;
try {
@ -1308,7 +1350,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
easySend(attache.getId(), ready);
}
} catch (final Exception e) {
logger.debug("Failed to send ready command:" + e.toString());
logger.debug("Failed to send ready command:", e);
}
return attache;
}
@ -1334,6 +1376,8 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
this.id = id;
this.resource = resource;
this.details = details;
this.uuid = uuid;
this.name = name;
}
@Override
@ -1382,10 +1426,11 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
startups[i] = (StartupCommand)_cmds[i];
}
final AgentAttache attache = handleConnectedAgent(_link, startups, _request);
final AgentAttache attache = handleConnectedAgent(_link, startups);
if (attache == null) {
logger.warn("Unable to create attache for agent: {}", _request);
}
unregisterNewConnection(_link.getSocketAddress());
}
}
@ -1402,7 +1447,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
break;
}
}
Response response = null;
Response response;
response = new Response(request, answers[0], _nodeId, -1);
try {
link.send(response.toBytes());
@ -1483,7 +1528,6 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
}
final long hostId = attache.getId();
final String hostName = attache.getName();
if (logger.isDebugEnabled()) {
if (cmd instanceof PingRoutingCommand) {
@ -1502,7 +1546,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
final Answer[] answers = new Answer[cmds.length];
for (int i = 0; i < cmds.length; i++) {
cmd = cmds[i];
Answer answer = null;
Answer answer;
try {
if (cmd instanceof StartupRoutingCommand) {
final StartupRoutingCommand startup = (StartupRoutingCommand) cmd;
@ -1536,7 +1580,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
final long cmdHostId = ((PingCommand)cmd).getHostId();
boolean requestStartupCommand = false;
final HostVO host = _hostDao.findById(Long.valueOf(cmdHostId));
final HostVO host = _hostDao.findById(cmdHostId);
boolean gatewayAccessible = true;
// if the router is sending a ping, verify the
// gateway was pingable
@ -1586,7 +1630,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
if (logD) {
logger.debug("SeqA {}-: Sending {}", attache.getId(), response.getSequence(), response);
} else {
logger.trace("SeqA {}-: Sending {}" + attache.getId(), response.getSequence(), response);
logger.trace("SeqA {}-: Sending {} {}", response.getSequence(), response, attache.getId());
}
try {
link.send(response.toBytes());
@ -1606,15 +1650,14 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
@Override
protected void doTask(final Task task) throws TaskExecutionException {
final TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.CLOUD_DB);
try {
try (TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.CLOUD_DB)) {
final Type type = task.getType();
if (type == Task.Type.DATA) {
if (type == Type.DATA) {
final byte[] data = task.getData();
try {
final Request event = Request.parse(data);
if (event instanceof Response) {
processResponse(task.getLink(), (Response)event);
processResponse(task.getLink(), (Response) event);
} else {
processRequest(task.getLink(), event);
}
@ -1626,10 +1669,10 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
logger.error(message);
throw new TaskExecutionException(message, e);
}
} else if (type == Task.Type.CONNECT) {
} else if (type == Task.Type.DISCONNECT) {
} else if (type == Type.CONNECT) {
} else if (type == Type.DISCONNECT) {
final Link link = task.getLink();
final AgentAttache attache = (AgentAttache)link.attachment();
final AgentAttache attache = (AgentAttache) link.attachment();
if (attache != null) {
disconnectWithInvestigation(attache, Event.AgentDisconnected);
} else {
@ -1638,8 +1681,6 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
link.terminated();
}
}
} finally {
txn.close();
}
}
}
@ -1668,21 +1709,16 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
@Override
public boolean agentStatusTransitTo(final HostVO host, final Status.Event e, final long msId) {
try {
_agentStatusLock.lock();
logger.debug("[Resource state = {}, Agent event = , Host = {}]",
host.getResourceState(), e.toString(), host);
logger.debug("[Resource state = {}, Agent event = , Host = {}]",
host.getResourceState(), e.toString(), host);
host.setManagementServerId(msId);
try {
return _statusStateMachine.transitTo(host, e, host.getId(), _hostDao);
} catch (final NoTransitionException e1) {
logger.debug("Cannot transit agent status with event {} for host {}, management server id is {}", e, host, msId);
throw new CloudRuntimeException(String.format(
"Cannot transit agent status with event %s for host %s, management server id is %d, %s", e, host, msId, e1.getMessage()));
}
} finally {
_agentStatusLock.unlock();
host.setManagementServerId(msId);
try {
return _statusStateMachine.transitTo(host, e, host.getId(), _hostDao);
} catch (final NoTransitionException e1) {
logger.debug("Cannot transit agent status with event {} for host {}, management server id is {}", e, host, msId);
throw new CloudRuntimeException(String.format(
"Cannot transit agent status with event %s for host %s, management server id is %d, %s", e, host, msId, e1.getMessage()));
}
}
@ -1871,7 +1907,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
}
protected List<Long> findAgentsBehindOnPing() {
final List<Long> agentsBehind = new ArrayList<Long>();
final List<Long> agentsBehind = new ArrayList<>();
final long cutoffTime = InaccurateClock.getTimeInSeconds() - mgmtServiceConf.getTimeout();
for (final Map.Entry<Long, Long> entry : _pingMap.entrySet()) {
if (entry.getValue() < cutoffTime) {
@ -1879,7 +1915,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
}
}
if (agentsBehind.size() > 0) {
if (!agentsBehind.isEmpty()) {
logger.info("Found the following agents behind on ping: {}", agentsBehind);
}
@ -1887,6 +1923,35 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
}
}
protected class AgentNewConnectionsMonitorTask extends ManagedContextRunnable {
@Override
protected void runInContext() {
logger.trace("Agent New Connections Monitor is started.");
final int cleanupTime = Wait.value();
Set<Map.Entry<String, Long>> entrySet = newAgentConnections.entrySet();
long cutOff = System.currentTimeMillis() - (cleanupTime * 60 * 1000L);
if (logger.isDebugEnabled()) {
List<String> expiredConnections = newAgentConnections.entrySet()
.stream()
.filter(e -> e.getValue() <= cutOff)
.map(Map.Entry::getKey)
.collect(Collectors.toList());
logger.debug("Currently {} active new connections, of which {} have expired - {}",
entrySet.size(),
expiredConnections.size(),
StringUtils.join(expiredConnections));
}
for (Map.Entry<String, Long> entry : entrySet) {
if (entry.getValue() <= cutOff) {
if (logger.isTraceEnabled()) {
logger.trace("Cleaning up new agent connection for {}", entry.getKey());
}
newAgentConnections.remove(entry.getKey());
}
}
}
}
protected class BehindOnPingListener implements Listener {
@Override
public boolean isRecurring() {
@ -1962,7 +2027,8 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
@Override
public ConfigKey<?>[] getConfigKeys() {
return new ConfigKey<?>[] { CheckTxnBeforeSending, Workers, Port, Wait, AlertWait, DirectAgentLoadSize,
DirectAgentPoolSize, DirectAgentThreadCap, EnableKVMAutoEnableDisable, ReadyCommandWait, GranularWaitTimeForCommands };
DirectAgentPoolSize, DirectAgentThreadCap, EnableKVMAutoEnableDisable, ReadyCommandWait,
GranularWaitTimeForCommands, RemoteAgentSslHandshakeTimeout, RemoteAgentMaxConcurrentNewConnections };
}
protected class SetHostParamsListener implements Listener {
@ -1997,7 +2063,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
}
if (((StartupRoutingCommand)cmd).getHypervisorType() == HypervisorType.KVM || ((StartupRoutingCommand)cmd).getHypervisorType() == HypervisorType.LXC) {
Map<String, String> params = new HashMap<String, String>();
Map<String, String> params = new HashMap<>();
params.put(Config.RouterAggregationCommandEachTimeout.toString(), _configDao.getValue(Config.RouterAggregationCommandEachTimeout.toString()));
params.put(Config.MigrateWait.toString(), _configDao.getValue(Config.MigrateWait.toString()));
params.put(NetworkOrchestrationService.TUNGSTEN_ENABLED.key(), String.valueOf(NetworkOrchestrationService.TUNGSTEN_ENABLED.valueIn(host.getDataCenterId())));
@ -2042,13 +2108,13 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
if (allHosts == null) {
return null;
}
Map<Long, List<Long>> hostsByZone = new HashMap<Long, List<Long>>();
Map<Long, List<Long>> hostsByZone = new HashMap<>();
for (HostVO host : allHosts) {
if (host.getHypervisorType() == HypervisorType.KVM || host.getHypervisorType() == HypervisorType.LXC) {
Long zoneId = host.getDataCenterId();
List<Long> hostIds = hostsByZone.get(zoneId);
if (hostIds == null) {
hostIds = new ArrayList<Long>();
hostIds = new ArrayList<>();
}
hostIds.add(host.getId());
hostsByZone.put(zoneId, hostIds);

143
engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java
View File

@ -56,6 +56,7 @@ import org.apache.cloudstack.maintenance.command.PrepareForShutdownManagementSer
import org.apache.cloudstack.maintenance.command.TriggerShutdownManagementServerHostCommand;
import org.apache.cloudstack.managed.context.ManagedContextRunnable;
import org.apache.cloudstack.managed.context.ManagedContextTimerTask;
import org.apache.cloudstack.management.ManagementServerHost;
import org.apache.cloudstack.outofbandmanagement.dao.OutOfBandManagementDao;
import org.apache.cloudstack.utils.identity.ManagementServerNode;
import org.apache.cloudstack.utils.security.SSLUtils;
@ -75,9 +76,6 @@ import com.cloud.cluster.ClusterManager;
import com.cloud.cluster.ClusterManagerListener;
import com.cloud.cluster.ClusterServicePdu;
import com.cloud.cluster.ClusteredAgentRebalanceService;
import org.apache.cloudstack.management.ManagementServerHost;
import org.apache.commons.collections.CollectionUtils;
import com.cloud.cluster.ManagementServerHostVO;
import com.cloud.cluster.agentlb.AgentLoadBalancerPlanner;
import com.cloud.cluster.agentlb.HostTransferMapVO;
@ -107,6 +105,8 @@ import com.cloud.utils.nio.Link;
import com.cloud.utils.nio.Task;
import com.google.gson.Gson;
import org.apache.commons.collections.CollectionUtils;
public class ClusteredAgentManagerImpl extends AgentManagerImpl implements ClusterManagerListener, ClusteredAgentRebalanceService {
private static ScheduledExecutorService s_transferExecutor = Executors.newScheduledThreadPool(2, new NamedThreadFactory("Cluster-AgentRebalancingExecutor"));
private final long rebalanceTimeOut = 300000; // 5 mins - after this time remove the agent from the transfer list
@ -114,7 +114,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
public final static long STARTUP_DELAY = 5000;
public final static long SCAN_INTERVAL = 90000; // 90 seconds, it takes 60 sec for xenserver to fail login
public final static int ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_COOPERATION = 5; // 5 seconds
protected Set<Long> _agentToTransferIds = new HashSet<Long>();
protected Set<Long> _agentToTransferIds = new HashSet<>();
Gson _gson;
protected HashMap<String, SocketChannel> _peers;
protected HashMap<String, SSLEngine> _sslEngines;
@ -151,17 +151,17 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
super();
}
protected final ConfigKey<Boolean> EnableLB = new ConfigKey<Boolean>(Boolean.class, "agent.lb.enabled", "Advanced", "false", "Enable agent load balancing between management server nodes", true);
protected final ConfigKey<Double> ConnectedAgentThreshold = new ConfigKey<Double>(Double.class, "agent.load.threshold", "Advanced", "0.7",
protected final ConfigKey<Boolean> EnableLB = new ConfigKey<>(Boolean.class, "agent.lb.enabled", "Advanced", "false", "Enable agent load balancing between management server nodes", true);
protected final ConfigKey<Double> ConnectedAgentThreshold = new ConfigKey<>(Double.class, "agent.load.threshold", "Advanced", "0.7",
"What percentage of the agents can be held by one management server before load balancing happens", true, EnableLB.key());
protected final ConfigKey<Integer> LoadSize = new ConfigKey<Integer>(Integer.class, "direct.agent.load.size", "Advanced", "16", "How many agents to connect to in each round", true);
protected final ConfigKey<Integer> ScanInterval = new ConfigKey<Integer>(Integer.class, "direct.agent.scan.interval", "Advanced", "90", "Interval between scans to load agents", false,
protected final ConfigKey<Integer> LoadSize = new ConfigKey<>(Integer.class, "direct.agent.load.size", "Advanced", "16", "How many agents to connect to in each round", true);
protected final ConfigKey<Integer> ScanInterval = new ConfigKey<>(Integer.class, "direct.agent.scan.interval", "Advanced", "90", "Interval between scans to load agents", false,
ConfigKey.Scope.Global, 1000);
@Override
public boolean configure(final String name, final Map<String, Object> xmlParams) throws ConfigurationException {
_peers = new HashMap<String, SocketChannel>(7);
_sslEngines = new HashMap<String, SSLEngine>(7);
_peers = new HashMap<>(7);
_sslEngines = new HashMap<>(7);
_nodeId = ManagementServerNode.getManagementServerId();
logger.info("Configuring ClusterAgentManagerImpl. management server node id(msid): {}", _nodeId);
@ -220,7 +220,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
if (hosts != null) {
hosts.addAll(appliances);
if (hosts.size() > 0) {
if (!hosts.isEmpty()) {
logger.debug("Found {} unmanaged direct hosts, processing connect for them...", hosts.size());
for (final HostVO host : hosts) {
try {
@ -234,12 +234,10 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
continue;
}
}
logger.debug("Loading directly connected host {}", host);
logger.debug("Loading directly connected {}", host);
loadDirectlyConnectedHost(host, false);
} catch (final Throwable e) {
logger.warn(" can not load directly connected host {}({}) due to ",
host, e);
logger.warn(" can not load directly connected {} due to ", host, e);
}
}
}
@ -267,10 +265,10 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
logger.debug("create forwarding ClusteredAgentAttache for {}", host);
long id = host.getId();
final AgentAttache attache = new ClusteredAgentAttache(this, id, host.getUuid(), host.getName());
AgentAttache old = null;
AgentAttache old;
synchronized (_agents) {
old = _agents.get(id);
_agents.put(id, attache);
old = _agents.get(host.getId());
_agents.put(host.getId(), attache);
}
if (old != null) {
logger.debug("Remove stale agent attache from current management server");
@ -284,7 +282,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
logger.debug("create ClusteredAgentAttache for {}", host);
final AgentAttache attache = new ClusteredAgentAttache(this, host.getId(), host.getUuid(), host.getName(), link, host.isInMaintenanceStates());
link.attach(attache);
AgentAttache old = null;
AgentAttache old;
synchronized (_agents) {
old = _agents.get(host.getId());
_agents.put(host.getId(), attache);
@ -299,7 +297,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
protected AgentAttache createAttacheForDirectConnect(final Host host, final ServerResource resource) {
logger.debug("Create ClusteredDirectAgentAttache for {}.", host);
final DirectAgentAttache attache = new ClusteredDirectAgentAttache(this, host.getId(), host.getUuid(), host.getName(), _nodeId, resource, host.isInMaintenanceStates());
AgentAttache old = null;
AgentAttache old;
synchronized (_agents) {
old = _agents.get(host.getId());
_agents.put(host.getId(), attache);
@ -418,12 +416,12 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
public boolean routeToPeer(final String peer, final byte[] bytes) {
int i = 0;
SocketChannel ch = null;
SSLEngine sslEngine = null;
SSLEngine sslEngine;
while (i++ < 5) {
ch = connectToPeer(peer, ch);
if (ch == null) {
try {
logD(bytes, "Unable to route to peer: " + Request.parse(bytes).toString());
logD(bytes, "Unable to route to peer: " + Request.parse(bytes));
} catch (ClassNotFoundException | UnsupportedVersionException e) {
// Request.parse thrown exception when we try to log it, log as much as we can
logD(bytes, "Unable to route to peer, and Request.parse further caught exception" + e.getMessage());
@ -441,7 +439,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
return true;
} catch (final IOException e) {
try {
logI(bytes, "Unable to route to peer: " + Request.parse(bytes).toString() + " due to " + e.getMessage());
logI(bytes, "Unable to route to peer: " + Request.parse(bytes) + " due to " + e.getMessage());
} catch (ClassNotFoundException | UnsupportedVersionException ex) {
// Request.parse thrown exception when we try to log it, log as much as we can
logI(bytes, "Unable to route to peer due to" + e.getMessage() + ". Also caught exception when parsing request: " + ex.getMessage());
@ -484,7 +482,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
public SocketChannel connectToPeer(final String peerName, final SocketChannel prevCh) {
synchronized (_peers) {
final SocketChannel ch = _peers.get(peerName);
SSLEngine sslEngine = null;
SSLEngine sslEngine;
if (prevCh != null) {
try {
prevCh.close();
@ -569,13 +567,13 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
AgentAttache agent = findAttache(hostId);
if (agent == null || !agent.forForward()) {
if (isHostOwnerSwitched(host)) {
logger.debug("Host {} has switched to another management server, need to update agent map with a forwarding agent attache", host);
logger.debug("{} has switched to another management server, need to update agent map with a forwarding agent attache", host);
agent = createAttache(host);
}
}
if (agent == null) {
final AgentUnavailableException ex = new AgentUnavailableException("Host with specified id is not in the right state: " + host.getStatus(), hostId);
ex.addProxyObject(_entityMgr.findById(Host.class, hostId).getUuid());
ex.addProxyObject(host.getUuid());
throw ex;
}
@ -617,9 +615,8 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
@Override
protected void doTask(final Task task) throws TaskExecutionException {
final TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.CLOUD_DB);
try {
if (task.getType() != Task.Type.DATA) {
try (TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.CLOUD_DB)) {
if (task.getType() != Type.DATA) {
super.doTask(task);
return;
}
@ -646,7 +643,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
}
final Request req = Request.parse(data);
final Command[] cmds = req.getCommands();
final CancelCommand cancel = (CancelCommand)cmds[0];
final CancelCommand cancel = (CancelCommand) cmds[0];
logD(data, "Cancel request received");
agent.cancel(cancel.getSequence());
final Long current = agent._currentSequence;
@ -670,10 +667,9 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
// to deserialize this and send it through the agent attache.
final Request req = Request.parse(data);
agent.send(req, null);
return;
} else {
if (agent instanceof Routable) {
final Routable cluster = (Routable)agent;
final Routable cluster = (Routable) agent;
cluster.routeToAgent(data);
} else {
agent.send(Request.parse(data));
@ -690,13 +686,12 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
if (mgmtId != -1 && mgmtId != _nodeId) {
routeToPeer(Long.toString(mgmtId), data);
if (Request.requiresSequentialExecution(data)) {
final AgentAttache attache = (AgentAttache)link.attachment();
final AgentAttache attache = (AgentAttache) link.attachment();
if (attache != null) {
attache.sendNext(Request.getSequence(data));
}
logD(data, "No attache to process " + Request.parse(data).toString());
logD(data, "No attache to process " + Request.parse(data));
}
return;
} else {
if (Request.isRequest(data)) {
super.doTask(task);
@ -712,7 +707,6 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
logger.info("SeqA {}-{}: Response is not processed: {}", attache.getId(), response.getSequence(), response.toString());
}
}
return;
}
}
} catch (final ClassNotFoundException e) {
@ -723,8 +717,6 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
final String message = String.format("UnsupportedVersionException occurred when executing tasks! Error '%s'", e.getMessage());
logger.error(message);
throw new TaskExecutionException(message, e);
} finally {
txn.close();
}
}
}
@ -768,7 +760,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
public boolean executeRebalanceRequest(final long agentId, final long currentOwnerId, final long futureOwnerId, final Event event, boolean isConnectionTransfer) throws AgentUnavailableException, OperationTimedoutException {
boolean result = false;
if (event == Event.RequestAgentRebalance) {
return setToWaitForRebalance(agentId, currentOwnerId, futureOwnerId);
return setToWaitForRebalance(agentId);
} else if (event == Event.StartAgentRebalance) {
try {
result = rebalanceHost(agentId, currentOwnerId, futureOwnerId, isConnectionTransfer);
@ -823,7 +815,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
sc.and(sc.entity().getType(), Op.EQ, Host.Type.Routing);
final List<HostVO> allManagedAgents = sc.list();
int avLoad = 0;
int avLoad;
if (!allManagedAgents.isEmpty() && !allMS.isEmpty()) {
avLoad = allManagedAgents.size() / allMS.size();
@ -841,7 +833,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
for (final ManagementServerHostVO node : allMS) {
if (node.getMsid() != _nodeId) {
List<HostVO> hostsToRebalance = new ArrayList<HostVO>();
List<HostVO> hostsToRebalance = new ArrayList<>();
for (final AgentLoadBalancerPlanner lbPlanner : _lbPlanners) {
hostsToRebalance = lbPlanner.getHostsToRebalance(node, avLoad);
if (hostsToRebalance != null && !hostsToRebalance.isEmpty()) {
@ -867,7 +859,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
HostTransferMapVO transfer = null;
try {
transfer = _hostTransferDao.startAgentTransfering(hostId, node.getMsid(), _nodeId);
final Answer[] answer = sendRebalanceCommand(node.getMsid(), hostId, node.getMsid(), _nodeId, Event.RequestAgentRebalance);
final Answer[] answer = sendRebalanceCommand(node.getMsid(), hostId, node.getMsid(), _nodeId);
if (answer == null) {
logger.warn("Failed to get host {} from management server {}", host, node);
result = false;
@ -894,8 +886,8 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
}
}
private Answer[] sendRebalanceCommand(final long peer, final long agentId, final long currentOwnerId, final long futureOwnerId, final Event event) {
return sendRebalanceCommand(peer, agentId, currentOwnerId, futureOwnerId, event, false);
private Answer[] sendRebalanceCommand(final long peer, final long agentId, final long currentOwnerId, final long futureOwnerId) {
return sendRebalanceCommand(peer, agentId, currentOwnerId, futureOwnerId, Event.RequestAgentRebalance, false);
}
private Answer[] sendRebalanceCommand(final long peer, final long agentId, final long currentOwnerId, final long futureOwnerId, final Event event, final boolean isConnectionTransfer) {
@ -910,8 +902,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
final String peerName = Long.toString(peer);
final String cmdStr = _gson.toJson(cmds);
final String ansStr = _clusterMgr.execute(peerName, agentId, cmdStr, true);
final Answer[] answers = _gson.fromJson(ansStr, Answer[].class);
return answers;
return _gson.fromJson(ansStr, Answer[].class);
} catch (final Exception e) {
logger.warn("Caught exception while talking to {}", currentOwnerId, e);
return null;
@ -960,7 +951,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
try {
logger.trace("Clustered agent transfer scan check, management server id: {}", _nodeId);
synchronized (_agentToTransferIds) {
if (_agentToTransferIds.size() > 0) {
if (!_agentToTransferIds.isEmpty()) {
logger.debug("Found {} agents to transfer", _agentToTransferIds.size());
// for (Long hostId : _agentToTransferIds) {
for (final Iterator<Long> iterator = _agentToTransferIds.iterator(); iterator.hasNext();) {
@ -984,7 +975,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
}
if (transferMap.getInitialOwner() != _nodeId || attache == null || attache.forForward()) {
logger.debug(String.format("Management server %d doesn't own host id=%d (%s) any more, skipping rebalance for the host", _nodeId, hostId, attache));
logger.debug("Management server {} doesn't own host id={} ({}) any more, skipping rebalance for the host", _nodeId, hostId, attache);
iterator.remove();
_hostTransferDao.completeAgentTransfer(hostId);
continue;
@ -1004,9 +995,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
_executor.execute(new RebalanceTask(hostId, transferMap.getInitialOwner(), transferMap.getFutureOwner()));
} catch (final RejectedExecutionException ex) {
logger.warn("Failed to submit rebalance task for host id={} ({}); postponing the execution", hostId, attache);
continue;
}
} else {
logger.debug("Agent {} ({}) can't be transferred yet as its request queue size is {} and listener queue size is {}",
hostId, attache, attache.getQueueSize(), attache.getNonRecurringListenersSize());
@ -1016,7 +1005,6 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
logger.trace("Found no agents to be transferred by the management server {}", _nodeId);
}
}
} catch (final Throwable e) {
logger.error("Problem with the clustered agent transfer scan check!", e);
}
@ -1024,7 +1012,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
};
}
private boolean setToWaitForRebalance(final long hostId, final long currentOwnerId, final long futureOwnerId) {
private boolean setToWaitForRebalance(final long hostId) {
logger.debug("Adding agent {} ({}) to the list of agents to transfer", hostId, findAttache(hostId));
synchronized (_agentToTransferIds) {
return _agentToTransferIds.add(hostId);
@ -1065,7 +1053,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
} else if (futureOwnerId == _nodeId) {
final HostVO host = _hostDao.findById(hostId);
try {
logger.debug("Disconnecting host {} as a part of rebalance process without notification", host);
logger.debug("Disconnecting {} as a part of rebalance process without notification", host);
final AgentAttache attache = findAttache(hostId);
if (attache != null) {
@ -1085,9 +1073,9 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
}
if (result) {
logger.debug("Successfully loaded directly connected host {} to the management server {} a part of rebalance process without notification", host, _nodeId);
logger.debug("Successfully loaded directly connected {} to the management server {} a part of rebalance process without notification", host, _nodeId);
} else {
logger.warn("Failed to load directly connected host {} to the management server {} a part of rebalance process without notification", host, _nodeId);
logger.warn("Failed to load directly connected {} to the management server {} a part of rebalance process without notification", host, _nodeId);
}
}
@ -1096,12 +1084,12 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
protected void finishRebalance(final long hostId, final long futureOwnerId, final Event event) {
final boolean success = event == Event.RebalanceCompleted ? true : false;
final boolean success = event == Event.RebalanceCompleted;
final AgentAttache attache = findAttache(hostId);
logger.debug("Finishing rebalancing for the agent {} ({}) with event {}", hostId, attache, event);
if (attache == null || !(attache instanceof ClusteredAgentAttache)) {
if (!(attache instanceof ClusteredAgentAttache)) {
logger.debug("Unable to find forward attache for the host id={} assuming that the agent disconnected already", hostId);
_hostTransferDao.completeAgentTransfer(hostId);
return;
@ -1197,9 +1185,9 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
}
protected class RebalanceTask extends ManagedContextRunnable {
Long hostId = null;
Long currentOwnerId = null;
Long futureOwnerId = null;
Long hostId;
Long currentOwnerId;
Long futureOwnerId;
public RebalanceTask(final long hostId, final long currentOwnerId, final long futureOwnerId) {
this.hostId = hostId;
@ -1268,7 +1256,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
final ChangeAgentCommand cmd = (ChangeAgentCommand)cmds[0];
logger.debug("Intercepting command for agent change: agent {} event: {}", cmd.getAgentId(), cmd.getEvent());
boolean result = false;
boolean result;
try {
result = executeAgentUserRequest(cmd.getAgentId(), cmd.getEvent());
logger.debug("Result is {}", result);
@ -1285,7 +1273,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
final TransferAgentCommand cmd = (TransferAgentCommand)cmds[0];
logger.debug("Intercepting command for agent rebalancing: agent: {}, event: {}, connection transfer: {}", cmd.getAgentId(), cmd.getEvent(), cmd.isConnectionTransfer());
boolean result = false;
boolean result;
try {
result = rebalanceAgent(cmd.getAgentId(), cmd.getEvent(), cmd.getCurrentOwner(), cmd.getFutureOwner(), cmd.isConnectionTransfer());
logger.debug("Result is {}", result);
@ -1305,7 +1293,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
logger.debug("Intercepting command to propagate event {} for host {} ({})", () -> cmd.getEvent().name(), cmd::getHostId, () -> _hostDao.findById(cmd.getHostId()));
boolean result = false;
boolean result;
try {
result = _resourceMgr.executeUserRequest(cmd.getHostId(), cmd.getEvent());
logger.debug("Result is {}", result);
@ -1403,23 +1391,23 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
@Override
public boolean transferDirectAgentsFromMS(String fromMsUuid, long fromMsId, long timeoutDurationInMs) {
if (timeoutDurationInMs <= 0) {
logger.debug(String.format("Not transferring direct agents from management server node %d (id: %s) to other nodes, invalid timeout duration", fromMsId, fromMsUuid));
logger.debug("Not transferring direct agents from management server node {} (id: {}) to other nodes, invalid timeout duration", fromMsId, fromMsUuid);
return false;
}
long transferStartTime = System.currentTimeMillis();
if (CollectionUtils.isEmpty(getDirectAgentHosts(fromMsId))) {
logger.info(String.format("No direct agent hosts available on management server node %d (id: %s), to transfer", fromMsId, fromMsUuid));
logger.info("No direct agent hosts available on management server node {} (id: {}), to transfer", fromMsId, fromMsUuid);
return true;
}
List<ManagementServerHostVO> msHosts = getUpMsHostsExcludingMs(fromMsId);
if (msHosts.isEmpty()) {
logger.warn(String.format("No management server nodes available to transfer agents from management server node %d (id: %s)", fromMsId, fromMsUuid));
logger.warn("No management server nodes available to transfer agents from management server node {} (id: {})", fromMsId, fromMsUuid);
return false;
}
logger.debug(String.format("Transferring direct agents from management server node %d (id: %s) to other nodes", fromMsId, fromMsUuid));
logger.debug("Transferring direct agents from management server node {} (id: {}) to other nodes", fromMsId, fromMsUuid);
int agentTransferFailedCount = 0;
List<DataCenterVO> dataCenterList = dcDao.listAll();
for (DataCenterVO dc : dataCenterList) {
@ -1427,11 +1415,11 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
if (CollectionUtils.isEmpty(directAgentHostsInDc)) {
continue;
}
logger.debug(String.format("Transferring %d direct agents from management server node %d (id: %s) of zone %s", directAgentHostsInDc.size(), fromMsId, fromMsUuid, dc.toString()));
logger.debug("Transferring {} direct agents from management server node {} (id: {}) of zone {}", directAgentHostsInDc.size(), fromMsId, fromMsUuid, dc);
for (HostVO host : directAgentHostsInDc) {
long transferElapsedTimeInMs = System.currentTimeMillis() - transferStartTime;
if (transferElapsedTimeInMs >= timeoutDurationInMs) {
logger.debug(String.format("Stop transferring remaining direct agents from management server node %d (id: %s), timed out", fromMsId, fromMsUuid));
logger.debug("Stop transferring remaining direct agents from management server node {} (id: {}), timed out", fromMsId, fromMsUuid);
return false;
}
@ -1449,7 +1437,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
updateLastManagementServer(host.getId(), fromMsId);
}
} catch (Exception e) {
logger.warn(String.format("Failed to transfer direct agent of the host %s from management server node %d (id: %s), due to %s", host, fromMsId, fromMsUuid, e.getMessage()));
logger.warn("Failed to transfer direct agent of the host {} from management server node {} (id: {}), due to {}", host, fromMsId, fromMsUuid, e.getMessage());
}
}
}
@ -1462,7 +1450,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
List<HostVO> hosts = _hostDao.listHostsByMs(msId);
for (HostVO host : hosts) {
AgentAttache agent = findAttache(host.getId());
if (agent != null && agent instanceof DirectAgentAttache) {
if (agent instanceof DirectAgentAttache) {
directAgentHosts.add(host);
}
}
@ -1475,7 +1463,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
List<HostVO> hosts = _hostDao.listHostsByMsAndDc(msId, dcId);
for (HostVO host : hosts) {
AgentAttache agent = findAttache(host.getId());
if (agent != null && agent instanceof DirectAgentAttache) {
if (agent instanceof DirectAgentAttache) {
directAgentHosts.add(host);
}
}
@ -1485,13 +1473,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
private List<ManagementServerHostVO> getUpMsHostsExcludingMs(long avoidMsId) {
final List<ManagementServerHostVO> msHosts = _mshostDao.listBy(ManagementServerHost.State.Up);
Iterator<ManagementServerHostVO> iterator = msHosts.iterator();
while (iterator.hasNext()) {
ManagementServerHostVO ms = iterator.next();
if (ms.getMsid() == avoidMsId || _mshostPeerDao.findByPeerMsAndState(ms.getId(), ManagementServerHost.State.Up) == null) {
iterator.remove();
}
}
msHosts.removeIf(ms -> ms.getMsid() == avoidMsId || _mshostPeerDao.findByPeerMsAndState(ms.getId(), ManagementServerHost.State.Up) == null);
return msHosts;
}
@ -1593,8 +1575,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
public ConfigKey<?>[] getConfigKeys() {
final ConfigKey<?>[] keys = super.getConfigKeys();
final List<ConfigKey<?>> keysLst = new ArrayList<ConfigKey<?>>();
keysLst.addAll(Arrays.asList(keys));
final List<ConfigKey<?>> keysLst = new ArrayList<>(Arrays.asList(keys));
keysLst.add(EnableLB);
keysLst.add(ConnectedAgentThreshold);
keysLst.add(LoadSize);

220
engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java
View File

@ -85,6 +85,7 @@ import org.apache.cloudstack.resource.ResourceCleanupService;
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
import org.apache.cloudstack.storage.to.VolumeObjectTO;
import org.apache.cloudstack.utils.cache.SingleCache;
import org.apache.cloudstack.utils.identity.ManagementServerNode;
import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils;
import org.apache.cloudstack.vm.UnmanagedVMsManager;
@ -406,6 +407,10 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
private DomainDao domainDao;
@Inject
ResourceCleanupService resourceCleanupService;
@Inject
VmWorkJobDao vmWorkJobDao;
private SingleCache<List<Long>> vmIdsInProgressCache;
VmWorkJobHandlerProxy _jobHandlerProxy = new VmWorkJobHandlerProxy(this);
@ -450,6 +455,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
Long.class, "systemvm.root.disk.size", "-1",
"Size of root volume (in GB) of system VMs and virtual routers", true);
private boolean syncTransitioningVmPowerState;
ScheduledExecutorService _executor = null;
private long _nodeId;
@ -816,6 +823,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
@Override
public boolean start() {
vmIdsInProgressCache = new SingleCache<>(10, vmWorkJobDao::listVmIdsWithPendingJob);
_executor.scheduleAtFixedRate(new CleanupTask(), 5, VmJobStateReportInterval.value(), TimeUnit.SECONDS);
_executor.scheduleAtFixedRate(new TransitionTask(), VmOpCleanupInterval.value(), VmOpCleanupInterval.value(), TimeUnit.SECONDS);
cancelWorkItems(_nodeId);
@ -843,6 +851,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
_messageBus.subscribe(VirtualMachineManager.Topics.VM_POWER_STATE, MessageDispatcher.getDispatcher(this));
syncTransitioningVmPowerState = Boolean.TRUE.equals(VmSyncPowerStateTransitioning.value());
return true;
}
@ -3506,7 +3516,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
if (MIGRATE_VM_ACROSS_CLUSTERS.valueIn(host.getDataCenterId()) &&
(HypervisorType.VMware.equals(host.getHypervisorType()) || !checkIfVmHasClusterWideVolumes(vm.getId()))) {
logger.info("Searching for hosts in the zone for vm migration");
List<Long> clustersToExclude = _clusterDao.listAllClusters(host.getDataCenterId());
List<Long> clustersToExclude = _clusterDao.listAllClusterIds(host.getDataCenterId());
List<ClusterVO> clusterList = _clusterDao.listByDcHyType(host.getDataCenterId(), host.getHypervisorType().toString());
for (ClusterVO cluster : clusterList) {
clustersToExclude.remove(cluster.getId());
@ -3800,7 +3810,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
if (ping.getHostVmStateReport() != null) {
_syncMgr.processHostVmStatePingReport(agentId, ping.getHostVmStateReport(), ping.getOutOfBand());
}
scanStalledVMInTransitionStateOnUpHost(agentId);
processed = true;
}
@ -4757,7 +4766,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
VmOpLockStateRetry, VmOpWaitInterval, ExecuteInSequence, VmJobCheckInterval, VmJobTimeout, VmJobStateReportInterval,
VmConfigDriveLabel, VmConfigDriveOnPrimaryPool, VmConfigDriveForceHostCacheUse, VmConfigDriveUseHostCacheOnUnsupportedPool,
HaVmRestartHostUp, ResourceCountRunningVMsonly, AllowExposeHypervisorHostname, AllowExposeHypervisorHostnameAccountLevel, SystemVmRootDiskSize,
AllowExposeDomainInMetadata, MetadataCustomCloudName, VmMetadataManufacturer, VmMetadataProductName
AllowExposeDomainInMetadata, MetadataCustomCloudName, VmMetadataManufacturer, VmMetadataProductName,
VmSyncPowerStateTransitioning
};
}
@ -4955,20 +4965,46 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
}
}
/**
* Scans stalled VMs in transition states on an UP host and processes them accordingly.
*
* <p>This method is executed only when the {@code syncTransitioningVmPowerState} flag is enabled. It identifies
* VMs stuck in specific states (e.g., Starting, Stopping, Migrating) on a host that is UP, except for those
* in the Expunging state, which require special handling.</p>
*
* <p>The following conditions are checked during the scan:
* <ul>
* <li>No pending {@code VmWork} job exists for the VM.</li>
* <li>The VM is associated with the given {@code hostId}, and the host is UP.</li>
* </ul>
* </p>
*
* <p>When a host is UP, a state report for the VMs will typically be received. However, certain scenarios
* (e.g., out-of-band changes or behavior specific to hypervisors like XenServer or KVM) might result in
* missing reports, preventing the state-sync logic from running. To address this, the method scans VMs
* based on their last update timestamp. If a VM remains stalled without a status update while its host is UP,
* it is assumed to be powered off, which is generally a safe assumption.</p>
*
* @param hostId the ID of the host to scan for stalled VMs in transition states.
*/
private void scanStalledVMInTransitionStateOnUpHost(final long hostId) {
final long stallThresholdInMs = VmJobStateReportInterval.value() + (VmJobStateReportInterval.value() >> 1);
final Date cutTime = new Date(DateUtil.currentGMTTime().getTime() - stallThresholdInMs);
final List<Long> mostlikelyStoppedVMs = listStalledVMInTransitionStateOnUpHost(hostId, cutTime);
for (final Long vmId : mostlikelyStoppedVMs) {
final VMInstanceVO vm = _vmDao.findById(vmId);
assert vm != null;
if (!syncTransitioningVmPowerState) {
return;
}
if (!_hostDao.isHostUp(hostId)) {
return;
}
final long stallThresholdInMs = VmJobStateReportInterval.value() * 2;
final long cutTime = new Date(DateUtil.currentGMTTime().getTime() - stallThresholdInMs).getTime();
final List<VMInstanceVO> hostTransitionVms = _vmDao.listByHostAndState(hostId, State.Starting, State.Stopping, State.Migrating);
final List<VMInstanceVO> mostLikelyStoppedVMs = listStalledVMInTransitionStateOnUpHost(hostTransitionVms, cutTime);
for (final VMInstanceVO vm : mostLikelyStoppedVMs) {
handlePowerOffReportWithNoPendingJobsOnVM(vm);
}
final List<Long> vmsWithRecentReport = listVMInTransitionStateWithRecentReportOnUpHost(hostId, cutTime);
for (final Long vmId : vmsWithRecentReport) {
final VMInstanceVO vm = _vmDao.findById(vmId);
assert vm != null;
final List<VMInstanceVO> vmsWithRecentReport = listVMInTransitionStateWithRecentReportOnUpHost(hostTransitionVms, cutTime);
for (final VMInstanceVO vm : vmsWithRecentReport) {
if (vm.getPowerState() == PowerState.PowerOn) {
handlePowerOnReportWithNoPendingJobsOnVM(vm);
} else {
@ -4977,6 +5013,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
}
}
private void scanStalledVMInTransitionStateOnDisconnectedHosts() {
final Date cutTime = new Date(DateUtil.currentGMTTime().getTime() - VmOpWaitInterval.value() * 1000);
final List<Long> stuckAndUncontrollableVMs = listStalledVMInTransitionStateOnDisconnectedHosts(cutTime);
@ -4989,89 +5026,58 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
}
}
private List<Long> listStalledVMInTransitionStateOnUpHost(final long hostId, final Date cutTime) {
final String sql = "SELECT i.* FROM vm_instance as i, host as h WHERE h.status = 'UP' " +
"AND h.id = ? AND i.power_state_update_time < ? AND i.host_id = h.id " +
"AND (i.state ='Starting' OR i.state='Stopping' OR i.state='Migrating') " +
"AND i.id NOT IN (SELECT w.vm_instance_id FROM vm_work_job AS w JOIN async_job AS j ON w.id = j.id WHERE j.job_status = ?)" +
"AND i.removed IS NULL";
final List<Long> l = new ArrayList<>();
try (TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.CLOUD_DB)) {
String cutTimeStr = DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutTime);
try {
PreparedStatement pstmt = txn.prepareAutoCloseStatement(sql);
pstmt.setLong(1, hostId);
pstmt.setString(2, cutTimeStr);
pstmt.setInt(3, JobInfo.Status.IN_PROGRESS.ordinal());
final ResultSet rs = pstmt.executeQuery();
while (rs.next()) {
l.add(rs.getLong(1));
}
} catch (SQLException e) {
logger.error("Unable to execute SQL [{}] with params {\"h.id\": {}, \"i.power_state_update_time\": \"{}\"} due to [{}].", sql, hostId, cutTimeStr, e.getMessage(), e);
}
private List<VMInstanceVO> listStalledVMInTransitionStateOnUpHost(
final List<VMInstanceVO> transitioningVms, final long cutTime) {
if (CollectionUtils.isEmpty(transitioningVms)) {
return transitioningVms;
}
return l;
List<Long> vmIdsInProgress = vmIdsInProgressCache.get();
return transitioningVms.stream()
.filter(v -> v.getPowerStateUpdateTime().getTime() < cutTime && !vmIdsInProgress.contains(v.getId()))
.collect(Collectors.toList());
}
private List<Long> listVMInTransitionStateWithRecentReportOnUpHost(final long hostId, final Date cutTime) {
final String sql = "SELECT i.* FROM vm_instance as i, host as h WHERE h.status = 'UP' " +
"AND h.id = ? AND i.power_state_update_time > ? AND i.host_id = h.id " +
"AND (i.state ='Starting' OR i.state='Stopping' OR i.state='Migrating') " +
"AND i.id NOT IN (SELECT w.vm_instance_id FROM vm_work_job AS w JOIN async_job AS j ON w.id = j.id WHERE j.job_status = ?)" +
"AND i.removed IS NULL";
final List<Long> l = new ArrayList<>();
try (TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.CLOUD_DB)) {
String cutTimeStr = DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutTime);
int jobStatusInProgress = JobInfo.Status.IN_PROGRESS.ordinal();
try {
PreparedStatement pstmt = txn.prepareAutoCloseStatement(sql);
pstmt.setLong(1, hostId);
pstmt.setString(2, cutTimeStr);
pstmt.setInt(3, jobStatusInProgress);
final ResultSet rs = pstmt.executeQuery();
while (rs.next()) {
l.add(rs.getLong(1));
}
} catch (final SQLException e) {
logger.error("Unable to execute SQL [{}] with params {\"h.id\": {}, \"i.power_state_update_time\": \"{}\", \"j.job_status\": {}} due to [{}].", sql, hostId, cutTimeStr, jobStatusInProgress, e.getMessage(), e);
}
return l;
private List<VMInstanceVO> listVMInTransitionStateWithRecentReportOnUpHost(
final List<VMInstanceVO> transitioningVms, final long cutTime) {
if (CollectionUtils.isEmpty(transitioningVms)) {
return transitioningVms;
}
List<Long> vmIdsInProgress = vmIdsInProgressCache.get();
return transitioningVms.stream()
.filter(v -> v.getPowerStateUpdateTime().getTime() > cutTime && !vmIdsInProgress.contains(v.getId()))
.collect(Collectors.toList());
}
private List<Long> listStalledVMInTransitionStateOnDisconnectedHosts(final Date cutTime) {
final String sql = "SELECT i.* FROM vm_instance as i, host as h WHERE h.status != 'UP' " +
"AND i.power_state_update_time < ? AND i.host_id = h.id " +
"AND (i.state ='Starting' OR i.state='Stopping' OR i.state='Migrating') " +
"AND i.id NOT IN (SELECT w.vm_instance_id FROM vm_work_job AS w JOIN async_job AS j ON w.id = j.id WHERE j.job_status = ?)" +
"AND i.removed IS NULL";
final String sql = "SELECT i.* " +
"FROM vm_instance AS i " +
"INNER JOIN host AS h ON i.host_id = h.id " +
"WHERE h.status != 'UP' " +
" AND i.power_state_update_time < ? " +
" AND i.state IN ('Starting', 'Stopping', 'Migrating') " +
" AND i.id NOT IN (SELECT vm_instance_id FROM vm_work_job AS w " +
" INNER JOIN async_job AS j ON w.id = j.id " +
" WHERE j.job_status = ?) " +
" AND i.removed IS NULL";
final List<Long> l = new ArrayList<>();
try (TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.CLOUD_DB)) {
String cutTimeStr = DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutTime);
int jobStatusInProgress = JobInfo.Status.IN_PROGRESS.ordinal();
TransactionLegacy txn = TransactionLegacy.currentTxn();
String cutTimeStr = DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutTime);
int jobStatusInProgress = JobInfo.Status.IN_PROGRESS.ordinal();
try {
PreparedStatement pstmt = txn.prepareAutoCloseStatement(sql);
try {
PreparedStatement pstmt = txn.prepareAutoCloseStatement(sql);
pstmt.setString(1, cutTimeStr);
pstmt.setInt(2, jobStatusInProgress);
final ResultSet rs = pstmt.executeQuery();
while (rs.next()) {
l.add(rs.getLong(1));
}
} catch (final SQLException e) {
logger.error("Unable to execute SQL [{}] with params {\"i.power_state_update_time\": \"{}\", \"j.job_status\": {}} due to [{}].", sql, cutTimeStr, jobStatusInProgress, e.getMessage(), e);
pstmt.setString(1, cutTimeStr);
pstmt.setInt(2, jobStatusInProgress);
final ResultSet rs = pstmt.executeQuery();
while (rs.next()) {
l.add(rs.getLong(1));
}
return l;
} catch (final SQLException e) {
logger.error("Unable to execute SQL [{}] with params {\"i.power_state_update_time\": \"{}\", \"j.job_status\": {}} due to [{}].", sql, cutTimeStr, jobStatusInProgress, e.getMessage(), e);
}
return l;
}
public class VmStateSyncOutcome extends OutcomeImpl<VirtualMachine> {
@ -5953,29 +5959,23 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
}
@Override
public HashMap<Long, ? extends VmStats> getVirtualMachineStatistics(long hostId, String hostName, List<Long> vmIds) {
public HashMap<Long, ? extends VmStats> getVirtualMachineStatistics(Host host, List<Long> vmIds) {
HashMap<Long, VmStatsEntry> vmStatsById = new HashMap<>();
if (CollectionUtils.isEmpty(vmIds)) {
return vmStatsById;
}
Map<Long, VMInstanceVO> vmMap = new HashMap<>();
for (Long vmId : vmIds) {
vmMap.put(vmId, _vmDao.findById(vmId));
}
return getVirtualMachineStatistics(hostId, hostName, vmMap);
Map<String, Long> vmMap = _vmDao.getNameIdMapForVmIds(vmIds);
return getVirtualMachineStatistics(host, vmMap);
}
@Override
public HashMap<Long, ? extends VmStats> getVirtualMachineStatistics(long hostId, String hostName, Map<Long, ? extends VirtualMachine> vmMap) {
public HashMap<Long, ? extends VmStats> getVirtualMachineStatistics(Host host, Map<String, Long> vmInstanceNameIdMap) {
HashMap<Long, VmStatsEntry> vmStatsById = new HashMap<>();
if (MapUtils.isEmpty(vmMap)) {
if (MapUtils.isEmpty(vmInstanceNameIdMap)) {
return vmStatsById;
}
Map<String, Long> vmNames = new HashMap<>();
for (Map.Entry<Long, ? extends VirtualMachine> vmEntry : vmMap.entrySet()) {
vmNames.put(vmEntry.getValue().getInstanceName(), vmEntry.getKey());
}
Answer answer = _agentMgr.easySend(hostId, new GetVmStatsCommand(new ArrayList<>(vmNames.keySet()), _hostDao.findById(hostId).getGuid(), hostName));
Answer answer = _agentMgr.easySend(host.getId(), new GetVmStatsCommand(
new ArrayList<>(vmInstanceNameIdMap.keySet()), host.getGuid(), host.getName()));
if (answer == null || !answer.getResult()) {
logger.warn("Unable to obtain VM statistics.");
return vmStatsById;
@ -5986,23 +5986,20 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
return vmStatsById;
}
for (Map.Entry<String, VmStatsEntry> entry : vmStatsByName.entrySet()) {
vmStatsById.put(vmNames.get(entry.getKey()), entry.getValue());
vmStatsById.put(vmInstanceNameIdMap.get(entry.getKey()), entry.getValue());
}
}
return vmStatsById;
}
@Override
public HashMap<Long, List<? extends VmDiskStats>> getVmDiskStatistics(long hostId, String hostName, Map<Long, ? extends VirtualMachine> vmMap) {
public HashMap<Long, List<? extends VmDiskStats>> getVmDiskStatistics(Host host, Map<String, Long> vmInstanceNameIdMap) {
HashMap<Long, List<? extends VmDiskStats>> vmDiskStatsById = new HashMap<>();
if (MapUtils.isEmpty(vmMap)) {
if (MapUtils.isEmpty(vmInstanceNameIdMap)) {
return vmDiskStatsById;
}
Map<String, Long> vmNames = new HashMap<>();
for (Map.Entry<Long, ? extends VirtualMachine> vmEntry : vmMap.entrySet()) {
vmNames.put(vmEntry.getValue().getInstanceName(), vmEntry.getKey());
}
Answer answer = _agentMgr.easySend(hostId, new GetVmDiskStatsCommand(new ArrayList<>(vmNames.keySet()), _hostDao.findById(hostId).getGuid(), hostName));
Answer answer = _agentMgr.easySend(host.getId(), new GetVmDiskStatsCommand(
new ArrayList<>(vmInstanceNameIdMap.keySet()), host.getGuid(), host.getName()));
if (answer == null || !answer.getResult()) {
logger.warn("Unable to obtain VM disk statistics.");
return vmDiskStatsById;
@ -6013,23 +6010,20 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
return vmDiskStatsById;
}
for (Map.Entry<String, List<VmDiskStatsEntry>> entry: vmDiskStatsByName.entrySet()) {
vmDiskStatsById.put(vmNames.get(entry.getKey()), entry.getValue());
vmDiskStatsById.put(vmInstanceNameIdMap.get(entry.getKey()), entry.getValue());
}
}
return vmDiskStatsById;
}
@Override
public HashMap<Long, List<? extends VmNetworkStats>> getVmNetworkStatistics(long hostId, String hostName, Map<Long, ? extends VirtualMachine> vmMap) {
public HashMap<Long, List<? extends VmNetworkStats>> getVmNetworkStatistics(Host host, Map<String, Long> vmInstanceNameIdMap) {
HashMap<Long, List<? extends VmNetworkStats>> vmNetworkStatsById = new HashMap<>();
if (MapUtils.isEmpty(vmMap)) {
if (MapUtils.isEmpty(vmInstanceNameIdMap)) {
return vmNetworkStatsById;
}
Map<String, Long> vmNames = new HashMap<>();
for (Map.Entry<Long, ? extends VirtualMachine> vmEntry : vmMap.entrySet()) {
vmNames.put(vmEntry.getValue().getInstanceName(), vmEntry.getKey());
}
Answer answer = _agentMgr.easySend(hostId, new GetVmNetworkStatsCommand(new ArrayList<>(vmNames.keySet()), _hostDao.findById(hostId).getGuid(), hostName));
Answer answer = _agentMgr.easySend(host.getId(), new GetVmNetworkStatsCommand(
new ArrayList<>(vmInstanceNameIdMap.keySet()), host.getGuid(), host.getName()));
if (answer == null || !answer.getResult()) {
logger.warn("Unable to obtain VM network statistics.");
return vmNetworkStatsById;
@ -6040,7 +6034,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
return vmNetworkStatsById;
}
for (Map.Entry<String, List<VmNetworkStatsEntry>> entry: vmNetworkStatsByName.entrySet()) {
vmNetworkStatsById.put(vmNames.get(entry.getKey()), entry.getValue());
vmNetworkStatsById.put(vmInstanceNameIdMap.get(entry.getKey()), entry.getValue());
}
}
return vmNetworkStatsById;

224
engine/orchestration/src/main/java/com/cloud/vm/VirtualMachinePowerStateSyncImpl.java
View File

@ -16,27 +16,29 @@
// under the License.
package com.cloud.vm;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import javax.inject.Inject;
import com.cloud.host.Host;
import com.cloud.host.HostVO;
import com.cloud.host.dao.HostDao;
import com.cloud.utils.Pair;
import org.apache.cloudstack.framework.messagebus.MessageBus;
import org.apache.cloudstack.framework.messagebus.PublishScope;
import org.apache.logging.log4j.Logger;
import org.apache.cloudstack.utils.cache.LazyCache;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import com.cloud.agent.api.HostVmStateReportEntry;
import com.cloud.configuration.ManagementServiceConfiguration;
import com.cloud.host.Host;
import com.cloud.host.HostVO;
import com.cloud.host.dao.HostDao;
import com.cloud.utils.DateUtil;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.vm.dao.VMInstanceDao;
public class VirtualMachinePowerStateSyncImpl implements VirtualMachinePowerStateSync {
@ -47,7 +49,12 @@ public class VirtualMachinePowerStateSyncImpl implements VirtualMachinePowerStat
@Inject HostDao hostDao;
@Inject ManagementServiceConfiguration mgmtServiceConf;
private LazyCache<Long, VMInstanceVO> vmCache;
private LazyCache<Long, HostVO> hostCache;
public VirtualMachinePowerStateSyncImpl() {
vmCache = new LazyCache<>(16, 10, this::getVmFromId);
hostCache = new LazyCache<>(16, 10, this::getHostFromId);
}
@Override
@ -58,130 +65,141 @@ public class VirtualMachinePowerStateSyncImpl implements VirtualMachinePowerStat
@Override
public void processHostVmStateReport(long hostId, Map<String, HostVmStateReportEntry> report) {
HostVO host = hostDao.findById(hostId);
logger.debug("Process host VM state report. host: {}", host);
Map<Long, Pair<VirtualMachine.PowerState, VMInstanceVO>> translatedInfo = convertVmStateReport(report);
processReport(host, translatedInfo, false);
logger.debug("Process host VM state report. host: {}", hostCache.get(hostId));
Map<Long, VirtualMachine.PowerState> translatedInfo = convertVmStateReport(report);
processReport(hostId, translatedInfo, false);
}
@Override
public void processHostVmStatePingReport(long hostId, Map<String, HostVmStateReportEntry> report, boolean force) {
HostVO host = hostDao.findById(hostId);
logger.debug("Process host VM state report from ping process. host: {}", host);
Map<Long, Pair<VirtualMachine.PowerState, VMInstanceVO>> translatedInfo = convertVmStateReport(report);
processReport(host, translatedInfo, force);
logger.debug("Process host VM state report from ping process. host: {}", hostCache.get(hostId));
Map<Long, VirtualMachine.PowerState> translatedInfo = convertVmStateReport(report);
processReport(hostId, translatedInfo, force);
}
private void processReport(HostVO host, Map<Long, Pair<VirtualMachine.PowerState, VMInstanceVO>> translatedInfo, boolean force) {
logger.debug("Process VM state report. host: {}, number of records in report: {}.", host, translatedInfo.size());
for (Map.Entry<Long, Pair<VirtualMachine.PowerState, VMInstanceVO>> entry : translatedInfo.entrySet()) {
logger.debug("VM state report. host: {}, vm: {}, power state: {}", host, entry.getValue().second(), entry.getValue().first());
if (_instanceDao.updatePowerState(entry.getKey(), host.getId(), entry.getValue().first(), DateUtil.currentGMTTime())) {
logger.debug("VM state report is updated. host: {}, vm: {}, power state: {}", host, entry.getValue().second(), entry.getValue().first());
_messageBus.publish(null, VirtualMachineManager.Topics.VM_POWER_STATE, PublishScope.GLOBAL, entry.getKey());
} else {
logger.trace("VM power state does not change, skip DB writing. vm: {}", entry.getValue().second());
}
private void updateAndPublishVmPowerStates(long hostId, Map<Long, VirtualMachine.PowerState> instancePowerStates,
Date updateTime) {
if (instancePowerStates.isEmpty()) {
return;
}
Set<Long> vmIds = instancePowerStates.keySet();
Map<Long, VirtualMachine.PowerState> notUpdated = _instanceDao.updatePowerState(instancePowerStates, hostId,
updateTime);
if (notUpdated.size() > vmIds.size()) {
return;
}
for (Long vmId : vmIds) {
if (!notUpdated.isEmpty() && !notUpdated.containsKey(vmId)) {
logger.debug("VM state report is updated. {}, {}, power state: {}",
() -> hostCache.get(hostId), () -> vmCache.get(vmId), () -> instancePowerStates.get(vmId));
_messageBus.publish(null, VirtualMachineManager.Topics.VM_POWER_STATE,
PublishScope.GLOBAL, vmId);
continue;
}
logger.trace("VM power state does not change, skip DB writing. {}", () -> vmCache.get(vmId));
}
}
private List<VMInstanceVO> filterOutdatedFromMissingVmReport(List<VMInstanceVO> vmsThatAreMissingReport) {
List<Long> outdatedVms = vmsThatAreMissingReport.stream()
.filter(v -> !_instanceDao.isPowerStateUpToDate(v))
.map(VMInstanceVO::getId)
.collect(Collectors.toList());
if (CollectionUtils.isEmpty(outdatedVms)) {
return vmsThatAreMissingReport;
}
_instanceDao.resetVmPowerStateTracking(outdatedVms);
return vmsThatAreMissingReport.stream()
.filter(v -> !outdatedVms.contains(v.getId()))
.collect(Collectors.toList());
}
private void processMissingVmReport(long hostId, Set<Long> vmIds, boolean force) {
// any state outdates should be checked against the time before this list was retrieved
Date startTime = DateUtil.currentGMTTime();
// for all running/stopping VMs, we provide monitoring of missing report
List<VMInstanceVO> vmsThatAreMissingReport = _instanceDao.findByHostInStates(host.getId(), VirtualMachine.State.Running,
VirtualMachine.State.Stopping, VirtualMachine.State.Starting);
java.util.Iterator<VMInstanceVO> it = vmsThatAreMissingReport.iterator();
while (it.hasNext()) {
VMInstanceVO instance = it.next();
if (translatedInfo.get(instance.getId()) != null)
it.remove();
List<VMInstanceVO> vmsThatAreMissingReport = _instanceDao.findByHostInStatesExcluding(hostId, vmIds,
VirtualMachine.State.Running, VirtualMachine.State.Stopping, VirtualMachine.State.Starting);
// here we need to be wary of out of band migration as opposed to other, more unexpected state changes
if (vmsThatAreMissingReport.isEmpty()) {
return;
}
Date currentTime = DateUtil.currentGMTTime();
logger.debug("Run missing VM report. current time: {}", currentTime.getTime());
if (!force) {
vmsThatAreMissingReport = filterOutdatedFromMissingVmReport(vmsThatAreMissingReport);
}
// here we need to be wary of out of band migration as opposed to other, more unexpected state changes
if (vmsThatAreMissingReport.size() > 0) {
Date currentTime = DateUtil.currentGMTTime();
logger.debug("Run missing VM report for host {}. current time: {}", host, currentTime.getTime());
// 2 times of sync-update interval for graceful period
long milliSecondsGracefullPeriod = mgmtServiceConf.getPingInterval() * 2000L;
for (VMInstanceVO instance : vmsThatAreMissingReport) {
// Make sure powerState is up to date for missing VMs
try {
if (!force && !_instanceDao.isPowerStateUpToDate(instance.getId())) {
logger.warn("Detected missing VM but power state is outdated, wait for another process report run for VM: {}", instance);
_instanceDao.resetVmPowerStateTracking(instance.getId());
continue;
}
} catch (CloudRuntimeException e) {
logger.warn("Checked for missing powerstate of a none existing vm {}", instance, e);
continue;
}
Date vmStateUpdateTime = instance.getPowerStateUpdateTime();
// 2 times of sync-update interval for graceful period
long milliSecondsGracefulPeriod = mgmtServiceConf.getPingInterval() * 2000L;
Map<Long, VirtualMachine.PowerState> instancePowerStates = new HashMap<>();
for (VMInstanceVO instance : vmsThatAreMissingReport) {
Date vmStateUpdateTime = instance.getPowerStateUpdateTime();
if (vmStateUpdateTime == null) {
logger.warn("VM power state update time is null, falling back to update time for {}", instance);
vmStateUpdateTime = instance.getUpdateTime();
if (vmStateUpdateTime == null) {
logger.warn("VM power state update time is null, falling back to update time for vm: {}", instance);
vmStateUpdateTime = instance.getUpdateTime();
if (vmStateUpdateTime == null) {
logger.warn("VM update time is null, falling back to creation time for vm: {}", instance);
vmStateUpdateTime = instance.getCreated();
}
}
String lastTime = new SimpleDateFormat("yyyy/MM/dd'T'HH:mm:ss.SSS'Z'").format(vmStateUpdateTime);
logger.debug("Detected missing VM. host: {}, vm: {}, power state: {}, last state update: {}",
host, instance, VirtualMachine.PowerState.PowerReportMissing, lastTime);
long milliSecondsSinceLastStateUpdate = currentTime.getTime() - vmStateUpdateTime.getTime();
if (force || milliSecondsSinceLastStateUpdate > milliSecondsGracefullPeriod) {
logger.debug("vm: {} - time since last state update({}ms) has passed graceful period", instance, milliSecondsSinceLastStateUpdate);
// this is were a race condition might have happened if we don't re-fetch the instance;
// between the startime of this job and the currentTime of this missing-branch
// an update might have occurred that we should not override in case of out of band migration
if (_instanceDao.updatePowerState(instance.getId(), host.getId(), VirtualMachine.PowerState.PowerReportMissing, startTime)) {
logger.debug("VM state report is updated. host: {}, vm: {}, power state: PowerReportMissing ", host, instance);
_messageBus.publish(null, VirtualMachineManager.Topics.VM_POWER_STATE, PublishScope.GLOBAL, instance.getId());
} else {
logger.debug("VM power state does not change, skip DB writing. vm: {}", instance);
}
} else {
logger.debug("vm: {} - time since last state update({} ms) has not passed graceful period yet", instance, milliSecondsSinceLastStateUpdate);
logger.warn("VM update time is null, falling back to creation time for {}", instance);
vmStateUpdateTime = instance.getCreated();
}
}
logger.debug("Detected missing VM. host: {}, vm id: {}({}), power state: {}, last state update: {}",
hostId,
instance.getId(),
instance.getUuid(),
VirtualMachine.PowerState.PowerReportMissing,
DateUtil.getOutputString(vmStateUpdateTime));
long milliSecondsSinceLastStateUpdate = currentTime.getTime() - vmStateUpdateTime.getTime();
if (force || (milliSecondsSinceLastStateUpdate > milliSecondsGracefulPeriod)) {
logger.debug("vm id: {} - time since last state update({} ms) has passed graceful period",
instance.getId(), milliSecondsSinceLastStateUpdate);
// this is where a race condition might have happened if we don't re-fetch the instance;
// between the startime of this job and the currentTime of this missing-branch
// an update might have occurred that we should not override in case of out of band migration
instancePowerStates.put(instance.getId(), VirtualMachine.PowerState.PowerReportMissing);
} else {
logger.debug("vm id: {} - time since last state update({} ms) has not passed graceful period yet",
instance.getId(), milliSecondsSinceLastStateUpdate);
}
}
logger.debug("Done with process of VM state report. host: {}", host);
updateAndPublishVmPowerStates(hostId, instancePowerStates, startTime);
}
public Map<Long, Pair<VirtualMachine.PowerState, VMInstanceVO>> convertVmStateReport(Map<String, HostVmStateReportEntry> states) {
final HashMap<Long, Pair<VirtualMachine.PowerState, VMInstanceVO>> map = new HashMap<>();
if (states == null) {
private void processReport(long hostId, Map<Long, VirtualMachine.PowerState> translatedInfo, boolean force) {
logger.debug("Process VM state report. {}, number of records in report: {}. VMs: [{}]",
() -> hostCache.get(hostId),
translatedInfo::size,
() -> translatedInfo.entrySet().stream().map(entry -> entry.getKey() + ":" + entry.getValue())
.collect(Collectors.joining(", ")) + "]");
updateAndPublishVmPowerStates(hostId, translatedInfo, DateUtil.currentGMTTime());
processMissingVmReport(hostId, translatedInfo.keySet(), force);
logger.debug("Done with process of VM state report. host: {}", () -> hostCache.get(hostId));
}
public Map<Long, VirtualMachine.PowerState> convertVmStateReport(Map<String, HostVmStateReportEntry> states) {
final HashMap<Long, VirtualMachine.PowerState> map = new HashMap<>();
if (MapUtils.isEmpty(states)) {
return map;
}
Map<String, Long> nameIdMap = _instanceDao.getNameIdMapForVmInstanceNames(states.keySet());
for (Map.Entry<String, HostVmStateReportEntry> entry : states.entrySet()) {
VMInstanceVO vm = findVM(entry.getKey());
if (vm != null) {
map.put(vm.getId(), new Pair<>(entry.getValue().getState(), vm));
Long id = nameIdMap.get(entry.getKey());
if (id != null) {
map.put(id, entry.getValue().getState());
} else {
logger.debug("Unable to find matched VM in CloudStack DB. name: {} powerstate: {}", entry.getKey(), entry.getValue());
}
}
return map;
}
private VMInstanceVO findVM(String vmName) {
return _instanceDao.findVMByInstanceName(vmName);
protected VMInstanceVO getVmFromId(long vmId) {
return _instanceDao.findById(vmId);
}
protected HostVO getHostFromId(long hostId) {
return hostDao.findById(hostId);
}
}

2
engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java
View File

@ -4872,7 +4872,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
@Override
public ConfigKey<?>[] getConfigKeys() {
return new ConfigKey<?>[]{NetworkGcWait, NetworkGcInterval, NetworkLockTimeout,
return new ConfigKey<?>[]{NetworkGcWait, NetworkGcInterval, NetworkLockTimeout, DeniedRoutes,
GuestDomainSuffix, NetworkThrottlingRate, MinVRVersion,
PromiscuousMode, MacAddressChanges, ForgedTransmits, MacLearning, RollingRestartEnabled,
TUNGSTEN_ENABLED, NSX_ENABLED };

2
engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDao.java
View File

@ -28,6 +28,8 @@ import com.cloud.utils.db.GenericDao;
public interface CapacityDao extends GenericDao<CapacityVO, Long> {
CapacityVO findByHostIdType(Long hostId, short capacityType);
List<CapacityVO> listByHostIdTypes(Long hostId, List<Short> capacityTypes);
List<Long> listClustersInZoneOrPodByHostCapacities(long id, long vmId, int requiredCpu, long requiredRam, short capacityTypeForOrdering, boolean isZone);
List<Long> listHostsWithEnoughCapacity(int requiredCpu, long requiredRam, Long clusterId, String hostType);

12
engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDaoImpl.java
View File

@ -671,6 +671,18 @@ public class CapacityDaoImpl extends GenericDaoBase<CapacityVO, Long> implements
return findOneBy(sc);
}
@Override
public List<CapacityVO> listByHostIdTypes(Long hostId, List<Short> capacityTypes) {
SearchBuilder<CapacityVO> sb = createSearchBuilder();
sb.and("hostId", sb.entity().getHostOrPoolId(), SearchCriteria.Op.EQ);
sb.and("type", sb.entity().getCapacityType(), SearchCriteria.Op.IN);
sb.done();
SearchCriteria<CapacityVO> sc = sb.create();
sc.setParameters("hostId", hostId);
sc.setParameters("type", capacityTypes.toArray());
return listBy(sc);
}
@Override
public List<Long> listClustersInZoneOrPodByHostCapacities(long id, long vmId, int requiredCpu, long requiredRam, short capacityTypeForOrdering, boolean isZone) {
TransactionLegacy txn = TransactionLegacy.currentTxn();

3
engine/schema/src/main/java/com/cloud/dc/ClusterDetailsDao.java
View File

@ -16,6 +16,7 @@
// under the License.
package com.cloud.dc;
import java.util.Collection;
import java.util.Map;
import com.cloud.utils.db.GenericDao;
@ -29,6 +30,8 @@ public interface ClusterDetailsDao extends GenericDao<ClusterDetailsVO, Long> {
ClusterDetailsVO findDetail(long clusterId, String name);
Map<String, String> findDetails(long clusterId, Collection<String> names);
void deleteDetails(long clusterId);
String getVmwareDcName(Long clusterId);

20
engine/schema/src/main/java/com/cloud/dc/ClusterDetailsDaoImpl.java
View File

@ -16,13 +16,16 @@
// under the License.
package com.cloud.dc;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.framework.config.ConfigKey.Scope;
import org.apache.cloudstack.framework.config.ScopedConfigStorage;
import org.apache.commons.collections.CollectionUtils;
import com.cloud.utils.crypt.DBEncryptionUtil;
import com.cloud.utils.db.GenericDaoBase;
@ -82,6 +85,23 @@ public class ClusterDetailsDaoImpl extends GenericDaoBase<ClusterDetailsVO, Long
return details;
}
@Override
public Map<String, String> findDetails(long clusterId, Collection<String> names) {
if (CollectionUtils.isEmpty(names)) {
return new HashMap<>();
}
SearchBuilder<ClusterDetailsVO> sb = createSearchBuilder();
sb.and("clusterId", sb.entity().getClusterId(), SearchCriteria.Op.EQ);
sb.and("name", sb.entity().getName(), SearchCriteria.Op.IN);
sb.done();
SearchCriteria<ClusterDetailsVO> sc = sb.create();
sc.setParameters("clusterId", clusterId);
sc.setParameters("name", names.toArray());
List<ClusterDetailsVO> results = search(sc, null);
return results.stream()
.collect(Collectors.toMap(ClusterDetailsVO::getName, ClusterDetailsVO::getValue));
}
@Override
public void deleteDetails(long clusterId) {
SearchCriteria<ClusterDetailsVO> sc = ClusterSearch.create();

16
engine/schema/src/main/java/com/cloud/dc/dao/ClusterDao.java
View File

@ -16,15 +16,15 @@
// under the License.
package com.cloud.dc.dao;
import java.util.List;
import java.util.Map;
import java.util.Set;
import com.cloud.cpu.CPU;
import com.cloud.dc.ClusterVO;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.utils.db.GenericDao;
import java.util.List;
import java.util.Map;
import java.util.Set;
public interface ClusterDao extends GenericDao<ClusterVO, Long> {
List<ClusterVO> listByPodId(long podId);
@ -36,7 +36,7 @@ public interface ClusterDao extends GenericDao<ClusterVO, Long> {
List<HypervisorType> getAvailableHypervisorInZone(Long zoneId);
Set<HypervisorType> getDistictAvailableHypervisorsAcrossClusters();
Set<HypervisorType> getDistinctAvailableHypervisorsAcrossClusters();
List<ClusterVO> listByDcHyType(long dcId, String hyType);
@ -46,9 +46,13 @@ public interface ClusterDao extends GenericDao<ClusterVO, Long> {
List<Long> listClustersWithDisabledPods(long zoneId);
Integer countAllByDcId(long zoneId);
Integer countAllManagedAndEnabledByDcId(long zoneId);
List<ClusterVO> listClustersByDcId(long zoneId);
List<Long> listAllClusters(Long zoneId);
List<Long> listAllClusterIds(Long zoneId);
boolean getSupportsResigning(long clusterId);

83
engine/schema/src/main/java/com/cloud/dc/dao/ClusterDaoImpl.java
View File

@ -16,25 +16,6 @@
// under the License.
package com.cloud.dc.dao;
import com.cloud.cpu.CPU;
import com.cloud.dc.ClusterDetailsDao;
import com.cloud.dc.ClusterDetailsVO;
import com.cloud.dc.ClusterVO;
import com.cloud.dc.HostPodVO;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.org.Grouping;
import com.cloud.utils.db.GenericDaoBase;
import com.cloud.utils.db.GenericSearchBuilder;
import com.cloud.utils.db.JoinBuilder;
import com.cloud.utils.db.SearchBuilder;
import com.cloud.utils.db.SearchCriteria;
import com.cloud.utils.db.SearchCriteria.Func;
import com.cloud.utils.db.SearchCriteria.Op;
import com.cloud.utils.db.TransactionLegacy;
import com.cloud.utils.exception.CloudRuntimeException;
import org.springframework.stereotype.Component;
import javax.inject.Inject;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
@ -46,6 +27,28 @@ import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import javax.inject.Inject;
import org.springframework.stereotype.Component;
import com.cloud.cpu.CPU;
import com.cloud.dc.ClusterDetailsDao;
import com.cloud.dc.ClusterDetailsVO;
import com.cloud.dc.ClusterVO;
import com.cloud.dc.HostPodVO;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.org.Grouping;
import com.cloud.org.Managed;
import com.cloud.utils.db.GenericDaoBase;
import com.cloud.utils.db.GenericSearchBuilder;
import com.cloud.utils.db.JoinBuilder;
import com.cloud.utils.db.SearchBuilder;
import com.cloud.utils.db.SearchCriteria;
import com.cloud.utils.db.SearchCriteria.Func;
import com.cloud.utils.db.SearchCriteria.Op;
import com.cloud.utils.db.TransactionLegacy;
import com.cloud.utils.exception.CloudRuntimeException;
@Component
public class ClusterDaoImpl extends GenericDaoBase<ClusterVO, Long> implements ClusterDao {
@ -58,7 +61,6 @@ public class ClusterDaoImpl extends GenericDaoBase<ClusterVO, Long> implements C
protected final SearchBuilder<ClusterVO> ClusterSearch;
protected final SearchBuilder<ClusterVO> ClusterDistinctArchSearch;
protected final SearchBuilder<ClusterVO> ClusterArchSearch;
protected GenericSearchBuilder<ClusterVO, Long> ClusterIdSearch;
private static final String GET_POD_CLUSTER_MAP_PREFIX = "SELECT pod_id, id FROM cloud.cluster WHERE cluster.id IN( ";
@ -98,6 +100,8 @@ public class ClusterDaoImpl extends GenericDaoBase<ClusterVO, Long> implements C
ZoneClusterSearch = createSearchBuilder();
ZoneClusterSearch.and("dataCenterId", ZoneClusterSearch.entity().getDataCenterId(), SearchCriteria.Op.EQ);
ZoneClusterSearch.and("allocationState", ZoneClusterSearch.entity().getAllocationState(), Op.EQ);
ZoneClusterSearch.and("managedState", ZoneClusterSearch.entity().getManagedState(), Op.EQ);
ZoneClusterSearch.done();
ClusterIdSearch = createSearchBuilder(Long.class);
@ -167,23 +171,15 @@ public class ClusterDaoImpl extends GenericDaoBase<ClusterVO, Long> implements C
sc.setParameters("zoneId", zoneId);
}
List<ClusterVO> clusters = listBy(sc);
List<HypervisorType> hypers = new ArrayList<HypervisorType>(4);
for (ClusterVO cluster : clusters) {
hypers.add(cluster.getHypervisorType());
}
return hypers;
return clusters.stream()
.map(ClusterVO::getHypervisorType)
.distinct()
.collect(Collectors.toList());
}
@Override
public Set<HypervisorType> getDistictAvailableHypervisorsAcrossClusters() {
SearchCriteria<ClusterVO> sc = ClusterSearch.create();
List<ClusterVO> clusters = listBy(sc);
Set<HypervisorType> hypers = new HashSet<>();
for (ClusterVO cluster : clusters) {
hypers.add(cluster.getHypervisorType());
}
return hypers;
public Set<HypervisorType> getDistinctAvailableHypervisorsAcrossClusters() {
return new HashSet<>(getAvailableHypervisorInZone(null));
}
@Override
@ -266,6 +262,23 @@ public class ClusterDaoImpl extends GenericDaoBase<ClusterVO, Long> implements C
return customSearch(sc, null);
}
@Override
public Integer countAllByDcId(long zoneId) {
SearchCriteria<ClusterVO> sc = ZoneClusterSearch.create();
sc.setParameters("dataCenterId", zoneId);
return getCount(sc);
}
@Override
public Integer countAllManagedAndEnabledByDcId(long zoneId) {
SearchCriteria<ClusterVO> sc = ZoneClusterSearch.create();
sc.setParameters("dataCenterId", zoneId);
sc.setParameters("allocationState", Grouping.AllocationState.Enabled);
sc.setParameters("managedState", Managed.ManagedState.Managed);
return getCount(sc);
}
@Override
public List<ClusterVO> listClustersByDcId(long zoneId) {
SearchCriteria<ClusterVO> sc = ZoneClusterSearch.create();
@ -289,7 +302,7 @@ public class ClusterDaoImpl extends GenericDaoBase<ClusterVO, Long> implements C
}
@Override
public List<Long> listAllClusters(Long zoneId) {
public List<Long> listAllClusterIds(Long zoneId) {
SearchCriteria<Long> sc = ClusterIdSearch.create();
if (zoneId != null) {
sc.setParameters("dataCenterId", zoneId);

3
engine/schema/src/main/java/com/cloud/dc/dao/DataCenterIpAddressDaoImpl.java
View File

@ -294,8 +294,7 @@ public class DataCenterIpAddressDaoImpl extends GenericDaoBase<DataCenterIpAddre
sc.addAnd("podId", SearchCriteria.Op.EQ, podId);
sc.addAnd("dataCenterId", SearchCriteria.Op.EQ, dcId);
List<DataCenterIpAddressVO> result = listBy(sc);
return result.size();
return getCount(sc);
}
public DataCenterIpAddressDaoImpl() {

2
engine/schema/src/main/java/com/cloud/dc/dao/DataCenterVnetDaoImpl.java
View File

@ -81,7 +81,7 @@ public class DataCenterVnetDaoImpl extends GenericDaoBase<DataCenterVnetVO, Long
public int countAllocatedVnets(long physicalNetworkId) {
SearchCriteria<DataCenterVnetVO> sc = DcSearchAllocated.create();
sc.setParameters("physicalNetworkId", physicalNetworkId);
return listBy(sc).size();
return getCount(sc);
}
@Override

46
engine/schema/src/main/java/com/cloud/host/dao/HostDao.java
View File

@ -27,6 +27,7 @@ import com.cloud.hypervisor.Hypervisor;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.info.RunningHostCountInfo;
import com.cloud.resource.ResourceState;
import com.cloud.utils.Pair;
import com.cloud.utils.db.GenericDao;
import com.cloud.utils.fsm.StateDao;
@ -39,8 +40,14 @@ public interface HostDao extends GenericDao<HostVO, Long>, StateDao<Status, Stat
Integer countAllByType(final Host.Type type);
Integer countAllInClusterByTypeAndStates(Long clusterId, final Host.Type type, List<Status> status);
Integer countAllByTypeInZone(long zoneId, final Host.Type type);
Integer countUpAndEnabledHostsInZone(long zoneId);
Pair<Integer, Integer> countAllHostsAndCPUSocketsByType(Type type);
/**
* Mark all hosts associated with a certain management server
* as disconnected.
@ -75,32 +82,41 @@ public interface HostDao extends GenericDao<HostVO, Long>, StateDao<Status, Stat
List<HostVO> findHypervisorHostInCluster(long clusterId);
HostVO findAnyStateHypervisorHostInCluster(long clusterId);
HostVO findOldestExistentHypervisorHostInCluster(long clusterId);
List<HostVO> listAllUpAndEnabledNonHAHosts(Type type, Long clusterId, Long podId, long dcId, String haTag);
List<HostVO> findByDataCenterId(Long zoneId);
List<Long> listIdsByDataCenterId(Long zoneId);
List<HostVO> findByPodId(Long podId);
List<Long> listIdsByPodId(Long podId);
List<HostVO> findByClusterId(Long clusterId);
List<Long> listIdsByClusterId(Long clusterId);
List<Long> listIdsForUpRouting(Long zoneId, Long podId, Long clusterId);
List<Long> listIdsByType(Type type);
List<Long> listIdsForUpEnabledByZoneAndHypervisor(Long zoneId, HypervisorType hypervisorType);
List<HostVO> findByClusterIdAndEncryptionSupport(Long clusterId);
/**
* Returns hosts that are 'Up' and 'Enabled' from the given Data Center/Zone
* Returns host Ids that are 'Up' and 'Enabled' from the given Data Center/Zone
*/
List<HostVO> listByDataCenterId(long id);
List<Long> listEnabledIdsByDataCenterId(long id);
/**
* Returns hosts that are from the given Data Center/Zone and at a given state (e.g. Creating, Enabled, Disabled, etc).
* Returns host Ids that are 'Up' and 'Disabled' from the given Data Center/Zone
*/
List<HostVO> listByDataCenterIdAndState(long id, ResourceState state);
/**
* Returns hosts that are 'Up' and 'Disabled' from the given Data Center/Zone
*/
List<HostVO> listDisabledByDataCenterId(long id);
List<Long> listDisabledIdsByDataCenterId(long id);
List<HostVO> listByDataCenterIdAndHypervisorType(long zoneId, Hypervisor.HypervisorType hypervisorType);
@ -110,8 +126,6 @@ public interface HostDao extends GenericDao<HostVO, Long>, StateDao<Status, Stat
List<HostVO> listAllHostsThatHaveNoRuleTag(Host.Type type, Long clusterId, Long podId, Long dcId);
List<HostVO> listAllHostsByType(Host.Type type);
HostVO findByPublicIp(String publicIp);
List<Long> listClustersByHostTag(String hostTagOnOffering);
@ -182,4 +196,14 @@ public interface HostDao extends GenericDao<HostVO, Long>, StateDao<Status, Stat
List<Long> findClustersThatMatchHostTagRule(String computeOfferingTags);
List<Long> listSsvmHostsWithPendingMigrateJobsOrderedByJobCount();
boolean isHostUp(long hostId);
List<Long> findHostIdsByZoneClusterResourceStateTypeAndHypervisorType(final Long zoneId, final Long clusterId,
final List<ResourceState> resourceStates, final List<Type> types,
final List<Hypervisor.HypervisorType> hypervisorTypes);
List<HypervisorType> listDistinctHypervisorTypes(final Long zoneId);
List<HostVO> listByIds(final List<Long> ids);
}

503
engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java
View File

@ -20,6 +20,7 @@ import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
@ -45,8 +46,8 @@ import com.cloud.dc.ClusterVO;
import com.cloud.dc.dao.ClusterDao;
import com.cloud.gpu.dao.HostGpuGroupsDao;
import com.cloud.gpu.dao.VGPUTypesDao;
import com.cloud.host.Host;
import com.cloud.host.DetailVO;
import com.cloud.host.Host;
import com.cloud.host.Host.Type;
import com.cloud.host.HostTagVO;
import com.cloud.host.HostVO;
@ -59,6 +60,8 @@ import com.cloud.org.Grouping;
import com.cloud.org.Managed;
import com.cloud.resource.ResourceState;
import com.cloud.utils.DateUtil;
import com.cloud.utils.Pair;
import com.cloud.utils.StringUtils;
import com.cloud.utils.db.Attribute;
import com.cloud.utils.db.DB;
import com.cloud.utils.db.Filter;
@ -74,19 +77,17 @@ import com.cloud.utils.db.TransactionLegacy;
import com.cloud.utils.db.UpdateBuilder;
import com.cloud.utils.exception.CloudRuntimeException;
import java.util.Arrays;
@DB
@TableGenerator(name = "host_req_sq", table = "op_host", pkColumnName = "id", valueColumnName = "sequence", allocationSize = 1)
public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao { //FIXME: , ExternalIdDao {
private static final String LIST_HOST_IDS_BY_COMPUTETAGS = "SELECT filtered.host_id, COUNT(filtered.tag) AS tag_count "
+ "FROM (SELECT host_id, tag, is_tag_a_rule FROM host_tags GROUP BY host_id,tag) AS filtered "
+ "WHERE tag IN(%s) AND is_tag_a_rule = 0 "
private static final String LIST_HOST_IDS_BY_HOST_TAGS = "SELECT filtered.host_id, COUNT(filtered.tag) AS tag_count "
+ "FROM (SELECT host_id, tag, is_tag_a_rule FROM host_tags GROUP BY host_id,tag,is_tag_a_rule) AS filtered "
+ "WHERE tag IN (%s) AND (is_tag_a_rule = 0 OR is_tag_a_rule IS NULL) "
+ "GROUP BY host_id "
+ "HAVING tag_count = %s ";
private static final String SEPARATOR = ",";
private static final String LIST_CLUSTERID_FOR_HOST_TAG = "select distinct cluster_id from host join ( %s ) AS selected_hosts ON host.id = selected_hosts.host_id";
private static final String LIST_CLUSTER_IDS_FOR_HOST_TAGS = "select distinct cluster_id from host join ( %s ) AS selected_hosts ON host.id = selected_hosts.host_id";
private static final String GET_HOSTS_OF_ACTIVE_VMS = "select h.id " +
"from vm_instance vm " +
"join host h on (vm.host_id=h.id) " +
@ -98,6 +99,7 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
protected SearchBuilder<HostVO> TypePodDcStatusSearch;
protected SearchBuilder<HostVO> IdsSearch;
protected SearchBuilder<HostVO> IdStatusSearch;
protected SearchBuilder<HostVO> TypeDcSearch;
protected SearchBuilder<HostVO> TypeDcStatusSearch;
@ -127,6 +129,7 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
protected SearchBuilder<HostVO> ResponsibleMsSearch;
protected SearchBuilder<HostVO> ResponsibleMsDcSearch;
protected GenericSearchBuilder<HostVO, String> ResponsibleMsIdSearch;
protected SearchBuilder<HostVO> HostTypeClusterCountSearch;
protected SearchBuilder<HostVO> HostTypeZoneCountSearch;
protected SearchBuilder<HostVO> ClusterStatusSearch;
protected SearchBuilder<HostVO> TypeNameZoneSearch;
@ -138,8 +141,7 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
protected SearchBuilder<HostVO> ManagedRoutingServersSearch;
protected SearchBuilder<HostVO> SecondaryStorageVMSearch;
protected GenericSearchBuilder<HostVO, Long> HostIdSearch;
protected GenericSearchBuilder<HostVO, Long> HostsInStatusSearch;
protected GenericSearchBuilder<HostVO, Long> HostsInStatusesSearch;
protected GenericSearchBuilder<HostVO, Long> CountRoutingByDc;
protected SearchBuilder<HostTransferMapVO> HostTransferSearch;
protected SearchBuilder<ClusterVO> ClusterManagedSearch;
@ -189,6 +191,8 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
HostTypeCountSearch = createSearchBuilder();
HostTypeCountSearch.and("type", HostTypeCountSearch.entity().getType(), SearchCriteria.Op.EQ);
HostTypeCountSearch.and("zoneId", HostTypeCountSearch.entity().getDataCenterId(), SearchCriteria.Op.EQ);
HostTypeCountSearch.and("resourceState", HostTypeCountSearch.entity().getResourceState(), SearchCriteria.Op.EQ);
HostTypeCountSearch.done();
ResponsibleMsSearch = createSearchBuilder();
@ -205,6 +209,13 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
ResponsibleMsIdSearch.and("managementServerId", ResponsibleMsIdSearch.entity().getManagementServerId(), SearchCriteria.Op.EQ);
ResponsibleMsIdSearch.done();
HostTypeClusterCountSearch = createSearchBuilder();
HostTypeClusterCountSearch.and("cluster", HostTypeClusterCountSearch.entity().getClusterId(), SearchCriteria.Op.EQ);
HostTypeClusterCountSearch.and("type", HostTypeClusterCountSearch.entity().getType(), SearchCriteria.Op.EQ);
HostTypeClusterCountSearch.and("status", HostTypeClusterCountSearch.entity().getStatus(), SearchCriteria.Op.IN);
HostTypeClusterCountSearch.and("removed", HostTypeClusterCountSearch.entity().getRemoved(), SearchCriteria.Op.NULL);
HostTypeClusterCountSearch.done();
HostTypeZoneCountSearch = createSearchBuilder();
HostTypeZoneCountSearch.and("type", HostTypeZoneCountSearch.entity().getType(), SearchCriteria.Op.EQ);
HostTypeZoneCountSearch.and("dc", HostTypeZoneCountSearch.entity().getDataCenterId(), SearchCriteria.Op.EQ);
@ -252,6 +263,10 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
TypeClusterStatusSearch.and("resourceState", TypeClusterStatusSearch.entity().getResourceState(), SearchCriteria.Op.EQ);
TypeClusterStatusSearch.done();
IdsSearch = createSearchBuilder();
IdsSearch.and("id", IdsSearch.entity().getId(), SearchCriteria.Op.IN);
IdsSearch.done();
IdStatusSearch = createSearchBuilder();
IdStatusSearch.and("id", IdStatusSearch.entity().getId(), SearchCriteria.Op.EQ);
IdStatusSearch.and("states", IdStatusSearch.entity().getStatus(), SearchCriteria.Op.IN);
@ -398,14 +413,14 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
AvailHypevisorInZone.groupBy(AvailHypevisorInZone.entity().getHypervisorType());
AvailHypevisorInZone.done();
HostsInStatusSearch = createSearchBuilder(Long.class);
HostsInStatusSearch.selectFields(HostsInStatusSearch.entity().getId());
HostsInStatusSearch.and("dc", HostsInStatusSearch.entity().getDataCenterId(), Op.EQ);
HostsInStatusSearch.and("pod", HostsInStatusSearch.entity().getPodId(), Op.EQ);
HostsInStatusSearch.and("cluster", HostsInStatusSearch.entity().getClusterId(), Op.EQ);
HostsInStatusSearch.and("type", HostsInStatusSearch.entity().getType(), Op.EQ);
HostsInStatusSearch.and("statuses", HostsInStatusSearch.entity().getStatus(), Op.IN);
HostsInStatusSearch.done();
HostsInStatusesSearch = createSearchBuilder(Long.class);
HostsInStatusesSearch.selectFields(HostsInStatusesSearch.entity().getId());
HostsInStatusesSearch.and("dc", HostsInStatusesSearch.entity().getDataCenterId(), Op.EQ);
HostsInStatusesSearch.and("pod", HostsInStatusesSearch.entity().getPodId(), Op.EQ);
HostsInStatusesSearch.and("cluster", HostsInStatusesSearch.entity().getClusterId(), Op.EQ);
HostsInStatusesSearch.and("type", HostsInStatusesSearch.entity().getType(), Op.EQ);
HostsInStatusesSearch.and("statuses", HostsInStatusesSearch.entity().getStatus(), Op.IN);
HostsInStatusesSearch.done();
CountRoutingByDc = createSearchBuilder(Long.class);
CountRoutingByDc.select(null, Func.COUNT, null);
@ -468,11 +483,6 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
HostsInClusterSearch.and("server", HostsInClusterSearch.entity().getManagementServerId(), SearchCriteria.Op.NNULL);
HostsInClusterSearch.done();
HostIdSearch = createSearchBuilder(Long.class);
HostIdSearch.selectFields(HostIdSearch.entity().getId());
HostIdSearch.and("dataCenterId", HostIdSearch.entity().getDataCenterId(), Op.EQ);
HostIdSearch.done();
searchBuilderFindByRuleTag = _hostTagsDao.createSearchBuilder();
searchBuilderFindByRuleTag.and("is_tag_a_rule", searchBuilderFindByRuleTag.entity().getIsTagARule(), Op.EQ);
searchBuilderFindByRuleTag.or("tagDoesNotExist", searchBuilderFindByRuleTag.entity().getIsTagARule(), Op.NULL);
@ -504,8 +514,7 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
sc.setParameters("resourceState", (Object[])states);
sc.setParameters("cluster", clusterId);
List<HostVO> hosts = listBy(sc);
return hosts.size();
return getCount(sc);
}
@Override
@ -516,36 +525,62 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
}
@Override
public Integer countAllByTypeInZone(long zoneId, Type type) {
SearchCriteria<HostVO> sc = HostTypeCountSearch.create();
sc.setParameters("type", type);
sc.setParameters("dc", zoneId);
public Integer countAllInClusterByTypeAndStates(Long clusterId, final Host.Type type, List<Status> status) {
SearchCriteria<HostVO> sc = HostTypeClusterCountSearch.create();
if (clusterId != null) {
sc.setParameters("cluster", clusterId);
}
if (type != null) {
sc.setParameters("type", type);
}
if (status != null) {
sc.setParameters("status", status.toArray());
}
return getCount(sc);
}
@Override
public List<HostVO> listByDataCenterId(long id) {
return listByDataCenterIdAndState(id, ResourceState.Enabled);
public Integer countAllByTypeInZone(long zoneId, Type type) {
SearchCriteria<HostVO> sc = HostTypeCountSearch.create();
sc.setParameters("type", type);
sc.setParameters("zoneId", zoneId);
return getCount(sc);
}
@Override
public List<HostVO> listByDataCenterIdAndState(long id, ResourceState state) {
SearchCriteria<HostVO> sc = scHostsFromZoneUpRouting(id);
sc.setParameters("resourceState", state);
return listBy(sc);
public Integer countUpAndEnabledHostsInZone(long zoneId) {
SearchCriteria<HostVO> sc = HostTypeCountSearch.create();
sc.setParameters("type", Type.Routing);
sc.setParameters("resourceState", ResourceState.Enabled);
sc.setParameters("zoneId", zoneId);
return getCount(sc);
}
@Override
public List<HostVO> listDisabledByDataCenterId(long id) {
return listByDataCenterIdAndState(id, ResourceState.Disabled);
public Pair<Integer, Integer> countAllHostsAndCPUSocketsByType(Type type) {
GenericSearchBuilder<HostVO, SumCount> sb = createSearchBuilder(SumCount.class);
sb.select("sum", Func.SUM, sb.entity().getCpuSockets());
sb.select("count", Func.COUNT, null);
sb.and("type", sb.entity().getType(), SearchCriteria.Op.EQ);
sb.done();
SearchCriteria<SumCount> sc = sb.create();
sc.setParameters("type", type);
SumCount result = customSearch(sc, null).get(0);
return new Pair<>((int)result.count, (int)result.sum);
}
private SearchCriteria<HostVO> scHostsFromZoneUpRouting(long id) {
SearchCriteria<HostVO> sc = DcSearch.create();
sc.setParameters("dc", id);
sc.setParameters("status", Status.Up);
sc.setParameters("type", Host.Type.Routing);
return sc;
private List<Long> listIdsForRoutingByZoneIdAndResourceState(long zoneId, ResourceState state) {
return listIdsBy(Type.Routing, Status.Up, state, null, zoneId, null, null);
}
@Override
public List<Long> listEnabledIdsByDataCenterId(long id) {
return listIdsForRoutingByZoneIdAndResourceState(id, ResourceState.Enabled);
}
@Override
public List<Long> listDisabledIdsByDataCenterId(long id) {
return listIdsForRoutingByZoneIdAndResourceState(id, ResourceState.Disabled);
}
@Override
@ -603,9 +638,7 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
sb.append(" ");
}
if (logger.isTraceEnabled()) {
logger.trace("Following hosts got reset: " + sb.toString());
}
logger.trace("Following hosts got reset: {}", sb);
}
/*
@ -615,8 +648,7 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
SearchCriteria<Long> sc = ClustersOwnedByMSSearch.create();
sc.setParameters("server", managementServerId);
List<Long> clusters = customSearch(sc, null);
return clusters;
return customSearch(sc, null);
}
/*
@ -626,13 +658,11 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
SearchCriteria<Long> sc = ClustersForHostsNotOwnedByAnyMSSearch.create();
sc.setJoinParameters("ClusterManagedSearch", "managed", Managed.ManagedState.Managed);
List<Long> clusters = customSearch(sc, null);
return clusters;
return customSearch(sc, null);
}
/**
* This determines if hosts belonging to cluster(@clusterId) are up for grabs
*
* This is used for handling following cases:
* 1. First host added in cluster
* 2. During MS restart all hosts in a cluster are without any MS
@ -642,9 +672,7 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
sc.setParameters("cluster", clusterId);
List<HostVO> hosts = search(sc, null);
boolean ownCluster = (hosts == null || hosts.size() == 0);
return ownCluster;
return (hosts == null || hosts.isEmpty());
}
@Override
@ -661,14 +689,14 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
logger.debug("Completed resetting hosts suitable for reconnect");
}
List<HostVO> assignedHosts = new ArrayList<HostVO>();
List<HostVO> assignedHosts = new ArrayList<>();
if (logger.isDebugEnabled()) {
logger.debug("Acquiring hosts for clusters already owned by this management server");
}
List<Long> clusters = findClustersOwnedByManagementServer(managementServerId);
txn.start();
if (clusters.size() > 0) {
if (!clusters.isEmpty()) {
// handle clusters already owned by @managementServerId
SearchCriteria<HostVO> sc = UnmanagedDirectConnectSearch.create();
sc.setParameters("lastPinged", lastPingSecondsAfter);
@ -683,13 +711,9 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
sb.append(host.getId());
sb.append(" ");
}
if (logger.isTraceEnabled()) {
logger.trace("Following hosts got acquired for clusters already owned: " + sb.toString());
}
}
if (logger.isDebugEnabled()) {
logger.debug("Completed acquiring hosts for clusters already owned by this management server");
logger.trace("Following hosts got acquired for clusters already owned: {}", sb);
}
logger.debug("Completed acquiring hosts for clusters already owned by this management server");
if (assignedHosts.size() < limit) {
if (logger.isDebugEnabled()) {
@ -701,7 +725,7 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
if (clusters.size() > limit) {
updatedClusters = clusters.subList(0, limit.intValue());
}
if (updatedClusters.size() > 0) {
if (!updatedClusters.isEmpty()) {
SearchCriteria<HostVO> sc = UnmanagedDirectConnectSearch.create();
sc.setParameters("lastPinged", lastPingSecondsAfter);
sc.setJoinParameters("ClusterManagedSearch", "managed", Managed.ManagedState.Managed);
@ -709,10 +733,10 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
List<HostVO> unmanagedHosts = lockRows(sc, null, true);
// group hosts based on cluster
Map<Long, List<HostVO>> hostMap = new HashMap<Long, List<HostVO>>();
Map<Long, List<HostVO>> hostMap = new HashMap<>();
for (HostVO host : unmanagedHosts) {
if (hostMap.get(host.getClusterId()) == null) {
hostMap.put(host.getClusterId(), new ArrayList<HostVO>());
hostMap.put(host.getClusterId(), new ArrayList<>());
}
hostMap.get(host.getClusterId()).add(host);
}
@ -733,13 +757,9 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
break;
}
}
if (logger.isTraceEnabled()) {
logger.trace("Following hosts got acquired from newly owned clusters: " + sb.toString());
}
}
if (logger.isDebugEnabled()) {
logger.debug("Completed acquiring hosts for clusters not owned by any management server");
logger.trace("Following hosts got acquired from newly owned clusters: {}", sb);
}
logger.debug("Completed acquiring hosts for clusters not owned by any management server");
}
txn.commit();
@ -794,6 +814,15 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
@Override
public List<HostVO> listByHostTag(Host.Type type, Long clusterId, Long podId, Long dcId, String hostTag) {
return listHostsWithOrWithoutHostTags(type, clusterId, podId, dcId, hostTag, true);
}
private List<HostVO> listHostsWithOrWithoutHostTags(Host.Type type, Long clusterId, Long podId, Long dcId, String hostTags, boolean withHostTags) {
if (StringUtils.isEmpty(hostTags)) {
logger.debug("Host tags not specified, to list hosts");
return new ArrayList<>();
}
SearchBuilder<HostVO> hostSearch = createSearchBuilder();
HostVO entity = hostSearch.entity();
hostSearch.and("type", entity.getType(), SearchCriteria.Op.EQ);
@ -804,7 +833,9 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
hostSearch.and("resourceState", entity.getResourceState(), SearchCriteria.Op.EQ);
SearchCriteria<HostVO> sc = hostSearch.create();
sc.setParameters("type", type.toString());
if (type != null) {
sc.setParameters("type", type.toString());
}
if (podId != null) {
sc.setParameters("pod", podId);
}
@ -817,27 +848,38 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
sc.setParameters("status", Status.Up.toString());
sc.setParameters("resourceState", ResourceState.Enabled.toString());
List<HostVO> tmpHosts = listBy(sc);
List<HostVO> correctHostsByHostTags = new ArrayList();
List<Long> hostIdsByComputeOffTags = findHostByComputeOfferings(hostTag);
List<HostVO> upAndEnabledHosts = listBy(sc);
if (CollectionUtils.isEmpty(upAndEnabledHosts)) {
return new ArrayList<>();
}
tmpHosts.forEach((host) -> { if(hostIdsByComputeOffTags.contains(host.getId())) correctHostsByHostTags.add(host);});
List<Long> hostIdsByHostTags = findHostIdsByHostTags(hostTags);
if (CollectionUtils.isEmpty(hostIdsByHostTags)) {
return withHostTags ? new ArrayList<>() : upAndEnabledHosts;
}
return correctHostsByHostTags;
if (withHostTags) {
List<HostVO> upAndEnabledHostsWithHostTags = new ArrayList<>();
upAndEnabledHosts.forEach((host) -> { if (hostIdsByHostTags.contains(host.getId())) upAndEnabledHostsWithHostTags.add(host);});
return upAndEnabledHostsWithHostTags;
} else {
List<HostVO> upAndEnabledHostsWithoutHostTags = new ArrayList<>();
upAndEnabledHosts.forEach((host) -> { if (!hostIdsByHostTags.contains(host.getId())) upAndEnabledHostsWithoutHostTags.add(host);});
return upAndEnabledHostsWithoutHostTags;
}
}
@Override
public List<HostVO> listAllUpAndEnabledNonHAHosts(Type type, Long clusterId, Long podId, long dcId, String haTag) {
if (StringUtils.isNotEmpty(haTag)) {
return listHostsWithOrWithoutHostTags(type, clusterId, podId, dcId, haTag, false);
}
SearchBuilder<HostTagVO> hostTagSearch = _hostTagsDao.createSearchBuilder();
hostTagSearch.and();
hostTagSearch.op("isTagARule", hostTagSearch.entity().getIsTagARule(), Op.EQ);
hostTagSearch.or("tagDoesNotExist", hostTagSearch.entity().getIsTagARule(), Op.NULL);
hostTagSearch.cp();
if (haTag != null && !haTag.isEmpty()) {
hostTagSearch.and().op("tag", hostTagSearch.entity().getTag(), SearchCriteria.Op.NEQ);
hostTagSearch.or("tagNull", hostTagSearch.entity().getTag(), SearchCriteria.Op.NULL);
hostTagSearch.cp();
}
SearchBuilder<HostVO> hostSearch = createSearchBuilder();
@ -848,18 +890,12 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
hostSearch.and("status", hostSearch.entity().getStatus(), SearchCriteria.Op.EQ);
hostSearch.and("resourceState", hostSearch.entity().getResourceState(), SearchCriteria.Op.EQ);
hostSearch.join("hostTagSearch", hostTagSearch, hostSearch.entity().getId(), hostTagSearch.entity().getHostId(), JoinBuilder.JoinType.LEFTOUTER);
SearchCriteria<HostVO> sc = hostSearch.create();
sc.setJoinParameters("hostTagSearch", "isTagARule", false);
if (haTag != null && !haTag.isEmpty()) {
sc.setJoinParameters("hostTagSearch", "tag", haTag);
}
if (type != null) {
sc.setParameters("type", type);
}
@ -899,12 +935,12 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
@DB
@Override
public List<HostVO> findLostHosts(long timeout) {
List<HostVO> result = new ArrayList<HostVO>();
List<HostVO> result = new ArrayList<>();
String sql = "select h.id from host h left join cluster c on h.cluster_id=c.id where h.mgmt_server_id is not null and h.last_ping < ? and h.status in ('Up', 'Updating', 'Disconnected', 'Connecting') and h.type not in ('ExternalFirewall', 'ExternalLoadBalancer', 'TrafficMonitor', 'SecondaryStorage', 'LocalSecondaryStorage', 'L2Networking') and (h.cluster_id is null or c.managed_state = 'Managed') ;";
try (TransactionLegacy txn = TransactionLegacy.currentTxn();
PreparedStatement pstmt = txn.prepareStatement(sql);) {
PreparedStatement pstmt = txn.prepareStatement(sql)) {
pstmt.setLong(1, timeout);
try (ResultSet rs = pstmt.executeQuery();) {
try (ResultSet rs = pstmt.executeQuery()) {
while (rs.next()) {
long id = rs.getLong(1); //ID column
result.add(findById(id));
@ -937,7 +973,7 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
HashMap<String, HashMap<String, VgpuTypesInfo>> groupDetails = host.getGpuGroupDetails();
if (groupDetails != null) {
// Create/Update GPU group entries
_hostGpuGroupsDao.persist(host.getId(), new ArrayList<String>(groupDetails.keySet()));
_hostGpuGroupsDao.persist(host.getId(), new ArrayList<>(groupDetails.keySet()));
// Create/Update VGPU types entries
_vgpuTypesDao.persist(host.getId(), groupDetails);
}
@ -980,7 +1016,7 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
boolean persisted = super.update(hostId, host);
if (!persisted) {
return persisted;
return false;
}
saveDetails(host);
@ -989,7 +1025,7 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
txn.commit();
return persisted;
return true;
}
@Override
@ -1000,11 +1036,10 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
+ "select h.data_center_id, h.type, count(*) as count from host as h INNER JOIN mshost as m ON h.mgmt_server_id=m.msid "
+ "where h.status='Up' and h.type='Routing' and m.last_update > ? " + "group by h.data_center_id, h.type) as t " + "ORDER by t.data_center_id, t.type";
ArrayList<RunningHostCountInfo> l = new ArrayList<RunningHostCountInfo>();
ArrayList<RunningHostCountInfo> l = new ArrayList<>();
TransactionLegacy txn = TransactionLegacy.currentTxn();
;
PreparedStatement pstmt = null;
PreparedStatement pstmt;
try {
pstmt = txn.prepareAutoCloseStatement(sql);
String gmtCutTime = DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutTime);
@ -1028,9 +1063,7 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
@Override
public long getNextSequence(long hostId) {
if (logger.isTraceEnabled()) {
logger.trace("getNextSequence(), hostId: " + hostId);
}
logger.trace("getNextSequence(), hostId: {}", hostId);
TableGenerator tg = _tgs.get("host_req_sq");
assert tg != null : "how can this be wrong!";
@ -1099,31 +1132,30 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
HostVO ho = findById(host.getId());
assert ho != null : "How how how? : " + host.getId();
// TODO handle this if(debug){}else{log.debug} it makes no sense
if (logger.isDebugEnabled()) {
StringBuilder str = new StringBuilder("Unable to update host for event:").append(event.toString());
str.append(". Name=").append(host.getName());
str.append("; New=[status=").append(newStatus.toString()).append(":msid=").append(newStatus.lostConnection() ? "null" : host.getManagementServerId())
.append(":lastpinged=").append(host.getLastPinged()).append("]");
str.append("; Old=[status=").append(oldStatus.toString()).append(":msid=").append(host.getManagementServerId()).append(":lastpinged=").append(oldPingTime)
.append("]");
str.append("; DB=[status=").append(vo.getStatus().toString()).append(":msid=").append(vo.getManagementServerId()).append(":lastpinged=").append(vo.getLastPinged())
.append(":old update count=").append(oldUpdateCount).append("]");
logger.debug(str.toString());
String str = "Unable to update host for event:" + event +
". Name=" + host.getName() +
"; New=[status=" + newStatus + ":msid=" + (newStatus.lostConnection() ? "null" : host.getManagementServerId()) +
":lastpinged=" + host.getLastPinged() + "]" +
"; Old=[status=" + oldStatus.toString() + ":msid=" + host.getManagementServerId() + ":lastpinged=" + oldPingTime +
"]" +
"; DB=[status=" + vo.getStatus().toString() + ":msid=" + vo.getManagementServerId() + ":lastpinged=" + vo.getLastPinged() +
":old update count=" + oldUpdateCount + "]";
logger.debug(str);
} else {
StringBuilder msg = new StringBuilder("Agent status update: [");
msg.append("id = " + host.getId());
msg.append("; name = " + host.getName());
msg.append("; old status = " + oldStatus);
msg.append("; event = " + event);
msg.append("; new status = " + newStatus);
msg.append("; old update count = " + oldUpdateCount);
msg.append("; new update count = " + newUpdateCount + "]");
logger.debug(msg.toString());
String msg = "Agent status update: [" + "id = " + host.getId() +
"; name = " + host.getName() +
"; old status = " + oldStatus +
"; event = " + event +
"; new status = " + newStatus +
"; old update count = " + oldUpdateCount +
"; new update count = " + newUpdateCount + "]";
logger.debug(msg);
}
if (ho.getState() == newStatus) {
logger.debug("Host " + ho.getName() + " state has already been updated to " + newStatus);
logger.debug("Host {} state has already been updated to {}", ho.getName(), newStatus);
return true;
}
}
@ -1149,25 +1181,24 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
int result = update(ub, sc, null);
assert result <= 1 : "How can this update " + result + " rows? ";
// TODO handle this if(debug){}else{log.debug} it makes no sense
if (logger.isDebugEnabled() && result == 0) {
HostVO ho = findById(host.getId());
assert ho != null : "How how how? : " + host.getId();
StringBuilder str = new StringBuilder("Unable to update resource state: [");
str.append("m = " + host.getId());
str.append("; name = " + host.getName());
str.append("; old state = " + oldState);
str.append("; event = " + event);
str.append("; new state = " + newState + "]");
logger.debug(str.toString());
String str = "Unable to update resource state: [" + "m = " + host.getId() +
"; name = " + host.getName() +
"; old state = " + oldState +
"; event = " + event +
"; new state = " + newState + "]";
logger.debug(str);
} else {
StringBuilder msg = new StringBuilder("Resource state update: [");
msg.append("id = " + host.getId());
msg.append("; name = " + host.getName());
msg.append("; old state = " + oldState);
msg.append("; event = " + event);
msg.append("; new state = " + newState + "]");
logger.debug(msg.toString());
String msg = "Resource state update: [" + "id = " + host.getId() +
"; name = " + host.getName() +
"; old state = " + oldState +
"; event = " + event +
"; new state = " + newState + "]";
logger.debug(msg);
}
return result > 0;
@ -1190,6 +1221,11 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
return listBy(sc);
}
@Override
public List<Long> listIdsByDataCenterId(Long zoneId) {
return listIdsBy(Type.Routing, null, null, null, zoneId, null, null);
}
@Override
public List<HostVO> findByPodId(Long podId) {
SearchCriteria<HostVO> sc = PodSearch.create();
@ -1197,6 +1233,11 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
return listBy(sc);
}
@Override
public List<Long> listIdsByPodId(Long podId) {
return listIdsBy(null, null, null, null, null, podId, null);
}
@Override
public List<HostVO> findByClusterId(Long clusterId) {
SearchCriteria<HostVO> sc = ClusterSearch.create();
@ -1204,6 +1245,63 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
return listBy(sc);
}
protected List<Long> listIdsBy(Host.Type type, Status status, ResourceState resourceState,
HypervisorType hypervisorType, Long zoneId, Long podId, Long clusterId) {
GenericSearchBuilder<HostVO, Long> sb = createSearchBuilder(Long.class);
sb.selectFields(sb.entity().getId());
sb.and("type", sb.entity().getType(), SearchCriteria.Op.EQ);
sb.and("status", sb.entity().getStatus(), SearchCriteria.Op.EQ);
sb.and("resourceState", sb.entity().getResourceState(), SearchCriteria.Op.EQ);
sb.and("hypervisorType", sb.entity().getHypervisorType(), SearchCriteria.Op.EQ);
sb.and("zoneId", sb.entity().getDataCenterId(), SearchCriteria.Op.EQ);
sb.and("podId", sb.entity().getPodId(), SearchCriteria.Op.EQ);
sb.and("clusterId", sb.entity().getClusterId(), SearchCriteria.Op.EQ);
sb.done();
SearchCriteria<Long> sc = sb.create();
if (type != null) {
sc.setParameters("type", type);
}
if (status != null) {
sc.setParameters("status", status);
}
if (resourceState != null) {
sc.setParameters("resourceState", resourceState);
}
if (hypervisorType != null) {
sc.setParameters("hypervisorType", hypervisorType);
}
if (zoneId != null) {
sc.setParameters("zoneId", zoneId);
}
if (podId != null) {
sc.setParameters("podId", podId);
}
if (clusterId != null) {
sc.setParameters("clusterId", clusterId);
}
return customSearch(sc, null);
}
@Override
public List<Long> listIdsByClusterId(Long clusterId) {
return listIdsBy(null, null, null, null, null, null, clusterId);
}
@Override
public List<Long> listIdsForUpRouting(Long zoneId, Long podId, Long clusterId) {
return listIdsBy(Type.Routing, Status.Up, null, null, zoneId, podId, clusterId);
}
@Override
public List<Long> listIdsByType(Type type) {
return listIdsBy(type, null, null, null, null, null, null);
}
@Override
public List<Long> listIdsForUpEnabledByZoneAndHypervisor(Long zoneId, HypervisorType hypervisorType) {
return listIdsBy(null, Status.Up, ResourceState.Enabled, hypervisorType, zoneId, null, null);
}
@Override
public List<HostVO> findByClusterIdAndEncryptionSupport(Long clusterId) {
SearchBuilder<DetailVO> hostCapabilitySearch = _detailsDao.createSearchBuilder();
@ -1256,6 +1354,15 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
return listBy(sc);
}
@Override
public HostVO findAnyStateHypervisorHostInCluster(long clusterId) {
SearchCriteria<HostVO> sc = TypeClusterStatusSearch.create();
sc.setParameters("type", Host.Type.Routing);
sc.setParameters("cluster", clusterId);
List<HostVO> list = listBy(sc, new Filter(1));
return list.isEmpty() ? null : list.get(0);
}
@Override
public HostVO findOldestExistentHypervisorHostInCluster(long clusterId) {
SearchCriteria<HostVO> sc = TypeClusterStatusSearch.create();
@ -1266,7 +1373,7 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
Filter orderByFilter = new Filter(HostVO.class, "created", true, null, null);
List<HostVO> hosts = search(sc, orderByFilter, null, false);
if (hosts != null && hosts.size() > 0) {
if (hosts != null && !hosts.isEmpty()) {
return hosts.get(0);
}
@ -1275,9 +1382,7 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
@Override
public List<Long> listAllHosts(long zoneId) {
SearchCriteria<Long> sc = HostIdSearch.create();
sc.addAnd("dataCenterId", SearchCriteria.Op.EQ, zoneId);
return customSearch(sc, null);
return listIdsBy(null, null, null, null, zoneId, null, null);
}
@Override
@ -1311,19 +1416,19 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
}
@Override
public List<Long> listClustersByHostTag(String computeOfferingTags) {
public List<Long> listClustersByHostTag(String hostTags) {
TransactionLegacy txn = TransactionLegacy.currentTxn();
String sql = this.LIST_CLUSTERID_FOR_HOST_TAG;
PreparedStatement pstmt = null;
List<Long> result = new ArrayList();
List<String> tags = Arrays.asList(computeOfferingTags.split(this.SEPARATOR));
String subselect = getHostIdsByComputeTags(tags);
sql = String.format(sql, subselect);
String selectStmtToListClusterIdsByHostTags = LIST_CLUSTER_IDS_FOR_HOST_TAGS;
PreparedStatement pstmt;
List<Long> result = new ArrayList<>();
List<String> tags = Arrays.asList(hostTags.split(SEPARATOR));
String selectStmtToListHostIdsByHostTags = getSelectStmtToListHostIdsByHostTags(tags);
selectStmtToListClusterIdsByHostTags = String.format(selectStmtToListClusterIdsByHostTags, selectStmtToListHostIdsByHostTags);
try {
pstmt = txn.prepareStatement(sql);
pstmt = txn.prepareStatement(selectStmtToListClusterIdsByHostTags);
for(int i = 0; i < tags.size(); i++){
for (int i = 0; i < tags.size(); i++){
pstmt.setString(i+1, tags.get(i));
}
@ -1334,20 +1439,20 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
pstmt.close();
return result;
} catch (SQLException e) {
throw new CloudRuntimeException("DB Exception on: " + sql, e);
throw new CloudRuntimeException("DB Exception on: " + selectStmtToListClusterIdsByHostTags, e);
}
}
private List<Long> findHostByComputeOfferings(String computeOfferingTags){
private List<Long> findHostIdsByHostTags(String hostTags){
TransactionLegacy txn = TransactionLegacy.currentTxn();
PreparedStatement pstmt = null;
List<Long> result = new ArrayList();
List<String> tags = Arrays.asList(computeOfferingTags.split(this.SEPARATOR));
String select = getHostIdsByComputeTags(tags);
PreparedStatement pstmt;
List<Long> result = new ArrayList<>();
List<String> tags = Arrays.asList(hostTags.split(SEPARATOR));
String selectStmtToListHostIdsByHostTags = getSelectStmtToListHostIdsByHostTags(tags);
try {
pstmt = txn.prepareStatement(select);
pstmt = txn.prepareStatement(selectStmtToListHostIdsByHostTags);
for(int i = 0; i < tags.size(); i++){
for (int i = 0; i < tags.size(); i++){
pstmt.setString(i+1, tags.get(i));
}
@ -1358,7 +1463,7 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
pstmt.close();
return result;
} catch (SQLException e) {
throw new CloudRuntimeException("DB Exception on: " + select, e);
throw new CloudRuntimeException("DB Exception on: " + selectStmtToListHostIdsByHostTags, e);
}
}
@ -1408,16 +1513,16 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
return result;
}
private String getHostIdsByComputeTags(List<String> offeringTags){
List<String> questionMarks = new ArrayList();
offeringTags.forEach((tag) -> { questionMarks.add("?"); });
return String.format(this.LIST_HOST_IDS_BY_COMPUTETAGS, String.join(",", questionMarks),questionMarks.size());
private String getSelectStmtToListHostIdsByHostTags(List<String> hostTags){
List<String> questionMarks = new ArrayList<>();
hostTags.forEach((tag) -> questionMarks.add("?"));
return String.format(LIST_HOST_IDS_BY_HOST_TAGS, String.join(SEPARATOR, questionMarks), questionMarks.size());
}
@Override
public List<HostVO> listHostsWithActiveVMs(long offeringId) {
TransactionLegacy txn = TransactionLegacy.currentTxn();
PreparedStatement pstmt = null;
PreparedStatement pstmt;
List<HostVO> result = new ArrayList<>();
StringBuilder sql = new StringBuilder(GET_HOSTS_OF_ACTIVE_VMS);
try {
@ -1466,7 +1571,7 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
@Override
public List<String> listOrderedHostsHypervisorVersionsInDatacenter(long datacenterId, HypervisorType hypervisorType) {
PreparedStatement pstmt = null;
PreparedStatement pstmt;
List<String> result = new ArrayList<>();
try {
TransactionLegacy txn = TransactionLegacy.currentTxn();
@ -1483,15 +1588,6 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
return result;
}
@Override
public List<HostVO> listAllHostsByType(Host.Type type) {
SearchCriteria<HostVO> sc = TypeSearch.create();
sc.setParameters("type", type);
sc.setParameters("resourceState", ResourceState.Enabled);
return listBy(sc);
}
@Override
public List<HostVO> listByType(Host.Type type) {
SearchCriteria<HostVO> sc = TypeSearch.create();
@ -1636,4 +1732,71 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
}
return String.format(sqlFindHostInZoneToExecuteCommand, hostResourceStatus);
}
@Override
public boolean isHostUp(long hostId) {
GenericSearchBuilder<HostVO, Status> sb = createSearchBuilder(Status.class);
sb.and("id", sb.entity().getId(), Op.EQ);
sb.selectFields(sb.entity().getStatus());
SearchCriteria<Status> sc = sb.create();
sc.setParameters("id", hostId);
List<Status> statuses = customSearch(sc, null);
return CollectionUtils.isNotEmpty(statuses) && Status.Up.equals(statuses.get(0));
}
@Override
public List<Long> findHostIdsByZoneClusterResourceStateTypeAndHypervisorType(final Long zoneId, final Long clusterId,
final List<ResourceState> resourceStates, final List<Type> types,
final List<Hypervisor.HypervisorType> hypervisorTypes) {
GenericSearchBuilder<HostVO, Long> sb = createSearchBuilder(Long.class);
sb.selectFields(sb.entity().getId());
sb.and("zoneId", sb.entity().getDataCenterId(), SearchCriteria.Op.EQ);
sb.and("clusterId", sb.entity().getClusterId(), SearchCriteria.Op.EQ);
sb.and("resourceState", sb.entity().getResourceState(), SearchCriteria.Op.IN);
sb.and("type", sb.entity().getType(), SearchCriteria.Op.IN);
if (CollectionUtils.isNotEmpty(hypervisorTypes)) {
sb.and().op(sb.entity().getHypervisorType(), SearchCriteria.Op.NULL);
sb.or("hypervisorTypes", sb.entity().getHypervisorType(), SearchCriteria.Op.IN);
sb.cp();
}
sb.done();
SearchCriteria<Long> sc = sb.create();
if (zoneId != null) {
sc.setParameters("zoneId", zoneId);
}
if (clusterId != null) {
sc.setParameters("clusterId", clusterId);
}
if (CollectionUtils.isNotEmpty(hypervisorTypes)) {
sc.setParameters("hypervisorTypes", hypervisorTypes.toArray());
}
sc.setParameters("resourceState", resourceStates.toArray());
sc.setParameters("type", types.toArray());
return customSearch(sc, null);
}
@Override
public List<HypervisorType> listDistinctHypervisorTypes(final Long zoneId) {
GenericSearchBuilder<HostVO, HypervisorType> sb = createSearchBuilder(HypervisorType.class);
sb.and("zoneId", sb.entity().getDataCenterId(), SearchCriteria.Op.EQ);
sb.and("type", sb.entity().getType(), SearchCriteria.Op.EQ);
sb.select(null, Func.DISTINCT, sb.entity().getHypervisorType());
sb.done();
SearchCriteria<HypervisorType> sc = sb.create();
if (zoneId != null) {
sc.setParameters("zoneId", zoneId);
}
sc.setParameters("type", Type.Routing);
return customSearch(sc, null);
}
@Override
public List<HostVO> listByIds(List<Long> ids) {
if (CollectionUtils.isEmpty(ids)) {
return new ArrayList<>();
}
SearchCriteria<HostVO> sc = IdsSearch.create();
sc.setParameters("id", ids.toArray());
return search(sc, null);
}
}

2
engine/schema/src/main/java/com/cloud/network/dao/IPAddressDaoImpl.java
View File

@ -421,7 +421,7 @@ public class IPAddressDaoImpl extends GenericDaoBase<IPAddressVO, Long> implemen
public long countFreeIpsInVlan(long vlanDbId) {
SearchCriteria<IPAddressVO> sc = VlanDbIdSearchUnallocated.create();
sc.setParameters("vlanDbId", vlanDbId);
return listBy(sc).size();
return getCount(sc);
}
@Override

3
engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java
View File

@ -415,8 +415,7 @@ public class NetworkDaoImpl extends GenericDaoBase<NetworkVO, Long>implements Ne
sc.setParameters("broadcastUri", broadcastURI);
sc.setParameters("guestType", guestTypes);
sc.setJoinParameters("persistent", "persistent", isPersistent);
List<NetworkVO> persistentNetworks = search(sc, null);
return persistentNetworks.size();
return getCount(sc);
}
@Override

3
engine/schema/src/main/java/com/cloud/secstorage/CommandExecLogDaoImpl.java
View File

@ -55,8 +55,7 @@ public class CommandExecLogDaoImpl extends GenericDaoBase<CommandExecLogVO, Long
SearchCriteria<CommandExecLogVO> sc = CommandSearch.create();
sc.setParameters("host_id", id);
sc.setParameters("command_name", "CopyCommand");
List<CommandExecLogVO> copyCmds = customSearch(sc, null);
return copyCmds.size();
return getCount(sc);
}
@Override

4
engine/schema/src/main/java/com/cloud/service/dao/ServiceOfferingDao.java
View File

@ -54,7 +54,7 @@ public interface ServiceOfferingDao extends GenericDao<ServiceOfferingVO, Long>
List<ServiceOfferingVO> listPublicByCpuAndMemory(Integer cpus, Integer memory);
List<ServiceOfferingVO> listByHostTag(String tag);
ServiceOfferingVO findServiceOfferingByComputeOnlyDiskOffering(long diskOfferingId, boolean includingRemoved);
List<Long> listIdsByHostTag(String tag);
}

11
engine/schema/src/main/java/com/cloud/service/dao/ServiceOfferingDaoImpl.java
View File

@ -34,6 +34,7 @@ import com.cloud.service.ServiceOfferingVO;
import com.cloud.storage.Storage.ProvisioningType;
import com.cloud.utils.db.DB;
import com.cloud.utils.db.GenericDaoBase;
import com.cloud.utils.db.GenericSearchBuilder;
import com.cloud.utils.db.SearchBuilder;
import com.cloud.utils.db.SearchCriteria;
import com.cloud.utils.exception.CloudRuntimeException;
@ -293,8 +294,9 @@ public class ServiceOfferingDaoImpl extends GenericDaoBase<ServiceOfferingVO, Lo
}
@Override
public List<ServiceOfferingVO> listByHostTag(String tag) {
SearchBuilder<ServiceOfferingVO> sb = createSearchBuilder();
public List<Long> listIdsByHostTag(String tag) {
GenericSearchBuilder<ServiceOfferingVO, Long> sb = createSearchBuilder(Long.class);
sb.selectFields(sb.entity().getId());
sb.and("tagNotNull", sb.entity().getHostTag(), SearchCriteria.Op.NNULL);
sb.and().op("tagEq", sb.entity().getHostTag(), SearchCriteria.Op.EQ);
sb.or("tagStartLike", sb.entity().getHostTag(), SearchCriteria.Op.LIKE);
@ -302,11 +304,12 @@ public class ServiceOfferingDaoImpl extends GenericDaoBase<ServiceOfferingVO, Lo
sb.or("tagEndLike", sb.entity().getHostTag(), SearchCriteria.Op.LIKE);
sb.cp();
sb.done();
SearchCriteria<ServiceOfferingVO> sc = sb.create();
SearchCriteria<Long> sc = sb.create();
sc.setParameters("tagEq", tag);
sc.setParameters("tagStartLike", tag + ",%");
sc.setParameters("tagMidLike", "%," + tag + ",%");
sc.setParameters("tagEndLike", "%," + tag);
return listBy(sc);
return customSearch(sc, null);
}
}

2
engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolHostDao.java
View File

@ -34,7 +34,7 @@ public interface StoragePoolHostDao extends GenericDao<StoragePoolHostVO, Long>
List<Long> findHostsConnectedToPools(List<Long> poolIds);
List<Pair<Long, Integer>> getDatacenterStoragePoolHostInfo(long dcId, boolean sharedOnly);
boolean hasDatacenterStoragePoolHostInfo(long dcId, boolean sharedOnly);
public void deletePrimaryRecordsForHost(long hostId);

24
engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolHostDaoImpl.java
View File

@ -55,11 +55,11 @@ public class StoragePoolHostDaoImpl extends GenericDaoBase<StoragePoolHostVO, Lo
protected static final String HOSTS_FOR_POOLS_SEARCH = "SELECT DISTINCT(ph.host_id) FROM storage_pool_host_ref ph, host h WHERE ph.host_id = h.id AND h.status = 'Up' AND resource_state = 'Enabled' AND ph.pool_id IN (?)";
protected static final String STORAGE_POOL_HOST_INFO = "SELECT p.data_center_id, count(ph.host_id) " + " FROM storage_pool p, storage_pool_host_ref ph "
+ " WHERE p.id = ph.pool_id AND p.data_center_id = ? " + " GROUP by p.data_center_id";
protected static final String STORAGE_POOL_HOST_INFO = "SELECT (SELECT id FROM storage_pool_host_ref ph WHERE " +
"ph.pool_id=p.id limit 1) AS sphr FROM storage_pool p WHERE p.data_center_id = ?";
protected static final String SHARED_STORAGE_POOL_HOST_INFO = "SELECT p.data_center_id, count(ph.host_id) " + " FROM storage_pool p, storage_pool_host_ref ph "
+ " WHERE p.id = ph.pool_id AND p.data_center_id = ? " + " AND p.pool_type NOT IN ('LVM', 'Filesystem')" + " GROUP by p.data_center_id";
protected static final String SHARED_STORAGE_POOL_HOST_INFO = "SELECT (SELECT id FROM storage_pool_host_ref ph " +
"WHERE ph.pool_id=p.id limit 1) AS sphr FROM storage_pool p WHERE p.data_center_id = ? AND p.pool_type NOT IN ('LVM', 'Filesystem')";
protected static final String DELETE_PRIMARY_RECORDS = "DELETE " + "FROM storage_pool_host_ref " + "WHERE host_id = ?";
@ -169,23 +169,23 @@ public class StoragePoolHostDaoImpl extends GenericDaoBase<StoragePoolHostVO, Lo
}
@Override
public List<Pair<Long, Integer>> getDatacenterStoragePoolHostInfo(long dcId, boolean sharedOnly) {
ArrayList<Pair<Long, Integer>> l = new ArrayList<Pair<Long, Integer>>();
public boolean hasDatacenterStoragePoolHostInfo(long dcId, boolean sharedOnly) {
Long poolCount = 0L;
String sql = sharedOnly ? SHARED_STORAGE_POOL_HOST_INFO : STORAGE_POOL_HOST_INFO;
TransactionLegacy txn = TransactionLegacy.currentTxn();
PreparedStatement pstmt = null;
try {
pstmt = txn.prepareAutoCloseStatement(sql);
try (PreparedStatement pstmt = txn.prepareAutoCloseStatement(sql)) {
pstmt.setLong(1, dcId);
ResultSet rs = pstmt.executeQuery();
while (rs.next()) {
l.add(new Pair<Long, Integer>(rs.getLong(1), rs.getInt(2)));
poolCount = rs.getLong(1);
if (poolCount > 0) {
return true;
}
}
} catch (SQLException e) {
logger.debug("SQLException: ", e);
}
return l;
return false;
}
/**

5
engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java
View File

@ -67,6 +67,8 @@ public interface VMTemplateDao extends GenericDao<VMTemplateVO, Long>, StateDao<
public List<VMTemplateVO> userIsoSearch(boolean listRemoved);
List<VMTemplateVO> listAllReadySystemVMTemplates(Long zoneId);
VMTemplateVO findSystemVMTemplate(long zoneId);
VMTemplateVO findSystemVMReadyTemplate(long zoneId, HypervisorType hypervisorType);
@ -91,6 +93,5 @@ public interface VMTemplateDao extends GenericDao<VMTemplateVO, Long>, StateDao<
List<VMTemplateVO> listByIds(List<Long> ids);
List<VMTemplateVO> listByTemplateTag(String tag);
List<Long> listIdsByTemplateTag(String tag);
}

60
engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java
View File

@ -344,19 +344,12 @@ public class VMTemplateDaoImpl extends GenericDaoBase<VMTemplateVO, Long> implem
readySystemTemplateSearch = createSearchBuilder();
readySystemTemplateSearch.and("state", readySystemTemplateSearch.entity().getState(), SearchCriteria.Op.EQ);
readySystemTemplateSearch.and("templateType", readySystemTemplateSearch.entity().getTemplateType(), SearchCriteria.Op.EQ);
readySystemTemplateSearch.and("hypervisorType", readySystemTemplateSearch.entity().getHypervisorType(), SearchCriteria.Op.IN);
SearchBuilder<TemplateDataStoreVO> templateDownloadSearch = _templateDataStoreDao.createSearchBuilder();
templateDownloadSearch.and("downloadState", templateDownloadSearch.entity().getDownloadState(), SearchCriteria.Op.IN);
readySystemTemplateSearch.join("vmTemplateJoinTemplateStoreRef", templateDownloadSearch, templateDownloadSearch.entity().getTemplateId(),
readySystemTemplateSearch.entity().getId(), JoinBuilder.JoinType.INNER);
SearchBuilder<HostVO> hostHyperSearch2 = _hostDao.createSearchBuilder();
hostHyperSearch2.and("type", hostHyperSearch2.entity().getType(), SearchCriteria.Op.EQ);
hostHyperSearch2.and("zoneId", hostHyperSearch2.entity().getDataCenterId(), SearchCriteria.Op.EQ);
hostHyperSearch2.and("removed", hostHyperSearch2.entity().getRemoved(), SearchCriteria.Op.NULL);
hostHyperSearch2.groupBy(hostHyperSearch2.entity().getHypervisorType());
readySystemTemplateSearch.join("tmplHyper", hostHyperSearch2, hostHyperSearch2.entity().getHypervisorType(), readySystemTemplateSearch.entity()
.getHypervisorType(), JoinBuilder.JoinType.INNER);
hostHyperSearch2.done();
readySystemTemplateSearch.groupBy(readySystemTemplateSearch.entity().getId());
readySystemTemplateSearch.done();
tmpltTypeHyperSearch2 = createSearchBuilder();
@ -556,29 +549,35 @@ public class VMTemplateDaoImpl extends GenericDaoBase<VMTemplateVO, Long> implem
}
@Override
public VMTemplateVO findSystemVMReadyTemplate(long zoneId, HypervisorType hypervisorType) {
public List<VMTemplateVO> listAllReadySystemVMTemplates(Long zoneId) {
List<HypervisorType> availableHypervisors = _hostDao.listDistinctHypervisorTypes(zoneId);
if (CollectionUtils.isEmpty(availableHypervisors)) {
return Collections.emptyList();
}
SearchCriteria<VMTemplateVO> sc = readySystemTemplateSearch.create();
sc.setParameters("templateType", Storage.TemplateType.SYSTEM);
sc.setParameters("state", VirtualMachineTemplate.State.Active);
sc.setJoinParameters("tmplHyper", "type", Host.Type.Routing);
sc.setJoinParameters("tmplHyper", "zoneId", zoneId);
sc.setJoinParameters("vmTemplateJoinTemplateStoreRef", "downloadState", new VMTemplateStorageResourceAssoc.Status[] {VMTemplateStorageResourceAssoc.Status.DOWNLOADED, VMTemplateStorageResourceAssoc.Status.BYPASSED});
sc.setParameters("hypervisorType", availableHypervisors.toArray());
sc.setJoinParameters("vmTemplateJoinTemplateStoreRef", "downloadState",
List.of(VMTemplateStorageResourceAssoc.Status.DOWNLOADED,
VMTemplateStorageResourceAssoc.Status.BYPASSED).toArray());
// order by descending order of id
List<VMTemplateVO> tmplts = listBy(sc, new Filter(VMTemplateVO.class, "id", false, null, null));
if (tmplts.size() > 0) {
if (hypervisorType == HypervisorType.Any) {
return tmplts.get(0);
}
for (VMTemplateVO tmplt : tmplts) {
if (tmplt.getHypervisorType() == hypervisorType) {
return tmplt;
}
}
return listBy(sc, new Filter(VMTemplateVO.class, "id", false, null, null));
}
@Override
public VMTemplateVO findSystemVMReadyTemplate(long zoneId, HypervisorType hypervisorType) {
List<VMTemplateVO> templates = listAllReadySystemVMTemplates(zoneId);
if (CollectionUtils.isEmpty(templates)) {
return null;
}
return null;
if (hypervisorType == HypervisorType.Any) {
return templates.get(0);
}
return templates.stream()
.filter(t -> t.getHypervisorType() == hypervisorType)
.findFirst()
.orElse(null);
}
@Override
@ -687,13 +686,14 @@ public class VMTemplateDaoImpl extends GenericDaoBase<VMTemplateVO, Long> implem
}
@Override
public List<VMTemplateVO> listByTemplateTag(String tag) {
SearchBuilder<VMTemplateVO> sb = createSearchBuilder();
public List<Long> listIdsByTemplateTag(String tag) {
GenericSearchBuilder<VMTemplateVO, Long> sb = createSearchBuilder(Long.class);
sb.selectFields(sb.entity().getId());
sb.and("tag", sb.entity().getTemplateTag(), SearchCriteria.Op.EQ);
sb.done();
SearchCriteria<VMTemplateVO> sc = sb.create();
SearchCriteria<Long> sc = sb.create();
sc.setParameters("tag", tag);
return listIncludingRemovedBy(sc);
return customSearchIncludingRemoved(sc, null);
}
@Override

8
engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java
View File

@ -571,14 +571,6 @@ public class VolumeDaoImpl extends GenericDaoBase<VolumeVO, Long> implements Vol
}
}
public static class SumCount {
public long sum;
public long count;
public SumCount() {
}
}
@Override
public List<VolumeVO> listVolumesToBeDestroyed() {
SearchCriteria<VolumeVO> sc = AllFieldsSearch.create();

2
engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java
View File

@ -870,7 +870,7 @@ public class SystemVmTemplateRegistration {
public void doInTransactionWithoutResult(final TransactionStatus status) {
Set<Hypervisor.HypervisorType> hypervisorsListInUse = new HashSet<Hypervisor.HypervisorType>();
try {
hypervisorsListInUse = clusterDao.getDistictAvailableHypervisorsAcrossClusters();
hypervisorsListInUse = clusterDao.getDistinctAvailableHypervisorsAcrossClusters();
} catch (final Exception e) {
LOGGER.error("updateSystemVmTemplates: Exception caught while getting hypervisor types from clusters: " + e.getMessage());

11
engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java
View File

@ -114,6 +114,17 @@ public class DatabaseAccessObject {
}
}
public void renameIndex(Connection conn, String tableName, String oldName, String newName) {
String stmt = String.format("ALTER TABLE %s RENAME INDEX %s TO %s", tableName, oldName, newName);
logger.debug("Statement: {}", stmt);
try (PreparedStatement pstmt = conn.prepareStatement(stmt)) {
pstmt.execute();
logger.debug("Renamed index {} to {}", oldName, newName);
} catch (SQLException e) {
logger.warn("Unable to rename index {} to {}", oldName, newName, e);
}
}
protected void closePreparedStatement(PreparedStatement pstmt, String errorMessage) {
try {
if (pstmt != null) {

6
engine/schema/src/main/java/com/cloud/upgrade/dao/DbUpgradeUtils.java
View File

@ -31,6 +31,12 @@ public class DbUpgradeUtils {
}
}
public static void renameIndexIfNeeded(Connection conn, String tableName, String oldName, String newName) {
if (!dao.indexExists(conn, tableName, oldName)) {
dao.renameIndex(conn, tableName, oldName, newName);
}
}
public static void addForeignKey(Connection conn, String tableName, String tableColumn, String foreignTableName, String foreignColumnName) {
dao.addForeignKey(conn, tableName, tableColumn, foreignTableName, foreignColumnName);
}

39
engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42000to42010.java
View File

@ -53,6 +53,7 @@ public class Upgrade42000to42010 extends DbUpgradeAbstractImpl implements DbUpgr
@Override
public void performDataMigration(Connection conn) {
addIndexes(conn);
}
@Override
@ -80,4 +81,42 @@ public class Upgrade42000to42010 extends DbUpgradeAbstractImpl implements DbUpgr
throw new CloudRuntimeException("Failed to find / register SystemVM template(s)");
}
}
private void addIndexes(Connection conn) {
DbUpgradeUtils.addIndexIfNeeded(conn, "host", "mgmt_server_id");
DbUpgradeUtils.addIndexIfNeeded(conn, "host", "resource");
DbUpgradeUtils.addIndexIfNeeded(conn, "host", "resource_state");
DbUpgradeUtils.addIndexIfNeeded(conn, "host", "type");
DbUpgradeUtils.renameIndexIfNeeded(conn, "user_ip_address", "public_ip_address", "uk_public_ip_address");
DbUpgradeUtils.addIndexIfNeeded(conn, "user_ip_address", "public_ip_address");
DbUpgradeUtils.addIndexIfNeeded(conn, "user_ip_address", "data_center_id");
DbUpgradeUtils.addIndexIfNeeded(conn, "user_ip_address", "vlan_db_id");
DbUpgradeUtils.addIndexIfNeeded(conn, "user_ip_address", "removed");
DbUpgradeUtils.addIndexIfNeeded(conn, "vlan", "vlan_type");
DbUpgradeUtils.addIndexIfNeeded(conn, "vlan", "data_center_id");
DbUpgradeUtils.addIndexIfNeeded(conn, "vlan", "removed");
DbUpgradeUtils.addIndexIfNeeded(conn, "network_offering_details", "name");
DbUpgradeUtils.addIndexIfNeeded(conn, "network_offering_details", "resource_id", "resource_type");
DbUpgradeUtils.addIndexIfNeeded(conn, "service_offering", "cpu");
DbUpgradeUtils.addIndexIfNeeded(conn, "service_offering", "speed");
DbUpgradeUtils.addIndexIfNeeded(conn, "service_offering", "ram_size");
DbUpgradeUtils.addIndexIfNeeded(conn, "op_host_planner_reservation", "resource_usage");
DbUpgradeUtils.addIndexIfNeeded(conn, "storage_pool", "pool_type");
DbUpgradeUtils.addIndexIfNeeded(conn, "storage_pool", "data_center_id", "status", "scope", "hypervisor");
DbUpgradeUtils.addIndexIfNeeded(conn, "router_network_ref", "guest_type");
DbUpgradeUtils.addIndexIfNeeded(conn, "domain_router", "role");
DbUpgradeUtils.addIndexIfNeeded(conn, "async_job", "instance_type", "job_status");
DbUpgradeUtils.addIndexIfNeeded(conn, "cluster", "managed_state");
}
}

2
engine/schema/src/main/java/com/cloud/vm/dao/ConsoleProxyDao.java
View File

@ -45,7 +45,7 @@ public interface ConsoleProxyDao extends GenericDao<ConsoleProxyVO, Long> {
public List<ConsoleProxyLoadInfo> getDatacenterSessionLoadMatrix();
public List<Pair<Long, Integer>> getDatacenterStoragePoolHostInfo(long dcId, boolean countAllPoolTypes);
public boolean hasDatacenterStoragePoolHostInfo(long dcId, boolean sharedOnly);
public List<Pair<Long, Integer>> getProxyLoadMatrix();

32
engine/schema/src/main/java/com/cloud/vm/dao/ConsoleProxyDaoImpl.java
View File

@ -23,7 +23,6 @@ import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import org.springframework.stereotype.Component;
import com.cloud.info.ConsoleProxyLoadInfo;
@ -76,11 +75,11 @@ public class ConsoleProxyDaoImpl extends GenericDaoBase<ConsoleProxyVO, Long> im
private static final String GET_PROXY_ACTIVE_LOAD = "SELECT active_session AS count" + " FROM console_proxy" + " WHERE id=?";
private static final String STORAGE_POOL_HOST_INFO = "SELECT p.data_center_id, count(ph.host_id) " + " FROM storage_pool p, storage_pool_host_ref ph "
+ " WHERE p.id = ph.pool_id AND p.data_center_id = ? " + " GROUP by p.data_center_id";
protected static final String STORAGE_POOL_HOST_INFO = "SELECT (SELECT id FROM storage_pool_host_ref ph WHERE " +
"ph.pool_id=p.id limit 1) AS sphr FROM storage_pool p WHERE p.data_center_id = ?";
private static final String SHARED_STORAGE_POOL_HOST_INFO = "SELECT p.data_center_id, count(ph.host_id) " + " FROM storage_pool p, storage_pool_host_ref ph "
+ " WHERE p.pool_type <> 'LVM' AND p.id = ph.pool_id AND p.data_center_id = ? " + " GROUP by p.data_center_id";
protected static final String SHARED_STORAGE_POOL_HOST_INFO = "SELECT (SELECT id FROM storage_pool_host_ref ph " +
"WHERE ph.pool_id=p.id limit 1) AS sphr FROM storage_pool p WHERE p.data_center_id = ? AND p.pool_type NOT IN ('LVM', 'Filesystem')";
protected SearchBuilder<ConsoleProxyVO> DataCenterStatusSearch;
protected SearchBuilder<ConsoleProxyVO> StateSearch;
@ -219,28 +218,23 @@ public class ConsoleProxyDaoImpl extends GenericDaoBase<ConsoleProxyVO, Long> im
}
@Override
public List<Pair<Long, Integer>> getDatacenterStoragePoolHostInfo(long dcId, boolean countAllPoolTypes) {
ArrayList<Pair<Long, Integer>> l = new ArrayList<Pair<Long, Integer>>();
public boolean hasDatacenterStoragePoolHostInfo(long dcId, boolean sharedOnly) {
Long poolCount = 0L;
String sql = sharedOnly ? SHARED_STORAGE_POOL_HOST_INFO : STORAGE_POOL_HOST_INFO;
TransactionLegacy txn = TransactionLegacy.currentTxn();
;
PreparedStatement pstmt = null;
try {
if (countAllPoolTypes) {
pstmt = txn.prepareAutoCloseStatement(STORAGE_POOL_HOST_INFO);
} else {
pstmt = txn.prepareAutoCloseStatement(SHARED_STORAGE_POOL_HOST_INFO);
}
try (PreparedStatement pstmt = txn.prepareAutoCloseStatement(sql)) {
pstmt.setLong(1, dcId);
ResultSet rs = pstmt.executeQuery();
while (rs.next()) {
l.add(new Pair<Long, Integer>(rs.getLong(1), rs.getInt(2)));
poolCount = rs.getLong(1);
if (poolCount > 0) {
return true;
}
}
} catch (SQLException e) {
logger.debug("Caught SQLException: ", e);
}
return l;
return false;
}
@Override

3
engine/schema/src/main/java/com/cloud/vm/dao/NicIpAliasDaoImpl.java
View File

@ -170,8 +170,7 @@ public class NicIpAliasDaoImpl extends GenericDaoBase<NicIpAliasVO, Long> implem
public Integer countAliasIps(long id) {
SearchCriteria<NicIpAliasVO> sc = AllFieldsSearch.create();
sc.setParameters("instanceId", id);
List<NicIpAliasVO> list = listBy(sc);
return list.size();
return getCount(sc);
}
@Override

21
engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java
View File

@ -16,6 +16,7 @@
// under the License.
package com.cloud.vm.dao;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
@ -81,7 +82,7 @@ public interface VMInstanceDao extends GenericDao<VMInstanceVO, Long>, StateDao<
List<VMInstanceVO> listByHostAndState(long hostId, State... states);
List<VMInstanceVO> listByTypes(VirtualMachine.Type... types);
int countByTypes(VirtualMachine.Type... types);
VMInstanceVO findByIdTypes(long id, VirtualMachine.Type... types);
@ -144,21 +145,28 @@ public interface VMInstanceDao extends GenericDao<VMInstanceVO, Long>, StateDao<
*/
List<String> listDistinctHostNames(long networkId, VirtualMachine.Type... types);
List<VMInstanceVO> findByHostInStatesExcluding(Long hostId, Collection<Long> excludingIds, State... states);
List<VMInstanceVO> findByHostInStates(Long hostId, State... states);
List<VMInstanceVO> listStartingWithNoHostId();
boolean updatePowerState(long instanceId, long powerHostId, VirtualMachine.PowerState powerState, Date wisdomEra);
Map<Long, VirtualMachine.PowerState> updatePowerState(Map<Long, VirtualMachine.PowerState> instancePowerStates,
long powerHostId, Date wisdomEra);
void resetVmPowerStateTracking(long instanceId);
void resetVmPowerStateTracking(List<Long> instanceId);
void resetHostPowerStateTracking(long hostId);
HashMap<String, Long> countVgpuVMs(Long dcId, Long podId, Long clusterId);
VMInstanceVO findVMByHostNameInZone(String hostName, long zoneId);
boolean isPowerStateUpToDate(long instanceId);
boolean isPowerStateUpToDate(VMInstanceVO instance);
List<VMInstanceVO> listNonMigratingVmsByHostEqualsLastHost(long hostId);
@ -170,4 +178,13 @@ public interface VMInstanceDao extends GenericDao<VMInstanceVO, Long>, StateDao<
List<Long> skippedVmIds);
Pair<List<VMInstanceVO>, Integer> listByVmsNotInClusterUsingPool(long clusterId, long poolId);
List<VMInstanceVO> listIdServiceOfferingForUpVmsByHostId(Long hostId);
List<VMInstanceVO> listIdServiceOfferingForVmsMigratingFromHost(Long hostId);
Map<String, Long> getNameIdMapForVmInstanceNames(Collection<String> names);
Map<String, Long> getNameIdMapForVmIds(Collection<Long> ids);
}

233
engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java
View File

@ -20,6 +20,7 @@ import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
@ -75,6 +76,7 @@ public class VMInstanceDaoImpl extends GenericDaoBase<VMInstanceVO, Long> implem
protected SearchBuilder<VMInstanceVO> LHVMClusterSearch;
protected SearchBuilder<VMInstanceVO> IdStatesSearch;
protected SearchBuilder<VMInstanceVO> AllFieldsSearch;
protected SearchBuilder<VMInstanceVO> IdServiceOfferingIdSelectSearch;
protected SearchBuilder<VMInstanceVO> ZoneTemplateNonExpungedSearch;
protected SearchBuilder<VMInstanceVO> TemplateNonExpungedSearch;
protected SearchBuilder<VMInstanceVO> NameLikeSearch;
@ -101,6 +103,7 @@ public class VMInstanceDaoImpl extends GenericDaoBase<VMInstanceVO, Long> implem
protected SearchBuilder<VMInstanceVO> BackupSearch;
protected SearchBuilder<VMInstanceVO> LastHostAndStatesSearch;
protected SearchBuilder<VMInstanceVO> VmsNotInClusterUsingPool;
protected SearchBuilder<VMInstanceVO> IdsPowerStateSelectSearch;
@Inject
ResourceTagDao tagsDao;
@ -175,6 +178,14 @@ public class VMInstanceDaoImpl extends GenericDaoBase<VMInstanceVO, Long> implem
AllFieldsSearch.and("account", AllFieldsSearch.entity().getAccountId(), Op.EQ);
AllFieldsSearch.done();
IdServiceOfferingIdSelectSearch = createSearchBuilder();
IdServiceOfferingIdSelectSearch.and("host", IdServiceOfferingIdSelectSearch.entity().getHostId(), Op.EQ);
IdServiceOfferingIdSelectSearch.and("lastHost", IdServiceOfferingIdSelectSearch.entity().getLastHostId(), Op.EQ);
IdServiceOfferingIdSelectSearch.and("state", IdServiceOfferingIdSelectSearch.entity().getState(), Op.EQ);
IdServiceOfferingIdSelectSearch.and("states", IdServiceOfferingIdSelectSearch.entity().getState(), Op.IN);
IdServiceOfferingIdSelectSearch.selectFields(IdServiceOfferingIdSelectSearch.entity().getId(), IdServiceOfferingIdSelectSearch.entity().getServiceOfferingId());
IdServiceOfferingIdSelectSearch.done();
ZoneTemplateNonExpungedSearch = createSearchBuilder();
ZoneTemplateNonExpungedSearch.and("zone", ZoneTemplateNonExpungedSearch.entity().getDataCenterId(), Op.EQ);
ZoneTemplateNonExpungedSearch.and("template", ZoneTemplateNonExpungedSearch.entity().getTemplateId(), Op.EQ);
@ -274,6 +285,7 @@ public class VMInstanceDaoImpl extends GenericDaoBase<VMInstanceVO, Long> implem
HostAndStateSearch = createSearchBuilder();
HostAndStateSearch.and("host", HostAndStateSearch.entity().getHostId(), Op.EQ);
HostAndStateSearch.and("states", HostAndStateSearch.entity().getState(), Op.IN);
HostAndStateSearch.and("idsNotIn", HostAndStateSearch.entity().getId(), Op.NIN);
HostAndStateSearch.done();
StartingWithNoHostSearch = createSearchBuilder();
@ -323,6 +335,15 @@ public class VMInstanceDaoImpl extends GenericDaoBase<VMInstanceVO, Long> implem
VmsNotInClusterUsingPool.join("hostSearch2", hostSearch2, hostSearch2.entity().getId(), VmsNotInClusterUsingPool.entity().getHostId(), JoinType.INNER);
VmsNotInClusterUsingPool.and("vmStates", VmsNotInClusterUsingPool.entity().getState(), Op.IN);
VmsNotInClusterUsingPool.done();
IdsPowerStateSelectSearch = createSearchBuilder();
IdsPowerStateSelectSearch.and("id", IdsPowerStateSelectSearch.entity().getId(), Op.IN);
IdsPowerStateSelectSearch.selectFields(IdsPowerStateSelectSearch.entity().getId(),
IdsPowerStateSelectSearch.entity().getPowerHostId(),
IdsPowerStateSelectSearch.entity().getPowerState(),
IdsPowerStateSelectSearch.entity().getPowerStateUpdateCount(),
IdsPowerStateSelectSearch.entity().getPowerStateUpdateTime());
IdsPowerStateSelectSearch.done();
}
@Override
@ -458,10 +479,10 @@ public class VMInstanceDaoImpl extends GenericDaoBase<VMInstanceVO, Long> implem
}
@Override
public List<VMInstanceVO> listByTypes(Type... types) {
public int countByTypes(Type... types) {
SearchCriteria<VMInstanceVO> sc = TypesSearch.create();
sc.setParameters("types", (Object[])types);
return listBy(sc);
return getCount(sc);
}
@Override
@ -897,6 +918,17 @@ public class VMInstanceDaoImpl extends GenericDaoBase<VMInstanceVO, Long> implem
return result;
}
@Override
public List<VMInstanceVO> findByHostInStatesExcluding(Long hostId, Collection<Long> excludingIds, State... states) {
SearchCriteria<VMInstanceVO> sc = HostAndStateSearch.create();
sc.setParameters("host", hostId);
if (excludingIds != null && !excludingIds.isEmpty()) {
sc.setParameters("idsNotIn", excludingIds.toArray());
}
sc.setParameters("states", (Object[])states);
return listBy(sc);
}
@Override
public List<VMInstanceVO> findByHostInStates(Long hostId, State... states) {
SearchCriteria<VMInstanceVO> sc = HostAndStateSearch.create();
@ -912,42 +944,109 @@ public class VMInstanceDaoImpl extends GenericDaoBase<VMInstanceVO, Long> implem
return listBy(sc);
}
@Override
public boolean updatePowerState(final long instanceId, final long powerHostId, final VirtualMachine.PowerState powerState, Date wisdomEra) {
return Transaction.execute(new TransactionCallback<>() {
@Override
public Boolean doInTransaction(TransactionStatus status) {
boolean needToUpdate = false;
VMInstanceVO instance = findById(instanceId);
if (instance != null
&& (null == instance.getPowerStateUpdateTime()
|| instance.getPowerStateUpdateTime().before(wisdomEra))) {
Long savedPowerHostId = instance.getPowerHostId();
if (instance.getPowerState() != powerState
|| savedPowerHostId == null
|| savedPowerHostId != powerHostId
|| !isPowerStateInSyncWithInstanceState(powerState, powerHostId, instance)) {
instance.setPowerState(powerState);
instance.setPowerHostId(powerHostId);
instance.setPowerStateUpdateCount(1);
instance.setPowerStateUpdateTime(DateUtil.currentGMTTime());
needToUpdate = true;
update(instanceId, instance);
} else {
// to reduce DB updates, consecutive same state update for more than 3 times
if (instance.getPowerStateUpdateCount() < MAX_CONSECUTIVE_SAME_STATE_UPDATE_COUNT) {
instance.setPowerStateUpdateCount(instance.getPowerStateUpdateCount() + 1);
instance.setPowerStateUpdateTime(DateUtil.currentGMTTime());
needToUpdate = true;
update(instanceId, instance);
}
}
}
return needToUpdate;
protected List<VMInstanceVO> listSelectPowerStateByIds(final List<Long> ids) {
if (CollectionUtils.isEmpty(ids)) {
return new ArrayList<>();
}
SearchCriteria<VMInstanceVO> sc = IdsPowerStateSelectSearch.create();
sc.setParameters("id", ids.toArray());
return customSearch(sc, null);
}
protected Integer getPowerUpdateCount(final VMInstanceVO instance, final long powerHostId,
final VirtualMachine.PowerState powerState, Date wisdomEra) {
if (instance.getPowerStateUpdateTime() == null || instance.getPowerStateUpdateTime().before(wisdomEra)) {
Long savedPowerHostId = instance.getPowerHostId();
boolean isStateMismatch = instance.getPowerState() != powerState
|| savedPowerHostId == null
|| !savedPowerHostId.equals(powerHostId)
|| !isPowerStateInSyncWithInstanceState(powerState, powerHostId, instance);
if (isStateMismatch) {
return 1;
} else if (instance.getPowerStateUpdateCount() < MAX_CONSECUTIVE_SAME_STATE_UPDATE_COUNT) {
return instance.getPowerStateUpdateCount() + 1;
}
}
return null;
}
@Override
public boolean updatePowerState(final long instanceId, final long powerHostId,
final VirtualMachine.PowerState powerState, Date wisdomEra) {
return Transaction.execute((TransactionCallback<Boolean>) status -> {
VMInstanceVO instance = findById(instanceId);
if (instance == null) {
return false;
}
// Check if we need to update based on powerStateUpdateTime
if (instance.getPowerStateUpdateTime() == null || instance.getPowerStateUpdateTime().before(wisdomEra)) {
Long savedPowerHostId = instance.getPowerHostId();
boolean isStateMismatch = instance.getPowerState() != powerState
|| savedPowerHostId == null
|| !savedPowerHostId.equals(powerHostId)
|| !isPowerStateInSyncWithInstanceState(powerState, powerHostId, instance);
if (isStateMismatch) {
instance.setPowerState(powerState);
instance.setPowerHostId(powerHostId);
instance.setPowerStateUpdateCount(1);
} else if (instance.getPowerStateUpdateCount() < MAX_CONSECUTIVE_SAME_STATE_UPDATE_COUNT) {
instance.setPowerStateUpdateCount(instance.getPowerStateUpdateCount() + 1);
} else {
// No need to update if power state is already in sync and count exceeded
return false;
}
instance.setPowerStateUpdateTime(DateUtil.currentGMTTime());
update(instanceId, instance);
return true; // Return true since an update occurred
}
return false;
});
}
@Override
public Map<Long, VirtualMachine.PowerState> updatePowerState(
final Map<Long, VirtualMachine.PowerState> instancePowerStates, long powerHostId, Date wisdomEra) {
Map<Long, VirtualMachine.PowerState> notUpdated = new HashMap<>();
List<VMInstanceVO> instances = listSelectPowerStateByIds(new ArrayList<>(instancePowerStates.keySet()));
Map<Long, Integer> updateCounts = new HashMap<>();
for (VMInstanceVO instance : instances) {
VirtualMachine.PowerState powerState = instancePowerStates.get(instance.getId());
Integer count = getPowerUpdateCount(instance, powerHostId, powerState, wisdomEra);
if (count != null) {
updateCounts.put(instance.getId(), count);
} else {
notUpdated.put(instance.getId(), powerState);
}
}
if (updateCounts.isEmpty()) {
return notUpdated;
}
StringBuilder sql = new StringBuilder("UPDATE `cloud`.`vm_instance` SET " +
"`power_host` = ?, `power_state_update_time` = now(), `power_state` = CASE ");
updateCounts.keySet().forEach(key -> {
sql.append("WHEN id = ").append(key).append(" THEN '").append(instancePowerStates.get(key)).append("' ");
});
sql.append("END, `power_state_update_count` = CASE ");
StringBuilder idList = new StringBuilder();
updateCounts.forEach((key, value) -> {
sql.append("WHEN `id` = ").append(key).append(" THEN ").append(value).append(" ");
idList.append(key).append(",");
});
idList.setLength(idList.length() - 1);
sql.append("END WHERE `id` IN (").append(idList).append(")");
TransactionLegacy txn = TransactionLegacy.currentTxn();
try (PreparedStatement pstmt = txn.prepareAutoCloseStatement(sql.toString())) {
pstmt.setLong(1, powerHostId);
pstmt.executeUpdate();
} catch (SQLException e) {
logger.error("Unable to execute update power states SQL from VMs {} due to: {}",
idList, e.getMessage(), e);
return instancePowerStates;
}
return notUpdated;
}
private boolean isPowerStateInSyncWithInstanceState(final VirtualMachine.PowerState powerState, final long powerHostId, final VMInstanceVO instance) {
State instanceState = instance.getState();
if ((powerState == VirtualMachine.PowerState.PowerOff && instanceState == State.Running)
@ -962,11 +1061,7 @@ public class VMInstanceDaoImpl extends GenericDaoBase<VMInstanceVO, Long> implem
}
@Override
public boolean isPowerStateUpToDate(final long instanceId) {
VMInstanceVO instance = findById(instanceId);
if(instance == null) {
throw new CloudRuntimeException("checking power state update count on non existing instance " + instanceId);
}
public boolean isPowerStateUpToDate(final VMInstanceVO instance) {
return instance.getPowerStateUpdateCount() < MAX_CONSECUTIVE_SAME_STATE_UPDATE_COUNT;
}
@ -985,6 +1080,25 @@ public class VMInstanceDaoImpl extends GenericDaoBase<VMInstanceVO, Long> implem
});
}
@Override
public void resetVmPowerStateTracking(List<Long> instanceIds) {
if (CollectionUtils.isEmpty(instanceIds)) {
return;
}
Transaction.execute(new TransactionCallbackNoReturn() {
@Override
public void doInTransactionWithoutResult(TransactionStatus status) {
SearchCriteria<VMInstanceVO> sc = IdsPowerStateSelectSearch.create();
sc.setParameters("id", instanceIds.toArray());
VMInstanceVO vm = createForUpdate();
vm.setPowerStateUpdateCount(0);
vm.setPowerStateUpdateTime(DateUtil.currentGMTTime());
UpdateBuilder ub = getUpdateBuilder(vm);
update(ub, sc, null);
}
});
}
@Override @DB
public void resetHostPowerStateTracking(final long hostId) {
Transaction.execute(new TransactionCallbackNoReturn() {
@ -1060,6 +1174,7 @@ public class VMInstanceDaoImpl extends GenericDaoBase<VMInstanceVO, Long> implem
return searchIncludingRemoved(sc, filter, null, false);
}
@Override
public Pair<List<VMInstanceVO>, Integer> listByVmsNotInClusterUsingPool(long clusterId, long poolId) {
SearchCriteria<VMInstanceVO> sc = VmsNotInClusterUsingPool.create();
sc.setParameters("vmStates", State.Starting, State.Running, State.Stopping, State.Migrating, State.Restoring);
@ -1069,4 +1184,44 @@ public class VMInstanceDaoImpl extends GenericDaoBase<VMInstanceVO, Long> implem
List<VMInstanceVO> uniqueVms = vms.stream().distinct().collect(Collectors.toList());
return new Pair<>(uniqueVms, uniqueVms.size());
}
@Override
public List<VMInstanceVO> listIdServiceOfferingForUpVmsByHostId(Long hostId) {
SearchCriteria<VMInstanceVO> sc = IdServiceOfferingIdSelectSearch.create();
sc.setParameters("host", hostId);
sc.setParameters("states", new Object[] {State.Starting, State.Running, State.Stopping, State.Migrating});
return customSearch(sc, null);
}
@Override
public List<VMInstanceVO> listIdServiceOfferingForVmsMigratingFromHost(Long hostId) {
SearchCriteria<VMInstanceVO> sc = IdServiceOfferingIdSelectSearch.create();
sc.setParameters("lastHost", hostId);
sc.setParameters("state", State.Migrating);
return customSearch(sc, null);
}
@Override
public Map<String, Long> getNameIdMapForVmInstanceNames(Collection<String> names) {
SearchBuilder<VMInstanceVO> sb = createSearchBuilder();
sb.and("name", sb.entity().getInstanceName(), Op.IN);
sb.selectFields(sb.entity().getId(), sb.entity().getInstanceName());
SearchCriteria<VMInstanceVO> sc = sb.create();
sc.setParameters("name", names.toArray());
List<VMInstanceVO> vms = customSearch(sc, null);
return vms.stream()
.collect(Collectors.toMap(VMInstanceVO::getInstanceName, VMInstanceVO::getId));
}
@Override
public Map<String, Long> getNameIdMapForVmIds(Collection<Long> ids) {
SearchBuilder<VMInstanceVO> sb = createSearchBuilder();
sb.and("id", sb.entity().getId(), Op.IN);
sb.selectFields(sb.entity().getId(), sb.entity().getInstanceName());
SearchCriteria<VMInstanceVO> sc = sb.create();
sc.setParameters("id", ids.toArray());
List<VMInstanceVO> vms = customSearch(sc, null);
return vms.stream()
.collect(Collectors.toMap(VMInstanceVO::getInstanceName, VMInstanceVO::getId));
}
}

2
engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/ResourceDetailsDao.java
View File

@ -88,6 +88,8 @@ public interface ResourceDetailsDao<R extends ResourceDetail> extends GenericDao
public Map<String, String> listDetailsKeyPairs(long resourceId);
Map<String, String> listDetailsKeyPairs(long resourceId, List<String> keys);
public Map<String, String> listDetailsKeyPairs(long resourceId, boolean forDisplay);
Map<String, Boolean> listDetailsVisibility(long resourceId);

15
engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/ResourceDetailsDaoBase.java
View File

@ -19,6 +19,7 @@ package org.apache.cloudstack.resourcedetail;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.apache.cloudstack.api.ResourceDetail;
import org.apache.commons.collections.CollectionUtils;
@ -91,6 +92,20 @@ public abstract class ResourceDetailsDaoBase<R extends ResourceDetail> extends G
return details;
}
@Override
public Map<String, String> listDetailsKeyPairs(long resourceId, List<String> keys) {
SearchBuilder<R> sb = createSearchBuilder();
sb.and("resourceId", sb.entity().getResourceId(), SearchCriteria.Op.EQ);
sb.and("name", sb.entity().getName(), SearchCriteria.Op.IN);
sb.done();
SearchCriteria<R> sc = sb.create();
sc.setParameters("resourceId", resourceId);
sc.setParameters("name", keys.toArray());
List<R> results = search(sc, null);
return results.stream().collect(Collectors.toMap(R::getName, R::getValue));
}
public Map<String, Boolean> listDetailsVisibility(long resourceId) {
SearchCriteria<R> sc = AllFieldsSearch.create();
sc.setParameters("resourceId", resourceId);

6
engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java
View File

@ -28,20 +28,20 @@ import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
import com.cloud.storage.Storage;
import com.cloud.utils.Pair;
import com.cloud.utils.db.Filter;
import org.apache.commons.collections.CollectionUtils;
import com.cloud.host.Status;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.storage.ScopeType;
import com.cloud.storage.Storage;
import com.cloud.storage.StoragePoolHostVO;
import com.cloud.storage.StoragePoolStatus;
import com.cloud.storage.StoragePoolTagVO;
import com.cloud.storage.dao.StoragePoolHostDao;
import com.cloud.storage.dao.StoragePoolTagsDao;
import com.cloud.utils.Pair;
import com.cloud.utils.db.DB;
import com.cloud.utils.db.Filter;
import com.cloud.utils.db.GenericDaoBase;
import com.cloud.utils.db.GenericSearchBuilder;
import com.cloud.utils.db.JoinBuilder;

8
engine/schema/src/main/resources/META-INF/db/views/cloud.network_offering_view.sql
View File

@ -76,13 +76,9 @@ SELECT
FROM
`cloud`.`network_offerings`
LEFT JOIN
`cloud`.`network_offering_details` AS `domain_details` ON `domain_details`.`network_offering_id` = `network_offerings`.`id` AND `domain_details`.`name`='domainid'
`cloud`.`domain` AS `domain` ON `domain`.id IN (SELECT value from `network_offering_details` where `name` = 'domainid' and `network_offering_id` = `network_offerings`.`id`)
LEFT JOIN
`cloud`.`domain` AS `domain` ON FIND_IN_SET(`domain`.`id`, `domain_details`.`value`)
LEFT JOIN
`cloud`.`network_offering_details` AS `zone_details` ON `zone_details`.`network_offering_id` = `network_offerings`.`id` AND `zone_details`.`name`='zoneid'
LEFT JOIN
`cloud`.`data_center` AS `zone` ON FIND_IN_SET(`zone`.`id`, `zone_details`.`value`)
`cloud`.`data_center` AS `zone` ON `zone`.`id` IN (SELECT value from `network_offering_details` where `name` = 'zoneid' and `network_offering_id` = `network_offerings`.`id`)
LEFT JOIN
`cloud`.`network_offering_details` AS `offering_details` ON `offering_details`.`network_offering_id` = `network_offerings`.`id` AND `offering_details`.`name`='internetProtocol'
GROUP BY

99
engine/schema/src/test/java/com/cloud/capacity/dao/CapacityDaoImplTest.java Normal file
View File

@ -0,0 +1,99 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package com.cloud.capacity.dao;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mockito;
import org.mockito.Spy;
import org.mockito.junit.MockitoJUnitRunner;
import com.cloud.capacity.CapacityVO;
import com.cloud.utils.db.SearchBuilder;
import com.cloud.utils.db.SearchCriteria;
@RunWith(MockitoJUnitRunner.class)
public class CapacityDaoImplTest {
@Spy
@InjectMocks
CapacityDaoImpl capacityDao = new CapacityDaoImpl();
private SearchBuilder<CapacityVO> searchBuilder;
private SearchCriteria<CapacityVO> searchCriteria;
@Before
public void setUp() {
searchBuilder = mock(SearchBuilder.class);
CapacityVO capacityVO = mock(CapacityVO.class);
when(searchBuilder.entity()).thenReturn(capacityVO);
searchCriteria = mock(SearchCriteria.class);
doReturn(searchBuilder).when(capacityDao).createSearchBuilder();
when(searchBuilder.create()).thenReturn(searchCriteria);
}
@Test
public void testListByHostIdTypes() {
// Prepare inputs
Long hostId = 1L;
List<Short> capacityTypes = Arrays.asList((short)1, (short)2);
CapacityVO capacity1 = new CapacityVO();
CapacityVO capacity2 = new CapacityVO();
List<CapacityVO> mockResult = Arrays.asList(capacity1, capacity2);
doReturn(mockResult).when(capacityDao).listBy(any(SearchCriteria.class));
List<CapacityVO> result = capacityDao.listByHostIdTypes(hostId, capacityTypes);
verify(searchBuilder).and(eq("hostId"), any(), eq(SearchCriteria.Op.EQ));
verify(searchBuilder).and(eq("type"), any(), eq(SearchCriteria.Op.IN));
verify(searchBuilder).done();
verify(searchCriteria).setParameters("hostId", hostId);
verify(searchCriteria).setParameters("type", capacityTypes.toArray());
verify(capacityDao).listBy(searchCriteria);
assertEquals(2, result.size());
assertSame(capacity1, result.get(0));
assertSame(capacity2, result.get(1));
}
@Test
public void testListByHostIdTypesEmptyResult() {
Long hostId = 1L;
List<Short> capacityTypes = Arrays.asList((short)1, (short)2);
doReturn(Collections.emptyList()).when(capacityDao).listBy(any(SearchCriteria.class));
List<CapacityVO> result = capacityDao.listByHostIdTypes(hostId, capacityTypes);
verify(searchBuilder).and(Mockito.eq("hostId"), any(), eq(SearchCriteria.Op.EQ));
verify(searchBuilder).and(eq("type"), any(), eq(SearchCriteria.Op.IN));
verify(searchBuilder).done();
verify(searchCriteria).setParameters("hostId", hostId);
verify(searchCriteria).setParameters("type", capacityTypes.toArray());
verify(capacityDao).listBy(searchCriteria);
assertTrue(result.isEmpty());
}
}

78
engine/schema/src/test/java/com/cloud/dc/dao/ClusterDaoImplTest.java Normal file
View File

@ -0,0 +1,78 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package com.cloud.dc.dao;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.isNull;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Spy;
import org.mockito.junit.MockitoJUnitRunner;
import com.cloud.dc.ClusterVO;
import com.cloud.utils.db.GenericSearchBuilder;
import com.cloud.utils.db.SearchBuilder;
@RunWith(MockitoJUnitRunner.class)
public class ClusterDaoImplTest {
@Spy
@InjectMocks
ClusterDaoImpl clusterDao = new ClusterDaoImpl();
private GenericSearchBuilder<ClusterVO, Long> genericSearchBuilder;
@Before
public void setUp() {
genericSearchBuilder = mock(SearchBuilder.class);
ClusterVO entityVO = mock(ClusterVO.class);
when(genericSearchBuilder.entity()).thenReturn(entityVO);
doReturn(genericSearchBuilder).when(clusterDao).createSearchBuilder(Long.class);
}
@Test
public void testListAllIds() {
List<Long> mockIds = Arrays.asList(1L, 2L, 3L);
doReturn(mockIds).when(clusterDao).customSearch(any(), isNull());
List<Long> result = clusterDao.listAllIds();
verify(clusterDao).customSearch(genericSearchBuilder.create(), null);
assertEquals(3, result.size());
assertEquals(Long.valueOf(1L), result.get(0));
assertEquals(Long.valueOf(2L), result.get(1));
assertEquals(Long.valueOf(3L), result.get(2));
}
@Test
public void testListAllIdsEmptyResult() {
doReturn(Collections.emptyList()).when(clusterDao).customSearch(any(), isNull());
List<Long> result = clusterDao.listAllIds();
verify(clusterDao).customSearch(genericSearchBuilder.create(), null);
assertTrue(result.isEmpty());
}
}

184
engine/schema/src/test/java/com/cloud/host/dao/HostDaoImplTest.java Normal file
View File

@ -0,0 +1,184 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package com.cloud.host.dao;
import java.util.List;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.Spy;
import org.mockito.junit.MockitoJUnitRunner;
import com.cloud.host.Host;
import com.cloud.host.HostVO;
import com.cloud.host.Status;
import com.cloud.hypervisor.Hypervisor;
import com.cloud.resource.ResourceState;
import com.cloud.utils.Pair;
import com.cloud.utils.db.GenericDaoBase;
import com.cloud.utils.db.GenericSearchBuilder;
import com.cloud.utils.db.SearchBuilder;
import com.cloud.utils.db.SearchCriteria;
@RunWith(MockitoJUnitRunner.class)
public class HostDaoImplTest {
@Spy
HostDaoImpl hostDao = new HostDaoImpl();
@Mock
private SearchBuilder<HostVO> mockSearchBuilder;
@Mock
private SearchCriteria<HostVO> mockSearchCriteria;
@Test
public void testCountUpAndEnabledHostsInZone() {
long testZoneId = 100L;
hostDao.HostTypeCountSearch = mockSearchBuilder;
Mockito.when(mockSearchBuilder.create()).thenReturn(mockSearchCriteria);
Mockito.doNothing().when(mockSearchCriteria).setParameters(Mockito.anyString(), Mockito.any());
int expected = 5;
Mockito.doReturn(expected).when(hostDao).getCount(mockSearchCriteria);
Integer count = hostDao.countUpAndEnabledHostsInZone(testZoneId);
Assert.assertSame(expected, count);
Mockito.verify(mockSearchCriteria).setParameters("type", Host.Type.Routing);
Mockito.verify(mockSearchCriteria).setParameters("resourceState", ResourceState.Enabled);
Mockito.verify(mockSearchCriteria).setParameters("zoneId", testZoneId);
Mockito.verify(hostDao).getCount(mockSearchCriteria);
}
@Test
public void testCountAllHostsAndCPUSocketsByType() {
Host.Type type = Host.Type.Routing;
GenericDaoBase.SumCount mockSumCount = new GenericDaoBase.SumCount();
mockSumCount.count = 10;
mockSumCount.sum = 20;
HostVO host = Mockito.mock(HostVO.class);
GenericSearchBuilder<HostVO, GenericDaoBase.SumCount> sb = Mockito.mock(GenericSearchBuilder.class);
Mockito.when(sb.entity()).thenReturn(host);
Mockito.doReturn(sb).when(hostDao).createSearchBuilder(GenericDaoBase.SumCount.class);
SearchCriteria<GenericDaoBase.SumCount> sc = Mockito.mock(SearchCriteria.class);
Mockito.when(sb.create()).thenReturn(sc);
Mockito.doReturn(List.of(mockSumCount)).when(hostDao).customSearch(Mockito.any(SearchCriteria.class), Mockito.any());
Pair<Integer, Integer> result = hostDao.countAllHostsAndCPUSocketsByType(type);
Assert.assertEquals(10, result.first().intValue());
Assert.assertEquals(20, result.second().intValue());
Mockito.verify(sc).setParameters("type", type);
}
@Test
public void testIsHostUp() {
long testHostId = 101L;
List<Status> statuses = List.of(Status.Up);
HostVO host = Mockito.mock(HostVO.class);
GenericSearchBuilder<HostVO, Status> sb = Mockito.mock(GenericSearchBuilder.class);
Mockito.when(sb.entity()).thenReturn(host);
SearchCriteria<Status> sc = Mockito.mock(SearchCriteria.class);
Mockito.when(sb.create()).thenReturn(sc);
Mockito.doReturn(sb).when(hostDao).createSearchBuilder(Status.class);
Mockito.doReturn(statuses).when(hostDao).customSearch(Mockito.any(SearchCriteria.class), Mockito.any());
boolean result = hostDao.isHostUp(testHostId);
Assert.assertTrue("Host should be up", result);
Mockito.verify(sc).setParameters("id", testHostId);
Mockito.verify(hostDao).customSearch(sc, null);
}
@Test
public void testFindHostIdsByZoneClusterResourceStateTypeAndHypervisorType() {
Long zoneId = 1L;
Long clusterId = 2L;
List<ResourceState> resourceStates = List.of(ResourceState.Enabled);
List<Host.Type> types = List.of(Host.Type.Routing);
List<Hypervisor.HypervisorType> hypervisorTypes = List.of(Hypervisor.HypervisorType.KVM);
List<Long> mockResults = List.of(1001L, 1002L); // Mocked result
HostVO host = Mockito.mock(HostVO.class);
GenericSearchBuilder<HostVO, Long> sb = Mockito.mock(GenericSearchBuilder.class);
Mockito.when(sb.entity()).thenReturn(host);
SearchCriteria<Long> sc = Mockito.mock(SearchCriteria.class);
Mockito.when(sb.create()).thenReturn(sc);
Mockito.when(sb.and()).thenReturn(sb);
Mockito.doReturn(sb).when(hostDao).createSearchBuilder(Long.class);
Mockito.doReturn(mockResults).when(hostDao).customSearch(Mockito.any(SearchCriteria.class), Mockito.any());
List<Long> hostIds = hostDao.findHostIdsByZoneClusterResourceStateTypeAndHypervisorType(
zoneId, clusterId, resourceStates, types, hypervisorTypes);
Assert.assertEquals(mockResults, hostIds);
Mockito.verify(sc).setParameters("zoneId", zoneId);
Mockito.verify(sc).setParameters("clusterId", clusterId);
Mockito.verify(sc).setParameters("resourceState", resourceStates.toArray());
Mockito.verify(sc).setParameters("type", types.toArray());
Mockito.verify(sc).setParameters("hypervisorTypes", hypervisorTypes.toArray());
}
@Test
public void testListDistinctHypervisorTypes() {
Long zoneId = 1L;
List<Hypervisor.HypervisorType> mockResults = List.of(Hypervisor.HypervisorType.KVM, Hypervisor.HypervisorType.XenServer);
HostVO host = Mockito.mock(HostVO.class);
GenericSearchBuilder<HostVO, Hypervisor.HypervisorType> sb = Mockito.mock(GenericSearchBuilder.class);
Mockito.when(sb.entity()).thenReturn(host);
SearchCriteria<Hypervisor.HypervisorType> sc = Mockito.mock(SearchCriteria.class);
Mockito.when(sb.create()).thenReturn(sc);
Mockito.doReturn(sb).when(hostDao).createSearchBuilder(Hypervisor.HypervisorType.class);
Mockito.doReturn(mockResults).when(hostDao).customSearch(Mockito.any(SearchCriteria.class), Mockito.any());
List<Hypervisor.HypervisorType> hypervisorTypes = hostDao.listDistinctHypervisorTypes(zoneId);
Assert.assertEquals(mockResults, hypervisorTypes);
Mockito.verify(sc).setParameters("zoneId", zoneId);
Mockito.verify(sc).setParameters("type", Host.Type.Routing);
}
@Test
public void testListByIds() {
List<Long> ids = List.of(101L, 102L);
List<HostVO> mockResults = List.of(Mockito.mock(HostVO.class), Mockito.mock(HostVO.class));
hostDao.IdsSearch = mockSearchBuilder;
Mockito.when(mockSearchBuilder.create()).thenReturn(mockSearchCriteria);
Mockito.doReturn(mockResults).when(hostDao).search(Mockito.any(SearchCriteria.class), Mockito.any());
List<HostVO> hosts = hostDao.listByIds(ids);
Assert.assertEquals(mockResults, hosts);
Mockito.verify(mockSearchCriteria).setParameters("id", ids.toArray());
Mockito.verify(hostDao).search(mockSearchCriteria, null);
}
@Test
public void testListIdsBy() {
Host.Type type = Host.Type.Routing;
Status status = Status.Up;
ResourceState resourceState = ResourceState.Enabled;
Hypervisor.HypervisorType hypervisorType = Hypervisor.HypervisorType.KVM;
Long zoneId = 1L, podId = 2L, clusterId = 3L;
List<Long> mockResults = List.of(1001L, 1002L);
HostVO host = Mockito.mock(HostVO.class);
GenericSearchBuilder<HostVO, Long> sb = Mockito.mock(GenericSearchBuilder.class);
Mockito.when(sb.entity()).thenReturn(host);
SearchCriteria<Long> sc = Mockito.mock(SearchCriteria.class);
Mockito.when(sb.create()).thenReturn(sc);
Mockito.doReturn(sb).when(hostDao).createSearchBuilder(Long.class);
Mockito.doReturn(mockResults).when(hostDao).customSearch(Mockito.any(SearchCriteria.class), Mockito.any());
List<Long> hostIds = hostDao.listIdsBy(type, status, resourceState, hypervisorType, zoneId, podId, clusterId);
Assert.assertEquals(mockResults, hostIds);
Mockito.verify(sc).setParameters("type", type);
Mockito.verify(sc).setParameters("status", status);
Mockito.verify(sc).setParameters("resourceState", resourceState);
Mockito.verify(sc).setParameters("hypervisorType", hypervisorType);
Mockito.verify(sc).setParameters("zoneId", zoneId);
Mockito.verify(sc).setParameters("podId", podId);
Mockito.verify(sc).setParameters("clusterId", clusterId);
}
}

7
engine/schema/src/test/java/com/cloud/usage/dao/UsageStorageDaoImplTest.java
View File

@ -23,12 +23,9 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.sql.PreparedStatement;
import com.cloud.utils.DateUtil;
import com.cloud.utils.db.TransactionLegacy;
import java.util.Date;
import java.util.TimeZone;
import com.cloud.usage.UsageStorageVO;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
@ -36,6 +33,10 @@ import org.mockito.MockedStatic;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
import com.cloud.usage.UsageStorageVO;
import com.cloud.utils.DateUtil;
import com.cloud.utils.db.TransactionLegacy;
@RunWith(MockitoJUnitRunner.class)
public class UsageStorageDaoImplTest {

181
engine/schema/src/test/java/org/apache/cloudstack/resourcedetail/ResourceDetailsDaoBaseTest.java Normal file
View File

@ -0,0 +1,181 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.resourcedetail;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.ArgumentMatchers.isNull;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
import org.apache.cloudstack.api.ResourceDetail;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Spy;
import org.mockito.junit.MockitoJUnitRunner;
import com.cloud.utils.db.SearchBuilder;
import com.cloud.utils.db.SearchCriteria;
@RunWith(MockitoJUnitRunner.class)
public class ResourceDetailsDaoBaseTest {
@Spy
@InjectMocks
TestDetailsDao testDetailsDao = new TestDetailsDao();
private SearchBuilder<TestDetailVO> searchBuilder;
private SearchCriteria<TestDetailVO> searchCriteria;
@Before
public void setUp() {
searchBuilder = mock(SearchBuilder.class);
searchCriteria = mock(SearchCriteria.class);
TestDetailVO entityVO = mock(TestDetailVO.class);
when(searchBuilder.entity()).thenReturn(entityVO);
searchCriteria = mock(SearchCriteria.class);
doReturn(searchBuilder).when(testDetailsDao).createSearchBuilder();
when(searchBuilder.create()).thenReturn(searchCriteria);
}
@Test
public void testListDetailsKeyPairs() {
long resourceId = 1L;
List<String> keys = Arrays.asList("key1", "key2");
TestDetailVO result1 = mock(TestDetailVO.class);
when(result1.getName()).thenReturn("key1");
when(result1.getValue()).thenReturn("value1");
TestDetailVO result2 = mock(TestDetailVO.class);
when(result2.getName()).thenReturn("key2");
when(result2.getValue()).thenReturn("value2");
List<TestDetailVO> mockResults = Arrays.asList(result1, result2);
doReturn(mockResults).when(testDetailsDao).search(any(SearchCriteria.class), isNull());
Map<String, String> result = testDetailsDao.listDetailsKeyPairs(resourceId, keys);
verify(searchBuilder).and(eq("resourceId"), any(), eq(SearchCriteria.Op.EQ));
verify(searchBuilder).and(eq("name"), any(), eq(SearchCriteria.Op.IN));
verify(searchBuilder).done();
verify(searchCriteria).setParameters("resourceId", resourceId);
verify(searchCriteria).setParameters("name", keys.toArray());
verify(testDetailsDao).search(searchCriteria, null);
assertEquals(2, result.size());
assertEquals("value1", result.get("key1"));
assertEquals("value2", result.get("key2"));
}
@Test
public void testListDetailsKeyPairsEmptyResult() {
long resourceId = 1L;
List<String> keys = Arrays.asList("key1", "key2");
doReturn(Collections.emptyList()).when(testDetailsDao).search(any(SearchCriteria.class), isNull());
Map<String, String> result = testDetailsDao.listDetailsKeyPairs(resourceId, keys);
verify(searchBuilder).and(eq("resourceId"), any(), eq(SearchCriteria.Op.EQ));
verify(searchBuilder).and(eq("name"), any(), eq(SearchCriteria.Op.IN));
verify(searchBuilder).done();
verify(searchCriteria).setParameters("resourceId", resourceId);
verify(searchCriteria).setParameters("name", keys.toArray());
verify(testDetailsDao).search(searchCriteria, null);
assertTrue(result.isEmpty());
}
protected static class TestDetailsDao extends ResourceDetailsDaoBase<TestDetailVO> {
@Override
public void addDetail(long resourceId, String key, String value, boolean display) {
super.addDetail(new TestDetailVO(resourceId, key, value, display));
}
}
@Entity
@Table(name = "test_details")
protected static class TestDetailVO implements ResourceDetail {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
@Column(name = "id")
private long id;
@Column(name = "resource_id")
private long resourceId;
@Column(name = "name")
private String name;
@Column(name = "value")
private String value;
@Column(name = "display")
private boolean display = true;
public TestDetailVO() {
}
public TestDetailVO(long resourceId, String name, String value, boolean display) {
this.resourceId = resourceId;
this.name = name;
this.value = value;
this.display = display;
}
@Override
public long getId() {
return id;
}
@Override
public String getName() {
return name;
}
@Override
public String getValue() {
return value;
}
@Override
public long getResourceId() {
return resourceId;
}
@Override
public boolean isDisplay() {
return display;
}
public void setName(String name) {
this.name = name;
}
public void setValue(String value) {
this.value = value;
}
}
}

39
engine/schema/src/test/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImplTest.java
View File

@ -17,12 +17,17 @@
package org.apache.cloudstack.storage.datastore.db;
import static org.mockito.ArgumentMatchers.nullable;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.isNull;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.IOException;
import java.sql.SQLException;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -34,13 +39,15 @@ import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.MockitoJUnitRunner;
import com.cloud.storage.ScopeType;
import com.cloud.storage.dao.StoragePoolHostDao;
import com.cloud.storage.dao.StoragePoolTagsDao;
import com.cloud.utils.db.GenericSearchBuilder;
import com.cloud.utils.db.SearchBuilder;
import junit.framework.TestCase;
import org.mockito.junit.MockitoJUnitRunner;
@RunWith(MockitoJUnitRunner.class)
public class PrimaryDataStoreDaoImplTest extends TestCase {
@ -59,6 +66,8 @@ public class PrimaryDataStoreDaoImplTest extends TestCase {
@Mock
StoragePoolVO storagePoolVO;
private GenericSearchBuilder<StoragePoolVO, Long> genericSearchBuilder;
private static final String STORAGE_TAG_1 = "NFS-A";
private static final String STORAGE_TAG_2 = "NFS-B";
private static final String[] STORAGE_TAGS_ARRAY = {STORAGE_TAG_1, STORAGE_TAG_2};
@ -155,4 +164,32 @@ public class PrimaryDataStoreDaoImplTest extends TestCase {
String expectedSql = primaryDataStoreDao.DetailsSqlPrefix + SQL_VALUES + primaryDataStoreDao.DetailsSqlSuffix;
verify(primaryDataStoreDao).searchStoragePoolsPreparedStatement(expectedSql, DATACENTER_ID, POD_ID, CLUSTER_ID, SCOPE, STORAGE_POOL_DETAILS.size());
}
@Test
public void testListAllIds() {
GenericSearchBuilder<StoragePoolVO, Long> genericSearchBuilder = mock(SearchBuilder.class);
StoragePoolVO entityVO = mock(StoragePoolVO.class);
when(genericSearchBuilder.entity()).thenReturn(entityVO);
doReturn(genericSearchBuilder).when(primaryDataStoreDao).createSearchBuilder(Long.class);
List<Long> mockIds = Arrays.asList(1L, 2L, 3L);
doReturn(mockIds).when(primaryDataStoreDao).customSearch(any(), isNull());
List<Long> result = primaryDataStoreDao.listAllIds();
verify(primaryDataStoreDao).customSearch(genericSearchBuilder.create(), null);
assertEquals(3, result.size());
assertEquals(Long.valueOf(1L), result.get(0));
assertEquals(Long.valueOf(2L), result.get(1));
assertEquals(Long.valueOf(3L), result.get(2));
}
@Test
public void testListAllIdsEmptyResult() {
GenericSearchBuilder<StoragePoolVO, Long> genericSearchBuilder = mock(SearchBuilder.class);
StoragePoolVO entityVO = mock(StoragePoolVO.class);
when(genericSearchBuilder.entity()).thenReturn(entityVO);
doReturn(genericSearchBuilder).when(primaryDataStoreDao).createSearchBuilder(Long.class);
doReturn(Collections.emptyList()).when(primaryDataStoreDao).customSearch(any(), isNull());
List<Long> result = primaryDataStoreDao.listAllIds();
verify(primaryDataStoreDao).customSearch(genericSearchBuilder.create(), null);
assertTrue(result.isEmpty());
}
}

4
framework/agent-lb/src/main/java/org/apache/cloudstack/agent/lb/IndirectAgentLBAlgorithm.java
View File

@ -42,4 +42,8 @@ public interface IndirectAgentLBAlgorithm {
* @return true if the lists are equal, false if not
*/
boolean compare(final List<String> msList, final List<String> receivedMsList);
default boolean isHostListNeeded() {
return false;
}
}

3
framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostPeerDaoImpl.java
View File

@ -121,8 +121,7 @@ public class ManagementServerHostPeerDaoImpl extends GenericDaoBase<ManagementSe
sc.setParameters("peerRunid", runid);
sc.setParameters("peerState", state);
List<ManagementServerHostPeerVO> l = listBy(sc);
return l.size();
return getCount(sc);
}
@Override

14
framework/config/src/main/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImpl.java
View File

@ -23,7 +23,6 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import javax.annotation.PostConstruct;
import javax.inject.Inject;
@ -36,6 +35,7 @@ import org.apache.cloudstack.framework.config.ScopedConfigStorage;
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.apache.cloudstack.framework.config.dao.ConfigurationGroupDao;
import org.apache.cloudstack.framework.config.dao.ConfigurationSubGroupDao;
import org.apache.cloudstack.utils.cache.LazyCache;
import org.apache.commons.lang.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
@ -44,8 +44,6 @@ import org.apache.logging.log4j.Logger;
import com.cloud.utils.Pair;
import com.cloud.utils.Ternary;
import com.cloud.utils.exception.CloudRuntimeException;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
/**
* ConfigDepotImpl implements the ConfigDepot and ConfigDepotAdmin interface.
@ -87,17 +85,15 @@ public class ConfigDepotImpl implements ConfigDepot, ConfigDepotAdmin {
List<ScopedConfigStorage> _scopedStorages;
Set<Configurable> _configured = Collections.synchronizedSet(new HashSet<Configurable>());
Set<String> newConfigs = Collections.synchronizedSet(new HashSet<>());
Cache<String, String> configCache;
LazyCache<String, String> configCache;
private HashMap<String, Pair<String, ConfigKey<?>>> _allKeys = new HashMap<String, Pair<String, ConfigKey<?>>>(1007);
HashMap<ConfigKey.Scope, Set<ConfigKey<?>>> _scopeLevelConfigsMap = new HashMap<ConfigKey.Scope, Set<ConfigKey<?>>>();
public ConfigDepotImpl() {
configCache = Caffeine.newBuilder()
.maximumSize(512)
.expireAfterWrite(CONFIG_CACHE_EXPIRE_SECONDS, TimeUnit.SECONDS)
.build();
configCache = new LazyCache<>(512,
CONFIG_CACHE_EXPIRE_SECONDS, this::getConfigStringValueInternal);
ConfigKey.init(this);
createEmptyScopeLevelMappings();
}
@ -311,7 +307,7 @@ public class ConfigDepotImpl implements ConfigDepot, ConfigDepotAdmin {
@Override
public String getConfigStringValue(String key, ConfigKey.Scope scope, Long scopeId) {
return configCache.get(getConfigCacheKey(key, scope, scopeId), this::getConfigStringValueInternal);
return configCache.get(getConfigCacheKey(key, scope, scopeId));
}
@Override

5
framework/db/src/main/java/com/cloud/utils/db/GenericDao.java
View File

@ -148,6 +148,11 @@ public interface GenericDao<T, ID extends Serializable> {
*/
List<T> listAll(Filter filter);
/**
* Look IDs for all active rows.
*/
List<ID> listAllIds();
/**
* Search for the entity beans
* @param sc

36
framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java
View File

@ -1218,6 +1218,35 @@ public abstract class GenericDaoBase<T, ID extends Serializable> extends Compone
return executeList(sql.toString());
}
private Object getIdObject() {
T entity = (T)_searchEnhancer.create();
try {
Method m = _entityBeanType.getMethod("getId");
return m.invoke(entity);
} catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException ignored) {
logger.warn("Unable to get ID object for entity: {}", _entityBeanType.getSimpleName());
}
return null;
}
@Override
public List<ID> listAllIds() {
Object idObj = getIdObject();
if (idObj == null) {
return Collections.emptyList();
}
Class<ID> clazz = (Class<ID>)idObj.getClass();
GenericSearchBuilder<T, ID> sb = createSearchBuilder(clazz);
try {
Method m = sb.entity().getClass().getMethod("getId");
sb.selectFields(m.invoke(sb.entity()));
} catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException ignored) {
return Collections.emptyList();
}
sb.done();
return customSearch(sb.create(), null);
}
@Override
public boolean expunge(final ID id) {
final TransactionLegacy txn = TransactionLegacy.currentTxn();
@ -2445,4 +2474,11 @@ public abstract class GenericDaoBase<T, ID extends Serializable> extends Compone
}
}
public static class SumCount {
public long sum;
public long count;
public SumCount() {
}
}
}

1
framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/VmWorkJobDao.java
View File

@ -40,4 +40,5 @@ public interface VmWorkJobDao extends GenericDao<VmWorkJobVO, Long> {
void expungeLeftoverWorkJobs(long msid);
int expungeByVmList(List<Long> vmIds, Long batchSize);
List<Long> listVmIdsWithPendingJob();
}

16
framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/VmWorkJobDaoImpl.java
View File

@ -24,6 +24,7 @@ import java.util.List;
import javax.annotation.PostConstruct;
import javax.inject.Inject;
import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO;
import org.apache.cloudstack.framework.jobs.impl.VmWorkJobVO;
import org.apache.cloudstack.framework.jobs.impl.VmWorkJobVO.Step;
import org.apache.cloudstack.jobs.JobInfo;
@ -32,6 +33,8 @@ import org.apache.commons.collections.CollectionUtils;
import com.cloud.utils.DateUtil;
import com.cloud.utils.db.Filter;
import com.cloud.utils.db.GenericDaoBase;
import com.cloud.utils.db.GenericSearchBuilder;
import com.cloud.utils.db.JoinBuilder;
import com.cloud.utils.db.SearchBuilder;
import com.cloud.utils.db.SearchCriteria;
import com.cloud.utils.db.SearchCriteria.Op;
@ -224,4 +227,17 @@ public class VmWorkJobDaoImpl extends GenericDaoBase<VmWorkJobVO, Long> implemen
sc.setParameters("vmIds", vmIds.toArray());
return batchExpunge(sc, batchSize);
}
@Override
public List<Long> listVmIdsWithPendingJob() {
GenericSearchBuilder<VmWorkJobVO, Long> sb = createSearchBuilder(Long.class);
SearchBuilder<AsyncJobVO> asyncJobSearch = _baseJobDao.createSearchBuilder();
asyncJobSearch.and("status", asyncJobSearch.entity().getStatus(), SearchCriteria.Op.EQ);
sb.join("asyncJobSearch", asyncJobSearch, sb.entity().getId(), asyncJobSearch.entity().getId(), JoinBuilder.JoinType.INNER);
sb.and("removed", sb.entity().getRemoved(), Op.NULL);
sb.selectFields(sb.entity().getVmInstanceId());
SearchCriteria<Long> sc = sb.create();
sc.setJoinParameters("asyncJobSearch", "status", JobInfo.Status.IN_PROGRESS);
return customSearch(sc, null);
}
}

94
framework/jobs/src/test/java/org/apache/cloudstack/framework/jobs/dao/VmWorkJobDaoImplTest.java
View File

@ -16,27 +16,69 @@
// under the License.
package org.apache.cloudstack.framework.jobs.dao;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.anyLong;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.eq;
import static org.mockito.Mockito.isNull;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO;
import org.apache.cloudstack.framework.jobs.impl.VmWorkJobVO;
import org.apache.cloudstack.jobs.JobInfo;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.MockitoJUnitRunner;
import org.mockito.stubbing.Answer;
import com.cloud.utils.db.GenericSearchBuilder;
import com.cloud.utils.db.JoinBuilder;
import com.cloud.utils.db.SearchBuilder;
import com.cloud.utils.db.SearchCriteria;
@RunWith(MockitoJUnitRunner.class)
public class VmWorkJobDaoImplTest {
@Mock
AsyncJobDao asyncJobDao;
@Spy
@InjectMocks
VmWorkJobDaoImpl vmWorkJobDaoImpl;
private GenericSearchBuilder<VmWorkJobVO, Long> genericVmWorkJobSearchBuilder;
private SearchBuilder<AsyncJobVO> asyncJobSearchBuilder;
private SearchCriteria<Long> searchCriteria;
@Before
public void setUp() {
genericVmWorkJobSearchBuilder = mock(GenericSearchBuilder.class);
VmWorkJobVO entityVO = mock(VmWorkJobVO.class);
when(genericVmWorkJobSearchBuilder.entity()).thenReturn(entityVO);
asyncJobSearchBuilder = mock(SearchBuilder.class);
AsyncJobVO asyncJobVO = mock(AsyncJobVO.class);
when(asyncJobSearchBuilder.entity()).thenReturn(asyncJobVO);
searchCriteria = mock(SearchCriteria.class);
when(vmWorkJobDaoImpl.createSearchBuilder(Long.class)).thenReturn(genericVmWorkJobSearchBuilder);
when(asyncJobDao.createSearchBuilder()).thenReturn(asyncJobSearchBuilder);
when(genericVmWorkJobSearchBuilder.create()).thenReturn(searchCriteria);
}
@Test
public void testExpungeByVmListNoVms() {
Assert.assertEquals(0, vmWorkJobDaoImpl.expungeByVmList(
@ -47,22 +89,52 @@ public class VmWorkJobDaoImplTest {
@Test
public void testExpungeByVmList() {
SearchBuilder<VmWorkJobVO> sb = Mockito.mock(SearchBuilder.class);
SearchCriteria<VmWorkJobVO> sc = Mockito.mock(SearchCriteria.class);
Mockito.when(sb.create()).thenReturn(sc);
Mockito.doAnswer((Answer<Integer>) invocationOnMock -> {
SearchBuilder<VmWorkJobVO> sb = mock(SearchBuilder.class);
SearchCriteria<VmWorkJobVO> sc = mock(SearchCriteria.class);
when(sb.create()).thenReturn(sc);
doAnswer((Answer<Integer>) invocationOnMock -> {
Long batchSize = (Long)invocationOnMock.getArguments()[1];
return batchSize == null ? 0 : batchSize.intValue();
}).when(vmWorkJobDaoImpl).batchExpunge(Mockito.any(SearchCriteria.class), Mockito.anyLong());
Mockito.when(vmWorkJobDaoImpl.createSearchBuilder()).thenReturn(sb);
final VmWorkJobVO mockedVO = Mockito.mock(VmWorkJobVO.class);
Mockito.when(sb.entity()).thenReturn(mockedVO);
}).when(vmWorkJobDaoImpl).batchExpunge(any(SearchCriteria.class), anyLong());
when(vmWorkJobDaoImpl.createSearchBuilder()).thenReturn(sb);
final VmWorkJobVO mockedVO = mock(VmWorkJobVO.class);
when(sb.entity()).thenReturn(mockedVO);
List<Long> vmIds = List.of(1L, 2L);
Object[] array = vmIds.toArray();
Long batchSize = 50L;
Assert.assertEquals(batchSize.intValue(), vmWorkJobDaoImpl.expungeByVmList(List.of(1L, 2L), batchSize));
Mockito.verify(sc).setParameters("vmIds", array);
Mockito.verify(vmWorkJobDaoImpl, Mockito.times(1))
verify(sc).setParameters("vmIds", array);
verify(vmWorkJobDaoImpl, times(1))
.batchExpunge(sc, batchSize);
}
@Test
public void testListVmIdsWithPendingJob() {
List<Long> mockVmIds = Arrays.asList(101L, 102L, 103L);
doReturn(mockVmIds).when(vmWorkJobDaoImpl).customSearch(any(SearchCriteria.class), isNull());
List<Long> result = vmWorkJobDaoImpl.listVmIdsWithPendingJob();
verify(genericVmWorkJobSearchBuilder).join(eq("asyncJobSearch"), eq(asyncJobSearchBuilder), any(), any(), eq(JoinBuilder.JoinType.INNER));
verify(genericVmWorkJobSearchBuilder).and(eq("removed"), any(), eq(SearchCriteria.Op.NULL));
verify(genericVmWorkJobSearchBuilder).create();
verify(asyncJobSearchBuilder).and(eq("status"), any(), eq(SearchCriteria.Op.EQ));
verify(searchCriteria).setJoinParameters(eq("asyncJobSearch"), eq("status"), eq(JobInfo.Status.IN_PROGRESS));
verify(vmWorkJobDaoImpl).customSearch(searchCriteria, null);
assertEquals(3, result.size());
assertEquals(Long.valueOf(101L), result.get(0));
assertEquals(Long.valueOf(102L), result.get(1));
assertEquals(Long.valueOf(103L), result.get(2));
}
@Test
public void testListVmIdsWithPendingJobEmptyResult() {
doReturn(Collections.emptyList()).when(vmWorkJobDaoImpl).customSearch(any(SearchCriteria.class), isNull());
List<Long> result = vmWorkJobDaoImpl.listVmIdsWithPendingJob();
verify(genericVmWorkJobSearchBuilder).join(eq("asyncJobSearch"), eq(asyncJobSearchBuilder), any(), any(), eq(JoinBuilder.JoinType.INNER));
verify(genericVmWorkJobSearchBuilder).and(eq("removed"), any(), eq(SearchCriteria.Op.NULL));
verify(genericVmWorkJobSearchBuilder).create();
verify(asyncJobSearchBuilder).and(eq("status"), any(), eq(SearchCriteria.Op.EQ));
verify(searchCriteria).setJoinParameters(eq("asyncJobSearch"), eq("status"), eq(JobInfo.Status.IN_PROGRESS));
verify(vmWorkJobDaoImpl).customSearch(searchCriteria, null);
assertTrue(result.isEmpty());
}
}

70
plugins/acl/dynamic-role-based/src/main/java/org/apache/cloudstack/acl/DynamicRoleBasedAPIAccessChecker.java
View File

@ -26,20 +26,21 @@ import java.util.Set;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
import org.apache.cloudstack.api.APICommand;
import org.apache.cloudstack.acl.RolePermissionEntity.Permission;
import org.apache.cloudstack.api.APICommand;
import org.apache.cloudstack.utils.cache.LazyCache;
import org.apache.commons.lang3.StringUtils;
import com.cloud.exception.PermissionDeniedException;
import com.cloud.exception.UnavailableCommandException;
import com.cloud.user.Account;
import com.cloud.user.AccountService;
import com.cloud.user.User;
import com.cloud.utils.Pair;
import com.cloud.utils.component.AdapterBase;
import com.cloud.utils.component.PluggableService;
import org.apache.commons.lang3.StringUtils;
public class DynamicRoleBasedAPIAccessChecker extends AdapterBase implements APIAclChecker {
@Inject
private AccountService accountService;
@Inject
@ -48,6 +49,9 @@ public class DynamicRoleBasedAPIAccessChecker extends AdapterBase implements API
private List<PluggableService> services;
private Map<RoleType, Set<String>> annotationRoleBasedApisMap = new HashMap<RoleType, Set<String>>();
private LazyCache<Long, Account> accountCache;
private LazyCache<Long, Pair<Role, List<RolePermission>>> rolePermissionsCache;
private int cachePeriod;
protected DynamicRoleBasedAPIAccessChecker() {
super();
@ -99,23 +103,66 @@ public class DynamicRoleBasedAPIAccessChecker extends AdapterBase implements API
annotationRoleBasedApisMap.get(role.getRoleType()).contains(apiName);
}
protected Account getAccountFromId(long accountId) {
return accountService.getAccount(accountId);
}
protected Pair<Role, List<RolePermission>> getRolePermissions(long roleId) {
final Role accountRole = roleService.findRole(roleId);
if (accountRole == null || accountRole.getId() < 1L) {
return new Pair<>(null, null);
}
if (accountRole.getRoleType() == RoleType.Admin && accountRole.getId() == RoleType.Admin.getId()) {
return new Pair<>(accountRole, null);
}
return new Pair<>(accountRole, roleService.findAllPermissionsBy(accountRole.getId()));
}
protected Pair<Role, List<RolePermission>> getRolePermissionsUsingCache(long roleId) {
if (cachePeriod > 0) {
return rolePermissionsCache.get(roleId);
}
return getRolePermissions(roleId);
}
protected Account getAccountFromIdUsingCache(long accountId) {
if (cachePeriod > 0) {
return accountCache.get(accountId);
}
return getAccountFromId(accountId);
}
@Override
public boolean checkAccess(User user, String commandName) throws PermissionDeniedException {
if (!isEnabled()) {
return true;
}
Account account = accountService.getAccount(user.getAccountId());
Account account = getAccountFromIdUsingCache(user.getAccountId());
if (account == null) {
throw new PermissionDeniedException(String.format("The account id [%s] for user id [%s] is null.", user.getAccountId(), user.getUuid()));
throw new PermissionDeniedException(String.format("Account for user id [%s] cannot be found", user.getUuid()));
}
return checkAccess(account, commandName);
Pair<Role, List<RolePermission>> roleAndPermissions = getRolePermissionsUsingCache(account.getRoleId());
final Role accountRole = roleAndPermissions.first();
if (accountRole == null) {
throw new PermissionDeniedException(String.format("Account role for user id [%s] cannot be found.", user.getUuid()));
}
if (accountRole.getRoleType() == RoleType.Admin && accountRole.getId() == RoleType.Admin.getId()) {
logger.info("Account for user id {} is Root Admin or Domain Admin, all APIs are allowed.", user.getUuid());
return true;
}
List<RolePermission> allPermissions = roleAndPermissions.second();
if (checkApiPermissionByRole(accountRole, commandName, allPermissions)) {
return true;
}
throw new UnavailableCommandException(String.format("The API [%s] does not exist or is not available for the account for user id [%s].", commandName, user.getUuid()));
}
public boolean checkAccess(Account account, String commandName) {
final Role accountRole = roleService.findRole(account.getRoleId());
if (accountRole == null || accountRole.getId() < 1L) {
Pair<Role, List<RolePermission>> roleAndPermissions = getRolePermissionsUsingCache(account.getRoleId());
final Role accountRole = roleAndPermissions.first();
if (accountRole == null) {
throw new PermissionDeniedException(String.format("The account [%s] has role null or unknown.", account));
}
@ -160,6 +207,9 @@ public class DynamicRoleBasedAPIAccessChecker extends AdapterBase implements API
@Override
public boolean configure(String name, Map<String, Object> params) throws ConfigurationException {
super.configure(name, params);
cachePeriod = Math.max(0, RoleService.DynamicApiCheckerCachePeriod.value());
accountCache = new LazyCache<>(32, cachePeriod, this::getAccountFromId);
rolePermissionsCache = new LazyCache<>(32, cachePeriod, this::getRolePermissions);
return true;
}

18
plugins/affinity-group-processors/explicit-dedication/src/main/java/org/apache/cloudstack/affinity/ExplicitDedicationProcessor.java
View File

@ -321,13 +321,13 @@ public class ExplicitDedicationProcessor extends AffinityProcessorBase implement
}
}
//add all hosts inside this in includeList
List<HostVO> hostList = _hostDao.listByDataCenterId(dr.getDataCenterId());
for (HostVO host : hostList) {
DedicatedResourceVO dHost = _dedicatedDao.findByHostId(host.getId());
List<Long> hostList = _hostDao.listEnabledIdsByDataCenterId(dr.getDataCenterId());
for (Long hostId : hostList) {
DedicatedResourceVO dHost = _dedicatedDao.findByHostId(hostId);
if (dHost != null && !dedicatedResources.contains(dHost)) {
avoidList.addHost(host.getId());
avoidList.addHost(hostId);
} else {
includeList.addHost(host.getId());
includeList.addHost(hostId);
}
}
}
@ -337,7 +337,7 @@ public class ExplicitDedicationProcessor extends AffinityProcessorBase implement
List<HostPodVO> pods = _podDao.listByDataCenterId(dc.getId());
List<ClusterVO> clusters = _clusterDao.listClustersByDcId(dc.getId());
List<HostVO> hosts = _hostDao.listByDataCenterId(dc.getId());
List<Long> hostIds = _hostDao.listEnabledIdsByDataCenterId(dc.getId());
Set<Long> podsInIncludeList = includeList.getPodsToAvoid();
Set<Long> clustersInIncludeList = includeList.getClustersToAvoid();
Set<Long> hostsInIncludeList = includeList.getHostsToAvoid();
@ -357,9 +357,9 @@ public class ExplicitDedicationProcessor extends AffinityProcessorBase implement
}
}
for (HostVO host : hosts) {
if (hostsInIncludeList != null && !hostsInIncludeList.contains(host.getId())) {
avoidList.addHost(host.getId());
for (Long hostId : hostIds) {
if (hostsInIncludeList != null && !hostsInIncludeList.contains(hostId)) {
avoidList.addHost(hostId);
}
}
return avoidList;

61
plugins/dedicated-resources/src/main/java/org/apache/cloudstack/dedicated/DedicatedResourceManagerImpl.java
View File

@ -23,7 +23,6 @@ import java.util.Map;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
import org.apache.commons.lang3.StringUtils;
import org.apache.cloudstack.affinity.AffinityGroup;
import org.apache.cloudstack.affinity.AffinityGroupService;
import org.apache.cloudstack.affinity.dao.AffinityGroupDao;
@ -45,8 +44,9 @@ import org.apache.cloudstack.api.response.DedicatePodResponse;
import org.apache.cloudstack.api.response.DedicateZoneResponse;
import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.apache.logging.log4j.Logger;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.stereotype.Component;
import com.cloud.configuration.Config;
@ -126,7 +126,7 @@ public class DedicatedResourceManagerImpl implements DedicatedService {
@ActionEvent(eventType = EventTypes.EVENT_DEDICATE_RESOURCE, eventDescription = "dedicating a Zone")
public List<DedicatedResourceVO> dedicateZone(final Long zoneId, final Long domainId, final String accountName) {
Long accountId = null;
List<HostVO> hosts = null;
List<Long> hostIds = null;
if (accountName != null) {
Account caller = CallContext.current().getCallingAccount();
Account owner = _accountMgr.finalizeOwner(caller, accountName, domainId, null);
@ -203,18 +203,20 @@ public class DedicatedResourceManagerImpl implements DedicatedService {
releaseDedicatedResource(null, null, dr.getClusterId(), null);
}
hosts = _hostDao.listByDataCenterId(dc.getId());
for (HostVO host : hosts) {
DedicatedResourceVO dHost = _dedicatedDao.findByHostId(host.getId());
hostIds = _hostDao.listEnabledIdsByDataCenterId(dc.getId());
for (Long hostId : hostIds) {
DedicatedResourceVO dHost = _dedicatedDao.findByHostId(hostId);
if (dHost != null) {
if (!(childDomainIds.contains(dHost.getDomainId()))) {
HostVO host = _hostDao.findById(hostId);
throw new CloudRuntimeException("Host " + host.getName() + " under this Zone " + dc.getName() + " is dedicated to different account/domain");
}
if (accountId != null) {
if (dHost.getAccountId().equals(accountId)) {
hostsToRelease.add(dHost);
} else {
logger.error(String.format("Host %s under this Zone %s is dedicated to different account/domain", host, dc));
HostVO host = _hostDao.findById(hostId);
logger.error("{} under {} is dedicated to different account/domain", host, dc);
throw new CloudRuntimeException("Host " + host.getName() + " under this Zone " + dc.getName() + " is dedicated to different account/domain");
}
} else {
@ -230,7 +232,7 @@ public class DedicatedResourceManagerImpl implements DedicatedService {
}
}
checkHostsSuitabilityForExplicitDedication(accountId, childDomainIds, hosts);
checkHostsSuitabilityForExplicitDedication(accountId, childDomainIds, hostIds);
final Long accountIdFinal = accountId;
return Transaction.execute(new TransactionCallback<List<DedicatedResourceVO>>() {
@ -284,7 +286,7 @@ public class DedicatedResourceManagerImpl implements DedicatedService {
childDomainIds.add(domainId);
checkAccountAndDomain(accountId, domainId);
HostPodVO pod = _podDao.findById(podId);
List<HostVO> hosts = null;
List<Long> hostIds = null;
if (pod == null) {
throw new InvalidParameterValueException("Unable to find pod by id " + podId);
} else {
@ -339,18 +341,20 @@ public class DedicatedResourceManagerImpl implements DedicatedService {
releaseDedicatedResource(null, null, dr.getClusterId(), null);
}
hosts = _hostDao.findByPodId(pod.getId());
for (HostVO host : hosts) {
DedicatedResourceVO dHost = _dedicatedDao.findByHostId(host.getId());
hostIds = _hostDao.listIdsByPodId(pod.getId());
for (Long hostId : hostIds) {
DedicatedResourceVO dHost = _dedicatedDao.findByHostId(hostId);
if (dHost != null) {
if (!(getDomainChildIds(domainId).contains(dHost.getDomainId()))) {
HostVO host = _hostDao.findById(hostId);
throw new CloudRuntimeException("Host " + host.getName() + " under this Pod " + pod.getName() + " is dedicated to different account/domain");
}
if (accountId != null) {
if (dHost.getAccountId().equals(accountId)) {
hostsToRelease.add(dHost);
} else {
logger.error(String.format("Host %s under this Pod %s is dedicated to different account/domain", host, pod));
HostVO host = _hostDao.findById(hostId);
logger.error("{} under this {} is dedicated to different account/domain", host, pod);
throw new CloudRuntimeException("Host " + host.getName() + " under this Pod " + pod.getName() + " is dedicated to different account/domain");
}
} else {
@ -366,7 +370,7 @@ public class DedicatedResourceManagerImpl implements DedicatedService {
}
}
checkHostsSuitabilityForExplicitDedication(accountId, childDomainIds, hosts);
checkHostsSuitabilityForExplicitDedication(accountId, childDomainIds, hostIds);
final Long accountIdFinal = accountId;
return Transaction.execute(new TransactionCallback<List<DedicatedResourceVO>>() {
@ -402,7 +406,7 @@ public class DedicatedResourceManagerImpl implements DedicatedService {
@ActionEvent(eventType = EventTypes.EVENT_DEDICATE_RESOURCE, eventDescription = "dedicating a Cluster")
public List<DedicatedResourceVO> dedicateCluster(final Long clusterId, final Long domainId, final String accountName) {
Long accountId = null;
List<HostVO> hosts = null;
List<Long> hostIds = null;
if (accountName != null) {
Account caller = CallContext.current().getCallingAccount();
Account owner = _accountMgr.finalizeOwner(caller, accountName, domainId, null);
@ -448,12 +452,13 @@ public class DedicatedResourceManagerImpl implements DedicatedService {
}
//check if any resource under this cluster is dedicated to different account or sub-domain
hosts = _hostDao.findByClusterId(cluster.getId());
hostIds = _hostDao.listIdsByClusterId(cluster.getId());
List<DedicatedResourceVO> hostsToRelease = new ArrayList<DedicatedResourceVO>();
for (HostVO host : hosts) {
DedicatedResourceVO dHost = _dedicatedDao.findByHostId(host.getId());
for (Long hostId : hostIds) {
DedicatedResourceVO dHost = _dedicatedDao.findByHostId(hostId);
if (dHost != null) {
if (!(childDomainIds.contains(dHost.getDomainId()))) {
HostVO host = _hostDao.findById(hostId);
throw new CloudRuntimeException("Host " + host.getName() + " under this Cluster " + cluster.getName() +
" is dedicated to different account/domain");
}
@ -479,7 +484,7 @@ public class DedicatedResourceManagerImpl implements DedicatedService {
}
}
checkHostsSuitabilityForExplicitDedication(accountId, childDomainIds, hosts);
checkHostsSuitabilityForExplicitDedication(accountId, childDomainIds, hostIds);
final Long accountIdFinal = accountId;
return Transaction.execute(new TransactionCallback<List<DedicatedResourceVO>>() {
@ -576,7 +581,7 @@ public class DedicatedResourceManagerImpl implements DedicatedService {
List<Long> childDomainIds = getDomainChildIds(domainId);
childDomainIds.add(domainId);
checkHostSuitabilityForExplicitDedication(accountId, childDomainIds, host);
checkHostSuitabilityForExplicitDedication(accountId, childDomainIds, host.getId());
final Long accountIdFinal = accountId;
return Transaction.execute(new TransactionCallback<List<DedicatedResourceVO>>() {
@ -662,13 +667,14 @@ public class DedicatedResourceManagerImpl implements DedicatedService {
return vms;
}
private boolean checkHostSuitabilityForExplicitDedication(Long accountId, List<Long> domainIds, Host host) {
private boolean checkHostSuitabilityForExplicitDedication(Long accountId, List<Long> domainIds, long hostId) {
boolean suitable = true;
List<UserVmVO> allVmsOnHost = getVmsOnHost(host.getId());
List<UserVmVO> allVmsOnHost = getVmsOnHost(hostId);
if (accountId != null) {
for (UserVmVO vm : allVmsOnHost) {
if (vm.getAccountId() != accountId) {
logger.info(String.format("Host %s found to be unsuitable for explicit dedication as it is running instances of another account", host));
Host host = _hostDao.findById(hostId);
logger.info("{} found to be unsuitable for explicit dedication as it is running instances of another account", host);
throw new CloudRuntimeException("Host " + host.getUuid() + " found to be unsuitable for explicit dedication as it is " +
"running instances of another account");
}
@ -676,7 +682,8 @@ public class DedicatedResourceManagerImpl implements DedicatedService {
} else {
for (UserVmVO vm : allVmsOnHost) {
if (!domainIds.contains(vm.getDomainId())) {
logger.info(String.format("Host %s found to be unsuitable for explicit dedication as it is running instances of another domain", host));
Host host = _hostDao.findById(hostId);
logger.info("{} found to be unsuitable for explicit dedication as it is running instances of another domain", host);
throw new CloudRuntimeException("Host " + host.getUuid() + " found to be unsuitable for explicit dedication as it is " +
"running instances of another domain");
}
@ -685,10 +692,10 @@ public class DedicatedResourceManagerImpl implements DedicatedService {
return suitable;
}
private boolean checkHostsSuitabilityForExplicitDedication(Long accountId, List<Long> domainIds, List<HostVO> hosts) {
private boolean checkHostsSuitabilityForExplicitDedication(Long accountId, List<Long> domainIds, List<Long> hostIds) {
boolean suitable = true;
for (HostVO host : hosts) {
checkHostSuitabilityForExplicitDedication(accountId, domainIds, host);
for (Long hostId : hostIds) {
checkHostSuitabilityForExplicitDedication(accountId, domainIds, hostId);
}
return suitable;
}

50
plugins/deployment-planners/implicit-dedication/src/main/java/com/cloud/deploy/ImplicitDedicationPlanner.java
View File

@ -21,14 +21,15 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
import org.apache.commons.collections.CollectionUtils;
import com.cloud.configuration.Config;
import com.cloud.exception.InsufficientServerCapacityException;
import com.cloud.host.HostVO;
import com.cloud.resource.ResourceManager;
import com.cloud.service.ServiceOfferingVO;
import com.cloud.service.dao.ServiceOfferingDao;
@ -38,7 +39,6 @@ import com.cloud.utils.DateUtil;
import com.cloud.utils.NumbersUtil;
import com.cloud.vm.VMInstanceVO;
import com.cloud.vm.VirtualMachineProfile;
import org.springframework.util.CollectionUtils;
public class ImplicitDedicationPlanner extends FirstFitPlanner implements DeploymentClusterPlanner {
@ -73,12 +73,11 @@ public class ImplicitDedicationPlanner extends FirstFitPlanner implements Deploy
boolean preferred = isServiceOfferingUsingPlannerInPreferredMode(vmProfile.getServiceOfferingId());
// Get the list of all the hosts in the given clusters
List<Long> allHosts = new ArrayList<Long>();
for (Long cluster : clusterList) {
List<HostVO> hostsInCluster = resourceMgr.listAllHostsInCluster(cluster);
for (HostVO hostVO : hostsInCluster) {
allHosts.add(hostVO.getId());
}
List<Long> allHosts = new ArrayList<>();
if (CollectionUtils.isNotEmpty(clusterList)) {
allHosts = clusterList.stream()
.flatMap(cluster -> hostDao.listIdsByClusterId(cluster).stream())
.collect(Collectors.toList());
}
// Go over all the hosts in the cluster and get a list of
@ -224,20 +223,15 @@ public class ImplicitDedicationPlanner extends FirstFitPlanner implements Deploy
}
private List<Long> getUpdatedClusterList(List<Long> clusterList, Set<Long> hostsSet) {
List<Long> updatedClusterList = new ArrayList<Long>();
for (Long cluster : clusterList) {
List<HostVO> hosts = resourceMgr.listAllHostsInCluster(cluster);
Set<Long> hostsInClusterSet = new HashSet<Long>();
for (HostVO host : hosts) {
hostsInClusterSet.add(host.getId());
}
if (!hostsSet.containsAll(hostsInClusterSet)) {
updatedClusterList.add(cluster);
}
if (CollectionUtils.isEmpty(clusterList)) {
return new ArrayList<>();
}
return updatedClusterList;
return clusterList.stream()
.filter(cluster -> {
Set<Long> hostsInClusterSet = new HashSet<>(hostDao.listIdsByClusterId(cluster));
return !hostsSet.containsAll(hostsInClusterSet);
})
.collect(Collectors.toList());
}
@Override
@ -257,15 +251,11 @@ public class ImplicitDedicationPlanner extends FirstFitPlanner implements Deploy
Account account = vmProfile.getOwner();
// Get the list of all the hosts in the given clusters
List<Long> allHosts = new ArrayList<Long>();
if (!CollectionUtils.isEmpty(clusterList)) {
for (Long cluster : clusterList) {
List<HostVO> hostsInCluster = resourceMgr.listAllHostsInCluster(cluster);
for (HostVO hostVO : hostsInCluster) {
allHosts.add(hostVO.getId());
}
}
List<Long> allHosts = new ArrayList<>();
if (CollectionUtils.isNotEmpty(clusterList)) {
allHosts = clusterList.stream()
.flatMap(cluster -> hostDao.listIdsByClusterId(cluster).stream())
.collect(Collectors.toList());
}
// Go over all the hosts in the cluster and get a list of
// 1. All empty hosts, not running any vms.

38
plugins/deployment-planners/implicit-dedication/src/test/java/org/apache/cloudstack/implicitplanner/ImplicitPlannerTest.java
View File

@ -16,11 +16,11 @@
// under the License.
package org.apache.cloudstack.implicitplanner;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.everyItem;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.everyItem;
import static org.hamcrest.Matchers.equalTo;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@ -36,7 +36,11 @@ import java.util.UUID;
import javax.inject.Inject;
import com.cloud.user.User;
import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
import org.apache.cloudstack.test.utils.SpringUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@ -54,12 +58,6 @@ import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import org.springframework.test.context.support.AnnotationConfigContextLoader;
import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
import org.apache.cloudstack.test.utils.SpringUtils;
import com.cloud.capacity.Capacity;
import com.cloud.capacity.CapacityManager;
import com.cloud.capacity.dao.CapacityDao;
@ -73,7 +71,6 @@ import com.cloud.deploy.DeploymentPlanner.ExcludeList;
import com.cloud.deploy.ImplicitDedicationPlanner;
import com.cloud.exception.InsufficientServerCapacityException;
import com.cloud.gpu.dao.HostGpuGroupsDao;
import com.cloud.host.HostVO;
import com.cloud.host.dao.HostDao;
import com.cloud.host.dao.HostDetailsDao;
import com.cloud.host.dao.HostTagsDao;
@ -90,6 +87,7 @@ import com.cloud.storage.dao.VolumeDao;
import com.cloud.user.Account;
import com.cloud.user.AccountManager;
import com.cloud.user.AccountVO;
import com.cloud.user.User;
import com.cloud.user.UserVO;
import com.cloud.utils.Pair;
import com.cloud.utils.component.ComponentContext;
@ -387,21 +385,9 @@ public class ImplicitPlannerTest {
when(serviceOfferingDetailsDao.listDetailsKeyPairs(offeringId)).thenReturn(details);
// Initialize hosts in clusters
HostVO host1 = mock(HostVO.class);
when(host1.getId()).thenReturn(5L);
HostVO host2 = mock(HostVO.class);
when(host2.getId()).thenReturn(6L);
HostVO host3 = mock(HostVO.class);
when(host3.getId()).thenReturn(7L);
List<HostVO> hostsInCluster1 = new ArrayList<HostVO>();
List<HostVO> hostsInCluster2 = new ArrayList<HostVO>();
List<HostVO> hostsInCluster3 = new ArrayList<HostVO>();
hostsInCluster1.add(host1);
hostsInCluster2.add(host2);
hostsInCluster3.add(host3);
when(resourceMgr.listAllHostsInCluster(1)).thenReturn(hostsInCluster1);
when(resourceMgr.listAllHostsInCluster(2)).thenReturn(hostsInCluster2);
when(resourceMgr.listAllHostsInCluster(3)).thenReturn(hostsInCluster3);
when(hostDao.listIdsByClusterId(1L)).thenReturn(List.of(5L));
when(hostDao.listIdsByClusterId(2L)).thenReturn(List.of(6L));
when(hostDao.listIdsByClusterId(3L)).thenReturn(List.of(7L));
// Mock vms on each host.
long offeringIdForVmsOfThisAccount = 15L;

4
plugins/hypervisors/simulator/src/main/java/com/cloud/resource/AgentRoutingResource.java
View File

@ -109,7 +109,7 @@ public class AgentRoutingResource extends AgentStorageResource {
public PingCommand getCurrentStatus(long id) {
TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.SIMULATOR_DB);
try {
MockConfigurationVO config = _simMgr.getMockConfigurationDao().findByNameBottomUP(agentHost.getDataCenterId(), agentHost.getPodId(), agentHost.getClusterId(), agentHost.getId(), "PingCommand");
MockConfigurationVO config = null;
if (config != null) {
Map<String, String> configParameters = config.getParameters();
for (Map.Entry<String, String> entry : configParameters.entrySet()) {
@ -122,7 +122,7 @@ public class AgentRoutingResource extends AgentStorageResource {
}
}
config = _simMgr.getMockConfigurationDao().findByNameBottomUP(agentHost.getDataCenterId(), agentHost.getPodId(), agentHost.getClusterId(), agentHost.getId(), "PingRoutingWithNwGroupsCommand");
config = null;
if (config != null) {
String message = config.getJsonResponse();
if (message != null) {

9
plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/discoverer/XcpServerDiscoverer.java
View File

@ -31,6 +31,7 @@ import javax.naming.ConfigurationException;
import javax.persistence.EntityExistsException;
import org.apache.cloudstack.hypervisor.xenserver.XenserverConfigs;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.maven.artifact.versioning.ComparableVersion;
import org.apache.xmlrpc.XmlRpcException;
@ -144,8 +145,8 @@ public class XcpServerDiscoverer extends DiscovererBase implements Discoverer, L
sc.and(sc.entity().getGuid(), Op.EQ, guid);
List<ClusterVO> clusters = sc.list();
ClusterVO clu = clusters.get(0);
List<HostVO> clusterHosts = _resourceMgr.listAllHostsInCluster(clu.getId());
if (clusterHosts == null || clusterHosts.size() == 0) {
List<Long> clusterHostIds = _hostDao.listIdsByClusterId(clu.getId());
if (CollectionUtils.isEmpty(clusterHostIds)) {
clu.setGuid(null);
_clusterDao.update(clu.getId(), clu);
_clusterDao.update(cluster.getId(), cluster);
@ -245,8 +246,8 @@ public class XcpServerDiscoverer extends DiscovererBase implements Discoverer, L
if (clu.getGuid() == null) {
setClusterGuid(clu, poolUuid);
} else {
List<HostVO> clusterHosts = _resourceMgr.listAllHostsInCluster(clusterId);
if (clusterHosts != null && clusterHosts.size() > 0) {
List<Long> clusterHostIds = _hostDao.listIdsByClusterId(clusterId);
if (CollectionUtils.isNotEmpty(clusterHostIds)) {
if (!clu.getGuid().equals(poolUuid)) {
String msg = "Please join the host " + hostIp + " to XS pool "
+ clu.getGuid() + " through XC/XS before adding it through CS UI";

4
plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java
View File

@ -298,8 +298,8 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp
metricsList.add(new ItemHostMemory(zoneName, zoneUuid, null, null, null, null, ALLOCATED, allocatedCapacityByTag.third(), 0, tag));
});
List<HostTagVO> allHostTagVOS = hostDao.listAll().stream()
.flatMap( h -> _hostTagsDao.getHostTags(h.getId()).stream())
List<HostTagVO> allHostTagVOS = hostDao.listAllIds().stream()
.flatMap( h -> _hostTagsDao.getHostTags(h).stream())
.distinct()
.collect(Collectors.toList());
List<String> allHostTags = new ArrayList<>();

8
plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsService.java
View File

@ -30,6 +30,7 @@ import org.apache.cloudstack.api.response.StoragePoolResponse;
import org.apache.cloudstack.api.response.UserVmResponse;
import org.apache.cloudstack.api.response.VolumeResponse;
import org.apache.cloudstack.api.response.ZoneResponse;
import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.response.ClusterMetricsResponse;
import org.apache.cloudstack.response.DbMetricsResponse;
import org.apache.cloudstack.response.HostMetricsResponse;
@ -47,6 +48,11 @@ import com.cloud.utils.Pair;
import com.cloud.utils.component.PluggableService;
public interface MetricsService extends PluggableService {
ConfigKey<Boolean> AllowListMetricsComputation = new ConfigKey<>("Advanced", Boolean.class, "allow.list.metrics.computation", "true",
"Whether the list zones and cluster metrics APIs are allowed metrics computation. Large environments may disabled this.",
true, ConfigKey.Scope.Global);
InfrastructureResponse listInfrastructure();
ListResponse<VmMetricsStatsResponse> searchForVmMetricsStats(ListVMsUsageHistoryCmd cmd);
@ -56,10 +62,10 @@ public interface MetricsService extends PluggableService {
List<VmMetricsResponse> listVmMetrics(List<UserVmResponse> vmResponses);
List<StoragePoolMetricsResponse> listStoragePoolMetrics(List<StoragePoolResponse> poolResponses);
List<HostMetricsResponse> listHostMetrics(List<HostResponse> poolResponses);
List<ManagementServerMetricsResponse> listManagementServerMetrics(List<ManagementServerResponse> poolResponses);
List<ClusterMetricsResponse> listClusterMetrics(Pair<List<ClusterResponse>, Integer> clusterResponses);
List<ZoneMetricsResponse> listZoneMetrics(List<ZoneResponse> poolResponses);
List<ManagementServerMetricsResponse> listManagementServerMetrics(List<ManagementServerResponse> poolResponses);
UsageServerMetricsResponse listUsageServerMetrics();
DbMetricsResponse listDbMetrics();
}

156
plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsServiceImpl.java
View File

@ -61,6 +61,8 @@ import org.apache.cloudstack.api.response.VolumeResponse;
import org.apache.cloudstack.api.response.ZoneResponse;
import org.apache.cloudstack.cluster.ClusterDrsAlgorithm;
import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.framework.config.Configurable;
import org.apache.cloudstack.management.ManagementServerHost.State;
import org.apache.cloudstack.response.ClusterMetricsResponse;
import org.apache.cloudstack.response.DbMetricsResponse;
@ -110,8 +112,6 @@ import com.cloud.host.Status;
import com.cloud.host.dao.HostDao;
import com.cloud.network.router.VirtualRouter;
import com.cloud.org.Cluster;
import com.cloud.org.Grouping;
import com.cloud.org.Managed;
import com.cloud.server.DbStatsCollection;
import com.cloud.server.ManagementServerHostStats;
import com.cloud.server.StatsCollector;
@ -141,8 +141,7 @@ import com.cloud.vm.dao.VMInstanceDao;
import com.cloud.vm.dao.VmStatsDao;
import com.google.gson.Gson;
public class MetricsServiceImpl extends MutualExclusiveIdsManagerBase implements MetricsService {
public class MetricsServiceImpl extends MutualExclusiveIdsManagerBase implements MetricsService, Configurable {
@Inject
private DataCenterDao dataCenterDao;
@Inject
@ -197,7 +196,6 @@ public class MetricsServiceImpl extends MutualExclusiveIdsManagerBase implements
}
private void updateHostMetrics(final HostMetrics hostMetrics, final HostJoinVO host) {
hostMetrics.incrTotalHosts();
hostMetrics.addCpuAllocated(host.getCpuReservedCapacity() + host.getCpuUsedCapacity());
hostMetrics.addMemoryAllocated(host.getMemReservedCapacity() + host.getMemUsedCapacity());
final HostStats hostStats = ApiDBUtils.getHostStatistics(host.getId());
@ -561,22 +559,17 @@ public class MetricsServiceImpl extends MutualExclusiveIdsManagerBase implements
response.setZones(dataCenterDao.countAll());
response.setPods(podDao.countAll());
response.setClusters(clusterDao.countAll());
response.setHosts(hostDao.countAllByType(Host.Type.Routing));
Pair<Integer, Integer> hostCountAndCpuSockets = hostDao.countAllHostsAndCPUSocketsByType(Host.Type.Routing);
response.setHosts(hostCountAndCpuSockets.first());
response.setStoragePools(storagePoolDao.countAll());
response.setImageStores(imageStoreDao.countAllImageStores());
response.setObjectStores(objectStoreDao.countAllObjectStores());
response.setSystemvms(vmInstanceDao.listByTypes(VirtualMachine.Type.ConsoleProxy, VirtualMachine.Type.SecondaryStorageVm).size());
response.setSystemvms(vmInstanceDao.countByTypes(VirtualMachine.Type.ConsoleProxy, VirtualMachine.Type.SecondaryStorageVm));
response.setRouters(domainRouterDao.countAllByRole(VirtualRouter.Role.VIRTUAL_ROUTER));
response.setInternalLbs(domainRouterDao.countAllByRole(VirtualRouter.Role.INTERNAL_LB_VM));
response.setAlerts(alertDao.countAll());
int cpuSockets = 0;
for (final Host host : hostDao.listByType(Host.Type.Routing)) {
if (host.getCpuSockets() != null) {
cpuSockets += host.getCpuSockets();
}
}
response.setCpuSockets(cpuSockets);
response.setManagementServers(managementServerHostDao.listAll().size());
response.setCpuSockets(hostCountAndCpuSockets.second());
response.setManagementServers(managementServerHostDao.countAll());
return response;
}
@ -764,38 +757,44 @@ public class MetricsServiceImpl extends MutualExclusiveIdsManagerBase implements
final CapacityDaoImpl.SummedCapacity cpuCapacity = getCapacity(Capacity.CAPACITY_TYPE_CPU, null, clusterId);
final CapacityDaoImpl.SummedCapacity memoryCapacity = getCapacity(Capacity.CAPACITY_TYPE_MEMORY, null, clusterId);
final HostMetrics hostMetrics = new HostMetrics(cpuCapacity, memoryCapacity);
hostMetrics.setUpResources(Long.valueOf(hostDao.countAllInClusterByTypeAndStates(clusterId, Host.Type.Routing, List.of(Status.Up))));
hostMetrics.setTotalResources(Long.valueOf(hostDao.countAllInClusterByTypeAndStates(clusterId, Host.Type.Routing, null)));
hostMetrics.setTotalHosts(hostMetrics.getTotalResources());
List<Ternary<Long, Long, Long>> cpuList = new ArrayList<>();
List<Ternary<Long, Long, Long>> memoryList = new ArrayList<>();
for (final Host host: hostDao.findByClusterId(clusterId)) {
if (host == null || host.getType() != Host.Type.Routing) {
continue;
if (AllowListMetricsComputation.value()) {
List<Ternary<Long, Long, Long>> cpuList = new ArrayList<>();
List<Ternary<Long, Long, Long>> memoryList = new ArrayList<>();
for (final Host host : hostDao.findByClusterId(clusterId)) {
if (host == null || host.getType() != Host.Type.Routing) {
continue;
}
updateHostMetrics(hostMetrics, hostJoinDao.findById(host.getId()));
HostJoinVO hostJoin = hostJoinDao.findById(host.getId());
cpuList.add(new Ternary<>(hostJoin.getCpuUsedCapacity(), hostJoin.getCpuReservedCapacity(), hostJoin.getCpus() * hostJoin.getSpeed()));
memoryList.add(new Ternary<>(hostJoin.getMemUsedCapacity(), hostJoin.getMemReservedCapacity(), hostJoin.getTotalMemory()));
}
if (host.getStatus() == Status.Up) {
hostMetrics.incrUpResources();
try {
Double imbalance = ClusterDrsAlgorithm.getClusterImbalance(clusterId, cpuList, memoryList, null);
metricsResponse.setDrsImbalance(imbalance.isNaN() ? null : 100.0 * imbalance);
} catch (ConfigurationException e) {
logger.warn("Failed to get cluster imbalance for cluster {}", clusterId, e);
}
} else {
if (cpuCapacity != null) {
hostMetrics.setCpuAllocated(cpuCapacity.getAllocatedCapacity());
}
if (memoryCapacity != null) {
hostMetrics.setMemoryAllocated(memoryCapacity.getAllocatedCapacity());
}
hostMetrics.incrTotalResources();
HostJoinVO hostJoin = hostJoinDao.findById(host.getId());
updateHostMetrics(hostMetrics, hostJoin);
cpuList.add(new Ternary<>(hostJoin.getCpuUsedCapacity(), hostJoin.getCpuReservedCapacity(), hostJoin.getCpus() * hostJoin.getSpeed()));
memoryList.add(new Ternary<>(hostJoin.getMemUsedCapacity(), hostJoin.getMemReservedCapacity(), hostJoin.getTotalMemory()));
}
try {
Double imbalance = ClusterDrsAlgorithm.getClusterImbalance(clusterId, cpuList, memoryList, null);
metricsResponse.setDrsImbalance(imbalance.isNaN() ? null : 100.0 * imbalance);
} catch (ConfigurationException e) {
logger.warn("Failed to get cluster imbalance for cluster " + clusterId, e);
}
metricsResponse.setState(clusterResponse.getAllocationState(), clusterResponse.getManagedState());
metricsResponse.setResources(hostMetrics.getUpResources(), hostMetrics.getTotalResources());
addHostCpuMetricsToResponse(metricsResponse, clusterId, hostMetrics);
addHostMemoryMetricsToResponse(metricsResponse, clusterId, hostMetrics);
metricsResponse.setHasAnnotation(clusterResponse.hasAnnotation());
metricsResponse.setState(clusterResponse.getAllocationState(), clusterResponse.getManagedState());
metricsResponse.setResources(hostMetrics.getUpResources(), hostMetrics.getTotalResources());
metricsResponses.add(metricsResponse);
}
return metricsResponses;
@ -944,35 +943,38 @@ public class MetricsServiceImpl extends MutualExclusiveIdsManagerBase implements
final CapacityDaoImpl.SummedCapacity cpuCapacity = getCapacity((int) Capacity.CAPACITY_TYPE_CPU, zoneId, null);
final CapacityDaoImpl.SummedCapacity memoryCapacity = getCapacity((int) Capacity.CAPACITY_TYPE_MEMORY, zoneId, null);
final HostMetrics hostMetrics = new HostMetrics(cpuCapacity, memoryCapacity);
hostMetrics.setUpResources(Long.valueOf(clusterDao.countAllManagedAndEnabledByDcId(zoneId)));
hostMetrics.setTotalResources(Long.valueOf(clusterDao.countAllByDcId(zoneId)));
hostMetrics.setTotalHosts(Long.valueOf(hostDao.countAllByTypeInZone(zoneId, Host.Type.Routing)));
for (final Cluster cluster : clusterDao.listClustersByDcId(zoneId)) {
if (cluster == null) {
continue;
}
hostMetrics.incrTotalResources();
if (cluster.getAllocationState() == Grouping.AllocationState.Enabled
&& cluster.getManagedState() == Managed.ManagedState.Managed) {
hostMetrics.incrUpResources();
}
for (final Host host: hostDao.findByClusterId(cluster.getId())) {
if (host == null || host.getType() != Host.Type.Routing) {
if (AllowListMetricsComputation.value()) {
for (final Cluster cluster : clusterDao.listClustersByDcId(zoneId)) {
if (cluster == null) {
continue;
}
updateHostMetrics(hostMetrics, hostJoinDao.findById(host.getId()));
for (final Host host: hostDao.findByClusterId(cluster.getId())) {
if (host == null || host.getType() != Host.Type.Routing) {
continue;
}
updateHostMetrics(hostMetrics, hostJoinDao.findById(host.getId()));
}
}
} else {
if (cpuCapacity != null) {
hostMetrics.setCpuAllocated(cpuCapacity.getAllocatedCapacity());
}
if (memoryCapacity != null) {
hostMetrics.setMemoryAllocated(memoryCapacity.getAllocatedCapacity());
}
}
addHostCpuMetricsToResponse(metricsResponse, null, hostMetrics);
addHostMemoryMetricsToResponse(metricsResponse, null, hostMetrics);
metricsResponse.setHasAnnotation(zoneResponse.hasAnnotation());
metricsResponse.setState(zoneResponse.getAllocationState());
metricsResponse.setResource(hostMetrics.getUpResources(), hostMetrics.getTotalResources());
final Long totalHosts = hostMetrics.getTotalHosts();
// CPU
addHostCpuMetricsToResponse(metricsResponse, null, hostMetrics);
// Memory
addHostMemoryMetricsToResponse(metricsResponse, null, hostMetrics);
metricsResponses.add(metricsResponse);
}
return metricsResponses;
@ -1030,12 +1032,14 @@ public class MetricsServiceImpl extends MutualExclusiveIdsManagerBase implements
private void getQueryHistory(DbMetricsResponse response) {
Map<String, Object> dbStats = ApiDBUtils.getDbStatistics();
if (dbStats != null) {
response.setQueries((Long)dbStats.get(DbStatsCollection.queries));
response.setUptime((Long)dbStats.get(DbStatsCollection.uptime));
if (dbStats == null) {
return;
}
List<Double> loadHistory = (List<Double>) dbStats.get(DbStatsCollection.loadAvarages);
response.setQueries((Long)dbStats.getOrDefault(DbStatsCollection.queries, -1L));
response.setUptime((Long)dbStats.getOrDefault(DbStatsCollection.uptime, -1L));
List<Double> loadHistory = (List<Double>) dbStats.getOrDefault(DbStatsCollection.loadAvarages, new ArrayList<Double>());
double[] loadAverages = new double[loadHistory.size()];
int index = 0;
@ -1110,6 +1114,16 @@ public class MetricsServiceImpl extends MutualExclusiveIdsManagerBase implements
return cmdList;
}
@Override
public String getConfigComponentName() {
return MetricsService.class.getSimpleName();
}
@Override
public ConfigKey<?>[] getConfigKeys() {
return new ConfigKey<?>[] {AllowListMetricsComputation};
}
private class HostMetrics {
// CPU metrics
private Long totalCpu = 0L;
@ -1135,6 +1149,14 @@ public class MetricsServiceImpl extends MutualExclusiveIdsManagerBase implements
}
}
public void setCpuAllocated(Long cpuAllocated) {
this.cpuAllocated = cpuAllocated;
}
public void setMemoryAllocated(Long memoryAllocated) {
this.memoryAllocated = memoryAllocated;
}
public void addCpuAllocated(Long cpuAllocated) {
this.cpuAllocated += cpuAllocated;
}
@ -1163,16 +1185,16 @@ public class MetricsServiceImpl extends MutualExclusiveIdsManagerBase implements
}
}
public void incrTotalHosts() {
this.totalHosts++;
public void setTotalHosts(Long totalHosts) {
this.totalHosts = totalHosts;
}
public void incrTotalResources() {
this.totalResources++;
public void setTotalResources(Long totalResources) {
this.totalResources = totalResources;
}
public void incrUpResources() {
this.upResources++;
public void setUpResources(Long upResources) {
this.upResources = upResources;
}
public Long getTotalCpu() {

152
plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java
View File

@ -18,6 +18,26 @@
*/
package org.apache.cloudstack.storage.datastore.lifecycle;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.stream.Collectors;
import javax.inject.Inject;
import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
import org.apache.cloudstack.engine.subsystem.api.storage.HostScope;
import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreInfo;
import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreLifeCycle;
import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreParameters;
import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope;
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
import org.apache.cloudstack.storage.volume.datastore.PrimaryDataStoreHelper;
import com.cloud.agent.AgentManager;
import com.cloud.agent.api.Answer;
import com.cloud.agent.api.CreateStoragePoolCommand;
@ -48,6 +68,7 @@ import com.cloud.storage.dao.StoragePoolWorkDao;
import com.cloud.storage.dao.VolumeDao;
import com.cloud.user.dao.UserDao;
import com.cloud.utils.NumbersUtil;
import com.cloud.utils.Pair;
import com.cloud.utils.db.DB;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.vm.VirtualMachineManager;
@ -56,23 +77,6 @@ import com.cloud.vm.dao.DomainRouterDao;
import com.cloud.vm.dao.SecondaryStorageVmDao;
import com.cloud.vm.dao.UserVmDao;
import com.cloud.vm.dao.VMInstanceDao;
import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
import org.apache.cloudstack.engine.subsystem.api.storage.HostScope;
import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreInfo;
import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreLifeCycle;
import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreParameters;
import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope;
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
import org.apache.cloudstack.storage.volume.datastore.PrimaryDataStoreHelper;
import javax.inject.Inject;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.UUID;
public class CloudStackPrimaryDataStoreLifeCycleImpl extends BasePrimaryDataStoreLifeCycleImpl implements PrimaryDataStoreLifeCycle {
@Inject
@ -326,18 +330,14 @@ public class CloudStackPrimaryDataStoreLifeCycleImpl extends BasePrimaryDataStor
}
private void validateVcenterDetails(Long zoneId, Long podId, Long clusterId, String storageHost) {
List<HostVO> allHosts =
_resourceMgr.listAllUpHosts(Host.Type.Routing, clusterId, podId, zoneId);
if (allHosts.isEmpty()) {
List<Long> allHostIds = _hostDao.listIdsForUpRouting(zoneId, podId, clusterId);
if (allHostIds.isEmpty()) {
throw new CloudRuntimeException(String.format("No host up to associate a storage pool with in zone: %s pod: %s cluster: %s",
zoneDao.findById(zoneId), podDao.findById(podId), clusterDao.findById(clusterId)));
}
boolean success = false;
for (HostVO h : allHosts) {
for (Long hId : allHostIds) {
ValidateVcenterDetailsCommand cmd = new ValidateVcenterDetailsCommand(storageHost);
final Answer answer = agentMgr.easySend(h.getId(), cmd);
final Answer answer = agentMgr.easySend(hId, cmd);
if (answer != null && answer.getResult()) {
logger.info("Successfully validated vCenter details provided");
return;
@ -346,7 +346,7 @@ public class CloudStackPrimaryDataStoreLifeCycleImpl extends BasePrimaryDataStor
throw new InvalidParameterValueException(String.format("Provided vCenter server details does not match with the existing vCenter in zone: %s",
zoneDao.findById(zoneId)));
} else {
logger.warn("Can not validate vCenter through host {} due to ValidateVcenterDetailsCommand returns null", h);
logger.warn("Can not validate vCenter through host {} due to ValidateVcenterDetailsCommand returns null", hostDao.findById(hId));
}
}
}
@ -385,85 +385,57 @@ public class CloudStackPrimaryDataStoreLifeCycleImpl extends BasePrimaryDataStor
}
}
private Pair<List<Long>, Boolean> prepareOcfs2NodesIfNeeded(PrimaryDataStoreInfo primaryStore) {
if (!StoragePoolType.OCFS2.equals(primaryStore.getPoolType())) {
return new Pair<>(_hostDao.listIdsForUpRouting(primaryStore.getDataCenterId(),
primaryStore.getPodId(), primaryStore.getClusterId()), true);
}
List<HostVO> allHosts = _resourceMgr.listAllUpHosts(Host.Type.Routing, primaryStore.getClusterId(),
primaryStore.getPodId(), primaryStore.getDataCenterId());
if (allHosts.isEmpty()) {
return new Pair<>(Collections.emptyList(), true);
}
List<Long> hostIds = allHosts.stream().map(HostVO::getId).collect(Collectors.toList());
if (!_ocfs2Mgr.prepareNodes(allHosts, primaryStore)) {
return new Pair<>(hostIds, false);
}
return new Pair<>(hostIds, true);
}
@Override
public boolean attachCluster(DataStore store, ClusterScope scope) {
PrimaryDataStoreInfo primarystore = (PrimaryDataStoreInfo)store;
// Check if there is host up in this cluster
List<HostVO> allHosts =
_resourceMgr.listAllUpHosts(Host.Type.Routing, primarystore.getClusterId(), primarystore.getPodId(), primarystore.getDataCenterId());
if (allHosts.isEmpty()) {
primaryDataStoreDao.expunge(primarystore.getId());
throw new CloudRuntimeException(String.format("No host up to associate a storage pool with in cluster %s", clusterDao.findById(primarystore.getClusterId())));
PrimaryDataStoreInfo primaryStore = (PrimaryDataStoreInfo)store;
Pair<List<Long>, Boolean> result = prepareOcfs2NodesIfNeeded(primaryStore);
List<Long> hostIds = result.first();
if (hostIds.isEmpty()) {
primaryDataStoreDao.expunge(primaryStore.getId());
throw new CloudRuntimeException("No host up to associate a storage pool with in cluster: " +
clusterDao.findById(primaryStore.getClusterId()));
}
if (primarystore.getPoolType() == StoragePoolType.OCFS2 && !_ocfs2Mgr.prepareNodes(allHosts, primarystore)) {
logger.warn("Can not create storage pool {} on cluster {}", primarystore::toString, () -> clusterDao.findById(primarystore.getClusterId()));
primaryDataStoreDao.expunge(primarystore.getId());
if (!result.second()) {
logger.warn("Can not create storage pool {} on {}", primaryStore,
clusterDao.findById(primaryStore.getClusterId()));
primaryDataStoreDao.expunge(primaryStore.getId());
return false;
}
boolean success = false;
for (HostVO h : allHosts) {
success = createStoragePool(h, primarystore);
if (success) {
for (Long hId : hostIds) {
HostVO host = _hostDao.findById(hId);
if (createStoragePool(host, primaryStore)) {
break;
}
}
logger.debug("In createPool Adding the pool to each of the hosts");
List<HostVO> poolHosts = new ArrayList<HostVO>();
for (HostVO h : allHosts) {
try {
storageMgr.connectHostToSharedPool(h, primarystore.getId());
poolHosts.add(h);
} catch (StorageConflictException se) {
primaryDataStoreDao.expunge(primarystore.getId());
throw new CloudRuntimeException("Storage has already been added as local storage");
} catch (Exception e) {
logger.warn("Unable to establish a connection between " + h + " and " + primarystore, e);
String reason = storageMgr.getStoragePoolMountFailureReason(e.getMessage());
if (reason != null) {
throw new CloudRuntimeException(reason);
}
}
}
if (poolHosts.isEmpty()) {
logger.warn("No host can access storage pool {} on cluster {}", primarystore::toString, () -> clusterDao.findById(primarystore.getClusterId()));
primaryDataStoreDao.expunge(primarystore.getId());
throw new CloudRuntimeException("Failed to access storage pool");
}
storageMgr.connectHostsToPool(store, hostIds, scope, true, true);
dataStoreHelper.attachCluster(store);
return true;
}
@Override
public boolean attachZone(DataStore dataStore, ZoneScope scope, HypervisorType hypervisorType) {
List<HostVO> hosts = _resourceMgr.listAllUpHostsInOneZoneByHypervisor(hypervisorType, scope.getScopeId());
public boolean attachZone(DataStore store, ZoneScope scope, HypervisorType hypervisorType) {
List<Long> hostIds = _hostDao.listIdsForUpEnabledByZoneAndHypervisor(scope.getScopeId(), hypervisorType);
logger.debug("In createPool. Attaching the pool to each of the hosts.");
List<HostVO> poolHosts = new ArrayList<HostVO>();
for (HostVO host : hosts) {
try {
storageMgr.connectHostToSharedPool(host, dataStore.getId());
poolHosts.add(host);
} catch (StorageConflictException se) {
primaryDataStoreDao.expunge(dataStore.getId());
throw new CloudRuntimeException(String.format("Storage has already been added as local storage to host: %s", host));
} catch (Exception e) {
logger.warn("Unable to establish a connection between " + host + " and " + dataStore, e);
String reason = storageMgr.getStoragePoolMountFailureReason(e.getMessage());
if (reason != null) {
throw new CloudRuntimeException(reason);
}
}
}
if (poolHosts.isEmpty()) {
logger.warn("No host can access storage pool " + dataStore + " in this zone.");
primaryDataStoreDao.expunge(dataStore.getId());
throw new CloudRuntimeException("Failed to create storage pool as it is not accessible to hosts.");
}
dataStoreHelper.attachZone(dataStore, hypervisorType);
storageMgr.connectHostsToPool(store, hostIds, scope, true, true);
dataStoreHelper.attachZone(store, hypervisorType);
return true;
}

77
plugins/storage/volume/default/src/test/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImplTest.java
View File

@ -19,23 +19,14 @@
package org.apache.cloudstack.storage.datastore.lifecycle;
import com.cloud.agent.AgentManager;
import com.cloud.agent.api.ModifyStoragePoolAnswer;
import com.cloud.agent.api.ModifyStoragePoolCommand;
import com.cloud.agent.api.StoragePoolInfo;
import com.cloud.exception.StorageConflictException;
import com.cloud.host.Host;
import com.cloud.host.HostVO;
import com.cloud.host.Status;
import com.cloud.resource.ResourceManager;
import com.cloud.resource.ResourceState;
import com.cloud.storage.DataStoreRole;
import com.cloud.storage.Storage;
import com.cloud.storage.StorageManager;
import com.cloud.storage.StorageManagerImpl;
import com.cloud.storage.dao.StoragePoolHostDao;
import com.cloud.utils.exception.CloudRuntimeException;
import junit.framework.TestCase;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.util.List;
import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
@ -58,14 +49,23 @@ import org.mockito.Mockito;
import org.mockito.MockitoAnnotations;
import org.mockito.junit.MockitoJUnitRunner;
import org.springframework.test.util.ReflectionTestUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.when;
import com.cloud.agent.AgentManager;
import com.cloud.agent.api.ModifyStoragePoolAnswer;
import com.cloud.agent.api.ModifyStoragePoolCommand;
import com.cloud.agent.api.StoragePoolInfo;
import com.cloud.exception.StorageConflictException;
import com.cloud.host.HostVO;
import com.cloud.host.dao.HostDao;
import com.cloud.resource.ResourceManager;
import com.cloud.storage.DataStoreRole;
import com.cloud.storage.Storage;
import com.cloud.storage.StorageManager;
import com.cloud.storage.StorageManagerImpl;
import com.cloud.storage.dao.StoragePoolHostDao;
import com.cloud.utils.exception.CloudRuntimeException;
import junit.framework.TestCase;
/**
* Created by ajna123 on 9/22/2015.
@ -118,6 +118,9 @@ public class CloudStackPrimaryDataStoreLifeCycleImplTest extends TestCase {
@Mock
PrimaryDataStoreHelper primaryDataStoreHelper;
@Mock
HostDao hostDao;
AutoCloseable closeable;
@Before
@ -129,17 +132,6 @@ public class CloudStackPrimaryDataStoreLifeCycleImplTest extends TestCase {
ReflectionTestUtils.setField(storageMgr, "_dataStoreMgr", _dataStoreMgr);
ReflectionTestUtils.setField(_cloudStackPrimaryDataStoreLifeCycle, "storageMgr", storageMgr);
List<HostVO> hostList = new ArrayList<HostVO>();
HostVO host1 = new HostVO(1L, "aa01", Host.Type.Routing, "192.168.1.1", "255.255.255.0", null, null, null, null, null, null, null, null, null, null,
UUID.randomUUID().toString(), Status.Up, "1.0", null, null, 1L, null, 0, 0, "aa", 0, Storage.StoragePoolType.NetworkFilesystem);
HostVO host2 = new HostVO(1L, "aa02", Host.Type.Routing, "192.168.1.1", "255.255.255.0", null, null, null, null, null, null, null, null, null, null,
UUID.randomUUID().toString(), Status.Up, "1.0", null, null, 1L, null, 0, 0, "aa", 0, Storage.StoragePoolType.NetworkFilesystem);
host1.setResourceState(ResourceState.Enabled);
host2.setResourceState(ResourceState.Disabled);
hostList.add(host1);
hostList.add(host2);
when(_dataStoreMgr.getDataStore(anyLong(), eq(DataStoreRole.Primary))).thenReturn(store);
when(store.getPoolType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem);
when(store.isShared()).thenReturn(true);
@ -152,7 +144,9 @@ public class CloudStackPrimaryDataStoreLifeCycleImplTest extends TestCase {
storageMgr.registerHostListener("default", hostListener);
when(_resourceMgr.listAllUpHosts(eq(Host.Type.Routing), anyLong(), anyLong(), anyLong())).thenReturn(hostList);
when(hostDao.listIdsForUpRouting(anyLong(), anyLong(), anyLong()))
.thenReturn(List.of(1L, 2L));
when(hostDao.findById(anyLong())).thenReturn(mock(HostVO.class));
when(agentMgr.easySend(anyLong(), Mockito.any(ModifyStoragePoolCommand.class))).thenReturn(answer);
when(answer.getResult()).thenReturn(true);
@ -171,18 +165,17 @@ public class CloudStackPrimaryDataStoreLifeCycleImplTest extends TestCase {
}
@Test
public void testAttachClusterException() throws Exception {
String exceptionString = "Mount failed due to incorrect mount options.";
public void testAttachClusterException() {
String mountFailureReason = "Incorrect mount option specified.";
CloudRuntimeException exception = new CloudRuntimeException(exceptionString);
ClusterScope scope = new ClusterScope(1L, 1L, 1L);
CloudRuntimeException exception = new CloudRuntimeException(mountFailureReason);
StorageManager storageManager = Mockito.mock(StorageManager.class);
Mockito.when(storageManager.connectHostToSharedPool(Mockito.any(), Mockito.anyLong())).thenThrow(exception);
Mockito.when(storageManager.getStoragePoolMountFailureReason(exceptionString)).thenReturn(mountFailureReason);
Mockito.doThrow(exception).when(storageManager).connectHostsToPool(Mockito.eq(store), Mockito.anyList(), Mockito.eq(scope), Mockito.eq(true), Mockito.eq(true));
ReflectionTestUtils.setField(_cloudStackPrimaryDataStoreLifeCycle, "storageMgr", storageManager);
try {
_cloudStackPrimaryDataStoreLifeCycle.attachCluster(store, new ClusterScope(1L, 1L, 1L));
_cloudStackPrimaryDataStoreLifeCycle.attachCluster(store, scope);
Assert.fail();
} catch (Exception e) {
Assert.assertEquals(e.getMessage(), mountFailureReason);

59
plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycle.java
View File

@ -24,17 +24,12 @@ import java.net.URISyntaxException;
import java.net.URLDecoder;
import java.security.KeyManagementException;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import javax.inject.Inject;
import org.apache.cloudstack.storage.datastore.client.ScaleIOGatewayClientConnectionPool;
import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao;
import org.apache.cloudstack.storage.datastore.util.ScaleIOUtil;
import org.apache.commons.collections.CollectionUtils;
import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.HostScope;
@ -44,9 +39,13 @@ import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreParame
import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope;
import org.apache.cloudstack.storage.datastore.api.StoragePoolStatistics;
import org.apache.cloudstack.storage.datastore.client.ScaleIOGatewayClient;
import org.apache.cloudstack.storage.datastore.client.ScaleIOGatewayClientConnectionPool;
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao;
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
import org.apache.cloudstack.storage.datastore.util.ScaleIOUtil;
import org.apache.cloudstack.storage.volume.datastore.PrimaryDataStoreHelper;
import org.apache.commons.collections.CollectionUtils;
import com.cloud.agent.AgentManager;
import com.cloud.agent.api.Answer;
@ -55,9 +54,9 @@ import com.cloud.agent.api.StoragePoolInfo;
import com.cloud.capacity.CapacityManager;
import com.cloud.dc.ClusterVO;
import com.cloud.dc.dao.ClusterDao;
import com.cloud.dc.dao.DataCenterDao;
import com.cloud.exception.InvalidParameterValueException;
import com.cloud.host.Host;
import com.cloud.host.HostVO;
import com.cloud.host.dao.HostDao;
import com.cloud.hypervisor.Hypervisor;
import com.cloud.resource.ResourceManager;
import com.cloud.storage.Storage;
@ -74,9 +73,13 @@ import com.cloud.utils.crypt.DBEncryptionUtil;
import com.cloud.utils.exception.CloudRuntimeException;
public class ScaleIOPrimaryDataStoreLifeCycle extends BasePrimaryDataStoreLifeCycleImpl implements PrimaryDataStoreLifeCycle {
@Inject
DataCenterDao dataCenterDao;
@Inject
private ClusterDao clusterDao;
@Inject
private HostDao hostDao;
@Inject
private PrimaryDataStoreDao primaryDataStoreDao;
@Inject
private StoragePoolDetailsDao storagePoolDetailsDao;
@ -258,28 +261,15 @@ public class ScaleIOPrimaryDataStoreLifeCycle extends BasePrimaryDataStoreLifeCy
}
PrimaryDataStoreInfo primaryDataStoreInfo = (PrimaryDataStoreInfo) dataStore;
List<HostVO> hostsInCluster = resourceManager.listAllUpAndEnabledHosts(Host.Type.Routing, primaryDataStoreInfo.getClusterId(),
primaryDataStoreInfo.getPodId(), primaryDataStoreInfo.getDataCenterId());
if (hostsInCluster.isEmpty()) {
List<Long> hostIds = hostDao.listIdsForUpRouting(primaryDataStoreInfo.getDataCenterId(),
primaryDataStoreInfo.getPodId(), primaryDataStoreInfo.getClusterId());
if (hostIds.isEmpty()) {
primaryDataStoreDao.expunge(primaryDataStoreInfo.getId());
throw new CloudRuntimeException("No hosts are Up to associate a storage pool with in cluster: " + cluster);
}
logger.debug("Attaching the pool to each of the hosts in the cluster: {}", cluster);
List<HostVO> poolHosts = new ArrayList<HostVO>();
for (HostVO host : hostsInCluster) {
try {
if (storageMgr.connectHostToSharedPool(host, primaryDataStoreInfo.getId())) {
poolHosts.add(host);
}
} catch (Exception e) {
logger.warn(String.format("Unable to establish a connection between host: %s and pool: %s on the cluster: %s", host, dataStore, cluster), e);
}
}
if (poolHosts.isEmpty()) {
logger.warn("No host can access storage pool '{}' on cluster '{}'.", primaryDataStoreInfo, cluster);
}
logger.debug("Attaching the pool to each of the hosts in the {}", cluster);
storageMgr.connectHostsToPool(dataStore, hostIds, scope, false, false);
dataStoreHelper.attachCluster(dataStore);
return true;
@ -296,21 +286,10 @@ public class ScaleIOPrimaryDataStoreLifeCycle extends BasePrimaryDataStoreLifeCy
throw new CloudRuntimeException("Unsupported hypervisor type: " + hypervisorType.toString());
}
logger.debug("Attaching the pool to each of the hosts in the zone: " + scope.getScopeId());
List<HostVO> hosts = resourceManager.listAllUpAndEnabledHostsInOneZoneByHypervisor(hypervisorType, scope.getScopeId());
List<HostVO> poolHosts = new ArrayList<HostVO>();
for (HostVO host : hosts) {
try {
if (storageMgr.connectHostToSharedPool(host, dataStore.getId())) {
poolHosts.add(host);
}
} catch (Exception e) {
logger.warn("Unable to establish a connection between host: " + host + " and pool: " + dataStore + "in the zone: " + scope.getScopeId(), e);
}
}
if (poolHosts.isEmpty()) {
logger.warn("No host can access storage pool " + dataStore + " in the zone: " + scope.getScopeId());
}
logger.debug("Attaching the pool to each of the hosts in the {}",
dataCenterDao.findById(scope.getScopeId()));
List<Long> hostIds = hostDao.listIdsForUpEnabledByZoneAndHypervisor(scope.getScopeId(), hypervisorType);
storageMgr.connectHostsToPool(dataStore, hostIds, scope, false, false);
dataStoreHelper.attachZone(dataStore);
return true;

32
plugins/storage/volume/scaleio/src/test/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycleTest.java
View File

@ -30,7 +30,6 @@ import static org.mockito.Mockito.when;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
@ -56,15 +55,13 @@ import org.mockito.MockedStatic;
import org.mockito.Mockito;
import org.mockito.MockitoAnnotations;
import org.mockito.junit.MockitoJUnitRunner;
import org.springframework.test.util.ReflectionTestUtils;
import com.cloud.host.Host;
import com.cloud.host.HostVO;
import com.cloud.host.Status;
import com.cloud.dc.DataCenterVO;
import com.cloud.dc.dao.DataCenterDao;
import com.cloud.host.dao.HostDao;
import com.cloud.hypervisor.Hypervisor;
import com.cloud.resource.ResourceManager;
import com.cloud.resource.ResourceState;
import com.cloud.storage.DataStoreRole;
import com.cloud.storage.Storage;
import com.cloud.storage.StorageManager;
import com.cloud.storage.StorageManagerImpl;
import com.cloud.storage.StoragePoolAutomation;
@ -73,7 +70,6 @@ import com.cloud.storage.VMTemplateStoragePoolVO;
import com.cloud.storage.dao.StoragePoolHostDao;
import com.cloud.template.TemplateManager;
import com.cloud.utils.exception.CloudRuntimeException;
import org.springframework.test.util.ReflectionTestUtils;
@RunWith(MockitoJUnitRunner.class)
public class ScaleIOPrimaryDataStoreLifeCycleTest {
@ -85,8 +81,6 @@ public class ScaleIOPrimaryDataStoreLifeCycleTest {
@Mock
private PrimaryDataStoreHelper dataStoreHelper;
@Mock
private ResourceManager resourceManager;
@Mock
private StoragePoolAutomation storagePoolAutomation;
@Mock
private StoragePoolHostDao storagePoolHostDao;
@ -100,6 +94,10 @@ public class ScaleIOPrimaryDataStoreLifeCycleTest {
private PrimaryDataStore store;
@Mock
private TemplateManager templateMgr;
@Mock
HostDao hostDao;
@Mock
DataCenterDao dataCenterDao;
@InjectMocks
private StorageManager storageMgr = new StorageManagerImpl();
@ -115,6 +113,7 @@ public class ScaleIOPrimaryDataStoreLifeCycleTest {
public void setUp() {
closeable = MockitoAnnotations.openMocks(this);
ReflectionTestUtils.setField(scaleIOPrimaryDataStoreLifeCycleTest, "storageMgr", storageMgr);
when(dataCenterDao.findById(anyLong())).thenReturn(mock(DataCenterVO.class));
}
@After
@ -137,17 +136,8 @@ public class ScaleIOPrimaryDataStoreLifeCycleTest {
final ZoneScope scope = new ZoneScope(1L);
List<HostVO> hostList = new ArrayList<HostVO>();
HostVO host1 = new HostVO(1L, "host01", Host.Type.Routing, "192.168.1.1", "255.255.255.0", null, null, null, null, null, null, null, null, null, null,
UUID.randomUUID().toString(), Status.Up, "1.0", null, null, 1L, null, 0, 0, "aa", 0, Storage.StoragePoolType.PowerFlex);
HostVO host2 = new HostVO(2L, "host02", Host.Type.Routing, "192.168.1.2", "255.255.255.0", null, null, null, null, null, null, null, null, null, null,
UUID.randomUUID().toString(), Status.Up, "1.0", null, null, 1L, null, 0, 0, "aa", 0, Storage.StoragePoolType.PowerFlex);
host1.setResourceState(ResourceState.Enabled);
host2.setResourceState(ResourceState.Enabled);
hostList.add(host1);
hostList.add(host2);
when(resourceManager.listAllUpAndEnabledHostsInOneZoneByHypervisor(Hypervisor.HypervisorType.KVM, 1L)).thenReturn(hostList);
when(hostDao.listIdsForUpEnabledByZoneAndHypervisor(scope.getScopeId(), Hypervisor.HypervisorType.KVM))
.thenReturn(List.of(1L, 2L));
when(dataStoreMgr.getDataStore(anyLong(), eq(DataStoreRole.Primary))).thenReturn(store);
when(store.isShared()).thenReturn(true);

14
plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/util/StorPoolHelper.java
View File

@ -219,17 +219,17 @@ public class StorPoolHelper {
}
public static Long findClusterIdByGlobalId(String globalId, ClusterDao clusterDao) {
List<ClusterVO> clusterVo = clusterDao.listAll();
if (clusterVo.size() == 1) {
List<Long> clusterIds = clusterDao.listAllIds();
if (clusterIds.size() == 1) {
StorPoolUtil.spLog("There is only one cluster, sending backup to secondary command");
return null;
}
for (ClusterVO clusterVO2 : clusterVo) {
if (globalId != null && StorPoolConfigurationManager.StorPoolClusterId.valueIn(clusterVO2.getId()) != null
&& globalId.contains(StorPoolConfigurationManager.StorPoolClusterId.valueIn(clusterVO2.getId()).toString())) {
StorPoolUtil.spLog("Found cluster with id=%s for object with globalId=%s", clusterVO2.getId(),
for (Long clusterId : clusterIds) {
if (globalId != null && StorPoolConfigurationManager.StorPoolClusterId.valueIn(clusterId) != null
&& globalId.contains(StorPoolConfigurationManager.StorPoolClusterId.valueIn(clusterId))) {
StorPoolUtil.spLog("Found cluster with id=%s for object with globalId=%s", clusterId,
globalId);
return clusterVO2.getId();
return clusterId;
}
}
throw new CloudRuntimeException(

96
server/src/main/java/com/cloud/alert/AlertManagerImpl.java
View File

@ -26,8 +26,11 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.Timer;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import javax.inject.Inject;
import javax.mail.MessagingException;
@ -75,12 +78,11 @@ import com.cloud.event.AlertGenerator;
import com.cloud.event.EventTypes;
import com.cloud.host.Host;
import com.cloud.host.HostVO;
import com.cloud.host.dao.HostDao;
import com.cloud.network.Ipv6Service;
import com.cloud.network.dao.IPAddressDao;
import com.cloud.org.Grouping.AllocationState;
import com.cloud.resource.ResourceManager;
import com.cloud.service.ServiceOfferingVO;
import com.cloud.service.dao.ServiceOfferingDao;
import com.cloud.storage.StorageManager;
import com.cloud.utils.Pair;
import com.cloud.utils.component.ManagerBase;
@ -124,9 +126,9 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager, Confi
@Inject
protected ConfigDepot _configDepot;
@Inject
ServiceOfferingDao _offeringsDao;
@Inject
Ipv6Service ipv6Service;
@Inject
HostDao hostDao;
private Timer _timer = null;
private long _capacityCheckPeriod = 60L * 60L * 1000L; // One hour by default.
@ -260,6 +262,66 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager, Confi
}
}
/**
* Recalculates the capacities of hosts, including CPU and RAM.
*/
protected void recalculateHostCapacities() {
List<Long> hostIds = hostDao.listIdsByType(Host.Type.Routing);
if (hostIds.isEmpty()) {
return;
}
ConcurrentHashMap<Long, Future<Void>> futures = new ConcurrentHashMap<>();
ExecutorService executorService = Executors.newFixedThreadPool(Math.max(1,
Math.min(CapacityManager.CapacityCalculateWorkers.value(), hostIds.size())));
for (Long hostId : hostIds) {
futures.put(hostId, executorService.submit(() -> {
final HostVO host = hostDao.findById(hostId);
_capacityMgr.updateCapacityForHost(host);
return null;
}));
}
for (Map.Entry<Long, Future<Void>> entry: futures.entrySet()) {
try {
entry.getValue().get();
} catch (InterruptedException | ExecutionException e) {
logger.error(String.format("Error during capacity calculation for host: %d due to : %s",
entry.getKey(), e.getMessage()), e);
}
}
executorService.shutdown();
}
protected void recalculateStorageCapacities() {
List<Long> storagePoolIds = _storagePoolDao.listAllIds();
if (storagePoolIds.isEmpty()) {
return;
}
ConcurrentHashMap<Long, Future<Void>> futures = new ConcurrentHashMap<>();
ExecutorService executorService = Executors.newFixedThreadPool(Math.max(1,
Math.min(CapacityManager.CapacityCalculateWorkers.value(), storagePoolIds.size())));
for (Long poolId: storagePoolIds) {
futures.put(poolId, executorService.submit(() -> {
final StoragePoolVO pool = _storagePoolDao.findById(poolId);
long disk = _capacityMgr.getAllocatedPoolCapacity(pool, null);
if (pool.isShared()) {
_storageMgr.createCapacityEntry(pool, Capacity.CAPACITY_TYPE_STORAGE_ALLOCATED, disk);
} else {
_storageMgr.createCapacityEntry(pool, Capacity.CAPACITY_TYPE_LOCAL_STORAGE, disk);
}
return null;
}));
}
for (Map.Entry<Long, Future<Void>> entry: futures.entrySet()) {
try {
entry.getValue().get();
} catch (InterruptedException | ExecutionException e) {
logger.error(String.format("Error during capacity calculation for storage pool: %d due to : %s",
entry.getKey(), e.getMessage()), e);
}
}
executorService.shutdown();
}
@Override
public void recalculateCapacity() {
// FIXME: the right way to do this is to register a listener (see RouterStatsListener, VMSyncListener)
@ -275,36 +337,14 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager, Confi
logger.debug("recalculating system capacity");
logger.debug("Executing cpu/ram capacity update");
}
// Calculate CPU and RAM capacities
// get all hosts...even if they are not in 'UP' state
List<HostVO> hosts = _resourceMgr.listAllNotInMaintenanceHostsInOneZone(Host.Type.Routing, null);
if (hosts != null) {
// prepare the service offerings
List<ServiceOfferingVO> offerings = _offeringsDao.listAllIncludingRemoved();
Map<Long, ServiceOfferingVO> offeringsMap = new HashMap<Long, ServiceOfferingVO>();
for (ServiceOfferingVO offering : offerings) {
offeringsMap.put(offering.getId(), offering);
}
for (HostVO host : hosts) {
_capacityMgr.updateCapacityForHost(host, offeringsMap);
}
}
recalculateHostCapacities();
if (logger.isDebugEnabled()) {
logger.debug("Done executing cpu/ram capacity update");
logger.debug("Executing storage capacity update");
}
// Calculate storage pool capacity
List<StoragePoolVO> storagePools = _storagePoolDao.listAll();
for (StoragePoolVO pool : storagePools) {
long disk = _capacityMgr.getAllocatedPoolCapacity(pool, null);
if (pool.isShared()) {
_storageMgr.createCapacityEntry(pool, Capacity.CAPACITY_TYPE_STORAGE_ALLOCATED, disk);
} else {
_storageMgr.createCapacityEntry(pool, Capacity.CAPACITY_TYPE_LOCAL_STORAGE, disk);
}
}
recalculateStorageCapacities();
if (logger.isDebugEnabled()) {
logger.debug("Done executing storage capacity update");
logger.debug("Executing capacity updates for public ip and Vlans");

6
server/src/main/java/com/cloud/api/query/QueryManagerImpl.java
View File

@ -2365,7 +2365,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q
// ids
hostSearchBuilder.and("id", hostSearchBuilder.entity().getId(), SearchCriteria.Op.EQ);
hostSearchBuilder.and("name", hostSearchBuilder.entity().getName(), SearchCriteria.Op.EQ);
hostSearchBuilder.and("type", hostSearchBuilder.entity().getType(), SearchCriteria.Op.LIKE);
hostSearchBuilder.and("type", hostSearchBuilder.entity().getType(), SearchCriteria.Op.EQ);
hostSearchBuilder.and("status", hostSearchBuilder.entity().getStatus(), SearchCriteria.Op.EQ);
hostSearchBuilder.and("dataCenterId", hostSearchBuilder.entity().getDataCenterId(), SearchCriteria.Op.EQ);
hostSearchBuilder.and("podId", hostSearchBuilder.entity().getPodId(), SearchCriteria.Op.EQ);
@ -2418,7 +2418,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q
sc.setParameters("name", name);
}
if (type != null) {
sc.setParameters("type", "%" + type);
sc.setParameters("type", type);
}
if (state != null) {
sc.setParameters("status", state);
@ -4575,7 +4575,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q
// check if zone is configured, if not, just return empty list
List<HypervisorType> hypers = null;
if (!isIso) {
hypers = _resourceMgr.listAvailHypervisorInZone(null, null);
hypers = _resourceMgr.listAvailHypervisorInZone(null);
if (hypers == null || hypers.isEmpty()) {
return new Pair<List<TemplateJoinVO>, Integer>(new ArrayList<TemplateJoinVO>(), 0);
}

4
server/src/main/java/com/cloud/api/query/dao/UserVmJoinDao.java
View File

@ -44,6 +44,6 @@ public interface UserVmJoinDao extends GenericDao<UserVmJoinVO, Long> {
List<UserVmJoinVO> listActiveByIsoId(Long isoId);
List<UserVmJoinVO> listByAccountServiceOfferingTemplateAndNotInState(long accountId, List<VirtualMachine.State> states,
List<Long> offeringIds, List<Long> templateIds);
List<UserVmJoinVO> listByAccountServiceOfferingTemplateAndNotInState(long accountId,
List<VirtualMachine.State> states, List<Long> offeringIds, List<Long> templateIds);
}

4
server/src/main/java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java
View File

@ -693,6 +693,8 @@ public class UserVmJoinDaoImpl extends GenericDaoBaseWithTagInformation<UserVmJo
public List<UserVmJoinVO> listByAccountServiceOfferingTemplateAndNotInState(long accountId, List<State> states,
List<Long> offeringIds, List<Long> templateIds) {
SearchBuilder<UserVmJoinVO> userVmSearch = createSearchBuilder();
userVmSearch.selectFields(userVmSearch.entity().getId(), userVmSearch.entity().getCpu(),
userVmSearch.entity().getRamSize());
userVmSearch.and("accountId", userVmSearch.entity().getAccountId(), Op.EQ);
userVmSearch.and("serviceOfferingId", userVmSearch.entity().getServiceOfferingId(), Op.IN);
userVmSearch.and("templateId", userVmSearch.entity().getTemplateId(), Op.IN);
@ -713,6 +715,6 @@ public class UserVmJoinDaoImpl extends GenericDaoBaseWithTagInformation<UserVmJo
sc.setParameters("state", states.toArray());
}
sc.setParameters("displayVm", 1);
return listBy(sc);
return customSearch(sc, null);
}
}

185
server/src/main/java/com/cloud/capacity/CapacityManagerImpl.java
View File

@ -23,6 +23,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
@ -37,6 +38,10 @@ import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.apache.cloudstack.framework.messagebus.MessageBus;
import org.apache.cloudstack.framework.messagebus.PublishScope;
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
import org.apache.cloudstack.utils.cache.LazyCache;
import org.apache.cloudstack.utils.cache.SingleCache;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import com.cloud.agent.AgentManager;
import com.cloud.agent.Listener;
@ -50,7 +55,6 @@ import com.cloud.capacity.dao.CapacityDao;
import com.cloud.configuration.Config;
import com.cloud.dc.ClusterDetailsDao;
import com.cloud.dc.ClusterDetailsVO;
import com.cloud.dc.ClusterVO;
import com.cloud.dc.dao.ClusterDao;
import com.cloud.deploy.DeploymentClusterPlanner;
import com.cloud.event.UsageEventVO;
@ -62,7 +66,6 @@ import com.cloud.host.dao.HostDao;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.hypervisor.dao.HypervisorCapabilitiesDao;
import com.cloud.offering.ServiceOffering;
import com.cloud.org.Cluster;
import com.cloud.resource.ResourceListener;
import com.cloud.resource.ResourceManager;
import com.cloud.resource.ResourceState;
@ -141,6 +144,9 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager,
@Inject
MessageBus _messageBus;
private LazyCache<Long, Pair<String, String>> clusterValuesCache;
private SingleCache<Map<Long, ServiceOfferingVO>> serviceOfferingsCache;
@Override
public boolean configure(String name, Map<String, Object> params) throws ConfigurationException {
_vmCapacityReleaseInterval = NumbersUtil.parseInt(_configDao.getValue(Config.CapacitySkipcountingHours.key()), 3600);
@ -156,6 +162,8 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager,
public boolean start() {
_resourceMgr.registerResourceEvent(ResourceListener.EVENT_PREPARE_MAINTENANCE_AFTER, this);
_resourceMgr.registerResourceEvent(ResourceListener.EVENT_CANCEL_MAINTENANCE_AFTER, this);
clusterValuesCache = new LazyCache<>(128, 60, this::getClusterValues);
serviceOfferingsCache = new SingleCache<>(60, this::getServiceOfferingsMap);
return true;
}
@ -209,8 +217,8 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager,
long reservedMem = capacityMemory.getReservedCapacity();
long reservedCpuCore = capacityCpuCore.getReservedCapacity();
long actualTotalCpu = capacityCpu.getTotalCapacity();
float cpuOvercommitRatio = Float.parseFloat(_clusterDetailsDao.findDetail(clusterIdFinal, "cpuOvercommitRatio").getValue());
float memoryOvercommitRatio = Float.parseFloat(_clusterDetailsDao.findDetail(clusterIdFinal, "memoryOvercommitRatio").getValue());
float cpuOvercommitRatio = Float.parseFloat(_clusterDetailsDao.findDetail(clusterIdFinal, VmDetailConstants.CPU_OVER_COMMIT_RATIO).getValue());
float memoryOvercommitRatio = Float.parseFloat(_clusterDetailsDao.findDetail(clusterIdFinal, VmDetailConstants.MEMORY_OVER_COMMIT_RATIO).getValue());
int vmCPU = svo.getCpu() * svo.getSpeed();
int vmCPUCore = svo.getCpu();
long vmMem = svo.getRamSize() * 1024L * 1024L;
@ -283,8 +291,8 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager,
final long hostId = vm.getHostId();
final HostVO host = _hostDao.findById(hostId);
final long clusterId = host.getClusterId();
final float cpuOvercommitRatio = Float.parseFloat(_clusterDetailsDao.findDetail(clusterId, "cpuOvercommitRatio").getValue());
final float memoryOvercommitRatio = Float.parseFloat(_clusterDetailsDao.findDetail(clusterId, "memoryOvercommitRatio").getValue());
final float cpuOvercommitRatio = Float.parseFloat(_clusterDetailsDao.findDetail(clusterId, VmDetailConstants.CPU_OVER_COMMIT_RATIO).getValue());
final float memoryOvercommitRatio = Float.parseFloat(_clusterDetailsDao.findDetail(clusterId, VmDetailConstants.MEMORY_OVER_COMMIT_RATIO).getValue());
final ServiceOfferingVO svo = _offeringsDao.findById(vm.getId(), vm.getServiceOfferingId());
@ -376,13 +384,13 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager,
toHumanReadableSize(capacityMem.getReservedCapacity()), toHumanReadableSize(ram), fromLastHost);
long cluster_id = host.getClusterId();
ClusterDetailsVO cluster_detail_cpu = _clusterDetailsDao.findDetail(cluster_id, "cpuOvercommitRatio");
ClusterDetailsVO cluster_detail_ram = _clusterDetailsDao.findDetail(cluster_id, "memoryOvercommitRatio");
ClusterDetailsVO cluster_detail_cpu = _clusterDetailsDao.findDetail(cluster_id, VmDetailConstants.CPU_OVER_COMMIT_RATIO);
ClusterDetailsVO cluster_detail_ram = _clusterDetailsDao.findDetail(cluster_id, VmDetailConstants.MEMORY_OVER_COMMIT_RATIO);
Float cpuOvercommitRatio = Float.parseFloat(cluster_detail_cpu.getValue());
Float memoryOvercommitRatio = Float.parseFloat(cluster_detail_ram.getValue());
boolean hostHasCpuCapability, hostHasCapacity = false;
hostHasCpuCapability = checkIfHostHasCpuCapability(host.getId(), cpucore, cpuspeed);
hostHasCpuCapability = checkIfHostHasCpuCapability(host, cpucore, cpuspeed);
if (hostHasCpuCapability) {
// first check from reserved capacity
@ -412,25 +420,16 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager,
}
@Override
public boolean checkIfHostHasCpuCapability(long hostId, Integer cpuNum, Integer cpuSpeed) {
public boolean checkIfHostHasCpuCapability(Host host, Integer cpuNum, Integer cpuSpeed) {
// Check host can support the Cpu Number and Speed.
Host host = _hostDao.findById(hostId);
boolean isCpuNumGood = host.getCpus().intValue() >= cpuNum;
boolean isCpuSpeedGood = host.getSpeed().intValue() >= cpuSpeed;
if (isCpuNumGood && isCpuSpeedGood) {
if (logger.isDebugEnabled()) {
logger.debug("Host: {} has cpu capability (cpu:{}, speed:{}) " +
"to support requested CPU: {} and requested speed: {}", host, host.getCpus(), host.getSpeed(), cpuNum, cpuSpeed);
}
return true;
} else {
if (logger.isDebugEnabled()) {
logger.debug("Host: {} doesn't have cpu capability (cpu:{}, speed:{})" +
" to support requested CPU: {} and requested speed: {}", host, host.getCpus(), host.getSpeed(), cpuNum, cpuSpeed);
}
return false;
}
boolean hasCpuCapability = isCpuNumGood && isCpuSpeedGood;
logger.debug("{} {} cpu capability (cpu: {}, speed: {} ) to support requested CPU: {} and requested speed: {}",
host, hasCpuCapability ? "has" : "doesn't have" ,host.getCpus(), host.getSpeed(), cpuNum, cpuSpeed);
return hasCpuCapability;
}
@Override
@ -628,21 +627,50 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager,
return totalAllocatedSize;
}
@DB
@Override
public void updateCapacityForHost(final Host host) {
// prepare the service offerings
List<ServiceOfferingVO> offerings = _offeringsDao.listAllIncludingRemoved();
Map<Long, ServiceOfferingVO> offeringsMap = new HashMap<Long, ServiceOfferingVO>();
for (ServiceOfferingVO offering : offerings) {
offeringsMap.put(offering.getId(), offering);
protected Pair<String, String> getClusterValues(long clusterId) {
Map<String, String> map = _clusterDetailsDao.findDetails(clusterId,
List.of(VmDetailConstants.CPU_OVER_COMMIT_RATIO, VmDetailConstants.MEMORY_OVER_COMMIT_RATIO));
return new Pair<>(map.get(VmDetailConstants.CPU_OVER_COMMIT_RATIO),
map.get(VmDetailConstants.MEMORY_OVER_COMMIT_RATIO));
}
protected Map<Long, ServiceOfferingVO> getServiceOfferingsMap() {
List<ServiceOfferingVO> serviceOfferings = _offeringsDao.listAllIncludingRemoved();
if (CollectionUtils.isEmpty(serviceOfferings)) {
return new HashMap<>();
}
updateCapacityForHost(host, offeringsMap);
return serviceOfferings.stream()
.collect(Collectors.toMap(
ServiceOfferingVO::getId,
offering -> offering
));
}
protected ServiceOfferingVO getServiceOffering(long id) {
Map <Long, ServiceOfferingVO> map = serviceOfferingsCache.get();
if (map.containsKey(id)) {
return map.get(id);
}
ServiceOfferingVO serviceOfferingVO = _offeringsDao.findByIdIncludingRemoved(id);
if (serviceOfferingVO != null) {
serviceOfferingsCache.invalidate();
}
return serviceOfferingVO;
}
protected Map<String, String> getVmDetailsForCapacityCalculation(long vmId) {
return _userVmDetailsDao.listDetailsKeyPairs(vmId,
List.of(VmDetailConstants.CPU_OVER_COMMIT_RATIO,
VmDetailConstants.MEMORY_OVER_COMMIT_RATIO,
UsageEventVO.DynamicParameters.memory.name(),
UsageEventVO.DynamicParameters.cpuNumber.name(),
UsageEventVO.DynamicParameters.cpuSpeed.name()));
}
@DB
@Override
public void updateCapacityForHost(final Host host, final Map<Long, ServiceOfferingVO> offeringsMap) {
public void updateCapacityForHost(final Host host) {
long usedCpuCore = 0;
long reservedCpuCore = 0;
long usedCpu = 0;
@ -651,32 +679,27 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager,
long reservedCpu = 0;
final CapacityState capacityState = (host.getResourceState() == ResourceState.Enabled) ? CapacityState.Enabled : CapacityState.Disabled;
List<VMInstanceVO> vms = _vmDao.listUpByHostId(host.getId());
if (logger.isDebugEnabled()) {
logger.debug("Found {} VMs on host {}", vms.size(), host);
}
List<VMInstanceVO> vms = _vmDao.listIdServiceOfferingForUpVmsByHostId(host.getId());
logger.debug("Found {} VMs on {}", vms.size(), host);
final List<VMInstanceVO> vosMigrating = _vmDao.listVmsMigratingFromHost(host.getId());
if (logger.isDebugEnabled()) {
logger.debug("Found {} VMs are Migrating from host {}", vosMigrating.size(), host);
}
final List<VMInstanceVO> vosMigrating = _vmDao.listIdServiceOfferingForVmsMigratingFromHost(host.getId());
logger.debug("Found {} VMs are Migrating from {}", vosMigrating.size(), host);
vms.addAll(vosMigrating);
ClusterVO cluster = _clusterDao.findById(host.getClusterId());
ClusterDetailsVO clusterDetailCpu = _clusterDetailsDao.findDetail(cluster.getId(), "cpuOvercommitRatio");
ClusterDetailsVO clusterDetailRam = _clusterDetailsDao.findDetail(cluster.getId(), "memoryOvercommitRatio");
Float clusterCpuOvercommitRatio = Float.parseFloat(clusterDetailCpu.getValue());
Float clusterRamOvercommitRatio = Float.parseFloat(clusterDetailRam.getValue());
Pair<String, String> clusterValues =
clusterValuesCache.get(host.getClusterId());
Float clusterCpuOvercommitRatio = Float.parseFloat(clusterValues.first());
Float clusterRamOvercommitRatio = Float.parseFloat(clusterValues.second());
for (VMInstanceVO vm : vms) {
Float cpuOvercommitRatio = 1.0f;
Float ramOvercommitRatio = 1.0f;
Map<String, String> vmDetails = _userVmDetailsDao.listDetailsKeyPairs(vm.getId());
String vmDetailCpu = vmDetails.get("cpuOvercommitRatio");
String vmDetailRam = vmDetails.get("memoryOvercommitRatio");
Map<String, String> vmDetails = getVmDetailsForCapacityCalculation(vm.getId());
String vmDetailCpu = vmDetails.get(VmDetailConstants.CPU_OVER_COMMIT_RATIO);
String vmDetailRam = vmDetails.get(VmDetailConstants.MEMORY_OVER_COMMIT_RATIO);
// if vmDetailCpu or vmDetailRam is not null it means it is running in a overcommitted cluster.
cpuOvercommitRatio = (vmDetailCpu != null) ? Float.parseFloat(vmDetailCpu) : clusterCpuOvercommitRatio;
ramOvercommitRatio = (vmDetailRam != null) ? Float.parseFloat(vmDetailRam) : clusterRamOvercommitRatio;
ServiceOffering so = offeringsMap.get(vm.getServiceOfferingId());
ServiceOffering so = getServiceOffering(vm.getServiceOfferingId());
if (so == null) {
so = _offeringsDao.findByIdIncludingRemoved(vm.getServiceOfferingId());
}
@ -702,26 +725,25 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager,
}
List<VMInstanceVO> vmsByLastHostId = _vmDao.listByLastHostId(host.getId());
if (logger.isDebugEnabled()) {
logger.debug("Found {} VM, not running on host {}", vmsByLastHostId.size(), host);
}
logger.debug("Found {} VM, not running on {}", vmsByLastHostId.size(), host);
for (VMInstanceVO vm : vmsByLastHostId) {
Float cpuOvercommitRatio = 1.0f;
Float ramOvercommitRatio = 1.0f;
long lastModificationTime = Optional.ofNullable(vm.getUpdateTime()).orElse(vm.getCreated()).getTime();
long secondsSinceLastUpdate = (DateUtil.currentGMTTime().getTime() - lastModificationTime) / 1000;
if (secondsSinceLastUpdate < _vmCapacityReleaseInterval) {
UserVmDetailVO vmDetailCpu = _userVmDetailsDao.findDetail(vm.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO);
UserVmDetailVO vmDetailRam = _userVmDetailsDao.findDetail(vm.getId(), VmDetailConstants.MEMORY_OVER_COMMIT_RATIO);
Map<String, String> vmDetails = getVmDetailsForCapacityCalculation(vm.getId());
String vmDetailCpu = vmDetails.get(VmDetailConstants.CPU_OVER_COMMIT_RATIO);
String vmDetailRam = vmDetails.get(VmDetailConstants.MEMORY_OVER_COMMIT_RATIO);
if (vmDetailCpu != null) {
//if vmDetail_cpu is not null it means it is running in a overcommited cluster.
cpuOvercommitRatio = Float.parseFloat(vmDetailCpu.getValue());
cpuOvercommitRatio = Float.parseFloat(vmDetailCpu);
}
if (vmDetailRam != null) {
ramOvercommitRatio = Float.parseFloat(vmDetailRam.getValue());
ramOvercommitRatio = Float.parseFloat(vmDetailRam);
}
ServiceOffering so = offeringsMap.get(vm.getServiceOfferingId());
Map<String, String> vmDetails = _userVmDetailsDao.listDetailsKeyPairs(vm.getId());
ServiceOffering so = getServiceOffering(vm.getServiceOfferingId());
if (so == null) {
so = _offeringsDao.findByIdIncludingRemoved(vm.getServiceOfferingId());
}
@ -761,9 +783,24 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager,
}
}
CapacityVO cpuCap = _capacityDao.findByHostIdType(host.getId(), Capacity.CAPACITY_TYPE_CPU);
CapacityVO memCap = _capacityDao.findByHostIdType(host.getId(), Capacity.CAPACITY_TYPE_MEMORY);
CapacityVO cpuCoreCap = _capacityDao.findByHostIdType(host.getId(), CapacityVO.CAPACITY_TYPE_CPU_CORE);
List<CapacityVO> capacities = _capacityDao.listByHostIdTypes(host.getId(), List.of(Capacity.CAPACITY_TYPE_CPU,
Capacity.CAPACITY_TYPE_MEMORY,
CapacityVO.CAPACITY_TYPE_CPU_CORE));
CapacityVO cpuCap = null;
CapacityVO memCap = null;
CapacityVO cpuCoreCap = null;
for (CapacityVO c : capacities) {
if (c.getCapacityType() == Capacity.CAPACITY_TYPE_CPU) {
cpuCap = c;
} else if (c.getCapacityType() == Capacity.CAPACITY_TYPE_MEMORY) {
memCap = c;
} else if (c.getCapacityType() == Capacity.CAPACITY_TYPE_CPU_CORE) {
cpuCoreCap = c;
}
if (ObjectUtils.allNotNull(cpuCap, memCap, cpuCoreCap)) {
break;
}
}
if (cpuCoreCap != null) {
long hostTotalCpuCore = host.getCpus().longValue();
@ -995,8 +1032,8 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager,
capacityCPU.addAnd("podId", SearchCriteria.Op.EQ, server.getPodId());
capacityCPU.addAnd("capacityType", SearchCriteria.Op.EQ, Capacity.CAPACITY_TYPE_CPU);
List<CapacityVO> capacityVOCpus = _capacityDao.search(capacitySC, null);
Float cpuovercommitratio = Float.parseFloat(_clusterDetailsDao.findDetail(server.getClusterId(), "cpuOvercommitRatio").getValue());
Float memoryOvercommitRatio = Float.parseFloat(_clusterDetailsDao.findDetail(server.getClusterId(), "memoryOvercommitRatio").getValue());
Float cpuovercommitratio = Float.parseFloat(_clusterDetailsDao.findDetail(server.getClusterId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO).getValue());
Float memoryOvercommitRatio = Float.parseFloat(_clusterDetailsDao.findDetail(server.getClusterId(), VmDetailConstants.MEMORY_OVER_COMMIT_RATIO).getValue());
if (capacityVOCpus != null && !capacityVOCpus.isEmpty()) {
CapacityVO CapacityVOCpu = capacityVOCpus.get(0);
@ -1053,9 +1090,9 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager,
String capacityOverProvisioningName = "";
if (capacityType == Capacity.CAPACITY_TYPE_CPU) {
capacityOverProvisioningName = "cpuOvercommitRatio";
capacityOverProvisioningName = VmDetailConstants.CPU_OVER_COMMIT_RATIO;
} else if (capacityType == Capacity.CAPACITY_TYPE_MEMORY) {
capacityOverProvisioningName = "memoryOvercommitRatio";
capacityOverProvisioningName = VmDetailConstants.MEMORY_OVER_COMMIT_RATIO;
} else {
throw new CloudRuntimeException("Invalid capacityType - " + capacityType);
}
@ -1093,13 +1130,11 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager,
public Pair<Boolean, Boolean> checkIfHostHasCpuCapabilityAndCapacity(Host host, ServiceOffering offering, boolean considerReservedCapacity) {
int cpu_requested = offering.getCpu() * offering.getSpeed();
long ram_requested = offering.getRamSize() * 1024L * 1024L;
Cluster cluster = _clusterDao.findById(host.getClusterId());
ClusterDetailsVO clusterDetailsCpuOvercommit = _clusterDetailsDao.findDetail(cluster.getId(), "cpuOvercommitRatio");
ClusterDetailsVO clusterDetailsRamOvercommmt = _clusterDetailsDao.findDetail(cluster.getId(), "memoryOvercommitRatio");
Float cpuOvercommitRatio = Float.parseFloat(clusterDetailsCpuOvercommit.getValue());
Float memoryOvercommitRatio = Float.parseFloat(clusterDetailsRamOvercommmt.getValue());
Pair<String, String> clusterDetails = getClusterValues(host.getClusterId());
Float cpuOvercommitRatio = Float.parseFloat(clusterDetails.first());
Float memoryOvercommitRatio = Float.parseFloat(clusterDetails.second());
boolean hostHasCpuCapability = checkIfHostHasCpuCapability(host.getId(), offering.getCpu(), offering.getSpeed());
boolean hostHasCpuCapability = checkIfHostHasCpuCapability(host, offering.getCpu(), offering.getSpeed());
boolean hostHasCapacity = checkIfHostHasCapacity(host, cpu_requested, ram_requested, false, cpuOvercommitRatio, memoryOvercommitRatio,
considerReservedCapacity);
@ -1241,6 +1276,6 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager,
public ConfigKey<?>[] getConfigKeys() {
return new ConfigKey<?>[] {CpuOverprovisioningFactor, MemOverprovisioningFactor, StorageCapacityDisableThreshold, StorageOverprovisioningFactor,
StorageAllocatedCapacityDisableThreshold, StorageOperationsExcludeCluster, ImageStoreNFSVersion, SecondaryStorageCapacityThreshold,
StorageAllocatedCapacityDisableThresholdForVolumeSize };
StorageAllocatedCapacityDisableThresholdForVolumeSize, CapacityCalculateWorkers };
}
}

12
server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java
View File

@ -16,6 +16,11 @@
// under the License.
package com.cloud.configuration;
import static com.cloud.configuration.Config.SecStorageAllowedInternalDownloadSites;
import static com.cloud.offering.NetworkOffering.RoutingMode.Dynamic;
import static com.cloud.offering.NetworkOffering.RoutingMode.Static;
import static org.apache.cloudstack.framework.config.ConfigKey.CATEGORY_SYSTEM;
import java.io.UnsupportedEncodingException;
import java.net.URI;
import java.net.URISyntaxException;
@ -308,11 +313,6 @@ import com.google.common.collect.Sets;
import com.googlecode.ipv6.IPv6Address;
import com.googlecode.ipv6.IPv6Network;
import static com.cloud.configuration.Config.SecStorageAllowedInternalDownloadSites;
import static com.cloud.offering.NetworkOffering.RoutingMode.Dynamic;
import static com.cloud.offering.NetworkOffering.RoutingMode.Static;
import static org.apache.cloudstack.framework.config.ConfigKey.CATEGORY_SYSTEM;
public class ConfigurationManagerImpl extends ManagerBase implements ConfigurationManager, ConfigurationService, Configurable {
public static final String PERACCOUNT = "peraccount";
public static final String PERZONE = "perzone";
@ -2521,7 +2521,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
// Check if there are any non-removed hosts in the zone.
if (!_hostDao.listByDataCenterId(zoneId).isEmpty()) {
if (!_hostDao.listEnabledIdsByDataCenterId(zoneId).isEmpty()) {
throw new CloudRuntimeException(errorMsg + "there are servers in this zone.");
}

12
server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java
View File

@ -869,11 +869,9 @@ public class ConsoleProxyManagerImpl extends ManagerBase implements ConsoleProxy
}
public boolean isZoneReady(Map<Long, ZoneHostInfo> zoneHostInfoMap, DataCenter dataCenter) {
List <HostVO> hosts = hostDao.listByDataCenterId(dataCenter.getId());
if (CollectionUtils.isEmpty(hosts)) {
if (logger.isDebugEnabled()) {
logger.debug("Zone {} has no host available which is enabled and in Up state", dataCenter);
}
Integer totalUpAndEnabledHosts = hostDao.countUpAndEnabledHostsInZone(dataCenter.getId());
if (totalUpAndEnabledHosts != null && totalUpAndEnabledHosts < 1) {
logger.debug("{} has no host available which is enabled and in Up state", dataCenter);
return false;
}
ZoneHostInfo zoneHostInfo = zoneHostInfoMap.get(dataCenter.getId());
@ -894,8 +892,8 @@ public class ConsoleProxyManagerImpl extends ManagerBase implements ConsoleProxy
if (templateHostRef != null) {
Boolean useLocalStorage = BooleanUtils.toBoolean(ConfigurationManagerImpl.SystemVMUseLocalStorage.valueIn(dataCenter.getId()));
List<Pair<Long, Integer>> l = consoleProxyDao.getDatacenterStoragePoolHostInfo(dataCenter.getId(), useLocalStorage);
if (CollectionUtils.isNotEmpty(l) && l.get(0).second() > 0) {
boolean hasDatacenterStoragePoolHostInfo = consoleProxyDao.hasDatacenterStoragePoolHostInfo(dataCenter.getId(), !useLocalStorage);
if (hasDatacenterStoragePoolHostInfo) {
return true;
} else {
if (logger.isDebugEnabled()) {

58
server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java
View File

@ -36,22 +36,7 @@ import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
import com.cloud.cpu.CPU;
import com.cloud.vm.UserVmManager;
import org.apache.cloudstack.affinity.AffinityGroupDomainMapVO;
import com.cloud.storage.VMTemplateVO;
import com.cloud.storage.dao.VMTemplateDao;
import com.cloud.user.AccountVO;
import com.cloud.user.dao.AccountDao;
import com.cloud.exception.StorageUnavailableException;
import com.cloud.utils.db.Filter;
import com.cloud.utils.fsm.StateMachine2;
import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.framework.config.Configurable;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.cloudstack.affinity.AffinityGroupProcessor;
import org.apache.cloudstack.affinity.AffinityGroupService;
import org.apache.cloudstack.affinity.AffinityGroupVMMapVO;
@ -64,6 +49,8 @@ import org.apache.cloudstack.engine.cloud.entity.api.db.dao.VMReservationDao;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
import org.apache.cloudstack.engine.subsystem.api.storage.StoragePoolAllocator;
import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.framework.config.Configurable;
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.apache.cloudstack.framework.messagebus.MessageBus;
import org.apache.cloudstack.framework.messagebus.MessageSubscriber;
@ -71,6 +58,9 @@ import org.apache.cloudstack.managed.context.ManagedContextTimerTask;
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
import org.apache.cloudstack.utils.identity.ManagementServerNode;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang3.StringUtils;
import com.cloud.agent.AgentManager;
import com.cloud.agent.Listener;
@ -85,6 +75,7 @@ import com.cloud.capacity.CapacityManager;
import com.cloud.capacity.dao.CapacityDao;
import com.cloud.configuration.Config;
import com.cloud.configuration.ConfigurationManagerImpl;
import com.cloud.cpu.CPU;
import com.cloud.dc.ClusterDetailsDao;
import com.cloud.dc.ClusterDetailsVO;
import com.cloud.dc.ClusterVO;
@ -102,6 +93,7 @@ import com.cloud.deploy.dao.PlannerHostReservationDao;
import com.cloud.exception.AffinityConflictException;
import com.cloud.exception.ConnectionException;
import com.cloud.exception.InsufficientServerCapacityException;
import com.cloud.exception.StorageUnavailableException;
import com.cloud.gpu.GPU;
import com.cloud.host.DetailVO;
import com.cloud.host.Host;
@ -122,15 +114,19 @@ import com.cloud.storage.ScopeType;
import com.cloud.storage.StorageManager;
import com.cloud.storage.StoragePool;
import com.cloud.storage.StoragePoolHostVO;
import com.cloud.storage.VMTemplateVO;
import com.cloud.storage.Volume;
import com.cloud.storage.VolumeVO;
import com.cloud.storage.dao.DiskOfferingDao;
import com.cloud.storage.dao.GuestOSCategoryDao;
import com.cloud.storage.dao.GuestOSDao;
import com.cloud.storage.dao.StoragePoolHostDao;
import com.cloud.storage.dao.VMTemplateDao;
import com.cloud.storage.dao.VolumeDao;
import com.cloud.template.VirtualMachineTemplate;
import com.cloud.user.AccountManager;
import com.cloud.user.AccountVO;
import com.cloud.user.dao.AccountDao;
import com.cloud.utils.DateUtil;
import com.cloud.utils.LogUtils;
import com.cloud.utils.NumbersUtil;
@ -138,13 +134,16 @@ import com.cloud.utils.Pair;
import com.cloud.utils.component.Manager;
import com.cloud.utils.component.ManagerBase;
import com.cloud.utils.db.DB;
import com.cloud.utils.db.Filter;
import com.cloud.utils.db.SearchCriteria;
import com.cloud.utils.db.Transaction;
import com.cloud.utils.db.TransactionCallback;
import com.cloud.utils.db.TransactionStatus;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.fsm.StateListener;
import com.cloud.utils.fsm.StateMachine2;
import com.cloud.vm.DiskProfile;
import com.cloud.vm.UserVmManager;
import com.cloud.vm.VMInstanceVO;
import com.cloud.vm.VirtualMachine;
import com.cloud.vm.VirtualMachine.Event;
@ -295,8 +294,9 @@ StateListener<State, VirtualMachine.Event, VirtualMachine>, Configurable {
return;
}
final Long lastHostClusterId = lastHost.getClusterId();
logger.warn("VM last host ID: {} belongs to zone ID: {} for which config - {} is false and storage migration would be needed for inter-cluster migration, therefore, adding all other clusters except ID: {} from this zone to avoid list", lastHost, vm.getDataCenterId(), ConfigurationManagerImpl.MIGRATE_VM_ACROSS_CLUSTERS.key(), lastHostClusterId);
List<Long> clusterIds = _clusterDao.listAllClusters(lastHost.getDataCenterId());
logger.warn(String.format("VM last host ID: %d belongs to zone ID: %s for which config - %s is false and storage migration would be needed for inter-cluster migration, therefore, adding all other clusters except ID: %d from this zone to avoid list",
lastHost.getId(), vm.getDataCenterId(), ConfigurationManagerImpl.MIGRATE_VM_ACROSS_CLUSTERS.key(), lastHostClusterId));
List<Long> clusterIds = _clusterDao.listAllClusterIds(lastHost.getDataCenterId());
Set<Long> existingAvoidedClusters = avoids.getClustersToAvoid();
clusterIds = clusterIds.stream().filter(x -> !Objects.equals(x, lastHostClusterId) && (existingAvoidedClusters == null || !existingAvoidedClusters.contains(x))).collect(Collectors.toList());
avoids.addClusterList(clusterIds);
@ -492,7 +492,7 @@ StateListener<State, VirtualMachine.Event, VirtualMachine>, Configurable {
float memoryOvercommitRatio = Float.parseFloat(cluster_detail_ram.getValue());
boolean hostHasCpuCapability, hostHasCapacity = false;
hostHasCpuCapability = _capacityMgr.checkIfHostHasCpuCapability(host.getId(), offering.getCpu(), offering.getSpeed());
hostHasCpuCapability = _capacityMgr.checkIfHostHasCpuCapability(host, offering.getCpu(), offering.getSpeed());
if (hostHasCpuCapability) {
// first check from reserved capacity
@ -736,12 +736,10 @@ StateListener<State, VirtualMachine.Event, VirtualMachine>, Configurable {
* Adds disabled Hosts to the ExcludeList in order to avoid them at the deployment planner.
*/
protected void avoidDisabledHosts(DataCenter dc, ExcludeList avoids) {
List<HostVO> disabledHosts = _hostDao.listDisabledByDataCenterId(dc.getId());
logger.debug("Adding hosts [{}] of datacenter [{}] to the avoid set, because these hosts are in the Disabled state.",
disabledHosts.stream().map(HostVO::getUuid).collect(Collectors.joining(", ")), dc);
for (HostVO host : disabledHosts) {
avoids.addHost(host.getId());
}
List<Long> disabledHostIds = _hostDao.listDisabledIdsByDataCenterId(dc.getId());
logger.debug("Adding hosts {} of datacenter [{}] to the avoid set, because these hosts are in the Disabled state.",
StringUtils.join(disabledHostIds), dc.getUuid());
disabledHostIds.forEach(avoids::addHost);
}
/**
@ -860,7 +858,7 @@ StateListener<State, VirtualMachine.Event, VirtualMachine>, Configurable {
List<Long> allDedicatedPods = _dedicatedDao.listAllPods();
allPodsInDc.retainAll(allDedicatedPods);
List<Long> allClustersInDc = _clusterDao.listAllClusters(dc.getId());
List<Long> allClustersInDc = _clusterDao.listAllClusterIds(dc.getId());
List<Long> allDedicatedClusters = _dedicatedDao.listAllClusters();
allClustersInDc.retainAll(allDedicatedClusters);
@ -1147,9 +1145,11 @@ StateListener<State, VirtualMachine.Event, VirtualMachine>, Configurable {
private void checkHostReservations() {
List<PlannerHostReservationVO> reservedHosts = _plannerHostReserveDao.listAllReservedHosts();
for (PlannerHostReservationVO hostReservation : reservedHosts) {
HostVO host = _hostDao.findById(hostReservation.getHostId());
List<HostVO> hosts = _hostDao.listByIds(reservedHosts
.stream()
.map(PlannerHostReservationVO::getHostId)
.collect(Collectors.toList()));
for (HostVO host : hosts) {
if (host != null && host.getManagementServerId() != null && host.getManagementServerId() == _nodeId) {
checkHostReservationRelease(host);
}
@ -1338,7 +1338,7 @@ StateListener<State, VirtualMachine.Event, VirtualMachine>, Configurable {
Pair<Host, Map<Volume, StoragePool>> potentialResources = findPotentialDeploymentResources(suitableHosts, suitableVolumeStoragePools, avoid,
resourceUsageRequired, readyAndReusedVolumes, plan.getPreferredHosts(), vmProfile.getVirtualMachine());
if (potentialResources != null) {
Host host = _hostDao.findById(potentialResources.first().getId());
Host host = potentialResources.first();
Map<Volume, StoragePool> storageVolMap = potentialResources.second();
// remove the reused vol<->pool from destination, since
// we don't have to prepare this volume.

66
server/src/main/java/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java
View File

@ -16,6 +16,29 @@
// under the License.
package com.cloud.hypervisor.kvm.discoverer;
import static com.cloud.configuration.ConfigurationManagerImpl.ADD_HOST_ON_SERVICE_RESTART_KVM;
import java.net.InetAddress;
import java.net.URI;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
import org.apache.cloudstack.agent.lb.IndirectAgentLB;
import org.apache.cloudstack.ca.CAManager;
import org.apache.cloudstack.ca.SetupCertificateCommand;
import org.apache.cloudstack.direct.download.DirectDownloadManager;
import org.apache.cloudstack.framework.ca.Certificate;
import org.apache.cloudstack.utils.cache.LazyCache;
import org.apache.cloudstack.utils.security.KeyStoreUtils;
import com.cloud.agent.AgentManager;
import com.cloud.agent.Listener;
import com.cloud.agent.api.AgentControlAnswer;
@ -32,6 +55,7 @@ import com.cloud.exception.DiscoveredWithErrorException;
import com.cloud.exception.DiscoveryException;
import com.cloud.exception.OperationTimedoutException;
import com.cloud.host.Host;
import com.cloud.host.HostInfo;
import com.cloud.host.HostVO;
import com.cloud.host.Status;
import com.cloud.host.dao.HostDao;
@ -48,26 +72,7 @@ import com.cloud.utils.StringUtils;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.ssh.SSHCmdHelper;
import com.trilead.ssh2.Connection;
import org.apache.cloudstack.agent.lb.IndirectAgentLB;
import org.apache.cloudstack.ca.CAManager;
import org.apache.cloudstack.ca.SetupCertificateCommand;
import org.apache.cloudstack.direct.download.DirectDownloadManager;
import org.apache.cloudstack.framework.ca.Certificate;
import org.apache.cloudstack.utils.security.KeyStoreUtils;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
import java.net.InetAddress;
import java.net.URI;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import static com.cloud.configuration.ConfigurationManagerImpl.ADD_HOST_ON_SERVICE_RESTART_KVM;
public abstract class LibvirtServerDiscoverer extends DiscovererBase implements Discoverer, Listener, ResourceStateAdapter {
private final int _waitTime = 5; /* wait for 5 minutes */
@ -89,6 +94,16 @@ public abstract class LibvirtServerDiscoverer extends DiscovererBase implements
@Inject
private HostDao hostDao;
private LazyCache<Long, HostVO> clusterExistingHostCache;
private HostVO getExistingHostForCluster(long clusterId) {
HostVO existingHostInCluster = _hostDao.findAnyStateHypervisorHostInCluster(clusterId);
if (existingHostInCluster != null) {
_hostDao.loadDetails(existingHostInCluster);
}
return existingHostInCluster;
}
@Override
public abstract Hypervisor.HypervisorType getHypervisorType();
@ -425,6 +440,9 @@ public abstract class LibvirtServerDiscoverer extends DiscovererBase implements
_kvmGuestNic = _kvmPrivateNic;
}
clusterExistingHostCache = new LazyCache<>(32, 30,
this::getExistingHostForCluster);
agentMgr.registerForHostEvents(this, true, false, false);
_resourceMgr.registerResourceStateAdapter(this.getClass().getSimpleName(), this);
return true;
@ -467,12 +485,10 @@ public abstract class LibvirtServerDiscoverer extends DiscovererBase implements
throw new IllegalArgumentException("cannot add host, due to can't find cluster: " + host.getClusterId());
}
List<HostVO> hostsInCluster = _resourceMgr.listAllHostsInCluster(clusterVO.getId());
if (!hostsInCluster.isEmpty()) {
HostVO oneHost = hostsInCluster.get(0);
_hostDao.loadDetails(oneHost);
String hostOsInCluster = oneHost.getDetail("Host.OS");
String hostOs = ssCmd.getHostDetails().get("Host.OS");
HostVO existingHostInCluster = clusterExistingHostCache.get(clusterVO.getId());
if (existingHostInCluster != null) {
String hostOsInCluster = existingHostInCluster.getDetail(HostInfo.HOST_OS);
String hostOs = ssCmd.getHostDetails().get(HostInfo.HOST_OS);
if (!isHostOsCompatibleWithOtherHost(hostOsInCluster, hostOs)) {
String msg = String.format("host: %s with hostOS, \"%s\"into a cluster, in which there are \"%s\" hosts added", firstCmd.getPrivateIpAddress(), hostOs, hostOsInCluster);
if (hostOs != null && hostOs.startsWith(hostOsInCluster)) {

24
server/src/main/java/com/cloud/network/NetworkServiceImpl.java
View File

@ -40,16 +40,6 @@ import java.util.UUID;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
import com.cloud.bgp.BGPService;
import com.cloud.dc.VlanDetailsVO;
import com.cloud.dc.dao.VlanDetailsDao;
import com.cloud.network.dao.NsxProviderDao;
import com.cloud.network.dao.PublicIpQuarantineDao;
import com.cloud.network.dao.VirtualRouterProviderDao;
import com.cloud.network.element.NsxProviderVO;
import com.cloud.network.element.VirtualRouterProviderVO;
import com.cloud.offering.ServiceOffering;
import com.cloud.service.dao.ServiceOfferingDao;
import org.apache.cloudstack.acl.ControlledEntity.ACLType;
import org.apache.cloudstack.acl.SecurityChecker.AccessType;
import org.apache.cloudstack.alert.AlertService;
@ -104,6 +94,7 @@ import com.cloud.alert.AlertManager;
import com.cloud.api.ApiDBUtils;
import com.cloud.api.query.dao.DomainRouterJoinDao;
import com.cloud.api.query.vo.DomainRouterJoinVO;
import com.cloud.bgp.BGPService;
import com.cloud.configuration.Config;
import com.cloud.configuration.ConfigurationManager;
import com.cloud.configuration.Resource;
@ -114,12 +105,14 @@ import com.cloud.dc.DataCenterVO;
import com.cloud.dc.DataCenterVnetVO;
import com.cloud.dc.DomainVlanMapVO;
import com.cloud.dc.Vlan.VlanType;
import com.cloud.dc.VlanDetailsVO;
import com.cloud.dc.VlanVO;
import com.cloud.dc.dao.AccountVlanMapDao;
import com.cloud.dc.dao.DataCenterDao;
import com.cloud.dc.dao.DataCenterVnetDao;
import com.cloud.dc.dao.DomainVlanMapDao;
import com.cloud.dc.dao.VlanDao;
import com.cloud.dc.dao.VlanDetailsDao;
import com.cloud.deploy.DeployDestination;
import com.cloud.domain.Domain;
import com.cloud.domain.DomainVO;
@ -165,6 +158,7 @@ import com.cloud.network.dao.NetworkDomainDao;
import com.cloud.network.dao.NetworkDomainVO;
import com.cloud.network.dao.NetworkServiceMapDao;
import com.cloud.network.dao.NetworkVO;
import com.cloud.network.dao.NsxProviderDao;
import com.cloud.network.dao.OvsProviderDao;
import com.cloud.network.dao.PhysicalNetworkDao;
import com.cloud.network.dao.PhysicalNetworkServiceProviderDao;
@ -172,9 +166,13 @@ import com.cloud.network.dao.PhysicalNetworkServiceProviderVO;
import com.cloud.network.dao.PhysicalNetworkTrafficTypeDao;
import com.cloud.network.dao.PhysicalNetworkTrafficTypeVO;
import com.cloud.network.dao.PhysicalNetworkVO;
import com.cloud.network.dao.PublicIpQuarantineDao;
import com.cloud.network.dao.VirtualRouterProviderDao;
import com.cloud.network.element.NetworkElement;
import com.cloud.network.element.NsxProviderVO;
import com.cloud.network.element.OvsProviderVO;
import com.cloud.network.element.VirtualRouterElement;
import com.cloud.network.element.VirtualRouterProviderVO;
import com.cloud.network.element.VpcVirtualRouterElement;
import com.cloud.network.guru.GuestNetworkGuru;
import com.cloud.network.guru.NetworkGuru;
@ -198,6 +196,7 @@ import com.cloud.network.vpc.dao.VpcDao;
import com.cloud.network.vpc.dao.VpcGatewayDao;
import com.cloud.network.vpc.dao.VpcOfferingDao;
import com.cloud.offering.NetworkOffering;
import com.cloud.offering.ServiceOffering;
import com.cloud.offerings.NetworkOfferingVO;
import com.cloud.offerings.dao.NetworkOfferingDao;
import com.cloud.offerings.dao.NetworkOfferingServiceMapDao;
@ -207,6 +206,7 @@ import com.cloud.projects.ProjectManager;
import com.cloud.server.ResourceTag;
import com.cloud.server.ResourceTag.ResourceObjectType;
import com.cloud.service.ServiceOfferingVO;
import com.cloud.service.dao.ServiceOfferingDao;
import com.cloud.tags.ResourceTagVO;
import com.cloud.tags.dao.ResourceTagDao;
import com.cloud.user.Account;
@ -1779,6 +1779,10 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C
throwInvalidIdException("Network offering with specified id doesn't support adding multiple ip ranges", ntwkOff.getUuid(), NETWORK_OFFERING_ID);
}
if (GuestType.Shared == ntwkOff.getGuestType() && !ntwkOff.isSpecifyVlan() && Objects.isNull(associatedNetworkId)) {
throw new CloudRuntimeException("Associated network must be provided when creating Shared networks when specifyVlan is false");
}
Pair<Integer, Integer> interfaceMTUs = validateMtuConfig(publicMtu, privateMtu, zone.getId());
mtuCheckForVpcNetwork(vpcId, interfaceMTUs, publicMtu, privateMtu);

2
server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java
View File

@ -2715,7 +2715,7 @@ public class AutoScaleManagerImpl extends ManagerBase implements AutoScaleManage
return vmStatsById;
}
try {
vmStatsById = virtualMachineManager.getVirtualMachineStatistics(host.getId(), host.getName(), vmIds);
vmStatsById = virtualMachineManager.getVirtualMachineStatistics(host, vmIds);
if (MapUtils.isEmpty(vmStatsById)) {
logger.warn("Got empty result for virtual machine statistics from host: " + host);
}

Some files were not shown because too many files have changed in this diff Show More