etan39
/
cloudstack
mirror of https://github.com/apache/cloudstack.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' of https://github.com/apache/cloudstack into nsx-integration

This commit is contained in:
Pearl Dsilva 2023-09-07 14:03:24 -04:00
parent d378130782 f6b2a58727
commit 271ed0f33a
19 changed files with 173 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java
View File

@ -183,7 +183,8 @@ public class HttpsDirectTemplateDownloader extends DirectTemplateDownloaderImpl
SSLContext context = getSSLContext();
urlConnection.setSSLSocketFactory(context.getSocketFactory());
urlConnection.connect();
return QCOW2Utils.getVirtualSize(urlConnection.getInputStream());
boolean isCompressed = !url.endsWith("qcow2");
return QCOW2Utils.getVirtualSize(urlObj.openStream(), isCompressed);
} catch (IOException e) {
throw new CloudRuntimeException(String.format("Cannot obtain qcow2 virtual size due to: %s", e.getMessage()), e);
}

2
debian/rules vendored
View File

@ -135,7 +135,7 @@ override_dh_auto_install:
install -D systemvm/dist/* $(DESTDIR)/usr/share/$(PACKAGE)-common/vms/
# We need jasypt for cloud-install-sys-tmplt, so this is a nasty hack to get it into the right place
install -D agent/target/dependencies/jasypt-1.9.3.jar $(DESTDIR)/usr/share/$(PACKAGE)-common/lib
install -D utils/target/cloud-utils-$(VERSION)-SHADED.jar $(DESTDIR)/usr/share/$(PACKAGE)-common/lib/$(PACKAGE)-utils.jar
install -D utils/target/cloud-utils-$(VERSION)-bundled.jar $(DESTDIR)/usr/share/$(PACKAGE)-common/lib/$(PACKAGE)-utils.jar
# cloudstack-python
mkdir -p $(DESTDIR)/usr/share/pyshared

29
engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java
View File

@ -18,6 +18,7 @@ package com.cloud.upgrade.dao;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import org.apache.log4j.Logger;
@ -84,6 +85,33 @@ public class DatabaseAccessObject {
return columnExists;
}
public String generateIndexName(String tableName, String columnName) {
return String.format("i_%s__%s", tableName, columnName);
}
public boolean indexExists(Connection conn, String tableName, String indexName) {
try (PreparedStatement pstmt = conn.prepareStatement(String.format("SHOW INDEXES FROM %s where Key_name = \"%s\"", tableName, indexName))) {
ResultSet result = pstmt.executeQuery();
if (result.next()) {
return true;
}
} catch (SQLException e) {
s_logger.debug(String.format("Index %s doesn't exist, ignoring exception:", indexName, e.getMessage()));
}
return false;
}
public void createIndex(Connection conn, String tableName, String columnName, String indexName) {
String stmt = String.format("CREATE INDEX %s on %s (%s)", indexName, tableName, columnName);
s_logger.debug("Statement: " + stmt);
try (PreparedStatement pstmt = conn.prepareStatement(stmt)) {
pstmt.execute();
s_logger.debug(String.format("Created index %s", indexName));
} catch (SQLException e) {
s_logger.warn(String.format("Unable to create index %s", indexName), e);
}
}
protected static void closePreparedStatement(PreparedStatement pstmt, String errorMessage) {
try {
if (pstmt != null) {
@ -93,5 +121,4 @@ public class DatabaseAccessObject {
s_logger.warn(errorMessage, e);
}
}
}

8
engine/schema/src/main/java/com/cloud/upgrade/dao/DbUpgradeUtils.java
View File

@ -23,6 +23,14 @@ public class DbUpgradeUtils {
private static DatabaseAccessObject dao = new DatabaseAccessObject();
public static void addIndexIfNeeded(Connection conn, String tableName, String columnName) {
String indexName = dao.generateIndexName(tableName, columnName);
if (!dao.indexExists(conn, tableName, indexName)) {
dao.createIndex(conn, tableName, columnName, indexName);
}
}
public static void addForeignKey(Connection conn, String tableName, String tableColumn, String foreignTableName, String foreignColumnName) {
dao.addForeignKey(conn, tableName, tableColumn, foreignTableName, foreignColumnName);
}

5
engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41800to41810.java
View File

@ -69,6 +69,7 @@ public class Upgrade41800to41810 implements DbUpgrade, DbUpgradeSystemVmTemplate
copyGuestOsMappingsToVMware80u1();
addForeignKeyToAutoscaleVmprofiles(conn);
mergeDuplicateGuestOSes();
addIndexes(conn);
}
private void mergeDuplicateGuestOSes() {
@ -242,4 +243,8 @@ public class Upgrade41800to41810 implements DbUpgrade, DbUpgradeSystemVmTemplate
private void addForeignKeyToAutoscaleVmprofiles(Connection conn) {
DbUpgradeUtils.addForeignKey(conn, "autoscale_vmprofiles", "user_data_id", "user_data", "id");
}
private void addIndexes(Connection conn) {
DbUpgradeUtils.addIndexIfNeeded(conn, "cluster_details", "name");
}
}

61
engine/schema/src/test/java/com/cloud/upgrade/dao/DatabaseAccessObjectTest.java
View File

@ -16,6 +16,7 @@
// under the License.
package com.cloud.upgrade.dao;
import static org.mockito.Matchers.startsWith;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyString;
import static org.mockito.Matchers.contains;
@ -27,9 +28,11 @@ import static org.mockito.Mockito.when;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import org.apache.log4j.Logger;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
@ -49,6 +52,9 @@ public class DatabaseAccessObjectTest {
@Mock
private Logger loggerMock;
@Mock
private ResultSet resultSetMock;
private final DatabaseAccessObject dao = new DatabaseAccessObject();
@Before
@ -83,6 +89,61 @@ public class DatabaseAccessObjectTest {
dao.dropKey(conn, tableName, key, isForeignKey);
}
@Test
public void generateIndexNameTest() {
String indexName = dao.generateIndexName("mytable","mycolumn");
Assert.assertEquals( "i_mytable__mycolumn", indexName);
}
@Test
public void indexExistsFalseTest() throws Exception {
when(resultSetMock.next()).thenReturn(false);
when(connectionMock.prepareStatement(startsWith("SHOW INDEXES FROM"))).thenReturn(preparedStatementMock);
when(preparedStatementMock.executeQuery()).thenReturn(resultSetMock);
Connection conn = connectionMock;
String tableName = "mytable";
String indexName = "myindex";
Assert.assertFalse(dao.indexExists(conn, tableName, indexName));
verify(connectionMock, times(1)).prepareStatement(anyString());
verify(preparedStatementMock, times(1)).executeQuery();
verify(preparedStatementMock, times(1)).close();
}
@Test
public void indexExistsTrueTest() throws Exception {
when(resultSetMock.next()).thenReturn(true);
when(connectionMock.prepareStatement(startsWith("SHOW INDEXES FROM"))).thenReturn(preparedStatementMock);
when(preparedStatementMock.executeQuery()).thenReturn(resultSetMock);
Connection conn = connectionMock;
String tableName = "mytable";
String indexName = "myindex";
Assert.assertTrue(dao.indexExists(conn, tableName, indexName));
verify(connectionMock, times(1)).prepareStatement(anyString());
verify(preparedStatementMock, times(1)).executeQuery();
verify(preparedStatementMock, times(1)).close();
}
@Test
public void createIndexTest() throws Exception {
when(connectionMock.prepareStatement(startsWith("CREATE INDEX"))).thenReturn(preparedStatementMock);
when(preparedStatementMock.execute()).thenReturn(true);
Connection conn = connectionMock;
String tableName = "mytable";
String columnName = "mycolumn";
String indexName = "myindex";
dao.createIndex(conn, tableName, columnName, indexName);
verify(connectionMock, times(1)).prepareStatement(anyString());
verify(preparedStatementMock, times(1)).execute();
verify(preparedStatementMock, times(1)).close();
verify(loggerMock, times(1)).debug("Created index myindex");
}
@Test
public void testDropKeyWhenTableNameIsNull() throws Exception {
SQLException sqlException = new SQLException();

2
packaging/centos7/cloud.spec
View File

@ -302,7 +302,7 @@ ln -sf log4j-cloud.xml ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/management/log4j
install python/bindir/cloud-external-ipallocator.py ${RPM_BUILD_ROOT}%{_bindir}/%{name}-external-ipallocator.py
install -D client/target/pythonlibs/jasypt-1.9.3.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/lib/jasypt-1.9.3.jar
install -D utils/target/cloud-utils-%{_maventag}-SHADED.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/lib/%{name}-utils.jar
install -D utils/target/cloud-utils-%{_maventag}-bundled.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/lib/%{name}-utils.jar
install -D packaging/centos7/cloud-ipallocator.rc ${RPM_BUILD_ROOT}%{_initrddir}/%{name}-ipallocator
install -D packaging/centos7/cloud.limits ${RPM_BUILD_ROOT}%{_sysconfdir}/security/limits.d/cloud

2
packaging/centos8/cloud.spec
View File

@ -284,7 +284,7 @@ ln -sf log4j-cloud.xml ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/management/log4j
install python/bindir/cloud-external-ipallocator.py ${RPM_BUILD_ROOT}%{_bindir}/%{name}-external-ipallocator.py
install -D client/target/pythonlibs/jasypt-1.9.3.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/lib/jasypt-1.9.3.jar
install -D utils/target/cloud-utils-%{_maventag}-SHADED.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/lib/%{name}-utils.jar
install -D utils/target/cloud-utils-%{_maventag}-bundled.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/lib/%{name}-utils.jar
install -D packaging/centos8/cloud-ipallocator.rc ${RPM_BUILD_ROOT}%{_initrddir}/%{name}-ipallocator
install -D packaging/centos8/cloud.limits ${RPM_BUILD_ROOT}%{_sysconfdir}/security/limits.d/cloud

2
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtConnection.java
View File

@ -21,7 +21,6 @@ import java.util.Map;
import org.apache.log4j.Logger;
import org.libvirt.Connect;
import org.libvirt.Library;
import org.libvirt.LibvirtException;
import com.cloud.hypervisor.Hypervisor;
@ -45,7 +44,6 @@ public class LibvirtConnection {
if (conn == null) {
s_logger.info("No existing libvirtd connection found. Opening a new one");
conn = new Connect(hypervisorURI, false);
Library.initEventLoop();
s_logger.debug("Successfully connected to libvirt at: " + hypervisorURI);
s_connections.put(hypervisorURI, conn);
} else {

5
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckUrlCommand.java
View File

@ -40,6 +40,11 @@ public class LibvirtCheckUrlCommand extends CommandWrapper<CheckUrlCommand, Chec
boolean checkResult = DirectDownloadHelper.checkUrlExistence(url);
if (checkResult) {
remoteSize = DirectDownloadHelper.getFileSize(url, cmd.getFormat());
if (remoteSize == null || remoteSize < 0) {
s_logger.error(String.format("Couldn't properly retrieve the remote size of the template on " +
"url %s, obtained size = %s", url, remoteSize));
return new CheckUrlAnswer(false, remoteSize);
}
}
return new CheckUrlAnswer(checkResult, remoteSize);
}

5
plugins/network-elements/vxlan/src/main/java/com/cloud/network/guru/VxlanGuestNetworkGuru.java
View File

@ -69,19 +69,16 @@ public class VxlanGuestNetworkGuru extends GuestNetworkGuru {
@Override
public Network design(NetworkOffering offering, DeploymentPlan plan, Network userSpecified, Account owner) {
NetworkVO network = (NetworkVO)super.design(offering, plan, userSpecified, owner);
if (network == null) {
return null;
}
if (offering.getGuestType() == GuestType.L2 && network.getBroadcastUri() != null) {
String vxlan = BroadcastDomainType.getValue(network.getBroadcastUri());
network.setBroadcastUri(BroadcastDomainType.Vxlan.toUri(vxlan));
}
network.setBroadcastDomainType(BroadcastDomainType.Vxlan);
return network;
return updateNetworkDesignForIPv6IfNeeded(network, userSpecified);
}
@Override

19
plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClientImpl.java
View File

@ -738,11 +738,20 @@ public class ScaleIOGatewayClientImpl implements ScaleIOGatewayClient {
try {
unmapVolumeFromAllSdcs(volumeId);
} catch (Exception ignored) {}
Boolean removeVolumeStatus = post(
"/instances/Volume::" + volumeId + "/action/removeVolume",
"{\"removeMode\":\"ONLY_ME\"}", Boolean.class);
if (removeVolumeStatus != null) {
return removeVolumeStatus;
try {
Boolean removeVolumeStatus = post(
"/instances/Volume::" + volumeId + "/action/removeVolume",
"{\"removeMode\":\"ONLY_ME\"}", Boolean.class);
if (removeVolumeStatus != null) {
return removeVolumeStatus;
}
} catch (Exception ex) {
if (ex instanceof ServerApiException && ex.getMessage().contains("Could not find the volume")) {
LOG.warn(String.format("API says deleting volume %s does not exist, handling gracefully", volumeId));
return true;
}
throw ex;
}
return false;
}

18
server/src/main/java/com/cloud/network/guru/ExternalGuestNetworkGuru.java
View File

@ -36,7 +36,6 @@ import com.cloud.event.EventTypes;
import com.cloud.event.EventVO;
import com.cloud.exception.InsufficientAddressCapacityException;
import com.cloud.exception.InsufficientVirtualNetworkCapacityException;
import com.cloud.exception.InvalidParameterValueException;
import com.cloud.network.IpAddressManager;
import com.cloud.network.Network;
import com.cloud.network.Network.GuestType;
@ -124,22 +123,7 @@ public class ExternalGuestNetworkGuru extends GuestNetworkGuru {
/* In order to revert userSpecified network setup */
config.setState(State.Allocated);
}
if (userSpecified == null) {
return config;
}
if ((userSpecified.getIp6Cidr() == null && userSpecified.getIp6Gateway() != null) ||
(userSpecified.getIp6Cidr() != null && userSpecified.getIp6Gateway() == null)) {
throw new InvalidParameterValueException("ip6gateway and ip6cidr must be specified together.");
}
if (userSpecified.getIp6Cidr() != null) {
config.setIp6Cidr(userSpecified.getIp6Cidr());
config.setIp6Gateway(userSpecified.getIp6Gateway());
}
if (userSpecified.getRouterIpv6() != null) {
config.setRouterIpv6(userSpecified.getRouterIpv6());
}
return config;
return updateNetworkDesignForIPv6IfNeeded(config, userSpecified);
}
@Override

18
server/src/main/java/com/cloud/network/guru/GuestNetworkGuru.java
View File

@ -545,4 +545,22 @@ public abstract class GuestNetworkGuru extends AdapterBase implements NetworkGur
public ConfigKey<?>[] getConfigKeys() {
return new ConfigKey<?>[]{UseSystemGuestVlans};
}
public Network updateNetworkDesignForIPv6IfNeeded(NetworkVO network, Network userSpecified) {
if (userSpecified == null) {
return network;
}
if ((userSpecified.getIp6Cidr() == null && userSpecified.getIp6Gateway() != null) ||
(userSpecified.getIp6Cidr() != null && userSpecified.getIp6Gateway() == null)) {
throw new InvalidParameterValueException("ip6gateway and ip6cidr must be specified together.");
}
if (userSpecified.getIp6Cidr() != null) {
network.setIp6Cidr(userSpecified.getIp6Cidr());
network.setIp6Gateway(userSpecified.getIp6Gateway());
}
if (userSpecified.getRouterIpv6() != null) {
network.setRouterIpv6(userSpecified.getRouterIpv6());
}
return network;
}
}

2
services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java
View File

@ -347,7 +347,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
// and as such can be easily read.
try (InputStream inputStream = td.getS3ObjectInputStream();) {
dnld.setTemplatesize(QCOW2Utils.getVirtualSize(inputStream));
dnld.setTemplatesize(QCOW2Utils.getVirtualSize(inputStream, false));
}
catch (IOException e) {
result = "Couldn't read QCOW2 virtual size. Error: " + e.getMessage();

4
ui/src/views/AutogenView.vue
View File

@ -968,10 +968,6 @@ export default {
console.log('DEBUG - Discarding API response as its `id` does not match the uuid on the browser path')
return
}
if (this.dataView && apiItemCount > 1) {
console.log('DEBUG - Discarding API response as got more than one item in data view', this.$route.params, this.items)
return
}
this.items = json[responseName][objectName]
if (!this.items || this.items.length === 0) {

2
utils/pom.xml
View File

@ -261,7 +261,7 @@
<goal>shade</goal>
</goals>
<configuration>
<finalName>${project.artifactId}-${project.version}-SHADED</finalName>
<finalName>${project.artifactId}-${project.version}-bundled</finalName>
<createDependencyReducedPom>false</createDependencyReducedPom>
<artifactSet>
<includes>

42
utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java
View File

@ -25,10 +25,9 @@ import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import org.apache.commons.compress.compressors.CompressorException;
import org.apache.commons.compress.compressors.CompressorInputStream;
import org.apache.commons.compress.compressors.CompressorStreamFactory;
import org.apache.log4j.Logger;
@ -57,7 +56,10 @@ public final class QCOW2Utils {
* @param inputStream The QCOW2 object in stream format.
* @return The virtual size of the QCOW2 object.
*/
public static long getVirtualSize(InputStream inputStream) throws IOException {
public static long getVirtualSize(InputStream inputStream, boolean isCompressed) throws IOException {
if (isCompressed) {
return getVirtualSizeFromInputStream(inputStream);
}
byte[] bytes = new byte[VIRTUALSIZE_HEADER_LENGTH];
if (inputStream.skip(VIRTUALSIZE_HEADER_LOCATION) != VIRTUALSIZE_HEADER_LOCATION) {
@ -71,20 +73,13 @@ public final class QCOW2Utils {
return NumbersUtil.bytesToLong(bytes);
}
public static long getVirtualSize(String urlStr) {
InputStream inputStream = null;
private static long getVirtualSizeFromInputStream(InputStream inputStream) throws IOException {
try {
URL url = new URL(urlStr);
BufferedInputStream bufferedInputStream = new BufferedInputStream(url.openStream());
inputStream = bufferedInputStream;
try {
CompressorInputStream compressorInputStream = new CompressorStreamFactory().createCompressorInputStream(bufferedInputStream);
inputStream = compressorInputStream;
BufferedInputStream bufferedInputStream = new BufferedInputStream(inputStream);
inputStream = new CompressorStreamFactory().createCompressorInputStream(bufferedInputStream);
} catch (CompressorException e) {
LOGGER.warn(e.getMessage());
inputStream = bufferedInputStream;
}
byte[] inputBytes = inputStream.readNBytes(VIRTUALSIZE_HEADER_LOCATION + VIRTUALSIZE_HEADER_LENGTH);
@ -93,7 +88,7 @@ public final class QCOW2Utils {
inputMagicBytes.put(inputBytes, 0, MAGIC_HEADER_LENGTH);
ByteBuffer qcow2MagicBytes = ByteBuffer.allocate(MAGIC_HEADER_LENGTH);
qcow2MagicBytes.put("QFI".getBytes(Charset.forName("UTF-8")));
qcow2MagicBytes.put("QFI".getBytes(StandardCharsets.UTF_8));
qcow2MagicBytes.put((byte)0xfb);
long virtualSize = 0L;
@ -105,12 +100,6 @@ public final class QCOW2Utils {
}
return virtualSize;
} catch (MalformedURLException e) {
LOGGER.warn("Failed to validate for qcow2, malformed URL: " + urlStr + ", error: " + e.getMessage());
throw new IllegalArgumentException("Invalid URL: " + urlStr);
} catch (IOException e) {
LOGGER.warn("Failed to validate for qcow2, error: " + e.getMessage());
throw new IllegalArgumentException("Failed to connect URL: " + urlStr);
} finally {
if (inputStream != null) {
try {
@ -121,4 +110,17 @@ public final class QCOW2Utils {
}
}
}
public static long getVirtualSize(String urlStr) {
try {
URL url = new URL(urlStr);
return getVirtualSizeFromInputStream(url.openStream());
} catch (MalformedURLException e) {
LOGGER.warn("Failed to validate for qcow2, malformed URL: " + urlStr + ", error: " + e.getMessage());
throw new IllegalArgumentException("Invalid URL: " + urlStr);
} catch (IOException e) {
LOGGER.warn("Failed to validate for qcow2, error: " + e.getMessage());
throw new IllegalArgumentException("Failed to connect URL: " + urlStr);
}
}
}

6
utils/src/test/java/com/cloud/utils/storage/QCOW2UtilsTest.java
View File

@ -24,7 +24,7 @@ import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import org.junit.Before;
import org.junit.Test;
@ -67,7 +67,7 @@ public class QCOW2UtilsTest {
ByteBuffer byteBuffer = ByteBuffer.allocate(72);
// Magic
byteBuffer.put("QFI".getBytes(Charset.forName("UTF-8")));
byteBuffer.put("QFI".getBytes(StandardCharsets.UTF_8));
byteBuffer.put((byte)0xfb);
// Version
@ -116,6 +116,6 @@ public class QCOW2UtilsTest {
@Test
public void getVirtualSizeTest() throws IOException {
assertEquals(virtualSize.longValue(), QCOW2Utils.getVirtualSize(inputStream));
assertEquals(virtualSize.longValue(), QCOW2Utils.getVirtualSize(inputStream, false));
}
}