diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java index 908bf22964b..2035aab2aca 100644 --- a/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java +++ b/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java @@ -183,7 +183,8 @@ public class HttpsDirectTemplateDownloader extends DirectTemplateDownloaderImpl SSLContext context = getSSLContext(); urlConnection.setSSLSocketFactory(context.getSocketFactory()); urlConnection.connect(); - return QCOW2Utils.getVirtualSize(urlConnection.getInputStream()); + boolean isCompressed = !url.endsWith("qcow2"); + return QCOW2Utils.getVirtualSize(urlObj.openStream(), isCompressed); } catch (IOException e) { throw new CloudRuntimeException(String.format("Cannot obtain qcow2 virtual size due to: %s", e.getMessage()), e); } diff --git a/debian/rules b/debian/rules index 2f160cc5c24..2d07c5e5728 100755 --- a/debian/rules +++ b/debian/rules @@ -135,7 +135,7 @@ override_dh_auto_install: install -D systemvm/dist/* $(DESTDIR)/usr/share/$(PACKAGE)-common/vms/ # We need jasypt for cloud-install-sys-tmplt, so this is a nasty hack to get it into the right place install -D agent/target/dependencies/jasypt-1.9.3.jar $(DESTDIR)/usr/share/$(PACKAGE)-common/lib - install -D utils/target/cloud-utils-$(VERSION)-SHADED.jar $(DESTDIR)/usr/share/$(PACKAGE)-common/lib/$(PACKAGE)-utils.jar + install -D utils/target/cloud-utils-$(VERSION)-bundled.jar $(DESTDIR)/usr/share/$(PACKAGE)-common/lib/$(PACKAGE)-utils.jar # cloudstack-python mkdir -p $(DESTDIR)/usr/share/pyshared diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java index 21d5a205a09..0b38acb5c21 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java @@ -18,6 +18,7 @@ package com.cloud.upgrade.dao; import java.sql.Connection; import java.sql.PreparedStatement; +import java.sql.ResultSet; import java.sql.SQLException; import org.apache.log4j.Logger; @@ -84,6 +85,33 @@ public class DatabaseAccessObject { return columnExists; } + public String generateIndexName(String tableName, String columnName) { + return String.format("i_%s__%s", tableName, columnName); + } + + public boolean indexExists(Connection conn, String tableName, String indexName) { + try (PreparedStatement pstmt = conn.prepareStatement(String.format("SHOW INDEXES FROM %s where Key_name = \"%s\"", tableName, indexName))) { + ResultSet result = pstmt.executeQuery(); + if (result.next()) { + return true; + } + } catch (SQLException e) { + s_logger.debug(String.format("Index %s doesn't exist, ignoring exception:", indexName, e.getMessage())); + } + return false; + } + + public void createIndex(Connection conn, String tableName, String columnName, String indexName) { + String stmt = String.format("CREATE INDEX %s on %s (%s)", indexName, tableName, columnName); + s_logger.debug("Statement: " + stmt); + try (PreparedStatement pstmt = conn.prepareStatement(stmt)) { + pstmt.execute(); + s_logger.debug(String.format("Created index %s", indexName)); + } catch (SQLException e) { + s_logger.warn(String.format("Unable to create index %s", indexName), e); + } + } + protected static void closePreparedStatement(PreparedStatement pstmt, String errorMessage) { try { if (pstmt != null) { @@ -93,5 +121,4 @@ public class DatabaseAccessObject { s_logger.warn(errorMessage, e); } } - } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/DbUpgradeUtils.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/DbUpgradeUtils.java index 02dad6250dc..6b4e1814de0 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/DbUpgradeUtils.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/DbUpgradeUtils.java @@ -23,6 +23,14 @@ public class DbUpgradeUtils { private static DatabaseAccessObject dao = new DatabaseAccessObject(); + public static void addIndexIfNeeded(Connection conn, String tableName, String columnName) { + String indexName = dao.generateIndexName(tableName, columnName); + + if (!dao.indexExists(conn, tableName, indexName)) { + dao.createIndex(conn, tableName, columnName, indexName); + } + } + public static void addForeignKey(Connection conn, String tableName, String tableColumn, String foreignTableName, String foreignColumnName) { dao.addForeignKey(conn, tableName, tableColumn, foreignTableName, foreignColumnName); } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41800to41810.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41800to41810.java index 8eb50666ec8..a58d9965259 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41800to41810.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41800to41810.java @@ -69,6 +69,7 @@ public class Upgrade41800to41810 implements DbUpgrade, DbUpgradeSystemVmTemplate copyGuestOsMappingsToVMware80u1(); addForeignKeyToAutoscaleVmprofiles(conn); mergeDuplicateGuestOSes(); + addIndexes(conn); } private void mergeDuplicateGuestOSes() { @@ -242,4 +243,8 @@ public class Upgrade41800to41810 implements DbUpgrade, DbUpgradeSystemVmTemplate private void addForeignKeyToAutoscaleVmprofiles(Connection conn) { DbUpgradeUtils.addForeignKey(conn, "autoscale_vmprofiles", "user_data_id", "user_data", "id"); } + + private void addIndexes(Connection conn) { + DbUpgradeUtils.addIndexIfNeeded(conn, "cluster_details", "name"); + } } diff --git a/engine/schema/src/test/java/com/cloud/upgrade/dao/DatabaseAccessObjectTest.java b/engine/schema/src/test/java/com/cloud/upgrade/dao/DatabaseAccessObjectTest.java index 2b8b2bd8f1c..7e78c3ec3e9 100644 --- a/engine/schema/src/test/java/com/cloud/upgrade/dao/DatabaseAccessObjectTest.java +++ b/engine/schema/src/test/java/com/cloud/upgrade/dao/DatabaseAccessObjectTest.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.upgrade.dao; +import static org.mockito.Matchers.startsWith; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyString; import static org.mockito.Matchers.contains; @@ -27,9 +28,11 @@ import static org.mockito.Mockito.when; import java.sql.Connection; import java.sql.PreparedStatement; +import java.sql.ResultSet; import java.sql.SQLException; import org.apache.log4j.Logger; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -49,6 +52,9 @@ public class DatabaseAccessObjectTest { @Mock private Logger loggerMock; + @Mock + private ResultSet resultSetMock; + private final DatabaseAccessObject dao = new DatabaseAccessObject(); @Before @@ -83,6 +89,61 @@ public class DatabaseAccessObjectTest { dao.dropKey(conn, tableName, key, isForeignKey); } + @Test + public void generateIndexNameTest() { + String indexName = dao.generateIndexName("mytable","mycolumn"); + Assert.assertEquals( "i_mytable__mycolumn", indexName); + } + + @Test + public void indexExistsFalseTest() throws Exception { + when(resultSetMock.next()).thenReturn(false); + when(connectionMock.prepareStatement(startsWith("SHOW INDEXES FROM"))).thenReturn(preparedStatementMock); + when(preparedStatementMock.executeQuery()).thenReturn(resultSetMock); + + Connection conn = connectionMock; + String tableName = "mytable"; + String indexName = "myindex"; + + Assert.assertFalse(dao.indexExists(conn, tableName, indexName)); + verify(connectionMock, times(1)).prepareStatement(anyString()); + verify(preparedStatementMock, times(1)).executeQuery(); + verify(preparedStatementMock, times(1)).close(); + } + + @Test + public void indexExistsTrueTest() throws Exception { + when(resultSetMock.next()).thenReturn(true); + when(connectionMock.prepareStatement(startsWith("SHOW INDEXES FROM"))).thenReturn(preparedStatementMock); + when(preparedStatementMock.executeQuery()).thenReturn(resultSetMock); + + Connection conn = connectionMock; + String tableName = "mytable"; + String indexName = "myindex"; + + Assert.assertTrue(dao.indexExists(conn, tableName, indexName)); + verify(connectionMock, times(1)).prepareStatement(anyString()); + verify(preparedStatementMock, times(1)).executeQuery(); + verify(preparedStatementMock, times(1)).close(); + } + + @Test + public void createIndexTest() throws Exception { + when(connectionMock.prepareStatement(startsWith("CREATE INDEX"))).thenReturn(preparedStatementMock); + when(preparedStatementMock.execute()).thenReturn(true); + + Connection conn = connectionMock; + String tableName = "mytable"; + String columnName = "mycolumn"; + String indexName = "myindex"; + + dao.createIndex(conn, tableName, columnName, indexName); + verify(connectionMock, times(1)).prepareStatement(anyString()); + verify(preparedStatementMock, times(1)).execute(); + verify(preparedStatementMock, times(1)).close(); + verify(loggerMock, times(1)).debug("Created index myindex"); + } + @Test public void testDropKeyWhenTableNameIsNull() throws Exception { SQLException sqlException = new SQLException(); diff --git a/packaging/centos7/cloud.spec b/packaging/centos7/cloud.spec index e498fed25a1..0fe1a81a6cf 100644 --- a/packaging/centos7/cloud.spec +++ b/packaging/centos7/cloud.spec @@ -302,7 +302,7 @@ ln -sf log4j-cloud.xml ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/management/log4j install python/bindir/cloud-external-ipallocator.py ${RPM_BUILD_ROOT}%{_bindir}/%{name}-external-ipallocator.py install -D client/target/pythonlibs/jasypt-1.9.3.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/lib/jasypt-1.9.3.jar -install -D utils/target/cloud-utils-%{_maventag}-SHADED.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/lib/%{name}-utils.jar +install -D utils/target/cloud-utils-%{_maventag}-bundled.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/lib/%{name}-utils.jar install -D packaging/centos7/cloud-ipallocator.rc ${RPM_BUILD_ROOT}%{_initrddir}/%{name}-ipallocator install -D packaging/centos7/cloud.limits ${RPM_BUILD_ROOT}%{_sysconfdir}/security/limits.d/cloud diff --git a/packaging/centos8/cloud.spec b/packaging/centos8/cloud.spec index 57b16543ba8..ce425d38165 100644 --- a/packaging/centos8/cloud.spec +++ b/packaging/centos8/cloud.spec @@ -284,7 +284,7 @@ ln -sf log4j-cloud.xml ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/management/log4j install python/bindir/cloud-external-ipallocator.py ${RPM_BUILD_ROOT}%{_bindir}/%{name}-external-ipallocator.py install -D client/target/pythonlibs/jasypt-1.9.3.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/lib/jasypt-1.9.3.jar -install -D utils/target/cloud-utils-%{_maventag}-SHADED.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/lib/%{name}-utils.jar +install -D utils/target/cloud-utils-%{_maventag}-bundled.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/lib/%{name}-utils.jar install -D packaging/centos8/cloud-ipallocator.rc ${RPM_BUILD_ROOT}%{_initrddir}/%{name}-ipallocator install -D packaging/centos8/cloud.limits ${RPM_BUILD_ROOT}%{_sysconfdir}/security/limits.d/cloud diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtConnection.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtConnection.java index 0f8031e3aaa..c70a72f399c 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtConnection.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtConnection.java @@ -21,7 +21,6 @@ import java.util.Map; import org.apache.log4j.Logger; import org.libvirt.Connect; -import org.libvirt.Library; import org.libvirt.LibvirtException; import com.cloud.hypervisor.Hypervisor; @@ -45,7 +44,6 @@ public class LibvirtConnection { if (conn == null) { s_logger.info("No existing libvirtd connection found. Opening a new one"); conn = new Connect(hypervisorURI, false); - Library.initEventLoop(); s_logger.debug("Successfully connected to libvirt at: " + hypervisorURI); s_connections.put(hypervisorURI, conn); } else { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckUrlCommand.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckUrlCommand.java index 939d43086f4..935bc8e113b 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckUrlCommand.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckUrlCommand.java @@ -40,6 +40,11 @@ public class LibvirtCheckUrlCommand extends CommandWrapper[] getConfigKeys() { return new ConfigKey[]{UseSystemGuestVlans}; } + + public Network updateNetworkDesignForIPv6IfNeeded(NetworkVO network, Network userSpecified) { + if (userSpecified == null) { + return network; + } + if ((userSpecified.getIp6Cidr() == null && userSpecified.getIp6Gateway() != null) || + (userSpecified.getIp6Cidr() != null && userSpecified.getIp6Gateway() == null)) { + throw new InvalidParameterValueException("ip6gateway and ip6cidr must be specified together."); + } + if (userSpecified.getIp6Cidr() != null) { + network.setIp6Cidr(userSpecified.getIp6Cidr()); + network.setIp6Gateway(userSpecified.getIp6Gateway()); + } + if (userSpecified.getRouterIpv6() != null) { + network.setRouterIpv6(userSpecified.getRouterIpv6()); + } + return network; + } } diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java index e06d7da210d..f647b497f58 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java @@ -347,7 +347,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager // and as such can be easily read. try (InputStream inputStream = td.getS3ObjectInputStream();) { - dnld.setTemplatesize(QCOW2Utils.getVirtualSize(inputStream)); + dnld.setTemplatesize(QCOW2Utils.getVirtualSize(inputStream, false)); } catch (IOException e) { result = "Couldn't read QCOW2 virtual size. Error: " + e.getMessage(); diff --git a/ui/src/views/AutogenView.vue b/ui/src/views/AutogenView.vue index 09a8e9eb285..ba1b87042b7 100644 --- a/ui/src/views/AutogenView.vue +++ b/ui/src/views/AutogenView.vue @@ -968,10 +968,6 @@ export default { console.log('DEBUG - Discarding API response as its `id` does not match the uuid on the browser path') return } - if (this.dataView && apiItemCount > 1) { - console.log('DEBUG - Discarding API response as got more than one item in data view', this.$route.params, this.items) - return - } this.items = json[responseName][objectName] if (!this.items || this.items.length === 0) { diff --git a/utils/pom.xml b/utils/pom.xml index c65de9cb901..39ab96b8bef 100755 --- a/utils/pom.xml +++ b/utils/pom.xml @@ -261,7 +261,7 @@ shade - ${project.artifactId}-${project.version}-SHADED + ${project.artifactId}-${project.version}-bundled false diff --git a/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java b/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java index afd4fe52e0a..32a54722aae 100644 --- a/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java +++ b/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java @@ -25,10 +25,9 @@ import java.io.InputStream; import java.net.MalformedURLException; import java.net.URL; import java.nio.ByteBuffer; -import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import org.apache.commons.compress.compressors.CompressorException; -import org.apache.commons.compress.compressors.CompressorInputStream; import org.apache.commons.compress.compressors.CompressorStreamFactory; import org.apache.log4j.Logger; @@ -57,7 +56,10 @@ public final class QCOW2Utils { * @param inputStream The QCOW2 object in stream format. * @return The virtual size of the QCOW2 object. */ - public static long getVirtualSize(InputStream inputStream) throws IOException { + public static long getVirtualSize(InputStream inputStream, boolean isCompressed) throws IOException { + if (isCompressed) { + return getVirtualSizeFromInputStream(inputStream); + } byte[] bytes = new byte[VIRTUALSIZE_HEADER_LENGTH]; if (inputStream.skip(VIRTUALSIZE_HEADER_LOCATION) != VIRTUALSIZE_HEADER_LOCATION) { @@ -71,20 +73,13 @@ public final class QCOW2Utils { return NumbersUtil.bytesToLong(bytes); } - public static long getVirtualSize(String urlStr) { - InputStream inputStream = null; - + private static long getVirtualSizeFromInputStream(InputStream inputStream) throws IOException { try { - URL url = new URL(urlStr); - BufferedInputStream bufferedInputStream = new BufferedInputStream(url.openStream()); - inputStream = bufferedInputStream; - try { - CompressorInputStream compressorInputStream = new CompressorStreamFactory().createCompressorInputStream(bufferedInputStream); - inputStream = compressorInputStream; + BufferedInputStream bufferedInputStream = new BufferedInputStream(inputStream); + inputStream = new CompressorStreamFactory().createCompressorInputStream(bufferedInputStream); } catch (CompressorException e) { LOGGER.warn(e.getMessage()); - inputStream = bufferedInputStream; } byte[] inputBytes = inputStream.readNBytes(VIRTUALSIZE_HEADER_LOCATION + VIRTUALSIZE_HEADER_LENGTH); @@ -93,7 +88,7 @@ public final class QCOW2Utils { inputMagicBytes.put(inputBytes, 0, MAGIC_HEADER_LENGTH); ByteBuffer qcow2MagicBytes = ByteBuffer.allocate(MAGIC_HEADER_LENGTH); - qcow2MagicBytes.put("QFI".getBytes(Charset.forName("UTF-8"))); + qcow2MagicBytes.put("QFI".getBytes(StandardCharsets.UTF_8)); qcow2MagicBytes.put((byte)0xfb); long virtualSize = 0L; @@ -105,12 +100,6 @@ public final class QCOW2Utils { } return virtualSize; - } catch (MalformedURLException e) { - LOGGER.warn("Failed to validate for qcow2, malformed URL: " + urlStr + ", error: " + e.getMessage()); - throw new IllegalArgumentException("Invalid URL: " + urlStr); - } catch (IOException e) { - LOGGER.warn("Failed to validate for qcow2, error: " + e.getMessage()); - throw new IllegalArgumentException("Failed to connect URL: " + urlStr); } finally { if (inputStream != null) { try { @@ -121,4 +110,17 @@ public final class QCOW2Utils { } } } + + public static long getVirtualSize(String urlStr) { + try { + URL url = new URL(urlStr); + return getVirtualSizeFromInputStream(url.openStream()); + } catch (MalformedURLException e) { + LOGGER.warn("Failed to validate for qcow2, malformed URL: " + urlStr + ", error: " + e.getMessage()); + throw new IllegalArgumentException("Invalid URL: " + urlStr); + } catch (IOException e) { + LOGGER.warn("Failed to validate for qcow2, error: " + e.getMessage()); + throw new IllegalArgumentException("Failed to connect URL: " + urlStr); + } + } } diff --git a/utils/src/test/java/com/cloud/utils/storage/QCOW2UtilsTest.java b/utils/src/test/java/com/cloud/utils/storage/QCOW2UtilsTest.java index 92881a63cda..0595b2f67db 100644 --- a/utils/src/test/java/com/cloud/utils/storage/QCOW2UtilsTest.java +++ b/utils/src/test/java/com/cloud/utils/storage/QCOW2UtilsTest.java @@ -24,7 +24,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; -import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import org.junit.Before; import org.junit.Test; @@ -67,7 +67,7 @@ public class QCOW2UtilsTest { ByteBuffer byteBuffer = ByteBuffer.allocate(72); // Magic - byteBuffer.put("QFI".getBytes(Charset.forName("UTF-8"))); + byteBuffer.put("QFI".getBytes(StandardCharsets.UTF_8)); byteBuffer.put((byte)0xfb); // Version @@ -116,6 +116,6 @@ public class QCOW2UtilsTest { @Test public void getVirtualSizeTest() throws IOException { - assertEquals(virtualSize.longValue(), QCOW2Utils.getVirtualSize(inputStream)); + assertEquals(virtualSize.longValue(), QCOW2Utils.getVirtualSize(inputStream, false)); } }