From 265f4cdc28804d8bde47274c19fd3b06de4ba928 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 12 May 2025 16:20:50 +0530 Subject: [PATCH 1/2] core: support chunked transfer for image files (#10820) --- .../template/HttpTemplateDownloader.java | 99 +++++++++++++------ 1 file changed, 71 insertions(+), 28 deletions(-) diff --git a/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java b/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java index 9b126846827..4c056f256cf 100755 --- a/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java +++ b/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java @@ -27,6 +27,7 @@ import java.io.InputStream; import java.io.RandomAccessFile; import java.net.URI; import java.net.URISyntaxException; +import java.util.Arrays; import java.util.Date; import java.util.List; @@ -80,6 +81,18 @@ public class HttpTemplateDownloader extends ManagedContextRunnable implements Te private ResourceType resourceType = ResourceType.TEMPLATE; private final HttpMethodRetryHandler myretryhandler; private boolean followRedirects = false; + private boolean isChunkedTransfer; + + protected static final List CUSTOM_HEADERS_FOR_CHUNKED_TRANSFER_SIZE = Arrays.asList( + "x-goog-stored-content-length", + "x-goog-meta-size", + "x-amz-meta-size", + "x-amz-meta-content-length", + "x-object-meta-size", + "x-original-content-length", + "x-oss-meta-content-length", + "x-file-size"); + private static final long MIN_FORMAT_VERIFICATION_SIZE = 1024 * 1024; public HttpTemplateDownloader(StorageLayer storageLayer, String downloadUrl, String toDir, DownloadCompleteCallback callback, long maxTemplateSizeInBytes, String user, String password, Proxy proxy, ResourceType resourceType) { @@ -205,13 +218,11 @@ public class HttpTemplateDownloader extends ManagedContextRunnable implements Te RandomAccessFile out = new RandomAccessFile(file, "rw"); ) { out.seek(localFileSize); - - logger.info("Starting download from " + downloadUrl + " to " + toFile + " remoteSize=" + toHumanReadableSize(remoteSize) + " , max size=" + toHumanReadableSize(maxTemplateSizeInBytes)); - - if (copyBytes(file, in, out)) return 0; - + logger.info("Starting download from {} to {} remoteSize={} , max size={}",downloadUrl, toFile, + toHumanReadableSize(remoteSize), toHumanReadableSize(maxTemplateSizeInBytes)); + boolean eof = copyBytes(file, in, out); Date finish = new Date(); - checkDowloadCompletion(); + checkDownloadCompletion(eof); downloadTime += finish.getTime() - start.getTime(); } finally { /* in.close() and out.close() */ } return totalBytes; @@ -237,28 +248,32 @@ public class HttpTemplateDownloader extends ManagedContextRunnable implements Te } private boolean copyBytes(File file, InputStream in, RandomAccessFile out) throws IOException { - int bytes; - byte[] block = new byte[CHUNK_SIZE]; + byte[] buffer = new byte[CHUNK_SIZE]; long offset = 0; - boolean done = false; VerifyFormat verifyFormat = new VerifyFormat(file); status = Status.IN_PROGRESS; - while (!done && status != Status.ABORTED && offset <= remoteSize) { - if ((bytes = in.read(block, 0, CHUNK_SIZE)) > -1) { - offset = writeBlock(bytes, out, block, offset); - if (!ResourceType.SNAPSHOT.equals(resourceType) && - !verifyFormat.isVerifiedFormat() && - (offset >= 1048576 || offset >= remoteSize)) { //let's check format after we get 1MB or full file - verifyFormat.invoke(); - } - } else { - done = true; + while (status != Status.ABORTED) { + int bytesRead = in.read(buffer, 0, CHUNK_SIZE); + if (bytesRead == -1) { + logger.debug("Reached EOF on input stream"); + break; + } + offset = writeBlock(bytesRead, out, buffer, offset); + if (!ResourceType.SNAPSHOT.equals(resourceType) + && !verifyFormat.isVerifiedFormat() + && (offset >= MIN_FORMAT_VERIFICATION_SIZE || offset >= remoteSize)) { + verifyFormat.invoke(); + } + if (offset >= remoteSize) { + logger.debug("Reached expected remote size limit: {} bytes", remoteSize); + break; } } out.getFD().sync(); - return false; + return !Status.ABORTED.equals(status); } + private long writeBlock(int bytes, RandomAccessFile out, byte[] block, long offset) throws IOException { out.write(block, 0, bytes); offset += bytes; @@ -267,11 +282,13 @@ public class HttpTemplateDownloader extends ManagedContextRunnable implements Te return offset; } - private void checkDowloadCompletion() { + private void checkDownloadCompletion(boolean eof) { String downloaded = "(incomplete download)"; - if (totalBytes >= remoteSize) { + if (eof && ((totalBytes >= remoteSize) || (isChunkedTransfer && remoteSize == maxTemplateSizeInBytes))) { status = Status.DOWNLOAD_FINISHED; - downloaded = "(download complete remote=" + toHumanReadableSize(remoteSize) + " bytes)"; + downloaded = "(download complete remote=" + + (remoteSize == maxTemplateSizeInBytes ? toHumanReadableSize(remoteSize) : "unknown") + + " bytes)"; } errorString = "Downloaded " + toHumanReadableSize(totalBytes) + " bytes " + downloaded; } @@ -293,18 +310,42 @@ public class HttpTemplateDownloader extends ManagedContextRunnable implements Te } } + protected long getRemoteSizeForChunkedTransfer() { + for (String headerKey : CUSTOM_HEADERS_FOR_CHUNKED_TRANSFER_SIZE) { + Header header = request.getResponseHeader(headerKey); + if (header == null) { + continue; + } + try { + return Long.parseLong(header.getValue()); + } catch (NumberFormatException ignored) {} + } + Header contentRangeHeader = request.getResponseHeader("Content-Range"); + if (contentRangeHeader != null) { + String contentRange = contentRangeHeader.getValue(); + if (contentRange != null && contentRange.contains("/")) { + String totalSize = contentRange.substring(contentRange.indexOf('/') + 1).trim(); + return Long.parseLong(totalSize); + } + } + return 0; + } + private boolean tryAndGetRemoteSize() { Header contentLengthHeader = request.getResponseHeader("content-length"); - boolean chunked = false; + isChunkedTransfer = false; long reportedRemoteSize = 0; if (contentLengthHeader == null) { Header chunkedHeader = request.getResponseHeader("Transfer-Encoding"); - if (chunkedHeader == null || !"chunked".equalsIgnoreCase(chunkedHeader.getValue())) { + if (chunkedHeader != null && "chunked".equalsIgnoreCase(chunkedHeader.getValue())) { + isChunkedTransfer = true; + reportedRemoteSize = getRemoteSizeForChunkedTransfer(); + logger.debug("{} is using chunked transfer encoding, possible remote size: {}", downloadUrl, + reportedRemoteSize); + } else { status = Status.UNRECOVERABLE_ERROR; errorString = " Failed to receive length of download "; return false; - } else if ("chunked".equalsIgnoreCase(chunkedHeader.getValue())) { - chunked = true; } } else { reportedRemoteSize = Long.parseLong(contentLengthHeader.getValue()); @@ -316,9 +357,11 @@ public class HttpTemplateDownloader extends ManagedContextRunnable implements Te return false; } } - if (remoteSize == 0) { remoteSize = reportedRemoteSize; + if (remoteSize != 0) { + logger.debug("Remote size for {} found to be {}", downloadUrl, toHumanReadableSize(remoteSize)); + } } return true; } From 011fced91ed4cdd6c839abf977fd2b0c8c27a544 Mon Sep 17 00:00:00 2001 From: Phsm Qwerty Date: Mon, 12 May 2025 16:16:54 +0200 Subject: [PATCH 2/2] ehancement: add password to configdrive vendor_data.json (#10061) --- .../configdrive/ConfigDriveBuilder.java | 43 ++++++++++++++++++- .../configdrive/ConfigDriveBuilderTest.java | 20 ++++----- 2 files changed, 51 insertions(+), 12 deletions(-) diff --git a/engine/storage/configdrive/src/main/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilder.java b/engine/storage/configdrive/src/main/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilder.java index 58cc341a87b..d57afbb0a23 100644 --- a/engine/storage/configdrive/src/main/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilder.java +++ b/engine/storage/configdrive/src/main/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilder.java @@ -126,7 +126,26 @@ public class ConfigDriveBuilder { File openStackFolder = new File(tempDirName + ConfigDrive.openStackConfigDriveName); - writeVendorEmptyJsonFile(openStackFolder); + /* + Try to find VM password in the vmData. + If it is found, then write it into vendor-data.json + */ + String vmPassword = ""; + for (String[] item : vmData) { + String dataType = item[CONFIGDATA_DIR]; + String fileName = item[CONFIGDATA_FILE]; + String content = item[CONFIGDATA_CONTENT]; + if (PASSWORD_FILE.equals(fileName)) { + vmPassword = content; + break; + } + } + if (vmPassword.equals("")) { + writeVendorDataJsonFile(openStackFolder); + } else { + writeVendorDataJsonFile(openStackFolder, vmPassword); + } + writeNetworkData(nics, supportedServices, openStackFolder); for (NicProfile nic: nics) { if (supportedServices.get(nic.getId()).contains(Network.Service.UserData)) { @@ -253,7 +272,7 @@ public class ConfigDriveBuilder { * * If the folder does not exist, and we cannot create it, we throw a {@link CloudRuntimeException}. */ - static void writeVendorEmptyJsonFile(File openStackFolder) { + static void writeVendorDataJsonFile(File openStackFolder) { if (openStackFolder.exists() || openStackFolder.mkdirs()) { writeFile(openStackFolder, "vendor_data.json", "{}"); } else { @@ -261,6 +280,26 @@ public class ConfigDriveBuilder { } } + /** + * Writes vendor data containing Cloudstack-generated password into vendor-data.json + * + * If the folder does not exist, and we cannot create it, we throw a {@link CloudRuntimeException}. + */ + static void writeVendorDataJsonFile(File openStackFolder, String password) { + if (openStackFolder.exists() || openStackFolder.mkdirs()) { + writeFile( + openStackFolder, + "vendor_data.json", + String.format( + "{\"cloud-init\": \"#cloud-config\\npassword: %s\\nchpasswd:\\n expire: False\"}", + password + ) + ); + } else { + throw new CloudRuntimeException("Failed to create folder " + openStackFolder); + } + } + /** * Creates the {@link JsonObject} with VM's metadata. The vmData is a list of arrays; we expect this list to have the following entries: *
    diff --git a/engine/storage/configdrive/src/test/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilderTest.java b/engine/storage/configdrive/src/test/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilderTest.java index 3effdb5ba21..c04ff0a1601 100644 --- a/engine/storage/configdrive/src/test/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilderTest.java +++ b/engine/storage/configdrive/src/test/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilderTest.java @@ -134,7 +134,7 @@ public class ConfigDriveBuilderTest { @Test(expected = CloudRuntimeException.class) public void buildConfigDriveTestIoException() { try (MockedStatic configDriveBuilderMocked = Mockito.mockStatic(ConfigDriveBuilder.class)) { - configDriveBuilderMocked.when(() -> ConfigDriveBuilder.writeVendorEmptyJsonFile(nullable(File.class))).thenThrow(CloudRuntimeException.class); + configDriveBuilderMocked.when(() -> ConfigDriveBuilder.writeVendorDataJsonFile(nullable(File.class))).thenThrow(CloudRuntimeException.class); Mockito.when(ConfigDriveBuilder.buildConfigDrive(null, new ArrayList<>(), "teste", "C:", null, supportedServices)).thenCallRealMethod(); ConfigDriveBuilder.buildConfigDrive(null, new ArrayList<>(), "teste", "C:", null, supportedServices); } @@ -144,7 +144,7 @@ public class ConfigDriveBuilderTest { public void buildConfigDriveTest() { try (MockedStatic configDriveBuilderMocked = Mockito.mockStatic(ConfigDriveBuilder.class)) { - configDriveBuilderMocked.when(() -> ConfigDriveBuilder.writeVendorEmptyJsonFile(Mockito.any(File.class))).then(invocationOnMock -> null); + configDriveBuilderMocked.when(() -> ConfigDriveBuilder.writeVendorDataJsonFile(Mockito.any(File.class))).then(invocationOnMock -> null); configDriveBuilderMocked.when(() -> ConfigDriveBuilder.writeVmMetadata(Mockito.anyList(), Mockito.anyString(), Mockito.any(File.class), anyMap())).then(invocationOnMock -> null); @@ -163,7 +163,7 @@ public class ConfigDriveBuilderTest { Assert.assertEquals("mockIsoDataBase64", returnedIsoData); configDriveBuilderMocked.verify(() -> { - ConfigDriveBuilder.writeVendorEmptyJsonFile(Mockito.any(File.class)); + ConfigDriveBuilder.writeVendorDataJsonFile(Mockito.any(File.class)); ConfigDriveBuilder.writeVmMetadata(Mockito.anyList(), Mockito.anyString(), Mockito.any(File.class), anyMap()); ConfigDriveBuilder.linkUserData(Mockito.anyString()); ConfigDriveBuilder.generateAndRetrieveIsoAsBase64Iso(Mockito.anyString(), Mockito.anyString(), Mockito.anyString()); @@ -172,23 +172,23 @@ public class ConfigDriveBuilderTest { } @Test(expected = CloudRuntimeException.class) - public void writeVendorEmptyJsonFileTestCannotCreateOpenStackFolder() { + public void writeVendorDataJsonFileTestCannotCreateOpenStackFolder() { File folderFileMock = Mockito.mock(File.class); Mockito.doReturn(false).when(folderFileMock).mkdirs(); - ConfigDriveBuilder.writeVendorEmptyJsonFile(folderFileMock); + ConfigDriveBuilder.writeVendorDataJsonFile(folderFileMock); } @Test(expected = CloudRuntimeException.class) - public void writeVendorEmptyJsonFileTest() { + public void writeVendorDataJsonFileTest() { File folderFileMock = Mockito.mock(File.class); Mockito.doReturn(false).when(folderFileMock).mkdirs(); - ConfigDriveBuilder.writeVendorEmptyJsonFile(folderFileMock); + ConfigDriveBuilder.writeVendorDataJsonFile(folderFileMock); } @Test - public void writeVendorEmptyJsonFileTestCreatingFolder() { + public void writeVendorDataJsonFileTestCreatingFolder() { try (MockedStatic configDriveBuilderMocked = Mockito.mockStatic(ConfigDriveBuilder.class)) { File folderFileMock = Mockito.mock(File.class); @@ -196,9 +196,9 @@ public class ConfigDriveBuilderTest { Mockito.doReturn(true).when(folderFileMock).mkdirs(); //force execution of real method - configDriveBuilderMocked.when(() -> ConfigDriveBuilder.writeVendorEmptyJsonFile(folderFileMock)).thenCallRealMethod(); + configDriveBuilderMocked.when(() -> ConfigDriveBuilder.writeVendorDataJsonFile(folderFileMock)).thenCallRealMethod(); - ConfigDriveBuilder.writeVendorEmptyJsonFile(folderFileMock); + ConfigDriveBuilder.writeVendorDataJsonFile(folderFileMock); Mockito.verify(folderFileMock).exists(); Mockito.verify(folderFileMock).mkdirs();