From b926c7474d8208c46f7575487843d1dea5fbfcb4 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 29 Jan 2026 17:51:19 +0530 Subject: [PATCH] server changes Signed-off-by: Abhishek Kumar --- .../GetImageTransferProgressAnswer.java | 8 +-- ...etImageTransferProgressCommandWrapper.java | 29 +++----- .../backup/IncrementalBackupServiceImpl.java | 72 +++++++++++++++---- systemvm/debian/opt/cloud/bin/image_server.py | 8 --- 4 files changed, 72 insertions(+), 45 deletions(-) diff --git a/core/src/main/java/org/apache/cloudstack/backup/GetImageTransferProgressAnswer.java b/core/src/main/java/org/apache/cloudstack/backup/GetImageTransferProgressAnswer.java index cc031abd21a..5b5713f4683 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/GetImageTransferProgressAnswer.java +++ b/core/src/main/java/org/apache/cloudstack/backup/GetImageTransferProgressAnswer.java @@ -22,7 +22,7 @@ import java.util.Map; import com.cloud.agent.api.Answer; public class GetImageTransferProgressAnswer extends Answer { - private Map progressMap; // transferId -> progress percentage (0-100) + private Map progressMap; // transferId -> progress percentage (0-100) public GetImageTransferProgressAnswer() { } @@ -32,16 +32,16 @@ public class GetImageTransferProgressAnswer extends Answer { } public GetImageTransferProgressAnswer(GetImageTransferProgressCommand cmd, boolean success, String details, - Map progressMap) { + Map progressMap) { super(cmd, success, details); this.progressMap = progressMap; } - public Map getProgressMap() { + public Map getProgressMap() { return progressMap; } - public void setProgressMap(Map progressMap) { + public void setProgressMap(Map progressMap) { this.progressMap = progressMap; } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetImageTransferProgressCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetImageTransferProgressCommandWrapper.java index 293e87f9cef..7e0cbf2934d 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetImageTransferProgressCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetImageTransferProgressCommandWrapper.java @@ -43,7 +43,7 @@ public class LibvirtGetImageTransferProgressCommandWrapper extends CommandWrappe List transferIds = cmd.getTransferIds(); Map volumePaths = cmd.getVolumePaths(); Map volumeSizes = cmd.getVolumeSizes(); - Map progressMap = new HashMap<>(); + Map progressMap = new HashMap<>(); if (transferIds == null || transferIds.isEmpty()) { return new GetImageTransferProgressAnswer(cmd, true, "No transfers to check", progressMap); @@ -54,16 +54,16 @@ public class LibvirtGetImageTransferProgressCommandWrapper extends CommandWrappe Long volumeSize = volumeSizes.get(transferId); if (volumePath == null || volumeSize == null || volumeSize == 0) { - logger.warn("Missing volume path or size for transferId: " + transferId); - progressMap.put(transferId, 0); + logger.warn("Missing volume path or size for transferId: {}", transferId); + progressMap.put(transferId, null); continue; } try { File file = new File(volumePath); if (!file.exists()) { - logger.warn("Volume file does not exist: " + volumePath); - progressMap.put(transferId, 0); + logger.warn("Volume file does not exist: {}", volumePath); + progressMap.put(transferId, null); continue; } @@ -71,24 +71,17 @@ public class LibvirtGetImageTransferProgressCommandWrapper extends CommandWrappe if (volumePath.endsWith(".qcow2") || volumePath.endsWith(".qcow")) { try { - long virtualSize = KVMPhysicalDisk.getVirtualSizeFromFile(volumePath); - currentSize = virtualSize; + currentSize = KVMPhysicalDisk.getVirtualSizeFromFile(volumePath); } catch (Exception e) { - logger.warn("Failed to get virtual size for qcow2 file: " + volumePath + ", using physical size", e); + logger.warn("Failed to get virtual size for qcow2 file: {}, using physical size", volumePath, e); } } - - int progress = 0; - if (volumeSize > 0) { - progress = (int) Math.min(100, Math.max(0, (currentSize * 100) / volumeSize)); - } - - progressMap.put(transferId, progress); - logger.debug("Transfer {} progress: {}% (current: {}, total: {})", transferId, progress, currentSize, volumeSize); + progressMap.put(transferId, currentSize); + logger.debug("Transfer {} progress, current: {})", transferId, currentSize, volumeSize); } catch (Exception e) { - logger.error("Error getting progress for transferId: " + transferId + ", path: " + volumePath, e); - progressMap.put(transferId, 0); + logger.error("Error getting progress for transferId: {}, path: {}", transferId, volumePath, e); + progressMap.put(transferId, null); } } diff --git a/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java b/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java index 97b28468bea..c87445afd42 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java @@ -30,6 +30,7 @@ import java.util.stream.Collectors; import javax.inject.Inject; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.command.admin.backup.CreateImageTransferCmd; import org.apache.cloudstack.api.command.admin.backup.DeleteVmCheckpointCmd; import org.apache.cloudstack.api.command.admin.backup.FinalizeBackupCmd; @@ -50,20 +51,27 @@ import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; import org.joda.time.DateTime; import org.springframework.stereotype.Component; import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; +import com.cloud.api.ApiDBUtils; import com.cloud.exception.AgentUnavailableException; import com.cloud.exception.OperationTimedoutException; import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; import com.cloud.storage.ScopeType; +import com.cloud.storage.Storage; import com.cloud.storage.Volume; +import com.cloud.storage.VolumeDetailVO; +import com.cloud.storage.VolumeStats; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.VolumeDao; +import com.cloud.storage.dao.VolumeDetailsDao; +import com.cloud.utils.NumbersUtil; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VMInstanceVO; @@ -85,6 +93,9 @@ public class IncrementalBackupServiceImpl extends ManagerBase implements Increme @Inject private VolumeDao volumeDao; + @Inject + private VolumeDetailsDao volumeDetailsDao; + @Inject private AgentManager agentManager; @@ -653,6 +664,7 @@ public class IncrementalBackupServiceImpl extends ManagerBase implements Increme Map> transfersByHost = transferringTransfers.stream() .collect(Collectors.groupingBy(ImageTransferVO::getHostId)); + Map transferVolumeMap = new HashMap<>(); for (Map.Entry> entry : transfersByHost.entrySet()) { Long hostId = entry.getKey(); @@ -669,6 +681,7 @@ public class IncrementalBackupServiceImpl extends ManagerBase implements Increme logger.warn("Volume not found for image transfer: " + transfer.getUuid()); continue; } + transferVolumeMap.put(transfer.getId(), volume); String transferId = transfer.getUuid(); transferIds.add(transferId); @@ -693,23 +706,27 @@ public class IncrementalBackupServiceImpl extends ManagerBase implements Increme GetImageTransferProgressCommand cmd = new GetImageTransferProgressCommand(transferIds, volumePaths, volumeSizes); GetImageTransferProgressAnswer answer = (GetImageTransferProgressAnswer) agentManager.send(hostId, cmd); - if (answer != null && answer.getResult() && answer.getProgressMap() != null) { - for (ImageTransferVO transfer : hostTransfers) { - String transferId = transfer.getUuid(); - Integer progress = answer.getProgressMap().get(transferId); - if (progress != null) { - transfer.setProgress(progress); - if (progress == 100) { - transfer.setPhase(ImageTransfer.Phase.finished); - logger.debug("Updated phase for image transfer {} to finished", transferId); - } - imageTransferDao.update(transfer.getId(), transfer); - logger.debug("Updated progress for image transfer {}: {}%", transferId, progress); - } - } - } else { + if (answer == null || !answer.getResult() || MapUtils.isEmpty(answer.getProgressMap())) { logger.warn("Failed to get progress for transfers on host {}: {}", hostId, answer != null ? answer.getDetails() : "null answer"); + return; + } + for (ImageTransferVO transfer : hostTransfers) { + String transferId = transfer.getUuid(); + Long currentSize = answer.getProgressMap().get(transferId); + if (currentSize == null) { + continue; + } + VolumeVO volume = transferVolumeMap.get(transfer.getId()); + long totalSize = getVolumeTotalSize(volume); + int progress = Math.max((int)((currentSize * 100) / totalSize), 100); + transfer.setProgress(progress); + if (currentSize >= 100) { + transfer.setPhase(ImageTransfer.Phase.finished); + logger.debug("Updated phase for image transfer {} to finished", transferId); + } + imageTransferDao.update(transfer.getId(), transfer); + logger.debug("Updated progress for image transfer {}: {}%", transferId, progress); } } catch (AgentUnavailableException | OperationTimedoutException e) { @@ -724,6 +741,31 @@ public class IncrementalBackupServiceImpl extends ManagerBase implements Increme } } + private long getVolumeTotalSize(VolumeVO volume) { + VolumeDetailVO detail = volumeDetailsDao.findDetail(volume.getId(), ApiConstants.VIRTUAL_SIZE); + if (detail != null) { + long size = NumbersUtil.parseLong(detail.getValue(), 0L); + if (size > 0) { + return size; + } + } + ApiDBUtils.getVolumeStatistics(volume.getPath()); + VolumeStats vs = null; + if (List.of(Storage.ImageFormat.VHD, Storage.ImageFormat.QCOW2, Storage.ImageFormat.RAW).contains(volume.getFormat())) { + if (volume.getPath() != null) { + vs = ApiDBUtils.getVolumeStatistics(volume.getPath()); + } + } else if (volume.getFormat() == Storage.ImageFormat.OVA) { + if (volume.getChainInfo() != null) { + vs = ApiDBUtils.getVolumeStatistics(volume.getChainInfo()); + } + } + if (vs != null && vs.getPhysicalSize() > 0) { + return vs.getPhysicalSize(); + } + return volume.getSize(); + } + @Override public String getConfigComponentName() { return IncrementalBackupService.class.getSimpleName(); diff --git a/systemvm/debian/opt/cloud/bin/image_server.py b/systemvm/debian/opt/cloud/bin/image_server.py index 440ea0593ee..7f3beb328db 100644 --- a/systemvm/debian/opt/cloud/bin/image_server.py +++ b/systemvm/debian/opt/cloud/bin/image_server.py @@ -586,14 +586,6 @@ class Handler(BaseHTTPRequestHandler): try: logging.info("PUT start image_id=%s content_length=%d", image_id, content_length) with _NbdConn(cfg["host"], int(cfg["port"]), cfg.get("export")) as conn: - size = conn.size() - if content_length != size: - self._send_error_json( - HTTPStatus.BAD_REQUEST, - f"Content-Length must equal image size ({size})", - ) - return - offset = 0 remaining = content_length while remaining > 0: