etan39
/
cloudstack
mirror of https://github.com/apache/cloudstack.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Support file backend for cow format: api and server

This commit is contained in:
Abhisar Sinha 2026-02-08 11:01:29 +05:30 committed by Abhishek Kumar
parent a3669298af
commit 586134d392
11 changed files with 182 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
api/src/main/java/org/apache/cloudstack/api/command/admin/backup/CreateImageTransferCmd.java
View File

@ -32,6 +32,8 @@ import org.apache.cloudstack.backup.ImageTransfer;
import org.apache.cloudstack.backup.IncrementalBackupService;
import org.apache.cloudstack.context.CallContext;
import com.cloud.utils.EnumUtils;
@APICommand(name = "createImageTransfer",
description = "Create image transfer for a disk in backup",
responseObject = ImageTransferResponse.class,
@ -61,6 +63,11 @@ public class CreateImageTransferCmd extends BaseCmd implements AdminCmd {
description = "Direction of the transfer: upload, download")
private String direction;
@Parameter(name = ApiConstants.FORMAT,
type = CommandType.STRING,
description = "Format of the image: cow/raw. Currently only raw is supported for download. Defaults to raw if not provided")
private String format;
public Long getBackupId() {
return backupId;
}
@ -73,7 +80,11 @@ public class CreateImageTransferCmd extends BaseCmd implements AdminCmd {
return ImageTransfer.Direction.valueOf(direction);
}
@Override
public ImageTransfer.Format getFormat() {
return EnumUtils.fromString(ImageTransfer.Format.class, format);
}
@Override
public void execute() {
ImageTransferResponse response = incrementalBackupService.createImageTransfer(this);
response.setResponseName(getCommandName());

12
api/src/main/java/org/apache/cloudstack/backup/ImageTransfer.java
View File

@ -27,6 +27,16 @@ public interface ImageTransfer extends ControlledEntity, InternalIdentity {
upload, download
}
public enum Format {
raw,
cow
}
public enum Backend {
nbd,
file
}
public enum Phase {
initializing, transferring, finished, failed
}
@ -47,5 +57,7 @@ public interface ImageTransfer extends ControlledEntity, InternalIdentity {
Direction getDirection();
Backend getBackend();
String getSignedTicketId();
}

2
api/src/main/java/org/apache/cloudstack/backup/IncrementalBackupService.java
View File

@ -62,7 +62,7 @@ public interface IncrementalBackupService extends Configurable, PluggableService
*/
ImageTransferResponse createImageTransfer(CreateImageTransferCmd cmd);
ImageTransfer createImageTransfer(long volumeId, Long backupId, ImageTransfer.Direction direction);
ImageTransfer createImageTransfer(long volumeId, Long backupId, ImageTransfer.Direction direction, ImageTransfer.Format format);
boolean cancelImageTransfer(long imageTransferId);

28
core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java
View File

@ -26,19 +26,35 @@ public class CreateImageTransferCommand extends Command {
private int nbdPort;
private String direction;
private String checkpointId;
private String file;
private ImageTransfer.Backend backend;
public CreateImageTransferCommand() {
}
public CreateImageTransferCommand(String transferId, String hostIpAddress, String exportName, int nbdPort, String direction, String checkpointId) {
private CreateImageTransferCommand(String transferId, String hostIpAddress, String direction) {
this.transferId = transferId;
this.hostIpAddress = hostIpAddress;
this.direction = direction;
}
public CreateImageTransferCommand(String transferId, String hostIpAddress, String direction, String exportName, int nbdPort, String checkpointId) {
this(transferId, hostIpAddress, direction);
this.backend = ImageTransfer.Backend.nbd;
this.exportName = exportName;
this.nbdPort = nbdPort;
this.direction = direction;
this.checkpointId = checkpointId;
}
public CreateImageTransferCommand(String transferId, String hostIpAddress, String direction, String file) {
this(transferId, hostIpAddress, direction);
if (direction == ImageTransfer.Direction.download.toString()) {
throw new IllegalArgumentException("File backend is only supported for upload");
}
this.backend = ImageTransfer.Backend.file;
this.file = file;
}
public String getExportName() {
return exportName;
}
@ -47,6 +63,14 @@ public class CreateImageTransferCommand extends Command {
return nbdPort;
}
public String getFile() {
return file;
}
public ImageTransfer.Backend getBackend() {
return backend;
}
public String getHostIpAddress() {
return hostIpAddress;
}

29
engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java
View File

@ -54,6 +54,9 @@ public class ImageTransferVO implements ImageTransfer {
@Column(name = "nbd_port")
private int nbdPort;
@Column(name = "file")
private String file;
@Column(name = "transfer_url")
private String transferUrl;
@ -65,6 +68,10 @@ public class ImageTransferVO implements ImageTransfer {
@Column(name = "direction")
private Direction direction;
@Enumerated(value = EnumType.STRING)
@Column(name = "backend")
private Backend backend;
@Column(name = "signed_ticket_id")
private String signedTicketId;
@ -95,12 +102,10 @@ public class ImageTransferVO implements ImageTransfer {
public ImageTransferVO() {
}
public ImageTransferVO(String uuid, Long backupId, long diskId, long hostId, int nbdPort, Phase phase, Direction direction, Long accountId, Long domainId, Long dataCenterId) {
private ImageTransferVO(String uuid, long diskId, long hostId, Phase phase, Direction direction, Long accountId, Long domainId, Long dataCenterId) {
this.uuid = uuid;
this.backupId = backupId;
this.diskId = diskId;
this.hostId = hostId;
this.nbdPort = nbdPort;
this.phase = phase;
this.direction = direction;
this.accountId = accountId;
@ -109,6 +114,19 @@ public class ImageTransferVO implements ImageTransfer {
this.created = new Date();
}
public ImageTransferVO(String uuid, Long backupId, long diskId, long hostId, int nbdPort, Phase phase, Direction direction, Long accountId, Long domainId, Long dataCenterId) {
this(uuid, diskId, hostId, phase, direction, accountId, domainId, dataCenterId);
this.backupId = backupId;
this.nbdPort = nbdPort;
this.backend = Backend.nbd;
}
public ImageTransferVO(String uuid, long diskId, long hostId, String file, Phase phase, Direction direction, Long accountId, Long domainId, Long dataCenterId) {
this(uuid, diskId, hostId, phase, direction, accountId, domainId, dataCenterId);
this.file = file;
this.backend = Backend.file;
}
@Override
public long getId() {
return id;
@ -183,6 +201,11 @@ public class ImageTransferVO implements ImageTransfer {
this.direction = direction;
}
@Override
public Backend getBackend() {
return backend;
}
@Override
public String getSignedTicketId() {
return signedTicketId;

1
engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDao.java
View File

@ -29,5 +29,6 @@ public interface ImageTransferDao extends GenericDao<ImageTransferVO, Long> {
ImageTransferVO findByUuid(String uuid);
ImageTransferVO findByNbdPort(int port);
ImageTransferVO findByVolume(Long volumeId);
ImageTransferVO findUnfinishedByVolume(Long volumeId);
List<ImageTransferVO> listByPhaseAndDirection(ImageTransfer.Phase phase, ImageTransfer.Direction direction);
}

14
engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDaoImpl.java
View File

@ -36,6 +36,7 @@ public class ImageTransferDaoImpl extends GenericDaoBase<ImageTransferVO, Long>
private SearchBuilder<ImageTransferVO> uuidSearch;
private SearchBuilder<ImageTransferVO> nbdPortSearch;
private SearchBuilder<ImageTransferVO> volumeSearch;
private SearchBuilder<ImageTransferVO> volumeUnfinishedSearch;
private SearchBuilder<ImageTransferVO> phaseDirectionSearch;
public ImageTransferDaoImpl() {
@ -59,6 +60,11 @@ public class ImageTransferDaoImpl extends GenericDaoBase<ImageTransferVO, Long>
volumeSearch.and("volumeId", volumeSearch.entity().getDiskId(), SearchCriteria.Op.EQ);
volumeSearch.done();
volumeUnfinishedSearch = createSearchBuilder();
volumeUnfinishedSearch.and("volumeId", volumeUnfinishedSearch.entity().getDiskId(), SearchCriteria.Op.EQ);
volumeUnfinishedSearch.and("phase", volumeUnfinishedSearch.entity().getPhase(), SearchCriteria.Op.NEQ);
volumeUnfinishedSearch.done();
phaseDirectionSearch = createSearchBuilder();
phaseDirectionSearch.and("phase", phaseDirectionSearch.entity().getPhase(), SearchCriteria.Op.EQ);
phaseDirectionSearch.and("direction", phaseDirectionSearch.entity().getDirection(), SearchCriteria.Op.EQ);
@ -93,6 +99,14 @@ public class ImageTransferDaoImpl extends GenericDaoBase<ImageTransferVO, Long>
return findOneBy(sc);
}
@Override
public ImageTransferVO findUnfinishedByVolume(Long volumeId) {
SearchCriteria<ImageTransferVO> sc = volumeUnfinishedSearch.create();
sc.setParameters("volumeId", volumeId);
sc.setParameters("phase", ImageTransferVO.Phase.finished.toString());
return findOneBy(sc);
}
@Override
public List<ImageTransferVO> listByPhaseAndDirection(ImageTransfer.Phase phase, ImageTransfer.Direction direction) {
SearchCriteria<ImageTransferVO> sc = phaseDirectionSearch.create();

5
engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql
View File

@ -18,7 +18,7 @@
--;
-- Schema upgrade from 4.21.0.0 to 4.22.0.0
--;
not supported for download
-- health check status as enum
CALL `cloud`.`IDEMPOTENT_CHANGE_COLUMN`('router_health_check', 'check_result', 'check_result', 'varchar(16) NOT NULL COMMENT "check executions result: SUCCESS, FAILURE, WARNING, UNKNOWN"');
@ -93,3 +93,6 @@ UPDATE `cloud`.`configuration` SET `scope` = 2 WHERE `name` = 'use.https.to.uplo
-- Delete the configuration for 'use.https.to.upload' from StoragePool
DELETE FROM `cloud`.`storage_pool_details` WHERE `name` = 'use.https.to.upload';
<<<<<<< HEAD
=======
>>>>>>> 1ec4e52fa6 (Support file backend for cow format: api and server)

2
engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql
View File

@ -141,8 +141,10 @@ CREATE TABLE IF NOT EXISTS `cloud`.`image_transfer`(
`host_id` bigint unsigned NOT NULL COMMENT 'Host ID',
`nbd_port` int NOT NULL COMMENT 'NBD port',
`transfer_url` varchar(255) COMMENT 'ImageIO transfer URL',
`file` varchar(255) COMMENT 'File for the file backend',
`phase` varchar(20) NOT NULL COMMENT 'Transfer phase: initializing, transferring, finished, failed',
`direction` varchar(20) NOT NULL COMMENT 'Direction: upload, download',
`backend` varchar(20) NOT NULL COMMENT 'Backend: nbd, file',
`progress` int COMMENT 'Transfer progress percentage (0-100)',
`signed_ticket_id` varchar(255) COMMENT 'Signed ticket ID from ImageIO',
`created` datetime NOT NULL COMMENT 'date created',

8
plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java
View File

@ -53,6 +53,7 @@ import org.apache.cloudstack.api.command.user.volume.ResizeVolumeCmd;
import org.apache.cloudstack.api.response.ListResponse;
import org.apache.cloudstack.api.response.ServiceOfferingResponse;
import org.apache.cloudstack.backup.ImageTransfer.Direction;
import org.apache.cloudstack.backup.ImageTransfer.Format;
import org.apache.cloudstack.backup.ImageTransferVO;
import org.apache.cloudstack.backup.IncrementalBackupService;
import org.apache.cloudstack.backup.dao.ImageTransferDao;
@ -753,7 +754,8 @@ public class ServerAdapter extends ManagerBase {
if (direction == null) {
throw new InvalidParameterValueException("Invalid or missing direction");
}
return createImageTransfer(null, volumeVO.getId(), direction);
Format format = EnumUtils.fromString(Format.class, request.getFormat());
return createImageTransfer(null, volumeVO.getId(), direction, format);
}
public boolean handleCancelImageTransfer(String uuid) {
@ -772,12 +774,12 @@ public class ServerAdapter extends ManagerBase {
return incrementalBackupService.finalizeImageTransfer(vo.getId());
}
private ImageTransfer createImageTransfer(Long backupId, Long volumeId, Direction direction) {
private ImageTransfer createImageTransfer(Long backupId, Long volumeId, Direction direction, Format format) {
Account serviceAccount = createServiceAccountIfNeeded();
CallContext.register(serviceAccount.getId(), serviceAccount.getId());
try {
org.apache.cloudstack.backup.ImageTransfer imageTransfer =
incrementalBackupService.createImageTransfer(volumeId, null, direction);
incrementalBackupService.createImageTransfer(volumeId, null, direction, format);
ImageTransferVO imageTransferVO = imageTransferDao.findById(imageTransfer.getId());
return ImageTransferVOToImageTransferConverter.toImageTransfer(imageTransferVO, this::getHostById, this::getVolumeById);
} finally {

121
server/src/main/java/org/apache/cloudstack/backup/IncrementalBackupServiceImpl.java
View File

@ -50,7 +50,6 @@ import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector;
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.joda.time.DateTime;
import org.springframework.stereotype.Component;
@ -251,8 +250,11 @@ public class IncrementalBackupServiceImpl extends ManagerBase implements Increme
boolean dummyOffering = isDummyOffering(backup.getBackupOfferingId());
List<ImageTransferVO> transfers = imageTransferDao.listByBackupId(backupId);
if (CollectionUtils.isNotEmpty(transfers)) {
throw new CloudRuntimeException("Image transfers not finalized for backup: " + backupId);
for (ImageTransferVO transfer : transfers) {
if (transfer.getPhase() != ImageTransferVO.Phase.finished) {
throw new CloudRuntimeException(String.format("Image transfer %s not finalized for backup: %s", transfer.getUuid(), backup.getUuid()));
}
imageTransferDao.remove(transfer.getId());
}
if (vm.getState() == State.Running) {
@ -294,13 +296,16 @@ public class IncrementalBackupServiceImpl extends ManagerBase implements Increme
}
private ImageTransferVO createDownloadImageTransfer(Long backupId, VolumeVO volume) {
private ImageTransferVO createDownloadImageTransfer(Long backupId, VolumeVO volume, ImageTransfer.Backend backend) {
final String direction = ImageTransfer.Direction.download.toString();
BackupVO backup = backupDao.findById(backupId);
if (backup == null) {
throw new CloudRuntimeException("Backup not found: " + backupId);
}
boolean dummyOffering = isDummyOffering(backup.getBackupOfferingId());
if (ImageTransfer.Backend.file.equals(backend)) {
throw new CloudRuntimeException("File backend is not supported for download");
}
String transferId = UUID.randomUUID().toString();
Host host = hostDao.findById(backup.getHostId());
@ -314,11 +319,10 @@ public class IncrementalBackupServiceImpl extends ManagerBase implements Increme
CreateImageTransferCommand transferCmd = new CreateImageTransferCommand(
transferId,
host.getPrivateIpAddress(),
direction,
volume.getUuid(),
backup.getNbdPort(),
direction,
backup.getFromCheckpointId()
);
backup.getFromCheckpointId());
try {
CreateImageTransferAnswer answer;
@ -396,50 +400,70 @@ public class IncrementalBackupServiceImpl extends ManagerBase implements Increme
return volumePath;
}
private ImageTransferVO createUploadImageTransfer(VolumeVO volume) {
private ImageTransferVO createUploadImageTransfer(VolumeVO volume, ImageTransfer.Backend backend) {
final String direction = ImageTransfer.Direction.upload.toString();
String transferId = UUID.randomUUID().toString();
int nbdPort = allocateNbdPort();
Long poolId = volume.getPoolId();
StoragePoolVO storagePoolVO = primaryDataStoreDao.findById(poolId);
Host host = getFirstHostFromStoragePool(storagePoolVO);
String volumePath = getVolumePathForFileBasedBackend(volume);
startNBDServer(transferId, direction, host, volume.getUuid(), volumePath, nbdPort);
ImageTransferVO imageTransfer;
CreateImageTransferCommand transferCmd;
if (backend.equals(ImageTransfer.Backend.file)) {
imageTransfer = new ImageTransferVO(
transferId,
volume.getId(),
host.getId(),
volumePath,
ImageTransferVO.Phase.transferring,
ImageTransfer.Direction.upload,
volume.getAccountId(),
volume.getDomainId(),
volume.getDataCenterId());
ImageTransferVO imageTransfer = new ImageTransferVO(
transferId,
null,
volume.getId(),
host.getId(),
nbdPort,
ImageTransferVO.Phase.transferring,
ImageTransfer.Direction.upload,
volume.getAccountId(),
volume.getDomainId(),
volume.getDataCenterId()
);
transferCmd = new CreateImageTransferCommand(
transferId,
host.getPrivateIpAddress(),
direction,
volumePath);
CreateImageTransferAnswer transferAnswer;
CreateImageTransferCommand transferCmd = new CreateImageTransferCommand(
transferId,
host.getPrivateIpAddress(),
volume.getUuid(),
nbdPort,
direction,
null
);
} else {
int nbdPort = allocateNbdPort();
startNBDServer(transferId, direction, host, volume.getUuid(), volumePath, nbdPort);
imageTransfer = new ImageTransferVO(
transferId,
null,
volume.getId(),
host.getId(),
nbdPort,
ImageTransferVO.Phase.transferring,
ImageTransfer.Direction.upload,
volume.getAccountId(),
volume.getDomainId(),
volume.getDataCenterId());
EndPoint ssvm = _epSelector.findSsvm(volume.getDataCenterId());
transferAnswer = (CreateImageTransferAnswer) ssvm.sendMessage(transferCmd);
if (!transferAnswer.getResult()) {
stopNbdServer(imageTransfer);
throw new CloudRuntimeException("Failed to create image transfer: " + transferAnswer.getDetails());
transferCmd = new CreateImageTransferCommand(
transferId,
host.getPrivateIpAddress(),
direction,
volume.getUuid(),
nbdPort,
null);
}
EndPoint ssvm = _epSelector.findSsvm(volume.getDataCenterId());
CreateImageTransferAnswer transferAnswer = (CreateImageTransferAnswer) ssvm.sendMessage(transferCmd);
if (!transferAnswer.getResult()) {
if (!backend.equals(ImageTransfer.Backend.file)) {
stopNbdServer(imageTransfer);
}
throw new CloudRuntimeException("Failed to create image transfer: " + transferAnswer.getDetails());
}
imageTransfer.setTransferUrl(transferAnswer.getTransferUrl());
imageTransfer.setSignedTicketId(transferAnswer.getImageTransferId());
imageTransfer = imageTransferDao.persist(imageTransfer);
@ -447,9 +471,21 @@ public class IncrementalBackupServiceImpl extends ManagerBase implements Increme
}
private ImageTransfer.Backend getImageTransferBackend(ImageTransfer.Format format, ImageTransfer.Direction direction) {
if (ImageTransfer.Format.cow.equals(format)) {
if (ImageTransfer.Direction.download.equals(direction)) {
logger.debug("Using NBD backend for download");
return ImageTransfer.Backend.nbd;
}
return ImageTransfer.Backend.file;
} else {
return ImageTransfer.Backend.nbd;
}
}
@Override
public ImageTransferResponse createImageTransfer(CreateImageTransferCmd cmd) {
ImageTransfer imageTransfer = createImageTransfer(cmd.getVolumeId(), cmd.getBackupId(), cmd.getDirection());
ImageTransfer imageTransfer = createImageTransfer(cmd.getVolumeId(), cmd.getBackupId(), cmd.getDirection(), cmd.getFormat());
if (imageTransfer instanceof ImageTransferVO) {
ImageTransferVO imageTransferVO = (ImageTransferVO) imageTransfer;
return toImageTransferResponse(imageTransferVO);
@ -458,19 +494,20 @@ public class IncrementalBackupServiceImpl extends ManagerBase implements Increme
}
@Override
public ImageTransfer createImageTransfer(long volumeId, Long backupId, ImageTransfer.Direction direction) {
public ImageTransfer createImageTransfer(long volumeId, Long backupId, ImageTransfer.Direction direction, ImageTransfer.Format format) {
ImageTransfer imageTransfer;
VolumeVO volume = volumeDao.findById(volumeId);
ImageTransferVO existingTransfer = imageTransferDao.findByVolume(volume.getId());
ImageTransferVO existingTransfer = imageTransferDao.findUnfinishedByVolume(volume.getId());
if (existingTransfer != null) {
throw new CloudRuntimeException("Image transfer already in progress for volume: " + volume.getUuid());
}
ImageTransfer.Backend backend = getImageTransferBackend(format, direction);
if (ImageTransfer.Direction.upload.equals(direction)) {
imageTransfer = createUploadImageTransfer(volume);
imageTransfer = createUploadImageTransfer(volume, backend);
} else if (ImageTransfer.Direction.download.equals(direction)) {
imageTransfer = createDownloadImageTransfer(backupId, volume);
imageTransfer = createDownloadImageTransfer(backupId, volume, backend);
} else {
throw new CloudRuntimeException("Invalid direction: " + direction);
}