add support for new gen clvm with template (qcow2) backing

This commit is contained in:
Pearl Dsilva 2026-03-05 13:09:56 -05:00
parent d51123de0c
commit 81bb667267
17 changed files with 491 additions and 69 deletions

View File

@ -170,6 +170,7 @@ public class Storage {
ISO(false, false, EncryptionSupport.Unsupported), // for iso image
LVM(false, false, EncryptionSupport.Unsupported), // XenServer local LVM SR
CLVM(true, false, EncryptionSupport.Unsupported),
CLVM_NG(true, false, EncryptionSupport.Hypervisor),
RBD(true, true, EncryptionSupport.Unsupported), // http://libvirt.org/storage.html#StorageBackendRBD
SharedMountPoint(true, true, EncryptionSupport.Hypervisor),
VMFS(true, true, EncryptionSupport.Unsupported), // VMware VMFS storage

View File

@ -3382,7 +3382,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
for (VolumeVO volume : volumes) {
StoragePoolVO pool = _storagePoolDao.findById(volume.getPoolId());
if (pool != null && pool.getPoolType() == Storage.StoragePoolType.CLVM) {
if (pool != null && ClvmLockManager.isClvmPoolType(pool.getPoolType())) {
clvmLockManager.setClvmLockHostId(volume.getId(), destHostId);
}
}

View File

@ -754,7 +754,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati
// For CLVM pools, set the lock host hint so volume is created on the correct host
// This avoids the need for shared mode activation and improves performance
if (pool.getPoolType() == Storage.StoragePoolType.CLVM && hostId != null) {
if (ClvmLockManager.isClvmPoolType(pool.getPoolType()) && hostId != null) {
logger.info("CLVM pool detected. Setting lock host {} for volume {} to route creation to correct host",
hostId, volumeInfo.getUuid());
volumeInfo.setDestinationHostId(hostId);
@ -818,7 +818,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati
}
StoragePoolVO pool = _storagePoolDao.findById(destPool.getId());
if (pool == null || pool.getPoolType() != Storage.StoragePoolType.CLVM) {
if (pool == null || pool.getPoolType() != Storage.StoragePoolType.CLVM || pool.getPoolType() != Storage.StoragePoolType.CLVM_NG) {
return;
}
@ -860,7 +860,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati
}
StoragePoolVO pool = _storagePoolDao.findById(volume.getPoolId());
if (pool != null && pool.getPoolType() == Storage.StoragePoolType.CLVM) {
if (pool != null && ClvmLockManager.isClvmPoolType(pool.getPoolType())) {
Long lockHostId = clvmLockManager.getClvmLockHostId(volume.getId(), volume.getUuid());
if (lockHostId != null) {
logger.debug("Found CLVM lock host {} from existing volume {} of VM {}",
@ -884,7 +884,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati
}
StoragePoolVO pool = _storagePoolDao.findById(volume.getPoolId());
if (pool == null || pool.getPoolType() != Storage.StoragePoolType.CLVM) {
if (pool == null || pool.getPoolType() != Storage.StoragePoolType.CLVM || pool.getPoolType() != Storage.StoragePoolType.CLVM_NG) {
continue;
}
@ -1324,8 +1324,8 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati
Long clusterId = storagePool.getClusterId();
logger.trace("storage-pool {}/{} is associated with cluster {}",storagePool.getName(), storagePool.getUuid(), clusterId);
Long hostId = vm.getHostId();
if (hostId == null && (storagePool.isLocal() || storagePool.getPoolType() == Storage.StoragePoolType.CLVM)) {
if (storagePool.getPoolType() == Storage.StoragePoolType.CLVM) {
if (hostId == null && (storagePool.isLocal() || ClvmLockManager.isClvmPoolType(storagePool.getPoolType()))) {
if (ClvmLockManager.isClvmPoolType(storagePool.getPoolType())) {
hostId = getClvmLockHostFromVmVolumes(vm.getId());
if (hostId != null) {
logger.debug("Using CLVM lock host {} from VM {}'s existing volumes for new volume creation",
@ -1998,7 +1998,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati
// For CLVM pools, set the destination host hint so volume is created on the correct host
// This avoids the need for shared mode activation and improves performance
StoragePoolVO poolVO = _storagePoolDao.findById(destPool.getId());
if (poolVO != null && poolVO.getPoolType() == Storage.StoragePoolType.CLVM) {
if (poolVO != null && ClvmLockManager.isClvmPoolType(poolVO.getPoolType())) {
Long hostId = vm.getVirtualMachine().getHostId();
if (hostId != null) {
volume.setDestinationHostId(hostId);

View File

@ -710,7 +710,7 @@ public class DefaultSnapshotStrategy extends SnapshotStrategyBase {
}
StoragePool pool = (StoragePool) dataStoreMgr.getDataStore(poolId, DataStoreRole.Primary);
if (pool == null || pool.getPoolType() != StoragePoolType.CLVM) {
if (pool == null || pool.getPoolType() != StoragePoolType.CLVM || pool.getPoolType() != StoragePoolType.CLVM_NG) {
return false;
}

View File

@ -32,6 +32,7 @@ import javax.inject.Inject;
import com.cloud.dc.DedicatedResourceVO;
import com.cloud.dc.dao.DedicatedResourceDao;
import com.cloud.storage.ClvmLockManager;
import com.cloud.storage.Storage;
import com.cloud.storage.VolumeDetailVO;
import com.cloud.storage.dao.VolumeDetailsDao;
@ -273,8 +274,6 @@ public class DefaultEndPointSelector implements EndPointSelector {
@Override
public EndPoint select(DataObject srcData, DataObject destData, boolean volumeEncryptionSupportRequired) {
// FOR CLVM: Check if destination is a volume with destinationHostId hint
// This ensures template-to-volume copy is routed to the correct host for optimal lock placement
if (destData instanceof VolumeInfo) {
EndPoint clvmEndpoint = selectClvmEndpointIfApplicable((VolumeInfo) destData, "template-to-volume copy");
if (clvmEndpoint != null) {
@ -424,7 +423,9 @@ public class DefaultEndPointSelector implements EndPointSelector {
// Check if this is a CLVM pool
StoragePoolVO pool = _storagePoolDao.findById(store.getId());
if (pool == null || pool.getPoolType() != Storage.StoragePoolType.CLVM) {
if (pool == null ||
(pool.getPoolType() != Storage.StoragePoolType.CLVM ||
pool.getPoolType() != Storage.StoragePoolType.CLVM_NG)) {
return null;
}
@ -450,7 +451,6 @@ public class DefaultEndPointSelector implements EndPointSelector {
public EndPoint select(DataObject object, boolean encryptionSupportRequired) {
DataStore store = object.getDataStore();
// For CLVM volumes with destination host hint, route to that specific host
// This ensures volumes are created on the correct host with exclusive locks
if (object instanceof VolumeInfo && store.getRole() == DataStoreRole.Primary) {
VolumeInfo volInfo = (VolumeInfo) object;
@ -467,6 +467,7 @@ public class DefaultEndPointSelector implements EndPointSelector {
throw new CloudRuntimeException(String.format("Storage role %s doesn't support encryption", store.getRole()));
}
@Override
public EndPoint select(DataObject object) {
DataStore store = object.getDataStore();
@ -475,7 +476,7 @@ public class DefaultEndPointSelector implements EndPointSelector {
if (object instanceof VolumeInfo && store.getRole() == DataStoreRole.Primary) {
VolumeInfo volume = (VolumeInfo) object;
StoragePoolVO pool = _storagePoolDao.findById(store.getId());
if (pool != null && pool.getPoolType() == Storage.StoragePoolType.CLVM) {
if (pool != null && ClvmLockManager.isClvmPoolType(pool.getPoolType())) {
Long lockHostId = getClvmLockHostId(volume);
if (lockHostId != null) {
logger.debug("Routing CLVM volume {} operation to lock holder host {}",
@ -589,7 +590,7 @@ public class DefaultEndPointSelector implements EndPointSelector {
DataStore store = volume.getDataStore();
if (store.getRole() == DataStoreRole.Primary) {
StoragePoolVO pool = _storagePoolDao.findById(store.getId());
if (pool != null && pool.getPoolType() == Storage.StoragePoolType.CLVM) {
if (pool != null && ClvmLockManager.isClvmPoolType(pool.getPoolType())) {
Long lockHostId = getClvmLockHostId(volume);
if (lockHostId != null) {
logger.info("Routing CLVM volume {} deletion to lock holder host {}",

View File

@ -37,6 +37,7 @@ import com.cloud.network.dao.NetworkDao;
import com.cloud.network.dao.NetworkVO;
import com.cloud.offerings.NetworkOfferingVO;
import com.cloud.offerings.dao.NetworkOfferingDao;
import com.cloud.storage.ClvmLockManager;
import com.cloud.storage.DataStoreRole;
import com.cloud.storage.Storage;
import com.cloud.storage.StorageManager;
@ -144,7 +145,7 @@ public class DefaultHostListener implements HypervisorHostListener {
// Propagate CLVM secure zero-fill setting to the host
// Note: This is done during host connection (agent start, MS restart, host reconnection)
// so the setting is non-dynamic. Changes require host reconnection to take effect.
if (pool.getPoolType() == Storage.StoragePoolType.CLVM) {
if (ClvmLockManager.isClvmPoolType(pool.getPoolType())) {
Boolean clvmSecureZeroFill = VolumeApiServiceImpl.CLVMSecureZeroFill.valueIn(poolId);
if (clvmSecureZeroFill != null) {
detailsMap.put("clvmsecurezerofill", String.valueOf(clvmSecureZeroFill));

View File

@ -2484,7 +2484,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
} else if ((poolType == StoragePoolType.NetworkFilesystem
|| poolType == StoragePoolType.SharedMountPoint
|| poolType == StoragePoolType.Filesystem
|| poolType == StoragePoolType.Gluster)
|| poolType == StoragePoolType.Gluster
|| poolType == StoragePoolType.CLVM_NG)
&& volFormat == PhysicalDiskFormat.QCOW2 ) {
return "QCOW2";
} else if (poolType == StoragePoolType.Linstor) {
@ -3680,13 +3681,19 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
final String glusterVolume = pool.getSourceDir().replace("/", "");
disk.defNetworkBasedDisk(glusterVolume + path.replace(mountpoint, ""), pool.getSourceHost(), pool.getSourcePort(), null,
null, devId, diskBusType, DiskProtocol.GLUSTER, DiskDef.DiskFmtType.QCOW2);
} else if (pool.getType() == StoragePoolType.CLVM || physicalDisk.getFormat() == PhysicalDiskFormat.RAW) {
} else if (pool.getType() == StoragePoolType.CLVM || pool.getType() == StoragePoolType.CLVM_NG || physicalDisk.getFormat() == PhysicalDiskFormat.RAW) {
// CLVM and CLVM_NG use block devices (/dev/vgname/volume)
if (volume.getType() == Volume.Type.DATADISK && !(isWindowsTemplate && isUefiEnabled)) {
disk.defBlockBasedDisk(physicalDisk.getPath(), devId, diskBusTypeData);
}
else {
} else {
disk.defBlockBasedDisk(physicalDisk.getPath(), devId, diskBusType);
}
// CLVM_NG uses QCOW2 format on block devices, override the default RAW format
if (pool.getType() == StoragePoolType.CLVM_NG) {
disk.setDiskFormatType(DiskDef.DiskFmtType.QCOW2);
}
if (pool.getType() == StoragePoolType.Linstor && isQemuDiscardBugFree(diskBusType)) {
disk.setDiscard(DiscardType.UNMAP);
}
@ -6566,21 +6573,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
if (isClvmVolume(disk, resource, vmSpec)) {
String volumePath = disk.getDiskPath();
try {
LOGGER.info("[CLVM Migration] {} for volume [{}]",
state.getLogMessage(), volumePath);
Script cmd = new Script("lvchange", Duration.standardSeconds(300), LOGGER);
cmd.add(state.getLvchangeFlag());
cmd.add(volumePath);
String result = cmd.execute();
if (result != null) {
LOGGER.error("[CLVM Migration] Failed to set volume [{}] to {} state. Command result: {}",
volumePath, state.getDescription(), result);
} else {
LOGGER.info("[CLVM Migration] Successfully set volume [{}] to {} state.",
volumePath, state.getDescription());
}
modifyClvmVolumeState(volumePath, state.getLvchangeFlag(), state.getDescription(), state.getLogMessage());
} catch (Exception e) {
LOGGER.error("[CLVM Migration] Exception while setting volume [{}] to {} state: {}",
volumePath, state.getDescription(), e.getMessage(), e);
@ -6589,6 +6582,53 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
}
}
private static void modifyClvmVolumeState(String volumePath, String lvchangeFlag,
String stateDescription, String logMessage) {
try {
LOGGER.info("[CLVM Migration] {} for volume [{}]", logMessage, volumePath);
Script cmd = new Script("lvchange", Duration.standardSeconds(300), LOGGER);
cmd.add(lvchangeFlag);
cmd.add(volumePath);
String result = cmd.execute();
if (result != null) {
String errorMsg = String.format(
"[CLVM Migration] Failed to set volume [%s] to %s state. Command result: %s",
volumePath, stateDescription, result);
LOGGER.error(errorMsg);
throw new CloudRuntimeException(errorMsg);
} else {
LOGGER.info("[CLVM Migration] Successfully set volume [{}] to {} state.",
volumePath, stateDescription);
}
} catch (CloudRuntimeException e) {
throw e;
} catch (Exception e) {
String errorMsg = String.format(
"[CLVM Migration] Exception while setting volume [%s] to %s state: %s",
volumePath, stateDescription, e.getMessage());
LOGGER.error(errorMsg, e);
throw new CloudRuntimeException(errorMsg, e);
}
}
public static void activateClvmVolumeExclusive(String volumePath) {
modifyClvmVolumeState(volumePath, ClvmVolumeState.EXCLUSIVE.getLvchangeFlag(),
ClvmVolumeState.EXCLUSIVE.getDescription(),
"Activating CLVM volume in exclusive mode for copy");
}
public static void deactivateClvmVolume(String volumePath) {
try {
modifyClvmVolumeState(volumePath, ClvmVolumeState.DEACTIVATE.getLvchangeFlag(),
ClvmVolumeState.DEACTIVATE.getDescription(),
"Deactivating CLVM volume after copy");
} catch (Exception e) {
LOGGER.warn("Failed to deactivate CLVM volume {}: {}", volumePath, e.getMessage());
}
}
/**
* Determines if a disk is on a CLVM storage pool by checking the actual pool type from VirtualMachineTO.
* This is the most reliable method as it uses CloudStack's own storage pool information.
@ -6613,8 +6653,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
DataStoreTO dataStore = volumeTO.getDataStore();
if (dataStore instanceof PrimaryDataStoreTO) {
PrimaryDataStoreTO primaryStore = (PrimaryDataStoreTO) dataStore;
boolean isClvm = StoragePoolType.CLVM == primaryStore.getPoolType();
LOGGER.debug("Disk {} identified as CLVM={} via VirtualMachineTO pool type: {}",
boolean isClvm = StoragePoolType.CLVM == primaryStore.getPoolType() ||
StoragePoolType.CLVM_NG == primaryStore.getPoolType();
LOGGER.debug("Disk {} identified as CLVM/CLVM_NG={} via VirtualMachineTO pool type: {}",
diskPath, isClvm, primaryStore.getPoolType());
return isClvm;
}

View File

@ -113,7 +113,8 @@ public final class LibvirtResizeVolumeCommandWrapper extends CommandWrapper<Resi
logger.debug("Resizing volume: " + path + ", from: " + toHumanReadableSize(currentSize) + ", to: " + toHumanReadableSize(newSize) + ", type: " + type + ", name: " + vmInstanceName + ", shrinkOk: " + shrinkOk);
/* libvirt doesn't support resizing (C)LVM devices, and corrupts QCOW2 in some scenarios, so we have to do these via qemu-img */
if (pool.getType() != StoragePoolType.CLVM && pool.getType() != StoragePoolType.Linstor && pool.getType() != StoragePoolType.PowerFlex
if (pool.getType() != StoragePoolType.CLVM && pool.getType() != StoragePoolType.CLVM_NG
&& pool.getType() != StoragePoolType.Linstor && pool.getType() != StoragePoolType.PowerFlex
&& vol.getFormat() != PhysicalDiskFormat.QCOW2) {
logger.debug("Volume " + path + " can be resized by libvirt. Asking libvirt to resize the volume.");
try {

View File

@ -117,7 +117,7 @@ public class LibvirtRevertSnapshotCommandWrapper extends CommandWrapper<RevertSn
secondaryStoragePool = storagePoolMgr.getStoragePoolByURI(snapshotImageStore.getUrl());
}
if (primaryPool.getType() == StoragePoolType.CLVM) {
if (primaryPool.getType() == StoragePoolType.CLVM || primaryPool.getType() == StoragePoolType.CLVM_NG) {
Script cmd = new Script(libvirtComputingResource.manageSnapshotPath(), libvirtComputingResource.getCmdsTimeout(), logger);
cmd.add("-v", getFullPathAccordingToStorage(secondaryStoragePool, snapshotRelPath));
cmd.add("-n", snapshotDisk.getName());

View File

@ -288,12 +288,34 @@ public class KVMStoragePoolManager {
}
if (pool instanceof LibvirtStoragePool) {
addPoolDetails(uuid, (LibvirtStoragePool) pool);
LibvirtStoragePool libvirtPool = (LibvirtStoragePool) pool;
addPoolDetails(uuid, libvirtPool);
updatePoolTypeIfApplicable(libvirtPool, pool, type, uuid);
}
return pool;
}
private void updatePoolTypeIfApplicable(LibvirtStoragePool libvirtPool, KVMStoragePool pool,
StoragePoolType type, String uuid) {
StoragePoolType correctType = type;
if (correctType == null || correctType == StoragePoolType.CLVM) {
StoragePoolInformation info = _storagePools.get(uuid);
if (info != null && info.getPoolType() != null) {
correctType = info.getPoolType();
}
}
if (correctType != null && correctType != pool.getType() &&
(correctType == StoragePoolType.CLVM || correctType == StoragePoolType.CLVM_NG) &&
(pool.getType() == StoragePoolType.CLVM || pool.getType() == StoragePoolType.CLVM_NG)) {
logger.debug("Correcting pool type from {} to {} for pool {} based on caller/cached information",
pool.getType(), correctType, uuid);
libvirtPool.setType(correctType);
}
}
/**
* As the class {@link LibvirtStoragePool} is constrained to the {@link org.libvirt.StoragePool} class, there is no way of saving a generic parameter such as the details, hence,
* this method was created to always make available the details of libvirt primary storages for when they are needed.
@ -450,6 +472,10 @@ public class KVMStoragePoolManager {
return adaptor.createDiskFromTemplate(template, name,
PhysicalDiskFormat.RAW, provisioningType,
size, destPool, timeout, passphrase);
} else if (destPool.getType() == StoragePoolType.CLVM_NG) {
return adaptor.createDiskFromTemplate(template, name,
PhysicalDiskFormat.QCOW2, provisioningType,
size, destPool, timeout, passphrase);
} else if (template.getFormat() == PhysicalDiskFormat.DIR) {
return adaptor.createDiskFromTemplate(template, name,
PhysicalDiskFormat.DIR, provisioningType,
@ -491,6 +517,11 @@ public class KVMStoragePoolManager {
return adaptor.createTemplateFromDirectDownloadFile(templateFilePath, destTemplatePath, destPool, format, timeout);
}
public void createTemplateOnClvmNg(String templatePath, String templateUuid, int timeout, KVMStoragePool pool) {
LibvirtStorageAdaptor adaptor = (LibvirtStorageAdaptor) getStorageAdaptor(pool.getType());
adaptor.createTemplateOnClvmNg(templatePath, templateUuid, timeout, pool);
}
public Ternary<Boolean, Map<String, String>, String> prepareStorageClient(StoragePoolType type, String uuid, Map<String, String> details) {
StorageAdaptor adaptor = getStorageAdaptor(type);
return adaptor.prepareStorageClient(uuid, details);

View File

@ -344,15 +344,28 @@ public class KVMStorageProcessor implements StorageProcessor {
path = destTempl.getUuid();
}
if (path != null && !storagePoolMgr.connectPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), path, details)) {
logger.warn("Failed to connect physical disk at path: {}, in storage pool [id: {}, name: {}]", path, primaryStore.getUuid(), primaryStore.getName());
return new PrimaryStorageDownloadAnswer("Failed to spool template disk at path: " + path + ", in storage pool id: " + primaryStore.getUuid());
}
if (primaryPool.getType() == StoragePoolType.CLVM_NG) {
logger.info("Copying template {} to CLVM_NG pool {}",
destTempl.getUuid(), primaryPool.getUuid());
primaryVol = storagePoolMgr.copyPhysicalDisk(tmplVol, path != null ? path : destTempl.getUuid(), primaryPool, cmd.getWaitInMillSeconds());
try {
storagePoolMgr.createTemplateOnClvmNg(tmplVol.getPath(), path, cmd.getWaitInMillSeconds(), primaryPool);
primaryVol = primaryPool.getPhysicalDisk("template-" + path);
} catch (Exception e) {
logger.error("Failed to create CLVM_NG template: {}", e.getMessage(), e);
return new PrimaryStorageDownloadAnswer("Failed to create CLVM_NG template: " + e.getMessage());
}
} else {
if (path != null && !storagePoolMgr.connectPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), path, details)) {
logger.warn("Failed to connect physical disk at path: {}, in storage pool [id: {}, name: {}]", path, primaryStore.getUuid(), primaryStore.getName());
return new PrimaryStorageDownloadAnswer("Failed to spool template disk at path: " + path + ", in storage pool id: " + primaryStore.getUuid());
}
if (!storagePoolMgr.disconnectPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), path)) {
logger.warn("Failed to disconnect physical disk at path: {}, in storage pool [id: {}, name: {}]", path, primaryStore.getUuid(), primaryStore.getName());
primaryVol = storagePoolMgr.copyPhysicalDisk(tmplVol, path != null ? path : destTempl.getUuid(), primaryPool, cmd.getWaitInMillSeconds());
if (!storagePoolMgr.disconnectPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), path)) {
logger.warn("Failed to disconnect physical disk at path: {}, in storage pool [id: {}, name: {}]", path, primaryStore.getUuid(), primaryStore.getName());
}
}
} else {
primaryVol = storagePoolMgr.copyPhysicalDisk(tmplVol, UUID.randomUUID().toString(), primaryPool, cmd.getWaitInMillSeconds());

View File

@ -35,6 +35,8 @@ import java.util.stream.Collectors;
import com.cloud.agent.properties.AgentProperties;
import com.cloud.agent.properties.AgentPropertiesFileHandler;
import com.cloud.utils.script.OutputInterpreter;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.utils.cryptsetup.KeyFile;
import org.apache.cloudstack.utils.qemu.QemuImageOptions;
@ -48,6 +50,7 @@ import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.joda.time.Duration;
import org.libvirt.Connect;
import org.libvirt.LibvirtException;
import org.libvirt.Secret;
@ -668,10 +671,10 @@ public class LibvirtStorageAdaptor implements StorageAdaptor {
try {
StorageVol vol = getVolume(libvirtPool.getPool(), volumeUuid);
// Check if volume was found - if null, treat as not found and trigger fallback for CLVM
// Check if volume was found - if null, treat as not found and trigger fallback for CLVM/CLVM_NG
if (vol == null) {
logger.debug("Volume " + volumeUuid + " not found in libvirt, will check for CLVM fallback");
if (pool.getType() == StoragePoolType.CLVM) {
logger.debug("Volume " + volumeUuid + " not found in libvirt, will check for CLVM/CLVM_NG fallback");
if (pool.getType() == StoragePoolType.CLVM || pool.getType() == StoragePoolType.CLVM_NG) {
return getPhysicalDisk(volumeUuid, pool, libvirtPool);
}
@ -709,8 +712,8 @@ public class LibvirtStorageAdaptor implements StorageAdaptor {
return disk;
} catch (LibvirtException e) {
logger.debug("Failed to get volume from libvirt: " + e.getMessage());
// For CLVM, try direct block device access as fallback
if (pool.getType() == StoragePoolType.CLVM) {
// For CLVM/CLVM_NG, try direct block device access as fallback
if (pool.getType() == StoragePoolType.CLVM || pool.getType() == StoragePoolType.CLVM_NG) {
return getPhysicalDisk(volumeUuid, pool, libvirtPool);
}
@ -745,7 +748,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor {
return getPhysicalDiskViaDirectBlockDevice(volumeUuid, pool);
}
private String getVgName(KVMStoragePool pool, String sourceDir) {
private String getVgName(String sourceDir) {
String vgName = sourceDir;
if (vgName.startsWith("/")) {
String[] parts = vgName.split("/");
@ -771,7 +774,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor {
if (sourceDir == null || sourceDir.isEmpty()) {
throw new CloudRuntimeException("CLVM pool sourceDir is not set, cannot determine VG name");
}
String vgName = getVgName(pool, sourceDir);
String vgName = getVgName(sourceDir);
logger.debug("Using VG name: {} (from sourceDir: {}) ", vgName, sourceDir);
// Check if the LV exists in LVM using lvs command
@ -841,12 +844,23 @@ public class LibvirtStorageAdaptor implements StorageAdaptor {
}
KVMPhysicalDisk disk = new KVMPhysicalDisk(lvPath, volumeUuid, pool);
disk.setFormat(PhysicalDiskFormat.RAW);
// Detect correct format based on pool type
PhysicalDiskFormat diskFormat = PhysicalDiskFormat.RAW; // Default for legacy CLVM
if (pool.getType() == StoragePoolType.CLVM_NG) {
// CLVM_NG uses QCOW2 format on LVM block devices
diskFormat = PhysicalDiskFormat.QCOW2;
logger.debug("CLVM_NG pool detected, setting disk format to QCOW2 for volume {}", volumeUuid);
} else {
logger.debug("CLVM pool detected, setting disk format to RAW for volume {}", volumeUuid);
}
disk.setFormat(diskFormat);
disk.setSize(size);
disk.setVirtualSize(size);
logger.info("Successfully accessed CLVM volume via direct block device: {} " +
"with size: {} bytes",lvPath, size);
logger.info("Successfully accessed CLVM/CLVM_NG volume via direct block device: {} " +
"with format: {} and size: {} bytes", lvPath, diskFormat, size);
return disk;
@ -982,7 +996,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor {
try {
sp = createNetfsStoragePool(PoolType.GLUSTERFS, conn, name, host, path, null);
} catch (LibvirtException e) {
logger.error("Failed to create glusterfs mount: " + host + ":" + path , e);
logger.error("Failed to create glusterlvm_fs mount: " + host + ":" + path , e);
logger.error(e.getStackTrace());
throw new CloudRuntimeException(e.toString());
}
@ -990,7 +1004,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor {
sp = createSharedStoragePool(conn, name, host, path);
} else if (type == StoragePoolType.RBD) {
sp = createRBDStoragePool(conn, name, host, port, userInfo, path);
} else if (type == StoragePoolType.CLVM) {
} else if (type == StoragePoolType.CLVM || type == StoragePoolType.CLVM_NG) {
sp = createCLVMStoragePool(conn, name, host, path);
}
}
@ -1276,14 +1290,14 @@ public class LibvirtStorageAdaptor implements StorageAdaptor {
@Override
public boolean connectPhysicalDisk(String name, KVMStoragePool pool, Map<String, String> details, boolean isVMMigrate) {
// this is for managed storage that needs to prep disks prior to use
if (pool.getType() == StoragePoolType.CLVM && isVMMigrate) {
logger.info("Activating CLVM volume {} at location: {} in shared mode for VM migration", name, pool.getLocalPath() + File.separator + name);
if ((pool.getType() == StoragePoolType.CLVM || pool.getType() == StoragePoolType.CLVM_NG) && isVMMigrate) {
logger.info("Activating CLVM/CLVM_NG volume {} at location: {} in shared mode for VM migration", name, pool.getLocalPath() + File.separator + name);
Script activateVolInSharedMode = new Script("lvchange", 5000, logger);
activateVolInSharedMode.add("-asy");
activateVolInSharedMode.add(pool.getLocalPath() + File.separator + name);
String result = activateVolInSharedMode.execute();
if (result != null) {
logger.error("Failed to activate CLVM volume {} in shared mode for VM migration. Command output: {}", name, result);
logger.error("Failed to activate CLVM/CLVM_NG volume {} in shared mode for VM migration. Command output: {}", name, result);
return false;
}
}
@ -1395,9 +1409,9 @@ public class LibvirtStorageAdaptor implements StorageAdaptor {
}
}
// For CLVM pools, always use direct LVM cleanup to ensure secure zero-fill
if (pool.getType() == StoragePoolType.CLVM) {
logger.info("CLVM pool detected - using direct LVM cleanup with secure zero-fill for volume {}", uuid);
// For CLVM/CLVM_NG pools, always use direct LVM cleanup to ensure secure zero-fill
if (pool.getType() == StoragePoolType.CLVM || pool.getType() == StoragePoolType.CLVM_NG) {
logger.info("CLVM/CLVM_NG pool detected - using direct LVM cleanup with secure zero-fill for volume {}", uuid);
return cleanupCLVMVolume(uuid, pool);
}
@ -1441,7 +1455,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor {
logger.debug("Source directory is null or empty, cannot determine VG name for CLVM pool {}, skipping direct cleanup", pool.getUuid());
return true;
}
String vgName = getVgName(pool, sourceDir);
String vgName = getVgName(sourceDir);
logger.info("Determined VG name: {} for pool: {}", vgName, pool.getUuid());
if (vgName == null || vgName.isEmpty()) {
@ -1538,6 +1552,14 @@ public class LibvirtStorageAdaptor implements StorageAdaptor {
Script.runSimpleBashScript("chmod 755 " + disk.getPath());
Script.runSimpleBashScript("tar -x -f " + template.getPath() + "/*.tar -C " + disk.getPath(), timeout);
} else if (format == PhysicalDiskFormat.QCOW2) {
if (destPool.getType() == StoragePoolType.CLVM_NG) {
logger.info("Creating CLVM_NG volume {} with backing file from template {}", newUuid, template.getName());
String backingFile = getClvmBackingFile(template, destPool);
disk = createClvmNgDiskWithBacking(newUuid, timeout, size, backingFile, destPool);
return disk;
}
QemuImg qemu = new QemuImg(timeout);
QemuImgFile destFile = new QemuImgFile(disk.getPath(), format);
if (size > template.getVirtualSize()) {
@ -1600,6 +1622,92 @@ public class LibvirtStorageAdaptor implements StorageAdaptor {
return disk;
}
private String getClvmBackingFile(KVMPhysicalDisk template, KVMStoragePool destPool) {
String templateLvName = "template-" + template.getName();
KVMPhysicalDisk templateOnPrimary = null;
try {
templateOnPrimary = destPool.getPhysicalDisk(templateLvName);
} catch (CloudRuntimeException e) {
logger.warn("Template {} not found on CLVM_NG pool {}.", templateLvName, destPool.getUuid());
}
String backingFile;
if (templateOnPrimary != null) {
backingFile = templateOnPrimary.getPath();
logger.info("Using template on primary storage as backing file: {}", backingFile);
ensureTemplateLvInSharedMode(backingFile);
} else {
logger.error("Template {} should be on primary storage before creating volumes from it", templateLvName);
throw new CloudRuntimeException(String.format("Template not found on CLVM_NG primary storage: {}." +
"Template must be copied to primary storage first.", templateLvName));
}
return backingFile;
}
/**
* Ensures a template LV is activated in shared mode so multiple VMs can use it as a backing file.
*
* @param templatePath The full path to the template LV (e.g., /dev/vgname/template-uuid)
* @param throwOnFailure If true, throws CloudRuntimeException on failure; if false, logs warning and continues
*/
private void ensureTemplateLvInSharedMode(String templatePath, boolean throwOnFailure) {
try {
Script checkLvs = new Script("lvs", Duration.millis(5000), logger);
checkLvs.add("--noheadings");
checkLvs.add("-o", "lv_attr");
checkLvs.add(templatePath);
OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
String result = checkLvs.execute(parser);
if (result == null && parser.getLines() != null && !parser.getLines().isEmpty()) {
String lvAttr = parser.getLines().trim();
if (lvAttr.length() >= 6) {
char activeChar = lvAttr.charAt(4); // 'a' = active, '-' = inactive
char sharedChar = lvAttr.charAt(5); // 's' = shared, 'e' = exclusive, '-' = not set
if (activeChar != 'a' || sharedChar != 's') {
logger.info("Template LV {} is not in shared mode (attr: {}). Activating in shared mode.",
templatePath, lvAttr);
Script lvchange = new Script("lvchange", Duration.millis(5000), logger);
lvchange.add("-asy");
lvchange.add(templatePath);
result = lvchange.execute();
if (result != null) {
String errorMsg = "Failed to activate template LV " + templatePath + " in shared mode: " + result;
if (throwOnFailure) {
throw new CloudRuntimeException(errorMsg);
} else {
logger.warn(errorMsg);
}
} else {
logger.info("Successfully activated template LV {} in shared mode", templatePath);
}
} else {
logger.debug("Template LV {} is already in shared mode", templatePath);
}
}
}
} catch (CloudRuntimeException e) {
throw e;
} catch (Exception e) {
String errorMsg = "Failed to check/ensure template LV shared mode for " + templatePath + ": " + e.getMessage();
if (throwOnFailure) {
throw new CloudRuntimeException(errorMsg, e);
} else {
logger.warn(errorMsg, e);
}
}
}
private void ensureTemplateLvInSharedMode(String templatePath) {
ensureTemplateLvInSharedMode(templatePath, false);
}
private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template,
String name, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, KVMStoragePool destPool, int timeout){
@ -2071,4 +2179,197 @@ public class LibvirtStorageAdaptor implements StorageAdaptor {
private void deleteDirVol(LibvirtStoragePool pool, StorageVol vol) throws LibvirtException {
Script.runSimpleBashScript("rm -r --interactive=never " + vol.getPath());
}
/**
* Get Physical Extent (PE) from the volume group
* @param vgName Volume group name
* @return PE size in bytes, defauts to 4MiB if it cannot be determined
*/
private long getVgPhysicalExtentSize(String vgName) {
String warningMessage = String.format("Failed to get PE size for VG %s, defaulting to 4MiB", vgName);
try {
Script vgDisplay = new Script("vgdisplay", 300000, logger);
vgDisplay.add("--units", "b"); // Output in bytes
vgDisplay.add("-C"); // Columnar output
vgDisplay.add("--noheadings");
vgDisplay.add("-o", "vg_extent_size");
vgDisplay.add(vgName);
String output = vgDisplay.execute();
if (output != null) {
output = output.trim();
if (output.endsWith("B")) {
output = output.substring(0, output.length() - 1);
}
logger.debug("Physical Extent size for VG {} is {} bytes", vgName, output);
return Long.parseLong(output);
} else {
logger.warn(warningMessage);
}
} catch (Exception e) {
logger.warn(warningMessage, e.getMessage());
}
final long DEFAULT_PE_SIZE = 4 * 1024 * 1024L;
logger.info("Using default PE size for VG {}: {} bytes (4 MiB)", vgName, DEFAULT_PE_SIZE);
return DEFAULT_PE_SIZE;
}
/**
* Calculate LVM LV size for QCOW2 image accounting for metadata overhead
* @param qcow2PhysicalSize Physical size in bytes from qemu-img info
* @param vgName Volume group name to query PE size
* @return Size in bytes to allocate for LV
*/
private long calculateClvmNgLvSize(long qcow2PhysicalSize, String vgName) {
long peSize = getVgPhysicalExtentSize(vgName);
long roundedSize = ((qcow2PhysicalSize + peSize - 1) / peSize) * peSize;
long finalSize = roundedSize + peSize;
logger.info("Calculated LV size for QCOW2 physical size {} bytes: {} bytes " +
"(rounded to {} PEs + 1 PE overhead, PE size = {} bytes)",
qcow2PhysicalSize, finalSize,
roundedSize / peSize, peSize);
return finalSize;
}
/**
* Get physical size of QCOW2 image
*/
private long getQcow2PhysicalSize(String imagePath) {
Script qemuImg = new Script("qemu-img", 300000, logger);
qemuImg.add("info");
qemuImg.add("--output=json");
qemuImg.add(imagePath);
String output = qemuImg.execute();
JsonObject info = JsonParser.parseString(output).getAsJsonObject();
return info.get("actual-size").getAsLong();
}
private KVMPhysicalDisk createClvmNgDiskWithBacking(String volumeUuid, int timeout, long virtualSize, String backingFile, KVMStoragePool pool) {
String vgName = getVgName(pool.getLocalPath());
long lvSize = calculateClvmNgLvSize(virtualSize, vgName);
String volumePath = "/dev/" + vgName + "/" + volumeUuid;
logger.debug("Creating CLVM_NG volume {} with LV size {} bytes (virtual size: {} bytes)", volumeUuid, lvSize, virtualSize);
Script lvcreate = new Script("lvcreate", Duration.millis(timeout), logger);
lvcreate.add("-n", volumeUuid);
lvcreate.add("-L", lvSize + "B");
lvcreate.add(vgName);
String result = lvcreate.execute();
if (result != null) {
throw new CloudRuntimeException("Failed to create LV for CLVM_NG volume: " + result);
}
Script qemuImg = new Script("qemu-img", Duration.millis(timeout), logger);
qemuImg.add("create");
qemuImg.add("-f", "qcow2");
StringBuilder qcow2Options = new StringBuilder();
qcow2Options.append("preallocation=metadata");
qcow2Options.append(",extended_l2=on");
qcow2Options.append(",cluster_size=128k");
if (backingFile != null && !backingFile.isEmpty()) {
qcow2Options.append(",backing_file=").append(backingFile);
qcow2Options.append(",backing_fmt=qcow2");
logger.debug("Creating CLVM_NG volume with backing file: {}", backingFile);
}
qemuImg.add("-o", qcow2Options.toString());
qemuImg.add(volumePath);
qemuImg.add(virtualSize + "");
result = qemuImg.execute();
if (result != null) {
removeLvOnFailure(volumePath, timeout);
throw new CloudRuntimeException("Failed to create QCOW2 on CLVM_NG volume: " + result);
}
KVMPhysicalDisk disk = new KVMPhysicalDisk(volumePath, volumeUuid, pool);
disk.setFormat(PhysicalDiskFormat.QCOW2);
disk.setSize(lvSize);
disk.setVirtualSize(virtualSize);
logger.info("Successfully created CLVM_NG volume {} with backing file (LV size: {}, virtual size: {})",
volumeUuid, lvSize, virtualSize);
return disk;
}
public void createTemplateOnClvmNg(String templatePath, String templateUuid, int timeout, KVMStoragePool pool) {
String vgName = getVgName(pool.getLocalPath());
String lvName = "template-" + templateUuid;
String lvPath = "/dev/" + vgName + "/" + lvName;
if (lvExists(lvPath)) {
logger.info("Template LV {} already exists in VG {}. Skipping creation.", lvName, vgName);
return;
}
logger.info("Creating new template LV {} in VG {} for template {}", lvName, vgName, templateUuid);
long physicalSize = getQcow2PhysicalSize(templatePath);
long lvSize = calculateClvmNgLvSize(physicalSize, vgName);
Script lvcreate = new Script("lvcreate", Duration.millis(timeout), logger);
lvcreate.add("-n", lvName);
lvcreate.add("-L", lvSize + "B");
lvcreate.add(vgName);
String result = lvcreate.execute();
if (result != null) {
throw new CloudRuntimeException("Failed to create LV for CLVM_NG template: " + result);
}
Script qemuImgConvert = new Script("qemu-img", Duration.millis(timeout), logger);
qemuImgConvert.add("convert");
qemuImgConvert.add(templatePath);
qemuImgConvert.add("-O", "qcow2");
qemuImgConvert.add(lvPath);
result = qemuImgConvert.execute();
if (result != null) {
removeLvOnFailure(lvPath, timeout);
throw new CloudRuntimeException("Failed to convert template to CLVM_NG volume: " + result);
}
logger.info("Created template LV {} with size {} bytes (physical: {}, overhead: {})",
lvName, lvSize, physicalSize, lvSize - physicalSize);
try {
ensureTemplateLvInSharedMode(lvPath, true);
} catch (CloudRuntimeException e) {
logger.error("Failed to activate template LV {} in shared mode. Cleaning up.", lvPath);
removeLvOnFailure(lvPath, timeout);
throw e;
}
KVMPhysicalDisk templateDisk = new KVMPhysicalDisk(lvPath, lvName, pool);
templateDisk.setFormat(PhysicalDiskFormat.QCOW2);
templateDisk.setVirtualSize(physicalSize);
templateDisk.setSize(lvSize);
}
private boolean lvExists(String lvPath) {
Script checkLv = new Script("lvs", Duration.millis(5000), logger);
checkLv.add("--noheadings");
checkLv.add("--unbuffered");
checkLv.add(lvPath);
String checkResult = checkLv.execute();
return checkResult == null;
}
private void removeLvOnFailure(String lvPath, int timeout) {
Script lvremove = new Script("lvremove", Duration.millis(timeout), logger);
lvremove.add("-f");
lvremove.add(lvPath);
lvremove.execute();
}
}

View File

@ -212,7 +212,7 @@ public class LibvirtStoragePool implements KVMStoragePool {
@Override
public boolean isExternalSnapshot() {
if (this.type == StoragePoolType.CLVM || type == StoragePoolType.RBD) {
if (this.type == StoragePoolType.CLVM || this.type == StoragePoolType.CLVM_NG || type == StoragePoolType.RBD) {
return true;
}
return false;
@ -277,6 +277,10 @@ public class LibvirtStoragePool implements KVMStoragePool {
return this.type;
}
public void setType(StoragePoolType type) {
this.type = type;
}
public StoragePool getPool() {
return this._pool;
}

View File

@ -233,6 +233,11 @@ public class CloudStackPrimaryDataStoreLifeCycleImpl extends BasePrimaryDataStor
parameters.setHost(storageHost);
parameters.setPort(0);
parameters.setPath(hostPath.replaceFirst("/", ""));
} else if (scheme.equalsIgnoreCase("clvm_ng")) {
parameters.setType(StoragePoolType.CLVM_NG);
parameters.setHost(storageHost);
parameters.setPort(0);
parameters.setPath(hostPath.replaceFirst("/", ""));
} else if (scheme.equalsIgnoreCase("rbd")) {
if (port == -1) {
port = 0;

View File

@ -18,6 +18,7 @@
*/
package com.cloud.storage;
import java.util.Arrays;
import javax.inject.Inject;
import com.cloud.agent.AgentManager;
@ -46,6 +47,10 @@ public class ClvmLockManager {
protected Logger logger = LogManager.getLogger(getClass());
public static boolean isClvmPoolType(Storage.StoragePoolType poolType) {
return Arrays.asList(Storage.StoragePoolType.CLVM, Storage.StoragePoolType.CLVM_NG).contains(poolType);
}
public Long getClvmLockHostId(Long volumeId, String volumeUuid) {
VolumeDetailVO detail = _volsDetailsDao.findDetail(volumeId, VolumeInfo.CLVM_LOCK_HOST_ID);
if (detail != null && detail.getValue() != null && !detail.getValue().isEmpty()) {

View File

@ -436,6 +436,12 @@ public class VMSnapshotManagerImpl extends MutualExclusiveIdsManagerBase impleme
vmSnapshotType = VMSnapshot.Type.Disk;
}
// CLVM_NG: Block VM snapshots until Phase 2 implementation is complete
if (rootVolumePool.getPoolType() == Storage.StoragePoolType.CLVM_NG) {
throw new InvalidParameterValueException("VM snapshots are not yet supported on CLVM_NG storage pools. " +
"This feature will be available in a future release.");
}
try {
return createAndPersistVMSnapshot(userVmVo, vsDescription, vmSnapshotName, vsDisplayName, vmSnapshotType);
} catch (Exception e) {

View File

@ -383,7 +383,7 @@
<a-input v-model:value="form.radossecret" :placeholder="$t('label.rados.secret')" />
</a-form-item>
</div>
<div v-if="form.protocol === 'CLVM'">
<div v-if="form.protocol === 'CLVM' || form.protocol === 'CLVM_NG'">
<a-form-item name="volumegroup" ref="volumegroup" :label="$t('label.volumegroup')">
<a-input v-model:value="form.volumegroup" :placeholder="$t('label.volumegroup')" />
</a-form-item>
@ -607,7 +607,7 @@ export default {
const cluster = this.clusters.find(cluster => cluster.id === this.form.cluster)
this.hypervisorType = cluster.hypervisortype
if (this.hypervisorType === 'KVM') {
this.protocols = ['nfs', 'SharedMountPoint', 'RBD', 'CLVM', 'Gluster', 'Linstor', 'custom', 'FiberChannel']
this.protocols = ['nfs', 'SharedMountPoint', 'RBD', 'CLVM', 'CLVM_NG', 'Gluster', 'Linstor', 'custom', 'FiberChannel']
if (this.form.scope === 'host') {
this.protocols.push('Filesystem')
}
@ -729,6 +729,15 @@ export default {
}
return url
},
clvmNgURL (vgname) {
var url
if (vgname.indexOf('://') === -1) {
url = 'clvm_ng://localhost/' + vgname
} else {
url = vgname
}
return url
},
vmfsURL (server, path) {
var url
if (server.indexOf('://') === -1) {
@ -853,6 +862,9 @@ export default {
} else if (values.protocol === 'CLVM') {
var vg = (values.volumegroup.substring(0, 1) !== '/') ? ('/' + values.volumegroup) : values.volumegroup
url = this.clvmURL(vg)
} else if (values.protocol === 'CLVM_NG') {
var vg = (values.volumegroup.substring(0, 1) !== '/') ? ('/' + values.volumegroup) : values.volumegroup
url = this.clvmNgURL(vg)
} else if (values.protocol === 'RBD') {
url = this.rbdURL(values.radosmonitor, values.radospool, values.radosuser, values.radossecret)
if (values.datapool) {