etan39
/
cloudstack
mirror of https://github.com/apache/cloudstack.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Recent changes of migration across clusters

This commit is contained in:
Harikrishna Patnala 2023-02-22 11:04:19 +05:30
parent c49b37f4a5
commit 61b451a20c
3 changed files with 188 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

121
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java
View File

@ -23,19 +23,35 @@ import com.cloud.agent.api.Answer;
import com.cloud.agent.api.storage.MigrateVolumeAnswer;
import com.cloud.agent.api.storage.MigrateVolumeCommand;
import com.cloud.agent.api.to.DiskTO;
import com.cloud.exception.InternalErrorException;
import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource;
import com.cloud.hypervisor.kvm.resource.LibvirtDomainXMLParser;
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef;
import com.cloud.hypervisor.kvm.storage.KVMPhysicalDisk;
import com.cloud.hypervisor.kvm.storage.KVMStoragePool;
import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager;
import com.cloud.resource.CommandWrapper;
import com.cloud.resource.ResourceWrapper;
import java.io.File;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import com.cloud.storage.Storage;
import org.apache.cloudstack.storage.datastore.client.ScaleIOGatewayClient;
import org.apache.cloudstack.storage.datastore.util.ScaleIOUtil;
import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
import org.apache.cloudstack.storage.to.VolumeObjectTO;
import org.apache.log4j.Logger;
import org.libvirt.Connect;
import org.libvirt.Domain;
import org.libvirt.DomainInfo;
import org.libvirt.TypedParameter;
import org.libvirt.LibvirtException;
import org.libvirt.event.BlockJobListener;
import org.libvirt.event.BlockJobStatus;
import org.libvirt.event.BlockJobType;
@ResourceWrapper(handles = MigrateVolumeCommand.class)
public final class LibvirtMigrateVolumeCommandWrapper extends CommandWrapper<MigrateVolumeCommand, Answer, LibvirtComputingResource> {
@ -44,13 +60,116 @@ public final class LibvirtMigrateVolumeCommandWrapper extends CommandWrapper<Mig
@Override
public Answer execute(final MigrateVolumeCommand command, final LibvirtComputingResource libvirtComputingResource) {
LOGGER.info("I'm here HARIIIII");
VolumeObjectTO srcVolumeObjectTO = (VolumeObjectTO)command.getSrcData();
PrimaryDataStoreTO srcPrimaryDataStore = (PrimaryDataStoreTO)srcVolumeObjectTO.getDataStore();
MigrateVolumeAnswer answer;
if (srcPrimaryDataStore.getPoolType().equals(Storage.StoragePoolType.PowerFlex)) {
answer = migrateVolumeInternal(command, libvirtComputingResource);
} else {
answer = migrateRegularVolume(command, libvirtComputingResource);
}
return answer;
}
private MigrateVolumeAnswer migrateVolumeInternal (final MigrateVolumeCommand command, final LibvirtComputingResource libvirtComputingResource) {
VolumeObjectTO srcVolumeObjectTO = (VolumeObjectTO)command.getSrcData();
PrimaryDataStoreTO srcPrimaryDataStore = (PrimaryDataStoreTO)srcVolumeObjectTO.getDataStore();
final String vmName = srcVolumeObjectTO.getVmName();
LOGGER.info("HARI VM name: "+ vmName);
VolumeObjectTO destVolumeObjectTO = (VolumeObjectTO)command.getDestData();
PrimaryDataStoreTO destPrimaryDataStore = (PrimaryDataStoreTO)destVolumeObjectTO.getDataStore();
String srcPath = srcVolumeObjectTO.getPath();
String destPath = destVolumeObjectTO.getPath();
Map<String, String> destDetails = command.getDestDetails();
final String srcVolumeId = ScaleIOUtil.getVolumePath(srcVolumeObjectTO.getPath());
LOGGER.info("HARI Source volume ID: "+ srcVolumeId);
final String destVolumeId = ScaleIOUtil.getVolumePath(destVolumeObjectTO.getPath());
LOGGER.info("HARI destination volume ID: "+ destVolumeId);
final String destSystemId = destDetails.get(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID);
LOGGER.info("HARI destination system ID: "+ destSystemId);
final String destDiskFileName = ScaleIOUtil.DISK_NAME_PREFIX + destSystemId + "-" + destVolumeId;
final String diskFilePath = ScaleIOUtil.DISK_PATH + File.separator + destDiskFileName;
Domain dm = null;
try {
final LibvirtUtilitiesHelper libvirtUtilitiesHelper = libvirtComputingResource.getLibvirtUtilitiesHelper();
Connect conn = libvirtUtilitiesHelper.getConnection();
dm = libvirtComputingResource.getDomain(conn, vmName);
if (dm == null) {
return new MigrateVolumeAnswer(command, false,
"Migrate volume failed due to can not find vm: " + vmName, null);
}
DomainInfo.DomainState domainState = dm.getInfo().state ;
if (domainState != DomainInfo.DomainState.VIR_DOMAIN_RUNNING) {
return new MigrateVolumeAnswer(command, false,
"Migrate volume failed due to VM is not running: " + vmName + " with domainState = " + domainState, null);
}
final LibvirtDomainXMLParser parser = new LibvirtDomainXMLParser();
final String domXml = dm.getXMLDesc(0);
parser.parseDomainXML(domXml);
LOGGER.info(String.format("VM [%s] with XML configuration [%s] will be migrated to host.", vmName, domXml));
List<LibvirtVMDef.DiskDef> disks = parser.getDisks();
LibvirtVMDef.DiskDef diskdef = null;
for (final LibvirtVMDef.DiskDef disk : disks) {
final String file = disk.getDiskPath();
LOGGER.info("HARIIII : " + file);
if (file != null && file.contains(srcVolumeId)) {
diskdef = disk;
break;
}
}
if (diskdef == null) {
throw new InternalErrorException("disk: " + srcPath + " is not attached before");
}
diskdef.setDiskPath(diskFilePath);
LOGGER.info("HARIIII Destination xml : " + diskdef.toString());
dm.blockCopy(srcPath, diskdef.toString(), new TypedParameter[]{}, 0);
BlockJobListener listener = new BlockJobListener() {
@Override
public void onEvent(Domain domain, String diskPath, BlockJobType type, BlockJobStatus status) {
}
};
return new MigrateVolumeAnswer(command, true, null, destPath);
} catch (LibvirtException e) {
String msg = "Migrate volume failed due to " + e.toString();
LOGGER.warn(msg, e);
return new MigrateVolumeAnswer(command, false, msg, null);
} catch (InternalErrorException e) {
throw new RuntimeException(e);
} finally {
if (dm != null) {
try {
dm.free();
} catch (LibvirtException l) {
LOGGER.trace("Ignoring libvirt error.", l);
};
}
}
}
private MigrateVolumeAnswer migrateRegularVolume(final MigrateVolumeCommand command, final LibvirtComputingResource libvirtComputingResource) {
KVMStoragePoolManager storagePoolManager = libvirtComputingResource.getStoragePoolMgr();
VolumeObjectTO srcVolumeObjectTO = (VolumeObjectTO)command.getSrcData();
PrimaryDataStoreTO srcPrimaryDataStore = (PrimaryDataStoreTO)srcVolumeObjectTO.getDataStore();
Map<String, String> srcDetails = command.getSrcDetails();
String srcPath = srcDetails != null ? srcDetails.get(DiskTO.IQN) : srcVolumeObjectTO.getPath();
VolumeObjectTO destVolumeObjectTO = (VolumeObjectTO)command.getDestData();

2
plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClientImpl.java
View File

@ -761,7 +761,7 @@ public class ScaleIOGatewayClientImpl implements ScaleIOGatewayClient {
}
String srcPoolId = volume.getStoragePoolId();
LOG.debug("Migrating the volume: " + srcVolumeId + " on the src pool: " + srcPoolId + " to the dest pool: " + destPoolId +
LOG.info("Migrating the volume: " + srcVolumeId + " on the src pool: " + srcPoolId + " to the dest pool: " + destPoolId +
" in the same PowerFlex cluster");
post("/instances/Volume::" + srcVolumeId + "/action/migrateVTree",

68
plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java
View File

@ -19,6 +19,7 @@ package org.apache.cloudstack.storage.datastore.driver;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import javax.inject.Inject;
@ -42,6 +43,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver
import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo;
import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService;
import org.apache.cloudstack.framework.async.AsyncCompletionCallback;
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.apache.cloudstack.storage.RemoteHostEndPoint;
@ -71,6 +73,7 @@ import com.cloud.agent.api.Answer;
import com.cloud.agent.api.to.DataObjectType;
import com.cloud.agent.api.to.DataStoreTO;
import com.cloud.agent.api.to.DataTO;
import com.cloud.agent.api.to.DiskTO;
import com.cloud.alert.AlertManager;
import com.cloud.configuration.Config;
import com.cloud.host.Host;
@ -127,6 +130,8 @@ public class ScaleIOPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
private HostDao hostDao;
@Inject
private VMInstanceDao vmInstanceDao;
@Inject
private VolumeService volumeService;
public ScaleIOPrimaryDataStoreDriver() {
@ -768,10 +773,23 @@ public class ScaleIOPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
final String srcVolumeId = ScaleIOUtil.getVolumePath(srcVolumePath);
final StoragePoolVO destStoragePool = storagePoolDao.findById(destPoolId);
final String destStoragePoolId = destStoragePool.getPath();
//CreateObjectAnswer createAnswer = createVolume((VolumeInfo) destData, destStore.getId());
//String destVolumePath = createAnswer.getData().getPath();
final String destVolumeId = UUID.randomUUID().toString();
final String destScaleIOVolumeName = String.format("%s-%s-%s-%s", ScaleIOUtil.VOLUME_PREFIX, srcData.getId(),
srcData.getUuid().split("-")[0].substring(4), ManagementServerImpl.customCsIdentifier.value());
String destVolumePath = String.format("%s:%s", destVolumeId, destScaleIOVolumeName);
VolumeObjectTO destVolTO = (VolumeObjectTO) destData.getTO();
destVolTO.setPath(destVolumePath);
Map<String, String> srcDetails = getVolumeDetails((VolumeInfo) srcData);
Map<String, String> destDetails = getVolumeDetails((VolumeInfo) destData);
String value = configDao.getValue(Config.MigrateWait.key());
int waitInterval = NumbersUtil.parseInt(value, Integer.parseInt(Config.MigrateWait.getDefaultValue()));
MigrateVolumeCommand migrateVolumeCommand = new MigrateVolumeCommand(srcData.getId(), srcVolumePath, destStoragePool, ((VolumeInfo) srcData).getAttachedVmName(), ((VolumeInfo) srcData).getVolumeType(), waitInterval);
MigrateVolumeCommand migrateVolumeCommand = new MigrateVolumeCommand(srcData.getTO(), destVolTO,
srcDetails, destDetails, waitInterval);
long hostId = 0;
VMInstanceVO instance = vmInstanceDao.findVMByInstanceName(((VolumeInfo) srcData).getAttachedVmName());
@ -860,6 +878,52 @@ public class ScaleIOPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
return answer;
}
private Map<String, String> getVolumeDetails(VolumeInfo volumeInfo) {
long storagePoolId = volumeInfo.getPoolId();
StoragePoolVO storagePoolVO = storagePoolDao.findById(storagePoolId);
if (!storagePoolVO.isManaged()) {
return null;
}
Map<String, String> volumeDetails = new HashMap<>();
VolumeVO volumeVO = volumeDao.findById(volumeInfo.getId());
volumeDetails.put(DiskTO.STORAGE_HOST, storagePoolVO.getHostAddress());
volumeDetails.put(DiskTO.STORAGE_PORT, String.valueOf(storagePoolVO.getPort()));
volumeDetails.put(DiskTO.IQN, volumeVO.get_iScsiName());
volumeDetails.put(DiskTO.PROTOCOL_TYPE, (volumeVO.getPoolType() != null) ? volumeVO.getPoolType().toString() : null);
volumeDetails.put(StorageManager.STORAGE_POOL_DISK_WAIT.toString(), String.valueOf(StorageManager.STORAGE_POOL_DISK_WAIT.valueIn(storagePoolVO.getId())));
volumeDetails.put(DiskTO.VOLUME_SIZE, String.valueOf(volumeVO.getSize()));
volumeDetails.put(DiskTO.SCSI_NAA_DEVICE_ID, getVolumeProperty(volumeInfo.getId(), DiskTO.SCSI_NAA_DEVICE_ID));
ChapInfo chapInfo = volumeService.getChapInfo(volumeInfo, volumeInfo.getDataStore());
if (chapInfo != null) {
volumeDetails.put(DiskTO.CHAP_INITIATOR_USERNAME, chapInfo.getInitiatorUsername());
volumeDetails.put(DiskTO.CHAP_INITIATOR_SECRET, chapInfo.getInitiatorSecret());
volumeDetails.put(DiskTO.CHAP_TARGET_USERNAME, chapInfo.getTargetUsername());
volumeDetails.put(DiskTO.CHAP_TARGET_SECRET, chapInfo.getTargetSecret());
}
String systemId = storagePoolDetailsDao.findDetail(storagePoolId, ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID).getValue();
volumeDetails.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID, systemId);
return volumeDetails;
}
private String getVolumeProperty(long volumeId, String property) {
VolumeDetailVO volumeDetails = volumeDetailsDao.findDetail(volumeId, property);
if (volumeDetails != null) {
return volumeDetails.getValue();
}
return null;
}
private Answer migrateVolume(DataObject srcData, DataObject destData) {
// Volume migration within same PowerFlex/ScaleIO cluster (with same System ID)
DataStore srcStore = srcData.getDataStore();
@ -875,6 +939,8 @@ public class ScaleIOPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
final StoragePoolVO destStoragePool = storagePoolDao.findById(destPoolId);
final String destStoragePoolId = destStoragePool.getPath();
int migrationTimeout = StorageManager.KvmStorageOfflineMigrationWait.value();
LOGGER.info("HARI source volume " + srcVolumeId);
LOGGER.info("HARI destination volume " + destStoragePoolId);
boolean migrateStatus = client.migrateVolume(srcVolumeId, destStoragePoolId, migrationTimeout);
if (migrateStatus) {
String newVolumeName = String.format("%s-%s-%s-%s", ScaleIOUtil.VOLUME_PREFIX, destData.getId(),