This commit is contained in:
Harikrishna 2026-01-22 09:59:55 +00:00 committed by GitHub
commit c4f37b9319
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
14 changed files with 509 additions and 54 deletions

View File

@ -29,6 +29,11 @@ import org.apache.cloudstack.api.response.ZoneResponse;
import com.cloud.exception.DiscoveryException;
import com.cloud.storage.ImageStore;
import com.cloud.user.Account;
import org.apache.commons.collections.MapUtils;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
@APICommand(name = "addSecondaryStorage", description = "Adds secondary storage.", responseObject = ImageStoreResponse.class,
requestHasSensitiveInfo = false, responseHasSensitiveInfo = false)
@ -44,6 +49,9 @@ public class AddSecondaryStorageCmd extends BaseCmd {
@Parameter(name = ApiConstants.ZONE_ID, type = CommandType.UUID, entityType = ZoneResponse.class, description = "The Zone ID for the secondary storage")
protected Long zoneId;
@Parameter(name = ApiConstants.DETAILS, type = CommandType.MAP, description = "Details in key/value pairs using format details[i].keyname=keyvalue. Example: details[0].copytemplatesfromothersecondarystorages=true")
protected Map details;
/////////////////////////////////////////////////////
/////////////////// Accessors ///////////////////////
/////////////////////////////////////////////////////
@ -56,6 +64,20 @@ public class AddSecondaryStorageCmd extends BaseCmd {
return zoneId;
}
public Map<String, String> getDetails() {
Map<String, String> detailsMap = new HashMap<>();
if (MapUtils.isNotEmpty(details)) {
Collection<?> props = details.values();
for (Object prop : props) {
HashMap<String, String> detail = (HashMap<String, String>) prop;
for (Map.Entry<String, String> entry: detail.entrySet()) {
detailsMap.put(entry.getKey(),entry.getValue());
}
}
}
return detailsMap;
}
/////////////////////////////////////////////////////
/////////////// API Implementation///////////////////
/////////////////////////////////////////////////////
@ -68,7 +90,7 @@ public class AddSecondaryStorageCmd extends BaseCmd {
@Override
public void execute(){
try{
ImageStore result = _storageService.discoverImageStore(null, getUrl(), "NFS", getZoneId(), null);
ImageStore result = _storageService.discoverImageStore(null, getUrl(), "NFS", getZoneId(), getDetails());
ImageStoreResponse storeResponse = null;
if (result != null ) {
storeResponse = _responseGenerator.createImageStoreResponse(result);

View File

@ -22,7 +22,6 @@ import java.util.concurrent.Future;
import org.apache.cloudstack.api.response.MigrationResponse;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo;
import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService.TemplateApiResult;
import org.apache.cloudstack.storage.ImageStoreService.MigrationPolicy;
@ -31,5 +30,5 @@ public interface StorageOrchestrationService {
MigrationResponse migrateResources(Long srcImgStoreId, Long destImgStoreId, List<Long> templateIdList, List<Long> snapshotIdList);
Future<TemplateApiResult> orchestrateTemplateCopyToImageStore(TemplateInfo source, DataStore destStore);
Future<TemplateApiResult> orchestrateTemplateCopyFromSecondaryStores(long templateId, DataStore destStore);
}

View File

@ -80,4 +80,6 @@ public interface TemplateService {
List<DatadiskTO> getTemplateDatadisksOnImageStore(TemplateInfo templateInfo, String configurationId);
AsyncCallFuture<TemplateApiResult> copyTemplateToImageStore(DataObject source, DataStore destStore);
}
void handleTemplateCopyFromSecondaryStores(long templateId, DataStore destStore);
}

View File

@ -220,8 +220,9 @@ public interface StorageManager extends StorageService {
"storage.pool.host.connect.workers", "1",
"Number of worker threads to be used to connect hosts to a primary storage", true);
ConfigKey<Boolean> COPY_PUBLIC_TEMPLATES_FROM_OTHER_STORAGES = new ConfigKey<>(Boolean.class, "copy.public.templates.from.other.storages",
"Storage", "true", "Allow SSVMs to try copying public templates from one secondary storage to another instead of downloading them from the source.",
ConfigKey<Boolean> COPY_TEMPLATES_FROM_OTHER_SECONDARY_STORAGES = new ConfigKey<>(Boolean.class, "copy.templates.from.other.secondary.storages",
"Storage", "true", "When enabled, this feature allows templates to be copied from existing Secondary Storage servers (within the same zone or across zones) " +
"while adding a new Secondary Storage. If the copy operation fails, the system falls back to downloading the template from the source URL.",
true, ConfigKey.Scope.Zone, null);
/**

View File

@ -36,6 +36,9 @@ import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
import com.cloud.dc.dao.DataCenterDao;
import com.cloud.storage.dao.VMTemplateDao;
import com.cloud.template.TemplateManager;
import org.apache.cloudstack.api.response.MigrationResponse;
import org.apache.cloudstack.engine.orchestration.service.StorageOrchestrationService;
import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
@ -45,6 +48,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.SecondaryStorageServic
import org.apache.cloudstack.engine.subsystem.api.storage.SecondaryStorageService.DataObjectResult;
import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory;
import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory;
import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo;
import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService;
import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService.TemplateApiResult;
@ -103,6 +107,15 @@ public class StorageOrchestrator extends ManagerBase implements StorageOrchestra
VolumeDataStoreDao volumeDataStoreDao;
@Inject
DataMigrationUtility migrationHelper;
@Inject
TemplateManager templateManager;
@Inject
VMTemplateDao templateDao;
@Inject
TemplateDataFactory templateDataFactory;
@Inject
DataCenterDao dcDao;
ConfigKey<Double> ImageStoreImbalanceThreshold = new ConfigKey<>("Advanced", Double.class,
"image.store.imbalance.threshold",
@ -304,8 +317,9 @@ public class StorageOrchestrator extends ManagerBase implements StorageOrchestra
}
@Override
public Future<TemplateApiResult> orchestrateTemplateCopyToImageStore(TemplateInfo source, DataStore destStore) {
return submit(destStore.getScope().getScopeId(), new CopyTemplateTask(source, destStore));
public Future<TemplateApiResult> orchestrateTemplateCopyFromSecondaryStores(long srcTemplateId, DataStore destStore) {
Long dstZoneId = destStore.getScope().getScopeId();
return submit(dstZoneId, new CopyTemplateFromSecondaryStorageTask(srcTemplateId, destStore));
}
protected Pair<String, Boolean> migrateCompleted(Long destDatastoreId, DataStore srcDatastore, List<DataObject> files, MigrationPolicy migrationPolicy, int skipped) {
@ -653,4 +667,33 @@ public class StorageOrchestrator extends ManagerBase implements StorageOrchestra
return result;
}
}
private class CopyTemplateFromSecondaryStorageTask implements Callable<TemplateApiResult> {
private final long srcTemplateId;
private final DataStore destStore;
private final String logid;
CopyTemplateFromSecondaryStorageTask(long srcTemplateId, DataStore destStore) {
this.srcTemplateId = srcTemplateId;
this.destStore = destStore;
this.logid = ThreadContext.get(LOGCONTEXTID);
}
@Override
public TemplateApiResult call() {
ThreadContext.put(LOGCONTEXTID, logid);
TemplateApiResult result;
long destZoneId = destStore.getScope().getScopeId();
TemplateInfo sourceTmpl = templateDataFactory.getTemplate(srcTemplateId, DataStoreRole.Image);
try {
templateService.handleTemplateCopyFromSecondaryStores(srcTemplateId, destStore);
result = new TemplateApiResult(sourceTmpl);
} finally {
tryCleaningUpExecutor(destZoneId);
ThreadContext.clearAll();
}
return result;
}
}
}

View File

@ -31,6 +31,8 @@ import java.util.concurrent.ExecutionException;
import javax.inject.Inject;
import com.cloud.exception.StorageUnavailableException;
import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.engine.orchestration.service.StorageOrchestrationService;
import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult;
import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult;
@ -70,6 +72,7 @@ import org.apache.cloudstack.storage.to.TemplateObjectTO;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.ThreadContext;
import org.springframework.stereotype.Component;
import com.cloud.agent.api.Answer;
@ -548,10 +551,7 @@ public class TemplateServiceImpl implements TemplateService {
}
if (availHypers.contains(tmplt.getHypervisorType())) {
boolean copied = isCopyFromOtherStoragesEnabled(zoneId) && tryCopyingTemplateToImageStore(tmplt, store);
if (!copied) {
tryDownloadingTemplateToImageStore(tmplt, store);
}
storageOrchestrator.orchestrateTemplateCopyFromSecondaryStores(tmplt.getId(), store);
} else {
logger.info("Skip downloading template {} since current data center does not have hypervisor {}", tmplt, tmplt.getHypervisorType());
}
@ -598,6 +598,16 @@ public class TemplateServiceImpl implements TemplateService {
}
@Override
public void handleTemplateCopyFromSecondaryStores(long templateId, DataStore destStore) {
VMTemplateVO template = _templateDao.findById(templateId);
long zoneId = destStore.getScope().getScopeId();
boolean copied = imageStoreDetailsUtil.isCopyTemplatesFromOtherStoragesEnabled(destStore.getId(), zoneId) && tryCopyingTemplateToImageStore(template, destStore);
if (!copied) {
tryDownloadingTemplateToImageStore(template, destStore);
}
}
protected void tryDownloadingTemplateToImageStore(VMTemplateVO tmplt, DataStore destStore) {
if (tmplt.getUrl() == null) {
logger.info("Not downloading template [{}] to image store [{}], as it has no URL.", tmplt.getUniqueName(),
@ -615,28 +625,134 @@ public class TemplateServiceImpl implements TemplateService {
}
protected boolean tryCopyingTemplateToImageStore(VMTemplateVO tmplt, DataStore destStore) {
Long zoneId = destStore.getScope().getScopeId();
List<DataStore> storesInZone = _storeMgr.getImageStoresByZoneIds(zoneId);
for (DataStore sourceStore : storesInZone) {
Map<String, TemplateProp> existingTemplatesInSourceStore = listTemplate(sourceStore);
if (existingTemplatesInSourceStore == null || !existingTemplatesInSourceStore.containsKey(tmplt.getUniqueName())) {
logger.debug("Template [{}] does not exist on image store [{}]; searching on another one.",
tmplt.getUniqueName(), sourceStore.getName());
continue;
}
TemplateObject sourceTmpl = (TemplateObject) _templateFactory.getTemplate(tmplt.getId(), sourceStore);
if (sourceTmpl.getInstallPath() == null) {
logger.warn("Can not copy template [{}] from image store [{}], as it returned a null install path.", tmplt.getUniqueName(),
sourceStore.getName());
continue;
}
storageOrchestrator.orchestrateTemplateCopyToImageStore(sourceTmpl, destStore);
if (searchAndCopyWithinZone(tmplt, destStore)) {
return true;
}
logger.debug("Can't copy template [{}] from another image store.", tmplt.getUniqueName());
Long destZoneId = destStore.getScope().getScopeId();
logger.debug("Template [{}] not found in any image store of zone [{}]. Checking other zones.",
tmplt.getUniqueName(), destZoneId);
return searchAndCopyAcrossZones(tmplt, destStore, destZoneId);
}
private boolean searchAndCopyAcrossZones(VMTemplateVO tmplt, DataStore destStore, Long destZoneId) {
List<Long> allZoneIds = _dcDao.listAllIds();
for (Long otherZoneId : allZoneIds) {
if (otherZoneId.equals(destZoneId)) {
continue;
}
List<DataStore> storesInOtherZone = _storeMgr.getImageStoresByZoneIds(otherZoneId);
logger.debug("Checking zone [{}] for template [{}]...", otherZoneId, tmplt.getUniqueName());
if (storesInOtherZone == null || storesInOtherZone.isEmpty()) {
logger.debug("Zone [{}] has no image stores. Skipping.", otherZoneId);
continue;
}
TemplateObject sourceTmpl = findUsableTemplate(tmplt, storesInOtherZone);
if (sourceTmpl == null) {
logger.debug("Template [{}] not found with a valid install path in any image store of zone [{}].",
tmplt.getUniqueName(), otherZoneId);
continue;
}
logger.info("Template [{}] found in zone [{}]. Initiating cross-zone copy to zone [{}].",
tmplt.getUniqueName(), otherZoneId, destZoneId);
return copyTemplateAcrossZones(destStore, sourceTmpl);
}
logger.debug("Template [{}] was not found in any zone. Cannot perform zone-to-zone copy.", tmplt.getUniqueName());
return false;
}
protected TemplateObject findUsableTemplate(VMTemplateVO tmplt, List<DataStore> imageStores) {
for (DataStore store : imageStores) {
Map<String, TemplateProp> templates = listTemplate(store);
if (templates == null || !templates.containsKey(tmplt.getUniqueName())) {
continue;
}
TemplateObject tmpl = (TemplateObject) _templateFactory.getTemplate(tmplt.getId(), store);
if (tmpl.getInstallPath() == null) {
logger.debug("Template [{}] found in image store [{}] but install path is null. Skipping.",
tmplt.getUniqueName(), store.getName());
continue;
}
return tmpl;
}
return null;
}
private boolean searchAndCopyWithinZone(VMTemplateVO tmplt, DataStore destStore) {
Long destZoneId = destStore.getScope().getScopeId();
List<DataStore> storesInSameZone = _storeMgr.getImageStoresByZoneIds(destZoneId);
TemplateObject sourceTmpl = findUsableTemplate(tmplt, storesInSameZone);
if (sourceTmpl == null) {
return false;
}
TemplateApiResult result;
AsyncCallFuture<TemplateApiResult> future = copyTemplateToImageStore(sourceTmpl, destStore);
try {
result = future.get();
} catch (ExecutionException | InterruptedException e) {
logger.warn("Exception while copying template [{}] from image store [{}] to image store [{}]: {}",
sourceTmpl.getUniqueName(), sourceTmpl.getDataStore().getName(), destStore.getName(), e.toString());
result = new TemplateApiResult(sourceTmpl);
result.setResult(e.getMessage());
}
return result.isSuccess();
}
private boolean copyTemplateAcrossZones(DataStore destStore, TemplateObject sourceTmpl) {
Long dstZoneId = destStore.getScope().getScopeId();
DataCenterVO dstZone = _dcDao.findById(dstZoneId);
if (dstZone == null) {
logger.warn("Destination zone [{}] not found for template [{}].", dstZoneId, sourceTmpl.getUniqueName());
return false;
}
TemplateApiResult result;
try {
VMTemplateVO template = _templateDao.findById(sourceTmpl.getId());
try {
DataStore sourceStore = sourceTmpl.getDataStore();
long userId = CallContext.current().getCallingUserId();
boolean success = _tmpltMgr.copy(userId, template, sourceStore, dstZone);
result = new TemplateApiResult(sourceTmpl);
if (!success) {
result.setResult("Cross-zone template copy failed");
}
} catch (StorageUnavailableException | ResourceAllocationException e) {
logger.error("Exception while copying template [{}] from zone [{}] to zone [{}]",
template,
sourceTmpl.getDataStore().getScope().getScopeId(),
dstZone.getId(),
e);
result = new TemplateApiResult(sourceTmpl);
result.setResult(e.getMessage());
} finally {
ThreadContext.clearAll();
}
} catch (Exception e) {
logger.error("Failed to copy template [{}] from zone [{}] to zone [{}].",
sourceTmpl.getUniqueName(),
sourceTmpl.getDataStore().getScope().getScopeId(),
dstZoneId,
e);
return false;
}
return result.isSuccess();
}
@Override
public AsyncCallFuture<TemplateApiResult> copyTemplateToImageStore(DataObject source, DataStore destStore) {
TemplateObject sourceTmpl = (TemplateObject) source;
@ -680,10 +796,6 @@ public class TemplateServiceImpl implements TemplateService {
return null;
}
protected boolean isCopyFromOtherStoragesEnabled(Long zoneId) {
return StorageManager.COPY_PUBLIC_TEMPLATES_FROM_OTHER_STORAGES.valueIn(zoneId);
}
protected void publishTemplateCreation(TemplateInfo tmplt) {
VMTemplateVO tmpltVo = _templateDao.findById(tmplt.getId());

View File

@ -18,12 +18,20 @@
*/
package org.apache.cloudstack.storage.image;
import com.cloud.dc.DataCenterVO;
import com.cloud.dc.dao.DataCenterDao;
import com.cloud.exception.ResourceAllocationException;
import com.cloud.exception.StorageUnavailableException;
import com.cloud.storage.dao.VMTemplateDao;
import com.cloud.storage.template.TemplateProp;
import com.cloud.template.TemplateManager;
import com.cloud.user.Account;
import com.cloud.user.User;
import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.engine.orchestration.service.StorageOrchestrationService;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
import org.apache.cloudstack.engine.subsystem.api.storage.Scope;
import org.apache.cloudstack.framework.async.AsyncCallFuture;
import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO;
import org.apache.cloudstack.storage.image.store.TemplateObject;
@ -45,6 +53,8 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.mockito.Mockito.mock;
@RunWith(MockitoJUnitRunner.class)
public class TemplateServiceImplTest {
@ -82,6 +92,15 @@ public class TemplateServiceImplTest {
@Mock
StorageOrchestrationService storageOrchestrator;
@Mock
DataCenterDao _dcDao;
@Mock
VMTemplateDao templateDao;
@Mock
TemplateManager _tmpltMgr;
Map<String, TemplateProp> templatesInSourceStore = new HashMap<>();
@Before
@ -94,7 +113,6 @@ public class TemplateServiceImplTest {
Mockito.doReturn(List.of(sourceStoreMock, destStoreMock)).when(dataStoreManagerMock).getImageStoresByZoneIds(zoneId);
Mockito.doReturn(templatesInSourceStore).when(templateService).listTemplate(sourceStoreMock);
Mockito.doReturn(null).when(templateService).listTemplate(destStoreMock);
Mockito.doReturn("install-path").when(templateInfoMock).getInstallPath();
Mockito.doReturn(templateInfoMock).when(templateDataFactoryMock).getTemplate(2L, sourceStoreMock);
}
@ -159,7 +177,7 @@ public class TemplateServiceImplTest {
boolean result = templateService.tryCopyingTemplateToImageStore(tmpltMock, destStoreMock);
Assert.assertFalse(result);
Mockito.verify(storageOrchestrator, Mockito.never()).orchestrateTemplateCopyToImageStore(Mockito.any(), Mockito.any());
Mockito.verify(storageOrchestrator, Mockito.never()).orchestrateTemplateCopyFromSecondaryStores(Mockito.anyLong(), Mockito.any());
}
@Test
@ -167,20 +185,161 @@ public class TemplateServiceImplTest {
templatesInSourceStore.put(tmpltMock.getUniqueName(), tmpltPropMock);
Mockito.doReturn(null).when(templateInfoMock).getInstallPath();
Scope scopeMock = Mockito.mock(Scope.class);
Mockito.doReturn(scopeMock).when(destStoreMock).getScope();
Mockito.doReturn(1L).when(scopeMock).getScopeId();
Mockito.doReturn(List.of(1L)).when(_dcDao).listAllIds();
boolean result = templateService.tryCopyingTemplateToImageStore(tmpltMock, destStoreMock);
Assert.assertFalse(result);
Mockito.verify(storageOrchestrator, Mockito.never()).orchestrateTemplateCopyToImageStore(Mockito.any(), Mockito.any());
Mockito.verify(storageOrchestrator, Mockito.never()).orchestrateTemplateCopyFromSecondaryStores(Mockito.anyLong(), Mockito.any());
}
@Test
public void tryCopyingTemplateToImageStoreTestReturnsTrueWhenTemplateExistsInAnotherStorageAndTaskWasScheduled() {
templatesInSourceStore.put(tmpltMock.getUniqueName(), tmpltPropMock);
Mockito.doReturn(new AsyncCallFuture<>()).when(storageOrchestrator).orchestrateTemplateCopyToImageStore(Mockito.any(), Mockito.any());
public void tryCopyingTemplateToImageStoreTestReturnsTrueWhenTemplateExistsInAnotherZone() throws StorageUnavailableException, ResourceAllocationException {
Scope scopeMock = Mockito.mock(Scope.class);
Mockito.doReturn(scopeMock).when(destStoreMock).getScope();
Mockito.doReturn(1L).when(scopeMock).getScopeId();
Mockito.doReturn(100L).when(tmpltMock).getId();
Mockito.doReturn("unique-name").when(tmpltMock).getUniqueName();
Mockito.doReturn(List.of(sourceStoreMock)).when(dataStoreManagerMock).getImageStoresByZoneIds(1L);
Mockito.doReturn(null).when(templateService).listTemplate(sourceStoreMock);
Mockito.doReturn(List.of(1L, 2L)).when(_dcDao).listAllIds();
DataStore otherZoneStoreMock = Mockito.mock(DataStore.class);
Mockito.doReturn(List.of(otherZoneStoreMock)).when(dataStoreManagerMock).getImageStoresByZoneIds(2L);
Map<String, TemplateProp> templatesInOtherZone = new HashMap<>();
templatesInOtherZone.put("unique-name", tmpltPropMock);
Mockito.doReturn(templatesInOtherZone).when(templateService).listTemplate(otherZoneStoreMock);
TemplateObject sourceTmplMock = Mockito.mock(TemplateObject.class);
Mockito.doReturn(sourceTmplMock).when(templateDataFactoryMock).getTemplate(100L, otherZoneStoreMock);
Mockito.doReturn("/mnt/secondary/template.qcow2").when(sourceTmplMock).getInstallPath();
DataCenterVO dstZoneMock = Mockito.mock(DataCenterVO.class);
Mockito.doReturn(dstZoneMock).when(_dcDao).findById(1L);
Mockito.doReturn(true).when(_tmpltMgr).copy(Mockito.anyLong(), Mockito.any(), Mockito.any(), Mockito.any());
boolean result = templateService.tryCopyingTemplateToImageStore(tmpltMock, destStoreMock);
Assert.assertTrue(result);
Mockito.verify(storageOrchestrator).orchestrateTemplateCopyToImageStore(Mockito.any(), Mockito.any());
}
@Test
public void tryCopyingTemplateToImageStoreTestReturnsFalseWhenDestinationZoneIsMissing() {
Scope scopeMock = Mockito.mock(Scope.class);
Mockito.doReturn(scopeMock).when(destStoreMock).getScope();
Mockito.doReturn(1L).when(scopeMock).getScopeId();
Mockito.doReturn(100L).when(tmpltMock).getId();
Mockito.doReturn("unique-name").when(tmpltMock).getUniqueName();
Mockito.doReturn(List.of(1L, 2L)).when(_dcDao).listAllIds();
Mockito.doReturn(List.of()).when(dataStoreManagerMock).getImageStoresByZoneIds(1L);
DataStore otherZoneStoreMock = Mockito.mock(DataStore.class);
Mockito.doReturn(List.of(otherZoneStoreMock)).when(dataStoreManagerMock).getImageStoresByZoneIds(2L);
Map<String, TemplateProp> templates = new HashMap<>();
templates.put("unique-name", tmpltPropMock);
Mockito.doReturn(templates).when(templateService).listTemplate(otherZoneStoreMock);
TemplateObject sourceTmplMock = Mockito.mock(TemplateObject.class);
Mockito.doReturn(sourceTmplMock).when(templateDataFactoryMock).getTemplate(100L, otherZoneStoreMock);
Mockito.doReturn("/mnt/secondary/template.qcow2").when(sourceTmplMock).getInstallPath();
Mockito.doReturn(null).when(_dcDao).findById(1L);
boolean result = templateService.tryCopyingTemplateToImageStore(tmpltMock, destStoreMock);
Assert.assertFalse(result);
}
@Test
public void tryCopyingTemplateToImageStoreTestReturnsTrueWhenCrossZoneCopyTaskIsScheduled() throws StorageUnavailableException, ResourceAllocationException {
Scope scopeMock = Mockito.mock(Scope.class);
Mockito.doReturn(scopeMock).when(destStoreMock).getScope();
Mockito.doReturn(1L).when(scopeMock).getScopeId();
Mockito.doReturn(100L).when(tmpltMock).getId();
Mockito.doReturn("unique-name").when(tmpltMock).getUniqueName();
Mockito.doReturn(List.of(1L, 2L)).when(_dcDao).listAllIds();
Mockito.doReturn(List.of()).when(dataStoreManagerMock).getImageStoresByZoneIds(1L);
DataStore otherZoneStoreMock = Mockito.mock(DataStore.class);
Mockito.doReturn(List.of(otherZoneStoreMock)).when(dataStoreManagerMock).getImageStoresByZoneIds(2L);
Map<String, TemplateProp> templates = new HashMap<>();
templates.put("unique-name", tmpltPropMock);
Mockito.doReturn(templates).when(templateService).listTemplate(otherZoneStoreMock);
TemplateObject sourceTmplMock = Mockito.mock(TemplateObject.class);
Mockito.doReturn(sourceTmplMock).when(templateDataFactoryMock).getTemplate(100L, otherZoneStoreMock);
Mockito.doReturn("/mnt/secondary/template.qcow2").when(sourceTmplMock).getInstallPath();
Mockito.doReturn(100L).when(sourceTmplMock).getId();
DataStore sourceStoreMock = Mockito.mock(DataStore.class);
Scope sourceScopeMock = Mockito.mock(Scope.class);
Mockito.doReturn(sourceStoreMock).when(sourceTmplMock).getDataStore();
DataCenterVO dstZoneMock = Mockito.mock(DataCenterVO.class);
Mockito.doReturn(dstZoneMock).when(_dcDao).findById(1L);
VMTemplateVO templateVoMock = Mockito.mock(VMTemplateVO.class);
Mockito.doReturn(templateVoMock).when(templateDao).findById(100L);
Mockito.doReturn(true).when(_tmpltMgr).copy(Mockito.anyLong(), Mockito.any(), Mockito.any(), Mockito.any());
Account account = mock(Account.class);
User user = mock(User.class);
CallContext callContext = mock(CallContext.class);
boolean result = templateService.tryCopyingTemplateToImageStore(tmpltMock, destStoreMock);
Assert.assertTrue(result);
}
@Test
public void tryCopyingTemplateToImageStoreTestReturnsFalseWhenTemplateNotFoundInAnyZone() {
Scope scopeMock = Mockito.mock(Scope.class);
Mockito.doReturn(scopeMock).when(destStoreMock).getScope();
Mockito.doReturn(1L).when(scopeMock).getScopeId();
Mockito.doReturn(List.of(1L, 2L)).when(_dcDao).listAllIds();
Mockito.doReturn(List.of(sourceStoreMock)).when(dataStoreManagerMock).getImageStoresByZoneIds(Mockito.anyLong());
Mockito.doReturn(null).when(templateService).listTemplate(Mockito.any());
boolean result = templateService.tryCopyingTemplateToImageStore(tmpltMock, destStoreMock);
Assert.assertFalse(result);
}
@Test
public void testFindUsableTemplateReturnsTemplateWithNonNullInstallPath() {
VMTemplateVO template = Mockito.mock(VMTemplateVO.class);
Mockito.when(template.getId()).thenReturn(10L);
Mockito.when(template.getUniqueName()).thenReturn("test-template");
DataStore storeWithNullPath = Mockito.mock(DataStore.class);
Mockito.when(storeWithNullPath.getName()).thenReturn("store-null");
DataStore storeWithValidPath = Mockito.mock(DataStore.class);
TemplateObject tmplWithNullPath = Mockito.mock(TemplateObject.class);
Mockito.when(tmplWithNullPath.getInstallPath()).thenReturn(null);
TemplateObject tmplWithValidPath = Mockito.mock(TemplateObject.class);
Mockito.when(tmplWithValidPath.getInstallPath()).thenReturn("/mnt/secondary/template.qcow2");
Mockito.doReturn(tmplWithNullPath).when(templateDataFactoryMock).getTemplate(10L, storeWithNullPath);
Mockito.doReturn(tmplWithValidPath).when(templateDataFactoryMock).getTemplate(10L, storeWithValidPath);
Map<String, TemplateProp> templates = new HashMap<>();
templates.put("test-template", Mockito.mock(TemplateProp.class));
Mockito.doReturn(templates).when(templateService).listTemplate(storeWithNullPath);
Mockito.doReturn(templates).when(templateService).listTemplate(storeWithValidPath);
List<DataStore> imageStores = List.of(storeWithNullPath, storeWithValidPath);
TemplateObject result = templateService.findUsableTemplate(template, imageStores);
Assert.assertNotNull(result);
Assert.assertEquals(tmplWithValidPath, result);
}
}

View File

@ -78,4 +78,15 @@ public class ImageStoreDetailsUtil {
return getGlobalDefaultNfsVersion();
}
public boolean isCopyTemplatesFromOtherStoragesEnabled(Long storeId, Long zoneId) {
final Map<String, String> storeDetails = imageStoreDetailsDao.getDetails(storeId);
final String keyWithoutDots = StorageManager.COPY_TEMPLATES_FROM_OTHER_SECONDARY_STORAGES.key()
.replace(".", "");
if (storeDetails != null && storeDetails.containsKey(keyWithoutDots)) {
return Boolean.parseBoolean(storeDetails.get(keyWithoutDots));
}
return StorageManager.COPY_TEMPLATES_FROM_OTHER_SECONDARY_STORAGES.valueIn(zoneId);
}
}

View File

@ -4202,7 +4202,7 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
DataStoreDownloadFollowRedirects,
AllowVolumeReSizeBeyondAllocation,
StoragePoolHostConnectWorkers,
COPY_PUBLIC_TEMPLATES_FROM_OTHER_STORAGES
COPY_TEMPLATES_FROM_OTHER_SECONDARY_STORAGES
};
}

View File

@ -838,6 +838,9 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
// Copy will just find one eligible image store for the destination zone
// and copy template there, not propagate to all image stores
// for that zone
boolean copied = false;
for (DataStore dstSecStore : dstSecStores) {
TemplateDataStoreVO dstTmpltStore = _tmplStoreDao.findByStoreTemplate(dstSecStore.getId(), tmpltId);
if (dstTmpltStore != null && dstTmpltStore.getDownloadState() == Status.DOWNLOADED) {
@ -852,9 +855,12 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
TemplateApiResult result = future.get();
if (result.isFailed()) {
logger.debug("Copy Template failed for image store {}: {}", dstSecStore, result.getResult());
_tmplStoreDao.removeByTemplateStore(tmpltId, dstSecStore.getId());
continue; // try next image store
}
copied = true;
_tmpltDao.addTemplateToZone(template, dstZoneId);
if (account.getId() != Account.ACCOUNT_ID_SYSTEM) {
@ -882,12 +888,14 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
}
}
}
return true;
} catch (Exception ex) {
logger.debug("Failed to copy Template to image store:{} ,will try next one", dstSecStore);
logger.debug("Failed to copy Template to image store:{} ,will try next one", dstSecStore, ex);
}
}
return true;
return copied;
}
@Override

View File

@ -590,6 +590,8 @@
"label.copy.consoleurl": "Copy console URL to clipboard",
"label.copyid": "Copy ID",
"label.copy.password": "Copy password",
"label.copy.templates.from.other.secondary.storages": "Copy Templates from other storages instead of fetching from URLs",
"label.copy.templates.from.other.secondary.storages.add.zone": "Copy Templates from other storages",
"label.core": "Core",
"label.core.zone.type": "Core Zone type",
"label.counter": "Counter",
@ -3018,7 +3020,7 @@
"message.desc.importmigratefromvmwarewizard": "By selecting an existing or external VMware Datacenter and an instance to import, CloudStack migrates the selected instance from VMware to KVM on a conversion host using virt-v2v and imports it into a KVM Cluster",
"message.desc.primary.storage": "Each Cluster must contain one or more primary storage servers. We will add the first one now. Primary storage contains the disk volumes for all the Instances running on hosts in the cluster. Use any standards-compliant protocol that is supported by the underlying hypervisor.",
"message.desc.reset.ssh.key.pair": "Please specify a ssh key pair that you would like to add to this Instance.",
"message.desc.secondary.storage": "Each Zone must have at least one NFS or secondary storage server. We will add the first one now. Secondary storage stores Instance Templates, ISO images, and Instance disk volume Snapshots. This server must be available to all hosts in the zone.<br/><br/>Provide the IP address and exported path.",
"message.desc.secondary.storage": "Each Zone must have at least one NFS or secondary storage server. We will add the first one now. Secondary storage stores Instance Templates, ISO images, and Instance disk volume Snapshots. This server must be available to all hosts in the zone.<br/><br/>Provide the IP address and exported path.<br/><br/> \"Copy templates from other secondary storages\" switch can be used to automatically copy existing templates from secondary storages in other zones instead of fetching from their URLs.",
"message.desc.register.user.data": "Please fill in the following to register new User Data.",
"message.desc.registered.user.data": "Registered a User Data.",
"message.desc.zone": "A Zone is the largest organizational unit in CloudStack, and it typically corresponds to a single datacenter. Zones provide physical isolation and redundancy. A zone consists of one or more Pods (each of which contains hosts and primary storage servers) and a secondary storage server which is shared by all pods in the zone.",

View File

@ -48,6 +48,7 @@
<a-form-item name="zone" ref="zone" :label="$t('label.zone')">
<a-select
v-model:value="form.zone"
@change="onZoneChange"
showSearch
optionFilterProp="label"
:filterOption="(input, option) => {
@ -105,6 +106,7 @@
<a-form-item name="zone" ref="zone" :label="$t('label.zone')">
<a-select
v-model:value="form.zone"
@change="onZoneChange"
showSearch
optionFilterProp="label"
:filterOption="(input, option) => {
@ -159,6 +161,17 @@
<a-input v-model:value="form.secondaryStorageNFSPath"/>
</a-form-item>
</div>
<div v-if="showCopyTemplatesToggle">
<a-form-item
name="copyTemplatesFromOtherSecondaryStorages"
ref="copyTemplatesFromOtherSecondaryStorages"
:label="$t('label.copy.templates.from.other.secondary.storages')">
<a-switch
v-model:checked="form.copyTemplatesFromOtherSecondaryStorages"
@change="onCopyTemplatesToggleChanged"
/>
</a-form-item>
</div>
<div :span="24" class="action-button">
<a-button @click="closeModal">{{ $t('label.cancel') }}</a-button>
<a-button type="primary" ref="submit" @click="handleSubmit">{{ $t('label.ok') }}</a-button>
@ -191,7 +204,9 @@ export default {
providers: ['NFS', 'SMB/CIFS', 'S3', 'Swift'],
zones: [],
loading: false,
secondaryStorageNFSStaging: false
secondaryStorageNFSStaging: false,
showCopyTemplatesToggle: false,
copyTemplatesTouched: false
}
},
created () {
@ -203,7 +218,8 @@ export default {
this.formRef = ref()
this.form = reactive({
provider: 'NFS',
secondaryStorageHttps: true
secondaryStorageHttps: true,
copyTemplatesFromOtherSecondaryStorages: true
})
this.rules = reactive({
zone: [{ required: true, message: this.$t('label.required') }],
@ -225,20 +241,56 @@ export default {
},
fetchData () {
this.listZones()
this.checkOtherSecondaryStorages()
},
closeModal () {
this.$emit('close-action')
},
fetchCopyTemplatesConfig () {
if (!this.form.zone) {
return
}
api('listConfigurations', {
name: 'copy.templates.from.other.secondary.storages',
zoneid: this.form.zone
}).then(json => {
const items =
json?.listconfigurationsresponse?.configuration || []
items.forEach(item => {
if (item.name === 'copy.templates.from.other.secondary.storages') {
this.form.copyTemplatesFromOtherSecondaryStorages =
item.value === 'true'
}
})
})
},
onZoneChange (val) {
this.form.zone = val
this.copyTemplatesTouched = false
this.fetchCopyTemplatesConfig()
},
listZones () {
api('listZones', { showicon: true }).then(json => {
if (json && json.listzonesresponse && json.listzonesresponse.zone) {
this.zones = json.listzonesresponse.zone
if (this.zones.length > 0) {
this.form.zone = this.zones[0].id || ''
}
this.zones = json.listzonesresponse.zone || []
if (this.zones.length > 0) {
this.form.zone = this.zones[0].id
this.fetchCopyTemplatesConfig()
}
})
},
checkOtherSecondaryStorages () {
api('listImageStores', { listall: true }).then(json => {
const stores = json?.listimagestoresresponse?.imagestore || []
this.showCopyTemplatesToggle = stores.length > 0
})
},
onCopyTemplatesToggleChanged (val) {
this.copyTemplatesTouched = true
},
nfsURL (server, path) {
var url
if (path.substring(0, 1) !== '/') {
@ -362,6 +414,22 @@ export default {
nfsParams.url = nfsUrl
}
if (
this.showCopyTemplatesToggle &&
this.copyTemplatesTouched
) {
const copyTemplatesKey = 'copytemplatesfromothersecondarystorages'
const detailIdx = Object.keys(data)
.filter(k => k.startsWith('details['))
.map(k => parseInt(k.match(/details\[(\d+)\]/)[1]))
.reduce((a, b) => Math.max(a, b), -1) + 1
data[`details[${detailIdx}].key`] = copyTemplatesKey
data[`details[${detailIdx}].value`] =
values.copyTemplatesFromOtherSecondaryStorages.toString()
}
this.loading = true
try {

View File

@ -840,6 +840,13 @@ export default {
display: {
secondaryStorageProvider: ['Swift']
}
},
{
title: 'label.copy.templates.from.other.secondary.storages.add.zone',
key: 'copyTemplatesFromOtherSecondaryStorages',
required: false,
switch: true,
checked: this.copytemplate
}
]
}
@ -860,7 +867,8 @@ export default {
}],
storageProviders: [],
currentStep: null,
options: ['primaryStorageScope', 'primaryStorageProtocol', 'provider', 'primaryStorageProvider']
options: ['primaryStorageScope', 'primaryStorageProtocol', 'provider', 'primaryStorageProvider'],
copytemplate: true
}
},
created () {
@ -885,6 +893,7 @@ export default {
primaryStorageScope: null
})
}
this.applyCopyTemplatesOptionFromGlobalSettingDuringSecondaryStorageAddition()
}
},
watch: {
@ -1108,6 +1117,20 @@ export default {
this.storageProviders = storageProviders
})
},
applyCopyTemplatesOptionFromGlobalSettingDuringSecondaryStorageAddition () {
api('listConfigurations', {
name: 'copy.templates.from.other.secondary.storages'
}).then(json => {
const config = json?.listconfigurationsresponse?.configuration?.[0]
if (!config || config.value === undefined) {
return
}
const value = String(config.value).toLowerCase() === 'true'
this.copytemplate = value
})
},
fetchPrimaryStorageProvider () {
this.primaryStorageProviders = []
api('listStorageProviders', { type: 'primary' }).then(json => {

View File

@ -1580,6 +1580,11 @@ export default {
params.provider = this.prefillContent.secondaryStorageProvider
params.zoneid = this.stepData.zoneReturned.id
params.url = url
if (this.prefillContent.copyTemplatesFromOtherSecondaryStorages !== undefined) {
params['details[0].key'] = 'copytemplatesfromothersecondarystorages'
params['details[0].value'] =
this.prefillContent.copyTemplatesFromOtherSecondaryStorages
}
} else if (this.prefillContent.secondaryStorageProvider === 'SMB') {
const nfsServer = this.prefillContent.secondaryStorageServer
const path = this.prefillContent.secondaryStoragePath