mirror of https://github.com/apache/cloudstack.git
CLOUDSTACK-4400:[object_store_refactor] Three entries for one template
in template_store_ref when MS was restarting during template download.
This commit is contained in:
Min Chen
parent
494ea5052f
commit
3937a7b5a5
|
|
@ -87,6 +87,7 @@ import com.cloud.user.Account;
|
|||
import com.cloud.user.AccountManager;
|
||||
import com.cloud.user.ResourceLimitService;
|
||||
import com.cloud.utils.UriUtils;
|
||||
import com.cloud.utils.db.GlobalLock;
|
||||
import com.cloud.utils.exception.CloudRuntimeException;
|
||||
|
||||
@Component
|
||||
|
|
@ -253,185 +254,203 @@ public class TemplateServiceImpl implements TemplateService {
|
|||
return;
|
||||
}
|
||||
long storeId = store.getId();
|
||||
Long zoneId = store.getScope().getScopeId();
|
||||
|
||||
Map<String, TemplateProp> templateInfos = listTemplate(store);
|
||||
if (templateInfos == null) {
|
||||
return;
|
||||
}
|
||||
// add lock to make template sync for a data store only be done once
|
||||
String lockString = "templatesync.storeId:" + storeId;
|
||||
GlobalLock syncLock = GlobalLock.getInternLock(lockString);
|
||||
try {
|
||||
if (syncLock.lock(3)) {
|
||||
try{
|
||||
Long zoneId = store.getScope().getScopeId();
|
||||
|
||||
Set<VMTemplateVO> toBeDownloaded = new HashSet<VMTemplateVO>();
|
||||
List<VMTemplateVO> allTemplates = null;
|
||||
if (zoneId == null) {
|
||||
// region wide store
|
||||
allTemplates = _templateDao.listAllActive();
|
||||
} else {
|
||||
// zone wide store
|
||||
allTemplates = _templateDao.listAllInZone(zoneId);
|
||||
}
|
||||
List<VMTemplateVO> rtngTmplts = _templateDao.listAllSystemVMTemplates();
|
||||
List<VMTemplateVO> defaultBuiltin = _templateDao.listDefaultBuiltinTemplates();
|
||||
|
||||
if (rtngTmplts != null) {
|
||||
for (VMTemplateVO rtngTmplt : rtngTmplts) {
|
||||
if (!allTemplates.contains(rtngTmplt)) {
|
||||
allTemplates.add(rtngTmplt);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (defaultBuiltin != null) {
|
||||
for (VMTemplateVO builtinTmplt : defaultBuiltin) {
|
||||
if (!allTemplates.contains(builtinTmplt)) {
|
||||
allTemplates.add(builtinTmplt);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
toBeDownloaded.addAll(allTemplates);
|
||||
|
||||
for (VMTemplateVO tmplt : allTemplates) {
|
||||
String uniqueName = tmplt.getUniqueName();
|
||||
TemplateDataStoreVO tmpltStore = _vmTemplateStoreDao.findByStoreTemplate(storeId, tmplt.getId());
|
||||
if (templateInfos.containsKey(uniqueName)) {
|
||||
TemplateProp tmpltInfo = templateInfos.remove(uniqueName);
|
||||
toBeDownloaded.remove(tmplt);
|
||||
if (tmpltStore != null) {
|
||||
s_logger.info("Template Sync found " + uniqueName + " already in the image store");
|
||||
if (tmpltStore.getDownloadState() != Status.DOWNLOADED) {
|
||||
tmpltStore.setErrorString("");
|
||||
Map<String, TemplateProp> templateInfos = listTemplate(store);
|
||||
if (templateInfos == null) {
|
||||
return;
|
||||
}
|
||||
if (tmpltInfo.isCorrupted()) {
|
||||
tmpltStore.setDownloadState(Status.DOWNLOAD_ERROR);
|
||||
String msg = "Template " + tmplt.getName() + ":" + tmplt.getId()
|
||||
+ " is corrupted on secondary storage " + tmpltStore.getId();
|
||||
tmpltStore.setErrorString(msg);
|
||||
s_logger.info("msg");
|
||||
if (tmplt.getUrl() == null) {
|
||||
msg = "Private Template (" + tmplt + ") with install path " + tmpltInfo.getInstallPath()
|
||||
+ "is corrupted, please check in image store: " + tmpltStore.getDataStoreId();
|
||||
s_logger.warn(msg);
|
||||
} else {
|
||||
s_logger.info("Removing template_store_ref entry for corrupted template " + tmplt.getName());
|
||||
_vmTemplateStoreDao.remove(tmpltStore.getId());
|
||||
toBeDownloaded.add(tmplt);
|
||||
}
|
||||
|
||||
Set<VMTemplateVO> toBeDownloaded = new HashSet<VMTemplateVO>();
|
||||
List<VMTemplateVO> allTemplates = null;
|
||||
if (zoneId == null) {
|
||||
// region wide store
|
||||
allTemplates = _templateDao.listAllActive();
|
||||
} else {
|
||||
tmpltStore.setDownloadPercent(100);
|
||||
tmpltStore.setDownloadState(Status.DOWNLOADED);
|
||||
tmpltStore.setState(ObjectInDataStoreStateMachine.State.Ready);
|
||||
tmpltStore.setInstallPath(tmpltInfo.getInstallPath());
|
||||
tmpltStore.setSize(tmpltInfo.getSize());
|
||||
tmpltStore.setPhysicalSize(tmpltInfo.getPhysicalSize());
|
||||
tmpltStore.setLastUpdated(new Date());
|
||||
// update size in vm_template table
|
||||
VMTemplateVO tmlpt = _templateDao.findById(tmplt.getId());
|
||||
tmlpt.setSize(tmpltInfo.getSize());
|
||||
_templateDao.update(tmplt.getId(), tmlpt);
|
||||
// zone wide store
|
||||
allTemplates = _templateDao.listAllInZone(zoneId);
|
||||
}
|
||||
List<VMTemplateVO> rtngTmplts = _templateDao.listAllSystemVMTemplates();
|
||||
List<VMTemplateVO> defaultBuiltin = _templateDao.listDefaultBuiltinTemplates();
|
||||
|
||||
// Skipping limit checks for SYSTEM Account and for the templates created from volumes or snapshots
|
||||
// which already got checked and incremented during createTemplate API call.
|
||||
if (tmpltInfo.getSize() > 0 && tmplt.getAccountId() != Account.ACCOUNT_ID_SYSTEM && tmplt.getUrl() != null) {
|
||||
long accountId = tmplt.getAccountId();
|
||||
try {
|
||||
_resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(accountId),
|
||||
com.cloud.configuration.Resource.ResourceType.secondary_storage,
|
||||
tmpltInfo.getSize() - UriUtils.getRemoteSize(tmplt.getUrl()));
|
||||
} catch (ResourceAllocationException e) {
|
||||
s_logger.warn(e.getMessage());
|
||||
_alertMgr.sendAlert(AlertManager.ALERT_TYPE_RESOURCE_LIMIT_EXCEEDED, zoneId, null,
|
||||
e.getMessage(), e.getMessage());
|
||||
} finally {
|
||||
_resourceLimitMgr.recalculateResourceCount(accountId, _accountMgr.getAccount(accountId)
|
||||
.getDomainId(), com.cloud.configuration.Resource.ResourceType.secondary_storage
|
||||
.getOrdinal());
|
||||
if (rtngTmplts != null) {
|
||||
for (VMTemplateVO rtngTmplt : rtngTmplts) {
|
||||
if (!allTemplates.contains(rtngTmplt)) {
|
||||
allTemplates.add(rtngTmplt);
|
||||
}
|
||||
}
|
||||
}
|
||||
_vmTemplateStoreDao.update(tmpltStore.getId(), tmpltStore);
|
||||
} else {
|
||||
tmpltStore = new TemplateDataStoreVO(storeId, tmplt.getId(), new Date(), 100, Status.DOWNLOADED,
|
||||
null, null, null, tmpltInfo.getInstallPath(), tmplt.getUrl());
|
||||
tmpltStore.setSize(tmpltInfo.getSize());
|
||||
tmpltStore.setPhysicalSize(tmpltInfo.getPhysicalSize());
|
||||
tmpltStore.setDataStoreRole(store.getRole());
|
||||
_vmTemplateStoreDao.persist(tmpltStore);
|
||||
|
||||
// update size in vm_template table
|
||||
VMTemplateVO tmlpt = _templateDao.findById(tmplt.getId());
|
||||
tmlpt.setSize(tmpltInfo.getSize());
|
||||
_templateDao.update(tmplt.getId(), tmlpt);
|
||||
associateTemplateToZone(tmplt.getId(), zoneId);
|
||||
if (defaultBuiltin != null) {
|
||||
for (VMTemplateVO builtinTmplt : defaultBuiltin) {
|
||||
if (!allTemplates.contains(builtinTmplt)) {
|
||||
allTemplates.add(builtinTmplt);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
toBeDownloaded.addAll(allTemplates);
|
||||
|
||||
for (VMTemplateVO tmplt : allTemplates) {
|
||||
String uniqueName = tmplt.getUniqueName();
|
||||
TemplateDataStoreVO tmpltStore = _vmTemplateStoreDao.findByStoreTemplate(storeId, tmplt.getId());
|
||||
if (templateInfos.containsKey(uniqueName)) {
|
||||
TemplateProp tmpltInfo = templateInfos.remove(uniqueName);
|
||||
toBeDownloaded.remove(tmplt);
|
||||
if (tmpltStore != null) {
|
||||
s_logger.info("Template Sync found " + uniqueName + " already in the image store");
|
||||
if (tmpltStore.getDownloadState() != Status.DOWNLOADED) {
|
||||
tmpltStore.setErrorString("");
|
||||
}
|
||||
if (tmpltInfo.isCorrupted()) {
|
||||
tmpltStore.setDownloadState(Status.DOWNLOAD_ERROR);
|
||||
String msg = "Template " + tmplt.getName() + ":" + tmplt.getId()
|
||||
+ " is corrupted on secondary storage " + tmpltStore.getId();
|
||||
tmpltStore.setErrorString(msg);
|
||||
s_logger.info("msg");
|
||||
if (tmplt.getUrl() == null) {
|
||||
msg = "Private Template (" + tmplt + ") with install path " + tmpltInfo.getInstallPath()
|
||||
+ "is corrupted, please check in image store: " + tmpltStore.getDataStoreId();
|
||||
s_logger.warn(msg);
|
||||
} else {
|
||||
s_logger.info("Removing template_store_ref entry for corrupted template " + tmplt.getName());
|
||||
_vmTemplateStoreDao.remove(tmpltStore.getId());
|
||||
toBeDownloaded.add(tmplt);
|
||||
}
|
||||
|
||||
} else {
|
||||
tmpltStore.setDownloadPercent(100);
|
||||
tmpltStore.setDownloadState(Status.DOWNLOADED);
|
||||
tmpltStore.setState(ObjectInDataStoreStateMachine.State.Ready);
|
||||
tmpltStore.setInstallPath(tmpltInfo.getInstallPath());
|
||||
tmpltStore.setSize(tmpltInfo.getSize());
|
||||
tmpltStore.setPhysicalSize(tmpltInfo.getPhysicalSize());
|
||||
tmpltStore.setLastUpdated(new Date());
|
||||
// update size in vm_template table
|
||||
VMTemplateVO tmlpt = _templateDao.findById(tmplt.getId());
|
||||
tmlpt.setSize(tmpltInfo.getSize());
|
||||
_templateDao.update(tmplt.getId(), tmlpt);
|
||||
|
||||
// Skipping limit checks for SYSTEM Account and for the templates created from volumes or snapshots
|
||||
// which already got checked and incremented during createTemplate API call.
|
||||
if (tmpltInfo.getSize() > 0 && tmplt.getAccountId() != Account.ACCOUNT_ID_SYSTEM && tmplt.getUrl() != null) {
|
||||
long accountId = tmplt.getAccountId();
|
||||
try {
|
||||
_resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(accountId),
|
||||
com.cloud.configuration.Resource.ResourceType.secondary_storage,
|
||||
tmpltInfo.getSize() - UriUtils.getRemoteSize(tmplt.getUrl()));
|
||||
} catch (ResourceAllocationException e) {
|
||||
s_logger.warn(e.getMessage());
|
||||
_alertMgr.sendAlert(AlertManager.ALERT_TYPE_RESOURCE_LIMIT_EXCEEDED, zoneId, null,
|
||||
e.getMessage(), e.getMessage());
|
||||
} finally {
|
||||
_resourceLimitMgr.recalculateResourceCount(accountId, _accountMgr.getAccount(accountId)
|
||||
.getDomainId(), com.cloud.configuration.Resource.ResourceType.secondary_storage
|
||||
.getOrdinal());
|
||||
}
|
||||
}
|
||||
}
|
||||
_vmTemplateStoreDao.update(tmpltStore.getId(), tmpltStore);
|
||||
} else {
|
||||
tmpltStore = new TemplateDataStoreVO(storeId, tmplt.getId(), new Date(), 100, Status.DOWNLOADED,
|
||||
null, null, null, tmpltInfo.getInstallPath(), tmplt.getUrl());
|
||||
tmpltStore.setSize(tmpltInfo.getSize());
|
||||
tmpltStore.setPhysicalSize(tmpltInfo.getPhysicalSize());
|
||||
tmpltStore.setDataStoreRole(store.getRole());
|
||||
_vmTemplateStoreDao.persist(tmpltStore);
|
||||
|
||||
// update size in vm_template table
|
||||
VMTemplateVO tmlpt = _templateDao.findById(tmplt.getId());
|
||||
tmlpt.setSize(tmpltInfo.getSize());
|
||||
_templateDao.update(tmplt.getId(), tmlpt);
|
||||
associateTemplateToZone(tmplt.getId(), zoneId);
|
||||
|
||||
|
||||
}
|
||||
} else {
|
||||
s_logger.info("Template Sync did not find " + uniqueName + " on image store " + storeId + ", may request download based on available hypervisor types");
|
||||
if (tmpltStore != null) {
|
||||
s_logger.info("Removing leftover template " + uniqueName + " entry from template store table");
|
||||
// remove those leftover entries
|
||||
_vmTemplateStoreDao.remove(tmpltStore.getId());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (toBeDownloaded.size() > 0) {
|
||||
/* Only download templates whose hypervirsor type is in the zone */
|
||||
List<HypervisorType> availHypers = _clusterDao.getAvailableHypervisorInZone(zoneId);
|
||||
if (availHypers.isEmpty()) {
|
||||
/*
|
||||
* This is for cloudzone, local secondary storage resource
|
||||
* started before cluster created
|
||||
*/
|
||||
availHypers.add(HypervisorType.KVM);
|
||||
}
|
||||
/* Baremetal need not to download any template */
|
||||
availHypers.remove(HypervisorType.BareMetal);
|
||||
availHypers.add(HypervisorType.None); // bug 9809: resume ISO
|
||||
// download.
|
||||
for (VMTemplateVO tmplt : toBeDownloaded) {
|
||||
if (tmplt.getUrl() == null) { // If url is null we can't
|
||||
s_logger.info("Skip downloading template " + tmplt.getUniqueName() + " since no url is specified.");
|
||||
continue;
|
||||
}
|
||||
|
||||
if (availHypers.contains(tmplt.getHypervisorType())) {
|
||||
s_logger.info("Downloading template " + tmplt.getUniqueName() + " to image store "
|
||||
+ store.getName());
|
||||
associateTemplateToZone(tmplt.getId(), zoneId);
|
||||
TemplateInfo tmpl = _templateFactory.getTemplate(tmplt.getId(), DataStoreRole.Image);
|
||||
createTemplateAsync(tmpl, store, null);
|
||||
} else {
|
||||
s_logger.info("Skip downloading template " + tmplt.getUniqueName() + " since current data center does not have hypervisor "
|
||||
+ tmplt.getHypervisorType().toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (String uniqueName : templateInfos.keySet()) {
|
||||
TemplateProp tInfo = templateInfos.get(uniqueName);
|
||||
if (_tmpltMgr.templateIsDeleteable(tInfo.getId())) {
|
||||
// we cannot directly call deleteTemplateSync here to
|
||||
// reuse delete logic since in this case, our db does not have
|
||||
// this template at all.
|
||||
TemplateObjectTO tmplTO = new TemplateObjectTO();
|
||||
tmplTO.setDataStore(store.getTO());
|
||||
tmplTO.setPath(tInfo.getInstallPath());
|
||||
tmplTO.setId(tInfo.getId());
|
||||
DeleteCommand dtCommand = new DeleteCommand(tmplTO);
|
||||
EndPoint ep = _epSelector.select(store);
|
||||
Answer answer = ep.sendMessage(dtCommand);
|
||||
if (answer == null || !answer.getResult()) {
|
||||
s_logger.info("Failed to deleted template at store: " + store.getName());
|
||||
|
||||
} else {
|
||||
String description = "Deleted template " + tInfo.getTemplateName() + " on secondary storage "
|
||||
+ storeId;
|
||||
s_logger.info(description);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
s_logger.info("Template Sync did not find " + uniqueName + " on image store " + storeId + ", may request download based on available hypervisor types");
|
||||
if (tmpltStore != null) {
|
||||
s_logger.info("Removing leftover template " + uniqueName + " entry from template store table");
|
||||
// remove those leftover entries
|
||||
_vmTemplateStoreDao.remove(tmpltStore.getId());
|
||||
finally{
|
||||
syncLock.unlock();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (toBeDownloaded.size() > 0) {
|
||||
/* Only download templates whose hypervirsor type is in the zone */
|
||||
List<HypervisorType> availHypers = _clusterDao.getAvailableHypervisorInZone(zoneId);
|
||||
if (availHypers.isEmpty()) {
|
||||
/*
|
||||
* This is for cloudzone, local secondary storage resource
|
||||
* started before cluster created
|
||||
*/
|
||||
availHypers.add(HypervisorType.KVM);
|
||||
}
|
||||
/* Baremetal need not to download any template */
|
||||
availHypers.remove(HypervisorType.BareMetal);
|
||||
availHypers.add(HypervisorType.None); // bug 9809: resume ISO
|
||||
// download.
|
||||
for (VMTemplateVO tmplt : toBeDownloaded) {
|
||||
if (tmplt.getUrl() == null) { // If url is null we can't
|
||||
s_logger.info("Skip downloading template " + tmplt.getUniqueName() + " since no url is specified.");
|
||||
continue;
|
||||
}
|
||||
|
||||
if (availHypers.contains(tmplt.getHypervisorType())) {
|
||||
s_logger.info("Downloading template " + tmplt.getUniqueName() + " to image store "
|
||||
+ store.getName());
|
||||
associateTemplateToZone(tmplt.getId(), zoneId);
|
||||
TemplateInfo tmpl = _templateFactory.getTemplate(tmplt.getId(), DataStoreRole.Image);
|
||||
createTemplateAsync(tmpl, store, null);
|
||||
} else {
|
||||
s_logger.info("Skip downloading template " + tmplt.getUniqueName() + " since current data center does not have hypervisor "
|
||||
+ tmplt.getHypervisorType().toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (String uniqueName : templateInfos.keySet()) {
|
||||
TemplateProp tInfo = templateInfos.get(uniqueName);
|
||||
if (_tmpltMgr.templateIsDeleteable(tInfo.getId())) {
|
||||
// we cannot directly call deleteTemplateSync here to
|
||||
// reuse delete logic since in this case, our db does not have
|
||||
// this template at all.
|
||||
TemplateObjectTO tmplTO = new TemplateObjectTO();
|
||||
tmplTO.setDataStore(store.getTO());
|
||||
tmplTO.setPath(tInfo.getInstallPath());
|
||||
tmplTO.setId(tInfo.getId());
|
||||
DeleteCommand dtCommand = new DeleteCommand(tmplTO);
|
||||
EndPoint ep = _epSelector.select(store);
|
||||
Answer answer = ep.sendMessage(dtCommand);
|
||||
if (answer == null || !answer.getResult()) {
|
||||
s_logger.info("Failed to deleted template at store: " + store.getName());
|
||||
|
||||
} else {
|
||||
String description = "Deleted template " + tInfo.getTemplateName() + " on secondary storage "
|
||||
+ storeId;
|
||||
s_logger.info(description);
|
||||
}
|
||||
|
||||
else {
|
||||
s_logger.info("Couldn't get global lock on " + lockString + ", another thread may be doing template sync on data store " + storeId + " now.");
|
||||
}
|
||||
} finally {
|
||||
syncLock.releaseRef();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -91,6 +91,7 @@ import com.cloud.user.AccountManager;
|
|||
import com.cloud.user.ResourceLimitService;
|
||||
import com.cloud.utils.NumbersUtil;
|
||||
import com.cloud.utils.db.DB;
|
||||
import com.cloud.utils.db.GlobalLock;
|
||||
import com.cloud.utils.exception.CloudRuntimeException;
|
||||
|
||||
@Component
|
||||
|
|
@ -304,7 +305,7 @@ public class VolumeServiceImpl implements VolumeService {
|
|||
apiResult.setResult(result.getResult());
|
||||
}
|
||||
} catch (Exception e) {
|
||||
s_logger.debug("ignore delete volume status update failure, it will be picked up by storage clean up thread later", e);
|
||||
s_logger.debug("ignore delete volume status update failure, it will be picked up by storage clean up thread later", e);
|
||||
}
|
||||
context.getFuture().complete(apiResult);
|
||||
return null;
|
||||
|
|
@ -1100,124 +1101,143 @@ public class VolumeServiceImpl implements VolumeService {
|
|||
@Override
|
||||
public void handleVolumeSync(DataStore store) {
|
||||
if (store == null) {
|
||||
s_logger.warn("Huh? ssHost is null");
|
||||
s_logger.warn("Huh? image store is null");
|
||||
return;
|
||||
}
|
||||
long storeId = store.getId();
|
||||
|
||||
Map<Long, TemplateProp> volumeInfos = listVolume(store);
|
||||
if (volumeInfos == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
List<VolumeDataStoreVO> dbVolumes = _volumeStoreDao.listByStoreId(storeId);
|
||||
List<VolumeDataStoreVO> toBeDownloaded = new ArrayList<VolumeDataStoreVO>(dbVolumes);
|
||||
for (VolumeDataStoreVO volumeStore : dbVolumes) {
|
||||
VolumeVO volume = _volumeDao.findById(volumeStore.getVolumeId());
|
||||
if (volume == null ){
|
||||
s_logger.warn("Volume_store_ref shows that volume " + volumeStore.getVolumeId() + " is on image store " + storeId
|
||||
+ ", but the volume is not found in volumes table, potentially some bugs in deleteVolume, so we just treat this volume to be deleted and mark it as destroyed");
|
||||
volumeStore.setDestroyed(true);
|
||||
_volumeStoreDao.update(volumeStore.getId(), volumeStore);
|
||||
continue;
|
||||
}
|
||||
// Exists then don't download
|
||||
if (volumeInfos.containsKey(volume.getId())) {
|
||||
TemplateProp volInfo = volumeInfos.remove(volume.getId());
|
||||
toBeDownloaded.remove(volumeStore);
|
||||
s_logger.info("Volume Sync found " + volume.getUuid() + " already in the volume image store table");
|
||||
if (volumeStore.getDownloadState() != Status.DOWNLOADED) {
|
||||
volumeStore.setErrorString("");
|
||||
}
|
||||
if (volInfo.isCorrupted()) {
|
||||
volumeStore.setDownloadState(Status.DOWNLOAD_ERROR);
|
||||
String msg = "Volume " + volume.getUuid() + " is corrupted on image store ";
|
||||
volumeStore.setErrorString(msg);
|
||||
s_logger.info("msg");
|
||||
if (volumeStore.getDownloadUrl() == null) {
|
||||
msg = "Volume (" + volume.getUuid() + ") with install path " + volInfo.getInstallPath()
|
||||
+ "is corrupted, please check in image store: " + volumeStore.getDataStoreId();
|
||||
s_logger.warn(msg);
|
||||
} else {
|
||||
toBeDownloaded.add(volumeStore);
|
||||
// add lock to make template sync for a data store only be done once
|
||||
String lockString = "volumesync.storeId:" + storeId;
|
||||
GlobalLock syncLock = GlobalLock.getInternLock(lockString);
|
||||
try {
|
||||
if ( syncLock.lock(3)){
|
||||
try {
|
||||
Map<Long, TemplateProp> volumeInfos = listVolume(store);
|
||||
if (volumeInfos == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
} else { // Put them in right status
|
||||
volumeStore.setDownloadPercent(100);
|
||||
volumeStore.setDownloadState(Status.DOWNLOADED);
|
||||
volumeStore.setInstallPath(volInfo.getInstallPath());
|
||||
volumeStore.setSize(volInfo.getSize());
|
||||
volumeStore.setPhysicalSize(volInfo.getPhysicalSize());
|
||||
volumeStore.setLastUpdated(new Date());
|
||||
_volumeStoreDao.update(volumeStore.getId(), volumeStore);
|
||||
List<VolumeDataStoreVO> dbVolumes = _volumeStoreDao.listByStoreId(storeId);
|
||||
List<VolumeDataStoreVO> toBeDownloaded = new ArrayList<VolumeDataStoreVO>(dbVolumes);
|
||||
for (VolumeDataStoreVO volumeStore : dbVolumes) {
|
||||
VolumeVO volume = _volumeDao.findById(volumeStore.getVolumeId());
|
||||
if (volume == null ){
|
||||
s_logger.warn("Volume_store_ref shows that volume " + volumeStore.getVolumeId() + " is on image store " + storeId
|
||||
+ ", but the volume is not found in volumes table, potentially some bugs in deleteVolume, so we just treat this volume to be deleted and mark it as destroyed");
|
||||
volumeStore.setDestroyed(true);
|
||||
_volumeStoreDao.update(volumeStore.getId(), volumeStore);
|
||||
continue;
|
||||
}
|
||||
// Exists then don't download
|
||||
if (volumeInfos.containsKey(volume.getId())) {
|
||||
TemplateProp volInfo = volumeInfos.remove(volume.getId());
|
||||
toBeDownloaded.remove(volumeStore);
|
||||
s_logger.info("Volume Sync found " + volume.getUuid() + " already in the volume image store table");
|
||||
if (volumeStore.getDownloadState() != Status.DOWNLOADED) {
|
||||
volumeStore.setErrorString("");
|
||||
}
|
||||
if (volInfo.isCorrupted()) {
|
||||
volumeStore.setDownloadState(Status.DOWNLOAD_ERROR);
|
||||
String msg = "Volume " + volume.getUuid() + " is corrupted on image store ";
|
||||
volumeStore.setErrorString(msg);
|
||||
s_logger.info("msg");
|
||||
if (volumeStore.getDownloadUrl() == null) {
|
||||
msg = "Volume (" + volume.getUuid() + ") with install path " + volInfo.getInstallPath()
|
||||
+ "is corrupted, please check in image store: " + volumeStore.getDataStoreId();
|
||||
s_logger.warn(msg);
|
||||
} else {
|
||||
s_logger.info("Removing volume_store_ref entry for corrupted volume " + volume.getName());
|
||||
_volumeStoreDao.remove(volumeStore.getId());
|
||||
toBeDownloaded.add(volumeStore);
|
||||
}
|
||||
|
||||
if (volume.getSize() == 0) {
|
||||
// Set volume size in volumes table
|
||||
volume.setSize(volInfo.getSize());
|
||||
_volumeDao.update(volumeStore.getVolumeId(), volume);
|
||||
} else { // Put them in right status
|
||||
volumeStore.setDownloadPercent(100);
|
||||
volumeStore.setDownloadState(Status.DOWNLOADED);
|
||||
volumeStore.setState(ObjectInDataStoreStateMachine.State.Ready);
|
||||
volumeStore.setInstallPath(volInfo.getInstallPath());
|
||||
volumeStore.setSize(volInfo.getSize());
|
||||
volumeStore.setPhysicalSize(volInfo.getPhysicalSize());
|
||||
volumeStore.setLastUpdated(new Date());
|
||||
_volumeStoreDao.update(volumeStore.getId(), volumeStore);
|
||||
|
||||
if (volume.getSize() == 0) {
|
||||
// Set volume size in volumes table
|
||||
volume.setSize(volInfo.getSize());
|
||||
_volumeDao.update(volumeStore.getVolumeId(), volume);
|
||||
}
|
||||
|
||||
if (volInfo.getSize() > 0) {
|
||||
try {
|
||||
_resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(volume.getAccountId()),
|
||||
com.cloud.configuration.Resource.ResourceType.secondary_storage, volInfo.getSize()
|
||||
- volInfo.getPhysicalSize());
|
||||
} catch (ResourceAllocationException e) {
|
||||
s_logger.warn(e.getMessage());
|
||||
_alertMgr.sendAlert(AlertManager.ALERT_TYPE_RESOURCE_LIMIT_EXCEEDED,
|
||||
volume.getDataCenterId(), volume.getPodId(), e.getMessage(), e.getMessage());
|
||||
} finally {
|
||||
_resourceLimitMgr.recalculateResourceCount(volume.getAccountId(), volume.getDomainId(),
|
||||
com.cloud.configuration.Resource.ResourceType.secondary_storage.getOrdinal());
|
||||
}
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
// Volume is not on secondary but we should download.
|
||||
if (volumeStore.getDownloadState() != Status.DOWNLOADED) {
|
||||
s_logger.info("Volume Sync did not find " + volume.getName() + " ready on image store " + storeId
|
||||
+ ", will request download to start/resume shortly");
|
||||
toBeDownloaded.add(volumeStore);
|
||||
}
|
||||
}
|
||||
|
||||
if (volInfo.getSize() > 0) {
|
||||
try {
|
||||
_resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(volume.getAccountId()),
|
||||
com.cloud.configuration.Resource.ResourceType.secondary_storage, volInfo.getSize()
|
||||
- volInfo.getPhysicalSize());
|
||||
} catch (ResourceAllocationException e) {
|
||||
s_logger.warn(e.getMessage());
|
||||
_alertMgr.sendAlert(AlertManager.ALERT_TYPE_RESOURCE_LIMIT_EXCEEDED,
|
||||
volume.getDataCenterId(), volume.getPodId(), e.getMessage(), e.getMessage());
|
||||
} finally {
|
||||
_resourceLimitMgr.recalculateResourceCount(volume.getAccountId(), volume.getDomainId(),
|
||||
com.cloud.configuration.Resource.ResourceType.secondary_storage.getOrdinal());
|
||||
// Download volumes which haven't been downloaded yet.
|
||||
if (toBeDownloaded.size() > 0) {
|
||||
for (VolumeDataStoreVO volumeHost : toBeDownloaded) {
|
||||
if (volumeHost.getDownloadUrl() == null) { // If url is null we
|
||||
s_logger.info("Skip downloading volume " + volumeHost.getVolumeId() + " since no download url is specified.");
|
||||
continue;
|
||||
}
|
||||
s_logger.debug("Volume " + volumeHost.getVolumeId() + " needs to be downloaded to " + store.getName());
|
||||
// TODO: pass a callback later
|
||||
VolumeInfo vol = volFactory.getVolume(volumeHost.getVolumeId());
|
||||
createVolumeAsync(vol, store);
|
||||
}
|
||||
}
|
||||
|
||||
// Delete volumes which are not present on DB.
|
||||
for (Long uniqueName : volumeInfos.keySet()) {
|
||||
TemplateProp tInfo = volumeInfos.get(uniqueName);
|
||||
|
||||
//we cannot directly call expungeVolumeAsync here to
|
||||
// reuse delete logic since in this case, our db does not have
|
||||
// this template at all.
|
||||
VolumeObjectTO tmplTO = new VolumeObjectTO();
|
||||
tmplTO.setDataStore(store.getTO());
|
||||
tmplTO.setPath(tInfo.getInstallPath());
|
||||
tmplTO.setId(tInfo.getId());
|
||||
DeleteCommand dtCommand = new DeleteCommand(tmplTO);
|
||||
EndPoint ep = _epSelector.select(store);
|
||||
Answer answer = ep.sendMessage(dtCommand);
|
||||
if (answer == null || !answer.getResult()) {
|
||||
s_logger.info("Failed to deleted volume at store: " + store.getName());
|
||||
|
||||
} else {
|
||||
String description = "Deleted volume " + tInfo.getTemplateName() + " on secondary storage " + storeId;
|
||||
s_logger.info(description);
|
||||
}
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
// Volume is not on secondary but we should download.
|
||||
if (volumeStore.getDownloadState() != Status.DOWNLOADED) {
|
||||
s_logger.info("Volume Sync did not find " + volume.getName() + " ready on image store " + storeId
|
||||
+ ", will request download to start/resume shortly");
|
||||
toBeDownloaded.add(volumeStore);
|
||||
}
|
||||
}
|
||||
|
||||
// Download volumes which haven't been downloaded yet.
|
||||
if (toBeDownloaded.size() > 0) {
|
||||
for (VolumeDataStoreVO volumeHost : toBeDownloaded) {
|
||||
if (volumeHost.getDownloadUrl() == null) { // If url is null we
|
||||
// can't initiate the
|
||||
// download
|
||||
continue;
|
||||
finally{
|
||||
syncLock.unlock();
|
||||
}
|
||||
s_logger.debug("Volume " + volumeHost.getVolumeId() + " needs to be downloaded to " + store.getName());
|
||||
// TODO: pass a callback later
|
||||
VolumeInfo vol = volFactory.getVolume(volumeHost.getVolumeId());
|
||||
createVolumeAsync(vol, store);
|
||||
}
|
||||
}
|
||||
|
||||
// Delete volumes which are not present on DB.
|
||||
for (Long uniqueName : volumeInfos.keySet()) {
|
||||
TemplateProp tInfo = volumeInfos.get(uniqueName);
|
||||
|
||||
//we cannot directly call expungeVolumeAsync here to
|
||||
// reuse delete logic since in this case, our db does not have
|
||||
// this template at all.
|
||||
VolumeObjectTO tmplTO = new VolumeObjectTO();
|
||||
tmplTO.setDataStore(store.getTO());
|
||||
tmplTO.setPath(tInfo.getInstallPath());
|
||||
tmplTO.setId(tInfo.getId());
|
||||
DeleteCommand dtCommand = new DeleteCommand(tmplTO);
|
||||
EndPoint ep = _epSelector.select(store);
|
||||
Answer answer = ep.sendMessage(dtCommand);
|
||||
if (answer == null || !answer.getResult()) {
|
||||
s_logger.info("Failed to deleted volume at store: " + store.getName());
|
||||
|
||||
} else {
|
||||
String description = "Deleted volume " + tInfo.getTemplateName() + " on secondary storage " + storeId;
|
||||
s_logger.info(description);
|
||||
else {
|
||||
s_logger.info("Couldn't get global lock on " + lockString + ", another thread may be doing volume sync on data store " + storeId + " now.");
|
||||
}
|
||||
} finally {
|
||||
syncLock.releaseRef();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue