From cfd8cdbf20ac436ce9b80804453e2ee3b57baf8b Mon Sep 17 00:00:00 2001 From: Edison Su Date: Wed, 2 Feb 2011 12:01:31 -0500 Subject: [PATCH] bug 8195: add on-demand download for buildin templates, only download these buildin centos template after when you adding a host for the hypervisor status 8195: resolved fixed --- .../storage/template/TemplateConstants.java | 2 - server/src/com/cloud/dc/dao/ClusterDao.java | 2 + .../src/com/cloud/dc/dao/ClusterDaoImpl.java | 21 +++++ server/src/com/cloud/host/dao/HostDao.java | 5 +- .../src/com/cloud/host/dao/HostDaoImpl.java | 23 ++++++ .../storage/download/DownloadListener.java | 32 ++++---- .../storage/download/DownloadMonitor.java | 4 + .../storage/download/DownloadMonitorImpl.java | 82 +++++++++++++++---- .../secondary/SecondaryStorageDiscoverer.java | 19 +---- 9 files changed, 138 insertions(+), 52 deletions(-) diff --git a/core/src/com/cloud/storage/template/TemplateConstants.java b/core/src/com/cloud/storage/template/TemplateConstants.java index 32008e843be..782ed1c5b28 100644 --- a/core/src/com/cloud/storage/template/TemplateConstants.java +++ b/core/src/com/cloud/storage/template/TemplateConstants.java @@ -26,8 +26,6 @@ public final class TemplateConstants { public static final String DEFAULT_TMPLT_FIRST_LEVEL_DIR = "tmpl/"; public static final String DEFAULT_SYSTEM_VM_TEMPLATE_PATH = "template/tmpl/1/"; - public static final long DEFAULT_SYSTEM_VM_DB_ID = 1L; - public static final long DEFAULT_BUILTIN_VM_DB_ID = 2L; public static final String DEFAULT_SYSTEM_VM_TMPLT_NAME = "routing"; diff --git a/server/src/com/cloud/dc/dao/ClusterDao.java b/server/src/com/cloud/dc/dao/ClusterDao.java index da7e574911b..f0689608263 100644 --- a/server/src/com/cloud/dc/dao/ClusterDao.java +++ b/server/src/com/cloud/dc/dao/ClusterDao.java @@ -20,6 +20,7 @@ package com.cloud.dc.dao; import java.util.List; import com.cloud.dc.ClusterVO; +import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.utils.db.GenericDao; public interface ClusterDao extends GenericDao { @@ -27,4 +28,5 @@ public interface ClusterDao extends GenericDao { ClusterVO findBy(String name, long podId); List listByHyTypeWithoutGuid(String hyType); List listByZoneId(long zoneId); + List getAvailableHypervisorInZone(long zoneId); } diff --git a/server/src/com/cloud/dc/dao/ClusterDaoImpl.java b/server/src/com/cloud/dc/dao/ClusterDaoImpl.java index a76e30186ae..c57b15ee3df 100644 --- a/server/src/com/cloud/dc/dao/ClusterDaoImpl.java +++ b/server/src/com/cloud/dc/dao/ClusterDaoImpl.java @@ -17,11 +17,13 @@ */ package com.cloud.dc.dao; +import java.util.ArrayList; import java.util.List; import javax.ejb.Local; import com.cloud.dc.ClusterVO; +import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; @@ -31,6 +33,7 @@ public class ClusterDaoImpl extends GenericDaoBase implements C protected final SearchBuilder PodSearch; protected final SearchBuilder HyTypeWithoutGuidSearch; + protected final SearchBuilder AvailHyperSearch; protected final SearchBuilder ZoneSearch; protected ClusterDaoImpl() { super(); @@ -48,6 +51,11 @@ public class ClusterDaoImpl extends GenericDaoBase implements C ZoneSearch = createSearchBuilder(); ZoneSearch.and("dataCenterId", ZoneSearch.entity().getPodId(), SearchCriteria.Op.EQ); ZoneSearch.done(); + + AvailHyperSearch = createSearchBuilder(); + AvailHyperSearch.and("zoneId", AvailHyperSearch.entity().getDataCenterId(), SearchCriteria.Op.EQ); + AvailHyperSearch.groupBy(AvailHyperSearch.entity().getHypervisorType()); + AvailHyperSearch.done(); } @Override @@ -81,4 +89,17 @@ public class ClusterDaoImpl extends GenericDaoBase implements C return listBy(sc); } + + @Override + public List getAvailableHypervisorInZone(long zoneId) { + SearchCriteria sc = AvailHyperSearch.create(); + sc.setParameters("zoneId", zoneId); + List clusters = listBy(sc); + List hypers = new ArrayList(4); + for (ClusterVO cluster : clusters) { + hypers.add(cluster.getHypervisorType()); + } + + return hypers; + } } diff --git a/server/src/com/cloud/host/dao/HostDao.java b/server/src/com/cloud/host/dao/HostDao.java index 85bd2994580..fff04e21067 100644 --- a/server/src/com/cloud/host/dao/HostDao.java +++ b/server/src/com/cloud/host/dao/HostDao.java @@ -25,6 +25,7 @@ import com.cloud.host.HostVO; import com.cloud.host.Status; import com.cloud.host.Host.Type; import com.cloud.host.Status.Event; +import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.info.RunningHostCountInfo; import com.cloud.utils.db.GenericDao; @@ -137,5 +138,7 @@ public interface HostDao extends GenericDao { void loadDetails(HostVO host); - HostVO findConsoleProxyHost(String name, Type type); + HostVO findConsoleProxyHost(String name, Type type); + + List getAvailHypervisorInZone(long hostId, long zoneId); } diff --git a/server/src/com/cloud/host/dao/HostDaoImpl.java b/server/src/com/cloud/host/dao/HostDaoImpl.java index af74072d31c..4e01bcbc98b 100644 --- a/server/src/com/cloud/host/dao/HostDaoImpl.java +++ b/server/src/com/cloud/host/dao/HostDaoImpl.java @@ -37,6 +37,7 @@ import com.cloud.host.HostVO; import com.cloud.host.Status; import com.cloud.host.Host.Type; import com.cloud.host.Status.Event; +import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.info.RunningHostCountInfo; import com.cloud.utils.DateUtil; import com.cloud.utils.component.ComponentLocator; @@ -81,6 +82,7 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao protected final SearchBuilder MaintenanceCountSearch; protected final SearchBuilder ClusterSearch; protected final SearchBuilder ConsoleProxyHostSearch; + protected final SearchBuilder AvailHypevisorInZone; protected final Attribute _statusAttr; protected final Attribute _msIdAttr; @@ -206,6 +208,13 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao UnmanagedExternalNetworkApplianceSearch.and("types", UnmanagedExternalNetworkApplianceSearch.entity().getType(), SearchCriteria.Op.IN); UnmanagedExternalNetworkApplianceSearch.and("lastPinged", UnmanagedExternalNetworkApplianceSearch.entity().getLastPinged(), SearchCriteria.Op.LTEQ); UnmanagedExternalNetworkApplianceSearch.done(); + + AvailHypevisorInZone = createSearchBuilder(); + AvailHypevisorInZone.and("zoneId", AvailHypevisorInZone.entity().getDataCenterId(), SearchCriteria.Op.EQ); + AvailHypevisorInZone.and("hostId", AvailHypevisorInZone.entity().getId(), SearchCriteria.Op.NEQ); + AvailHypevisorInZone.and("type", AvailHypevisorInZone.entity().getType(), SearchCriteria.Op.EQ); + AvailHypevisorInZone.groupBy(AvailHypevisorInZone.entity().getHypervisorType()); + AvailHypevisorInZone.done(); _statusAttr = _allAttributes.get("status"); _msIdAttr = _allAttributes.get("managementServerId"); @@ -616,6 +625,20 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao assert tg != null : "how can this be wrong!"; return s_seqFetcher.getNextSequence(Long.class, tg, hostId); + } + + @Override + public List getAvailHypervisorInZone(long hostId, long zoneId) { + SearchCriteria sc = AvailHypevisorInZone.create(); + sc.setParameters("zoneId", zoneId); + sc.setParameters("hostId", hostId); + sc.setParameters("type", Host.Type.Routing); + List hosts = listBy(sc); + List hypers = new ArrayList(4); + for (HostVO host : hosts) { + hypers.add(host.getHypervisorType()); + } + return hypers; } } diff --git a/server/src/com/cloud/storage/download/DownloadListener.java b/server/src/com/cloud/storage/download/DownloadListener.java index 61a117ac9f1..152ee474cd4 100644 --- a/server/src/com/cloud/storage/download/DownloadListener.java +++ b/server/src/com/cloud/storage/download/DownloadListener.java @@ -32,6 +32,7 @@ import com.cloud.agent.api.AgentControlCommand; import com.cloud.agent.api.Answer; import com.cloud.agent.api.Command; import com.cloud.agent.api.StartupCommand; +import com.cloud.agent.api.StartupRoutingCommand; import com.cloud.agent.api.StartupStorageCommand; import com.cloud.agent.api.storage.DownloadAnswer; import com.cloud.agent.api.storage.DownloadCommand; @@ -276,24 +277,25 @@ public class DownloadListener implements Listener { @Override public void processConnect(HostVO agent, StartupCommand cmd) throws ConnectionException { - if (!(cmd instanceof StartupStorageCommand)) { + if (!((cmd instanceof StartupStorageCommand) || (cmd instanceof StartupRoutingCommand))) { return; } - if (cmd.getGuid().startsWith("iso:")) { - //FIXME: do not download template for ISO secondary - return; - } - - long agentId = agent.getId(); - - StartupStorageCommand storage = (StartupStorageCommand)cmd; - if (storage.getResourceType() == Storage.StorageResourceType.STORAGE_HOST || - storage.getResourceType() == Storage.StorageResourceType.SECONDARY_STORAGE ) - { - downloadMonitor.handleTemplateSync(agentId, storage.getTemplateInfo()); + if (cmd instanceof StartupRoutingCommand) { + downloadMonitor.handleSysTemplateDownload(agent); } else { - //downloadMonitor.handlePoolTemplateSync(storage.getPoolInfo(), storage.getTemplateInfo()); - //no need to do anything. The storagepoolmonitor will initiate template sync. + if (cmd.getGuid().startsWith("iso:")) { + //FIXME: do not download template for ISO secondary + return; + } + + long agentId = agent.getId(); + + StartupStorageCommand storage = (StartupStorageCommand)cmd; + if (storage.getResourceType() == Storage.StorageResourceType.STORAGE_HOST || + storage.getResourceType() == Storage.StorageResourceType.SECONDARY_STORAGE ) + { + downloadMonitor.handleTemplateSync(agentId, storage.getTemplateInfo()); + } } } diff --git a/server/src/com/cloud/storage/download/DownloadMonitor.java b/server/src/com/cloud/storage/download/DownloadMonitor.java index 46f3a3f8ee9..739685e9722 100644 --- a/server/src/com/cloud/storage/download/DownloadMonitor.java +++ b/server/src/com/cloud/storage/download/DownloadMonitor.java @@ -18,6 +18,7 @@ package com.cloud.storage.download; +import java.util.List; import java.util.Map; import com.cloud.exception.InvalidParameterValueException; @@ -43,4 +44,7 @@ public interface DownloadMonitor extends Manager{ public void copyTemplate(VMTemplateVO template, HostVO sourceServer, HostVO destServer) throws InvalidParameterValueException, StorageUnavailableException; + /*When new host added, take a look at if there are templates needed to be downloaded for the same hypervisor as the host*/ + void handleSysTemplateDownload(HostVO hostId); + } \ No newline at end of file diff --git a/server/src/com/cloud/storage/download/DownloadMonitorImpl.java b/server/src/com/cloud/storage/download/DownloadMonitorImpl.java index 0cfb1dc7724..21c41475de1 100755 --- a/server/src/com/cloud/storage/download/DownloadMonitorImpl.java +++ b/server/src/com/cloud/storage/download/DownloadMonitorImpl.java @@ -40,6 +40,7 @@ import com.cloud.agent.api.storage.DownloadProgressCommand.RequestType; import com.cloud.alert.AlertManager; import com.cloud.configuration.dao.ConfigurationDao; import com.cloud.dc.DataCenterVO; +import com.cloud.dc.dao.ClusterDao; import com.cloud.dc.dao.DataCenterDao; import com.cloud.event.EventTypes; import com.cloud.event.UsageEventVO; @@ -105,6 +106,11 @@ public class DownloadMonitorImpl implements DownloadMonitor { @Inject private UsageEventDao _usageEventDao; + + @Inject + private ClusterDao _clusterDao; + @Inject + private HostDao _hostDao; private String _name; private Boolean _sslCopy = new Boolean(false); @@ -381,7 +387,47 @@ public class DownloadMonitorImpl implements DownloadMonitor { } } + + @Override + public void handleSysTemplateDownload(HostVO host) { + List hypers = _hostDao.getAvailHypervisorInZone(host.getId(), host.getDataCenterId()); + HypervisorType hostHyper = host.getHypervisorType(); + if (hypers.contains(hostHyper)) { + return; + } + Set toBeDownloaded = new HashSet(); + List ssHosts = _hostDao.listBy(Host.Type.SecondaryStorage, host.getDataCenterId()); + if (ssHosts == null || ssHosts.isEmpty()) { + return; + } + HostVO sshost = ssHosts.get(0); + /*Download all the templates in zone with the same hypervisortype*/ + + List rtngTmplts = _templateDao.listAllSystemVMTemplates(); + List defaultBuiltin = _templateDao.listDefaultBuiltinTemplates(); + + + for (VMTemplateVO rtngTmplt : rtngTmplts) { + if (rtngTmplt.getHypervisorType() == hostHyper) { + toBeDownloaded.add(rtngTmplt); + } + } + + for (VMTemplateVO builtinTmplt : defaultBuiltin) { + if (builtinTmplt.getHypervisorType() == hostHyper) { + toBeDownloaded.add(builtinTmplt); + } + } + + for (VMTemplateVO template: toBeDownloaded) { + VMTemplateHostVO tmpltHost = _vmTemplateHostDao.findByHostTemplate(sshost.getId(), template.getId()); + if (tmpltHost == null) { + downloadTemplateToStorage(template, sshost); + } + } + } + @Override public void handleTemplateSync(long sserverId, Map templateInfos) { HostVO storageHost = _serverDao.findById(sserverId); @@ -389,26 +435,26 @@ public class DownloadMonitorImpl implements DownloadMonitor { s_logger.warn("Huh? Agent id " + sserverId + " does not correspond to a row in hosts table?"); return; } - + Set toBeDownloaded = new HashSet(); List allTemplates = _templateDao.listAllInZone(storageHost.getDataCenterId()); List rtngTmplts = _templateDao.listAllSystemVMTemplates(); List defaultBuiltin = _templateDao.listDefaultBuiltinTemplates(); - - if (rtngTmplts != null) { - for (VMTemplateVO rtngTmplt : rtngTmplts) { - if (!allTemplates.contains(rtngTmplt)) { - allTemplates.add(rtngTmplt); - } - } - } + if (rtngTmplts != null) { + for (VMTemplateVO rtngTmplt : rtngTmplts) { + if (!allTemplates.contains(rtngTmplt)) { + allTemplates.add(rtngTmplt); + } + } + } + if (defaultBuiltin != null) { - for (VMTemplateVO builtinTmplt : defaultBuiltin) { - if (!allTemplates.contains(builtinTmplt)) { - allTemplates.add(builtinTmplt); - } - } + for (VMTemplateVO builtinTmplt : defaultBuiltin) { + if (!allTemplates.contains(builtinTmplt)) { + allTemplates.add(builtinTmplt); + } + } } toBeDownloaded.addAll(allTemplates); @@ -471,9 +517,13 @@ public class DownloadMonitorImpl implements DownloadMonitor { if (sserver == null) { throw new CloudRuntimeException("Unable to find host from id"); } + /*Only download templates whose hypervirsor type is in the zone*/ + List availHypers = _clusterDao.getAvailableHypervisorInZone(sserver.getDataCenterId()); for (VMTemplateVO tmplt: toBeDownloaded) { - s_logger.debug("Template " + tmplt.getName() + " needs to be downloaded to " + sserver.getName()); - downloadTemplateToStorage(tmplt, sserver); + if (availHypers.contains(tmplt.getHypervisorType())) { + s_logger.debug("Template " + tmplt.getName() + " needs to be downloaded to " + sserver.getName()); + downloadTemplateToStorage(tmplt, sserver); + } } } diff --git a/server/src/com/cloud/storage/secondary/SecondaryStorageDiscoverer.java b/server/src/com/cloud/storage/secondary/SecondaryStorageDiscoverer.java index 30b0555c28c..d37f6f0c271 100644 --- a/server/src/com/cloud/storage/secondary/SecondaryStorageDiscoverer.java +++ b/server/src/com/cloud/storage/secondary/SecondaryStorageDiscoverer.java @@ -258,7 +258,6 @@ public class SecondaryStorageDiscoverer extends DiscovererBase implements Discov if (_useServiceVM) { for (HostVO h: hosts) { _hostDao.disconnect(h, Event.AgentDisconnected, msId); - associateSystemVmTemplate(h.getId(), h.getDataCenterId()); } } for (HostVO h: hosts) { @@ -267,24 +266,8 @@ public class SecondaryStorageDiscoverer extends DiscovererBase implements Discov } - protected void associateSystemVmTemplate(long hostId, long dcId) { - VMTemplateVO tmplt = _tmpltDao.findById(TemplateConstants.DEFAULT_SYSTEM_VM_DB_ID); - if (tmplt == null) { - throw new CloudRuntimeException("Cannot find routing template in vm_template table. Check your configuration"); - } - VMTemplateHostVO tmpltHost = _vmTemplateHostDao.findByHostTemplate(hostId, TemplateConstants.DEFAULT_SYSTEM_VM_DB_ID); - if (tmpltHost == null) { - VMTemplateHostVO vmTemplateHost = new VMTemplateHostVO(hostId, TemplateConstants.DEFAULT_SYSTEM_VM_DB_ID, new Date(), 100, VMTemplateStorageResourceAssoc.Status.DOWNLOADED, null, null, null, TemplateConstants.DEFAULT_SYSTEM_VM_TEMPLATE_PATH, null); - _vmTemplateHostDao.persist(vmTemplateHost); - } - } - private void associateTemplatesToZone(long hostId, long dcId){ - VMTemplateZoneVO tmpltZone = _vmTemplateZoneDao.findByZoneTemplate(dcId, TemplateConstants.DEFAULT_SYSTEM_VM_DB_ID); - if (tmpltZone == null) { - VMTemplateZoneVO vmTemplateZone = new VMTemplateZoneVO(dcId, TemplateConstants.DEFAULT_SYSTEM_VM_DB_ID, new Date()); - _vmTemplateZoneDao.persist(vmTemplateZone); - } + VMTemplateZoneVO tmpltZone; List allTemplates = _vmTemplateDao.listAll(); for (VMTemplateVO vt: allTemplates){