From 44d4113ae6fb0455a073fbf69566bc974fef8d6c Mon Sep 17 00:00:00 2001 From: Sheng Yang Date: Tue, 28 Jun 2011 14:00:27 -0700 Subject: [PATCH] bug 10429: backport redundant router Part 2 commit 797839360c65cd348d2eb20630521177ab0919de bug 9154: redundant virtual router commit 8ff7f230204d4d3a7a4adee75523a9a84f4276fe bug 9154: Replace domain_router.is_master with domain_router.redundant_state in DB commit 230b99e9e0b152648f1dd2a5eab6f22315b8e7b4 bug 9154: Add redundant state to DomainRouterResponse commit ccefb5ff5e83d713798a347c99bce1a0d04b4317 bug 9154: Add router fault state report commit 7a3090378f9785caecf741b70554f6ea17c41764 bug 9154: Send alert if found two virtual routers in master state commit 66831056e4bf27665871bccd24e6159071564847 bug 9154: Code clean up commit bf3f58a85741fa7118bd848a42d8b21baa4478d4 bug 9154: Add isRedundantRouter to DomainRouterResponse --- .../api/response/DomainRouterResponse.java | 21 ++++++- .../cloud/network/router/VirtualRouter.java | 8 +++ .../VirtualRoutingResource.java | 7 +-- .../xen/resource/CitrixResourceBase.java | 14 +++++ core/src/com/cloud/vm/DomainRouterVO.java | 19 +++--- scripts/vm/hypervisor/xenserver/vmops | 15 ++++- .../vm/hypervisor/xenserver/xenserver56/patch | 1 + .../src/com/cloud/api/ApiResponseHelper.java | 2 + .../VirtualNetworkApplianceManagerImpl.java | 62 ++++++++++++++----- setup/db/create-schema.sql | 2 +- 10 files changed, 122 insertions(+), 29 deletions(-) diff --git a/api/src/com/cloud/api/response/DomainRouterResponse.java b/api/src/com/cloud/api/response/DomainRouterResponse.java index 33ce06a53ac..59d8dcc50eb 100644 --- a/api/src/com/cloud/api/response/DomainRouterResponse.java +++ b/api/src/com/cloud/api/response/DomainRouterResponse.java @@ -119,8 +119,11 @@ public class DomainRouterResponse extends BaseResponse { @SerializedName("serviceofferingname") @Param(description="the name of the service offering of the virtual machine") private String serviceOfferingName; - + @SerializedName("isredundantrouter") @Param(description="if this router is an redundant virtual router") + private boolean isRedundantRouter; + @SerializedName("redundantstate") @Param(description="the state of redundant virtual router") + private String redundantState; @Override public Long getObjectId() { @@ -374,4 +377,20 @@ public class DomainRouterResponse extends BaseResponse { public void setServiceOfferingName(String serviceOfferingName) { this.serviceOfferingName = serviceOfferingName; } + + public String getRedundantState() { + return redundantState; + } + + public void setRedundantState(String redundantState) { + this.redundantState = redundantState; + } + + public boolean getIsRedundantRouter() { + return isRedundantRouter; + } + + public void setIsRedundantRouter(boolean isRedundantRouter) { + this.isRedundantRouter = isRedundantRouter; + } } diff --git a/api/src/com/cloud/network/router/VirtualRouter.java b/api/src/com/cloud/network/router/VirtualRouter.java index 32009620c88..50048e29305 100755 --- a/api/src/com/cloud/network/router/VirtualRouter.java +++ b/api/src/com/cloud/network/router/VirtualRouter.java @@ -29,4 +29,12 @@ public interface VirtualRouter extends VirtualMachine { DHCP_USERDATA } Role getRole(); + boolean getIsRedundantRouter(); + public enum RedundantState { + UNKNOWN, + MASTER, + BACKUP, + FAULT + } + RedundantState getRedundantState(); } diff --git a/core/src/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java b/core/src/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java index 810fe79fbe2..7f17325eb92 100755 --- a/core/src/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java +++ b/core/src/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java @@ -366,11 +366,10 @@ public class VirtualRoutingResource implements Manager { final String routerPrivateIPAddress = cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP); final String result = getRouterStatus(routerPrivateIPAddress); - CheckRouterAnswer answer = new CheckRouterAnswer(cmd, false, "Router return: " + result); - if (result != null) { - answer.setIsMaster(result.equals("Status: MASTER")); + if (result == null || result.isEmpty()) { + return new CheckRouterAnswer(cmd, "CheckRouterCommand failed"); } - return answer; + return new CheckRouterAnswer(cmd, result.equals("Status: MASTER"), result); } protected Answer execute(final CheckConsoleProxyLoadCommand cmd) { diff --git a/core/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java b/core/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java index dfca85c8c50..db3a6fd2ebe 100644 --- a/core/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java +++ b/core/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java @@ -65,6 +65,8 @@ import com.cloud.agent.api.CheckHealthAnswer; import com.cloud.agent.api.CheckHealthCommand; import com.cloud.agent.api.CheckOnHostAnswer; import com.cloud.agent.api.CheckOnHostCommand; +import com.cloud.agent.api.CheckRouterAnswer; +import com.cloud.agent.api.CheckRouterCommand; import com.cloud.agent.api.CheckVirtualMachineAnswer; import com.cloud.agent.api.CheckVirtualMachineCommand; import com.cloud.agent.api.CleanupNetworkRulesCmd; @@ -484,6 +486,8 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe return execute((OvsDestroyTunnelCommand)cmd); } else if (clazz == UpdateHostPasswordCommand.class) { return execute((UpdateHostPasswordCommand)cmd); + } else if (cmd instanceof CheckRouterCommand) { + return execute((CheckRouterCommand)cmd); } else { return Answer.createUnsupportedCommandAnswer(cmd); } @@ -1156,6 +1160,16 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe return new Answer(cmd); } + private CheckRouterAnswer execute(CheckRouterCommand cmd) { + Connection conn = getConnection(); + String args = cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP); + String result = callHostPlugin(conn, "vmops", "checkRouter", "args", args); + if (result == null || result.isEmpty()) { + return new CheckRouterAnswer(cmd, "CheckRouterCommand failed"); + } + return new CheckRouterAnswer(cmd, result.equals("Status: MASTER"), result); + } + protected MaintainAnswer execute(MaintainCommand cmd) { Connection conn = getConnection(); try { diff --git a/core/src/com/cloud/vm/DomainRouterVO.java b/core/src/com/cloud/vm/DomainRouterVO.java index 5a42b6354f0..34129cc4e3c 100755 --- a/core/src/com/cloud/vm/DomainRouterVO.java +++ b/core/src/com/cloud/vm/DomainRouterVO.java @@ -57,8 +57,9 @@ public class DomainRouterVO extends VMInstanceVO implements VirtualRouter { @Column(name="priority") int priority; - @Column(name="is_master") - boolean isMaster; + @Column(name="redundant_state") + @Enumerated(EnumType.STRING) + private RedundantState redundantState; @Column(name="role") @Enumerated(EnumType.STRING) @@ -75,13 +76,13 @@ public class DomainRouterVO extends VMInstanceVO implements VirtualRouter { long networkId, boolean isRedundantRouter, int priority, - boolean isMaster, + RedundantState redundantState, boolean haEnabled) { super(id, serviceOfferingId, name, name, Type.DomainRouter, templateId, hypervisorType, guestOSId, domainId, accountId, haEnabled); this.networkId = networkId; this.isRedundantRouter = isRedundantRouter; this.priority = priority; - this.isMaster = isMaster; + this.redundantState = redundantState; } public void setPublicIpAddress(String publicIpAddress) { @@ -138,6 +139,7 @@ public class DomainRouterVO extends VMInstanceVO implements VirtualRouter { this.role = role; } + @Override public boolean getIsRedundantRouter() { return this.isRedundantRouter; } @@ -159,12 +161,13 @@ public class DomainRouterVO extends VMInstanceVO implements VirtualRouter { this.priority = priority; } - public boolean getIsMaster() { - return this.isMaster; + @Override + public RedundantState getRedundantState() { + return this.redundantState; } - public void setIsMaster(boolean isMaster) { - this.isMaster = isMaster; + public void setRedundantState(RedundantState redundantState) { + this.redundantState = redundantState; } public void setServiceOfferingId(long serviceOfferingId) { diff --git a/scripts/vm/hypervisor/xenserver/vmops b/scripts/vm/hypervisor/xenserver/vmops index 8a3c0a40c6a..d1f63f50fe1 100755 --- a/scripts/vm/hypervisor/xenserver/vmops +++ b/scripts/vm/hypervisor/xenserver/vmops @@ -1028,8 +1028,21 @@ def network_rules(session, args): except: util.SMlog("Failed to network rule !") +@echo +def checkRouter(session, args): + sargs = args['args'] + cmd = sargs.split(' ') + cmd.insert(0, "/opt/xensource/bin/getRouterStatus.sh") + cmd.insert(0, "/bin/bash") + try: + txt = util.pread2(cmd) + except: + util.SMlog(" check router status fail! ") + txt = '' + return txt if __name__ == "__main__": - XenAPIPlugin.dispatch({"pingtest": pingtest, "setup_iscsi":setup_iscsi, "gethostvmstats": gethostvmstats, "getvncport": getvncport, "getgateway": getgateway, "preparemigration": preparemigration, "setIptables": setIptables, "pingdomr": pingdomr, "pingxenserver": pingxenserver, "ipassoc": ipassoc, "vm_data": vm_data, "savePassword": savePassword, "saveDhcpEntry": saveDhcpEntry, "setFirewallRule": setFirewallRule, "setLoadBalancerRule": setLoadBalancerRule, "createFile": createFile, "deleteFile": deleteFile, "networkUsage": networkUsage, "network_rules":network_rules, "can_bridge_firewall":can_bridge_firewall, "default_network_rules":default_network_rules, "destroy_network_rules_for_vm":destroy_network_rules_for_vm, "default_network_rules_systemvm":default_network_rules_systemvm, "get_rule_logs_for_vms":get_rule_logs_for_vms, "setLinkLocalIP":setLinkLocalIP, "cleanup_rules":cleanup_rules}) + XenAPIPlugin.dispatch({"pingtest": pingtest, "setup_iscsi":setup_iscsi, "gethostvmstats": gethostvmstats, "getvncport": getvncport, "getgateway": getgateway, "preparemigration": preparemigration, "setIptables": setIptables, "pingdomr": pingdomr, "pingxenserver": pingxenserver, "ipassoc": ipassoc, "vm_data": vm_data, "savePassword": savePassword, "saveDhcpEntry": saveDhcpEntry, "setFirewallRule": setFirewallRule, "setLoadBalancerRule": setLoadBalancerRule, "createFile": createFile, "deleteFile": deleteFile, "networkUsage": networkUsage, "network_rules":network_rules, "can_bridge_firewall":can_bridge_firewall, "default_network_rules":default_network_rules, "destroy_network_rules_for_vm":destroy_network_rules_for_vm, "default_network_rules_systemvm":default_network_rules_systemvm, "get_rule_logs_for_vms":get_rule_logs_for_vms, "setLinkLocalIP":setLinkLocalIP, "cleanup_rules":cleanup_rules, "checkRouter": checkRouter}) + diff --git a/scripts/vm/hypervisor/xenserver/xenserver56/patch b/scripts/vm/hypervisor/xenserver/xenserver56/patch index fd7409f4139..913d60fc948 100644 --- a/scripts/vm/hypervisor/xenserver/xenserver56/patch +++ b/scripts/vm/hypervisor/xenserver/xenserver56/patch @@ -42,3 +42,4 @@ InterfaceReconfigure.py=.,0755,/opt/xensource/bin fsimage.so=..,0755,/usr/lib/fs/ext2fs-lib create_privatetemplate_from_snapshot.sh=..,0755,/opt/xensource/bin upgrade_snapshot.sh=..,0755,/opt/xensource/bin +getRouterStatus.sh=../../../../network/domr/,0755,/opt/xensource/bin diff --git a/server/src/com/cloud/api/ApiResponseHelper.java b/server/src/com/cloud/api/ApiResponseHelper.java index bb99d12a3e9..71bf4f6cbf9 100755 --- a/server/src/com/cloud/api/ApiResponseHelper.java +++ b/server/src/com/cloud/api/ApiResponseHelper.java @@ -1083,6 +1083,8 @@ public class ApiResponseHelper implements ResponseGenerator { routerResponse.setTemplateId(router.getTemplateId()); routerResponse.setCreated(router.getCreated()); routerResponse.setState(router.getState()); + routerResponse.setIsRedundantRouter(router.getIsRedundantRouter()); + routerResponse.setRedundantState(router.getRedundantState().toString()); if (router.getHostId() != null) { routerResponse.setHostId(router.getHostId()); diff --git a/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java b/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java index 5f60e25f5b0..d4727212586 100755 --- a/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java +++ b/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java @@ -115,6 +115,7 @@ import com.cloud.network.dao.NetworkRuleConfigDao; import com.cloud.network.lb.LoadBalancingRule; import com.cloud.network.lb.LoadBalancingRule.LbDestination; import com.cloud.network.lb.LoadBalancingRulesManager; +import com.cloud.network.router.VirtualRouter.RedundantState; import com.cloud.network.router.VirtualRouter.Role; import com.cloud.network.rules.FirewallRule; import com.cloud.network.rules.FirewallRule.Purpose; @@ -700,12 +701,7 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian public CheckRouterTask() { } - @Override - public void run() { - - final List routers = _routerDao.listVirtualUpByHostId(null); - s_logger.debug("Found " + routers.size() + " running routers. "); - + private void updateRoutersRedundantState(List routers) { for (DomainRouterVO router : routers) { if (!router.getIsRedundantRouter()) { continue; @@ -721,15 +717,21 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian command.setAccessDetail(NetworkElementCommand.ROUTER_IP, router.getPrivateIpAddress()); command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); final CheckRouterAnswer answer = (CheckRouterAnswer) _agentMgr.easySend(router.getHostId(), command); - if (answer != null) { - if (answer.getResult()) { - router.setIsMaster(answer.getIsMaster()); + RedundantState state = RedundantState.UNKNOWN; + if (answer != null && answer.getResult()) { + if (answer.getIsMaster()) { + state = RedundantState.MASTER; } else { - router.setIsMaster(false); + if (answer.getDetails() != null) { + if (answer.getDetails().equals("Status: BACKUP")) { + state = RedundantState.BACKUP; + } else if (answer.getDetails().startsWith("Status: FAULT")) { + state = RedundantState.FAULT; + } + } } - } else { - router.setIsMaster(false); } + router.setRedundantState(state); Transaction txn = Transaction.open(Transaction.CLOUD_DB); try { txn.start(); @@ -744,6 +746,38 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian } } } + + private void checkDuplicateMaster(List routers) { + Map networkRouterMaps = new HashMap(); + for (DomainRouterVO router : routers) { + if (router.getRedundantState() == RedundantState.MASTER) { + if (networkRouterMaps.containsKey(router.getNetworkId())) { + DomainRouterVO dupRouter = networkRouterMaps.get(router.getNetworkId()); + String title = "More than one redundant virtual router is in MASTER state! Router " + router.getHostName() + " and router " + dupRouter.getHostName(); + String context = "Virtual router (name: " + router.getHostName() + ", id: " + router.getId() + " and router (name: " + + dupRouter.getHostName() + ", id: " + router.getId() + ") are both in MASTER state! If the problem persist, restart both of routers. "; + + _alertMgr.sendAlert(AlertManager.ALERT_TYPE_DOMAIN_ROUTER, router.getDataCenterIdToDeployIn(), router.getPodIdToDeployIn(), title, context); + _alertMgr.sendAlert(AlertManager.ALERT_TYPE_DOMAIN_ROUTER, dupRouter.getDataCenterIdToDeployIn(), dupRouter.getPodIdToDeployIn(), title, context); + } else { + networkRouterMaps.put(router.getNetworkId(), router); + } + } + } + } + + @Override + public void run() { + + final List routers = _routerDao.listVirtualUpByHostId(null); + s_logger.debug("Found " + routers.size() + " running routers. "); + + updateRoutersRedundantState(routers); + + /* FIXME assumed the a pair of redundant routers managed by same mgmt server, + * then the update above can get the latest status */ + checkDuplicateMaster(routers); + } } public static boolean isAdmin(short accountType) { @@ -838,7 +872,7 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian priority = 100 - routers.size() * 20; } router = new DomainRouterVO(id, _offering.getId(), VirtualMachineName.getRouterName(id, _instance), template.getId(), template.getHypervisorType(), template.getGuestOSId(), - owner.getDomainId(), owner.getId(), guestNetwork.getId(), isRedundant, priority, false, _offering.getOfferHA()); + owner.getDomainId(), owner.getId(), guestNetwork.getId(), isRedundant, priority, RedundantState.UNKNOWN, _offering.getOfferHA()); router = _itMgr.allocate(router, template, _offering, networks, plan, null, owner); // Creating stats entry for router UserStatisticsVO stats = _userStatsDao.findBy(owner.getId(), plan.getDataCenterId(), router.getNetworkId(), null, router.getId(), router.getType().toString()); @@ -952,7 +986,7 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian } router = new DomainRouterVO(id, _offering.getId(), VirtualMachineName.getRouterName(id, _instance), template.getId(), template.getHypervisorType(), template.getGuestOSId(), - owner.getDomainId(), owner.getId(), guestNetwork.getId(), false, 0, false,_offering.getOfferHA()); + owner.getDomainId(), owner.getId(), guestNetwork.getId(), false, 0, RedundantState.UNKNOWN,_offering.getOfferHA()); router.setRole(Role.DHCP_USERDATA); router = _itMgr.allocate(router, template, _offering, networks, plan, null, owner); // Creating stats entry for router diff --git a/setup/db/create-schema.sql b/setup/db/create-schema.sql index ed7ac75a101..3787c584aa6 100755 --- a/setup/db/create-schema.sql +++ b/setup/db/create-schema.sql @@ -896,7 +896,7 @@ CREATE TABLE `cloud`.`domain_router` ( `network_id` bigint unsigned NOT NULL COMMENT 'network configuration that this domain router belongs to', `is_redundant_router` int(1) unsigned NOT NULL COMMENT 'if in redundant router mode', `priority` int(4) unsigned COMMENT 'priority of router in the redundant router mode', - `is_master` int(1) unsigned DEFAULT 0 COMMENT 'if the router is master in redundant router mode', + `redundant_state` varchar(64) NOT NULL COMMENT 'the state of redundant virtual router', `role` varchar(64) NOT NULL COMMENT 'type of role played by this router', PRIMARY KEY (`id`), CONSTRAINT `fk_domain_router__id` FOREIGN KEY `fk_domain_router__id` (`id`) REFERENCES `vm_instance`(`id`) ON DELETE CASCADE