From 94ea2736f4a1614e45f4bc56388aad7adeb22a08 Mon Sep 17 00:00:00 2001 From: Likitha Shetty Date: Sun, 10 Nov 2013 23:31:56 +0530 Subject: [PATCH 001/169] CLOUDSTACK-5927. Storage garbage collector is trying to destroy the SSVM/CPVM root disk that is in use. In case of VMware VM during root volume preparation if we are switching to a new volume, force expunge root disk that was created from the old template. Because otherwise storage garbage collector will later try to expunge the old disk marked for expunge and fail with 'Cannot delete file' exception since in VMware the new root vmdk has the same name and is now in use. --- .../engine/orchestration/VolumeOrchestrator.java | 11 ++++++++++- .../storage/resource/VmwareStorageProcessor.java | 4 +--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java index 46199692563..a3ab5a4164a 100644 --- a/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java +++ b/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java @@ -808,7 +808,16 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati } catch (NoTransitionException e) { s_logger.debug("Unable to destroy existing volume: " + e.toString()); } - + // In case of VMware VM will continue to use the old root disk until expunged, so force expunge old root disk + if (vm.getHypervisorType() == HypervisorType.VMware) { + s_logger.info("Expunging volume " + existingVolume.getId() + " from primary data store"); + AsyncCallFuture future = volService.expungeVolumeAsync(volFactory.getVolume(existingVolume.getId())); + try { + future.get(); + } catch (Exception e) { + s_logger.debug("Failed to expunge volume:" + existingVolume.getId(), e); + } + } return newVolume; } }); diff --git a/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageProcessor.java index 1e4f8d7f01b..2089c084d73 100644 --- a/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -363,9 +363,7 @@ public class VmwareStorageProcessor implements StorageProcessor { // restoreVM - move the new ROOT disk into corresponding VM folder String vmInternalCSName = volume.getVmName(); if (dsMo.folderExists(String.format("[%s]", dsMo.getName()), vmInternalCSName)) { - String oldRootDisk = VmwareStorageLayoutHelper.getVmwareDatastorePathFromVmdkFileName(dsMo, vmInternalCSName, vmdkFileBaseName); - if (oldRootDisk != null) - VmwareStorageLayoutHelper.syncVolumeToVmDefaultFolder(dcMo, vmInternalCSName, dsMo, vmdkFileBaseName); + VmwareStorageLayoutHelper.syncVolumeToVmDefaultFolder(dcMo, vmInternalCSName, dsMo, vmdkName); } VolumeObjectTO newVol = new VolumeObjectTO(); From 78f62c63479a81383ff8957a12c88ee4c181cfda Mon Sep 17 00:00:00 2001 From: Alex Hitchins Date: Wed, 22 Jan 2014 16:49:33 +0000 Subject: [PATCH 002/169] Added required dependancy to agent setup. --- packaging/centos63/cloud.spec | 1 + 1 file changed, 1 insertion(+) diff --git a/packaging/centos63/cloud.spec b/packaging/centos63/cloud.spec index 7132d4f2cc4..8029b61cc31 100644 --- a/packaging/centos63/cloud.spec +++ b/packaging/centos63/cloud.spec @@ -111,6 +111,7 @@ The Apache CloudStack files shared between agent and management server %package agent Summary: CloudStack Agent for KVM hypervisors +Requires: openssh-clients Requires: java >= 1.6.0 Requires: %{name}-common = %{_ver} Requires: libvirt From 0679af343453bf888f0ddefdd36937713aacbc28 Mon Sep 17 00:00:00 2001 From: Mike Tutkowski Date: Wed, 22 Jan 2014 11:08:34 -0700 Subject: [PATCH 003/169] CLOUDSTACK-5873: [Automation] Failed to attach volume to VM, if the vm is created with option startvm=false --- .../src/com/cloud/vm/VirtualMachineManagerImpl.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java index df51a3c5266..6dc58944d07 100755 --- a/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java @@ -1121,6 +1121,13 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac VolumeVO volume = _volsDao.findById(volumeId); disk.setPath(volume.get_iScsiName()); + + if (disk.getData() instanceof VolumeObjectTO) { + VolumeObjectTO volTo = (VolumeObjectTO)disk.getData(); + + volTo.setPath(volume.get_iScsiName()); + } + volume.setPath(volume.get_iScsiName()); _volsDao.update(volumeId, volume); From 50f311ed947bd22404465ca5df416abe0fdcfc4e Mon Sep 17 00:00:00 2001 From: Kelven Yang Date: Wed, 22 Jan 2014 11:09:45 -0800 Subject: [PATCH 004/169] CLOUDSTACK-5928: disable host delta sync when new VM sync is enabled --- .../cloud/vm/VirtualMachineManagerImpl.java | 38 +++++++-- .../vmware/manager/VmwareManagerImpl.java | 85 ++++++++++--------- .../cloud/storage/VolumeApiServiceImpl.java | 2 +- .../vm/snapshot/VMSnapshotManagerImpl.java | 2 +- 4 files changed, 75 insertions(+), 52 deletions(-) diff --git a/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java index 6dc58944d07..40b9151442a 100755 --- a/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java @@ -337,7 +337,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac // TODO, remove it after transient period is over static final ConfigKey VmJobEnabled = new ConfigKey("Advanced", - Boolean.class, "vm.job.enabled", "false", + Boolean.class, "vm.job.enabled", "true", "True to enable new VM sync model. false to use the old way", false); static final ConfigKey VmJobCheckInterval = new ConfigKey("Advanced", @@ -2944,12 +2944,14 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac if (cmd instanceof PingRoutingCommand) { PingRoutingCommand ping = (PingRoutingCommand)cmd; if (ping.getNewStates() != null && ping.getNewStates().size() > 0) { - Commands commands = deltaHostSync(agentId, ping.getNewStates()); - if (commands.size() > 0) { - try { - _agentMgr.send(agentId, commands, this); - } catch (final AgentUnavailableException e) { - s_logger.warn("Agent is now unavailable", e); + if (!VmJobEnabled.value()) { + Commands commands = deltaHostSync(agentId, ping.getNewStates()); + if (commands.size() > 0) { + try { + _agentMgr.send(agentId, commands, this); + } catch (final AgentUnavailableException e) { + s_logger.warn("Agent is now unavailable", e); + } } } } @@ -3973,6 +3975,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac s_logger.warn("VM " + vmId + " no longer exists when processing VM state report"); } } else { + s_logger.info("There is pending job working on the VM. vm id: " + vmId + ", postpone power-change report by resetting power-change counters"); + // reset VM power state tracking so that we won't lost signal when VM has // been translated to _vmDao.resetVmPowerStateTracking(vmId); @@ -3981,19 +3985,23 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac private void handlePowerOnReportWithNoPendingJobsOnVM(VMInstanceVO vm) { // - // 1) handle left-over transitional VM states + // 1) handle left-over transitional VM states // 2) handle out of band VM live migration // 3) handle out of sync stationary states, marking VM from Stopped to Running with // alert messages // switch (vm.getState()) { case Starting: + s_logger.info("VM " + vm.getInstanceName() + " is at " + vm.getState() + " and we received a power-on report while there is no pending jobs on it"); + try { stateTransitTo(vm, VirtualMachine.Event.FollowAgentPowerOnReport, vm.getPowerHostId()); } catch (NoTransitionException e) { s_logger.warn("Unexpected VM state transition exception, race-condition?", e); } + s_logger.info("VM " + vm.getInstanceName() + " is sync-ed to at Running state according to power-on report from hypervisor"); + // we need to alert admin or user about this risky state transition _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_SYNC, vm.getDataCenterId(), vm.getPodIdToDeployIn(), VM_SYNC_ALERT_SUBJECT, "VM " + vm.getHostName() + "(" + vm.getInstanceName() @@ -4008,10 +4016,13 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } catch (NoTransitionException e) { s_logger.warn("Unexpected VM state transition exception, race-condition?", e); } + break; case Stopping: case Stopped: + s_logger.info("VM " + vm.getInstanceName() + " is at " + vm.getState() + " and we received a power-on report while there is no pending jobs on it"); + try { stateTransitTo(vm, VirtualMachine.Event.FollowAgentPowerOnReport, vm.getPowerHostId()); } catch (NoTransitionException e) { @@ -4020,6 +4031,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_SYNC, vm.getDataCenterId(), vm.getPodIdToDeployIn(), VM_SYNC_ALERT_SUBJECT, "VM " + vm.getHostName() + "(" + vm.getInstanceName() + ") state is sync-ed (" + vm.getState() + " -> Running) from out-of-context transition. VM network environment may need to be reset"); + + s_logger.info("VM " + vm.getInstanceName() + " is sync-ed to at Running state according to power-on report from hypervisor"); break; case Destroyed: @@ -4029,11 +4042,13 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac break; case Migrating: + s_logger.info("VM " + vm.getInstanceName() + " is at " + vm.getState() + " and we received a power-on report while there is no pending jobs on it"); try { stateTransitTo(vm, VirtualMachine.Event.FollowAgentPowerOnReport, vm.getPowerHostId()); } catch (NoTransitionException e) { s_logger.warn("Unexpected VM state transition exception, race-condition?", e); } + s_logger.info("VM " + vm.getInstanceName() + " is sync-ed to at Running state according to power-on report from hypervisor"); break; case Error: @@ -4046,7 +4061,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac private void handlePowerOffReportWithNoPendingJobsOnVM(VMInstanceVO vm) { - // 1) handle left-over transitional VM states + // 1) handle left-over transitional VM states // 2) handle out of sync stationary states, schedule force-stop to release resources // switch (vm.getState()) { @@ -4055,14 +4070,19 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac case Running: case Stopped: case Migrating: + s_logger.info("VM " + vm.getInstanceName() + " is at " + vm.getState() + " and we received a power-off report while there is no pending jobs on it"); try { stateTransitTo(vm, VirtualMachine.Event.FollowAgentPowerOffReport, vm.getPowerHostId()); } catch (NoTransitionException e) { s_logger.warn("Unexpected VM state transition exception, race-condition?", e); } + _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_SYNC, vm.getDataCenterId(), vm.getPodIdToDeployIn(), VM_SYNC_ALERT_SUBJECT, "VM " + vm.getHostName() + "(" + vm.getInstanceName() + ") state is sync-ed (" + vm.getState() + " -> Stopped) from out-of-context transition."); + + s_logger.info("VM " + vm.getInstanceName() + " is sync-ed to at Stopped state according to power-on report from hypervisor"); + // TODO: we need to forcely release all resource allocation break; diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java index bf56c54440b..028fa942d57 100755 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java @@ -132,8 +132,8 @@ import com.cloud.vm.DomainRouterVO; public class VmwareManagerImpl extends ManagerBase implements VmwareManager, VmwareStorageMount, Listener, VmwareDatacenterService { private static final Logger s_logger = Logger.getLogger(VmwareManagerImpl.class); - private static final int STARTUP_DELAY = 60000; // 60 seconds - private static final long DEFAULT_HOST_SCAN_INTERVAL = 600000; // every 10 minutes + private static final int STARTUP_DELAY = 60000; // 60 seconds + private static final long DEFAULT_HOST_SCAN_INTERVAL = 600000; // every 10 minutes private long _hostScanInterval = DEFAULT_HOST_SCAN_INTERVAL; int _timeout; @@ -173,7 +173,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw String _managemetPortGroupName; String _defaultSystemVmNicAdapterType = VirtualEthernetCardType.E1000.toString(); String _recycleHungWorker = "false"; - long _hungWorkerTimeout = 7200000; // 2 hour + long _hungWorkerTimeout = 7200000; // 2 hour int _additionalPortRangeStart; int _additionalPortRangeSize; int _routerExtraPublicNics = 2; @@ -296,10 +296,10 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw if(_recycleHungWorker == null || _recycleHungWorker.isEmpty()) { _recycleHungWorker = "false"; } - + value = _configDao.getValue(Config.VmwareHungWorkerTimeout.key()); if(value != null) - _hungWorkerTimeout = Long.parseLong(value) * 1000; + _hungWorkerTimeout = Long.parseLong(value) * 1000; _rootDiskController = _configDao.getValue(Config.VmwareRootDiskControllerType.key()); if(_rootDiskController == null || _rootDiskController.isEmpty()) { @@ -459,7 +459,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw hostSpec.setUserName(userName); hostSpec.setPassword(password); hostSpec.setHostName(host); - hostSpec.setForce(true); // forcely take over the host + hostSpec.setForce(true); // forcely take over the host ManagedObjectReference morTask = serviceContext.getService().addHostTask(morCluster, hostSpec, true, null, null); boolean taskResult = vclient.waitForTask(morTask); @@ -547,49 +547,49 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw public void gcLeftOverVMs(VmwareContext context) { VmwareCleanupMaid.gcLeftOverVMs(context); } - + @Override public boolean needRecycle(String workerTag) { - if(s_logger.isInfoEnabled()) - s_logger.info("Check to see if a worker VM with tag " + workerTag + " needs to be recycled"); - - if(workerTag == null || workerTag.isEmpty()) { - s_logger.error("Invalid worker VM tag " + workerTag); - return false; - } - - String tokens[] = workerTag.split("-"); - if(tokens.length != 3) { - s_logger.error("Invalid worker VM tag " + workerTag); - return false; - } - - long startTick = Long.parseLong(tokens[0]); - long msid = Long.parseLong(tokens[1]); - long runid = Long.parseLong(tokens[2]); - + if (s_logger.isInfoEnabled()) + s_logger.info("Check to see if a worker VM with tag " + workerTag + " needs to be recycled"); + + if (workerTag == null || workerTag.isEmpty()) { + s_logger.error("Invalid worker VM tag " + workerTag); + return false; + } + + String tokens[] = workerTag.split("-"); + if (tokens.length != 3) { + s_logger.error("Invalid worker VM tag " + workerTag); + return false; + } + + long startTick = Long.parseLong(tokens[0]); + long msid = Long.parseLong(tokens[1]); + long runid = Long.parseLong(tokens[2]); + if(_mshostPeerDao.countStateSeenInPeers(msid, runid, ManagementServerHost.State.Down) > 0) { - if(s_logger.isInfoEnabled()) - s_logger.info("Worker VM's owner management server node has been detected down from peer nodes, recycle it"); - return true; + if (s_logger.isInfoEnabled()) + s_logger.info("Worker VM's owner management server node has been detected down from peer nodes, recycle it"); + return true; } - + if(msid == _clusterMgr.getManagementNodeId() && runid != _clusterMgr.getCurrentRunId()) { - if(s_logger.isInfoEnabled()) - s_logger.info("Worker VM's owner management server has changed runid, recycle it"); - return true; + if (s_logger.isInfoEnabled()) + s_logger.info("Worker VM's owner management server has changed runid, recycle it"); + return true; } - + // disable time-out check until we have found out a VMware API that can check if // there are pending tasks on the subject VM -/* - if(System.currentTimeMillis() - startTick > _hungWorkerTimeout) { - if(s_logger.isInfoEnabled()) - s_logger.info("Worker VM expired, seconds elapsed: " + (System.currentTimeMillis() - startTick) / 1000); - return true; - } -*/ - return false; + /* + if(System.currentTimeMillis() - startTick > _hungWorkerTimeout) { + if(s_logger.isInfoEnabled()) + s_logger.info("Worker VM expired, seconds elapsed: " + (System.currentTimeMillis() - startTick) / 1000); + return true; + } + */ + return false; } @Override @@ -619,6 +619,9 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw ", destination: " + destIso.getAbsolutePath()); try { FileUtil.copyfile(srcIso, destIso); + + s_logger.info("System VM patch ISO file is copied to secondary storage. source ISO: " + srcIso.getAbsolutePath() + + ", destination: " + destIso.getAbsolutePath()); } catch(IOException e) { s_logger.error("Unexpected exception ", e); diff --git a/server/src/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/com/cloud/storage/VolumeApiServiceImpl.java index f46f1e9e0d0..6be43808008 100644 --- a/server/src/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/com/cloud/storage/VolumeApiServiceImpl.java @@ -333,7 +333,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic // TODO static final ConfigKey VmJobEnabled = new ConfigKey("Advanced", - Boolean.class, "vm.job.enabled", "false", + Boolean.class, "vm.job.enabled", "true", "True to enable new VM sync model. false to use the old way", false); static final ConfigKey VmJobCheckInterval = new ConfigKey("Advanced", Long.class, "vm.job.check.interval", "3000", diff --git a/server/src/com/cloud/vm/snapshot/VMSnapshotManagerImpl.java b/server/src/com/cloud/vm/snapshot/VMSnapshotManagerImpl.java index e273e1a2338..ceded0867dc 100644 --- a/server/src/com/cloud/vm/snapshot/VMSnapshotManagerImpl.java +++ b/server/src/com/cloud/vm/snapshot/VMSnapshotManagerImpl.java @@ -136,7 +136,7 @@ public class VMSnapshotManagerImpl extends ManagerBase implements VMSnapshotMana // TODO static final ConfigKey VmJobEnabled = new ConfigKey("Advanced", - Boolean.class, "vm.job.enabled", "false", + Boolean.class, "vm.job.enabled", "true", "True to enable new VM sync model. false to use the old way", false); static final ConfigKey VmJobCheckInterval = new ConfigKey("Advanced", Long.class, "vm.job.check.interval", "3000", From f160b31f51f9165a94c6912bba3b4f6447b73854 Mon Sep 17 00:00:00 2001 From: Alena Prokharchyk Date: Tue, 7 Jan 2014 12:15:37 -0800 Subject: [PATCH 005/169] CLOUDSTACK-5930: Changed "authenticate" method to return both - result of authentication, and action to perform when authentication failed - to the accountManagerImpl. Only if authenicators request INCREMENT_INCORRECT_LOGIN_ATTEMPT_COUNT, the incorrect_login_attempts parameter will be increased Signed-off-by: Alena Prokharchyk Conflicts: plugins/user-authenticators/ldap/src/org/apache/cloudstack/ldap/LdapAuthenticator.java plugins/user-authenticators/md5/src/com/cloud/server/auth/MD5UserAuthenticator.java plugins/user-authenticators/plain-text/src/com/cloud/server/auth/PlainTextUserAuthenticator.java plugins/user-authenticators/sha256salted/src/com/cloud/server/auth/SHA256SaltedUserAuthenticator.java server/src/com/cloud/server/auth/UserAuthenticator.java server/src/com/cloud/user/AccountManagerImpl.java --- .../cloudstack/ldap/LdapAuthenticator.java | 31 +++++---- .../server/auth/MD5UserAuthenticator.java | 64 ++++++++++--------- .../auth/PlainTextUserAuthenticator.java | 40 ++++++------ .../auth/SHA256SaltedUserAuthenticator.java | 13 ++-- .../server/auth/test/AuthenticatorTest.java | 10 +-- .../cloud/server/auth/UserAuthenticator.java | 35 +++++----- .../com/cloud/user/AccountManagerImpl.java | 25 +++++--- 7 files changed, 122 insertions(+), 96 deletions(-) diff --git a/plugins/user-authenticators/ldap/src/org/apache/cloudstack/ldap/LdapAuthenticator.java b/plugins/user-authenticators/ldap/src/org/apache/cloudstack/ldap/LdapAuthenticator.java index 559a9794b2d..dac917bba6d 100644 --- a/plugins/user-authenticators/ldap/src/org/apache/cloudstack/ldap/LdapAuthenticator.java +++ b/plugins/user-authenticators/ldap/src/org/apache/cloudstack/ldap/LdapAuthenticator.java @@ -25,6 +25,7 @@ import org.apache.log4j.Logger; import com.cloud.server.auth.DefaultUserAuthenticator; import com.cloud.user.UserAccount; import com.cloud.user.dao.UserAccountDao; +import com.cloud.utils.Pair; public class LdapAuthenticator extends DefaultUserAuthenticator { private static final Logger s_logger = Logger @@ -46,23 +47,27 @@ public class LdapAuthenticator extends DefaultUserAuthenticator { _userAccountDao = userAccountDao; } - @Override - public boolean authenticate(final String username, final String password, - final Long domainId, final Map requestParameters) { + @Override + public Pair authenticate(final String username, final String password, final Long domainId, final Map requestParameters) { final UserAccount user = _userAccountDao.getUserAccount(username, domainId); - if (user == null) { - s_logger.debug("Unable to find user with " + username - + " in domain " + domainId); - return false; - } else if (_ldapManager.isLdapEnabled()) { - return _ldapManager.canAuthenticate(username, password); - } else { - return false; - } - } + if (user == null) { + s_logger.debug("Unable to find user with " + username + " in domain " + domainId); + return new Pair(false, null); + } else if (_ldapManager.isLdapEnabled()) { + boolean result = _ldapManager.canAuthenticate(username, password); + ActionOnFailedAuthentication action = null; + if (result == false) { + action = ActionOnFailedAuthentication.INCREMENT_INCORRECT_LOGIN_ATTEMPT_COUNT; + } + return new Pair(result, action); + + } else { + return new Pair(false, ActionOnFailedAuthentication.INCREMENT_INCORRECT_LOGIN_ATTEMPT_COUNT); + } + } @Override public String encode(final String password) { diff --git a/plugins/user-authenticators/md5/src/com/cloud/server/auth/MD5UserAuthenticator.java b/plugins/user-authenticators/md5/src/com/cloud/server/auth/MD5UserAuthenticator.java index 63583af4ad6..f8f75ed675a 100644 --- a/plugins/user-authenticators/md5/src/com/cloud/server/auth/MD5UserAuthenticator.java +++ b/plugins/user-authenticators/md5/src/com/cloud/server/auth/MD5UserAuthenticator.java @@ -27,6 +27,7 @@ import org.apache.log4j.Logger; import com.cloud.user.UserAccount; import com.cloud.user.dao.UserAccountDao; +import com.cloud.utils.Pair; import com.cloud.utils.exception.CloudRuntimeException; /** @@ -34,47 +35,48 @@ import com.cloud.utils.exception.CloudRuntimeException; * comparing it against the local database. * */ -@Local(value={UserAuthenticator.class}) +@Local(value = {UserAuthenticator.class}) public class MD5UserAuthenticator extends DefaultUserAuthenticator { - public static final Logger s_logger = Logger.getLogger(MD5UserAuthenticator.class); - - @Inject private UserAccountDao _userAccountDao; - - @Override - public boolean authenticate(String username, String password, Long domainId, Map requestParameters ) { - if (s_logger.isDebugEnabled()) { + public static final Logger s_logger = Logger.getLogger(MD5UserAuthenticator.class); + + @Inject + private UserAccountDao _userAccountDao; + + @Override + public Pair authenticate(String username, String password, Long domainId, Map requestParameters) { + if (s_logger.isDebugEnabled()) { s_logger.debug("Retrieving user: " + username); } UserAccount user = _userAccountDao.getUserAccount(username, domainId); if (user == null) { s_logger.debug("Unable to find user with " + username + " in domain " + domainId); - return false; + return new Pair(false, null); } - + if (!user.getPassword().equals(encode(password))) { s_logger.debug("Password does not match"); - return false; + return new Pair(false, ActionOnFailedAuthentication.INCREMENT_INCORRECT_LOGIN_ATTEMPT_COUNT); } - return true; - } + return new Pair(true, null); + } - public String encode(String password) { - MessageDigest md5 = null; - try { - md5 = MessageDigest.getInstance("MD5"); - } catch (NoSuchAlgorithmException e) { - throw new CloudRuntimeException("Unable to hash password", e); - } + public String encode(String password) { + MessageDigest md5 = null; + try { + md5 = MessageDigest.getInstance("MD5"); + } catch (NoSuchAlgorithmException e) { + throw new CloudRuntimeException("Unable to hash password", e); + } - md5.reset(); - BigInteger pwInt = new BigInteger(1, md5.digest(password.getBytes())); - String pwStr = pwInt.toString(16); - int padding = 32 - pwStr.length(); - StringBuffer sb = new StringBuffer(); - for (int i = 0; i < padding; i++) { - sb.append('0'); // make sure the MD5 password is 32 digits long - } - sb.append(pwStr); - return sb.toString(); - } + md5.reset(); + BigInteger pwInt = new BigInteger(1, md5.digest(password.getBytes())); + String pwStr = pwInt.toString(16); + int padding = 32 - pwStr.length(); + StringBuffer sb = new StringBuffer(); + for (int i = 0; i < padding; i++) { + sb.append('0'); // make sure the MD5 password is 32 digits long + } + sb.append(pwStr); + return sb.toString(); + } } diff --git a/plugins/user-authenticators/plain-text/src/com/cloud/server/auth/PlainTextUserAuthenticator.java b/plugins/user-authenticators/plain-text/src/com/cloud/server/auth/PlainTextUserAuthenticator.java index 849e82e093d..6a5f415bf73 100644 --- a/plugins/user-authenticators/plain-text/src/com/cloud/server/auth/PlainTextUserAuthenticator.java +++ b/plugins/user-authenticators/plain-text/src/com/cloud/server/auth/PlainTextUserAuthenticator.java @@ -24,36 +24,38 @@ import org.apache.log4j.Logger; import com.cloud.user.UserAccount; import com.cloud.user.dao.UserAccountDao; +import com.cloud.utils.Pair; - -@Local(value={UserAuthenticator.class}) +@Local(value = {UserAuthenticator.class}) public class PlainTextUserAuthenticator extends DefaultUserAuthenticator { - public static final Logger s_logger = Logger.getLogger(PlainTextUserAuthenticator.class); - - @Inject private UserAccountDao _userAccountDao; - - @Override - public boolean authenticate(String username, String password, Long domainId, Map requestParameters ) { - if (s_logger.isDebugEnabled()) { + public static final Logger s_logger = Logger.getLogger(PlainTextUserAuthenticator.class); + + @Inject + private UserAccountDao _userAccountDao; + + @Override + public Pair authenticate(String username, String password, Long domainId, Map requestParameters) { + if (s_logger.isDebugEnabled()) { s_logger.debug("Retrieving user: " + username); } UserAccount user = _userAccountDao.getUserAccount(username, domainId); if (user == null) { s_logger.debug("Unable to find user with " + username + " in domain " + domainId); - return false; + return new Pair(false, null); } - + if (!user.getPassword().equals(password)) { s_logger.debug("Password does not match"); - return false; + return new Pair(false, ActionOnFailedAuthentication.INCREMENT_INCORRECT_LOGIN_ATTEMPT_COUNT); } - return true; - } - @Override - public String encode(String password) { - // Plaintext so no encoding at all - return password; - } + return new Pair(true, null); + } + + @Override + public String encode(String password) { + // Plaintext so no encoding at all + return password; + } } diff --git a/plugins/user-authenticators/sha256salted/src/com/cloud/server/auth/SHA256SaltedUserAuthenticator.java b/plugins/user-authenticators/sha256salted/src/com/cloud/server/auth/SHA256SaltedUserAuthenticator.java index 3592ddc8169..36305f18c99 100644 --- a/plugins/user-authenticators/sha256salted/src/com/cloud/server/auth/SHA256SaltedUserAuthenticator.java +++ b/plugins/user-authenticators/sha256salted/src/com/cloud/server/auth/SHA256SaltedUserAuthenticator.java @@ -30,9 +30,10 @@ import org.bouncycastle.util.encoders.Base64; import com.cloud.user.UserAccount; import com.cloud.user.dao.UserAccountDao; +import com.cloud.utils.Pair; import com.cloud.utils.exception.CloudRuntimeException; -@Local(value={UserAuthenticator.class}) +@Local(value = {UserAuthenticator.class}) public class SHA256SaltedUserAuthenticator extends DefaultUserAuthenticator { public static final Logger s_logger = Logger.getLogger(SHA256SaltedUserAuthenticator.class); private static final String s_defaultPassword = "000000000000000000000000000="; @@ -45,8 +46,7 @@ public class SHA256SaltedUserAuthenticator extends DefaultUserAuthenticator { * @see com.cloud.server.auth.UserAuthenticator#authenticate(java.lang.String, java.lang.String, java.lang.Long, java.util.Map) */ @Override - public boolean authenticate(String username, String password, - Long domainId, Map requestParameters) { + public Pair authenticate(String username, String password, Long domainId, Map requestParameters) { if (s_logger.isDebugEnabled()) { s_logger.debug("Retrieving user: " + username); } @@ -72,7 +72,12 @@ public class SHA256SaltedUserAuthenticator extends DefaultUserAuthenticator { try { String hashedPassword = encode(password, salt); /* constantTimeEquals comes first in boolean since we need to thwart timing attacks */ - return constantTimeEquals(realPassword, hashedPassword) && realUser; + boolean result = constantTimeEquals(realPassword, hashedPassword) && realUser; + ActionOnFailedAuthentication action = null; + if (!result && realUser) { + action = ActionOnFailedAuthentication.INCREMENT_INCORRECT_LOGIN_ATTEMPT_COUNT; + } + return new Pair(result, action); } catch (NoSuchAlgorithmException e) { throw new CloudRuntimeException("Unable to hash password", e); } catch (UnsupportedEncodingException e) { diff --git a/plugins/user-authenticators/sha256salted/test/src/com/cloud/server/auth/test/AuthenticatorTest.java b/plugins/user-authenticators/sha256salted/test/src/com/cloud/server/auth/test/AuthenticatorTest.java index 4c4c152326d..39751c7b61d 100644 --- a/plugins/user-authenticators/sha256salted/test/src/com/cloud/server/auth/test/AuthenticatorTest.java +++ b/plugins/user-authenticators/sha256salted/test/src/com/cloud/server/auth/test/AuthenticatorTest.java @@ -92,11 +92,11 @@ public class AuthenticatorTest { @Test public void testAuthentication() throws UnsupportedEncodingException, NoSuchAlgorithmException { Map dummyMap = new HashMap(); - assertEquals("32 byte salt authenticated", true, authenticator.authenticate("admin", "password", 0L, dummyMap)); - assertEquals("20 byte salt authenticated", true, authenticator.authenticate("admin20Byte", "password", 0L, dummyMap)); - assertEquals("fake user not authenticated", false, authenticator.authenticate("fake", "fake", 0L, dummyMap)); - assertEquals("bad password not authenticated", false, authenticator.authenticate("admin", "fake", 0L, dummyMap)); - assertEquals("20 byte user bad password not authenticated", false, authenticator.authenticate("admin20Byte", "fake", 0L, dummyMap)); + assertEquals("32 byte salt authenticated", true, authenticator.authenticate("admin", "password", 0L, dummyMap).first()); + assertEquals("20 byte salt authenticated", true, authenticator.authenticate("admin20Byte", "password", 0L, dummyMap).first()); + assertEquals("fake user not authenticated", false, authenticator.authenticate("fake", "fake", 0L, dummyMap).first()); + assertEquals("bad password not authenticated", false, authenticator.authenticate("admin", "fake", 0L, dummyMap).first()); + assertEquals("20 byte user bad password not authenticated", false, authenticator.authenticate("admin20Byte", "fake", 0L, dummyMap).first()); } // @Test diff --git a/server/src/com/cloud/server/auth/UserAuthenticator.java b/server/src/com/cloud/server/auth/UserAuthenticator.java index 95c4f0e4707..cc527b8c74f 100644 --- a/server/src/com/cloud/server/auth/UserAuthenticator.java +++ b/server/src/com/cloud/server/auth/UserAuthenticator.java @@ -18,26 +18,29 @@ package com.cloud.server.auth; import java.util.Map; +import com.cloud.utils.Pair; import com.cloud.utils.component.Adapter; /** * which UserAuthenticator to user in components.xml. - * */ public interface UserAuthenticator extends Adapter { - - /** - * - * @param username - * @param password - * @param domainId - * @return true if the user has been successfully authenticated, false otherwise - */ - public boolean authenticate(String username, String password, Long domainId, Map requestParameters); - - /** - * @param password - * @return the encoded password - */ - public String encode(String password); + public enum ActionOnFailedAuthentication { + INCREMENT_INCORRECT_LOGIN_ATTEMPT_COUNT; + } + + /** + * + * @param username + * @param password + * @param domainId + * @return the pair of 2 booleans - first identifies the success of authenciation, the second - whether to increase incorrect login attempts count in case of failed authentication + */ + public Pair authenticate(String username, String password, Long domainId, Map requestParameters); + + /** + * @param password + * @return the encoded password + */ + public String encode(String password); } diff --git a/server/src/com/cloud/user/AccountManagerImpl.java b/server/src/com/cloud/user/AccountManagerImpl.java index 0655f8c688b..d36765368cb 100755 --- a/server/src/com/cloud/user/AccountManagerImpl.java +++ b/server/src/com/cloud/user/AccountManagerImpl.java @@ -21,6 +21,7 @@ import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -110,6 +111,7 @@ import com.cloud.projects.ProjectVO; import com.cloud.projects.dao.ProjectAccountDao; import com.cloud.projects.dao.ProjectDao; import com.cloud.server.auth.UserAuthenticator; +import com.cloud.server.auth.UserAuthenticator.ActionOnFailedAuthentication; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.Volume; import com.cloud.storage.VolumeApiService; @@ -1960,13 +1962,19 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M } boolean authenticated = false; + HashSet actionsOnFailedAuthenticaion = new HashSet(); for (UserAuthenticator authenticator : _userAuthenticators) { - if (authenticator.authenticate(username, password, domainId, requestParameters)) { + Pair result = authenticator.authenticate(username, password, domainId, requestParameters); + if (result.first()) { authenticated = true; break; + } else if (result.second() != null) { + actionsOnFailedAuthenticaion.add(result.second()); } } + boolean updateIncorrectLoginCount = actionsOnFailedAuthenticaion.contains(ActionOnFailedAuthentication.INCREMENT_INCORRECT_LOGIN_ATTEMPT_COUNT); + if (authenticated) { UserAccount userAccount = _userAccountDao.getUserAccount(username, domainId); if (userAccount == null) { @@ -2003,13 +2011,14 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M if (!isInternalAccount(userAccount.getType())) { // Internal accounts are not disabled int attemptsMade = userAccount.getLoginAttempts() + 1; - if (attemptsMade < _allowedLoginAttempts) { - updateLoginAttempts(userAccount.getId(), attemptsMade, false); - s_logger.warn("Login attempt failed. You have " + (_allowedLoginAttempts - attemptsMade) + " attempt(s) remaining"); - } else { - updateLoginAttempts(userAccount.getId(), _allowedLoginAttempts, true); - s_logger.warn("User " + userAccount.getUsername() + " has been disabled due to multiple failed login attempts." + - " Please contact admin."); + if (updateIncorrectLoginCount) { + if (attemptsMade < _allowedLoginAttempts) { + updateLoginAttempts(userAccount.getId(), attemptsMade, false); + s_logger.warn("Login attempt failed. You have " + (_allowedLoginAttempts - attemptsMade) + " attempt(s) remaining"); + } else { + updateLoginAttempts(userAccount.getId(), _allowedLoginAttempts, true); + s_logger.warn("User " + userAccount.getUsername() + " has been disabled due to multiple failed login attempts." + " Please contact admin."); + } } } } else { From 29e44b1b67fa1dcc8289f15b6ed82fa9edb88c7b Mon Sep 17 00:00:00 2001 From: Kelven Yang Date: Wed, 22 Jan 2014 14:08:58 -0800 Subject: [PATCH 006/169] CLOUDSTACK-5928: Use legacy vmsync logic by default --- .../src/com/cloud/vm/VirtualMachineManagerImpl.java | 2 +- server/src/com/cloud/storage/VolumeApiServiceImpl.java | 2 +- server/src/com/cloud/vm/snapshot/VMSnapshotManagerImpl.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java index 40b9151442a..d32f144b80b 100755 --- a/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java @@ -337,7 +337,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac // TODO, remove it after transient period is over static final ConfigKey VmJobEnabled = new ConfigKey("Advanced", - Boolean.class, "vm.job.enabled", "true", + Boolean.class, "vm.job.enabled", "false", "True to enable new VM sync model. false to use the old way", false); static final ConfigKey VmJobCheckInterval = new ConfigKey("Advanced", diff --git a/server/src/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/com/cloud/storage/VolumeApiServiceImpl.java index 6be43808008..f46f1e9e0d0 100644 --- a/server/src/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/com/cloud/storage/VolumeApiServiceImpl.java @@ -333,7 +333,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic // TODO static final ConfigKey VmJobEnabled = new ConfigKey("Advanced", - Boolean.class, "vm.job.enabled", "true", + Boolean.class, "vm.job.enabled", "false", "True to enable new VM sync model. false to use the old way", false); static final ConfigKey VmJobCheckInterval = new ConfigKey("Advanced", Long.class, "vm.job.check.interval", "3000", diff --git a/server/src/com/cloud/vm/snapshot/VMSnapshotManagerImpl.java b/server/src/com/cloud/vm/snapshot/VMSnapshotManagerImpl.java index ceded0867dc..e273e1a2338 100644 --- a/server/src/com/cloud/vm/snapshot/VMSnapshotManagerImpl.java +++ b/server/src/com/cloud/vm/snapshot/VMSnapshotManagerImpl.java @@ -136,7 +136,7 @@ public class VMSnapshotManagerImpl extends ManagerBase implements VMSnapshotMana // TODO static final ConfigKey VmJobEnabled = new ConfigKey("Advanced", - Boolean.class, "vm.job.enabled", "true", + Boolean.class, "vm.job.enabled", "false", "True to enable new VM sync model. false to use the old way", false); static final ConfigKey VmJobCheckInterval = new ConfigKey("Advanced", Long.class, "vm.job.check.interval", "3000", From 8ef47e8ff3fd0997c2680f4068c4dadb7bb3b511 Mon Sep 17 00:00:00 2001 From: Devdeep Singh Date: Thu, 23 Jan 2014 19:00:27 +0530 Subject: [PATCH 007/169] CLOUDSTACK-5936: Hyper-V agent should log to the windows event log. Changed the default log appender for the agent. --- .../ServerResource/AgentShell/AgentService.Designer.cs | 2 +- .../hyperv/DotNet/ServerResource/AgentShell/App.config | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/plugins/hypervisors/hyperv/DotNet/ServerResource/AgentShell/AgentService.Designer.cs b/plugins/hypervisors/hyperv/DotNet/ServerResource/AgentShell/AgentService.Designer.cs index 854427732aa..4bf424fa4b9 100644 --- a/plugins/hypervisors/hyperv/DotNet/ServerResource/AgentShell/AgentService.Designer.cs +++ b/plugins/hypervisors/hyperv/DotNet/ServerResource/AgentShell/AgentService.Designer.cs @@ -45,7 +45,7 @@ namespace CloudStack.Plugin.AgentShell private void InitializeComponent() { components = new System.ComponentModel.Container(); - this.ServiceName = "CloudStack ServerResource"; + this.ServiceName = Program.serviceName; } #endregion diff --git a/plugins/hypervisors/hyperv/DotNet/ServerResource/AgentShell/App.config b/plugins/hypervisors/hyperv/DotNet/ServerResource/AgentShell/App.config index 68ab80ee555..b783dfecd63 100644 --- a/plugins/hypervisors/hyperv/DotNet/ServerResource/AgentShell/App.config +++ b/plugins/hypervisors/hyperv/DotNet/ServerResource/AgentShell/App.config @@ -10,9 +10,9 @@ - + - + @@ -25,13 +25,14 @@ + - + From d31fa09c7e5bb04fcc4594c0352d9b91171e9d1a Mon Sep 17 00:00:00 2001 From: Santhosh Edukulla Date: Thu, 23 Jan 2014 17:24:43 +0530 Subject: [PATCH 008/169] Added Fix for CLOUDSTACK-5875 Added fix for exception and listing. Mentioned details under bug. Post the fix, simulator works fine. Signed-off-by: Santhosh Edukulla Signed-off-by: Koushik Das --- .../src/com/cloud/agent/manager/MockAgentManagerImpl.java | 7 +++++++ setup/db/templates.simulator.sql | 8 ++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/plugins/hypervisors/simulator/src/com/cloud/agent/manager/MockAgentManagerImpl.java b/plugins/hypervisors/simulator/src/com/cloud/agent/manager/MockAgentManagerImpl.java index 4071aeb4c46..447482b9642 100644 --- a/plugins/hypervisors/simulator/src/com/cloud/agent/manager/MockAgentManagerImpl.java +++ b/plugins/hypervisors/simulator/src/com/cloud/agent/manager/MockAgentManagerImpl.java @@ -32,6 +32,8 @@ import javax.ejb.Local; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.user.AccountManager; +import org.apache.cloudstack.context.CallContext; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -87,6 +89,7 @@ public class MockAgentManagerImpl extends ManagerBase implements MockAgentManage MockStorageManager _storageMgr = null; @Inject ResourceManager _resourceMgr; + @Inject private AccountManager _accountMgr; SimulatorSecondaryDiscoverer discoverer; @Inject @@ -306,8 +309,10 @@ public class MockAgentManagerImpl extends ManagerBase implements MockAgentManage @Override @DB public void run() { + CallContext.register(_accountMgr.getSystemUser(), _accountMgr.getSystemAccount()); if (this.mode.equalsIgnoreCase("Stop")) { handleSystemVMStop(); + CallContext.unregister(); return; } @@ -363,10 +368,12 @@ public class MockAgentManagerImpl extends ManagerBase implements MockAgentManage _resourceMgr.discoverHosts(cmd); } catch (DiscoveryException e) { s_logger.debug("Failed to discover host: " + e.toString()); + CallContext.unregister(); return; } } catch (ConfigurationException e) { s_logger.debug("Failed to load secondary storage resource: " + e.toString()); + CallContext.unregister(); return; } } diff --git a/setup/db/templates.simulator.sql b/setup/db/templates.simulator.sql index 5f9b67d1f7e..25e91bd9c5f 100755 --- a/setup/db/templates.simulator.sql +++ b/setup/db/templates.simulator.sql @@ -16,7 +16,7 @@ -- under the License. -INSERT INTO `cloud`.`vm_template` (id, uuid, unique_name, name, public, created, type, hvm, bits, account_id, url, checksum, enable_password, display_text, format, guest_os_id, featured, cross_zones, hypervisor_type) - VALUES (100, UUID(), 'simulator-domR', 'SystemVM Template (simulator)', 0, now(), 'SYSTEM', 0, 64, 1, 'http://nfs1.lab.vmops.com/templates/routing/debian/latest/systemvm.vhd.bz2', '', 0, 'SystemVM Template (simulator)', 'VHD', 15, 0, 1, 'Simulator'); -INSERT INTO `cloud`.`vm_template` (id, uuid, unique_name, name, public, created, type, hvm, bits, account_id, url, checksum, enable_password, display_text, format, guest_os_id, featured, cross_zones, hypervisor_type) - VALUES (111, UUID(), 'simulator-Centos', 'CentOS 5.3(64-bit) no GUI (Simulator)', 1, now(), 'BUILTIN', 0, 64, 1, 'http://nfs1.lab.vmops.com/templates/centos53-x86_64/latest/f59f18fb-ae94-4f97-afd2-f84755767aca.vhd.bz2', '', 0, 'CentOS 5.3(64-bit) no GUI (Simulator)', 'VHD', 11, 1, 1, 'Simulator'); +INSERT INTO `cloud`.`vm_template` (id, uuid, unique_name, name, public, created, type, hvm, bits, account_id, url, checksum, enable_password, display_text, format, guest_os_id, featured, cross_zones, hypervisor_type, state) + VALUES (100, UUID(), 'simulator-domR', 'SystemVM Template (simulator)', 0, now(), 'SYSTEM', 0, 64, 1, 'http://nfs1.lab.vmops.com/templates/routing/debian/latest/systemvm.vhd.bz2', '', 0, 'SystemVM Template (simulator)', 'VHD', 15, 0, 1, 'Simulator','Active'); +INSERT INTO `cloud`.`vm_template` (id, uuid, unique_name, name, public, created, type, hvm, bits, account_id, url, checksum, enable_password, display_text, format, guest_os_id, featured, cross_zones, hypervisor_type,state) + VALUES (111, UUID(), 'simulator-Centos', 'CentOS 5.3(64-bit) no GUI (Simulator)', 1, now(), 'BUILTIN', 0, 64, 1, 'http://nfs1.lab.vmops.com/templates/centos53-x86_64/latest/f59f18fb-ae94-4f97-afd2-f84755767aca.vhd.bz2', '', 0, 'CentOS 5.3(64-bit) no GUI (Simulator)', 'VHD', 11, 1, 1, 'Simulator','Active'); From 6d997bd6893f0ab649899f88456d6e80b23bb873 Mon Sep 17 00:00:00 2001 From: Devdeep Singh Date: Fri, 24 Jan 2014 20:53:42 +0530 Subject: [PATCH 009/169] CLOUDSTACK-5942: The agent at places logs the password of the user. It should mask the password from the message string before writing to the log. Made a change to do so. --- .../HypervResource/CloudStackTypes.cs | 4 ++-- .../HypervResourceController.cs | 22 +++++++++---------- .../ServerResource/HypervResource/Utils.cs | 10 +++++++++ 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/CloudStackTypes.cs b/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/CloudStackTypes.cs index 847380cb115..d54295ccdaa 100644 --- a/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/CloudStackTypes.cs +++ b/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/CloudStackTypes.cs @@ -254,7 +254,7 @@ namespace HypervResource // Assert if (result.dataStore == null || (result.primaryDataStore == null && result.nfsDataStore == null)) { - String errMsg = "VolumeObjectTO missing dataStore in spec " + volumeObjectTOJson.ToString(); + String errMsg = "VolumeObjectTO missing dataStore in spec " + Utils.CleanString(volumeObjectTOJson.ToString()); logger.Error(errMsg); throw new ArgumentNullException(errMsg); } @@ -292,7 +292,7 @@ namespace HypervResource } else { - String errMsg = "VolumeObjectTO missing dataStore in spec " + volInfo.ToString(); + String errMsg = "VolumeObjectTO missing dataStore in spec " + Utils.CleanString(volInfo.ToString()); logger.Error(errMsg); throw new ArgumentNullException(errMsg); } diff --git a/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/HypervResourceController.cs b/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/HypervResourceController.cs index f39cc97c8fc..81ab42a7393 100644 --- a/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/HypervResourceController.cs +++ b/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/HypervResourceController.cs @@ -207,7 +207,7 @@ namespace HypervResource { using (log4net.NDC.Push(Guid.NewGuid().ToString())) { - logger.Info(CloudStackTypes.AttachCommand + cmd.ToString()); + logger.Info(CloudStackTypes.AttachCommand + Utils.CleanString(cmd.ToString())); string details = null; bool result = false; @@ -268,7 +268,7 @@ namespace HypervResource { using (log4net.NDC.Push(Guid.NewGuid().ToString())) { - logger.Info(CloudStackTypes.DettachCommand + cmd.ToString()); + logger.Info(CloudStackTypes.DettachCommand + Utils.CleanString(cmd.ToString())); string details = null; bool result = false; @@ -485,7 +485,7 @@ namespace HypervResource { JObject ansObj = Utils.CreateCloudStackObject(ansType, ansContent); JArray answer = new JArray(ansObj); - logger.Info(ansObj.ToString()); + logger.Info(Utils.CleanString(ansObj.ToString())); return answer; } @@ -496,7 +496,7 @@ namespace HypervResource { using (log4net.NDC.Push(Guid.NewGuid().ToString())) { - logger.Info(CloudStackTypes.CreateCommand + cmd.ToString()); + logger.Info(CloudStackTypes.CreateCommand + Utils.CleanString(cmd.ToString())); string details = null; bool result = false; @@ -603,7 +603,7 @@ namespace HypervResource { using (log4net.NDC.Push(Guid.NewGuid().ToString())) { - logger.Info(CloudStackTypes.PrimaryStorageDownloadCommand + cmd.ToString()); + logger.Info(CloudStackTypes.PrimaryStorageDownloadCommand + Utils.CleanString(cmd.ToString())); string details = null; bool result = false; long size = 0; @@ -871,7 +871,7 @@ namespace HypervResource { using (log4net.NDC.Push(Guid.NewGuid().ToString())) { - logger.Info(CloudStackTypes.CreateStoragePoolCommand + cmd.ToString()); + logger.Info(CloudStackTypes.CreateStoragePoolCommand + Utils.CleanString(cmd.ToString())); object ansContent = new { result = true, @@ -889,7 +889,7 @@ namespace HypervResource { using (log4net.NDC.Push(Guid.NewGuid().ToString())) { - logger.Info(CloudStackTypes.ModifyStoragePoolCommand + cmd.ToString()); + logger.Info(CloudStackTypes.ModifyStoragePoolCommand + Utils.CleanString(cmd.ToString())); string details = null; string localPath; StoragePoolType poolType; @@ -1045,7 +1045,7 @@ namespace HypervResource { using (log4net.NDC.Push(Guid.NewGuid().ToString())) { - logger.Info(CloudStackTypes.StartCommand + cmd.ToString()); // TODO: Security hole? VM data printed to log + logger.Info(CloudStackTypes.StartCommand + Utils.CleanString(cmd.ToString())); string details = null; bool result = false; @@ -1144,7 +1144,7 @@ namespace HypervResource { using (log4net.NDC.Push(Guid.NewGuid().ToString())) { - logger.Info(CloudStackTypes.CreateObjectCommand + cmd.ToString()); + logger.Info(CloudStackTypes.CreateObjectCommand + Utils.CleanString(cmd.ToString())); bool result = false; string details = null; @@ -1315,7 +1315,7 @@ namespace HypervResource using (log4net.NDC.Push(Guid.NewGuid().ToString())) { // Log command *after* we've removed security details from the command. - logger.Info(CloudStackTypes.CopyCommand + cmd.ToString()); + logger.Info(CloudStackTypes.CopyCommand + Utils.CleanString(cmd.ToString())); bool result = false; string details = null; @@ -1691,7 +1691,7 @@ namespace HypervResource { using (log4net.NDC.Push(Guid.NewGuid().ToString())) { - logger.Info(CloudStackTypes.GetStorageStatsCommand + cmd.ToString()); + logger.Info(CloudStackTypes.GetStorageStatsCommand + Utils.CleanString(cmd.ToString())); bool result = false; string details = null; long capacity = 0; diff --git a/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/Utils.cs b/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/Utils.cs index c4b39ba24c2..6ebc5bf7f23 100644 --- a/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/Utils.cs +++ b/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/Utils.cs @@ -164,6 +164,16 @@ namespace HypervResource capacity = totalNumberOfBytes > 0 ? (long)totalNumberOfBytes : 0; } + public static string CleanString(string stringToClean) + { + string cleanString = null; + string regexQueryString = "(&|%26)?(password|accesskey|secretkey)(=|%3D).*?(?=(%26|[&'\"]))"; + string regexJson = "\"(password|accesskey|secretkey)\":\".*?\",?"; + cleanString = System.Text.RegularExpressions.Regex.Replace(stringToClean, regexQueryString, ""); + cleanString = System.Text.RegularExpressions.Regex.Replace(cleanString, regexJson, ""); + return cleanString; + } + // from http://stackoverflow.com/a/2541569/939250 #region imports [DllImport("advapi32.dll", SetLastError = true)] From 81a83b5b31f0a237ae56449c60350160915f28f8 Mon Sep 17 00:00:00 2001 From: Jessica Wang Date: Fri, 24 Jan 2014 18:23:42 -0800 Subject: [PATCH 010/169] CLOUDSTACK-5614: UI - Infrastructure > Sockets > listView > (1) split "XenServer" into "XenServer 6.2.0" and "XenServer"(prior to 6.2.0). (2) change label of Hyperv to Hyper-V. --- ui/scripts/system.js | 215 ++++++++++++++++++++++++++++--------------- 1 file changed, 142 insertions(+), 73 deletions(-) diff --git a/ui/scripts/system.js b/ui/scripts/system.js index 0c4f45d413b..960f1aa488a 100644 --- a/ui/scripts/system.js +++ b/ui/scripts/system.js @@ -397,7 +397,7 @@ var returnedHostCount = 0; var returnedHostCpusocketsSum = 0; - var callListHostsWithPage = function(setTotalHostCount) { + var callListHostsWithPage = function() { $.ajax({ url: createURL('listHosts'), async: false, @@ -411,10 +411,8 @@ if (json.listhostsresponse.count == undefined) { return; } - - if (setTotalHostCount) { - totalHostCount = json.listhostsresponse.count; - } + + totalHostCount = json.listhostsresponse.count; returnedHostCount += json.listhostsresponse.host.length; var items = json.listhostsresponse.host; @@ -426,13 +424,13 @@ if (returnedHostCount < totalHostCount) { currentPage++; - callListHostsWithPage(false); + callListHostsWithPage(); } } }); } - callListHostsWithPage(true); + callListHostsWithPage(); socketCount += returnedHostCpusocketsSum; }) @@ -7468,73 +7466,144 @@ sockets: { label: 'label.sockets' } }, dataProvider: function(args) { - $.ajax({ - url: createURL('listHypervisors'), - success: function(json) { - var supportSocketHypervisors = { - "Hyperv": 1, - "KVM": 1, - "XenServer": 1, - "VMware": 1 - }; - - args.response.success({ - data: $(json.listhypervisorsresponse.hypervisor).map(function(index, hypervisor) { - var totalHostCount = 0; - var currentPage = 1; - var returnedHostCount = 0; - var returnedHostCpusocketsSum = 0; - - var callListHostsWithPage = function(setTotalHostCount) { - $.ajax({ - url: createURL('listHosts'), - async: false, - data: { - type: 'routing', - hypervisor: hypervisor.name, - page: currentPage, - pagesize: pageSize //global variable - }, - success: function(json) { - if (json.listhostsresponse.count == undefined) { - return; - } - - if (setTotalHostCount) { - totalHostCount = json.listhostsresponse.count; - } - returnedHostCount += json.listhostsresponse.host.length; - - var items = json.listhostsresponse.host; - for (var i = 0; i < items.length; i++) { - if (items[i].cpusockets != undefined && isNaN(items[i].cpusockets) == false) { - returnedHostCpusocketsSum += items[i].cpusockets; - } - } - - if (returnedHostCount < totalHostCount) { - currentPage++; - callListHostsWithPage(false); - } - } - }); - } - - callListHostsWithPage(true); - - if ((hypervisor.name in supportSocketHypervisors) == false) { - returnedHostCpusocketsSum = 'N/A'; - } - - return { - hypervisor: hypervisor.name, - hosts: totalHostCount, - sockets: returnedHostCpusocketsSum - }; - }) - }); - } + var array1 = []; + + // ***** non XenServer (begin) ***** + var hypervisors = ["Hyperv", "KVM", "VMware", "BareMetal", "Ovm", "LXC"]; + + var supportSocketHypervisors = { + "Hyperv": 1, + "KVM": 1, + "VMware": 1 + }; + + for (var h = 0; h < hypervisors.length; h++) { + var totalHostCount = 0; + var currentPage = 1; + var returnedHostCount = 0; + var returnedHostCpusocketsSum = 0; + + var callListHostsWithPage = function() { + $.ajax({ + url: createURL('listHosts'), + async: false, + data: { + type: 'routing', + hypervisor: hypervisors[h], + page: currentPage, + pagesize: pageSize //global variable + }, + success: function(json) { + if (json.listhostsresponse.count == undefined) { + return; + } + + totalHostCount = json.listhostsresponse.count; + returnedHostCount += json.listhostsresponse.host.length; + + var items = json.listhostsresponse.host; + for (var i = 0; i < items.length; i++) { + if (items[i].cpusockets != undefined && isNaN(items[i].cpusockets) == false) { + returnedHostCpusocketsSum += items[i].cpusockets; + } + } + + if (returnedHostCount < totalHostCount) { + currentPage++; + callListHostsWithPage(); + } + } + }); + } + + callListHostsWithPage(); + + if ((hypervisors[h] in supportSocketHypervisors) == false) { + returnedHostCpusocketsSum = 'N/A'; + } + + var hypervisorName = hypervisors[h]; + if (hypervisorName == "Hyperv") { + hypervisorName = "Hyper-V"; + } + + array1.push({ + hypervisor: hypervisorName, + hosts: totalHostCount, + sockets: returnedHostCpusocketsSum + }); + } + // ***** non XenServer (end) ***** + + + // ***** XenServer (begin) ***** + var totalHostCount = 0; + var currentPage = 1; + var returnedHostCount = 0; + + var returnedHostCountForXenServer = 0; + var returnedHostCountForXenServer620 = 0; + var returnedHostCpusocketsSumForXenServer620 = 0; + + var callListHostsWithPage = function() { + $.ajax({ + url: createURL('listHosts'), + async: false, + data: { + type: 'routing', + hypervisor: 'XenServer', + page: currentPage, + pagesize: pageSize //global variable + }, + success: function(json) { + if (json.listhostsresponse.count == undefined) { + return; + } + + totalHostCount = json.listhostsresponse.count; + returnedHostCount += json.listhostsresponse.host.length; + + var items = json.listhostsresponse.host; + for (var i = 0; i < items.length; i++) { + //"hypervisorversion" == "6.2.0" + if (items[i].hypervisorversion == "6.2.0") { + returnedHostCountForXenServer620 ++; + if (items[i].cpusockets != undefined && isNaN(items[i].cpusockets) == false) { + returnedHostCpusocketsSumForXenServer620 += items[i].cpusockets; + } + } else { + returnedHostCountForXenServer++; + } + } + + if (returnedHostCount < totalHostCount) { + currentPage++; + callListHostsWithPage(); + } + } + }); + } + + callListHostsWithPage(); + + array1.push({ + hypervisor: 'XenServer', + hosts: returnedHostCountForXenServer, + sockets: 'N/A' }); + + array1.push({ + hypervisor: 'XenServer 6.2.0', + hosts: returnedHostCountForXenServer620, + sockets: returnedHostCpusocketsSumForXenServer620 + }); + // ***** XenServer (end) ***** + + + args.response.success({ + data: array1 + }); + } }; From f18c5a1910b6370585a1d61638b8310c3ecba5ef Mon Sep 17 00:00:00 2001 From: Hugo Trippaers Date: Fri, 24 Jan 2014 11:58:53 +0000 Subject: [PATCH 011/169] Findbugs finding : Incorrect use of equality (cherry picked from commit 10cc5645e8b9713bc7c86d543413d396ce203cc6) --- .../src/com/cloud/hypervisor/vmware/VmwareServerDiscoverer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/VmwareServerDiscoverer.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/VmwareServerDiscoverer.java index 89bbeb93bb9..54e98151272 100755 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/VmwareServerDiscoverer.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/VmwareServerDiscoverer.java @@ -168,7 +168,7 @@ public class VmwareServerDiscoverer extends DiscovererBase implements _clusterDetailsDao.persist(clusterId, clusterDetails); } String updatedInventoryPath = validateCluster(url, vmwareDc); - if (url.getPath() != updatedInventoryPath) { + if (!url.getPath().equals(updatedInventoryPath)) { // If url from API doesn't specify DC then update url in database with DC associated with this zone. clusterDetails.put("url", url.getScheme() + "://" + url.getHost() + updatedInventoryPath); _clusterDetailsDao.persist(clusterId, clusterDetails); From 60ac12780bfa1604902a89d5dc7937a8b9334e0d Mon Sep 17 00:00:00 2001 From: Hugo Trippaers Date: Fri, 24 Jan 2014 13:01:08 +0000 Subject: [PATCH 012/169] Backport fixed for issues found by Findbugs --- .../cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java | 2 +- .../cloud/hypervisor/vmware/resource/VmwareResource.java | 2 +- .../storage/motion/VmwareStorageMotionStrategy.java | 2 +- .../cloud/hypervisor/xen/resource/CitrixResourceBase.java | 4 ++-- .../storage/motion/XenServerStorageMotionStrategy.java | 2 +- utils/src/com/cloud/utils/net/NetUtils.java | 2 +- .../src/com/cloud/hypervisor/vmware/mo/DatastoreMO.java | 6 ++++++ 7 files changed, 13 insertions(+), 7 deletions(-) diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java index 012c8a5e25c..bdccc101e98 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java @@ -960,7 +960,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { } if (disk == null) { - throw new CloudRuntimeException("Failed to create " + disk.getPath() + " from template " + template.getName()); + throw new CloudRuntimeException("Failed to create disk from template " + template.getName()); } return disk; diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 1d4c43f41bc..bfb95fd1a5d 100755 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -3508,7 +3508,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa public int compare(DiskTO arg0, DiskTO arg1) { if (arg0.getDiskSeq() < arg1.getDiskSeq()) { return -1; - } else if (arg0.getDiskSeq() == arg1.getDiskSeq()) { + } else if (arg0.getDiskSeq().equals(arg1.getDiskSeq())) { return 0; } diff --git a/plugins/hypervisors/vmware/src/org/apache/cloudstack/storage/motion/VmwareStorageMotionStrategy.java b/plugins/hypervisors/vmware/src/org/apache/cloudstack/storage/motion/VmwareStorageMotionStrategy.java index 4668f67a24f..446093f99a4 100644 --- a/plugins/hypervisors/vmware/src/org/apache/cloudstack/storage/motion/VmwareStorageMotionStrategy.java +++ b/plugins/hypervisors/vmware/src/org/apache/cloudstack/storage/motion/VmwareStorageMotionStrategy.java @@ -95,7 +95,7 @@ public class VmwareStorageMotionStrategy implements DataMotionStrategy { try { VMInstanceVO instance = instanceDao.findById(vmTo.getId()); if (instance != null) { - if (srcHost.getClusterId() == destHost.getClusterId()) { + if (srcHost.getClusterId().equals(destHost.getClusterId())) { answer = migrateVmWithVolumesWithinCluster(instance, vmTo, srcHost, destHost, volumeMap); } else { answer = migrateVmWithVolumesAcrossCluster(instance, vmTo, srcHost, destHost, volumeMap); diff --git a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java index 16fef1d86a1..f065d58dff0 100644 --- a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java +++ b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java @@ -846,7 +846,7 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe Set dom0Vifs = dom0.getVIFs(conn); for (VIF vif:dom0Vifs) { vif.getRecord(conn); - if (vif.getNetwork(conn).getUuid(conn) == nw.getUuid(conn)) { + if (vif.getNetwork(conn).getUuid(conn).equals(nw.getUuid(conn))) { dom0vif = vif; s_logger.debug("A VIF for dom0 has already been found - No need to create one"); } @@ -3427,7 +3427,7 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe Set vbds = vm.getVBDs(conn); for( VBD vbd : vbds) { VBD.Record vbdRec = vbd.getRecord(conn); - if( vbdRec.type.equals(Types.VbdType.CD.toString()) && !vbdRec.empty ) { + if( vbdRec.type.equals(Types.VbdType.CD) && !vbdRec.empty ) { vbd.eject(conn); break; } diff --git a/plugins/hypervisors/xen/src/org/apache/cloudstack/storage/motion/XenServerStorageMotionStrategy.java b/plugins/hypervisors/xen/src/org/apache/cloudstack/storage/motion/XenServerStorageMotionStrategy.java index 3dc7dd86cbc..8d119407526 100644 --- a/plugins/hypervisors/xen/src/org/apache/cloudstack/storage/motion/XenServerStorageMotionStrategy.java +++ b/plugins/hypervisors/xen/src/org/apache/cloudstack/storage/motion/XenServerStorageMotionStrategy.java @@ -103,7 +103,7 @@ public class XenServerStorageMotionStrategy implements DataMotionStrategy { try { VMInstanceVO instance = instanceDao.findById(vmTo.getId()); if (instance != null) { - if (srcHost.getClusterId() == destHost.getClusterId()) { + if (srcHost.getClusterId().equals(destHost.getClusterId())) { answer = migrateVmWithVolumesWithinCluster(instance, vmTo, srcHost, destHost, volumeMap); } else { answer = migrateVmWithVolumesAcrossCluster(instance, vmTo, srcHost, destHost, volumeMap); diff --git a/utils/src/com/cloud/utils/net/NetUtils.java b/utils/src/com/cloud/utils/net/NetUtils.java index a72aded7389..792d606e76b 100755 --- a/utils/src/com/cloud/utils/net/NetUtils.java +++ b/utils/src/com/cloud/utils/net/NetUtils.java @@ -840,7 +840,7 @@ public class NetUtils { //this implies cidrA is super set of cidrB return supersetOrSubset.isSuperset; } - else if (cidrALong[1] == cidrBLong[1]) { + else if (cidrALong[1].equals(cidrBLong[1])) { //this implies both the cidrs are equal return supersetOrSubset.sameSubnet; } diff --git a/vmware-base/src/com/cloud/hypervisor/vmware/mo/DatastoreMO.java b/vmware-base/src/com/cloud/hypervisor/vmware/mo/DatastoreMO.java index 4b2ce088ea5..716bc20e908 100755 --- a/vmware-base/src/com/cloud/hypervisor/vmware/mo/DatastoreMO.java +++ b/vmware-base/src/com/cloud/hypervisor/vmware/mo/DatastoreMO.java @@ -32,6 +32,7 @@ import com.vmware.vim25.PropertySpec; import com.vmware.vim25.SelectionSpec; import com.vmware.vim25.TraversalSpec; +import com.cloud.exception.CloudException; import com.cloud.hypervisor.vmware.util.VmwareContext; import com.cloud.utils.Pair; @@ -349,7 +350,12 @@ public class DatastoreMO extends BaseMO { ArrayList results = browserMo.searchDatastoreSubFolders("[" + getName() + "]", fileName, caseInsensitive); if (results != null && results.size() > 1) { s_logger.warn("Multiple files with name " + fileName + " exists in datastore " + datastorePath + ". Trying to choose first file found in search attempt."); + } else if (results == null) { + String msg = "No file found with name " + fileName + " found in datastore " + datastorePath; + s_logger.error(msg); + throw new CloudException(msg); } + for (HostDatastoreBrowserSearchResults result : results) { List info = result.getFile(); if (info != null && info.size() > 0) { From c307de312dccb55cb4d57f0ebde7859606f983c0 Mon Sep 17 00:00:00 2001 From: Hugo Trippaers Date: Sat, 25 Jan 2014 10:51:10 +0100 Subject: [PATCH 013/169] Backport findbugs configuration to 4.3 so we can run a findbugs report in jenkins. --- pom.xml | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/pom.xml b/pom.xml index 68d8dad6c70..7af713970a0 100644 --- a/pom.xml +++ b/pom.xml @@ -87,6 +87,7 @@ 1.0.10 4.0.0 2.10 + 2.5.3 @@ -444,6 +445,21 @@ ${basedir}/${cs.target.dir}/classes ${basedir}/${cs.target.dir}/test-classes + + + org.codehaus.mojo + findbugs-maven-plugin + + + cloudstack-findbugs + none + + check + + + + + @@ -712,6 +728,25 @@ + + org.codehaus.mojo + findbugs-maven-plugin + ${cs.findbugs.version} + + Max + High + true + false + + + + cloudstack-findbugs + + check + + + + @@ -771,5 +806,23 @@ vmware-base + + enablefindbugs + + + + org.codehaus.mojo + findbugs-maven-plugin + + + cloudstack-findbugs + process-classes + true + + + + + + From 43ba36f97950aa8d09399a28bb50c6a22209f15e Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Sat, 25 Jan 2014 23:26:53 +0100 Subject: [PATCH 014/169] nullpointer dereference guarded --- .../impl/UserConcentratedAllocator.java | 60 ++++++++----------- 1 file changed, 25 insertions(+), 35 deletions(-) diff --git a/server/src/com/cloud/agent/manager/allocator/impl/UserConcentratedAllocator.java b/server/src/com/cloud/agent/manager/allocator/impl/UserConcentratedAllocator.java index 3f1994e0d8e..8101da94c0d 100755 --- a/server/src/com/cloud/agent/manager/allocator/impl/UserConcentratedAllocator.java +++ b/server/src/com/cloud/agent/manager/allocator/impl/UserConcentratedAllocator.java @@ -32,30 +32,18 @@ import org.apache.log4j.Logger; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import com.cloud.agent.manager.allocator.PodAllocator; -import com.cloud.capacity.CapacityVO; -import com.cloud.capacity.dao.CapacityDao; import com.cloud.dc.DataCenter; -import com.cloud.dc.HostPodVO; import com.cloud.dc.Pod; -import com.cloud.dc.dao.HostPodDao; import com.cloud.offering.ServiceOffering; -import com.cloud.service.dao.ServiceOfferingDao; -import com.cloud.storage.VolumeVO; -import com.cloud.storage.dao.VMTemplatePoolDao; -import com.cloud.storage.dao.VolumeDao; import com.cloud.template.VirtualMachineTemplate; import com.cloud.utils.DateUtil; import com.cloud.utils.NumbersUtil; import com.cloud.utils.Pair; import com.cloud.utils.component.AdapterBase; import com.cloud.utils.db.SearchCriteria; -import com.cloud.vm.UserVmVO; -import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachine.State; import com.cloud.vm.VirtualMachineProfile; -import com.cloud.vm.dao.UserVmDao; -import com.cloud.vm.dao.VMInstanceDao; @Local(value = PodAllocator.class) public class UserConcentratedAllocator extends AdapterBase implements PodAllocator { @@ -246,35 +234,37 @@ public class UserConcentratedAllocator extends AdapterBase implements PodAllocat // List vms = _vmInstanceDao.listByLastHostId(hostId); List vms = null; long usedCapacity = 0; - for (VMInstanceVO vm : vms) { - if (skipCalculation(vm)) { - continue; - } - - ServiceOffering so = null; - - if (vm.getType() == VirtualMachine.Type.User) { - UserVmVO userVm = _vmDao.findById(vm.getId()); - if (userVm == null) { + if (vms != null) { + for (VMInstanceVO vm : vms) { + if (skipCalculation(vm)) { continue; } - } - so = _offeringDao.findById(vm.getId(), vm.getServiceOfferingId()); + ServiceOffering so = null; - if (capacityType == CapacityVO.CAPACITY_TYPE_MEMORY) { - usedCapacity += so.getRamSize() * 1024L * 1024L; - - if (s_logger.isDebugEnabled()) { - s_logger.debug("Counting memory capacity used by vm: " + vm.getId() + ", size: " + so.getRamSize() + "MB, host: " + hostId - + ", currently counted: " + usedCapacity + " Bytes"); + if (vm.getType() == VirtualMachine.Type.User) { + UserVmVO userVm = _vmDao.findById(vm.getId()); + if (userVm == null) { + continue; + } } - } else if (capacityType == CapacityVO.CAPACITY_TYPE_CPU) { - usedCapacity += so.getCpu() * so.getSpeed(); - if (s_logger.isDebugEnabled()) { - s_logger.debug("Counting cpu capacity used by vm: " + vm.getId() + ", cpu: " + so.getCpu() + ", speed: " + so.getSpeed() - + ", currently counted: " + usedCapacity + " Bytes"); + so = _offeringDao.findById(vm.getId(), vm.getServiceOfferingId()); + + if (capacityType == CapacityVO.CAPACITY_TYPE_MEMORY) { + usedCapacity += so.getRamSize() * 1024L * 1024L; + + if (s_logger.isDebugEnabled()) { + s_logger.debug("Counting memory capacity used by vm: " + vm.getId() + ", size: " + so.getRamSize() + "MB, host: " + hostId + + ", currently counted: " + usedCapacity + " Bytes"); + } + } else if (capacityType == CapacityVO.CAPACITY_TYPE_CPU) { + usedCapacity += so.getCpu() * so.getSpeed(); + + if (s_logger.isDebugEnabled()) { + s_logger.debug("Counting cpu capacity used by vm: " + vm.getId() + ", cpu: " + so.getCpu() + ", speed: " + so.getSpeed() + + ", currently counted: " + usedCapacity + " Bytes"); + } } } } From f3529a19a9aad36dbd92e311018643629f19c748 Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Sat, 25 Jan 2014 23:46:18 +0100 Subject: [PATCH 015/169] moved call on static object to synchronized block --- server/src/com/cloud/api/ApiDispatcher.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/com/cloud/api/ApiDispatcher.java b/server/src/com/cloud/api/ApiDispatcher.java index 44fb6765a34..9624c61cb9e 100755 --- a/server/src/com/cloud/api/ApiDispatcher.java +++ b/server/src/com/cloud/api/ApiDispatcher.java @@ -429,9 +429,9 @@ public class ApiDispatcher { } } } else { - DateFormat format = BaseCmd.INPUT_FORMAT; - format.setLenient(false); + final DateFormat format = BaseCmd.INPUT_FORMAT; synchronized (format) { + format.setLenient(false); field.set(cmdObj, format.parse(paramObj.toString())); } } From 437ff438af23f7ff499200e212966f275a309a75 Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Sun, 26 Jan 2014 00:34:41 +0100 Subject: [PATCH 016/169] restored imports --- .../allocator/impl/UserConcentratedAllocator.java | 12 ++++++++++++ server/src/com/cloud/api/ApiResponseHelper.java | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/server/src/com/cloud/agent/manager/allocator/impl/UserConcentratedAllocator.java b/server/src/com/cloud/agent/manager/allocator/impl/UserConcentratedAllocator.java index 8101da94c0d..e2e06c2b4b1 100755 --- a/server/src/com/cloud/agent/manager/allocator/impl/UserConcentratedAllocator.java +++ b/server/src/com/cloud/agent/manager/allocator/impl/UserConcentratedAllocator.java @@ -32,18 +32,30 @@ import org.apache.log4j.Logger; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import com.cloud.agent.manager.allocator.PodAllocator; +import com.cloud.capacity.CapacityVO; +import com.cloud.capacity.dao.CapacityDao; import com.cloud.dc.DataCenter; +import com.cloud.dc.HostPodVO; import com.cloud.dc.Pod; +import com.cloud.dc.dao.HostPodDao; import com.cloud.offering.ServiceOffering; +import com.cloud.service.dao.ServiceOfferingDao; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.VMTemplatePoolDao; +import com.cloud.storage.dao.VolumeDao; import com.cloud.template.VirtualMachineTemplate; import com.cloud.utils.DateUtil; import com.cloud.utils.NumbersUtil; import com.cloud.utils.Pair; import com.cloud.utils.component.AdapterBase; import com.cloud.utils.db.SearchCriteria; +import com.cloud.vm.UserVmVO; +import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachine.State; import com.cloud.vm.VirtualMachineProfile; +import com.cloud.vm.dao.UserVmDao; +import com.cloud.vm.dao.VMInstanceDao; @Local(value = PodAllocator.class) public class UserConcentratedAllocator extends AdapterBase implements PodAllocator { diff --git a/server/src/com/cloud/api/ApiResponseHelper.java b/server/src/com/cloud/api/ApiResponseHelper.java index 8cfc8596aff..ceb4ad6c408 100755 --- a/server/src/com/cloud/api/ApiResponseHelper.java +++ b/server/src/com/cloud/api/ApiResponseHelper.java @@ -387,7 +387,7 @@ public class ApiResponseHelper implements ResponseGenerator { } resourceLimitResponse.setResourceType(Integer.valueOf(limit.getType().getOrdinal()).toString()); if((limit.getType() == ResourceType.primary_storage || limit.getType() == ResourceType.secondary_storage) && limit.getMax() >= 0) { - resourceLimitResponse.setMax((long) Math.ceil(limit.getMax()/ResourceType.bytesToGiB)); + resourceLimitResponse.setMax((long)Math.ceil((limit.getMax() / ResourceType.bytesToGiB))); } else { resourceLimitResponse.setMax(limit.getMax()); } From 92b4f66d73562e4211d2d787554ff229dbeb5705 Mon Sep 17 00:00:00 2001 From: Ian Duffy Date: Sun, 26 Jan 2014 00:34:25 +0000 Subject: [PATCH 017/169] Fix findbug issues within LDAP authenticator --- .../cloudstack/api/command/LDAPConfigCmd.java | 2 +- .../api/command/LdapCreateAccountCmd.java | 206 +++++++++--------- .../api/command/LdapImportUsersCmd.java | 17 +- .../cloudstack/ldap/LdapContextFactory.java | 2 - .../ldap/LdapAuthenticatorSpec.groovy | 8 +- .../ldap/LdapManagerImplSpec.groovy | 16 +- 6 files changed, 118 insertions(+), 133 deletions(-) diff --git a/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/command/LDAPConfigCmd.java b/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/command/LDAPConfigCmd.java index 3faf8b768b2..a4883c51a01 100644 --- a/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/command/LDAPConfigCmd.java +++ b/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/command/LDAPConfigCmd.java @@ -209,7 +209,7 @@ public class LDAPConfigCmd extends BaseCmd { } private boolean updateLDAP() { - LdapConfigurationResponse response = _ldapManager.addConfiguration(hostname, port); + _ldapManager.addConfiguration(hostname, port); /** * There is no query filter now. It is derived from ldap.user.object and ldap.search.group.principle diff --git a/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/command/LdapCreateAccountCmd.java b/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/command/LdapCreateAccountCmd.java index 981e72e64e1..b78b484627f 100644 --- a/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/command/LdapCreateAccountCmd.java +++ b/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/command/LdapCreateAccountCmd.java @@ -35,7 +35,6 @@ import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.ldap.LdapManager; import org.apache.cloudstack.ldap.LdapUser; import org.apache.log4j.Logger; -import org.bouncycastle.util.encoders.Base64; import com.cloud.user.Account; import com.cloud.user.AccountService; @@ -43,125 +42,126 @@ import com.cloud.user.UserAccount; @APICommand(name = "ldapCreateAccount", description = "Creates an account from an LDAP user", responseObject = AccountResponse.class, since = "4.2.0") public class LdapCreateAccountCmd extends BaseCmd { - public static final Logger s_logger = Logger - .getLogger(LdapCreateAccountCmd.class.getName()); - private static final String s_name = "createaccountresponse"; + public static final Logger s_logger = Logger + .getLogger(LdapCreateAccountCmd.class.getName()); + private static final String s_name = "createaccountresponse"; - @Inject - private LdapManager _ldapManager; + @Inject + private LdapManager _ldapManager; - @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, description = "Creates the user under the specified account. If no account is specified, the username will be used as the account name.") - private String accountName; + @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, description = "Creates the user under the specified account. If no account is specified, the username will be used as the account name.") + private String accountName; - @Parameter(name = ApiConstants.ACCOUNT_TYPE, type = CommandType.SHORT, required = true, description = "Type of the account. Specify 0 for user, 1 for root admin, and 2 for domain admin") - private Short accountType; + @Parameter(name = ApiConstants.ACCOUNT_TYPE, type = CommandType.SHORT, required = true, description = "Type of the account. Specify 0 for user, 1 for root admin, and 2 for domain admin") + private Short accountType; - @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, entityType = DomainResponse.class, description = "Creates the user under the specified domain.") - private Long domainId; + @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, entityType = DomainResponse.class, description = "Creates the user under the specified domain.") + private Long domainId; - @Parameter(name = ApiConstants.TIMEZONE, type = CommandType.STRING, description = "Specifies a timezone for this command. For more information on the timezone parameter, see Time Zone Format.") - private String timezone; + @Parameter(name = ApiConstants.TIMEZONE, type = CommandType.STRING, description = "Specifies a timezone for this command. For more information on the timezone parameter, see Time Zone Format.") + private String timezone; - @Parameter(name = ApiConstants.USERNAME, type = CommandType.STRING, required = true, description = "Unique username.") - private String username; + @Parameter(name = ApiConstants.USERNAME, type = CommandType.STRING, required = true, description = "Unique username.") + private String username; - @Parameter(name = ApiConstants.NETWORK_DOMAIN, type = CommandType.STRING, description = "Network domain for the account's networks") - private String networkDomain; + @Parameter(name = ApiConstants.NETWORK_DOMAIN, type = CommandType.STRING, description = "Network domain for the account's networks") + private String networkDomain; - @Parameter(name = ApiConstants.ACCOUNT_DETAILS, type = CommandType.MAP, description = "details for account used to store specific parameters") - private Map details; + @Parameter(name = ApiConstants.ACCOUNT_DETAILS, type = CommandType.MAP, description = "details for account used to store specific parameters") + private Map details; - @Parameter(name = ApiConstants.ACCOUNT_ID, type = CommandType.STRING, description = "Account UUID, required for adding account from external provisioning system") - private String accountUUID; + @Parameter(name = ApiConstants.ACCOUNT_ID, type = CommandType.STRING, description = "Account UUID, required for adding account from external provisioning system") + private String accountUUID; - @Parameter(name = ApiConstants.USER_ID, type = CommandType.STRING, description = "User UUID, required for adding account from external provisioning system") - private String userUUID; + @Parameter(name = ApiConstants.USER_ID, type = CommandType.STRING, description = "User UUID, required for adding account from external provisioning system") + private String userUUID; - public LdapCreateAccountCmd() { - super(); - } + public LdapCreateAccountCmd() { + super(); + } - public LdapCreateAccountCmd(final LdapManager ldapManager, - final AccountService accountService) { - super(); - _ldapManager = ldapManager; - _accountService = accountService; - } + public LdapCreateAccountCmd(final LdapManager ldapManager, + final AccountService accountService) { + super(); + _ldapManager = ldapManager; + _accountService = accountService; + } - UserAccount createCloudstackUserAccount(final LdapUser user) { - return _accountService.createUserAccount(username, generatePassword(), - user.getFirstname(), user.getLastname(), user.getEmail(), - timezone, accountName, accountType, domainId, networkDomain, - details, accountUUID, userUUID); - } + UserAccount createCloudstackUserAccount(final LdapUser user) { + return _accountService.createUserAccount(username, generatePassword(), + user.getFirstname(), user.getLastname(), user.getEmail(), + timezone, accountName, accountType, domainId, networkDomain, + details, accountUUID, userUUID); + } - @Override - public void execute() throws ServerApiException { - final CallContext callContext = getCurrentContext(); - callContext.setEventDetails("Account Name: " + accountName - + ", Domain Id:" + domainId); - try { - final LdapUser user = _ldapManager.getUser(username); - validateUser(user); - final UserAccount userAccount = createCloudstackUserAccount(user); - if (userAccount != null) { - final AccountResponse response = _responseGenerator - .createUserAccountResponse(userAccount); - response.setResponseName(getCommandName()); - setResponseObject(response); - } else { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, - "Failed to create a user account"); - } - } catch (final NamingException e) { - throw new ServerApiException( - ApiErrorCode.RESOURCE_UNAVAILABLE_ERROR, - "No LDAP user exists with the username of " + username); - } - } + @Override + public void execute() throws ServerApiException { + final CallContext callContext = getCurrentContext(); + callContext.setEventDetails("Account Name: " + accountName + + ", Domain Id:" + domainId); + try { + final LdapUser user = _ldapManager.getUser(username); + validateUser(user); + final UserAccount userAccount = createCloudstackUserAccount(user); + if (userAccount != null) { + final AccountResponse response = _responseGenerator + .createUserAccountResponse(userAccount); + response.setResponseName(getCommandName()); + setResponseObject(response); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, + "Failed to create a user account"); + } + } catch (final NamingException e) { + throw new ServerApiException( + ApiErrorCode.RESOURCE_UNAVAILABLE_ERROR, + "No LDAP user exists with the username of " + username); + } + } - private String generatePassword() throws ServerApiException { - try { - final SecureRandom randomGen = SecureRandom.getInstance("SHA1PRNG"); - final byte bytes[] = new byte[20]; - randomGen.nextBytes(bytes); - return Base64.encode(bytes).toString(); - } catch (final NoSuchAlgorithmException e) { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, - "Failed to generate random password"); - } - } + private String generatePassword() throws ServerApiException { + final SecureRandom random = new SecureRandom(); + final int length = 20; + final String characters = "abcdefghjkmnpqrstuvwxyzABCDEFGHJKMNPQRSTUVWXYZ23456789!@£$%^&*()_+="; - @Override - public String getCommandName() { - return s_name; - } + String password = ""; + for (int i = 0; i < length; i++) { + int index = (int) (random.nextDouble() * characters.length()); + password += characters.charAt(index); + } + return password; + } - CallContext getCurrentContext() { - return CallContext.current(); - } + @Override + public String getCommandName() { + return s_name; + } - @Override - public long getEntityOwnerId() { - return Account.ACCOUNT_ID_SYSTEM; - } + CallContext getCurrentContext() { + return CallContext.current(); + } - private boolean validateUser(final LdapUser user) throws ServerApiException { - if (user.getEmail() == null) { - throw new ServerApiException( - ApiErrorCode.RESOURCE_UNAVAILABLE_ERROR, username - + " has no email address set within LDAP"); - } - if (user.getFirstname() == null) { - throw new ServerApiException( - ApiErrorCode.RESOURCE_UNAVAILABLE_ERROR, username - + " has no firstname set within LDAP"); - } - if (user.getLastname() == null) { - throw new ServerApiException( - ApiErrorCode.RESOURCE_UNAVAILABLE_ERROR, username - + " has no lastname set within LDAP"); - } - return true; - } + @Override + public long getEntityOwnerId() { + return Account.ACCOUNT_ID_SYSTEM; + } + + private boolean validateUser(final LdapUser user) throws ServerApiException { + if (user.getEmail() == null) { + throw new ServerApiException( + ApiErrorCode.RESOURCE_UNAVAILABLE_ERROR, username + + " has no email address set within LDAP"); + } + if (user.getFirstname() == null) { + throw new ServerApiException( + ApiErrorCode.RESOURCE_UNAVAILABLE_ERROR, username + + " has no firstname set within LDAP"); + } + if (user.getLastname() == null) { + throw new ServerApiException( + ApiErrorCode.RESOURCE_UNAVAILABLE_ERROR, username + + " has no lastname set within LDAP"); + } + return true; + } } \ No newline at end of file diff --git a/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/command/LdapImportUsersCmd.java b/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/command/LdapImportUsersCmd.java index 1855d5d41c5..d82276ca18f 100644 --- a/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/command/LdapImportUsersCmd.java +++ b/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/command/LdapImportUsersCmd.java @@ -34,7 +34,6 @@ import org.apache.cloudstack.ldap.LdapUser; import org.apache.cloudstack.ldap.NoLdapUserMatchingQueryException; import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; -import org.bouncycastle.util.encoders.Base64; import com.cloud.domain.Domain; import com.cloud.exception.*; @@ -171,13 +170,15 @@ public class LdapImportUsersCmd extends BaseListCmd { } private String generatePassword() throws ServerApiException { - try { - final SecureRandom randomGen = SecureRandom.getInstance("SHA1PRNG"); - final byte bytes[] = new byte[20]; - randomGen.nextBytes(bytes); - return Base64.encode(bytes).toString(); - } catch (final NoSuchAlgorithmException e) { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to generate random password"); + final SecureRandom random = new SecureRandom(); + final int length = 20; + final String characters = "abcdefghjkmnpqrstuvwxyzABCDEFGHJKMNPQRSTUVWXYZ23456789!@£$%^&*()_+="; + + String password = ""; + for (int i = 0; i < length; i++) { + int index = (int) (random.nextDouble() * characters.length()); + password += characters.charAt(index); } + return password; } } diff --git a/plugins/user-authenticators/ldap/src/org/apache/cloudstack/ldap/LdapContextFactory.java b/plugins/user-authenticators/ldap/src/org/apache/cloudstack/ldap/LdapContextFactory.java index ceeed6862fb..c0d5d509a81 100644 --- a/plugins/user-authenticators/ldap/src/org/apache/cloudstack/ldap/LdapContextFactory.java +++ b/plugins/user-authenticators/ldap/src/org/apache/cloudstack/ldap/LdapContextFactory.java @@ -95,8 +95,6 @@ public class LdapContextFactory { environment.put(Context.INITIAL_CONTEXT_FACTORY, factory); environment.put(Context.PROVIDER_URL, url); - environment.put("com.sun.jndi.ldap.read.timeout", "500"); - environment.put("com.sun.jndi.ldap.connect.pool", "true"); enableSSL(environment); setAuthentication(environment, isSystemContext); diff --git a/plugins/user-authenticators/ldap/test/groovy/org/apache/cloudstack/ldap/LdapAuthenticatorSpec.groovy b/plugins/user-authenticators/ldap/test/groovy/org/apache/cloudstack/ldap/LdapAuthenticatorSpec.groovy index 416c1330359..51f8e84559a 100644 --- a/plugins/user-authenticators/ldap/test/groovy/org/apache/cloudstack/ldap/LdapAuthenticatorSpec.groovy +++ b/plugins/user-authenticators/ldap/test/groovy/org/apache/cloudstack/ldap/LdapAuthenticatorSpec.groovy @@ -34,7 +34,7 @@ class LdapAuthenticatorSpec extends spock.lang.Specification { when: "A user authentications" def result = ldapAuthenticator.authenticate("rmurphy", "password", 0, null) then: "their authentication fails" - result == false + result.first() == false } def "Test failed authentication due to ldap bind being unsuccessful"() { @@ -51,7 +51,7 @@ class LdapAuthenticatorSpec extends spock.lang.Specification { def result = ldapAuthenticator.authenticate("rmurphy", "password", 0, null) then: "their authentication fails" - result == false + result.first() == false } def "Test failed authentication due to ldap not being configured"() { @@ -66,7 +66,7 @@ class LdapAuthenticatorSpec extends spock.lang.Specification { when: "The user authenticates" def result = ldapAuthenticator.authenticate("rmurphy", "password", 0, null) then: "their authentication fails" - result == false + result.first() == false } def "Test successful authentication"() { @@ -83,7 +83,7 @@ class LdapAuthenticatorSpec extends spock.lang.Specification { def result = ldapAuthenticator.authenticate("rmurphy", "password", 0, null) then: "their authentication passes" - result == true + result.first() == true } def "Test that encode doesn't change the input"() { diff --git a/plugins/user-authenticators/ldap/test/groovy/org/apache/cloudstack/ldap/LdapManagerImplSpec.groovy b/plugins/user-authenticators/ldap/test/groovy/org/apache/cloudstack/ldap/LdapManagerImplSpec.groovy index 42988e0caef..c0ca2e8a45f 100644 --- a/plugins/user-authenticators/ldap/test/groovy/org/apache/cloudstack/ldap/LdapManagerImplSpec.groovy +++ b/plugins/user-authenticators/ldap/test/groovy/org/apache/cloudstack/ldap/LdapManagerImplSpec.groovy @@ -297,30 +297,16 @@ class LdapManagerImplSpec extends spock.lang.Specification { thrown InvalidParameterValueException } - def supportedLdapCommands() { - List> cmdList = new ArrayList>(); - cmdList.add(LdapUserSearchCmd.class); - cmdList.add(LdapListUsersCmd.class); - cmdList.add(LdapAddConfigurationCmd.class); - cmdList.add(LdapDeleteConfigurationCmd.class); - cmdList.add(LdapListConfigurationCmd.class); - cmdList.add(LdapCreateAccountCmd.class); - cmdList.add(LdapImportUsersCmd.class); - return cmdList - } - def "Test that getCommands isn't empty"() { given: "We have an LdapConfigurationDao, LdapContextFactory, LdapUserManager and LdapManager" def ldapConfigurationDao = Mock(LdapConfigurationDaoImpl) def ldapContextFactory = Mock(LdapContextFactory) def ldapUserManager = Mock(LdapUserManager) - final List> cmdList = supportedLdapCommands() def ldapManager = new LdapManagerImpl(ldapConfigurationDao, ldapContextFactory, ldapUserManager) when: "Get commands is called" def result = ldapManager.getCommands() - then: "it must return all the commands" + then: "it must contain commands" result.size() > 0 - result == cmdList } def "Testing of listConfigurations"() { From 9a1b882d0eb871c64fe0f0f3fbafbabae89188fa Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Sun, 26 Jan 2014 12:25:18 +0100 Subject: [PATCH 018/169] Findbugs issues in ApiResponseHelper fixes --- .../com/cloud/network/rules/FirewallRule.java | 3 ++- .../com/cloud/network/vpc/StaticRoute.java | 3 ++- .../src/com/cloud/api/ApiResponseHelper.java | 21 ++++++++----------- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/api/src/com/cloud/network/rules/FirewallRule.java b/api/src/com/cloud/network/rules/FirewallRule.java index 042665cdcf9..0c74d02e224 100644 --- a/api/src/com/cloud/network/rules/FirewallRule.java +++ b/api/src/com/cloud/network/rules/FirewallRule.java @@ -41,7 +41,8 @@ public interface FirewallRule extends ControlledEntity, Identity, InternalIdenti Staged, // Rule been created but has never got through network rule conflict detection. Rules in this state can not be sent to network elements. Add, // Add means the rule has been created and has gone through network rule conflict detection. Active, // Rule has been sent to the network elements and reported to be active. - Revoke // Revoke means this rule has been revoked. If this rule has been sent to the network elements, the rule will be deleted from database. + Revoke, // Revoke means this rule has been revoked. If this rule has been sent to the network elements, the rule will be deleted from database. + Deleting // rule has been revoked and is scheduled for deletion } enum TrafficType { diff --git a/api/src/com/cloud/network/vpc/StaticRoute.java b/api/src/com/cloud/network/vpc/StaticRoute.java index ccdbec899b8..5707ca14024 100644 --- a/api/src/com/cloud/network/vpc/StaticRoute.java +++ b/api/src/com/cloud/network/vpc/StaticRoute.java @@ -25,7 +25,8 @@ public interface StaticRoute extends ControlledEntity, Identity, InternalIdentit Staged, // route been created but has never got through network rule conflict detection. Routes in this state can not be sent to VPC virtual router. Add, // Add means the route has been created and has gone through network rule conflict detection. Active, // Route has been sent to the VPC router and reported to be active. - Revoke // Revoke means this route has been revoked. If this route has been sent to the VPC router, the route will be deleted from database. + Revoke, // Revoke means this route has been revoked. If this route has been sent to the VPC router, the route will be deleted from database. + Deleting // rule has been revoked and is scheduled for deletion } /** diff --git a/server/src/com/cloud/api/ApiResponseHelper.java b/server/src/com/cloud/api/ApiResponseHelper.java index ceb4ad6c408..5bd8241f2ab 100755 --- a/server/src/com/cloud/api/ApiResponseHelper.java +++ b/server/src/com/cloud/api/ApiResponseHelper.java @@ -31,6 +31,8 @@ import java.util.TimeZone; import javax.inject.Inject; +import org.apache.log4j.Logger; + import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.affinity.AffinityGroup; @@ -146,7 +148,6 @@ import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.usage.Usage; import org.apache.cloudstack.usage.UsageService; import org.apache.cloudstack.usage.UsageTypes; -import org.apache.log4j.Logger; import com.cloud.api.query.ViewResponseHelper; import com.cloud.api.query.vo.AccountJoinVO; @@ -285,7 +286,6 @@ import com.cloud.uservm.UserVm; import com.cloud.utils.Pair; import com.cloud.utils.StringUtils; import com.cloud.utils.db.EntityManager; -import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.net.Ip; import com.cloud.utils.net.NetUtils; import com.cloud.vm.ConsoleProxyVO; @@ -387,7 +387,7 @@ public class ApiResponseHelper implements ResponseGenerator { } resourceLimitResponse.setResourceType(Integer.valueOf(limit.getType().getOrdinal()).toString()); if((limit.getType() == ResourceType.primary_storage || limit.getType() == ResourceType.secondary_storage) && limit.getMax() >= 0) { - resourceLimitResponse.setMax((long)Math.ceil((limit.getMax() / ResourceType.bytesToGiB))); + resourceLimitResponse.setMax((long)Math.ceil((double)limit.getMax() / ResourceType.bytesToGiB)); } else { resourceLimitResponse.setMax(limit.getMax()); } @@ -1951,8 +1951,6 @@ public class ApiResponseHelper implements ResponseGenerator { public SecurityGroupResponse createSecurityGroupResponseFromSecurityGroupRule(List securityRules) { SecurityGroupResponse response = new SecurityGroupResponse(); Map securiytGroupAccounts = new HashMap(); - Map allowedSecurityGroups = new HashMap(); - Map allowedSecuriytGroupAccounts = new HashMap(); if ((securityRules != null) && !securityRules.isEmpty()) { SecurityGroupJoinVO securityGroup = ApiDBUtils.findSecurityGroupViewById(securityRules.get(0).getSecurityGroupId()).get(0); @@ -3137,11 +3135,10 @@ public class ApiResponseHelper implements ResponseGenerator { response.setCidr(result.getCidr()); StaticRoute.State state = result.getState(); - String stateToSet = state.toString(); - if (state.equals(FirewallRule.State.Revoke)) { - stateToSet = "Deleting"; + if (state.equals(StaticRoute.State.Revoke)) { + state = StaticRoute.State.Deleting; } - response.setState(stateToSet); + response.setState(state.toString()); populateAccount(response, result.getAccountId()); populateDomain(response, result.getDomainId()); @@ -3629,11 +3626,11 @@ public class ApiResponseHelper implements ResponseGenerator { ApplicationLoadBalancerRuleResponse ruleResponse = new ApplicationLoadBalancerRuleResponse(); ruleResponse.setInstancePort(lb.getDefaultPortStart()); ruleResponse.setSourcePort(lb.getSourcePortStart()); - String stateToSet = lb.getState().toString(); + FirewallRule.State stateToSet = lb.getState(); if (stateToSet.equals(FirewallRule.State.Revoke)) { - stateToSet = "Deleting"; + stateToSet = FirewallRule.State.Deleting; } - ruleResponse.setState(stateToSet); + ruleResponse.setState(stateToSet.toString()); ruleResponse.setObjectName("loadbalancerrule"); ruleResponses.add(ruleResponse); lbResponse.setLbRules(ruleResponses); From 9aced41d708acd22b43ef0e512fa2cc657a6c0a2 Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Sun, 26 Jan 2014 13:27:37 +0100 Subject: [PATCH 019/169] findbugs fixes for ApiServer, ApiSerializerHelper and ApiXmlDocWriter --- .../com/cloud/api/ApiSerializerHelper.java | 2 +- server/src/com/cloud/api/ApiServer.java | 22 ++++++++++++++----- server/src/com/cloud/api/ApiServlet.java | 3 ++- .../cloud/api/EncodedStringTypeAdapter.java | 2 +- server/src/com/cloud/api/doc/Alert.java | 12 +++++++--- server/src/com/cloud/api/doc/Command.java | 13 +++++++---- .../api/response/ApiResponseSerializer.java | 2 +- 7 files changed, 40 insertions(+), 16 deletions(-) diff --git a/server/src/com/cloud/api/ApiSerializerHelper.java b/server/src/com/cloud/api/ApiSerializerHelper.java index e65d5e7e8da..67b9a3891b6 100644 --- a/server/src/com/cloud/api/ApiSerializerHelper.java +++ b/server/src/com/cloud/api/ApiSerializerHelper.java @@ -24,7 +24,7 @@ import org.apache.cloudstack.api.ResponseObject; public class ApiSerializerHelper { public static final Logger s_logger = Logger.getLogger(ApiSerializerHelper.class.getName()); - public static String token = "/"; + private static String token = "/"; public static String toSerializedString(Object result) { if (result != null) { diff --git a/server/src/com/cloud/api/ApiServer.java b/server/src/com/cloud/api/ApiServer.java index bf987be860d..c6a581ce420 100755 --- a/server/src/com/cloud/api/ApiServer.java +++ b/server/src/com/cloud/api/ApiServer.java @@ -159,9 +159,9 @@ public class ApiServer extends ManagerBase implements HttpRequestHandler, ApiSer private static final Logger s_logger = Logger.getLogger(ApiServer.class.getName()); private static final Logger s_accessLogger = Logger.getLogger("apiserver." + ApiServer.class.getName()); - public static boolean encodeApiResponse = false; - public static String jsonContentType = "text/javascript"; - public static String controlCharacters = "[\000-\011\013-\014\016-\037\177]"; // Non-printable ASCII characters - numbers 0 to 31 and 127 decimal + private static boolean encodeApiResponse = false; + private static String jsonContentType = "text/javascript"; + private static String controlCharacters = "[\000-\011\013-\014\016-\037\177]"; // Non-printable ASCII characters - numbers 0 to 31 and 127 decimal @Inject ApiDispatcher _dispatcher; @Inject private AccountManager _accountMgr; @@ -237,7 +237,7 @@ public class ApiServer extends ManagerBase implements HttpRequestHandler, ApiSer _apiNameCmdClassMap.put(apiName, cmdClass); } - encodeApiResponse = Boolean.valueOf(_configDao.getValue(Config.EncodeApiResponse.key())); + setEncodeApiResponse(Boolean.valueOf(_configDao.getValue(Config.EncodeApiResponse.key()))); String jsonType = _configDao.getValue(Config.JavaScriptDefaultContentType.key()); if (jsonType != null) { jsonContentType = jsonType; @@ -375,7 +375,7 @@ public class ApiServer extends ManagerBase implements HttpRequestHandler, ApiSer buildAuditTrail(auditTrailSb, command[0], response); } else { if (!command[0].equalsIgnoreCase("login") && !command[0].equalsIgnoreCase("logout")) { - String errorString = "Unknown API command: " + ((command == null) ? "null" : command[0]); + String errorString = "Unknown API command: " + command[0]; s_logger.warn(errorString); auditTrailSb.append(" " + errorString); throw new ServerApiException(ApiErrorCode.UNSUPPORTED_ACTION_ERROR, errorString); @@ -1115,4 +1115,16 @@ public class ApiServer extends ManagerBase implements HttpRequestHandler, ApiSer public void setApiAccessCheckers(List _apiAccessCheckers) { this._apiAccessCheckers = _apiAccessCheckers; } + + public static boolean isEncodeApiResponse() { + return encodeApiResponse; + } + + private static void setEncodeApiResponse(boolean encodeApiResponse) { + ApiServer.encodeApiResponse = encodeApiResponse; + } + + public static String getJsonContentType() { + return jsonContentType; + } } diff --git a/server/src/com/cloud/api/ApiServlet.java b/server/src/com/cloud/api/ApiServlet.java index def18d0919d..6dd826a6613 100755 --- a/server/src/com/cloud/api/ApiServlet.java +++ b/server/src/com/cloud/api/ApiServlet.java @@ -34,6 +34,7 @@ import javax.servlet.http.HttpSession; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; import org.springframework.web.context.support.SpringBeanAutowiringSupport; + import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.ServerApiException; @@ -366,7 +367,7 @@ public class ApiServlet extends HttpServlet { private void writeResponse(HttpServletResponse resp, String response, int responseCode, String responseType) { try { if (BaseCmd.RESPONSE_TYPE_JSON.equalsIgnoreCase(responseType)) { - resp.setContentType(ApiServer.jsonContentType + "; charset=UTF-8"); + resp.setContentType(ApiServer.getJsonContentType() + "; charset=UTF-8"); } else { resp.setContentType("text/xml; charset=UTF-8"); } diff --git a/server/src/com/cloud/api/EncodedStringTypeAdapter.java b/server/src/com/cloud/api/EncodedStringTypeAdapter.java index daec4fd4389..dca58287c89 100644 --- a/server/src/com/cloud/api/EncodedStringTypeAdapter.java +++ b/server/src/com/cloud/api/EncodedStringTypeAdapter.java @@ -36,7 +36,7 @@ public class EncodedStringTypeAdapter implements JsonSerializer{ } private static String encodeString(String value) { - if (!ApiServer.encodeApiResponse) { + if (!ApiServer.isEncodeApiResponse()) { return value; } try { diff --git a/server/src/com/cloud/api/doc/Alert.java b/server/src/com/cloud/api/doc/Alert.java index 211f2e9d973..bfd3b2ac728 100644 --- a/server/src/com/cloud/api/doc/Alert.java +++ b/server/src/com/cloud/api/doc/Alert.java @@ -16,9 +16,15 @@ // under the License. package com.cloud.api.doc; -public class Alert { - private String type; - private int value; +import java.io.Serializable; + +public class Alert implements Serializable{ + /** + * + */ + private static final long serialVersionUID = 960408026527837920L; + private final String type; + private final int value; public Alert(String type, int value) { this.type = type; diff --git a/server/src/com/cloud/api/doc/Command.java b/server/src/com/cloud/api/doc/Command.java index a267605216e..1946b30d54e 100644 --- a/server/src/com/cloud/api/doc/Command.java +++ b/server/src/com/cloud/api/doc/Command.java @@ -16,11 +16,16 @@ // under the License. package com.cloud.api.doc; +import java.io.Serializable; import java.util.ArrayList; -public class Command { +public class Command implements Serializable{ - private String name; + /** + * + */ + private static final long serialVersionUID = -4318310162503004975L; + private String name; private String description; private String usage; private boolean isAsync; @@ -84,7 +89,7 @@ public class Command { } public Argument getReqArgByName(String name){ - for (Argument a : this.getRequest()) { + for (Argument a : getRequest()) { if (a.getName().equals(name)) { return a; } @@ -93,7 +98,7 @@ public class Command { } public Argument getResArgByName(String name){ - for (Argument a : this.getResponse()) { + for (Argument a : getResponse()) { if (a.getName().equals(name)) { return a; } diff --git a/server/src/com/cloud/api/response/ApiResponseSerializer.java b/server/src/com/cloud/api/response/ApiResponseSerializer.java index fb32e7d7cdd..a3e6f0f012f 100644 --- a/server/src/com/cloud/api/response/ApiResponseSerializer.java +++ b/server/src/com/cloud/api/response/ApiResponseSerializer.java @@ -277,7 +277,7 @@ public class ApiResponseSerializer { } private static String encodeParam(String value) { - if (!ApiServer.encodeApiResponse) { + if (!ApiServer.isEncodeApiResponse()) { return value; } try { From d30ddf6d59732fa30855811e7fe0aa639ee3849a Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Sun, 26 Jan 2014 15:11:56 +0100 Subject: [PATCH 020/169] findbugs: null check for gateway and netmask instead of gateway twice --- .../com/cloud/configuration/ConfigurationManagerImpl.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/src/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/com/cloud/configuration/ConfigurationManagerImpl.java index 99f2c194299..d2e5aaa39a1 100755 --- a/server/src/com/cloud/configuration/ConfigurationManagerImpl.java +++ b/server/src/com/cloud/configuration/ConfigurationManagerImpl.java @@ -36,6 +36,8 @@ import javax.ejb.Local; import javax.inject.Inject; import javax.naming.ConfigurationException; +import org.apache.log4j.Logger; + import org.apache.cloudstack.acl.SecurityChecker; import org.apache.cloudstack.affinity.AffinityGroup; import org.apache.cloudstack.affinity.AffinityGroupService; @@ -83,7 +85,6 @@ import org.apache.cloudstack.region.dao.RegionDao; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; -import org.apache.log4j.Logger; import com.cloud.alert.AlertManager; import com.cloud.api.ApiDBUtils; @@ -2754,7 +2755,7 @@ ConfigurationManagerImpl extends ManagerBase implements ConfigurationManager, Co } } return NetUtils.supersetOrSubset.neitherSubetNorSuperset; - } else if (newVlanGateway == null || newVlanGateway == null) { + } else if (newVlanGateway == null || newVlanNetmask == null) { throw new InvalidParameterValueException( "either both netmask and gateway should be passed or both should me omited."); } else { From 390325c50b4f401a0dd91fc3b6b0b03f21db0bbc Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Sun, 26 Jan 2014 15:20:24 +0100 Subject: [PATCH 021/169] findbugs: remove compatible objects from collection --- .../deploy/DeploymentPlanningManagerImpl.java | 37 +++++++++---------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/server/src/com/cloud/deploy/DeploymentPlanningManagerImpl.java b/server/src/com/cloud/deploy/DeploymentPlanningManagerImpl.java index a41f5ee4100..160e9cfde42 100644 --- a/server/src/com/cloud/deploy/DeploymentPlanningManagerImpl.java +++ b/server/src/com/cloud/deploy/DeploymentPlanningManagerImpl.java @@ -30,6 +30,8 @@ import javax.ejb.Local; import javax.inject.Inject; import javax.naming.ConfigurationException; +import org.apache.log4j.Logger; + import org.apache.cloudstack.affinity.AffinityGroupProcessor; import org.apache.cloudstack.affinity.AffinityGroupService; import org.apache.cloudstack.affinity.AffinityGroupVMMapVO; @@ -48,8 +50,16 @@ import org.apache.cloudstack.managed.context.ManagedContextTimerTask; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.utils.identity.ManagementServerNode; -import org.apache.log4j.Logger; +import com.cloud.agent.AgentManager; +import com.cloud.agent.Listener; +import com.cloud.agent.api.AgentControlAnswer; +import com.cloud.agent.api.AgentControlCommand; +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.Command; +import com.cloud.agent.api.StartupCommand; +import com.cloud.agent.api.StartupRoutingCommand; +import com.cloud.agent.manager.allocator.HostAllocator; import com.cloud.capacity.CapacityManager; import com.cloud.capacity.dao.CapacityDao; import com.cloud.configuration.Config; @@ -70,7 +80,6 @@ import com.cloud.deploy.dao.PlannerHostReservationDao; import com.cloud.exception.AffinityConflictException; import com.cloud.exception.ConnectionException; import com.cloud.exception.InsufficientServerCapacityException; -import com.cloud.exception.PermissionDeniedException; import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.host.Status; @@ -102,7 +111,6 @@ import com.cloud.utils.db.DB; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.Transaction; import com.cloud.utils.db.TransactionCallback; -import com.cloud.utils.db.TransactionCallbackNoReturn; import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.fsm.StateListener; @@ -111,19 +119,10 @@ import com.cloud.vm.ReservationContext; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachine.Event; -import com.cloud.vm.VirtualMachineProfile; import com.cloud.vm.VirtualMachine.State; +import com.cloud.vm.VirtualMachineProfile; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.VMInstanceDao; -import com.cloud.agent.AgentManager; -import com.cloud.agent.Listener; -import com.cloud.agent.api.AgentControlAnswer; -import com.cloud.agent.api.AgentControlCommand; -import com.cloud.agent.api.Answer; -import com.cloud.agent.api.Command; -import com.cloud.agent.api.StartupCommand; -import com.cloud.agent.api.StartupRoutingCommand; -import com.cloud.agent.manager.allocator.HostAllocator; @Local(value = { DeploymentPlanningManager.class }) @@ -207,7 +206,7 @@ public class DeploymentPlanningManagerImpl extends ManagerBase implements Deploy return _affinityProcessors; } public void setAffinityGroupProcessors(List affinityProcessors) { - this._affinityProcessors = affinityProcessors; + _affinityProcessors = affinityProcessors; } @Override @@ -951,7 +950,7 @@ public class DeploymentPlanningManagerImpl extends ManagerBase implements Deploy } // Cluster can be put in avoid set in following scenarios: - // 1. If storage allocators haven't put any pools in avoid set means either no pools in cluster + // 1. If storage allocators haven't put any pools in avoid set means either no pools in cluster // or pools not suitable for the allocators to handle or there is no // linkage of any suitable host to any of the pools in cluster // 2. If all 'shared' or 'local' pools are in avoid set @@ -1152,9 +1151,9 @@ public class DeploymentPlanningManagerImpl extends ManagerBase implements Deploy List suitablePools = new ArrayList(); StoragePool pool = null; if (toBeCreated.getPoolId() != null) { - pool = (StoragePool) this.dataStoreMgr.getPrimaryDataStore(toBeCreated.getPoolId()); + pool = (StoragePool) dataStoreMgr.getPrimaryDataStore(toBeCreated.getPoolId()); } else { - pool = (StoragePool) this.dataStoreMgr.getPrimaryDataStore(plan.getPoolId()); + pool = (StoragePool) dataStoreMgr.getPrimaryDataStore(plan.getPoolId()); } if (!pool.isInMaintenance()) { @@ -1167,7 +1166,7 @@ public class DeploymentPlanningManagerImpl extends ManagerBase implements Deploy && plan.getClusterId() == exstPoolClusterId) { canReusePool = true; } else if (plan.getDataCenterId() == exstPoolDcId) { - DataStore dataStore = this.dataStoreMgr.getPrimaryDataStore(pool.getId()); + DataStore dataStore = dataStoreMgr.getPrimaryDataStore(pool.getId()); if (dataStore != null && dataStore.getScope() != null && dataStore.getScope().getScopeType() == ScopeType.ZONE) { canReusePool = true; @@ -1268,7 +1267,7 @@ public class DeploymentPlanningManagerImpl extends ManagerBase implements Deploy } } if (suitableVolumeStoragePools.values() != null) { - poolsToAvoidOutput.removeAll(suitableVolumeStoragePools.values()); + poolsToAvoidOutput.removeAll(suitableVolumeStoragePools.keySet()); } if (avoid.getPoolsToAvoid() != null) { avoid.getPoolsToAvoid().addAll(poolsToAvoidOutput); From df6f40fe737b68bb11afb075c8fae32029e7199d Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Sun, 26 Jan 2014 15:25:19 +0100 Subject: [PATCH 022/169] Revert "findbugs: remove compatible objects from collection" This reverts commit 390325c50b4f401a0dd91fc3b6b0b03f21db0bbc. --- .../deploy/DeploymentPlanningManagerImpl.java | 37 ++++++++++--------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/server/src/com/cloud/deploy/DeploymentPlanningManagerImpl.java b/server/src/com/cloud/deploy/DeploymentPlanningManagerImpl.java index 160e9cfde42..a41f5ee4100 100644 --- a/server/src/com/cloud/deploy/DeploymentPlanningManagerImpl.java +++ b/server/src/com/cloud/deploy/DeploymentPlanningManagerImpl.java @@ -30,8 +30,6 @@ import javax.ejb.Local; import javax.inject.Inject; import javax.naming.ConfigurationException; -import org.apache.log4j.Logger; - import org.apache.cloudstack.affinity.AffinityGroupProcessor; import org.apache.cloudstack.affinity.AffinityGroupService; import org.apache.cloudstack.affinity.AffinityGroupVMMapVO; @@ -50,16 +48,8 @@ import org.apache.cloudstack.managed.context.ManagedContextTimerTask; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.utils.identity.ManagementServerNode; +import org.apache.log4j.Logger; -import com.cloud.agent.AgentManager; -import com.cloud.agent.Listener; -import com.cloud.agent.api.AgentControlAnswer; -import com.cloud.agent.api.AgentControlCommand; -import com.cloud.agent.api.Answer; -import com.cloud.agent.api.Command; -import com.cloud.agent.api.StartupCommand; -import com.cloud.agent.api.StartupRoutingCommand; -import com.cloud.agent.manager.allocator.HostAllocator; import com.cloud.capacity.CapacityManager; import com.cloud.capacity.dao.CapacityDao; import com.cloud.configuration.Config; @@ -80,6 +70,7 @@ import com.cloud.deploy.dao.PlannerHostReservationDao; import com.cloud.exception.AffinityConflictException; import com.cloud.exception.ConnectionException; import com.cloud.exception.InsufficientServerCapacityException; +import com.cloud.exception.PermissionDeniedException; import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.host.Status; @@ -111,6 +102,7 @@ import com.cloud.utils.db.DB; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.Transaction; import com.cloud.utils.db.TransactionCallback; +import com.cloud.utils.db.TransactionCallbackNoReturn; import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.fsm.StateListener; @@ -119,10 +111,19 @@ import com.cloud.vm.ReservationContext; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachine.Event; -import com.cloud.vm.VirtualMachine.State; import com.cloud.vm.VirtualMachineProfile; +import com.cloud.vm.VirtualMachine.State; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.VMInstanceDao; +import com.cloud.agent.AgentManager; +import com.cloud.agent.Listener; +import com.cloud.agent.api.AgentControlAnswer; +import com.cloud.agent.api.AgentControlCommand; +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.Command; +import com.cloud.agent.api.StartupCommand; +import com.cloud.agent.api.StartupRoutingCommand; +import com.cloud.agent.manager.allocator.HostAllocator; @Local(value = { DeploymentPlanningManager.class }) @@ -206,7 +207,7 @@ public class DeploymentPlanningManagerImpl extends ManagerBase implements Deploy return _affinityProcessors; } public void setAffinityGroupProcessors(List affinityProcessors) { - _affinityProcessors = affinityProcessors; + this._affinityProcessors = affinityProcessors; } @Override @@ -950,7 +951,7 @@ public class DeploymentPlanningManagerImpl extends ManagerBase implements Deploy } // Cluster can be put in avoid set in following scenarios: - // 1. If storage allocators haven't put any pools in avoid set means either no pools in cluster + // 1. If storage allocators haven't put any pools in avoid set means either no pools in cluster // or pools not suitable for the allocators to handle or there is no // linkage of any suitable host to any of the pools in cluster // 2. If all 'shared' or 'local' pools are in avoid set @@ -1151,9 +1152,9 @@ public class DeploymentPlanningManagerImpl extends ManagerBase implements Deploy List suitablePools = new ArrayList(); StoragePool pool = null; if (toBeCreated.getPoolId() != null) { - pool = (StoragePool) dataStoreMgr.getPrimaryDataStore(toBeCreated.getPoolId()); + pool = (StoragePool) this.dataStoreMgr.getPrimaryDataStore(toBeCreated.getPoolId()); } else { - pool = (StoragePool) dataStoreMgr.getPrimaryDataStore(plan.getPoolId()); + pool = (StoragePool) this.dataStoreMgr.getPrimaryDataStore(plan.getPoolId()); } if (!pool.isInMaintenance()) { @@ -1166,7 +1167,7 @@ public class DeploymentPlanningManagerImpl extends ManagerBase implements Deploy && plan.getClusterId() == exstPoolClusterId) { canReusePool = true; } else if (plan.getDataCenterId() == exstPoolDcId) { - DataStore dataStore = dataStoreMgr.getPrimaryDataStore(pool.getId()); + DataStore dataStore = this.dataStoreMgr.getPrimaryDataStore(pool.getId()); if (dataStore != null && dataStore.getScope() != null && dataStore.getScope().getScopeType() == ScopeType.ZONE) { canReusePool = true; @@ -1267,7 +1268,7 @@ public class DeploymentPlanningManagerImpl extends ManagerBase implements Deploy } } if (suitableVolumeStoragePools.values() != null) { - poolsToAvoidOutput.removeAll(suitableVolumeStoragePools.keySet()); + poolsToAvoidOutput.removeAll(suitableVolumeStoragePools.values()); } if (avoid.getPoolsToAvoid() != null) { avoid.getPoolsToAvoid().addAll(poolsToAvoidOutput); From b40156313adce56ae80a622e885ecd9ee403c2f6 Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Sun, 26 Jan 2014 16:03:56 +0100 Subject: [PATCH 023/169] findbugs: serialized class serializable --- server/src/com/cloud/api/doc/Argument.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/server/src/com/cloud/api/doc/Argument.java b/server/src/com/cloud/api/doc/Argument.java index 29c361e23fd..c9f07e784a2 100644 --- a/server/src/com/cloud/api/doc/Argument.java +++ b/server/src/com/cloud/api/doc/Argument.java @@ -16,10 +16,15 @@ // under the License. package com.cloud.api.doc; +import java.io.Serializable; import java.util.List; -public class Argument implements Comparable{ - private String name; +public class Argument implements Comparable, Serializable{ + /** + * + */ + private static final long serialVersionUID = -5696100838854848492L; + private String name; private String description; private Boolean required; private String type; @@ -42,7 +47,7 @@ public class Argument implements Comparable{ } public String getType() { - return this.type; + return type; } public void setType(String type) { @@ -89,11 +94,12 @@ public class Argument implements Comparable{ this.sinceVersion = sinceVersion; } + @Override public int compareTo(Object anotherAgrument) throws ClassCastException { if (!(anotherAgrument instanceof Argument)) throw new ClassCastException("An Argument object expected."); Argument argument = (Argument)anotherAgrument; - return this.getName().compareToIgnoreCase(argument.getName()); + return getName().compareToIgnoreCase(argument.getName()); } public boolean hasArguments() { From f3b9a7f1232e8f39e619bf99155a4e36ff2d88f9 Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Sun, 26 Jan 2014 16:07:59 +0100 Subject: [PATCH 024/169] findbugs: retrieve the objects to remove from the collection(s) --- .../deploy/DeploymentPlanningManagerImpl.java | 43 +++++++++++-------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/server/src/com/cloud/deploy/DeploymentPlanningManagerImpl.java b/server/src/com/cloud/deploy/DeploymentPlanningManagerImpl.java index a41f5ee4100..9c3aa974eed 100644 --- a/server/src/com/cloud/deploy/DeploymentPlanningManagerImpl.java +++ b/server/src/com/cloud/deploy/DeploymentPlanningManagerImpl.java @@ -30,6 +30,8 @@ import javax.ejb.Local; import javax.inject.Inject; import javax.naming.ConfigurationException; +import org.apache.log4j.Logger; + import org.apache.cloudstack.affinity.AffinityGroupProcessor; import org.apache.cloudstack.affinity.AffinityGroupService; import org.apache.cloudstack.affinity.AffinityGroupVMMapVO; @@ -48,8 +50,16 @@ import org.apache.cloudstack.managed.context.ManagedContextTimerTask; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.utils.identity.ManagementServerNode; -import org.apache.log4j.Logger; +import com.cloud.agent.AgentManager; +import com.cloud.agent.Listener; +import com.cloud.agent.api.AgentControlAnswer; +import com.cloud.agent.api.AgentControlCommand; +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.Command; +import com.cloud.agent.api.StartupCommand; +import com.cloud.agent.api.StartupRoutingCommand; +import com.cloud.agent.manager.allocator.HostAllocator; import com.cloud.capacity.CapacityManager; import com.cloud.capacity.dao.CapacityDao; import com.cloud.configuration.Config; @@ -70,7 +80,6 @@ import com.cloud.deploy.dao.PlannerHostReservationDao; import com.cloud.exception.AffinityConflictException; import com.cloud.exception.ConnectionException; import com.cloud.exception.InsufficientServerCapacityException; -import com.cloud.exception.PermissionDeniedException; import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.host.Status; @@ -102,7 +111,6 @@ import com.cloud.utils.db.DB; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.Transaction; import com.cloud.utils.db.TransactionCallback; -import com.cloud.utils.db.TransactionCallbackNoReturn; import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.fsm.StateListener; @@ -111,19 +119,10 @@ import com.cloud.vm.ReservationContext; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachine.Event; -import com.cloud.vm.VirtualMachineProfile; import com.cloud.vm.VirtualMachine.State; +import com.cloud.vm.VirtualMachineProfile; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.VMInstanceDao; -import com.cloud.agent.AgentManager; -import com.cloud.agent.Listener; -import com.cloud.agent.api.AgentControlAnswer; -import com.cloud.agent.api.AgentControlCommand; -import com.cloud.agent.api.Answer; -import com.cloud.agent.api.Command; -import com.cloud.agent.api.StartupCommand; -import com.cloud.agent.api.StartupRoutingCommand; -import com.cloud.agent.manager.allocator.HostAllocator; @Local(value = { DeploymentPlanningManager.class }) @@ -207,7 +206,7 @@ public class DeploymentPlanningManagerImpl extends ManagerBase implements Deploy return _affinityProcessors; } public void setAffinityGroupProcessors(List affinityProcessors) { - this._affinityProcessors = affinityProcessors; + _affinityProcessors = affinityProcessors; } @Override @@ -951,7 +950,7 @@ public class DeploymentPlanningManagerImpl extends ManagerBase implements Deploy } // Cluster can be put in avoid set in following scenarios: - // 1. If storage allocators haven't put any pools in avoid set means either no pools in cluster + // 1. If storage allocators haven't put any pools in avoid set means either no pools in cluster // or pools not suitable for the allocators to handle or there is no // linkage of any suitable host to any of the pools in cluster // 2. If all 'shared' or 'local' pools are in avoid set @@ -1152,9 +1151,9 @@ public class DeploymentPlanningManagerImpl extends ManagerBase implements Deploy List suitablePools = new ArrayList(); StoragePool pool = null; if (toBeCreated.getPoolId() != null) { - pool = (StoragePool) this.dataStoreMgr.getPrimaryDataStore(toBeCreated.getPoolId()); + pool = (StoragePool) dataStoreMgr.getPrimaryDataStore(toBeCreated.getPoolId()); } else { - pool = (StoragePool) this.dataStoreMgr.getPrimaryDataStore(plan.getPoolId()); + pool = (StoragePool) dataStoreMgr.getPrimaryDataStore(plan.getPoolId()); } if (!pool.isInMaintenance()) { @@ -1167,7 +1166,7 @@ public class DeploymentPlanningManagerImpl extends ManagerBase implements Deploy && plan.getClusterId() == exstPoolClusterId) { canReusePool = true; } else if (plan.getDataCenterId() == exstPoolDcId) { - DataStore dataStore = this.dataStoreMgr.getPrimaryDataStore(pool.getId()); + DataStore dataStore = dataStoreMgr.getPrimaryDataStore(pool.getId()); if (dataStore != null && dataStore.getScope() != null && dataStore.getScope().getScopeType() == ScopeType.ZONE) { canReusePool = true; @@ -1268,7 +1267,13 @@ public class DeploymentPlanningManagerImpl extends ManagerBase implements Deploy } } if (suitableVolumeStoragePools.values() != null) { - poolsToAvoidOutput.removeAll(suitableVolumeStoragePools.values()); + HashSet toRemove = new HashSet(); + for (List lsp : suitableVolumeStoragePools.values()) { + for (StoragePool sp : lsp) { + toRemove.add(sp.getId()); + } + } + poolsToAvoidOutput.removeAll(toRemove); } if (avoid.getPoolsToAvoid() != null) { avoid.getPoolsToAvoid().addAll(poolsToAvoidOutput); From c58e509924e97dfced80981da5c27b9f3aae9b33 Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Sun, 26 Jan 2014 16:35:48 +0100 Subject: [PATCH 025/169] findbugs: null-checks --- server/src/com/cloud/acl/DomainChecker.java | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/server/src/com/cloud/acl/DomainChecker.java b/server/src/com/cloud/acl/DomainChecker.java index 7b47bae4f25..87ff27a732d 100755 --- a/server/src/com/cloud/acl/DomainChecker.java +++ b/server/src/com/cloud/acl/DomainChecker.java @@ -19,11 +19,12 @@ package com.cloud.acl; import javax.ejb.Local; import javax.inject.Inject; +import org.springframework.stereotype.Component; + import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.acl.SecurityChecker; import org.apache.cloudstack.affinity.AffinityGroup; import org.apache.cloudstack.api.BaseCmd; -import org.springframework.stereotype.Component; import com.cloud.dc.DataCenter; import com.cloud.dc.DedicatedResourceVO; @@ -145,7 +146,7 @@ public class DomainChecker extends AdapterBase implements SecurityChecker { if (caller.getId() != entity.getAccountId()) { throw new PermissionDeniedException(caller + " does not have permission to operate with resource " + entity); } - } + } } } @@ -166,7 +167,7 @@ public class DomainChecker extends AdapterBase implements SecurityChecker { //admin has all permissions if (account.getType() == Account.ACCOUNT_TYPE_ADMIN) { return true; - } + } //if account is normal user or domain admin //check if account's domain is a child of zone's domain (Note: This is made consistent with the list command for disk offering) else if (account.getType() == Account.ACCOUNT_TYPE_NORMAL || account.getType() == Account.ACCOUNT_TYPE_RESOURCE_DOMAIN_ADMIN || account.getType() == Account.ACCOUNT_TYPE_DOMAIN_ADMIN) { @@ -192,7 +193,7 @@ public class DomainChecker extends AdapterBase implements SecurityChecker { } //not found return false; - } + } @Override public boolean checkAccess(Account account, ServiceOffering so) throws PermissionDeniedException { @@ -202,7 +203,7 @@ public class DomainChecker extends AdapterBase implements SecurityChecker { //admin has all permissions if (account.getType() == Account.ACCOUNT_TYPE_ADMIN) { return true; - } + } //if account is normal user or domain admin //check if account's domain is a child of zone's domain (Note: This is made consistent with the list command for service offering) else if (account.getType() == Account.ACCOUNT_TYPE_NORMAL || account.getType() == Account.ACCOUNT_TYPE_RESOURCE_DOMAIN_ADMIN || account.getType() == Account.ACCOUNT_TYPE_DOMAIN_ADMIN) { @@ -228,7 +229,7 @@ public class DomainChecker extends AdapterBase implements SecurityChecker { } //not found return false; - } + } @Override public boolean checkAccess(Account account, DataCenter zone) throws PermissionDeniedException { @@ -238,7 +239,7 @@ public class DomainChecker extends AdapterBase implements SecurityChecker { //admin has all permissions if (account.getType() == Account.ACCOUNT_TYPE_ADMIN) { return true; - } + } //if account is normal user //check if account's domain is a child of zone's domain else if (account.getType() == Account.ACCOUNT_TYPE_NORMAL || account.getType() == Account.ACCOUNT_TYPE_PROJECT) { @@ -298,7 +299,9 @@ public class DomainChecker extends AdapterBase implements SecurityChecker { } } //didn't find in upper tree - if (zoneDomainRecord.getPath().contains(accountDomainRecord.getPath())) { + if (zoneDomainRecord != null && + accountDomainRecord != null && + zoneDomainRecord.getPath().contains(accountDomainRecord.getPath())) { return true; } } From 23a3d99fc0836839a88584d7aa6a978d99c5d126 Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Sun, 26 Jan 2014 16:38:53 +0100 Subject: [PATCH 026/169] findbugs: byte[].toString -> Arrays.toString(byte[]) --- .../com/cloud/consoleproxy/ConsoleProxyManagerImpl.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java b/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java index 8a31d73fba0..3572127f20c 100755 --- a/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java +++ b/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java @@ -17,6 +17,7 @@ package com.cloud.consoleproxy; import java.nio.charset.Charset; +import java.util.Arrays; import java.util.Date; import java.util.HashMap; import java.util.Iterator; @@ -118,10 +119,10 @@ import com.cloud.utils.component.ManagerBase; import com.cloud.utils.db.DB; import com.cloud.utils.db.GlobalLock; import com.cloud.utils.db.QueryBuilder; -import com.cloud.utils.db.TransactionCallbackNoReturn; -import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.db.SearchCriteria.Op; import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallbackNoReturn; +import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.events.SubscriptionMgr; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.net.NetUtils; @@ -502,7 +503,7 @@ VirtualMachineGuru, SystemVmLoadScanHandler, ResourceStateAdapter { byte[] details = proxy.getSessionDetails(); status = gson.fromJson(details != null ? new String(details, Charset.forName("US-ASCII")) : null, ConsoleProxyStatus.class); } catch (Throwable e) { - s_logger.warn("Unable to parse proxy session details : " + proxy.getSessionDetails()); + s_logger.warn("Unable to parse proxy session details : " + Arrays.toString(proxy.getSessionDetails())); } if (status != null && status.getConnections() != null) { @@ -1708,7 +1709,7 @@ VirtualMachineGuru, SystemVmLoadScanHandler, ResourceStateAdapter { @Inject public void setConsoleProxyAllocators(List consoleProxyAllocators) { - this._consoleProxyAllocators = consoleProxyAllocators; + _consoleProxyAllocators = consoleProxyAllocators; } } From 866a539b067663b346bd8d087c578987a86fe834 Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Sun, 26 Jan 2014 16:49:18 +0100 Subject: [PATCH 027/169] findbugs: removed fields injected in parent --- .../kvm/discoverer/LibvirtServerDiscoverer.java | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/server/src/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java b/server/src/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java index d7b5ab8318b..abd75fcfec9 100644 --- a/server/src/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java +++ b/server/src/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java @@ -28,8 +28,6 @@ import javax.naming.ConfigurationException; import org.apache.log4j.Logger; -import org.apache.cloudstack.framework.config.dao.ConfigurationDao; - import com.cloud.agent.AgentManager; import com.cloud.agent.Listener; import com.cloud.agent.api.AgentControlAnswer; @@ -41,7 +39,6 @@ import com.cloud.agent.api.StartupCommand; import com.cloud.agent.api.StartupRoutingCommand; import com.cloud.configuration.Config; import com.cloud.dc.ClusterVO; -import com.cloud.dc.dao.ClusterDao; import com.cloud.exception.AgentUnavailableException; import com.cloud.exception.DiscoveredWithErrorException; import com.cloud.exception.DiscoveryException; @@ -49,10 +46,8 @@ import com.cloud.exception.OperationTimedoutException; import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.host.Status; -import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor; import com.cloud.hypervisor.Hypervisor.HypervisorType; -import com.cloud.network.NetworkModel; import com.cloud.network.PhysicalNetworkSetupInfo; import com.cloud.resource.Discoverer; import com.cloud.resource.DiscovererBase; @@ -71,18 +66,11 @@ public abstract class LibvirtServerDiscoverer extends DiscovererBase implements private String _kvmPublicNic; private String _kvmGuestNic; @Inject - HostDao _hostDao = null; - @Inject - ClusterDao _clusterDao; - @Inject ResourceManager _resourceMgr; @Inject AgentManager _agentMgr; - @Inject - ConfigurationDao _configDao; - @Inject - NetworkModel _networkMgr; + @Override public abstract Hypervisor.HypervisorType getHypervisorType(); @Override From 0b13f8e59db1e681e1dff0baa828bb0711842e9f Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Sun, 26 Jan 2014 23:14:17 +0100 Subject: [PATCH 028/169] Findbugs: different issues solved --- .../com/cloud/api/query/vo/UserVmJoinVO.java | 23 ++++--- .../com/cloud/consoleproxy/AgentHookBase.java | 27 ++++---- .../consoleproxy/ConsoleProxyManagerImpl.java | 6 +- .../src/com/cloud/dc/DedicatedResourceVO.java | 25 ++++++-- .../cloud/network/IpAddressManagerImpl.java | 64 ++++++++++--------- .../com/cloud/network/NetworkModelImpl.java | 11 ++-- .../network/guru/ControlNetworkGuru.java | 8 +-- .../com/cloud/network/vpc/VpcManagerImpl.java | 4 +- .../cloud/template/TemplateManagerImpl.java | 4 +- 9 files changed, 97 insertions(+), 75 deletions(-) diff --git a/server/src/com/cloud/api/query/vo/UserVmJoinVO.java b/server/src/com/cloud/api/query/vo/UserVmJoinVO.java index 5aae820d810..dc6b30d50e1 100644 --- a/server/src/com/cloud/api/query/vo/UserVmJoinVO.java +++ b/server/src/com/cloud/api/query/vo/UserVmJoinVO.java @@ -36,6 +36,7 @@ import com.cloud.storage.Storage.StoragePoolType; import com.cloud.storage.Volume; import com.cloud.utils.db.Encrypt; import com.cloud.utils.db.GenericDao; +import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachine.State; @@ -48,10 +49,10 @@ public class UserVmJoinVO extends BaseViewVO implements ControlledViewEntity { private long id; @Column(name="name", updatable=false, nullable=false, length=255) - private String name = null; + private final String name = null; @Column(name="display_name", updatable=false, nullable=false, length=255) - private String displayName = null; + private final String displayName = null; @Column(name="account_id") private long accountId; @@ -60,7 +61,7 @@ public class UserVmJoinVO extends BaseViewVO implements ControlledViewEntity { private String accountUuid; @Column(name="account_name") - private String accountName = null; + private final String accountName = null; @Column(name="account_type") private short accountType; @@ -72,10 +73,10 @@ public class UserVmJoinVO extends BaseViewVO implements ControlledViewEntity { private String domainUuid; @Column(name="domain_name") - private String domainName = null; + private final String domainName = null; @Column(name="domain_path") - private String domainPath = null; + private final String domainPath = null; @Column(name="instance_group_id") private long instanceGroupId; @@ -97,7 +98,7 @@ public class UserVmJoinVO extends BaseViewVO implements ControlledViewEntity { */ @Enumerated(value=EnumType.STRING) @Column(name="state", updatable=true, nullable=false, length=32) - private State state = null; + private final State state = null; @Column(name=GenericDao.CREATED_COLUMN) private Date created; @@ -154,7 +155,7 @@ public class UserVmJoinVO extends BaseViewVO implements ControlledViewEntity { private String dataCenterUuid; @Column(name="data_center_name") - private String dataCenterName = null; + private final String dataCenterName = null; @Column(name="security_group_enabled") private boolean securityGroupEnabled; @@ -230,7 +231,7 @@ public class UserVmJoinVO extends BaseViewVO implements ControlledViewEntity { private String volume_uuid; @Column(name = "volume_device_id") - private Long volumeDeviceId = null; + private final Long volumeDeviceId = null; @Column(name = "volume_type") @Enumerated(EnumType.STRING) @@ -735,9 +736,11 @@ public class UserVmJoinVO extends BaseViewVO implements ControlledViewEntity { } public String getDetail(String name) { - assert (details != null) : "Did you forget to load the details?"; + if (details == null) { + throw new CloudRuntimeException("No details to get. Did you forget to load the details?"); + } - return details != null ? details.get(name) : null; + return details.get(name); } public String getUserData() { diff --git a/server/src/com/cloud/consoleproxy/AgentHookBase.java b/server/src/com/cloud/consoleproxy/AgentHookBase.java index aa9e4f8a8fb..660d1f2026f 100644 --- a/server/src/com/cloud/consoleproxy/AgentHookBase.java +++ b/server/src/com/cloud/consoleproxy/AgentHookBase.java @@ -22,6 +22,11 @@ import java.util.Random; import org.apache.log4j.Logger; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; + +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; + import com.cloud.agent.AgentManager; import com.cloud.agent.api.AgentControlAnswer; import com.cloud.agent.api.Answer; @@ -44,14 +49,10 @@ import com.cloud.server.ManagementServer; import com.cloud.servlet.ConsoleProxyPasswordBasedEncryptor; import com.cloud.servlet.ConsoleProxyServlet; import com.cloud.utils.Ternary; +import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VirtualMachine; import com.cloud.vm.dao.VMInstanceDao; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; - -import org.apache.cloudstack.framework.config.dao.ConfigurationDao; - /** * Utility class to manage interactions with agent-based console access * Extracted from ConsoleProxyManagerImpl so that other console proxy managers @@ -72,12 +73,12 @@ public abstract class AgentHookBase implements AgentHook { public AgentHookBase(VMInstanceDao instanceDao, HostDao hostDao, ConfigurationDao cfgDao, KeystoreManager ksMgr, AgentManager agentMgr, ManagementServer ms) { - this._instanceDao = instanceDao; - this._hostDao = hostDao; - this._agentMgr = agentMgr; - this._configDao = cfgDao; - this._ksMgr = ksMgr; - this._ms = ms; + _instanceDao = instanceDao; + _hostDao = hostDao; + _agentMgr = agentMgr; + _configDao = cfgDao; + _ksMgr = ksMgr; + _ms = ms; } @Override @@ -204,7 +205,9 @@ public abstract class AgentHookBase implements AgentHook { assert (ksBits != null); if (ksBits == null) { - s_logger.error("Could not find and construct a valid SSL certificate"); + String msg = "Could not find and construct a valid SSL certificate"; + s_logger.error(msg); + throw new CloudRuntimeException(msg); } cmd = new StartConsoleProxyAgentHttpHandlerCommand(ksBits, storePassword); cmd.setEncryptorPassword(getEncryptorPassword()); diff --git a/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java b/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java index 3572127f20c..6a607ddd9da 100755 --- a/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java +++ b/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java @@ -971,11 +971,7 @@ VirtualMachineGuru, SystemVmLoadScanHandler, ResourceStateAdapter { } } else { if (s_logger.isDebugEnabled()) { - if (template == null) { - s_logger.debug("Zone host is ready, but console proxy template is null"); - } else { - s_logger.debug("Zone host is ready, but console proxy template: " + template.getId() + " is not ready on secondary storage."); - } + s_logger.debug("Zone host is ready, but console proxy template: " + template.getId() + " is not ready on secondary storage."); } } } diff --git a/server/src/com/cloud/dc/DedicatedResourceVO.java b/server/src/com/cloud/dc/DedicatedResourceVO.java index 9caa6b04487..1f66ecf9e42 100644 --- a/server/src/com/cloud/dc/DedicatedResourceVO.java +++ b/server/src/com/cloud/dc/DedicatedResourceVO.java @@ -20,7 +20,6 @@ import java.util.UUID; import javax.persistence.Column; import javax.persistence.Entity; - import javax.persistence.GeneratedValue; import javax.persistence.GenerationType; import javax.persistence.Id; @@ -31,6 +30,11 @@ import javax.persistence.Table; @Table(name="dedicated_resources") public class DedicatedResourceVO implements DedicatedResources{ + /** + * + */ + private static final long serialVersionUID = -6659510127145101917L; + @Id @GeneratedValue(strategy = GenerationType.IDENTITY) @Column(name="id") @@ -61,7 +65,7 @@ public class DedicatedResourceVO implements DedicatedResources{ private long affinityGroupId; public DedicatedResourceVO() { - this.uuid = UUID.randomUUID().toString(); + uuid = UUID.randomUUID().toString(); } public DedicatedResourceVO(Long dataCenterId, Long podId, Long clusterId, Long hostId, Long domainId, @@ -72,14 +76,16 @@ public class DedicatedResourceVO implements DedicatedResources{ this.hostId = hostId; this.domainId = domainId; this.accountId = accountId; - this.uuid = UUID.randomUUID().toString(); + uuid = UUID.randomUUID().toString(); this.affinityGroupId = affinityGroupId; } + @Override public long getId() { return id; } + @Override public Long getDataCenterId() { return dataCenterId; } @@ -88,6 +94,7 @@ public class DedicatedResourceVO implements DedicatedResources{ this.dataCenterId = dataCenterId; } + @Override public Long getPodId() { return podId; } @@ -96,6 +103,7 @@ public class DedicatedResourceVO implements DedicatedResources{ this.podId = podId; } + @Override public Long getClusterId() { return clusterId; } @@ -104,6 +112,7 @@ public class DedicatedResourceVO implements DedicatedResources{ this.clusterId = clusterId; } + @Override public Long getHostId() { return hostId; } @@ -113,9 +122,10 @@ public class DedicatedResourceVO implements DedicatedResources{ } public DedicatedResourceVO(long dedicatedResourceId) { - this.id = dedicatedResourceId; + id = dedicatedResourceId; } + @Override public Long getDomainId() { return domainId; } @@ -124,6 +134,7 @@ public class DedicatedResourceVO implements DedicatedResources{ this.domainId = domainId; } + @Override public Long getAccountId() { return accountId; } @@ -132,14 +143,16 @@ public class DedicatedResourceVO implements DedicatedResources{ this.accountId = accountId; } + @Override public String getUuid() { - return this.uuid; + return uuid; } public void setUuid(String uuid) { this.uuid = uuid; } + @Override public long getAffinityGroupId() { return affinityGroupId; } @@ -147,7 +160,7 @@ public class DedicatedResourceVO implements DedicatedResources{ @Override public boolean equals(Object obj) { if (obj instanceof DedicatedResourceVO) { - return ((DedicatedResourceVO) obj).getId() == this.getId(); + return ((DedicatedResourceVO) obj).getId() == getId(); } else { return false; } diff --git a/server/src/com/cloud/network/IpAddressManagerImpl.java b/server/src/com/cloud/network/IpAddressManagerImpl.java index f4bff584213..1ccd0d604a2 100644 --- a/server/src/com/cloud/network/IpAddressManagerImpl.java +++ b/server/src/com/cloud/network/IpAddressManagerImpl.java @@ -17,6 +17,7 @@ package com.cloud.network; import java.util.ArrayList; +import java.util.Arrays; import java.util.Date; import java.util.HashMap; import java.util.HashSet; @@ -29,6 +30,7 @@ import java.util.UUID; import javax.inject.Inject; import org.apache.log4j.Logger; + import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.acl.SecurityChecker.AccessType; import org.apache.cloudstack.context.CallContext; @@ -138,14 +140,14 @@ import com.cloud.utils.db.DB; import com.cloud.utils.db.EntityManager; import com.cloud.utils.db.Filter; import com.cloud.utils.db.GlobalLock; -import com.cloud.utils.db.TransactionCallback; -import com.cloud.utils.db.TransactionCallbackNoReturn; -import com.cloud.utils.db.TransactionCallbackWithException; import com.cloud.utils.db.JoinBuilder.JoinType; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.SearchCriteria.Op; import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallback; +import com.cloud.utils.db.TransactionCallbackNoReturn; +import com.cloud.utils.db.TransactionCallbackWithException; import com.cloud.utils.db.TransactionCallbackWithExceptionNoReturn; import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; @@ -578,8 +580,9 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage IPAddressVO ip = markIpAsUnavailable(addrId); - assert (ip != null) : "Unable to mark the ip address id=" + addrId + " as unavailable."; if (ip == null) { + String msg = "Unable to mark the ip address id=" + addrId + " as unavailable."; + s_logger.error(msg); return true; } @@ -692,10 +695,10 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage if (dedicatedVlanDbIds != null && !dedicatedVlanDbIds.isEmpty()) { fetchFromDedicatedRange = true; sc.setParameters("vlanId", dedicatedVlanDbIds.toArray()); - errorMessage.append(", vlanId id=" + dedicatedVlanDbIds.toArray()); + errorMessage.append(", vlanId id=" + Arrays.toString(dedicatedVlanDbIds.toArray())); } else if (nonDedicatedVlanDbIds != null && !nonDedicatedVlanDbIds.isEmpty()) { sc.setParameters("vlanId", nonDedicatedVlanDbIds.toArray()); - errorMessage.append(", vlanId id=" + nonDedicatedVlanDbIds.toArray()); + errorMessage.append(", vlanId id=" + Arrays.toString(nonDedicatedVlanDbIds.toArray())); } else { if (podId != null) { InsufficientAddressCapacityException ex = new InsufficientAddressCapacityException("Insufficient address capacity", Pod.class, podId); @@ -735,7 +738,7 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage if (useSystemIps && nonDedicatedVlanDbIds != null && !nonDedicatedVlanDbIds.isEmpty()) { fetchFromDedicatedRange = false; sc.setParameters("vlanId", nonDedicatedVlanDbIds.toArray()); - errorMessage.append(", vlanId id=" + nonDedicatedVlanDbIds.toArray()); + errorMessage.append(", vlanId id=" + Arrays.toString(nonDedicatedVlanDbIds.toArray())); addrs = _ipAddressDao.lockRows(sc, filter, true); } } @@ -868,6 +871,11 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage public PublicIp assignDedicateIpAddress(Account owner, final Long guestNtwkId, final Long vpcId, final long dcId, final boolean isSourceNat) throws ConcurrentOperationException, InsufficientAddressCapacityException { + if (owner == null) { + s_logger.error("No account to assign an ip to."); + return null; + } + final long ownerId = owner.getId(); PublicIp ip = null; @@ -899,13 +907,11 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage return ip; } finally { - if (owner != null) { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Releasing lock account " + ownerId); - } - - _accountDao.releaseFromLockTable(ownerId); + if (s_logger.isDebugEnabled()) { + s_logger.debug("Releasing lock account " + ownerId); } + _accountDao.releaseFromLockTable(ownerId); + if (ip == null) { s_logger.error("Unable to get source nat ip address for account " + ownerId); } @@ -1253,7 +1259,7 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage s_logger.debug("Associating ip " + ipToAssoc + " to network " + network); - IPAddressVO ip = _ipAddressDao.findById(ipId); + IPAddressVO ip = ipToAssoc; //_ipAddressDao.findById(ipId); //update ip address with networkId ip.setAssociatedWithNetworkId(networkId); ip.setSourceNat(isSourceNat); @@ -1270,18 +1276,16 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage return ip; } finally { if (!success && releaseOnFailure) { - if (ip != null) { - try { - s_logger.warn("Failed to associate ip address, so releasing ip from the database " + ip); - _ipAddressDao.markAsUnavailable(ip.getId()); - if (!applyIpAssociations(network, true)) { - // if fail to apply ip assciations again, unassign ip address without updating resource - // count and generating usage event as there is no need to keep it in the db - _ipAddressDao.unassignIpAddress(ip.getId()); - } - } catch (Exception e) { - s_logger.warn("Unable to disassociate ip address for recovery", e); + try { + s_logger.warn("Failed to associate ip address, so releasing ip from the database " + ip); + _ipAddressDao.markAsUnavailable(ip.getId()); + if (!applyIpAssociations(network, true)) { + // if fail to apply ip assciations again, unassign ip address without updating resource + // count and generating usage event as there is no need to keep it in the db + _ipAddressDao.unassignIpAddress(ip.getId()); } + } catch (Exception e) { + s_logger.warn("Unable to disassociate ip address for recovery", e); } } } @@ -1362,7 +1366,7 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage PublicIp publicIp = PublicIp.createFromAddrAndVlan(ipToAssoc, _vlanDao.findById(ipToAssoc.getVlanId())); ipList.add(publicIp); Map> ipToServices = _networkModel.getIpToServices(ipList, false, true); - if (ipToServices != null & !ipToServices.isEmpty()) { + if (ipToServices != null && !ipToServices.isEmpty()) { Set services = ipToServices.get(publicIp); if (services != null && !services.isEmpty()) { throw new InvalidParameterValueException("IP " + ipToAssoc + " has services and rules associated in the network " + networkId); @@ -1403,7 +1407,7 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage PublicIp publicIp = PublicIp.createFromAddrAndVlan(ip, _vlanDao.findById(ip.getVlanId())); ipList.add(publicIp); Map> ipToServices = _networkModel.getIpToServices(ipList, false, true); - if (ipToServices != null & !ipToServices.isEmpty()) { + if (ipToServices != null && !ipToServices.isEmpty()) { Set ipServices = ipToServices.get(publicIp); if (ipServices != null && !ipServices.isEmpty()) { return false; @@ -1732,14 +1736,14 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage @Override public boolean applyStaticNats(List staticNats, boolean continueOnError, boolean forRevoke) throws ResourceUnavailableException { - Network network = _networksDao.findById(staticNats.get(0).getNetworkId()); - boolean success = true; - if (staticNats == null || staticNats.size() == 0) { s_logger.debug("There are no static nat rules for the network elements"); return true; } + Network network = _networksDao.findById(staticNats.get(0).getNetworkId()); + boolean success = true; + // get the list of public ip's owned by the network List userIps = _ipAddressDao.listByAssociatedNetwork(network.getId(), null); List publicIps = new ArrayList(); diff --git a/server/src/com/cloud/network/NetworkModelImpl.java b/server/src/com/cloud/network/NetworkModelImpl.java index 27b200633e5..7281beb79b2 100755 --- a/server/src/com/cloud/network/NetworkModelImpl.java +++ b/server/src/com/cloud/network/NetworkModelImpl.java @@ -1024,14 +1024,14 @@ public class NetworkModelImpl extends ManagerBase implements NetworkModel { List map = _ntwkOfferingSrvcDao.listByNetworkOfferingId(networkOfferingId); for (NetworkOfferingServiceMapVO instance : map) { - String service = instance.getService(); + Service service = Network.Service.getService(instance.getService()); Set providers; providers = serviceProviderMap.get(service); if (providers == null) { providers = new HashSet(); } providers.add(Provider.getProvider(instance.getProvider())); - serviceProviderMap.put(Service.getService(service), providers); + serviceProviderMap.put(service, providers); } return serviceProviderMap; @@ -1526,12 +1526,15 @@ public class NetworkModelImpl extends ManagerBase implements NetworkModel { @Override public void checkNetworkPermissions(Account owner, Network network) { + if (network == null) { + throw new CloudRuntimeException("no network to check permissions for."); + } // Perform account permission check if (network.getGuestType() != Network.GuestType.Shared || (network.getGuestType() == Network.GuestType.Shared && network.getAclType() == ACLType.Account)) { AccountVO networkOwner = _accountDao.findById(network.getAccountId()); if(networkOwner == null) - throw new PermissionDeniedException("Unable to use network with id= " + ((network != null)? ((NetworkVO)network).getUuid() : "") + ", network does not have an owner"); + throw new PermissionDeniedException("Unable to use network with id= " + ((NetworkVO)network).getUuid() + ", network does not have an owner"); if(owner.getType() != Account.ACCOUNT_TYPE_PROJECT && networkOwner.getType() == Account.ACCOUNT_TYPE_PROJECT){ if(!_projectAccountDao.canAccessProjectAccount(owner.getAccountId(), network.getAccountId())){ throw new PermissionDeniedException("Unable to use network with id= " + ((network != null)? ((NetworkVO)network).getUuid() : "") + ", permission denied"); @@ -1545,7 +1548,7 @@ public class NetworkModelImpl extends ManagerBase implements NetworkModel { } else { if (!isNetworkAvailableInDomain(network.getId(), owner.getDomainId())) { - throw new PermissionDeniedException("Shared network id=" + ((network != null)? ((NetworkVO)network).getUuid() : "") + " is not available in domain id=" + owner.getDomainId()); + throw new PermissionDeniedException("Shared network id=" + ((NetworkVO)network).getUuid() + " is not available in domain id=" + owner.getDomainId()); } } } diff --git a/server/src/com/cloud/network/guru/ControlNetworkGuru.java b/server/src/com/cloud/network/guru/ControlNetworkGuru.java index 26caa61e09a..a8274d3f4ea 100755 --- a/server/src/com/cloud/network/guru/ControlNetworkGuru.java +++ b/server/src/com/cloud/network/guru/ControlNetworkGuru.java @@ -134,7 +134,7 @@ public class ControlNetworkGuru extends PodBasedNetworkGuru implements NetworkGu assert nic.getTrafficType() == TrafficType.Control; // we have to get management/private ip for the control nic for vmware/hyperv due ssh issues. - HypervisorType hType = dest.getHost().getHypervisorType(); + HypervisorType hType = dest.getHost().getHypervisorType(); if ( ( (hType == HypervisorType.VMware) || (hType == HypervisorType.Hyperv) )&& isRouterVm(vm)) { if(dest.getDataCenter().getNetworkType() != NetworkType.Basic) { super.reserve(nic, config, vm, dest, context); @@ -168,7 +168,7 @@ public class ControlNetworkGuru extends PodBasedNetworkGuru implements NetworkGu @Override public boolean release(NicProfile nic, VirtualMachineProfile vm, String reservationId) { assert nic.getTrafficType() == TrafficType.Control; - HypervisorType hType = vm.getHypervisorType(); + HypervisorType hType = vm.getHypervisorType(); if ( ( (hType == HypervisorType.VMware) || (hType == HypervisorType.Hyperv) )&& isRouterVm(vm)) { long dcId = vm.getVirtualMachine().getDataCenterId(); DataCenterVO dcVo = _dcDao.findById(dcId); @@ -218,12 +218,12 @@ public class ControlNetworkGuru extends PodBasedNetworkGuru implements NetworkGu Map dbParams = _configDao.getConfiguration(params); - _cidr = dbParams.get(Config.ControlCidr); + _cidr = dbParams.get(Config.ControlCidr.toString()); if (_cidr == null) { _cidr = "169.254.0.0/16"; } - _gateway = dbParams.get(Config.ControlGateway); + _gateway = dbParams.get(Config.ControlGateway.toString()); if (_gateway == null) { _gateway = NetUtils.getLinkLocalGateway(); } diff --git a/server/src/com/cloud/network/vpc/VpcManagerImpl.java b/server/src/com/cloud/network/vpc/VpcManagerImpl.java index b9dd729a6b1..3ae194f97d9 100644 --- a/server/src/com/cloud/network/vpc/VpcManagerImpl.java +++ b/server/src/com/cloud/network/vpc/VpcManagerImpl.java @@ -432,14 +432,14 @@ public class VpcManagerImpl extends ManagerBase implements VpcManager, VpcProvis List map = _vpcOffSvcMapDao.listByVpcOffId(vpcOffId); for (VpcOfferingServiceMapVO instance : map) { - String service = instance.getService(); + Service service = Service.getService(instance.getService()); Set providers; providers = serviceProviderMap.get(service); if (providers == null) { providers = new HashSet(); } providers.add(Provider.getProvider(instance.getProvider())); - serviceProviderMap.put(Service.getService(service), providers); + serviceProviderMap.put(service, providers); } return serviceProviderMap; diff --git a/server/src/com/cloud/template/TemplateManagerImpl.java b/server/src/com/cloud/template/TemplateManagerImpl.java index 681be4af65b..33ba667b988 100755 --- a/server/src/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/com/cloud/template/TemplateManagerImpl.java @@ -425,7 +425,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, if (isISO) { desc = Upload.Type.ISO.toString(); } - eventId = eventId == null ? 0 : eventId; + eventId = (eventId == null ? 0 : eventId); if (!_accountMgr.isRootAdmin(caller.getType()) && _disableExtraction) { throw new PermissionDeniedException("Extraction has been disabled by admin"); @@ -1801,7 +1801,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } // Don't allow to modify system template - if (id == Long.valueOf(1)) { + if (Long.valueOf(1).equals(id)) { InvalidParameterValueException ex = new InvalidParameterValueException("Unable to update template/iso of specified id"); ex.addProxyObject(String.valueOf(id), "templateId"); throw ex; From 20127e09dc0e341a2f790a8a52cded4c5f1f0cc1 Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Sun, 26 Jan 2014 23:49:56 +0100 Subject: [PATCH 029/169] findbugs: assign result of string manipulations --- .../network/security/SecurityGroupManagerImpl.java | 7 ++++--- server/src/com/cloud/test/IPRangeConfig.java | 13 ++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/server/src/com/cloud/network/security/SecurityGroupManagerImpl.java b/server/src/com/cloud/network/security/SecurityGroupManagerImpl.java index c7b6e1eb3f7..e48bb344ecd 100755 --- a/server/src/com/cloud/network/security/SecurityGroupManagerImpl.java +++ b/server/src/com/cloud/network/security/SecurityGroupManagerImpl.java @@ -42,6 +42,7 @@ import javax.naming.ConfigurationException; import org.apache.commons.codec.digest.DigestUtils; import org.apache.log4j.Logger; + import org.apache.cloudstack.api.command.user.securitygroup.AuthorizeSecurityGroupEgressCmd; import org.apache.cloudstack.api.command.user.securitygroup.AuthorizeSecurityGroupIngressCmd; import org.apache.cloudstack.api.command.user.securitygroup.CreateSecurityGroupCmd; @@ -376,7 +377,7 @@ public class SecurityGroupManagerImpl extends ManagerBase implements SecurityGro protected String generateRulesetSignature(Map> ingress, Map> egress) { String ruleset = ingress.toString(); - ruleset.concat(egress.toString()); + ruleset = ruleset.concat(egress.toString()); return DigestUtils.md5Hex(ruleset); } @@ -401,7 +402,7 @@ public class SecurityGroupManagerImpl extends ManagerBase implements SecurityGro if (s_logger.isTraceEnabled()) { s_logger.trace("Security Group Mgr: scheduling ruleset updates for " + affectedVms.size() + " vms"); } - boolean locked = _workLock.lock(_globalWorkLockTimeout); + boolean locked = _workLock.lock(_globalWorkLockTimeout); if (!locked) { s_logger.warn("Security Group Mgr: failed to acquire global work lock"); return; @@ -885,7 +886,7 @@ public class SecurityGroupManagerImpl extends ManagerBase implements SecurityGro _serverId = ManagementServerNode.getManagementServerId(); - s_logger.info("SecurityGroupManager: num worker threads=" + _numWorkerThreads + + s_logger.info("SecurityGroupManager: num worker threads=" + _numWorkerThreads + ", time between cleanups=" + _timeBetweenCleanups + " global lock timeout=" + _globalWorkLockTimeout); createThreadPools(); diff --git a/server/src/com/cloud/test/IPRangeConfig.java b/server/src/com/cloud/test/IPRangeConfig.java index 23ca1bba3ee..0669c55d377 100755 --- a/server/src/com/cloud/test/IPRangeConfig.java +++ b/server/src/com/cloud/test/IPRangeConfig.java @@ -28,7 +28,6 @@ import java.util.Vector; import com.cloud.utils.component.ComponentContext; import com.cloud.utils.db.DB; -import com.cloud.utils.db.Transaction; import com.cloud.utils.db.TransactionLegacy; import com.cloud.utils.net.NetUtils; @@ -72,7 +71,7 @@ public class IPRangeConfig { long zoneId = PodZoneConfig.getZoneId(zone); result = changeRange(op, "public", -1, zoneId, startIP, endIP, null, -1); - result.replaceAll("
", "/n"); + result = result.replaceAll("
", "/n"); System.out.println(result); } else if (type.equals("private")) { if (args.length != 5 && args.length != 6) { @@ -94,7 +93,7 @@ public class IPRangeConfig { long podId = PodZoneConfig.getPodId(pod, zone); long zoneId = PodZoneConfig.getZoneId(zone); result = changeRange(op, "private", podId, zoneId, startIP, endIP, null, -1); - result.replaceAll("
", "/n"); + result = result.replaceAll("
", "/n"); System.out.println(result); } else { printError(usage()); @@ -515,7 +514,7 @@ public class IPRangeConfig { } catch (Exception ex) { System.out.print(ex.getMessage()); ex.printStackTrace(); - } + } return problemIPs; } @@ -574,7 +573,7 @@ public class IPRangeConfig { // return DatabaseConfig.getDatabaseValueString("SELECT * FROM `cloud`.`configuration` WHERE name = \"guest.ip.network\"", "value", // "Unable to start DB connection to read guest IP network. Please contact Cloud Support."); // } -// +// // public static String getGuestNetmask() { // return DatabaseConfig.getDatabaseValueString("SELECT * FROM `cloud`.`configuration` WHERE name = \"guest.netmask\"", "value", // "Unable to start DB connection to read guest netmask. Please contact Cloud Support."); @@ -583,10 +582,10 @@ public class IPRangeConfig { // public static String getGuestSubnet() { // String guestIpNetwork = getGuestIpNetwork(); // String guestNetmask = getGuestNetmask(); -// +// // if (guestIpNetwork == null || guestIpNetwork.isEmpty()) printError("Please enter a valid guest IP network address."); // if (guestNetmask == null || guestNetmask.isEmpty()) printError("Please enter a valid guest IP network netmask"); -// +// // return NetUtils.getSubNet(guestIpNetwork, guestNetmask); // } From 791b7f8f7fe8f7d342e46fbb61a3f421e87fca59 Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Mon, 27 Jan 2014 00:23:57 +0100 Subject: [PATCH 030/169] findbugs: nullpointer guards and wrappertype comparisons --- .../ResourceLimitManagerImpl.java | 18 ++++---- .../cloud/storage/VolumeApiServiceImpl.java | 42 +++++++++---------- .../src/com/cloud/vm/UserVmManagerImpl.java | 12 +++--- 3 files changed, 35 insertions(+), 37 deletions(-) diff --git a/server/src/com/cloud/resourcelimit/ResourceLimitManagerImpl.java b/server/src/com/cloud/resourcelimit/ResourceLimitManagerImpl.java index fc9099dfe9c..0b86c0c6ce0 100755 --- a/server/src/com/cloud/resourcelimit/ResourceLimitManagerImpl.java +++ b/server/src/com/cloud/resourcelimit/ResourceLimitManagerImpl.java @@ -17,6 +17,7 @@ package com.cloud.resourcelimit; import java.util.ArrayList; +import java.util.Arrays; import java.util.EnumMap; import java.util.List; import java.util.Map; @@ -29,6 +30,9 @@ import javax.ejb.Local; import javax.inject.Inject; import javax.naming.ConfigurationException; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; + import org.apache.cloudstack.acl.SecurityChecker.AccessType; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; @@ -38,8 +42,6 @@ import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; -import org.apache.log4j.Logger; -import org.springframework.stereotype.Component; import com.cloud.alert.AlertManager; import com.cloud.configuration.Config; @@ -72,12 +74,10 @@ import com.cloud.service.ServiceOfferingVO; import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.storage.DataStoreRole; import com.cloud.storage.SnapshotVO; -import com.cloud.storage.VMTemplateHostVO; import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.dao.SnapshotDao; import com.cloud.storage.dao.VMTemplateDao; -import com.cloud.storage.dao.VMTemplateHostDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.storage.dao.VolumeDaoImpl.SumCount; import com.cloud.user.Account; @@ -95,12 +95,12 @@ import com.cloud.utils.db.GenericSearchBuilder; import com.cloud.utils.db.JoinBuilder; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; -import com.cloud.utils.db.TransactionCallback; -import com.cloud.utils.db.TransactionCallbackWithExceptionNoReturn; -import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.db.SearchCriteria.Func; import com.cloud.utils.db.SearchCriteria.Op; import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallback; +import com.cloud.utils.db.TransactionCallbackWithExceptionNoReturn; +import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.UserVmVO; import com.cloud.vm.VirtualMachine; @@ -108,8 +108,6 @@ import com.cloud.vm.VirtualMachine.State; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.VMInstanceDao; -import java.util.Arrays; - @Component @Local(value = { ResourceLimitService.class }) public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLimitService { @@ -871,7 +869,7 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim } _resourceCountDao.setResourceCount(accountId, ResourceOwnerType.Account, type, (newCount == null) ? 0 : newCount.longValue()); - if (oldCount != newCount) { + if (!Long.valueOf(oldCount).equals(newCount)) { s_logger.info("Discrepency in the resource count " + "(original count=" + oldCount + " correct count = " + newCount + ") for type " + type + " for account ID " + accountId + " is fixed during resource count recalculation."); } diff --git a/server/src/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/com/cloud/storage/VolumeApiServiceImpl.java index f46f1e9e0d0..bc319b95064 100644 --- a/server/src/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/com/cloud/storage/VolumeApiServiceImpl.java @@ -445,29 +445,29 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic return Transaction.execute(new TransactionCallback() { @Override public VolumeVO doInTransaction(TransactionStatus status) { - VolumeVO volume = new VolumeVO(volumeName, zoneId, -1, -1, -1, new Long(-1), null, null, 0, Volume.Type.DATADISK); - volume.setPoolId(null); - volume.setDataCenterId(zoneId); - volume.setPodId(null); - volume.setAccountId(owner.getAccountId()); - volume.setDomainId(owner.getDomainId()); - long diskOfferingId = _diskOfferingDao.findByUniqueName("Cloud.com-Custom").getId(); - volume.setDiskOfferingId(diskOfferingId); - // volume.setSize(size); - volume.setInstanceId(null); - volume.setUpdated(new Date()); - volume.setDomainId((owner == null) ? Domain.ROOT_DOMAIN : owner.getDomainId()); - volume.setFormat(ImageFormat.valueOf(format)); - volume = _volsDao.persist(volume); - CallContext.current().setEventDetails("Volume Id: " + volume.getId()); + VolumeVO volume = new VolumeVO(volumeName, zoneId, -1, -1, -1, new Long(-1), null, null, 0, Volume.Type.DATADISK); + volume.setPoolId(null); + volume.setDataCenterId(zoneId); + volume.setPodId(null); + volume.setAccountId((owner == null) ? null : owner.getAccountId()); + volume.setDomainId((owner == null) ? Domain.ROOT_DOMAIN : owner.getDomainId()); + long diskOfferingId = _diskOfferingDao.findByUniqueName("Cloud.com-Custom").getId(); + volume.setDiskOfferingId(diskOfferingId); + // volume.setSize(size); + volume.setInstanceId(null); + volume.setUpdated(new Date()); + volume.setDomainId((owner == null) ? Domain.ROOT_DOMAIN : owner.getDomainId()); + volume.setFormat(ImageFormat.valueOf(format)); + volume = _volsDao.persist(volume); + CallContext.current().setEventDetails("Volume Id: " + volume.getId()); - // Increment resource count during allocation; if actual creation fails, - // decrement it - _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.volume); - _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.secondary_storage, UriUtils.getRemoteSize(url)); + // Increment resource count during allocation; if actual creation fails, + // decrement it + _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.volume); + _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.secondary_storage, UriUtils.getRemoteSize(url)); - return volume; - } + return volume; + } }); } diff --git a/server/src/com/cloud/vm/UserVmManagerImpl.java b/server/src/com/cloud/vm/UserVmManagerImpl.java index ea38d12ee51..363c663250f 100755 --- a/server/src/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/com/cloud/vm/UserVmManagerImpl.java @@ -34,6 +34,9 @@ import javax.ejb.Local; import javax.inject.Inject; import javax.naming.ConfigurationException; +import org.apache.commons.codec.binary.Base64; +import org.apache.log4j.Logger; + import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.acl.SecurityChecker.AccessType; import org.apache.cloudstack.affinity.AffinityGroupService; @@ -79,8 +82,6 @@ import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.to.TemplateObjectTO; -import org.apache.commons.codec.binary.Base64; -import org.apache.log4j.Logger; import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; @@ -455,7 +456,6 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir protected boolean _dailyOrHourly = false; private int capacityReleaseInterval; - protected String _name; protected String _instance; protected String _zone; protected boolean _instanceNameFlag; @@ -1315,7 +1315,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir _accountMgr.checkAccess(caller, null, true, vmInstance); //Check if its a scale "up" - ServiceOfferingVO newServiceOffering = (ServiceOfferingVO) _offeringDao.findById(newServiceOfferingId); + ServiceOfferingVO newServiceOffering = _offeringDao.findById(newServiceOfferingId); if (newServiceOffering.isDynamic()) { newServiceOffering.setDynamicFlag(true); validateCustomParameters(newServiceOffering, customParameters); @@ -4287,13 +4287,13 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir //if hosts are dedicated to different account/domains, raise an alert if (srcExplDedicated && destExplDedicated) { - if((accountOfDedicatedHost(srcHost) != null) && (accountOfDedicatedHost(srcHost)!= accountOfDedicatedHost(destHost))) { + if (!((accountOfDedicatedHost(srcHost) == null) || (accountOfDedicatedHost(srcHost).equals(accountOfDedicatedHost(destHost))))) { String msg = "VM is being migrated from host " + srcHost.getName() + " explicitly dedicated to account " + accountOfDedicatedHost(srcHost) + " to host " + destHost.getName() + " explicitly dedicated to account " + accountOfDedicatedHost(destHost); _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_USERVM, vm.getDataCenterId(), vm.getPodIdToDeployIn(), msg, msg); s_logger.warn(msg); } - if((domainOfDedicatedHost(srcHost) != null) && (domainOfDedicatedHost(srcHost)!= domainOfDedicatedHost(destHost))) { + if (!((domainOfDedicatedHost(srcHost) == null) || (domainOfDedicatedHost(srcHost).equals(domainOfDedicatedHost(destHost))))) { String msg = "VM is being migrated from host " + srcHost.getName() + " explicitly dedicated to domain " + domainOfDedicatedHost(srcHost) + " to host " + destHost.getName() + " explicitly dedicated to domain " + domainOfDedicatedHost(destHost); _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_USERVM, vm.getDataCenterId(), vm.getPodIdToDeployIn(), msg, msg); From 9772693dca9a91a438078de11b1be1e6044aa6f3 Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Mon, 27 Jan 2014 00:54:23 +0100 Subject: [PATCH 031/169] findbugs: throw exceptions when found --- .../cloud/projects/ProjectManagerImpl.java | 46 ++++++++++--------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/server/src/com/cloud/projects/ProjectManagerImpl.java b/server/src/com/cloud/projects/ProjectManagerImpl.java index 22e2020ad22..1b18453382c 100755 --- a/server/src/com/cloud/projects/ProjectManagerImpl.java +++ b/server/src/com/cloud/projects/ProjectManagerImpl.java @@ -5,7 +5,7 @@ // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, @@ -39,12 +39,17 @@ import javax.mail.URLName; import javax.mail.internet.InternetAddress; import javax.naming.ConfigurationException; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; + +import com.sun.mail.smtp.SMTPMessage; +import com.sun.mail.smtp.SMTPSSLTransport; +import com.sun.mail.smtp.SMTPTransport; + import org.apache.cloudstack.acl.SecurityChecker.AccessType; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.managed.context.ManagedContextRunnable; -import org.apache.log4j.Logger; -import org.springframework.stereotype.Component; import com.cloud.api.ApiDBUtils; import com.cloud.api.query.dao.ProjectAccountJoinDao; @@ -77,7 +82,6 @@ import com.cloud.user.User; import com.cloud.user.dao.AccountDao; import com.cloud.utils.DateUtil; import com.cloud.utils.NumbersUtil; -import com.cloud.utils.component.Manager; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.concurrency.NamedThreadFactory; import com.cloud.utils.db.DB; @@ -87,9 +91,6 @@ import com.cloud.utils.db.TransactionCallbackNoReturn; import com.cloud.utils.db.TransactionCallbackWithExceptionNoReturn; import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; -import com.sun.mail.smtp.SMTPMessage; -import com.sun.mail.smtp.SMTPSSLTransport; -import com.sun.mail.smtp.SMTPTransport; @Component @Local(value = { ProjectService.class, ProjectManager.class }) @@ -108,7 +109,7 @@ public class ProjectManagerImpl extends ManagerBase implements ProjectManager { @Inject DomainManager _domainMgr; @Inject - ConfigurationManager _configMgr; + ConfigurationManager _configMgr; @Inject ResourceLimitService _resourceLimitMgr; @Inject @@ -257,7 +258,7 @@ public class ProjectManagerImpl extends ManagerBase implements ProjectManager { @Override - @ActionEvent(eventType = EventTypes.EVENT_PROJECT_DELETE, eventDescription = "deleting project", async = true) + @ActionEvent(eventType = EventTypes.EVENT_PROJECT_DELETE, eventDescription = "deleting project", async = true) public boolean deleteProject(long projectId) { CallContext ctx = CallContext.current(); @@ -269,7 +270,7 @@ public class ProjectManagerImpl extends ManagerBase implements ProjectManager { _accountMgr.checkAccess(ctx.getCallingAccount(),AccessType.ModifyProject, true, _accountMgr.getAccount(project.getProjectAccountId())); - return deleteProject(ctx.getCallingAccount(), ctx.getCallingUserId(), project); + return deleteProject(ctx.getCallingAccount(), ctx.getCallingUserId(), project); } @DB @@ -309,7 +310,7 @@ public class ProjectManagerImpl extends ManagerBase implements ProjectManager { @DB private boolean cleanupProject(final Project project, AccountVO caller, Long callerUserId) { - boolean result=true; + boolean result=true; //Delete project's account AccountVO account = _accountDao.findById(project.getProjectAccountId()); s_logger.debug("Deleting projects " + project + " internal account id=" + account.getId() + " as a part of project cleanup..."); @@ -527,7 +528,7 @@ public class ProjectManagerImpl extends ManagerBase implements ProjectManager { if (project == null) { InvalidParameterValueException ex = new InvalidParameterValueException("Unable to find project with specified id"); - ex.addProxyObject(String.valueOf(projectId), "projectId"); + ex.addProxyObject(String.valueOf(projectId), "projectId"); throw ex; } @@ -586,7 +587,7 @@ public class ProjectManagerImpl extends ManagerBase implements ProjectManager { } else { s_logger.warn("Failed to generate invitation for account " + account.getAccountName() + " to project id=" + project); return false; - } + } } if (email != null) { @@ -597,7 +598,7 @@ public class ProjectManagerImpl extends ManagerBase implements ProjectManager { } else { s_logger.warn("Failed to generate invitation for email " + email + " to project id=" + project); return false; - } + } } return false; @@ -613,7 +614,7 @@ public class ProjectManagerImpl extends ManagerBase implements ProjectManager { if (project == null) { InvalidParameterValueException ex = new InvalidParameterValueException("Unable to find project with specified id"); - ex.addProxyObject(String.valueOf(projectId), "projectId"); + ex.addProxyObject(String.valueOf(projectId), "projectId"); throw ex; } @@ -626,7 +627,8 @@ public class ProjectManagerImpl extends ManagerBase implements ProjectManager { if ( domain != null ){ domainUuid = domain.getUuid(); } - ex.addProxyObject(domainUuid, "domainId"); + ex.addProxyObject(domainUuid, "domainId"); + throw ex; } //verify permissions @@ -654,7 +656,7 @@ public class ProjectManagerImpl extends ManagerBase implements ProjectManager { - public ProjectInvitation createAccountInvitation(Project project, Long accountId) { + public ProjectInvitation createAccountInvitation(Project project, Long accountId) { if (activeInviteExists(project, accountId, null)) { throw new InvalidParameterValueException("There is already a pending invitation for account id=" + accountId + " to the project id=" + project); } @@ -678,7 +680,7 @@ public class ProjectManagerImpl extends ManagerBase implements ProjectManager { } if (invite != null) { - if (invite.getState() == ProjectInvitation.State.Completed || + if (invite.getState() == ProjectInvitation.State.Completed || (invite.getState() == ProjectInvitation.State.Pending && _projectInvitationDao.isActive(invite.getId(), _invitationTimeOut))) { return true; } else { @@ -796,7 +798,7 @@ public class ProjectManagerImpl extends ManagerBase implements ProjectManager { if (projectAccount != null) { s_logger.debug("Account " + accountNameFinal + " already added to the project id=" + projectId); } else { - assignAccountToProject(project, accountIdFinal, ProjectAccount.Role.Regular); + assignAccountToProject(project, accountIdFinal, ProjectAccount.Role.Regular); } } else { s_logger.warn("Failed to update project invitation " + inviteFinal + " with state " + newState); @@ -842,7 +844,7 @@ public class ProjectManagerImpl extends ManagerBase implements ProjectManager { if (currentState == State.Active) { s_logger.debug("The project id=" + projectId + " is already active, no need to activate it again"); return project; - } + } if (currentState != State.Suspended) { throw new InvalidParameterValueException("Can't activate the project in " + currentState + " state"); @@ -966,7 +968,7 @@ public class ProjectManagerImpl extends ManagerBase implements ProjectManager { } } - public void sendInvite(String token, String email, long projectId) throws MessagingException, UnsupportedEncodingException { + public void sendInvite(String token, String email, long projectId) throws MessagingException, UnsupportedEncodingException { if (_smtpSession != null) { InternetAddress address = null; if (email != null) { @@ -1025,7 +1027,7 @@ public class ProjectManagerImpl extends ManagerBase implements ProjectManager { return true; } else { s_logger.debug("Failed to remove project invitation id=" + id); - return false; + return false; } } From 9495c68c1378e379880433d45ec43bfda75ec3f9 Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Mon, 27 Jan 2014 00:55:19 +0100 Subject: [PATCH 032/169] findbugs: removed obfuscating fields --- .../com/cloud/network/ExternalIpAddressAllocator.java | 9 ++++----- .../src/com/cloud/vm/snapshot/VMSnapshotManagerImpl.java | 1 - 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/server/src/com/cloud/network/ExternalIpAddressAllocator.java b/server/src/com/cloud/network/ExternalIpAddressAllocator.java index ac8f64ec3e5..2cce15e2478 100644 --- a/server/src/com/cloud/network/ExternalIpAddressAllocator.java +++ b/server/src/com/cloud/network/ExternalIpAddressAllocator.java @@ -40,7 +40,6 @@ import com.cloud.utils.exception.CloudRuntimeException; @Local(value=IpAddrAllocator.class) public class ExternalIpAddressAllocator extends AdapterBase implements IpAddrAllocator{ private static final Logger s_logger = Logger.getLogger(ExternalIpAddressAllocator.class); - String _name; @Inject ConfigurationDao _configDao = null; @Inject IPAddressDao _ipAddressDao = null; @Inject VlanDao _vlanDao; @@ -50,10 +49,10 @@ public class ExternalIpAddressAllocator extends AdapterBase implements IpAddrAll @Override public IpAddr getPrivateIpAddress(String macAddr, long dcId, long podId) { - if (_externalIpAllocatorUrl == null || this._externalIpAllocatorUrl.equalsIgnoreCase("")) { + if (_externalIpAllocatorUrl == null || _externalIpAllocatorUrl.equalsIgnoreCase("")) { return new IpAddr(); } - String urlString = this._externalIpAllocatorUrl + "?command=getIpAddr&mac=" + macAddr + "&dc=" + dcId + "&pod=" + podId; + String urlString = _externalIpAllocatorUrl + "?command=getIpAddr&mac=" + macAddr + "&dc=" + dcId + "&pod=" + podId; s_logger.debug("getIP:" + urlString); BufferedReader in = null; @@ -99,11 +98,11 @@ public class ExternalIpAddressAllocator extends AdapterBase implements IpAddrAll @Override public boolean releasePrivateIpAddress(String ip, long dcId, long podId) { /*TODO: call API to release the ip address from external DHCP server*/ - if (_externalIpAllocatorUrl == null || this._externalIpAllocatorUrl.equalsIgnoreCase("")) { + if (_externalIpAllocatorUrl == null || _externalIpAllocatorUrl.equalsIgnoreCase("")) { return false; } - String urlString = this._externalIpAllocatorUrl + "?command=releaseIpAddr&ip=" + ip + "&dc=" + dcId + "&pod=" + podId; + String urlString = _externalIpAllocatorUrl + "?command=releaseIpAddr&ip=" + ip + "&dc=" + dcId + "&pod=" + podId; s_logger.debug("releaseIP:" + urlString); BufferedReader in = null; diff --git a/server/src/com/cloud/vm/snapshot/VMSnapshotManagerImpl.java b/server/src/com/cloud/vm/snapshot/VMSnapshotManagerImpl.java index e273e1a2338..5ae09fde6d6 100644 --- a/server/src/com/cloud/vm/snapshot/VMSnapshotManagerImpl.java +++ b/server/src/com/cloud/vm/snapshot/VMSnapshotManagerImpl.java @@ -105,7 +105,6 @@ public class VMSnapshotManagerImpl extends ManagerBase implements VMSnapshotMana public static final String VM_WORK_JOB_HANDLER = VMSnapshotManagerImpl.class.getSimpleName(); - String _name; @Inject VMInstanceDao _vmInstanceDao; @Inject VMSnapshotDao _vmSnapshotDao; From 3fae917ebefb56ba280c2e6c5b6ff0f1720d46ed Mon Sep 17 00:00:00 2001 From: Koushik Das Date: Mon, 27 Jan 2014 15:23:28 +0530 Subject: [PATCH 033/169] CLOUDSTACK-5949: Remove unused field vnc_password from user_vm_view Removed unused field. This will ensure that list VM queries are not blocked. --- .../com/cloud/api/query/vo/UserVmJoinVO.java | 9 - setup/db/db/schema-421to430.sql | 200 ++++++++++++++++++ 2 files changed, 200 insertions(+), 9 deletions(-) diff --git a/server/src/com/cloud/api/query/vo/UserVmJoinVO.java b/server/src/com/cloud/api/query/vo/UserVmJoinVO.java index dc6b30d50e1..2bb0783595e 100644 --- a/server/src/com/cloud/api/query/vo/UserVmJoinVO.java +++ b/server/src/com/cloud/api/query/vo/UserVmJoinVO.java @@ -34,7 +34,6 @@ import com.cloud.network.Networks.TrafficType; import com.cloud.server.ResourceTag.ResourceObjectType; import com.cloud.storage.Storage.StoragePoolType; import com.cloud.storage.Volume; -import com.cloud.utils.db.Encrypt; import com.cloud.utils.db.GenericDao; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VirtualMachine; @@ -122,10 +121,6 @@ public class UserVmJoinVO extends BaseViewVO implements ControlledViewEntity { @Column(name="ha_enabled", updatable=true, nullable=true) private boolean haEnabled; - @Encrypt - @Column(name="vnc_password", updatable=true, nullable=false, length=255) - protected String vncPassword; - @Column(name="limit_cpu_use", updatable=true, nullable=true) private boolean limitCpuUse; @@ -519,10 +514,6 @@ public class UserVmJoinVO extends BaseViewVO implements ControlledViewEntity { return haEnabled; } - public String getVncPassword() { - return vncPassword; - } - public String getPrivateIpAddress() { return privateIpAddress; } diff --git a/setup/db/db/schema-421to430.sql b/setup/db/db/schema-421to430.sql index 8bfec373163..4f8a52e9f3a 100644 --- a/setup/db/db/schema-421to430.sql +++ b/setup/db/db/schema-421to430.sql @@ -890,3 +890,203 @@ ALTER TABLE `cloud`.`vm_instance` DROP COLUMN `speed`; INSERT IGNORE INTO `cloud`.`hypervisor_capabilities`(uuid, hypervisor_type, hypervisor_version, max_guests_limit, security_group_enabled, max_data_volumes_limit, max_hosts_per_cluster, storage_motion_supported, vm_snapshot_enabled) VALUES (UUID(), 'VMware', '5.5', 128, 0, 13, 32, 1, 1); ALTER TABLE `cloud`.`network_acl_item` modify `cidr` varchar(2048); + +DROP VIEW IF EXISTS `cloud`.`user_vm_view`; +CREATE VIEW `cloud`.`user_vm_view` AS + select + vm_instance.id id, + vm_instance.name name, + user_vm.display_name display_name, + user_vm.user_data user_data, + account.id account_id, + account.uuid account_uuid, + account.account_name account_name, + account.type account_type, + domain.id domain_id, + domain.uuid domain_uuid, + domain.name domain_name, + domain.path domain_path, + projects.id project_id, + projects.uuid project_uuid, + projects.name project_name, + instance_group.id instance_group_id, + instance_group.uuid instance_group_uuid, + instance_group.name instance_group_name, + vm_instance.uuid uuid, + vm_instance.last_host_id last_host_id, + vm_instance.vm_type type, + vm_instance.limit_cpu_use limit_cpu_use, + vm_instance.created created, + vm_instance.state state, + vm_instance.removed removed, + vm_instance.ha_enabled ha_enabled, + vm_instance.hypervisor_type hypervisor_type, + vm_instance.instance_name instance_name, + vm_instance.guest_os_id guest_os_id, + vm_instance.display_vm display_vm, + guest_os.uuid guest_os_uuid, + vm_instance.pod_id pod_id, + host_pod_ref.uuid pod_uuid, + vm_instance.private_ip_address private_ip_address, + vm_instance.private_mac_address private_mac_address, + vm_instance.vm_type vm_type, + data_center.id data_center_id, + data_center.uuid data_center_uuid, + data_center.name data_center_name, + data_center.is_security_group_enabled security_group_enabled, + data_center.networktype data_center_type, + host.id host_id, + host.uuid host_uuid, + host.name host_name, + vm_template.id template_id, + vm_template.uuid template_uuid, + vm_template.name template_name, + vm_template.display_text template_display_text, + vm_template.enable_password password_enabled, + iso.id iso_id, + iso.uuid iso_uuid, + iso.name iso_name, + iso.display_text iso_display_text, + service_offering.id service_offering_id, + disk_offering.uuid service_offering_uuid, + Case + When (`cloud`.`service_offering`.`cpu` is null) then (`custom_cpu`.`value`) + Else ( `cloud`.`service_offering`.`cpu`) + End as `cpu`, + Case + When (`cloud`.`service_offering`.`speed` is null) then (`custom_speed`.`value`) + Else ( `cloud`.`service_offering`.`speed`) + End as `speed`, + Case + When (`cloud`.`service_offering`.`ram_size` is null) then (`custom_ram_size`.`value`) + Else ( `cloud`.`service_offering`.`ram_size`) + END as `ram_size`, + disk_offering.name service_offering_name, + storage_pool.id pool_id, + storage_pool.uuid pool_uuid, + storage_pool.pool_type pool_type, + volumes.id volume_id, + volumes.uuid volume_uuid, + volumes.device_id volume_device_id, + volumes.volume_type volume_type, + security_group.id security_group_id, + security_group.uuid security_group_uuid, + security_group.name security_group_name, + security_group.description security_group_description, + nics.id nic_id, + nics.uuid nic_uuid, + nics.network_id network_id, + nics.ip4_address ip_address, + nics.ip6_address ip6_address, + nics.ip6_gateway ip6_gateway, + nics.ip6_cidr ip6_cidr, + nics.default_nic is_default_nic, + nics.gateway gateway, + nics.netmask netmask, + nics.mac_address mac_address, + nics.broadcast_uri broadcast_uri, + nics.isolation_uri isolation_uri, + vpc.id vpc_id, + vpc.uuid vpc_uuid, + networks.uuid network_uuid, + networks.name network_name, + networks.traffic_type traffic_type, + networks.guest_type guest_type, + user_ip_address.id public_ip_id, + user_ip_address.uuid public_ip_uuid, + user_ip_address.public_ip_address public_ip_address, + ssh_keypairs.keypair_name keypair_name, + resource_tags.id tag_id, + resource_tags.uuid tag_uuid, + resource_tags.key tag_key, + resource_tags.value tag_value, + resource_tags.domain_id tag_domain_id, + resource_tags.account_id tag_account_id, + resource_tags.resource_id tag_resource_id, + resource_tags.resource_uuid tag_resource_uuid, + resource_tags.resource_type tag_resource_type, + resource_tags.customer tag_customer, + async_job.id job_id, + async_job.uuid job_uuid, + async_job.job_status job_status, + async_job.account_id job_account_id, + affinity_group.id affinity_group_id, + affinity_group.uuid affinity_group_uuid, + affinity_group.name affinity_group_name, + affinity_group.description affinity_group_description, + vm_instance.dynamically_scalable dynamically_scalable, + all_details.name detail_name, + all_details.value detail_value + from + `cloud`.`user_vm` + inner join + `cloud`.`vm_instance` ON vm_instance.id = user_vm.id + and vm_instance.removed is NULL + inner join + `cloud`.`account` ON vm_instance.account_id = account.id + inner join + `cloud`.`domain` ON vm_instance.domain_id = domain.id + left join + `cloud`.`guest_os` ON vm_instance.guest_os_id = guest_os.id + left join + `cloud`.`host_pod_ref` ON vm_instance.pod_id = host_pod_ref.id + left join + `cloud`.`projects` ON projects.project_account_id = account.id + left join + `cloud`.`instance_group_vm_map` ON vm_instance.id = instance_group_vm_map.instance_id + left join + `cloud`.`instance_group` ON instance_group_vm_map.group_id = instance_group.id + left join + `cloud`.`data_center` ON vm_instance.data_center_id = data_center.id + left join + `cloud`.`host` ON vm_instance.host_id = host.id + left join + `cloud`.`vm_template` ON vm_instance.vm_template_id = vm_template.id + left join + `cloud`.`vm_template` iso ON iso.id = user_vm.iso_id + left join + `cloud`.`service_offering` ON vm_instance.service_offering_id = service_offering.id + left join + `cloud`.`disk_offering` ON vm_instance.service_offering_id = disk_offering.id + left join + `cloud`.`volumes` ON vm_instance.id = volumes.instance_id + left join + `cloud`.`storage_pool` ON volumes.pool_id = storage_pool.id + left join + `cloud`.`security_group_vm_map` ON vm_instance.id = security_group_vm_map.instance_id + left join + `cloud`.`security_group` ON security_group_vm_map.security_group_id = security_group.id + left join + `cloud`.`nics` ON vm_instance.id = nics.instance_id and nics.removed is null + left join + `cloud`.`networks` ON nics.network_id = networks.id + left join + `cloud`.`vpc` ON networks.vpc_id = vpc.id and vpc.removed is null + left join + `cloud`.`user_ip_address` ON user_ip_address.vm_id = vm_instance.id + left join + `cloud`.`user_vm_details` as ssh_details ON ssh_details.vm_id = vm_instance.id + and ssh_details.name = 'SSH.PublicKey' + left join + `cloud`.`ssh_keypairs` ON ssh_keypairs.public_key = ssh_details.value + left join + `cloud`.`resource_tags` ON resource_tags.resource_id = vm_instance.id + and resource_tags.resource_type = 'UserVm' + left join + `cloud`.`async_job` ON async_job.instance_id = vm_instance.id + and async_job.instance_type = 'VirtualMachine' + and async_job.job_status = 0 + left join + `cloud`.`affinity_group_vm_map` ON vm_instance.id = affinity_group_vm_map.instance_id + left join + `cloud`.`affinity_group` ON affinity_group_vm_map.affinity_group_id = affinity_group.id + left join + `cloud`.`user_vm_details` as all_details ON all_details.vm_id = vm_instance.id + left join + `cloud`.`user_vm_details` `custom_cpu` ON (((`custom_cpu`.`vm_id` = `cloud`.`vm_instance`.`id`) and (`custom_cpu`.`name` = 'CpuNumber'))) + left join + `cloud`.`user_vm_details` `custom_speed` ON (((`custom_speed`.`vm_id` = `cloud`.`vm_instance`.`id`) and (`custom_speed`.`name` = 'CpuSpeed'))) + left join + `cloud`.`user_vm_details` `custom_ram_size` ON (((`custom_ram_size`.`vm_id` = `cloud`.`vm_instance`.`id`) and (`custom_ram_size`.`name` = 'memory'))); + + From 6d1475182cd9ea17010a142043dd9fe9c923cf81 Mon Sep 17 00:00:00 2001 From: Min Chen Date: Mon, 27 Jan 2014 11:54:38 -0800 Subject: [PATCH 034/169] CLOUDSTACK-5960: Domain admin or user cannot register a template using S3/Swift object store. --- server/src/com/cloud/template/TemplateAdapterBase.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/server/src/com/cloud/template/TemplateAdapterBase.java b/server/src/com/cloud/template/TemplateAdapterBase.java index 6194e6e1f4c..c75c1e4137b 100755 --- a/server/src/com/cloud/template/TemplateAdapterBase.java +++ b/server/src/com/cloud/template/TemplateAdapterBase.java @@ -162,8 +162,14 @@ public abstract class TemplateAdapterBase extends AdapterBase implements Templat } boolean isAdmin = _accountDao.findById(templateOwner.getId()).getType() == Account.ACCOUNT_TYPE_ADMIN; - - if (!isAdmin && zoneId == null) { + boolean isRegionStore = false; + List stores = _imgStoreDao.findRegionImageStores(); + if (stores != null && stores.size() > 0) { + isRegionStore = true; + } + + if (!isAdmin && zoneId == null && !isRegionStore ) { + // domain admin and user should also be able to register template on a region store throw new InvalidParameterValueException("Please specify a valid zone Id."); } From e8ba2d08500b582a5e2e52e694fa6d9ec89d923b Mon Sep 17 00:00:00 2001 From: Likitha Shetty Date: Mon, 27 Jan 2014 17:52:16 +0530 Subject: [PATCH 035/169] CLOUDSTACK-5796. [VMware] Size column is not getting updated in snapshot_store_ref table when a snapshot is backed up in secondary storage. Calculate and update the size of a backed up snapshot. This snapshot size is in turn used to update the secondary_storage count for an account. --- .../resource/VmwareStorageProcessor.java | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageProcessor.java index 2089c084d73..24cfe537b95 100644 --- a/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -985,7 +985,7 @@ public class VmwareStorageProcessor implements StorageProcessor { String backupUuid = UUID.randomUUID().toString(); Pair snapshotInfo = exportVolumeToSecondaryStroage(vmMo, volumePath, secStorageUrl, installPath, backupUuid, workerVmName); - return new Ternary(backupUuid + "/" + backupUuid, snapshotInfo.first(), snapshotInfo.second()); + return new Ternary(backupUuid, snapshotInfo.first(), snapshotInfo.second()); } @Override @@ -1067,8 +1067,25 @@ public class VmwareStorageProcessor implements StorageProcessor { answer = new CopyCmdAnswer(details); } else { details = "Successfully backedUp the snapshot with Uuid: " + snapshotUuid + " to secondary storage."; + + // Get snapshot physical size + long physicalSize = 0l; + String secondaryMountPoint = mountService.getMountPoint(secondaryStorageUrl); + String snapshotDir = destSnapshot.getPath() + "/" + snapshotBackupUuid; + File[] files = new File(secondaryMountPoint + "/" + snapshotDir).listFiles(); + if(files != null) { + for(File file : files) { + String fileName = file.getName(); + if(fileName.toLowerCase().startsWith(snapshotBackupUuid) && fileName.toLowerCase().endsWith(".vmdk")) { + physicalSize = new File(secondaryMountPoint + "/" + snapshotDir + "/" + fileName).length(); + break; + } + } + } + SnapshotObjectTO newSnapshot = new SnapshotObjectTO(); - newSnapshot.setPath(destSnapshot.getPath() + "/" + snapshotBackupUuid); + newSnapshot.setPath(snapshotDir + "/" + snapshotBackupUuid); + newSnapshot.setPhysicalSize(physicalSize); answer = new CopyCmdAnswer(newSnapshot); } } finally { From 1f4e2a51a42e566c11ab35cb38a81fd177feff16 Mon Sep 17 00:00:00 2001 From: Sanjay Tripathi Date: Mon, 27 Jan 2014 14:18:53 +0530 Subject: [PATCH 036/169] CS-5950: Wrong storage tags are used for ISO deployed VMs. --- .../com/cloud/deploy/DeploymentPlanningManagerImpl.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/server/src/com/cloud/deploy/DeploymentPlanningManagerImpl.java b/server/src/com/cloud/deploy/DeploymentPlanningManagerImpl.java index 9c3aa974eed..0a2580542e1 100644 --- a/server/src/com/cloud/deploy/DeploymentPlanningManagerImpl.java +++ b/server/src/com/cloud/deploy/DeploymentPlanningManagerImpl.java @@ -17,6 +17,7 @@ package com.cloud.deploy; import java.util.ArrayList; +import java.util.Arrays; import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; @@ -91,6 +92,7 @@ import com.cloud.org.Grouping; import com.cloud.resource.ResourceState; import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.ScopeType; +import com.cloud.storage.Storage; import com.cloud.storage.StorageManager; import com.cloud.storage.StoragePool; import com.cloud.storage.StoragePoolHostVO; @@ -1215,8 +1217,12 @@ public class DeploymentPlanningManagerImpl extends ManagerBase implements Deploy s_logger.debug("Calling StoragePoolAllocators to find suitable pools"); DiskOfferingVO diskOffering = _diskOfferingDao.findById(toBeCreated.getDiskOfferingId()); - DiskProfile diskProfile = new DiskProfile(toBeCreated, diskOffering, vmProfile.getHypervisorType()); + if (vmProfile.getTemplate().getFormat() == Storage.ImageFormat.ISO && vmProfile.getServiceOffering().getTagsArray().length != 0) { + diskOffering.setTagsArray(Arrays.asList(vmProfile.getServiceOffering().getTagsArray())); + } + + DiskProfile diskProfile = new DiskProfile(toBeCreated, diskOffering, vmProfile.getHypervisorType()); boolean useLocalStorage = false; if (vmProfile.getType() != VirtualMachine.Type.User) { String ssvmUseLocalStorage = _configDao.getValue(Config.SystemVMUseLocalStorage.key()); From 0e4b93c6c12f38d5418accac38fbdd28edb86a2c Mon Sep 17 00:00:00 2001 From: Sanjay Tripathi Date: Tue, 28 Jan 2014 19:09:26 +0530 Subject: [PATCH 037/169] CLOUDSTACK-5900: Failed to delete template/ISO that failed to download. --- .../resource/NfsSecondaryStorageResource.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index f90309dfa35..8c1a7e25c1b 100755 --- a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -1740,16 +1740,16 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S parent += File.separator; } String absoluteTemplatePath = parent + relativeTemplatePath; - File tmpltParent = new File(absoluteTemplatePath).getParentFile(); + File tmpltDir = new File(absoluteTemplatePath); String details = null; - if (!tmpltParent.exists()) { - details = "template parent directory " + tmpltParent.getName() + " doesn't exist"; + if (!tmpltDir.exists()) { + details = "template parent directory " + tmpltDir.getName() + " doesn't exist"; s_logger.debug(details); return new Answer(cmd, true, details); } - File[] tmpltFiles = tmpltParent.listFiles(); + File[] tmpltFiles = tmpltDir.listFiles(); if (tmpltFiles == null || tmpltFiles.length == 0) { - details = "No files under template parent directory " + tmpltParent.getName(); + details = "No files under template parent directory " + tmpltDir.getName(); s_logger.debug(details); } else { boolean found = false; @@ -1776,12 +1776,12 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S } if (!found) { - details = "Can not find template.properties under " + tmpltParent.getName(); + details = "Can not find template.properties under " + tmpltDir.getName(); s_logger.debug(details); } } - if (!tmpltParent.delete()) { - details = "Unable to delete directory " + tmpltParent.getName() + " under Template path " + if (!tmpltDir.delete()) { + details = "Unable to delete directory " + tmpltDir.getName() + " under Template path " + relativeTemplatePath; s_logger.debug(details); return new Answer(cmd, false, details); From 782265e713d5eaa41ca3d6040d8994639ddbce10 Mon Sep 17 00:00:00 2001 From: Jayapal Date: Tue, 28 Jan 2014 19:20:44 +0530 Subject: [PATCH 038/169] CLOUDSTACK-5966 Added enable/disable monitoring service in global config --- .../api/routing/NetworkElementCommand.java | 2 ++ .../VirtualRoutingResource.java | 9 ++++-- .../vmware/resource/VmwareResource.java | 4 +++ .../xen/resource/CitrixResourceBase.java | 5 ++++ .../src/com/cloud/configuration/Config.java | 1 + .../VirtualNetworkApplianceManagerImpl.java | 16 ++++++++-- setup/db/db/schema-421to430.sql | 1 + .../config/opt/cloud/bin/monitor_service.sh | 29 +++++++++++++++++-- 8 files changed, 60 insertions(+), 7 deletions(-) diff --git a/core/src/com/cloud/agent/api/routing/NetworkElementCommand.java b/core/src/com/cloud/agent/api/routing/NetworkElementCommand.java index 843d213896e..23b75fd671b 100644 --- a/core/src/com/cloud/agent/api/routing/NetworkElementCommand.java +++ b/core/src/com/cloud/agent/api/routing/NetworkElementCommand.java @@ -34,6 +34,8 @@ public abstract class NetworkElementCommand extends Command { public static final String GUEST_BRIDGE = "guest.bridge"; public static final String VPC_PRIVATE_GATEWAY = "vpc.gateway.private"; public static final String FIREWALL_EGRESS_DEFAULT = "firewall.egress.default"; + public static final String ROUTER_MONITORING_DISABLE = "router.monitor.disable"; + protected NetworkElementCommand() { diff --git a/core/src/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java b/core/src/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java index 0eac223f96e..035a5b3e130 100755 --- a/core/src/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java +++ b/core/src/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java @@ -821,9 +821,12 @@ public class VirtualRoutingResource implements Manager { return new ConsoleProxyLoadAnswer(cmd, proxyVmId, proxyVmName, success, result); } - public String configureMonitor(final String routerIP, final String config) { + public String configureMonitor(final String routerIP, final String config, final String disable) { String args= " -c " + config; + if (disable != null) { + args = args + "-d"; + } return routerProxy("monitor_service.sh", routerIP, args); } @@ -968,9 +971,11 @@ public class VirtualRoutingResource implements Manager { private Answer execute(SetMonitorServiceCommand cmd) { String routerIp = cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP); + String disable = cmd.getAccessDetail(NetworkElementCommand.ROUTER_MONITORING_DISABLE); String config = cmd.getConfiguration(); - String result = configureMonitor(routerIp, config); + + String result = configureMonitor(routerIp, config, disable); if (result != null) { return new Answer(cmd, false, "SetMonitorServiceCommand failed"); diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java index bfb95fd1a5d..7439f12a932 100755 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -806,10 +806,14 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa String controlIp = getRouterSshControlIp(cmd); String config = cmd.getConfiguration(); + String disable = cmd.getAccessDetail(NetworkElementCommand.ROUTER_MONITORING_DISABLE); String args = ""; args += " -c " + config; + if (disable != null) { + args = args + " -d "; + } try { VmwareManager mgr = getServiceContext().getStockObject(VmwareManager.CONTEXT_STOCK_NAME); diff --git a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java index f065d58dff0..43c7d8b683f 100644 --- a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java +++ b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java @@ -7550,6 +7550,7 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe Connection conn = getConnection(); String routerIp = cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP); + String disable = cmd.getAccessDetail(NetworkElementCommand.ROUTER_MONITORING_DISABLE); if (routerIp == null) { return new Answer(cmd); @@ -7557,6 +7558,10 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe String args = "monitor_service.sh " + routerIp; args += " -c " + config; + if (disable != null) { + args += " -d "; + } + String result = callHostPlugin(conn, "vmops", "routerProxy", "args", args); if (result == null || result.isEmpty()) { diff --git a/server/src/com/cloud/configuration/Config.java b/server/src/com/cloud/configuration/Config.java index d2713c022ed..da79995dde0 100755 --- a/server/src/com/cloud/configuration/Config.java +++ b/server/src/com/cloud/configuration/Config.java @@ -105,6 +105,7 @@ public enum Config { SecurityGroupDefaultAdding("Network", ManagementServer.class, Boolean.class, "network.securitygroups.defaultadding", "true", "If true, the user VM would be added to the default security group by default", null), GuestOSNeedGatewayOnNonDefaultNetwork("Network", NetworkOrchestrationService.class, String.class, "network.dhcp.nondefaultnetwork.setgateway.guestos", "Windows", "The guest OS's name start with this fields would result in DHCP server response gateway information even when the network it's on is not default network. Names are separated by comma.", null), + EnableServiceMonitoring("Network", ManagementServer.class, Boolean.class, "network.router.enableserviceMonitoring", "false", "service monitoring in router enable/disable option, default false", null), //VPN RemoteAccessVpnPskLength("Network", AgentManager.class, Integer.class, "remote.access.vpn.psk.length", "24", "The length of the ipsec preshared key (minimum 8, maximum 256)", null), diff --git a/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java b/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java index af7ef831019..c372e7b3678 100755 --- a/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java +++ b/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java @@ -2223,12 +2223,20 @@ public class VirtualNetworkApplianceManagerImpl extends ManagerBase implements V finalizeUserDataAndDhcpOnStart(cmds, router, provider, guestNetworkId); } - finalizeMonitorServiceOnStrat(cmds, profile, router, provider, routerGuestNtwkIds.get(0)); + + String serviceMonitringSet = _configDao.getValue(Config.EnableServiceMonitoring.key()); + + if (serviceMonitringSet != null && serviceMonitringSet.equalsIgnoreCase("true")) { + finalizeMonitorServiceOnStrat(cmds, profile, router, provider, routerGuestNtwkIds.get(0), true); + } else { + finalizeMonitorServiceOnStrat(cmds, profile, router, provider, routerGuestNtwkIds.get(0), false); + } + return true; } - private void finalizeMonitorServiceOnStrat(Commands cmds, VirtualMachineProfile profile, DomainRouterVO router, Provider provider, long networkId) { + private void finalizeMonitorServiceOnStrat(Commands cmds, VirtualMachineProfile profile, DomainRouterVO router, Provider provider, long networkId, Boolean add) { NetworkVO network = _networkDao.findById(networkId); @@ -2270,6 +2278,10 @@ public class VirtualNetworkApplianceManagerImpl extends ManagerBase implements V command.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(networkId, router.getId())); command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); + if (!add) { + command.setAccessDetail(NetworkElementCommand.ROUTER_MONITORING_DISABLE, add.toString()); + } + cmds.addCommand("monitor", command); } diff --git a/setup/db/db/schema-421to430.sql b/setup/db/db/schema-421to430.sql index 4f8a52e9f3a..1ed6c6b6cc9 100644 --- a/setup/db/db/schema-421to430.sql +++ b/setup/db/db/schema-421to430.sql @@ -1089,4 +1089,5 @@ CREATE VIEW `cloud`.`user_vm_view` AS left join `cloud`.`user_vm_details` `custom_ram_size` ON (((`custom_ram_size`.`vm_id` = `cloud`.`vm_instance`.`id`) and (`custom_ram_size`.`name` = 'memory'))); +INSERT INTO `cloud`.`configuration`(category, instance, component, name, value, description, default_value) VALUES ('NetworkManager', 'DEFAULT', 'management-server', 'network.router.EnableServiceMonitoring', 'false', 'service monitoring in router enable/disable option, default false', 'false') ON DUPLICATE KEY UPDATE category='NetworkManager'; diff --git a/systemvm/patches/debian/config/opt/cloud/bin/monitor_service.sh b/systemvm/patches/debian/config/opt/cloud/bin/monitor_service.sh index 51b69235318..809c5b7cdbd 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/monitor_service.sh +++ b/systemvm/patches/debian/config/opt/cloud/bin/monitor_service.sh @@ -25,7 +25,7 @@ then exit 1 fi -set -x +#set -x usage() { printf "Usage: %s: -c config string \n" $(basename $0) >&2 } @@ -58,13 +58,36 @@ done config=$2 +if [ -n "$3" ] +then #delete cron job before updating config file -crontab -l | grep -v monitorServices.py | crontab - +crontab -l|grep "monitorServices.py" + +if [ $? -eq 0 ] +then + t=`date +%s`; + touch /tmp/monitor-$t.txt; + conf=/tmp/monitor-$t.txt + crontab -l >$conf + sed -i /#monitoringConfig/,+3d $conf + crontab $conf + rm $conf +fi + + +logger -t cloud "deleted crontab entry for monitoring services" +unlock_exit 0 $lock $locked +fi create_config $config #add cron job -(crontab -l ;echo -e "SHELL=/bin/bash\nPATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin\n */3 * * * * /usr/bin/python /root/monitorServices.py") | crontab - +crontab -l|grep "monitorServices.py" +if [ $? -ne 0 ] + then + (crontab -l ;echo -e "#monitoringConfig\nSHELL=/bin/bash\nPATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin\n */3 * * * * /usr/bin/python /root/monitorServices.py") | crontab - + logger -t cloud "added crontab entry for monitoring services" + fi unlock_exit 0 $lock $locked From 77aeb2871ba393adb311f590185f2a34055c1aa1 Mon Sep 17 00:00:00 2001 From: Min Chen Date: Tue, 28 Jan 2014 13:09:57 -0800 Subject: [PATCH 039/169] CLOUDSTACK-5971:Templates created from a snapshots can't be copied to other zones. --- .../cloudstack/storage/image/TemplateServiceImpl.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java b/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java index 7514f437ffd..a649bb72123 100644 --- a/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java +++ b/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java @@ -75,6 +75,7 @@ import com.cloud.dc.dao.DataCenterDao; import com.cloud.exception.ResourceAllocationException; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.storage.DataStoreRole; +import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.Storage.TemplateType; import com.cloud.storage.StoragePool; import com.cloud.storage.VMTemplateStorageResourceAssoc; @@ -736,6 +737,13 @@ public class TemplateServiceImpl implements TemplateService { @Override public AsyncCallFuture copyTemplate(TemplateInfo srcTemplate, DataStore destStore) { + // for vmware template, we need to check if ova packing is needed, since template created from snapshot does not have .ova file + // we invoke createEntityExtractURL to trigger ova packing. Ideally, we can directly use extractURL to pass to following createTemplate. + // Need to understand what is the background to use two different urls for copy and extract. + if (srcTemplate.getFormat() == ImageFormat.OVA){ + ImageStoreEntity tmpltStore = (ImageStoreEntity)srcTemplate.getDataStore(); + tmpltStore.createEntityExtractUrl(srcTemplate.getInstallPath(), srcTemplate.getFormat(), srcTemplate); + } // generate a URL from source template ssvm to download to destination data store String url = generateCopyUrl(srcTemplate); if (url == null) { From e624632dfd532ba2885a8e7dffcb40d1436d02c1 Mon Sep 17 00:00:00 2001 From: Marcus Sorensen Date: Tue, 28 Jan 2014 15:32:37 -0700 Subject: [PATCH 040/169] CLOUDSTACK-5968 create vm.memballoon.disable agent parameter --- .../hypervisor/kvm/resource/LibvirtComputingResource.java | 8 +++++++- .../com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java | 2 ++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index d3c87d14baf..6e0c6fa1a85 100755 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -374,6 +374,7 @@ ServerResource { private boolean _can_bridge_firewall; protected String _localStoragePath; protected String _localStorageUUID; + protected boolean _noMemBalloon = false; protected String _guestCpuMode; protected String _guestCpuModel; private final Map _pifs = new HashMap(); @@ -749,6 +750,11 @@ ServerResource { value = (String) params.get("cmds.timeout"); _cmdsTimeout = NumbersUtil.parseInt(value, 7200) * 1000; + value = (String) params.get("vm.memballoon.disable"); + if (Boolean.parseBoolean(value)) { + _noMemBalloon = true; + } + value = (String) params.get("host.reserved.mem.mb"); _dom0MinMem = NumbersUtil.parseInt(value, 0) * 1024 * 1024; @@ -3489,7 +3495,7 @@ ServerResource { GuestResourceDef grd = new GuestResourceDef(); - if (vmTO.getMinRam() != vmTO.getMaxRam()){ + if (vmTO.getMinRam() != vmTO.getMaxRam() && !_noMemBalloon) { grd.setMemBalloning(true); grd.setCurrentMem(vmTO.getMinRam()/1024); grd.setMemorySize(vmTO.getMaxRam()/1024); diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java index 0ceea248a8f..6a5db84d9e8 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java @@ -166,6 +166,8 @@ public class LibvirtVMDef { } if (_memBalloning){ resBuidler.append("\n" + "\n" + "\n"); + } else { + resBuidler.append("\n" + "\n" + "\n"); } if (_vcpu != -1) { resBuidler.append("" + _vcpu + "\n"); From 661a19bde3bf9726954f5ba88b8830f8c5a17926 Mon Sep 17 00:00:00 2001 From: Marcus Sorensen Date: Tue, 28 Jan 2014 16:03:52 -0700 Subject: [PATCH 041/169] CLOUDSTACK-5968 create vm.memballoon.disable agent parameter --- agent/conf/agent.properties | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/agent/conf/agent.properties b/agent/conf/agent.properties index 5f5f3682afd..8bd9756ef27 100644 --- a/agent/conf/agent.properties +++ b/agent/conf/agent.properties @@ -114,3 +114,7 @@ domr.scripts.dir=scripts/network/domr/kvm # for examples:"Conroe" "Penryn", "Nehalem", "Westmere", "pentiumpro" and so # on,run virsh capabilities for more details. # guest.cpu.model= +# +# vm.memballoon.disable=true +# Disable memory ballooning on vm guests for overcommit, by default overcommit +# feature enables balloon and sets currentMemory to a minimum value From f5ba6a34f11f9717957acec4c7720f8ce9200d7a Mon Sep 17 00:00:00 2001 From: Sanjay Tripathi Date: Wed, 29 Jan 2014 12:56:03 +0530 Subject: [PATCH 042/169] Revert "CLOUDSTACK-5900: Failed to delete template/ISO that failed to download." This reverts commit 0e4b93c6c12f38d5418accac38fbdd28edb86a2c. --- .../resource/NfsSecondaryStorageResource.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index 8c1a7e25c1b..f90309dfa35 100755 --- a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -1740,16 +1740,16 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S parent += File.separator; } String absoluteTemplatePath = parent + relativeTemplatePath; - File tmpltDir = new File(absoluteTemplatePath); + File tmpltParent = new File(absoluteTemplatePath).getParentFile(); String details = null; - if (!tmpltDir.exists()) { - details = "template parent directory " + tmpltDir.getName() + " doesn't exist"; + if (!tmpltParent.exists()) { + details = "template parent directory " + tmpltParent.getName() + " doesn't exist"; s_logger.debug(details); return new Answer(cmd, true, details); } - File[] tmpltFiles = tmpltDir.listFiles(); + File[] tmpltFiles = tmpltParent.listFiles(); if (tmpltFiles == null || tmpltFiles.length == 0) { - details = "No files under template parent directory " + tmpltDir.getName(); + details = "No files under template parent directory " + tmpltParent.getName(); s_logger.debug(details); } else { boolean found = false; @@ -1776,12 +1776,12 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S } if (!found) { - details = "Can not find template.properties under " + tmpltDir.getName(); + details = "Can not find template.properties under " + tmpltParent.getName(); s_logger.debug(details); } } - if (!tmpltDir.delete()) { - details = "Unable to delete directory " + tmpltDir.getName() + " under Template path " + if (!tmpltParent.delete()) { + details = "Unable to delete directory " + tmpltParent.getName() + " under Template path " + relativeTemplatePath; s_logger.debug(details); return new Answer(cmd, false, details); From ae22d1990e081267fdb68b2e014153a8207083c5 Mon Sep 17 00:00:00 2001 From: Sanjay Tripathi Date: Wed, 29 Jan 2014 13:52:03 +0530 Subject: [PATCH 043/169] CLOUDSTACK-5900: Failed to delete template/ISO that failed to download. --- .../storage/resource/NfsSecondaryStorageResource.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index f90309dfa35..0ce36ed9beb 100755 --- a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -1740,7 +1740,12 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S parent += File.separator; } String absoluteTemplatePath = parent + relativeTemplatePath; - File tmpltParent = new File(absoluteTemplatePath).getParentFile(); + File tmpltParent; + if(absoluteTemplatePath.endsWith(String.valueOf(obj.getId()))) { + tmpltParent = new File(absoluteTemplatePath); + } else { + tmpltParent = new File(absoluteTemplatePath).getParentFile(); + } String details = null; if (!tmpltParent.exists()) { details = "template parent directory " + tmpltParent.getName() + " doesn't exist"; From caf17c2f46bf317ec8966b08aaff728a73fda14a Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Wed, 29 Jan 2014 17:30:19 +0100 Subject: [PATCH 044/169] CLOUDSTACK-5979 vlan parsing fixed --- .../cloud/network/element/NiciraNvpElement.java | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/plugins/network-elements/nicira-nvp/src/com/cloud/network/element/NiciraNvpElement.java b/plugins/network-elements/nicira-nvp/src/com/cloud/network/element/NiciraNvpElement.java index 3e9e16a8a6f..c062abd42d4 100644 --- a/plugins/network-elements/nicira-nvp/src/com/cloud/network/element/NiciraNvpElement.java +++ b/plugins/network-elements/nicira-nvp/src/com/cloud/network/element/NiciraNvpElement.java @@ -259,8 +259,20 @@ NiciraNvpElementService, ResourceStateAdapter, IpDeployer { + NetUtils.getCidrSize(sourceNatIp.getVlanNetmask()); String internalCidr = network.getGateway() + "/" + network.getCidr().split("/")[1]; - long vlanid = (Vlan.UNTAGGED.equals(sourceNatIp.getVlanTag())) ? 0 - : Long.parseLong(sourceNatIp.getVlanTag()); + // assuming a vlan: + String vtag = sourceNatIp.getVlanTag(); + BroadcastDomainType tiep = null; + try { + tiep = BroadcastDomainType.getTypeOf(vtag); + } catch (URISyntaxException use) { + throw new CloudRuntimeException("vlantag for sourceNatIp is not valid: " + vtag, use); + } + if (tiep == BroadcastDomainType.Vlan) { + vtag = BroadcastDomainType.Vlan.getValueFrom(BroadcastDomainType.fromString(vtag)); + } else if (!(tiep == BroadcastDomainType.UnDecided || tiep == BroadcastDomainType.Native)) { + throw new CloudRuntimeException("only vlans are supported for sourceNatIp, at this moment: " + vtag); + } + long vlanid = (Vlan.UNTAGGED.equals(vtag)) ? 0 : Long.parseLong(vtag); CreateLogicalRouterCommand cmd = new CreateLogicalRouterCommand( niciraNvpHost.getDetail("l3gatewayserviceuuid"), vlanid, From b1ec82cdecece8b29bc9234473ff9eb7d6abe143 Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Wed, 29 Jan 2014 17:55:01 +0100 Subject: [PATCH 045/169] Revert "CLOUDSTACK-5979 vlan parsing fixed" This reverts commit caf17c2f46bf317ec8966b08aaff728a73fda14a. --- .../cloud/network/element/NiciraNvpElement.java | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/plugins/network-elements/nicira-nvp/src/com/cloud/network/element/NiciraNvpElement.java b/plugins/network-elements/nicira-nvp/src/com/cloud/network/element/NiciraNvpElement.java index c062abd42d4..3e9e16a8a6f 100644 --- a/plugins/network-elements/nicira-nvp/src/com/cloud/network/element/NiciraNvpElement.java +++ b/plugins/network-elements/nicira-nvp/src/com/cloud/network/element/NiciraNvpElement.java @@ -259,20 +259,8 @@ NiciraNvpElementService, ResourceStateAdapter, IpDeployer { + NetUtils.getCidrSize(sourceNatIp.getVlanNetmask()); String internalCidr = network.getGateway() + "/" + network.getCidr().split("/")[1]; - // assuming a vlan: - String vtag = sourceNatIp.getVlanTag(); - BroadcastDomainType tiep = null; - try { - tiep = BroadcastDomainType.getTypeOf(vtag); - } catch (URISyntaxException use) { - throw new CloudRuntimeException("vlantag for sourceNatIp is not valid: " + vtag, use); - } - if (tiep == BroadcastDomainType.Vlan) { - vtag = BroadcastDomainType.Vlan.getValueFrom(BroadcastDomainType.fromString(vtag)); - } else if (!(tiep == BroadcastDomainType.UnDecided || tiep == BroadcastDomainType.Native)) { - throw new CloudRuntimeException("only vlans are supported for sourceNatIp, at this moment: " + vtag); - } - long vlanid = (Vlan.UNTAGGED.equals(vtag)) ? 0 : Long.parseLong(vtag); + long vlanid = (Vlan.UNTAGGED.equals(sourceNatIp.getVlanTag())) ? 0 + : Long.parseLong(sourceNatIp.getVlanTag()); CreateLogicalRouterCommand cmd = new CreateLogicalRouterCommand( niciraNvpHost.getDetail("l3gatewayserviceuuid"), vlanid, From d38a50343130f6f147e2c9c7b2e32a43f31c7e6b Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Wed, 29 Jan 2014 18:20:16 +0100 Subject: [PATCH 046/169] check for BroadcastDomainTypes on sourceNatIp(cherry picked from commit 6d0dbf59683e1878533c8c985f26f18a1d69e1c1) Conflicts: plugins/network-elements/nicira-nvp/src/com/cloud/network/element/NiciraNvpElement.java --- api/test/com/cloud/network/NetworksTest.java | 15 +++++++ .../network/element/NiciraNvpElement.java | 41 +++++++++++-------- 2 files changed, 38 insertions(+), 18 deletions(-) diff --git a/api/test/com/cloud/network/NetworksTest.java b/api/test/com/cloud/network/NetworksTest.java index 87f69d62192..860cff571cf 100644 --- a/api/test/com/cloud/network/NetworksTest.java +++ b/api/test/com/cloud/network/NetworksTest.java @@ -23,6 +23,7 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import com.cloud.dc.Vlan; import com.cloud.network.Networks.BroadcastDomainType; import com.cloud.network.Networks.IsolationType; import com.cloud.utils.exception.CloudRuntimeException; @@ -59,6 +60,20 @@ public class NetworksTest { Assert.assertEquals("id2 should be \"2\"", "2", id2); } + @Test + public void vlanValueTest() throws URISyntaxException { + String uri1 = "vlan://1"; + String uri2 = "1"; + String vtag = BroadcastDomainType.Vlan.getValueFrom(BroadcastDomainType.fromString(uri1)); + Assert.assertEquals("vtag should be \"1\"", "1", vtag); + BroadcastDomainType tiep1 = BroadcastDomainType.getTypeOf(uri1); + Assert.assertEquals("the type of uri1 should be 'Vlan'", BroadcastDomainType.Vlan, tiep1); + BroadcastDomainType tiep2 = BroadcastDomainType.getTypeOf(uri2); + Assert.assertEquals("the type of uri1 should be 'Undecided'", BroadcastDomainType.UnDecided, tiep2); + BroadcastDomainType tiep3 = BroadcastDomainType.getTypeOf(Vlan.UNTAGGED); + Assert.assertEquals("the type of uri1 should be 'vlan'", BroadcastDomainType.Native, tiep3); + } + @Test public void vlanIsolationTypeTest() throws URISyntaxException { String uri1 = "vlan://1"; diff --git a/plugins/network-elements/nicira-nvp/src/com/cloud/network/element/NiciraNvpElement.java b/plugins/network-elements/nicira-nvp/src/com/cloud/network/element/NiciraNvpElement.java index 3e9e16a8a6f..bcbcc8290b1 100644 --- a/plugins/network-elements/nicira-nvp/src/com/cloud/network/element/NiciraNvpElement.java +++ b/plugins/network-elements/nicira-nvp/src/com/cloud/network/element/NiciraNvpElement.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.network.element; +import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -253,25 +254,29 @@ NiciraNvpElementService, ResourceStateAdapter, IpDeployer { network.getId(), Service.SourceNat, Provider.NiciraNvp)) { s_logger.debug("Apparently we are supposed to provide SourceNat on this network"); - PublicIp sourceNatIp = _ipAddrMgr - .assignSourceNatIpAddressToGuestNetwork(owner, network); - String publicCidr = sourceNatIp.getAddress().addr() + "/" - + NetUtils.getCidrSize(sourceNatIp.getVlanNetmask()); - String internalCidr = network.getGateway() + "/" - + network.getCidr().split("/")[1]; - long vlanid = (Vlan.UNTAGGED.equals(sourceNatIp.getVlanTag())) ? 0 - : Long.parseLong(sourceNatIp.getVlanTag()); + PublicIp sourceNatIp = _ipAddrMgr.assignSourceNatIpAddressToGuestNetwork(owner, network); + String publicCidr = sourceNatIp.getAddress().addr() + "/" + NetUtils.getCidrSize(sourceNatIp.getVlanNetmask()); + String internalCidr = network.getGateway() + "/" + network.getCidr().split("/")[1]; + // assuming a vlan: + String vtag = sourceNatIp.getVlanTag(); + BroadcastDomainType tiep = null; + try { + tiep = BroadcastDomainType.getTypeOf(vtag); + } catch (URISyntaxException use) { + throw new CloudRuntimeException("vlantag for sourceNatIp is not valid: " + vtag, use); + } + if (tiep == BroadcastDomainType.Vlan) { + vtag = BroadcastDomainType.Vlan.getValueFrom(BroadcastDomainType.fromString(vtag)); + } else if (!(tiep == BroadcastDomainType.UnDecided || tiep == BroadcastDomainType.Native)) { + throw new CloudRuntimeException("only vlans are supported for sourceNatIp, at this moment: " + vtag); + } + long vlanid = (Vlan.UNTAGGED.equals(vtag)) ? 0 : Long.parseLong(vtag); - CreateLogicalRouterCommand cmd = new CreateLogicalRouterCommand( - niciraNvpHost.getDetail("l3gatewayserviceuuid"), vlanid, - BroadcastDomainType.getValue(network.getBroadcastUri()), - "router-" + network.getDisplayText(), publicCidr, - sourceNatIp.getGateway(), internalCidr, context - .getDomain().getName() - + "-" - + context.getAccount().getAccountName()); - CreateLogicalRouterAnswer answer = (CreateLogicalRouterAnswer)_agentMgr - .easySend(niciraNvpHost.getId(), cmd); + CreateLogicalRouterCommand cmd = + new CreateLogicalRouterCommand(niciraNvpHost.getDetail("l3gatewayserviceuuid"), vlanid, BroadcastDomainType.getValue(network.getBroadcastUri()), + "router-" + network.getDisplayText(), publicCidr, sourceNatIp.getGateway(), internalCidr, context.getDomain().getName() + "-" + + context.getAccount().getAccountName()); + CreateLogicalRouterAnswer answer = (CreateLogicalRouterAnswer)_agentMgr.easySend(niciraNvpHost.getId(), cmd); if (answer.getResult() == false) { s_logger.error("Failed to create Logical Router for network " + network.getDisplayText()); From 1ad7bce69b764d3d833a8682ec5016bfbeea394c Mon Sep 17 00:00:00 2001 From: Jessica Wang Date: Wed, 29 Jan 2014 15:40:14 -0800 Subject: [PATCH 047/169] CLOUDSTACK-5970: UI > network menu > guest network section > select non-VPC isolated network with SourceNAT enabled > IP Address section > select sourceNAT IP > show Configuration tab. --- ui/scripts/network.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/scripts/network.js b/ui/scripts/network.js index 4e1ce05ff91..2cb3973d3eb 100755 --- a/ui/scripts/network.js +++ b/ui/scripts/network.js @@ -2226,7 +2226,7 @@ } } - if (ipAddress.issourcenat) { + if (ipAddress.vpcid != null && ipAddress.issourcenat) { //don't show Configuration(ipRules) tab on VPC sourceNAT IP disableIpRules = true; } From b1d4c06ad1d8931f5c810dcd8b1f1df01db174f3 Mon Sep 17 00:00:00 2001 From: Alena Prokharchyk Date: Wed, 29 Jan 2014 17:12:46 -0800 Subject: [PATCH 048/169] LOUDSTACK-5987: when listTemplates with filter=community/featured, get public templates from all domains in the system, not just parent/child subdomains of the caller Conflicts: server/src/com/cloud/api/query/QueryManagerImpl.java --- .../com/cloud/api/query/QueryManagerImpl.java | 30 ++++++++++++------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/server/src/com/cloud/api/query/QueryManagerImpl.java b/server/src/com/cloud/api/query/QueryManagerImpl.java index 3cbe7275f4b..6671ab9ba6f 100644 --- a/server/src/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/com/cloud/api/query/QueryManagerImpl.java @@ -26,10 +26,6 @@ import java.util.Set; import javax.ejb.Local; import javax.inject.Inject; -import org.apache.cloudstack.engine.subsystem.api.storage.*; -import org.apache.log4j.Logger; -import org.springframework.stereotype.Component; - import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.affinity.AffinityGroupDomainMapVO; import org.apache.cloudstack.affinity.AffinityGroupResponse; @@ -85,8 +81,15 @@ import org.apache.cloudstack.api.response.UserVmResponse; import org.apache.cloudstack.api.response.VolumeResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreCapabilities; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreDriver; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.TemplateState; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.query.QueryService; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; import com.cloud.api.query.dao.AccountJoinDao; import com.cloud.api.query.dao.AffinityGroupJoinDao; @@ -172,8 +175,8 @@ import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VolumeDetailsDao; import com.cloud.tags.ResourceTagVO; import com.cloud.tags.dao.ResourceTagDao; -import com.cloud.template.VirtualMachineTemplate.TemplateFilter; import com.cloud.template.VirtualMachineTemplate.State; +import com.cloud.template.VirtualMachineTemplate.TemplateFilter; import com.cloud.user.Account; import com.cloud.user.AccountManager; import com.cloud.user.DomainManager; @@ -2930,10 +2933,17 @@ public class QueryManagerImpl extends ManagerBase implements QueryService { if (!permittedAccounts.isEmpty()) { for (Account account : permittedAccounts) { permittedAccountIds.add(account.getId()); - DomainVO accountDomain = _domainDao.findById(account.getDomainId()); + boolean publicTemplates = (templateFilter == TemplateFilter.featured || templateFilter == TemplateFilter.community); // get all parent domain ID's all the way till root domain - DomainVO domainTreeNode = accountDomain; + DomainVO domainTreeNode = null; + //if template filter is featured, or community, all child domains should be included in search + if (publicTemplates) { + domainTreeNode = _domainDao.findById(Domain.ROOT_DOMAIN); + + } else { + domainTreeNode = _domainDao.findById(account.getDomainId()); + } relatedDomainIds.add(domainTreeNode.getId()); while (domainTreeNode.getParent() != null) { domainTreeNode = _domainDao.findById(domainTreeNode.getParent()); @@ -2941,10 +2951,8 @@ public class QueryManagerImpl extends ManagerBase implements QueryService { } // get all child domain ID's - if (_accountMgr.isAdmin(account.getType()) - || (templateFilter == TemplateFilter.featured || templateFilter == TemplateFilter.community)) { - List allChildDomains = _domainDao.findAllChildren(accountDomain.getPath(), - accountDomain.getId()); + if (_accountMgr.isAdmin(account.getType()) || publicTemplates) { + List allChildDomains = _domainDao.findAllChildren(domainTreeNode.getPath(), domainTreeNode.getId()); for (DomainVO childDomain : allChildDomains) { relatedDomainIds.add(childDomain.getId()); } From 441d02dd402e632c3eba1062a55fc0d752e76003 Mon Sep 17 00:00:00 2001 From: Sheng Yang Date: Wed, 29 Jan 2014 16:15:24 -0800 Subject: [PATCH 049/169] CLOUDSTACK-5986: Make dnsmasq handle dnsmasq.leases when dhcp_release is available The original issue has been exposed due to CloudStack VR would modify the dnsmasq.leases, thus make it unsync with dnsmasq's memory lease. Make the modification to let dnsmasq handle the lease file if dhcp_release is available. --- .../config/etc/init.d/cloud-early-config | 10 ++++ .../patches/debian/config/root/edithosts.sh | 53 +++++++++---------- 2 files changed, 35 insertions(+), 28 deletions(-) diff --git a/systemvm/patches/debian/config/etc/init.d/cloud-early-config b/systemvm/patches/debian/config/etc/init.d/cloud-early-config index d6dc8f02031..77e0f17361f 100755 --- a/systemvm/patches/debian/config/etc/init.d/cloud-early-config +++ b/systemvm/patches/debian/config/etc/init.d/cloud-early-config @@ -750,6 +750,16 @@ setup_dnsmasq() { sed -i -e "/^dhcp-client-update/d" /etc/dnsmasq.conf echo 'dhcp-client-update' >> /etc/dnsmasq.conf fi + + command -v dhcp_release > /dev/null 2>&1 + no_dhcp_release=$? + if [ $no_dhcp_release -eq 0 -a -z "$ETH0_IP6" ] + then + echo 1 > /var/cache/cloud/dnsmasq_managed_lease + sed -i -e "/^leasefile-ro/d" /etc/dnsmasq.conf + else + echo 0 > /var/cache/cloud/dnsmasq_managed_lease + fi } setup_sshd(){ diff --git a/systemvm/patches/debian/config/root/edithosts.sh b/systemvm/patches/debian/config/root/edithosts.sh index b82fb8e79be..8e7ddac8fd8 100755 --- a/systemvm/patches/debian/config/root/edithosts.sh +++ b/systemvm/patches/debian/config/root/edithosts.sh @@ -80,8 +80,7 @@ fi grep "redundant_router=1" /var/cache/cloud/cmdline > /dev/null no_redundant=$? -command -v dhcp_release > /dev/null 2>&1 -no_dhcp_release=$? +dnsmasq_managed_lease=`cat /var/cache/cloud/dnsmasq_managed_lease` wait_for_dnsmasq () { local _pid=$(pidof dnsmasq) @@ -96,12 +95,7 @@ wait_for_dnsmasq () { return 1 } -if [ $ipv6 ] -then - no_dhcp_release=1 -fi - -if [ $no_dhcp_release -eq 0 ] +if [ $dnsmasq_managed_lease ] then #release previous dhcp lease if present logger -t cloud "edithosts: releasing $ipv4" @@ -145,27 +139,30 @@ then fi fi -#delete leases to supplied mac and ip addresses -if [ $ipv4 ] +if [ $dnsmasq_managed_lease -eq 0 ] then - sed -i /$mac/d $DHCP_LEASES - sed -i /"$ipv4 "/d $DHCP_LEASES -fi -if [ $ipv6 ] -then - sed -i /$duid/d $DHCP_LEASES - sed -i /"$ipv6 "/d $DHCP_LEASES -fi -sed -i /"$host "/d $DHCP_LEASES + #delete leases to supplied mac and ip addresses + if [ $ipv4 ] + then + sed -i /$mac/d $DHCP_LEASES + sed -i /"$ipv4 "/d $DHCP_LEASES + fi + if [ $ipv6 ] + then + sed -i /$duid/d $DHCP_LEASES + sed -i /"$ipv6 "/d $DHCP_LEASES + fi + sed -i /"$host "/d $DHCP_LEASES -#put in the new entry -if [ $ipv4 ] -then - echo "0 $mac $ipv4 $host *" >> $DHCP_LEASES -fi -if [ $ipv6 ] -then - echo "0 $duid $ipv6 $host *" >> $DHCP_LEASES + #put in the new entry + if [ $ipv4 ] + then + echo "0 $mac $ipv4 $host *" >> $DHCP_LEASES + fi + if [ $ipv6 ] + then + echo "0 $duid $ipv6 $host *" >> $DHCP_LEASES + fi fi #edit hosts file as well @@ -215,7 +212,7 @@ pid=$(pidof dnsmasq) if [ "$pid" != "" ] then # use SIGHUP to avoid service outage if dhcp_release is available. - if [ $no_dhcp_release -eq 0 ] + if [ $dnsmasq_managed_lease ] then kill -HUP $pid else From 8024f2cc70c31ab22efe574e866cc9de7a48da28 Mon Sep 17 00:00:00 2001 From: Sanjay Tripathi Date: Thu, 30 Jan 2014 13:12:59 +0530 Subject: [PATCH 050/169] CLOUDSTACK-5900: Failed to delete template/ISO that failed to download. --- .../storage/resource/NfsSecondaryStorageResource.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index 0ce36ed9beb..39d5b484dde 100755 --- a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -1740,11 +1740,12 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S parent += File.separator; } String absoluteTemplatePath = parent + relativeTemplatePath; - File tmpltParent; - if(absoluteTemplatePath.endsWith(String.valueOf(obj.getId()))) { - tmpltParent = new File(absoluteTemplatePath); + File tmpltPath = new File(absoluteTemplatePath); + File tmpltParent = null; + if(tmpltPath.exists() && tmpltPath.isDirectory()) { + tmpltParent = tmpltPath; } else { - tmpltParent = new File(absoluteTemplatePath).getParentFile(); + tmpltParent = tmpltPath.getParentFile(); } String details = null; if (!tmpltParent.exists()) { From 7255e50f2051aff1cb5e1991e2dde4ef48f57454 Mon Sep 17 00:00:00 2001 From: Jayapal Date: Thu, 30 Jan 2014 14:16:58 +0530 Subject: [PATCH 051/169] CLOUDSTACK-5966 Updated service monitoring config to start by default. Global settings EnableServiceMonitoring param set to true --- server/src/com/cloud/configuration/Config.java | 2 +- setup/db/db/schema-421to430.sql | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/server/src/com/cloud/configuration/Config.java b/server/src/com/cloud/configuration/Config.java index da79995dde0..2d55f4634e0 100755 --- a/server/src/com/cloud/configuration/Config.java +++ b/server/src/com/cloud/configuration/Config.java @@ -105,7 +105,7 @@ public enum Config { SecurityGroupDefaultAdding("Network", ManagementServer.class, Boolean.class, "network.securitygroups.defaultadding", "true", "If true, the user VM would be added to the default security group by default", null), GuestOSNeedGatewayOnNonDefaultNetwork("Network", NetworkOrchestrationService.class, String.class, "network.dhcp.nondefaultnetwork.setgateway.guestos", "Windows", "The guest OS's name start with this fields would result in DHCP server response gateway information even when the network it's on is not default network. Names are separated by comma.", null), - EnableServiceMonitoring("Network", ManagementServer.class, Boolean.class, "network.router.enableserviceMonitoring", "false", "service monitoring in router enable/disable option, default false", null), + EnableServiceMonitoring("Network", ManagementServer.class, Boolean.class, "network.router.enableserviceMonitoring", "true", "service monitoring in router enable/disable option, default true", null), //VPN RemoteAccessVpnPskLength("Network", AgentManager.class, Integer.class, "remote.access.vpn.psk.length", "24", "The length of the ipsec preshared key (minimum 8, maximum 256)", null), diff --git a/setup/db/db/schema-421to430.sql b/setup/db/db/schema-421to430.sql index 1ed6c6b6cc9..4f8a52e9f3a 100644 --- a/setup/db/db/schema-421to430.sql +++ b/setup/db/db/schema-421to430.sql @@ -1089,5 +1089,4 @@ CREATE VIEW `cloud`.`user_vm_view` AS left join `cloud`.`user_vm_details` `custom_ram_size` ON (((`custom_ram_size`.`vm_id` = `cloud`.`vm_instance`.`id`) and (`custom_ram_size`.`name` = 'memory'))); -INSERT INTO `cloud`.`configuration`(category, instance, component, name, value, description, default_value) VALUES ('NetworkManager', 'DEFAULT', 'management-server', 'network.router.EnableServiceMonitoring', 'false', 'service monitoring in router enable/disable option, default false', 'false') ON DUPLICATE KEY UPDATE category='NetworkManager'; From cff312ef743763c69b0c3b0cfd9faf906f3cf22c Mon Sep 17 00:00:00 2001 From: Sanjay Tripathi Date: Thu, 30 Jan 2014 16:17:19 +0530 Subject: [PATCH 052/169] CLOUDSTACK-5660: Migrate vm live migration succeeds but throws error as "Failed to migrate the system vm". --- server/src/com/cloud/vm/UserVmManagerImpl.java | 7 ++++++- ui/scripts/system.js | 12 ++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/server/src/com/cloud/vm/UserVmManagerImpl.java b/server/src/com/cloud/vm/UserVmManagerImpl.java index 363c663250f..d9923d238e1 100755 --- a/server/src/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/com/cloud/vm/UserVmManagerImpl.java @@ -4216,7 +4216,12 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir collectVmDiskStatistics(uservm); } _itMgr.migrate(vm.getUuid(), srcHostId, dest); - return _vmDao.findById(vmId); + VMInstanceVO vmInstance = _vmInstanceDao.findById(vmId); + if (vmInstance.getType().equals(VirtualMachine.Type.User)) { + return _vmDao.findById(vmId); + } else { + return vmInstance; + } } private boolean checkIfHostIsDedicated(HostVO host) { diff --git a/ui/scripts/system.js b/ui/scripts/system.js index 960f1aa488a..72f9e338d5a 100644 --- a/ui/scripts/system.js +++ b/ui/scripts/system.js @@ -2508,7 +2508,7 @@ getUpdatedItem: function(json) { //return json.queryasyncjobresultresponse.jobresult.systemvminstance; //not all properties returned in systemvminstance $.ajax({ - url: createURL("listRouters&id=" + json.queryasyncjobresultresponse.jobresult.systemvminstance.id), + url: createURL("listRouters&id=" + json.queryasyncjobresultresponse.jobresult.systemvm.id), dataType: "json", async: false, success: function(json) { @@ -3030,7 +3030,7 @@ getUpdatedItem: function(json) { //return json.queryasyncjobresultresponse.jobresult.systemvminstance; //not all properties returned in systemvminstance $.ajax({ - url: createURL("listInternalLoadBalancerVMs&id=" + json.queryasyncjobresultresponse.jobresult.systemvminstance.id), + url: createURL("listInternalLoadBalancerVMs&id=" + json.queryasyncjobresultresponse.jobresult.systemvm.id), dataType: "json", async: false, success: function(json) { @@ -3624,7 +3624,7 @@ getUpdatedItem: function(json) { //return json.queryasyncjobresultresponse.jobresult.systemvminstance; //not all properties returned in systemvminstance $.ajax({ - url: createURL("listRouters&id=" + json.queryasyncjobresultresponse.jobresult.systemvminstance.id), + url: createURL("listRouters&id=" + json.queryasyncjobresultresponse.jobresult.systemvm.id), dataType: "json", async: false, success: function(json) { @@ -6788,7 +6788,7 @@ getUpdatedItem: function(json) { //return json.queryasyncjobresultresponse.jobresult.systemvminstance; //not all properties returned in systemvminstance $.ajax({ - url: createURL("listSystemVms&id=" + json.queryasyncjobresultresponse.jobresult.systemvminstance.id), + url: createURL("listSystemVms&id=" + json.queryasyncjobresultresponse.jobresult.systemvm.id), dataType: "json", async: false, success: function(json) { @@ -7941,7 +7941,7 @@ getUpdatedItem: function(json) { //return json.queryasyncjobresultresponse.jobresult.systemvminstance; //not all properties returned in systemvminstance $.ajax({ - url: createURL("listRouters&id=" + json.queryasyncjobresultresponse.jobresult.systemvminstance.id), + url: createURL("listRouters&id=" + json.queryasyncjobresultresponse.jobresult.systemvm.id), dataType: "json", async: false, success: function(json) { @@ -9167,7 +9167,7 @@ getUpdatedItem: function(json) { //return json.queryasyncjobresultresponse.jobresult.systemvminstance; //not all properties returned in systemvminstance $.ajax({ - url: createURL("listSystemVms&id=" + json.queryasyncjobresultresponse.jobresult.systemvminstance.id), + url: createURL("listSystemVms&id=" + json.queryasyncjobresultresponse.jobresult.systemvm.id), dataType: "json", async: false, success: function(json) { From 1e9f066df2fd4f6b27e9866eefd01611d7184e45 Mon Sep 17 00:00:00 2001 From: Hiroaki KAWAI Date: Thu, 30 Jan 2014 20:14:55 +0900 Subject: [PATCH 053/169] Fix mock socket client/server test. Normally a server will bind to 0.0.0.0, which is not an address that a client can specify to connect to. 127.0.0.1 or ::1 will do. --- .../rdpconsole/src/test/java/rdpclient/MockServerTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/console-proxy-rdp/rdpconsole/src/test/java/rdpclient/MockServerTest.java b/services/console-proxy-rdp/rdpconsole/src/test/java/rdpclient/MockServerTest.java index a40cdb8ed79..17bc4bde0c4 100755 --- a/services/console-proxy-rdp/rdpconsole/src/test/java/rdpclient/MockServerTest.java +++ b/services/console-proxy-rdp/rdpconsole/src/test/java/rdpclient/MockServerTest.java @@ -58,7 +58,7 @@ public class MockServerTest extends TestCase { Socket socket = SocketFactory.getDefault().createSocket(); try { - socket.connect(server.getAddress()); + socket.connect(new InetSocketAddress("127.0.0.1", server.getAddress().getPort())); InputStream is = socket.getInputStream(); OutputStream os = socket.getOutputStream(); @@ -127,7 +127,7 @@ public class MockServerTest extends TestCase { Socket socket = SocketFactory.getDefault().createSocket(); try { - InetSocketAddress address = server.getAddress(); + InetSocketAddress address = new InetSocketAddress("127.0.0.1", server.getAddress().getPort()); socket.connect(address); // Send hello data over plain connection From db3dc2ee17c699f1abf42ea2fbe09ec2748caa4a Mon Sep 17 00:00:00 2001 From: Hiroaki KAWAI Date: Thu, 30 Jan 2014 20:24:57 +0900 Subject: [PATCH 054/169] Some rework stratosphere ssp plugin * add missing command entry in commands.properties * migrate httpclient 3.x to 4.x * fix the broken SspClient * add webapp session checking in mock ssp server --- client/tomcatconf/commands.properties.in | 5 + .../cloudstack/network/element/SspClient.java | 278 ++++++++---------- .../network/element/SspElement.java | 10 +- .../stratosphere-ssp/sspmock/sspmock.py | 58 ++-- .../network/element/SspClientTest.java | 75 ++--- 5 files changed, 207 insertions(+), 219 deletions(-) diff --git a/client/tomcatconf/commands.properties.in b/client/tomcatconf/commands.properties.in index 28789c16b2b..add35ddaed1 100644 --- a/client/tomcatconf/commands.properties.in +++ b/client/tomcatconf/commands.properties.in @@ -587,6 +587,11 @@ addBigSwitchVnsDevice=1 deleteBigSwitchVnsDevice=1 listBigSwitchVnsDevices=1 +#### stratosphere ssp commands + +addStratosphereSsp=1 +deleteStratoshereSsp=1 + #### host simulator commands configureSimulator=1 diff --git a/plugins/network-elements/stratosphere-ssp/src/org/apache/cloudstack/network/element/SspClient.java b/plugins/network-elements/stratosphere-ssp/src/org/apache/cloudstack/network/element/SspClient.java index ee4d4558bff..c0db92ccab0 100644 --- a/plugins/network-elements/stratosphere-ssp/src/org/apache/cloudstack/network/element/SspClient.java +++ b/plugins/network-elements/stratosphere-ssp/src/org/apache/cloudstack/network/element/SspClient.java @@ -17,26 +17,33 @@ package org.apache.cloudstack.network.element; import java.io.IOException; +import java.io.InputStreamReader; import java.io.UnsupportedEncodingException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Arrays; -import org.apache.commons.httpclient.HttpClient; -import org.apache.commons.httpclient.HttpConnectionManager; -import org.apache.commons.httpclient.HttpException; -import org.apache.commons.httpclient.HttpMethod; -import org.apache.commons.httpclient.HttpStatus; -import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager; -import org.apache.commons.httpclient.URIException; -import org.apache.commons.httpclient.cookie.CookiePolicy; -import org.apache.commons.httpclient.methods.DeleteMethod; -import org.apache.commons.httpclient.methods.EntityEnclosingMethod; -import org.apache.commons.httpclient.methods.PostMethod; -import org.apache.commons.httpclient.methods.PutMethod; -import org.apache.commons.httpclient.methods.RequestEntity; -import org.apache.commons.httpclient.methods.StringRequestEntity; -import org.apache.commons.httpclient.params.HttpClientParams; +import org.apache.http.HttpResponse; +import org.apache.http.HttpStatus; +import org.apache.http.client.HttpClient; +import org.apache.http.client.entity.UrlEncodedFormEntity; +import org.apache.http.client.methods.HttpDelete; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpPut; +import org.apache.http.client.methods.HttpRequestBase; +import org.apache.http.client.params.ClientPNames; +import org.apache.http.client.params.CookiePolicy; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.http.impl.conn.PoolingClientConnectionManager; +import org.apache.http.message.BasicNameValuePair; +import org.apache.http.params.CoreConnectionPNames; import org.apache.log4j.Logger; import com.google.gson.Gson; +import com.google.gson.JsonIOException; +import com.google.gson.JsonSyntaxException; import com.google.gson.annotations.SerializedName; /** @@ -44,117 +51,75 @@ import com.google.gson.annotations.SerializedName; */ public class SspClient { private static final Logger s_logger = Logger.getLogger(SspClient.class); - private static final HttpConnectionManager s_httpclient_manager = new MultiThreadedHttpConnectionManager(); - private static final HttpClientParams s_httpclient_params = new HttpClientParams(); + private static final HttpClient s_client = new DefaultHttpClient( + new PoolingClientConnectionManager()); static { - s_httpclient_params.setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY); + s_client.getParams() + .setParameter(ClientPNames.COOKIE_POLICY, CookiePolicy.BROWSER_COMPATIBILITY) + .setParameter(CoreConnectionPNames.SO_TIMEOUT, 10000); } private final String apiUrl; private final String username; private final String password; - protected HttpClient client; - protected PostMethod postMethod; - protected DeleteMethod deleteMethod; - protected PutMethod putMethod; - public SspClient(String apiUrl, String username, String password) { super(); this.apiUrl = apiUrl; this.username = username; this.password = password; - client = new HttpClient(s_httpclient_params, s_httpclient_manager); - postMethod = new PostMethod(apiUrl); - deleteMethod = new DeleteMethod(apiUrl); - putMethod = new PutMethod(apiUrl); } - public boolean login(){ - PostMethod method = postMethod; - method.setPath("/ws.v1/login"); // NOTE: /ws.v1/login is correct - method.addParameter("username", username); - method.addParameter("password", password); + protected HttpClient getHttpClient() { // for mock test + return s_client; + } + private HttpResponse innerExecuteMethod(HttpRequestBase req, String path) { try { - client.executeMethod(method); - } catch (HttpException e) { - s_logger.info("Login "+username+" to "+apiUrl+" failed", e); - return false; + URI base = new URI(apiUrl); + req.setURI(new URI(base.getScheme(), base.getUserInfo(), base.getHost(), + base.getPort(), path, null, null)); + } catch (URISyntaxException e) { + s_logger.error("invalid API URL " + apiUrl + " path " + path, e); + return null; + } + HttpResponse res = null; + try { + res = getHttpClient().execute(req); + s_logger.info("ssp api call:" + req + " status=" + res.getStatusLine()); } catch (IOException e) { - s_logger.info("Login "+username+" to "+apiUrl+" failed", e); - return false; - } finally { - method.releaseConnection(); + s_logger.error("ssp api call failed: " + req, e); } - String apiCallPath = null; + return res; + } + + private HttpResponse executeMethod(HttpRequestBase req, String path) { + HttpResponse res = innerExecuteMethod(req, path); + if (res.getStatusLine().getStatusCode() == HttpStatus.SC_UNAUTHORIZED && login()) { + req.reset(); + res = innerExecuteMethod(req, path); + } + return res; + } + + public boolean login() { + HttpPost method = new HttpPost(); try { - apiCallPath = method.getName() + " " + method.getURI().toString(); - } catch (URIException e) { - s_logger.error("method getURI failed", e); + method.setEntity(new UrlEncodedFormEntity(Arrays.asList( + new BasicNameValuePair("username", username), + new BasicNameValuePair("password", password)))); + } catch (UnsupportedEncodingException e) { + s_logger.error("invalid username or password", e); + return false; } - s_logger.info("ssp api call:" + apiCallPath + " user="+username+" status="+method.getStatusLine()); - if(method.getStatusCode() == HttpStatus.SC_OK){ + + HttpResponse res = this.innerExecuteMethod(method, "/ws.v1/login"); + if (res != null && res.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { return true; } return false; } - private String executeMethod(HttpMethod method){ - String apiCallPath = null; - try { - apiCallPath = method.getName() + " " + method.getURI().toString(); - } catch (URIException e) { - s_logger.error("method getURI failed", e); - } - - String response = null; - try { - client.executeMethod(method); - response = method.getResponseBodyAsString(); - } catch (HttpException e) { - s_logger.error("ssp api call failed "+apiCallPath, e); - return null; - } catch (IOException e) { - s_logger.error("ssp api call failed "+apiCallPath, e); - return null; - } finally { - method.releaseConnection(); - } - - if(method.getStatusCode() == HttpStatus.SC_UNAUTHORIZED){ - if(!login()){ - return null; - } - - try { - client.executeMethod(method); - response = method.getResponseBodyAsString(); - } catch (HttpException e) { - s_logger.error("ssp api call failed "+apiCallPath, e); - return null; - } catch (IOException e) { - s_logger.error("ssp api call failed "+apiCallPath, e); - return null; - } finally { - method.releaseConnection(); - } - } - s_logger.info("ssp api call:" + apiCallPath + " user="+username+" status="+method.getStatusLine()); - if(method instanceof EntityEnclosingMethod){ - EntityEnclosingMethod emethod = (EntityEnclosingMethod)method; - RequestEntity reqEntity = emethod.getRequestEntity(); - if(reqEntity instanceof StringRequestEntity){ - StringRequestEntity strReqEntity = (StringRequestEntity)reqEntity; - s_logger.debug("ssp api request body:"+strReqEntity.getContent()); - }else{ - s_logger.debug("ssp api request body:"+emethod.getRequestEntity()); - } - } - s_logger.debug("ssp api response body:" + response); - return response; - } - public class TenantNetwork { public String uuid; public String name; @@ -162,35 +127,36 @@ public class SspClient { public String tenantUuid; } - public TenantNetwork createTenantNetwork(String tenantUuid, String networkName){ + public TenantNetwork createTenantNetwork(String tenantUuid, String networkName) { TenantNetwork req = new TenantNetwork(); req.name = networkName; req.tenantUuid = tenantUuid; - PostMethod method = postMethod; - method.setPath("/ssp.v1/tenant-networks"); - StringRequestEntity entity = null; - try { - entity = new StringRequestEntity(new Gson().toJson(req), "application/json", "UTF-8"); - } catch (UnsupportedEncodingException e) { - s_logger.error("failed creating http request body", e); + HttpPost method = new HttpPost(); + method.setEntity(new StringEntity(new Gson().toJson(req), ContentType.APPLICATION_JSON)); + HttpResponse res = executeMethod(method, "/ssp.v1/tenant-networks"); + if (res == null || res.getStatusLine().getStatusCode() != HttpStatus.SC_CREATED) { return null; } - method.setRequestEntity(entity); - - String response = executeMethod(method); - if(response != null && method.getStatusCode() == HttpStatus.SC_CREATED){ - return new Gson().fromJson(response, TenantNetwork.class); + try { + return new Gson().fromJson(new InputStreamReader(res.getEntity().getContent()), + TenantNetwork.class); + } catch (JsonSyntaxException e) { + s_logger.error("reading response body failed", e); + } catch (JsonIOException e) { + s_logger.error("reading response body failed", e); + } catch (IllegalStateException e) { + s_logger.error("reading response body failed", e); + } catch (IOException e) { + s_logger.error("reading response body failed", e); } return null; } - public boolean deleteTenantNetwork(String tenantNetworkUuid){ - DeleteMethod method = deleteMethod; - method.setPath("/ssp.v1/tenant-networks/"+tenantNetworkUuid); - - executeMethod(method); - if(method.getStatusCode() == HttpStatus.SC_NO_CONTENT){ + public boolean deleteTenantNetwork(String tenantNetworkUuid) { + HttpDelete method = new HttpDelete(); + HttpResponse res = executeMethod(method, "/ssp.v1/tenant-networks/" + tenantNetworkUuid); + if (res != null && res.getStatusLine().getStatusCode() == HttpStatus.SC_NO_CONTENT) { return true; } return false; @@ -209,63 +175,69 @@ public class SspClient { public Integer vlanId; } - public TenantPort createTenantPort(String tenantNetworkUuid){ + public TenantPort createTenantPort(String tenantNetworkUuid) { TenantPort req = new TenantPort(); req.networkUuid = tenantNetworkUuid; req.attachmentType = "NoAttachment"; - PostMethod method = postMethod; - method.setPath("/ssp.v1/tenant-ports"); - StringRequestEntity entity = null; - try { - entity = new StringRequestEntity(new Gson().toJson(req), "application/json", "UTF-8"); - } catch (UnsupportedEncodingException e) { - s_logger.error("failed creating http request body", e); + HttpPost method = new HttpPost(); + method.setEntity(new StringEntity(new Gson().toJson(req), ContentType.APPLICATION_JSON)); + HttpResponse res = executeMethod(method, "/ssp.v1/tenant-ports"); + + if (res == null || res.getStatusLine().getStatusCode() != HttpStatus.SC_CREATED) { return null; } - method.setRequestEntity(entity); - - String response = executeMethod(method); - if(response != null && method.getStatusCode() == HttpStatus.SC_CREATED){ - return new Gson().fromJson(response, TenantPort.class); + try { + return new Gson().fromJson(new InputStreamReader(res.getEntity().getContent()), + TenantPort.class); + } catch (JsonSyntaxException e) { + s_logger.error("reading response body failed", e); + } catch (JsonIOException e) { + s_logger.error("reading response body failed", e); + } catch (IllegalStateException e) { + s_logger.error("reading response body failed", e); + } catch (IOException e) { + s_logger.error("reading response body failed", e); } return null; } - public boolean deleteTenantPort(String tenantPortUuid){ - DeleteMethod method = deleteMethod; - method.setPath("/ssp.v1/tenant-ports/"+tenantPortUuid); + public boolean deleteTenantPort(String tenantPortUuid) { + HttpDelete method = new HttpDelete(); + HttpResponse res = executeMethod(method, "/ssp.v1/tenant-ports/" + tenantPortUuid); - executeMethod(method); - if(method.getStatusCode() == HttpStatus.SC_NO_CONTENT){ + if (res != null && res.getStatusLine().getStatusCode() == HttpStatus.SC_NO_CONTENT) { return true; } return false; } - public TenantPort updateTenantVifBinding(String portUuid, String hypervisorIpAddress){ + public TenantPort updateTenantVifBinding(String portUuid, String hypervisorIpAddress) { TenantPort req = new TenantPort(); - if(hypervisorIpAddress != null){ + if (hypervisorIpAddress != null) { req.attachmentType = "VifAttachment"; req.hypervisorIpAddress = hypervisorIpAddress; - }else{ + } else { req.attachmentType = "NoAttachment"; } - PutMethod method = putMethod; - method.setPath("/ssp.v1/tenant-ports/"+portUuid); - StringRequestEntity entity = null; - try { - entity = new StringRequestEntity(new Gson().toJson(req), "application/json", "UTF-8"); - } catch (UnsupportedEncodingException e) { - s_logger.error("failed creating http request body", e); + HttpPut method = new HttpPut(); + method.setEntity(new StringEntity(new Gson().toJson(req), ContentType.APPLICATION_JSON)); + HttpResponse res = executeMethod(method, "/ssp.v1/tenant-ports/" + portUuid); + if (res == null || res.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { return null; } - method.setRequestEntity(entity); - - String response = executeMethod(method); - if(response != null && method.getStatusCode() == HttpStatus.SC_OK){ - return new Gson().fromJson(response, TenantPort.class); + try { + return new Gson().fromJson(new InputStreamReader(res.getEntity().getContent()), + TenantPort.class); + } catch (JsonSyntaxException e) { + s_logger.error("reading response body failed", e); + } catch (JsonIOException e) { + s_logger.error("reading response body failed", e); + } catch (IllegalStateException e) { + s_logger.error("reading response body failed", e); + } catch (IOException e) { + s_logger.error("reading response body failed", e); } return null; } diff --git a/plugins/network-elements/stratosphere-ssp/src/org/apache/cloudstack/network/element/SspElement.java b/plugins/network-elements/stratosphere-ssp/src/org/apache/cloudstack/network/element/SspElement.java index fa0488d870a..0364e6894a4 100644 --- a/plugins/network-elements/stratosphere-ssp/src/org/apache/cloudstack/network/element/SspElement.java +++ b/plugins/network-elements/stratosphere-ssp/src/org/apache/cloudstack/network/element/SspElement.java @@ -189,14 +189,10 @@ public class SspElement extends AdapterBase implements ConnectivityProvider, Ssp public boolean isReady(PhysicalNetworkServiceProvider provider) { PhysicalNetwork physicalNetwork = _physicalNetworkDao.findById(provider.getPhysicalNetworkId()); assert(physicalNetwork!=null); - if(physicalNetwork != null){ - if(fetchSspClients(physicalNetwork.getId(), physicalNetwork.getDataCenterId(), false).size() > 0){ - return true; - } - s_logger.warn("Ssp api endpoint not found. "+physicalNetwork.toString()); - }else{ - s_logger.warn("PhysicalNetwork is NULL."); + if(fetchSspClients(physicalNetwork.getId(), physicalNetwork.getDataCenterId(), false).size() > 0){ + return true; } + s_logger.warn("Ssp api endpoint not found. "+physicalNetwork.toString()); return false; } diff --git a/plugins/network-elements/stratosphere-ssp/sspmock/sspmock.py b/plugins/network-elements/stratosphere-ssp/sspmock/sspmock.py index 5e7e61067c6..9ac646e244e 100644 --- a/plugins/network-elements/stratosphere-ssp/sspmock/sspmock.py +++ b/plugins/network-elements/stratosphere-ssp/sspmock/sspmock.py @@ -17,7 +17,8 @@ import json import uuid -from flask import Flask +from flask import Flask,request,make_response +from beaker.middleware import SessionMiddleware app = Flask(__name__) tenant_networks = [] @@ -25,50 +26,59 @@ tenant_ports = [] @app.route("/ws.v1/login", methods=["POST",]) def login(): - response.content_type = "application/json" - return "" + assert "username" in request.form + assert "password" in request.form + request.environ["beaker.session"]["login"] = True + res = make_response("", 200) + res.headers["Content-type"] = "application/json" + return res @app.route("/ssp.v1/tenant-networks", methods=["POST",]) def create_tenant_network(): - response.content_type = "application/json" - response.status = 201 + if "login" not in request.environ["beaker.session"]: + return make_response("", 401) obj = request.json obj["uuid"] = str(uuid.uuid1()) tenant_networks.append(obj) - return json.dumps(obj) + res = make_response(json.dumps(obj), 201) + res.headers["Content-type"] = "application/json" + return res @app.route("/ssp.v1/tenant-networks/", methods=["DELETE",]) def delete_tenant_network(tenant_net_uuid): + if "login" not in request.environ["beaker.session"]: + return make_response("", 401) for net in tenant_networks: if net["uuid"] == tenant_net_uuid: tenant_networks.remove(net) - response.status = 204 - return "" - response.status = 404 - return "" + return make_response("", 204) + return make_response("", 404) @app.route("/ssp.v1/tenant-ports", methods=["POST",]) def create_tenant_port(): - response.content_type = "application/json" - response.status = 201 + if "login" not in request.environ["beaker.session"]: + return make_response("", 401) obj = request.json obj["uuid"] = str(uuid.uuid1()) tenant_ports.append(obj) - return json.dumps(obj) + res = make_response(json.dumps(obj), 201) + res.headers["Content-type"] = "application/json" + return res @app.route("/ssp.v1/tenant-ports/", methods=["DELETE",]) def delete_tenant_port(tenant_port_uuid): + if "login" not in request.environ["beaker.session"]: + return make_response("", 401) for port in tenant_ports: if port["uuid"] == tenant_port_uuid: tenant_ports.remove(port) - response.status = 204 - return "" - response.status = 404 - return "" + return make_response("", 204) + return make_response("", 404) @app.route("/ssp.v1/tenant-ports/", methods=["PUT",]) def update_tenant_port(tenant_port_uuid): - response.content_type = "application/json" + if "login" not in request.environ["beaker.session"]: + return make_response("", 401) for port in tenant_ports: if port["uuid"] == tenant_port_uuid: obj = request.json @@ -76,10 +86,14 @@ def update_tenant_port(tenant_port_uuid): obj["vlan_id"] = 100 tenant_ports.remove(port) tenant_ports.append(obj) - response.status = 200 - return json.dumps(obj) - response.status = 404 - return "" + res = make_response(json.dumps(obj), 200) + res.headers["Content-type"] = "application/json" + return res + return make_response("", 404) if __name__=="__main__": + app.wsgi_app = SessionMiddleware(app.wsgi_app, { + "session.auto":True, + "session.type":"cookie", + "session.validate_key":"hoge"}) app.run(host="0.0.0.0", port=9080, debug=True) diff --git a/plugins/network-elements/stratosphere-ssp/test/org/apache/cloudstack/network/element/SspClientTest.java b/plugins/network-elements/stratosphere-ssp/test/org/apache/cloudstack/network/element/SspClientTest.java index 795d1c169ab..627cc878f59 100644 --- a/plugins/network-elements/stratosphere-ssp/test/org/apache/cloudstack/network/element/SspClientTest.java +++ b/plugins/network-elements/stratosphere-ssp/test/org/apache/cloudstack/network/element/SspClientTest.java @@ -16,49 +16,41 @@ // under the License. package org.apache.cloudstack.network.element; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.RETURNS_DEEP_STUBS; + +import java.io.ByteArrayInputStream; import java.util.UUID; -import org.apache.cloudstack.network.element.SspClient; -import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.HttpStatus; -import org.apache.commons.httpclient.URI; -import org.apache.commons.httpclient.methods.DeleteMethod; -import org.apache.commons.httpclient.methods.PostMethod; -import org.apache.commons.httpclient.methods.PutMethod; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpUriRequest; import org.junit.Test; -import static org.junit.Assert.*; -import static org.mockito.Mockito.*; - public class SspClientTest { - HttpClient _client = mock(HttpClient.class); - PostMethod _postMethod = mock(PostMethod.class); - PutMethod _putMethod = mock(PutMethod.class); - DeleteMethod _deleteMethod = mock(DeleteMethod.class); - String uuid = UUID.randomUUID().toString(); String apiUrl = "http://a.example.jp/"; String username = "foo"; String password = "bar"; - SspClient sspClient = new SspClient(apiUrl, username, password){ - { - client = _client; - postMethod = _postMethod; - putMethod = _putMethod; - deleteMethod = _deleteMethod; - } - }; - - @SuppressWarnings("deprecation") - private URI getUri() throws Exception{ - return new URI(apiUrl); - } @Test public void loginTest() throws Exception { - when(_postMethod.getURI()).thenReturn(getUri()); - when(_postMethod.getStatusCode()).thenReturn(HttpStatus.SC_OK); + SspClient sspClient = spy(new SspClient(apiUrl, username, password)); + + HttpClient client = mock(HttpClient.class); + HttpResponse res = mock(HttpResponse.class, RETURNS_DEEP_STUBS); + doReturn(client).when(sspClient).getHttpClient(); + when(client.execute(any(HttpUriRequest.class))).thenReturn(res); + when(res.getStatusLine().getStatusCode()).thenReturn(HttpStatus.SC_OK); + assertTrue(sspClient.login()); assertTrue(sspClient.login()); assertTrue(sspClient.login()); @@ -68,13 +60,18 @@ public class SspClientTest { public void createNetworkTest() throws Exception { String networkName = "example network 1"; String tenant_net_uuid = UUID.randomUUID().toString(); + SspClient sspClient = spy(new SspClient(apiUrl, username, password)); + + HttpClient client = mock(HttpClient.class); + HttpResponse res = mock(HttpResponse.class, RETURNS_DEEP_STUBS); + doReturn(client).when(sspClient).getHttpClient(); + when(client.execute(any(HttpUriRequest.class))).thenReturn(res); + when(res.getStatusLine().getStatusCode()).thenReturn(HttpStatus.SC_CREATED); + String body = "{\"uuid\":\"" + tenant_net_uuid + "\",\"name\":\"" + networkName + + "\",\"tenant_uuid\":\"" + uuid + "\"}"; + when(res.getEntity().getContent()).thenReturn( + new ByteArrayInputStream(body.getBytes("UTF-8"))); - when(_postMethod.getURI()).thenReturn(getUri()); - when(_postMethod.getStatusCode()).thenReturn(HttpStatus.SC_CREATED); - when(_postMethod.getResponseBodyAsString()).thenReturn( - "{\"uuid\":\""+tenant_net_uuid+ - "\",\"name\":\""+networkName+ - "\",\"tenant_uuid\":\""+uuid+"\"}"); SspClient.TenantNetwork tnet = sspClient.createTenantNetwork(uuid, networkName); assertEquals(tnet.name, networkName); assertEquals(tnet.uuid, tenant_net_uuid); @@ -84,9 +81,13 @@ public class SspClientTest { @Test public void deleteNetworkTest() throws Exception { String tenant_net_uuid = UUID.randomUUID().toString(); + SspClient sspClient = spy(new SspClient(apiUrl, username, password)); - when(_deleteMethod.getURI()).thenReturn(getUri()); - when(_deleteMethod.getStatusCode()).thenReturn(HttpStatus.SC_NO_CONTENT); + HttpClient client = mock(HttpClient.class); + HttpResponse res = mock(HttpResponse.class, RETURNS_DEEP_STUBS); + doReturn(client).when(sspClient).getHttpClient(); + when(client.execute(any(HttpUriRequest.class))).thenReturn(res); + when(res.getStatusLine().getStatusCode()).thenReturn(HttpStatus.SC_NO_CONTENT); sspClient.deleteTenantNetwork(tenant_net_uuid); } From 775fa0f0d02f182e3cba02faed569272b00e9926 Mon Sep 17 00:00:00 2001 From: Koushik Das Date: Thu, 30 Jan 2014 18:07:24 +0530 Subject: [PATCH 055/169] CLOUDSTACK-5989: Trying to start a vm while 'vm snapshot' is in progress results in inconsistency The fix is to fail the start operation if a vm snapshot is in progress --- .../src/com/cloud/vm/VirtualMachineManagerImpl.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java index d32f144b80b..3405d4449d4 100755 --- a/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java @@ -1529,6 +1529,11 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } protected boolean stateTransitTo(VMInstanceVO vm, VirtualMachine.Event e, Long hostId, String reservationId) throws NoTransitionException { + // if there are active vm snapshots task, state change is not allowed + if (_vmSnapshotMgr.hasActiveVMSnapshotTasks(vm.getId())) { + s_logger.error("State transit with event: " + e + " failed due to: " + vm.getInstanceName() + " has active VM snapshots tasks"); + return false; + } vm.setReservationId(reservationId); return _stateMachine.transitTo(vm, e, new Pair(vm.getHostId(), hostId), _vmDao); } From 16c3f53793517e3c924df5ff6d9dc872dd508731 Mon Sep 17 00:00:00 2001 From: Likitha Shetty Date: Thu, 30 Jan 2014 18:01:28 +0530 Subject: [PATCH 056/169] CLOUDSTACK-5994. Hitting IndexOutOfBoundsException in GetVmStatsCommand after upgrade. To obtain network read/write statistics, multiply sample duration with the average of the particular performance metric obtained over the sample period. --- .../cloud/hypervisor/vmware/resource/VmwareResource.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 7439f12a932..d1fea92c0bd 100755 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -6690,11 +6690,16 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa if(vals.get(vi) instanceof PerfMetricIntSeries) { PerfMetricIntSeries val = (PerfMetricIntSeries)vals.get(vi); List perfValues = val.getValue(); + Long sumRate = 0L; + for (int j = 0; j < infos.size(); j++) { // Size of the array matches the size as the PerfSampleInfo + sumRate += perfValues.get(j); + } + Long averageRate = sumRate / infos.size(); if (vals.get(vi).getId().getCounterId() == rxPerfCounterInfo.getKey()) { - networkReadKBs = sampleDuration * perfValues.get(3); //get the average RX rate multiplied by sampled duration + networkReadKBs = sampleDuration * averageRate; //get the average RX rate multiplied by sampled duration } if (vals.get(vi).getId().getCounterId() == txPerfCounterInfo.getKey()) { - networkWriteKBs = sampleDuration * perfValues.get(3);//get the average TX rate multiplied by sampled duration + networkWriteKBs = sampleDuration * averageRate; //get the average TX rate multiplied by sampled duration } } } From 5651081ac0a32ba88c7446f079a5d3a8dca47feb Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Thu, 30 Jan 2014 14:55:02 +0100 Subject: [PATCH 057/169] Revert "Fix findbug issues within LDAP authenticator" This reverts commit 92b4f66d73562e4211d2d787554ff229dbeb5705. --- .../cloudstack/api/command/LDAPConfigCmd.java | 2 +- .../api/command/LdapCreateAccountCmd.java | 206 +++++++++--------- .../api/command/LdapImportUsersCmd.java | 17 +- .../cloudstack/ldap/LdapContextFactory.java | 2 + .../ldap/LdapAuthenticatorSpec.groovy | 8 +- .../ldap/LdapManagerImplSpec.groovy | 16 +- 6 files changed, 133 insertions(+), 118 deletions(-) diff --git a/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/command/LDAPConfigCmd.java b/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/command/LDAPConfigCmd.java index a4883c51a01..3faf8b768b2 100644 --- a/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/command/LDAPConfigCmd.java +++ b/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/command/LDAPConfigCmd.java @@ -209,7 +209,7 @@ public class LDAPConfigCmd extends BaseCmd { } private boolean updateLDAP() { - _ldapManager.addConfiguration(hostname, port); + LdapConfigurationResponse response = _ldapManager.addConfiguration(hostname, port); /** * There is no query filter now. It is derived from ldap.user.object and ldap.search.group.principle diff --git a/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/command/LdapCreateAccountCmd.java b/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/command/LdapCreateAccountCmd.java index b78b484627f..981e72e64e1 100644 --- a/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/command/LdapCreateAccountCmd.java +++ b/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/command/LdapCreateAccountCmd.java @@ -35,6 +35,7 @@ import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.ldap.LdapManager; import org.apache.cloudstack.ldap.LdapUser; import org.apache.log4j.Logger; +import org.bouncycastle.util.encoders.Base64; import com.cloud.user.Account; import com.cloud.user.AccountService; @@ -42,126 +43,125 @@ import com.cloud.user.UserAccount; @APICommand(name = "ldapCreateAccount", description = "Creates an account from an LDAP user", responseObject = AccountResponse.class, since = "4.2.0") public class LdapCreateAccountCmd extends BaseCmd { - public static final Logger s_logger = Logger - .getLogger(LdapCreateAccountCmd.class.getName()); - private static final String s_name = "createaccountresponse"; + public static final Logger s_logger = Logger + .getLogger(LdapCreateAccountCmd.class.getName()); + private static final String s_name = "createaccountresponse"; - @Inject - private LdapManager _ldapManager; + @Inject + private LdapManager _ldapManager; - @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, description = "Creates the user under the specified account. If no account is specified, the username will be used as the account name.") - private String accountName; + @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, description = "Creates the user under the specified account. If no account is specified, the username will be used as the account name.") + private String accountName; - @Parameter(name = ApiConstants.ACCOUNT_TYPE, type = CommandType.SHORT, required = true, description = "Type of the account. Specify 0 for user, 1 for root admin, and 2 for domain admin") - private Short accountType; + @Parameter(name = ApiConstants.ACCOUNT_TYPE, type = CommandType.SHORT, required = true, description = "Type of the account. Specify 0 for user, 1 for root admin, and 2 for domain admin") + private Short accountType; - @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, entityType = DomainResponse.class, description = "Creates the user under the specified domain.") - private Long domainId; + @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, entityType = DomainResponse.class, description = "Creates the user under the specified domain.") + private Long domainId; - @Parameter(name = ApiConstants.TIMEZONE, type = CommandType.STRING, description = "Specifies a timezone for this command. For more information on the timezone parameter, see Time Zone Format.") - private String timezone; + @Parameter(name = ApiConstants.TIMEZONE, type = CommandType.STRING, description = "Specifies a timezone for this command. For more information on the timezone parameter, see Time Zone Format.") + private String timezone; - @Parameter(name = ApiConstants.USERNAME, type = CommandType.STRING, required = true, description = "Unique username.") - private String username; + @Parameter(name = ApiConstants.USERNAME, type = CommandType.STRING, required = true, description = "Unique username.") + private String username; - @Parameter(name = ApiConstants.NETWORK_DOMAIN, type = CommandType.STRING, description = "Network domain for the account's networks") - private String networkDomain; + @Parameter(name = ApiConstants.NETWORK_DOMAIN, type = CommandType.STRING, description = "Network domain for the account's networks") + private String networkDomain; - @Parameter(name = ApiConstants.ACCOUNT_DETAILS, type = CommandType.MAP, description = "details for account used to store specific parameters") - private Map details; + @Parameter(name = ApiConstants.ACCOUNT_DETAILS, type = CommandType.MAP, description = "details for account used to store specific parameters") + private Map details; - @Parameter(name = ApiConstants.ACCOUNT_ID, type = CommandType.STRING, description = "Account UUID, required for adding account from external provisioning system") - private String accountUUID; + @Parameter(name = ApiConstants.ACCOUNT_ID, type = CommandType.STRING, description = "Account UUID, required for adding account from external provisioning system") + private String accountUUID; - @Parameter(name = ApiConstants.USER_ID, type = CommandType.STRING, description = "User UUID, required for adding account from external provisioning system") - private String userUUID; + @Parameter(name = ApiConstants.USER_ID, type = CommandType.STRING, description = "User UUID, required for adding account from external provisioning system") + private String userUUID; - public LdapCreateAccountCmd() { - super(); - } + public LdapCreateAccountCmd() { + super(); + } - public LdapCreateAccountCmd(final LdapManager ldapManager, - final AccountService accountService) { - super(); - _ldapManager = ldapManager; - _accountService = accountService; - } + public LdapCreateAccountCmd(final LdapManager ldapManager, + final AccountService accountService) { + super(); + _ldapManager = ldapManager; + _accountService = accountService; + } - UserAccount createCloudstackUserAccount(final LdapUser user) { - return _accountService.createUserAccount(username, generatePassword(), - user.getFirstname(), user.getLastname(), user.getEmail(), - timezone, accountName, accountType, domainId, networkDomain, - details, accountUUID, userUUID); - } + UserAccount createCloudstackUserAccount(final LdapUser user) { + return _accountService.createUserAccount(username, generatePassword(), + user.getFirstname(), user.getLastname(), user.getEmail(), + timezone, accountName, accountType, domainId, networkDomain, + details, accountUUID, userUUID); + } - @Override - public void execute() throws ServerApiException { - final CallContext callContext = getCurrentContext(); - callContext.setEventDetails("Account Name: " + accountName - + ", Domain Id:" + domainId); - try { - final LdapUser user = _ldapManager.getUser(username); - validateUser(user); - final UserAccount userAccount = createCloudstackUserAccount(user); - if (userAccount != null) { - final AccountResponse response = _responseGenerator - .createUserAccountResponse(userAccount); - response.setResponseName(getCommandName()); - setResponseObject(response); - } else { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, - "Failed to create a user account"); - } - } catch (final NamingException e) { - throw new ServerApiException( - ApiErrorCode.RESOURCE_UNAVAILABLE_ERROR, - "No LDAP user exists with the username of " + username); - } - } + @Override + public void execute() throws ServerApiException { + final CallContext callContext = getCurrentContext(); + callContext.setEventDetails("Account Name: " + accountName + + ", Domain Id:" + domainId); + try { + final LdapUser user = _ldapManager.getUser(username); + validateUser(user); + final UserAccount userAccount = createCloudstackUserAccount(user); + if (userAccount != null) { + final AccountResponse response = _responseGenerator + .createUserAccountResponse(userAccount); + response.setResponseName(getCommandName()); + setResponseObject(response); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, + "Failed to create a user account"); + } + } catch (final NamingException e) { + throw new ServerApiException( + ApiErrorCode.RESOURCE_UNAVAILABLE_ERROR, + "No LDAP user exists with the username of " + username); + } + } - private String generatePassword() throws ServerApiException { - final SecureRandom random = new SecureRandom(); - final int length = 20; - final String characters = "abcdefghjkmnpqrstuvwxyzABCDEFGHJKMNPQRSTUVWXYZ23456789!@£$%^&*()_+="; + private String generatePassword() throws ServerApiException { + try { + final SecureRandom randomGen = SecureRandom.getInstance("SHA1PRNG"); + final byte bytes[] = new byte[20]; + randomGen.nextBytes(bytes); + return Base64.encode(bytes).toString(); + } catch (final NoSuchAlgorithmException e) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, + "Failed to generate random password"); + } + } - String password = ""; - for (int i = 0; i < length; i++) { - int index = (int) (random.nextDouble() * characters.length()); - password += characters.charAt(index); - } - return password; - } + @Override + public String getCommandName() { + return s_name; + } - @Override - public String getCommandName() { - return s_name; - } + CallContext getCurrentContext() { + return CallContext.current(); + } - CallContext getCurrentContext() { - return CallContext.current(); - } + @Override + public long getEntityOwnerId() { + return Account.ACCOUNT_ID_SYSTEM; + } - @Override - public long getEntityOwnerId() { - return Account.ACCOUNT_ID_SYSTEM; - } - - private boolean validateUser(final LdapUser user) throws ServerApiException { - if (user.getEmail() == null) { - throw new ServerApiException( - ApiErrorCode.RESOURCE_UNAVAILABLE_ERROR, username - + " has no email address set within LDAP"); - } - if (user.getFirstname() == null) { - throw new ServerApiException( - ApiErrorCode.RESOURCE_UNAVAILABLE_ERROR, username - + " has no firstname set within LDAP"); - } - if (user.getLastname() == null) { - throw new ServerApiException( - ApiErrorCode.RESOURCE_UNAVAILABLE_ERROR, username - + " has no lastname set within LDAP"); - } - return true; - } + private boolean validateUser(final LdapUser user) throws ServerApiException { + if (user.getEmail() == null) { + throw new ServerApiException( + ApiErrorCode.RESOURCE_UNAVAILABLE_ERROR, username + + " has no email address set within LDAP"); + } + if (user.getFirstname() == null) { + throw new ServerApiException( + ApiErrorCode.RESOURCE_UNAVAILABLE_ERROR, username + + " has no firstname set within LDAP"); + } + if (user.getLastname() == null) { + throw new ServerApiException( + ApiErrorCode.RESOURCE_UNAVAILABLE_ERROR, username + + " has no lastname set within LDAP"); + } + return true; + } } \ No newline at end of file diff --git a/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/command/LdapImportUsersCmd.java b/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/command/LdapImportUsersCmd.java index d82276ca18f..1855d5d41c5 100644 --- a/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/command/LdapImportUsersCmd.java +++ b/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/command/LdapImportUsersCmd.java @@ -34,6 +34,7 @@ import org.apache.cloudstack.ldap.LdapUser; import org.apache.cloudstack.ldap.NoLdapUserMatchingQueryException; import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; +import org.bouncycastle.util.encoders.Base64; import com.cloud.domain.Domain; import com.cloud.exception.*; @@ -170,15 +171,13 @@ public class LdapImportUsersCmd extends BaseListCmd { } private String generatePassword() throws ServerApiException { - final SecureRandom random = new SecureRandom(); - final int length = 20; - final String characters = "abcdefghjkmnpqrstuvwxyzABCDEFGHJKMNPQRSTUVWXYZ23456789!@£$%^&*()_+="; - - String password = ""; - for (int i = 0; i < length; i++) { - int index = (int) (random.nextDouble() * characters.length()); - password += characters.charAt(index); + try { + final SecureRandom randomGen = SecureRandom.getInstance("SHA1PRNG"); + final byte bytes[] = new byte[20]; + randomGen.nextBytes(bytes); + return Base64.encode(bytes).toString(); + } catch (final NoSuchAlgorithmException e) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to generate random password"); } - return password; } } diff --git a/plugins/user-authenticators/ldap/src/org/apache/cloudstack/ldap/LdapContextFactory.java b/plugins/user-authenticators/ldap/src/org/apache/cloudstack/ldap/LdapContextFactory.java index c0d5d509a81..ceeed6862fb 100644 --- a/plugins/user-authenticators/ldap/src/org/apache/cloudstack/ldap/LdapContextFactory.java +++ b/plugins/user-authenticators/ldap/src/org/apache/cloudstack/ldap/LdapContextFactory.java @@ -95,6 +95,8 @@ public class LdapContextFactory { environment.put(Context.INITIAL_CONTEXT_FACTORY, factory); environment.put(Context.PROVIDER_URL, url); + environment.put("com.sun.jndi.ldap.read.timeout", "500"); + environment.put("com.sun.jndi.ldap.connect.pool", "true"); enableSSL(environment); setAuthentication(environment, isSystemContext); diff --git a/plugins/user-authenticators/ldap/test/groovy/org/apache/cloudstack/ldap/LdapAuthenticatorSpec.groovy b/plugins/user-authenticators/ldap/test/groovy/org/apache/cloudstack/ldap/LdapAuthenticatorSpec.groovy index 51f8e84559a..416c1330359 100644 --- a/plugins/user-authenticators/ldap/test/groovy/org/apache/cloudstack/ldap/LdapAuthenticatorSpec.groovy +++ b/plugins/user-authenticators/ldap/test/groovy/org/apache/cloudstack/ldap/LdapAuthenticatorSpec.groovy @@ -34,7 +34,7 @@ class LdapAuthenticatorSpec extends spock.lang.Specification { when: "A user authentications" def result = ldapAuthenticator.authenticate("rmurphy", "password", 0, null) then: "their authentication fails" - result.first() == false + result == false } def "Test failed authentication due to ldap bind being unsuccessful"() { @@ -51,7 +51,7 @@ class LdapAuthenticatorSpec extends spock.lang.Specification { def result = ldapAuthenticator.authenticate("rmurphy", "password", 0, null) then: "their authentication fails" - result.first() == false + result == false } def "Test failed authentication due to ldap not being configured"() { @@ -66,7 +66,7 @@ class LdapAuthenticatorSpec extends spock.lang.Specification { when: "The user authenticates" def result = ldapAuthenticator.authenticate("rmurphy", "password", 0, null) then: "their authentication fails" - result.first() == false + result == false } def "Test successful authentication"() { @@ -83,7 +83,7 @@ class LdapAuthenticatorSpec extends spock.lang.Specification { def result = ldapAuthenticator.authenticate("rmurphy", "password", 0, null) then: "their authentication passes" - result.first() == true + result == true } def "Test that encode doesn't change the input"() { diff --git a/plugins/user-authenticators/ldap/test/groovy/org/apache/cloudstack/ldap/LdapManagerImplSpec.groovy b/plugins/user-authenticators/ldap/test/groovy/org/apache/cloudstack/ldap/LdapManagerImplSpec.groovy index c0ca2e8a45f..42988e0caef 100644 --- a/plugins/user-authenticators/ldap/test/groovy/org/apache/cloudstack/ldap/LdapManagerImplSpec.groovy +++ b/plugins/user-authenticators/ldap/test/groovy/org/apache/cloudstack/ldap/LdapManagerImplSpec.groovy @@ -297,16 +297,30 @@ class LdapManagerImplSpec extends spock.lang.Specification { thrown InvalidParameterValueException } + def supportedLdapCommands() { + List> cmdList = new ArrayList>(); + cmdList.add(LdapUserSearchCmd.class); + cmdList.add(LdapListUsersCmd.class); + cmdList.add(LdapAddConfigurationCmd.class); + cmdList.add(LdapDeleteConfigurationCmd.class); + cmdList.add(LdapListConfigurationCmd.class); + cmdList.add(LdapCreateAccountCmd.class); + cmdList.add(LdapImportUsersCmd.class); + return cmdList + } + def "Test that getCommands isn't empty"() { given: "We have an LdapConfigurationDao, LdapContextFactory, LdapUserManager and LdapManager" def ldapConfigurationDao = Mock(LdapConfigurationDaoImpl) def ldapContextFactory = Mock(LdapContextFactory) def ldapUserManager = Mock(LdapUserManager) + final List> cmdList = supportedLdapCommands() def ldapManager = new LdapManagerImpl(ldapConfigurationDao, ldapContextFactory, ldapUserManager) when: "Get commands is called" def result = ldapManager.getCommands() - then: "it must contain commands" + then: "it must return all the commands" result.size() > 0 + result == cmdList } def "Testing of listConfigurations"() { From 888b90677623da745a01db0a5087272f0dabe42c Mon Sep 17 00:00:00 2001 From: Jayapal Date: Thu, 30 Jan 2014 19:29:54 +0530 Subject: [PATCH 058/169] CLOUDSTACK-5966 Update the schema-421-430.sql for EnableServiceMonitoring global setting --- setup/db/db/schema-421to430.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/db/db/schema-421to430.sql b/setup/db/db/schema-421to430.sql index 4f8a52e9f3a..316cbe94613 100644 --- a/setup/db/db/schema-421to430.sql +++ b/setup/db/db/schema-421to430.sql @@ -1090,3 +1090,4 @@ CREATE VIEW `cloud`.`user_vm_view` AS `cloud`.`user_vm_details` `custom_ram_size` ON (((`custom_ram_size`.`vm_id` = `cloud`.`vm_instance`.`id`) and (`custom_ram_size`.`name` = 'memory'))); +INSERT IGNORE INTO `cloud`.`configuration`(category, instance, component, name, value, description, default_value) VALUES ('NetworkManager', 'DEFAULT', 'management-server', 'network.router.EnableServiceMonitoring', 'true', 'service monitoring in router enable/disable option, default true', 'true') ON DUPLICATE KEY UPDATE category='NetworkManager'; From ce7c6d23575d455a8273b77702a56b7bb419b37b Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Thu, 30 Jan 2014 15:25:19 +0100 Subject: [PATCH 059/169] findbugs replaced byte[].toString() with Arrays.toString(byte[]) --- .../api/command/LdapCreateAccountCmd.java | 3 ++- .../api/command/LdapImportUsersCmd.java | 17 ++++++++++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/command/LdapCreateAccountCmd.java b/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/command/LdapCreateAccountCmd.java index 981e72e64e1..3afa22b3b5c 100644 --- a/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/command/LdapCreateAccountCmd.java +++ b/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/command/LdapCreateAccountCmd.java @@ -18,6 +18,7 @@ package org.apache.cloudstack.api.command; import java.security.NoSuchAlgorithmException; import java.security.SecureRandom; +import java.util.Arrays; import java.util.Map; import javax.inject.Inject; @@ -125,7 +126,7 @@ public class LdapCreateAccountCmd extends BaseCmd { final SecureRandom randomGen = SecureRandom.getInstance("SHA1PRNG"); final byte bytes[] = new byte[20]; randomGen.nextBytes(bytes); - return Base64.encode(bytes).toString(); + return Arrays.toString(Base64.encode(bytes)); } catch (final NoSuchAlgorithmException e) { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to generate random password"); diff --git a/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/command/LdapImportUsersCmd.java b/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/command/LdapImportUsersCmd.java index 1855d5d41c5..41eacf0e37e 100644 --- a/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/command/LdapImportUsersCmd.java +++ b/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/command/LdapImportUsersCmd.java @@ -19,13 +19,19 @@ package org.apache.cloudstack.api.command; import java.security.NoSuchAlgorithmException; import java.security.SecureRandom; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.UUID; import javax.inject.Inject; -import org.apache.cloudstack.api.*; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseListCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.response.DomainResponse; import org.apache.cloudstack.api.response.LdapUserResponse; import org.apache.cloudstack.api.response.ListResponse; @@ -37,7 +43,12 @@ import org.apache.log4j.Logger; import org.bouncycastle.util.encoders.Base64; import com.cloud.domain.Domain; -import com.cloud.exception.*; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; import com.cloud.user.AccountService; import com.cloud.user.DomainService; @@ -175,7 +186,7 @@ public class LdapImportUsersCmd extends BaseListCmd { final SecureRandom randomGen = SecureRandom.getInstance("SHA1PRNG"); final byte bytes[] = new byte[20]; randomGen.nextBytes(bytes); - return Base64.encode(bytes).toString(); + return Arrays.toString(Base64.encode(bytes)); } catch (final NoSuchAlgorithmException e) { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to generate random password"); } From 54f2234e2191533be21b328534dc20b1871ca3ad Mon Sep 17 00:00:00 2001 From: Alena Prokharchyk Date: Thu, 30 Jan 2014 10:42:52 -0800 Subject: [PATCH 060/169] Fixed API doc that got broken by adding new commands for stratosphere ssp plugin --- tools/apidoc/gen_toc.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tools/apidoc/gen_toc.py b/tools/apidoc/gen_toc.py index 4cfbeb44815..9e4cd31236e 100644 --- a/tools/apidoc/gen_toc.py +++ b/tools/apidoc/gen_toc.py @@ -6,9 +6,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -126,7 +126,7 @@ known_categories = { 'Project': 'Project', 'Lun': 'Storage', 'Pool': 'Pool', - 'VPC': 'VPC', + 'VPC': 'VPC', 'PrivateGateway': 'VPC', 'Simulator': 'simulator', 'StaticRoute': 'VPC', @@ -159,7 +159,8 @@ known_categories = { 'UCS' : 'UCS', 'Ucs' : 'UCS', 'CacheStores' : 'Cache Stores', - 'CacheStore' : 'Cache Store' + 'CacheStore' : 'Cache Store', + 'StratosphereSsp' : ' Stratosphere SSP' } @@ -204,7 +205,7 @@ for f in sys.argv: pass except IndexError, e: print fn - + def xml_for(command): name = command['name'] @@ -219,7 +220,7 @@ def xml_for(command): def write_xml(out, user): with file(out, 'w') as f: cat_strings = [] - + for category in categories.keys(): strings = [] for command in categories[category]: @@ -241,7 +242,7 @@ def write_xml(out, user): %(all_strings)s - + ''' % locals() @@ -267,7 +268,7 @@ def java_for_user(user): for command in categories[category]: if command['user'] == user: strings.append(java_for(command, user)) - func = user_to_func[user] + func = user_to_func[user] all_strings = ''.join(strings) return ''' public void %(func)s() { From 713240baef97ddd62b646ad35c86e1e3ab6702d6 Mon Sep 17 00:00:00 2001 From: Jessica Wang Date: Thu, 30 Jan 2014 14:26:04 -0800 Subject: [PATCH 061/169] CLOUDSTACK-5996: UI - In project view, the first listRotuers API(without projectid=-1) will return the same objects as the second listRouters API(with projectid=-1), because in project view, all API calls are appended with projectid=[projectID]. Therefore, we only call the second listRouters API(with projectid=-1) in non-project view. --- ui/scripts/system.js | 113 +++++++++++++++++++++++++++++-------------- 1 file changed, 78 insertions(+), 35 deletions(-) diff --git a/ui/scripts/system.js b/ui/scripts/system.js index 72f9e338d5a..6b89182ed7e 100644 --- a/ui/scripts/system.js +++ b/ui/scripts/system.js @@ -318,7 +318,6 @@ virtualRouterCount: function(data) { var data2 = { - projectid: -1, page: 1, pagesize: 1 //specifying pagesize as 1 because we don't need any embedded objects to be returned here. The only thing we need from API response is "count" property. }; @@ -327,24 +326,35 @@ data: data2, success: function(json) { var total1 = json.listroutersresponse.count ? json.listroutersresponse.count : 0; + var total2 = 0; //reset + /* + * In project view, the first listRotuers API(without projectid=-1) will return the same objects as the second listRouters API(with projectid=-1), + * because in project view, all API calls are appended with projectid=[projectID]. + * Therefore, we only call the second listRouters API(with projectid=-1) in non-project view. + */ + if (cloudStack.context && cloudStack.context.projects == null) { //non-project view var data3 = { listAll: true, + projectid: -1, page: 1, pagesize: 1 //specifying pagesize as 1 because we don't need any embedded objects to be returned here. The only thing we need from API response is "count" property. }; $.ajax({ url: createURL('listRouters'), data: data3, + async: false, success: function(json) { - var total2 = json.listroutersresponse.count ? json.listroutersresponse.count : 0; + total2 = json.listroutersresponse.count ? json.listroutersresponse.count : 0; + } + }); + } + dataFns.capacity($.extend(data, { virtualRouterCount: (total1 + total2) })); } }); - } - }); }, capacity: function(data) { @@ -2317,10 +2327,16 @@ routers.push(item); }); - // Get project routers + /* + * In project view, the first listRotuers API(without projectid=-1) will return the same objects as the second listRouters API(with projectid=-1), + * because in project view, all API calls are appended with projectid=[projectID]. + * Therefore, we only call the second listRouters API(with projectid=-1) in non-project view. + */ + if (cloudStack.context && cloudStack.context.projects == null) { //non-project view $.ajax({ url: createURL("listRouters&zoneid=" + selectedZoneObj.id + "&listAll=true&page=" + args.page + "&pagesize=" + pageSize + array1.join("") + "&projectid=-1"), data: data2, + async: false, success: function(json) { var items = json.listroutersresponse.router ? json.listroutersresponse.router : []; @@ -2328,14 +2344,16 @@ $(items).map(function(index, item) { routers.push(item); }); + } + }); + } + args.response.success({ actionFilter: routerActionfilter, data: $(routers).map(mapRouterType) }); } }); - } - }); }, detailView: { name: 'Virtual applicance details', @@ -3397,25 +3415,32 @@ routers.push(item); }); - // Get project routers + /* + * In project view, the first listRotuers API(without projectid=-1) will return the same objects as the second listRouters API(with projectid=-1), + * because in project view, all API calls are appended with projectid=[projectID]. + * Therefore, we only call the second listRouters API(with projectid=-1) in non-project view. + */ + if (cloudStack.context && cloudStack.context.projects == null) { //non-project view $.ajax({ url: createURL("listRouters&zoneid=" + selectedZoneObj.id + "&listAll=true&page=" + args.page + "&pagesize=" + pageSize + array1.join("") + "&projectid=-1"), dataType: 'json', data: data2, - async: true, + async: false, success: function(json) { var items = json.listroutersresponse.router; $(items).map(function(index, item) { routers.push(item); }); + } + }); + } + args.response.success({ actionFilter: routerActionfilter, data: $(routers).map(mapRouterType) }); } }); - } - }); }, detailView: { name: 'Virtual applicance details', @@ -7392,32 +7417,41 @@ $(items).map(function(index, item) { routers.push(item); }); - - //get project-owned routers - var toSearchByProjectid = true; + + //if account is specified in advanced search, don't search project-owned routers + var accountIsNotSpecifiedInAdvSearch = true; if (args.filterBy != null) { if (args.filterBy.advSearch != null && typeof(args.filterBy.advSearch) == "object") { //advanced search - if ('account' in args.filterBy.advSearch && args.filterBy.advSearch.account.length > 0) { //if account is specified in advanced search, don't search project-owned routers - toSearchByProjectid = false; //since account and projectid can't be specified together + if ('account' in args.filterBy.advSearch && args.filterBy.advSearch.account.length > 0) { + accountIsNotSpecifiedInAdvSearch = false; //since account and projectid can't be specified together } } } - if (toSearchByProjectid) { - $.ajax({ - url: createURL('listRouters'), - data: $.extend(data,{ - listAll: true, - projectid: -1 - }), - async: false, - success: function(json) { - var items = json.listroutersresponse.router ? json.listroutersresponse.router : []; - $(items).map(function(index, item) { - routers.push(item); - }); - } - }); - } + if (accountIsNotSpecifiedInAdvSearch) { + /* + * In project view, the first listRotuers API(without projectid=-1) will return the same objects as the second listRouters API(with projectid=-1), + * because in project view, all API calls are appended with projectid=[projectID]. + * Therefore, we only call the second listRouters API(with projectid=-1) in non-project view. + */ + if (cloudStack.context && cloudStack.context.projects == null) { //non-project view + $.ajax({ + url: createURL("listRouters&listAll=true&page=" + args.page + "&pagesize=" + pageSize + "&projectid=-1"), + async: false, + success: function(json) { + var items = json.listroutersresponse.router ? json.listroutersresponse.router : []; + $(items).map(function(index, item) { + routers.push(item); + }); + } + }); + + } + } + + args.response.success({ + actionFilter: routerActionfilter, + data: $(routers).map(mapRouterType) + }); } }); @@ -7681,10 +7715,17 @@ $(items).map(function(index, item) { routers.push(item); }); - // Get project routers + + /* + * In project view, the first listRotuers API(without projectid=-1) will return the same objects as the second listRouters API(with projectid=-1), + * because in project view, all API calls are appended with projectid=[projectID]. + * Therefore, we only call the second listRouters API(with projectid=-1) in non-project view. + */ + if (cloudStack.context && cloudStack.context.projects == null) { //non-project view $.ajax({ url: createURL("listRouters&listAll=true&page=" + args.page + "&pagesize=" + pageSize + array1.join("") + "&projectid=-1"), data: data2, + async: false, success: function(json) { var items = json.listroutersresponse.router ? json.listroutersresponse.router : []; @@ -7692,14 +7733,16 @@ $(items).map(function(index, item) { routers.push(item); }); + } + }); + } + args.response.success({ actionFilter: routerActionfilter, data: $(routers).map(mapRouterType) }); } }); - } - }); }, detailView: { name: 'Virtual applicance details', From 1ae02009b562692a7da54f2694a9962f55660aa3 Mon Sep 17 00:00:00 2001 From: Koushik Das Date: Fri, 31 Jan 2014 10:35:19 +0530 Subject: [PATCH 062/169] CLOUDSTACK-5993: Cloud agent fails to start on 32-bit system vms (cpvm and ssvm) created with 4GB RAM offering Capped the max heap size of the JVM at 2600M for 32-bit system VMs. For 64-bit, existing logic works fine. --- systemvm/scripts/_run.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/systemvm/scripts/_run.sh b/systemvm/scripts/_run.sh index bd063468cfb..1a37c7d372c 100755 --- a/systemvm/scripts/_run.sh +++ b/systemvm/scripts/_run.sh @@ -61,4 +61,11 @@ then maxmem=$eightypcnt fi +if [ "$(uname -m | grep '64')" == "" ]; then + let "maxmem32bit=2600" + if [ $maxmem -gt $maxmem32bit ]; then + maxmem=$maxmem32bit + fi +fi + java -Djavax.net.ssl.trustStore=./certs/realhostip.keystore -Dlog.home=$LOGHOME -mx${maxmem}m -cp $CP com.cloud.agent.AgentShell $keyvalues $@ From 1aa1b3e19cfb1e06c35840d6c5bae450d9890a12 Mon Sep 17 00:00:00 2001 From: Marcus Sorensen Date: Thu, 30 Jan 2014 23:33:27 -0700 Subject: [PATCH 063/169] CLOUDSTACK-6000: When expunging fails, move volume state back to Destroy, so that the cleanup thread will try again on next run. --- api/src/com/cloud/storage/Volume.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/com/cloud/storage/Volume.java b/api/src/com/cloud/storage/Volume.java index b1a253f3b91..fd374dce3e3 100755 --- a/api/src/com/cloud/storage/Volume.java +++ b/api/src/com/cloud/storage/Volume.java @@ -82,7 +82,7 @@ public interface Volume extends ControlledEntity, Identity, InternalIdentity, Ba s_fsm.addTransition(Destroy, Event.ExpungingRequested, Expunging); s_fsm.addTransition(Expunging, Event.ExpungingRequested, Expunging); s_fsm.addTransition(Expunging, Event.OperationSucceeded, Expunged); - s_fsm.addTransition(Expunging, Event.OperationFailed, Expunging); + s_fsm.addTransition(Expunging, Event.OperationFailed, Destroy); s_fsm.addTransition(Ready, Event.SnapshotRequested, Snapshotting); s_fsm.addTransition(Snapshotting, Event.OperationSucceeded, Ready); s_fsm.addTransition(Snapshotting, Event.OperationFailed, Ready); From 353060e578cf97f266512b0370ca69dfdbf5461a Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Fri, 31 Jan 2014 08:41:24 +0100 Subject: [PATCH 064/169] Revert "Findbugs: different issues solved" This reverts commit dcda79e2c02fda4e0fe52f61bf61712742e1f0db. --- .../com/cloud/api/query/vo/UserVmJoinVO.java | 23 +++---- .../com/cloud/consoleproxy/AgentHookBase.java | 27 ++++---- .../consoleproxy/ConsoleProxyManagerImpl.java | 6 +- .../src/com/cloud/dc/DedicatedResourceVO.java | 25 ++------ .../cloud/network/IpAddressManagerImpl.java | 64 +++++++++---------- .../com/cloud/network/NetworkModelImpl.java | 11 ++-- .../network/guru/ControlNetworkGuru.java | 8 +-- .../com/cloud/network/vpc/VpcManagerImpl.java | 4 +- .../cloud/template/TemplateManagerImpl.java | 4 +- 9 files changed, 75 insertions(+), 97 deletions(-) diff --git a/server/src/com/cloud/api/query/vo/UserVmJoinVO.java b/server/src/com/cloud/api/query/vo/UserVmJoinVO.java index 2bb0783595e..25079ca59f5 100644 --- a/server/src/com/cloud/api/query/vo/UserVmJoinVO.java +++ b/server/src/com/cloud/api/query/vo/UserVmJoinVO.java @@ -35,7 +35,6 @@ import com.cloud.server.ResourceTag.ResourceObjectType; import com.cloud.storage.Storage.StoragePoolType; import com.cloud.storage.Volume; import com.cloud.utils.db.GenericDao; -import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachine.State; @@ -48,10 +47,10 @@ public class UserVmJoinVO extends BaseViewVO implements ControlledViewEntity { private long id; @Column(name="name", updatable=false, nullable=false, length=255) - private final String name = null; + private String name = null; @Column(name="display_name", updatable=false, nullable=false, length=255) - private final String displayName = null; + private String displayName = null; @Column(name="account_id") private long accountId; @@ -60,7 +59,7 @@ public class UserVmJoinVO extends BaseViewVO implements ControlledViewEntity { private String accountUuid; @Column(name="account_name") - private final String accountName = null; + private String accountName = null; @Column(name="account_type") private short accountType; @@ -72,10 +71,10 @@ public class UserVmJoinVO extends BaseViewVO implements ControlledViewEntity { private String domainUuid; @Column(name="domain_name") - private final String domainName = null; + private String domainName = null; @Column(name="domain_path") - private final String domainPath = null; + private String domainPath = null; @Column(name="instance_group_id") private long instanceGroupId; @@ -97,7 +96,7 @@ public class UserVmJoinVO extends BaseViewVO implements ControlledViewEntity { */ @Enumerated(value=EnumType.STRING) @Column(name="state", updatable=true, nullable=false, length=32) - private final State state = null; + private State state = null; @Column(name=GenericDao.CREATED_COLUMN) private Date created; @@ -150,7 +149,7 @@ public class UserVmJoinVO extends BaseViewVO implements ControlledViewEntity { private String dataCenterUuid; @Column(name="data_center_name") - private final String dataCenterName = null; + private String dataCenterName = null; @Column(name="security_group_enabled") private boolean securityGroupEnabled; @@ -226,7 +225,7 @@ public class UserVmJoinVO extends BaseViewVO implements ControlledViewEntity { private String volume_uuid; @Column(name = "volume_device_id") - private final Long volumeDeviceId = null; + private Long volumeDeviceId = null; @Column(name = "volume_type") @Enumerated(EnumType.STRING) @@ -727,11 +726,9 @@ public class UserVmJoinVO extends BaseViewVO implements ControlledViewEntity { } public String getDetail(String name) { - if (details == null) { - throw new CloudRuntimeException("No details to get. Did you forget to load the details?"); - } + assert (details != null) : "Did you forget to load the details?"; - return details.get(name); + return details != null ? details.get(name) : null; } public String getUserData() { diff --git a/server/src/com/cloud/consoleproxy/AgentHookBase.java b/server/src/com/cloud/consoleproxy/AgentHookBase.java index 660d1f2026f..aa9e4f8a8fb 100644 --- a/server/src/com/cloud/consoleproxy/AgentHookBase.java +++ b/server/src/com/cloud/consoleproxy/AgentHookBase.java @@ -22,11 +22,6 @@ import java.util.Random; import org.apache.log4j.Logger; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; - -import org.apache.cloudstack.framework.config.dao.ConfigurationDao; - import com.cloud.agent.AgentManager; import com.cloud.agent.api.AgentControlAnswer; import com.cloud.agent.api.Answer; @@ -49,10 +44,14 @@ import com.cloud.server.ManagementServer; import com.cloud.servlet.ConsoleProxyPasswordBasedEncryptor; import com.cloud.servlet.ConsoleProxyServlet; import com.cloud.utils.Ternary; -import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VirtualMachine; import com.cloud.vm.dao.VMInstanceDao; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; + +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; + /** * Utility class to manage interactions with agent-based console access * Extracted from ConsoleProxyManagerImpl so that other console proxy managers @@ -73,12 +72,12 @@ public abstract class AgentHookBase implements AgentHook { public AgentHookBase(VMInstanceDao instanceDao, HostDao hostDao, ConfigurationDao cfgDao, KeystoreManager ksMgr, AgentManager agentMgr, ManagementServer ms) { - _instanceDao = instanceDao; - _hostDao = hostDao; - _agentMgr = agentMgr; - _configDao = cfgDao; - _ksMgr = ksMgr; - _ms = ms; + this._instanceDao = instanceDao; + this._hostDao = hostDao; + this._agentMgr = agentMgr; + this._configDao = cfgDao; + this._ksMgr = ksMgr; + this._ms = ms; } @Override @@ -205,9 +204,7 @@ public abstract class AgentHookBase implements AgentHook { assert (ksBits != null); if (ksBits == null) { - String msg = "Could not find and construct a valid SSL certificate"; - s_logger.error(msg); - throw new CloudRuntimeException(msg); + s_logger.error("Could not find and construct a valid SSL certificate"); } cmd = new StartConsoleProxyAgentHttpHandlerCommand(ksBits, storePassword); cmd.setEncryptorPassword(getEncryptorPassword()); diff --git a/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java b/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java index 6a607ddd9da..3572127f20c 100755 --- a/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java +++ b/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java @@ -971,7 +971,11 @@ VirtualMachineGuru, SystemVmLoadScanHandler, ResourceStateAdapter { } } else { if (s_logger.isDebugEnabled()) { - s_logger.debug("Zone host is ready, but console proxy template: " + template.getId() + " is not ready on secondary storage."); + if (template == null) { + s_logger.debug("Zone host is ready, but console proxy template is null"); + } else { + s_logger.debug("Zone host is ready, but console proxy template: " + template.getId() + " is not ready on secondary storage."); + } } } } diff --git a/server/src/com/cloud/dc/DedicatedResourceVO.java b/server/src/com/cloud/dc/DedicatedResourceVO.java index 1f66ecf9e42..9caa6b04487 100644 --- a/server/src/com/cloud/dc/DedicatedResourceVO.java +++ b/server/src/com/cloud/dc/DedicatedResourceVO.java @@ -20,6 +20,7 @@ import java.util.UUID; import javax.persistence.Column; import javax.persistence.Entity; + import javax.persistence.GeneratedValue; import javax.persistence.GenerationType; import javax.persistence.Id; @@ -30,11 +31,6 @@ import javax.persistence.Table; @Table(name="dedicated_resources") public class DedicatedResourceVO implements DedicatedResources{ - /** - * - */ - private static final long serialVersionUID = -6659510127145101917L; - @Id @GeneratedValue(strategy = GenerationType.IDENTITY) @Column(name="id") @@ -65,7 +61,7 @@ public class DedicatedResourceVO implements DedicatedResources{ private long affinityGroupId; public DedicatedResourceVO() { - uuid = UUID.randomUUID().toString(); + this.uuid = UUID.randomUUID().toString(); } public DedicatedResourceVO(Long dataCenterId, Long podId, Long clusterId, Long hostId, Long domainId, @@ -76,16 +72,14 @@ public class DedicatedResourceVO implements DedicatedResources{ this.hostId = hostId; this.domainId = domainId; this.accountId = accountId; - uuid = UUID.randomUUID().toString(); + this.uuid = UUID.randomUUID().toString(); this.affinityGroupId = affinityGroupId; } - @Override public long getId() { return id; } - @Override public Long getDataCenterId() { return dataCenterId; } @@ -94,7 +88,6 @@ public class DedicatedResourceVO implements DedicatedResources{ this.dataCenterId = dataCenterId; } - @Override public Long getPodId() { return podId; } @@ -103,7 +96,6 @@ public class DedicatedResourceVO implements DedicatedResources{ this.podId = podId; } - @Override public Long getClusterId() { return clusterId; } @@ -112,7 +104,6 @@ public class DedicatedResourceVO implements DedicatedResources{ this.clusterId = clusterId; } - @Override public Long getHostId() { return hostId; } @@ -122,10 +113,9 @@ public class DedicatedResourceVO implements DedicatedResources{ } public DedicatedResourceVO(long dedicatedResourceId) { - id = dedicatedResourceId; + this.id = dedicatedResourceId; } - @Override public Long getDomainId() { return domainId; } @@ -134,7 +124,6 @@ public class DedicatedResourceVO implements DedicatedResources{ this.domainId = domainId; } - @Override public Long getAccountId() { return accountId; } @@ -143,16 +132,14 @@ public class DedicatedResourceVO implements DedicatedResources{ this.accountId = accountId; } - @Override public String getUuid() { - return uuid; + return this.uuid; } public void setUuid(String uuid) { this.uuid = uuid; } - @Override public long getAffinityGroupId() { return affinityGroupId; } @@ -160,7 +147,7 @@ public class DedicatedResourceVO implements DedicatedResources{ @Override public boolean equals(Object obj) { if (obj instanceof DedicatedResourceVO) { - return ((DedicatedResourceVO) obj).getId() == getId(); + return ((DedicatedResourceVO) obj).getId() == this.getId(); } else { return false; } diff --git a/server/src/com/cloud/network/IpAddressManagerImpl.java b/server/src/com/cloud/network/IpAddressManagerImpl.java index 1ccd0d604a2..f4bff584213 100644 --- a/server/src/com/cloud/network/IpAddressManagerImpl.java +++ b/server/src/com/cloud/network/IpAddressManagerImpl.java @@ -17,7 +17,6 @@ package com.cloud.network; import java.util.ArrayList; -import java.util.Arrays; import java.util.Date; import java.util.HashMap; import java.util.HashSet; @@ -30,7 +29,6 @@ import java.util.UUID; import javax.inject.Inject; import org.apache.log4j.Logger; - import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.acl.SecurityChecker.AccessType; import org.apache.cloudstack.context.CallContext; @@ -140,14 +138,14 @@ import com.cloud.utils.db.DB; import com.cloud.utils.db.EntityManager; import com.cloud.utils.db.Filter; import com.cloud.utils.db.GlobalLock; +import com.cloud.utils.db.TransactionCallback; +import com.cloud.utils.db.TransactionCallbackNoReturn; +import com.cloud.utils.db.TransactionCallbackWithException; import com.cloud.utils.db.JoinBuilder.JoinType; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.SearchCriteria.Op; import com.cloud.utils.db.Transaction; -import com.cloud.utils.db.TransactionCallback; -import com.cloud.utils.db.TransactionCallbackNoReturn; -import com.cloud.utils.db.TransactionCallbackWithException; import com.cloud.utils.db.TransactionCallbackWithExceptionNoReturn; import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; @@ -580,9 +578,8 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage IPAddressVO ip = markIpAsUnavailable(addrId); + assert (ip != null) : "Unable to mark the ip address id=" + addrId + " as unavailable."; if (ip == null) { - String msg = "Unable to mark the ip address id=" + addrId + " as unavailable."; - s_logger.error(msg); return true; } @@ -695,10 +692,10 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage if (dedicatedVlanDbIds != null && !dedicatedVlanDbIds.isEmpty()) { fetchFromDedicatedRange = true; sc.setParameters("vlanId", dedicatedVlanDbIds.toArray()); - errorMessage.append(", vlanId id=" + Arrays.toString(dedicatedVlanDbIds.toArray())); + errorMessage.append(", vlanId id=" + dedicatedVlanDbIds.toArray()); } else if (nonDedicatedVlanDbIds != null && !nonDedicatedVlanDbIds.isEmpty()) { sc.setParameters("vlanId", nonDedicatedVlanDbIds.toArray()); - errorMessage.append(", vlanId id=" + Arrays.toString(nonDedicatedVlanDbIds.toArray())); + errorMessage.append(", vlanId id=" + nonDedicatedVlanDbIds.toArray()); } else { if (podId != null) { InsufficientAddressCapacityException ex = new InsufficientAddressCapacityException("Insufficient address capacity", Pod.class, podId); @@ -738,7 +735,7 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage if (useSystemIps && nonDedicatedVlanDbIds != null && !nonDedicatedVlanDbIds.isEmpty()) { fetchFromDedicatedRange = false; sc.setParameters("vlanId", nonDedicatedVlanDbIds.toArray()); - errorMessage.append(", vlanId id=" + Arrays.toString(nonDedicatedVlanDbIds.toArray())); + errorMessage.append(", vlanId id=" + nonDedicatedVlanDbIds.toArray()); addrs = _ipAddressDao.lockRows(sc, filter, true); } } @@ -871,11 +868,6 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage public PublicIp assignDedicateIpAddress(Account owner, final Long guestNtwkId, final Long vpcId, final long dcId, final boolean isSourceNat) throws ConcurrentOperationException, InsufficientAddressCapacityException { - if (owner == null) { - s_logger.error("No account to assign an ip to."); - return null; - } - final long ownerId = owner.getId(); PublicIp ip = null; @@ -907,11 +899,13 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage return ip; } finally { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Releasing lock account " + ownerId); - } - _accountDao.releaseFromLockTable(ownerId); + if (owner != null) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Releasing lock account " + ownerId); + } + _accountDao.releaseFromLockTable(ownerId); + } if (ip == null) { s_logger.error("Unable to get source nat ip address for account " + ownerId); } @@ -1259,7 +1253,7 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage s_logger.debug("Associating ip " + ipToAssoc + " to network " + network); - IPAddressVO ip = ipToAssoc; //_ipAddressDao.findById(ipId); + IPAddressVO ip = _ipAddressDao.findById(ipId); //update ip address with networkId ip.setAssociatedWithNetworkId(networkId); ip.setSourceNat(isSourceNat); @@ -1276,16 +1270,18 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage return ip; } finally { if (!success && releaseOnFailure) { - try { - s_logger.warn("Failed to associate ip address, so releasing ip from the database " + ip); - _ipAddressDao.markAsUnavailable(ip.getId()); - if (!applyIpAssociations(network, true)) { - // if fail to apply ip assciations again, unassign ip address without updating resource - // count and generating usage event as there is no need to keep it in the db - _ipAddressDao.unassignIpAddress(ip.getId()); + if (ip != null) { + try { + s_logger.warn("Failed to associate ip address, so releasing ip from the database " + ip); + _ipAddressDao.markAsUnavailable(ip.getId()); + if (!applyIpAssociations(network, true)) { + // if fail to apply ip assciations again, unassign ip address without updating resource + // count and generating usage event as there is no need to keep it in the db + _ipAddressDao.unassignIpAddress(ip.getId()); + } + } catch (Exception e) { + s_logger.warn("Unable to disassociate ip address for recovery", e); } - } catch (Exception e) { - s_logger.warn("Unable to disassociate ip address for recovery", e); } } } @@ -1366,7 +1362,7 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage PublicIp publicIp = PublicIp.createFromAddrAndVlan(ipToAssoc, _vlanDao.findById(ipToAssoc.getVlanId())); ipList.add(publicIp); Map> ipToServices = _networkModel.getIpToServices(ipList, false, true); - if (ipToServices != null && !ipToServices.isEmpty()) { + if (ipToServices != null & !ipToServices.isEmpty()) { Set services = ipToServices.get(publicIp); if (services != null && !services.isEmpty()) { throw new InvalidParameterValueException("IP " + ipToAssoc + " has services and rules associated in the network " + networkId); @@ -1407,7 +1403,7 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage PublicIp publicIp = PublicIp.createFromAddrAndVlan(ip, _vlanDao.findById(ip.getVlanId())); ipList.add(publicIp); Map> ipToServices = _networkModel.getIpToServices(ipList, false, true); - if (ipToServices != null && !ipToServices.isEmpty()) { + if (ipToServices != null & !ipToServices.isEmpty()) { Set ipServices = ipToServices.get(publicIp); if (ipServices != null && !ipServices.isEmpty()) { return false; @@ -1736,14 +1732,14 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage @Override public boolean applyStaticNats(List staticNats, boolean continueOnError, boolean forRevoke) throws ResourceUnavailableException { + Network network = _networksDao.findById(staticNats.get(0).getNetworkId()); + boolean success = true; + if (staticNats == null || staticNats.size() == 0) { s_logger.debug("There are no static nat rules for the network elements"); return true; } - Network network = _networksDao.findById(staticNats.get(0).getNetworkId()); - boolean success = true; - // get the list of public ip's owned by the network List userIps = _ipAddressDao.listByAssociatedNetwork(network.getId(), null); List publicIps = new ArrayList(); diff --git a/server/src/com/cloud/network/NetworkModelImpl.java b/server/src/com/cloud/network/NetworkModelImpl.java index 7281beb79b2..27b200633e5 100755 --- a/server/src/com/cloud/network/NetworkModelImpl.java +++ b/server/src/com/cloud/network/NetworkModelImpl.java @@ -1024,14 +1024,14 @@ public class NetworkModelImpl extends ManagerBase implements NetworkModel { List map = _ntwkOfferingSrvcDao.listByNetworkOfferingId(networkOfferingId); for (NetworkOfferingServiceMapVO instance : map) { - Service service = Network.Service.getService(instance.getService()); + String service = instance.getService(); Set providers; providers = serviceProviderMap.get(service); if (providers == null) { providers = new HashSet(); } providers.add(Provider.getProvider(instance.getProvider())); - serviceProviderMap.put(service, providers); + serviceProviderMap.put(Service.getService(service), providers); } return serviceProviderMap; @@ -1526,15 +1526,12 @@ public class NetworkModelImpl extends ManagerBase implements NetworkModel { @Override public void checkNetworkPermissions(Account owner, Network network) { - if (network == null) { - throw new CloudRuntimeException("no network to check permissions for."); - } // Perform account permission check if (network.getGuestType() != Network.GuestType.Shared || (network.getGuestType() == Network.GuestType.Shared && network.getAclType() == ACLType.Account)) { AccountVO networkOwner = _accountDao.findById(network.getAccountId()); if(networkOwner == null) - throw new PermissionDeniedException("Unable to use network with id= " + ((NetworkVO)network).getUuid() + ", network does not have an owner"); + throw new PermissionDeniedException("Unable to use network with id= " + ((network != null)? ((NetworkVO)network).getUuid() : "") + ", network does not have an owner"); if(owner.getType() != Account.ACCOUNT_TYPE_PROJECT && networkOwner.getType() == Account.ACCOUNT_TYPE_PROJECT){ if(!_projectAccountDao.canAccessProjectAccount(owner.getAccountId(), network.getAccountId())){ throw new PermissionDeniedException("Unable to use network with id= " + ((network != null)? ((NetworkVO)network).getUuid() : "") + ", permission denied"); @@ -1548,7 +1545,7 @@ public class NetworkModelImpl extends ManagerBase implements NetworkModel { } else { if (!isNetworkAvailableInDomain(network.getId(), owner.getDomainId())) { - throw new PermissionDeniedException("Shared network id=" + ((NetworkVO)network).getUuid() + " is not available in domain id=" + owner.getDomainId()); + throw new PermissionDeniedException("Shared network id=" + ((network != null)? ((NetworkVO)network).getUuid() : "") + " is not available in domain id=" + owner.getDomainId()); } } } diff --git a/server/src/com/cloud/network/guru/ControlNetworkGuru.java b/server/src/com/cloud/network/guru/ControlNetworkGuru.java index a8274d3f4ea..26caa61e09a 100755 --- a/server/src/com/cloud/network/guru/ControlNetworkGuru.java +++ b/server/src/com/cloud/network/guru/ControlNetworkGuru.java @@ -134,7 +134,7 @@ public class ControlNetworkGuru extends PodBasedNetworkGuru implements NetworkGu assert nic.getTrafficType() == TrafficType.Control; // we have to get management/private ip for the control nic for vmware/hyperv due ssh issues. - HypervisorType hType = dest.getHost().getHypervisorType(); + HypervisorType hType = dest.getHost().getHypervisorType(); if ( ( (hType == HypervisorType.VMware) || (hType == HypervisorType.Hyperv) )&& isRouterVm(vm)) { if(dest.getDataCenter().getNetworkType() != NetworkType.Basic) { super.reserve(nic, config, vm, dest, context); @@ -168,7 +168,7 @@ public class ControlNetworkGuru extends PodBasedNetworkGuru implements NetworkGu @Override public boolean release(NicProfile nic, VirtualMachineProfile vm, String reservationId) { assert nic.getTrafficType() == TrafficType.Control; - HypervisorType hType = vm.getHypervisorType(); + HypervisorType hType = vm.getHypervisorType(); if ( ( (hType == HypervisorType.VMware) || (hType == HypervisorType.Hyperv) )&& isRouterVm(vm)) { long dcId = vm.getVirtualMachine().getDataCenterId(); DataCenterVO dcVo = _dcDao.findById(dcId); @@ -218,12 +218,12 @@ public class ControlNetworkGuru extends PodBasedNetworkGuru implements NetworkGu Map dbParams = _configDao.getConfiguration(params); - _cidr = dbParams.get(Config.ControlCidr.toString()); + _cidr = dbParams.get(Config.ControlCidr); if (_cidr == null) { _cidr = "169.254.0.0/16"; } - _gateway = dbParams.get(Config.ControlGateway.toString()); + _gateway = dbParams.get(Config.ControlGateway); if (_gateway == null) { _gateway = NetUtils.getLinkLocalGateway(); } diff --git a/server/src/com/cloud/network/vpc/VpcManagerImpl.java b/server/src/com/cloud/network/vpc/VpcManagerImpl.java index 3ae194f97d9..b9dd729a6b1 100644 --- a/server/src/com/cloud/network/vpc/VpcManagerImpl.java +++ b/server/src/com/cloud/network/vpc/VpcManagerImpl.java @@ -432,14 +432,14 @@ public class VpcManagerImpl extends ManagerBase implements VpcManager, VpcProvis List map = _vpcOffSvcMapDao.listByVpcOffId(vpcOffId); for (VpcOfferingServiceMapVO instance : map) { - Service service = Service.getService(instance.getService()); + String service = instance.getService(); Set providers; providers = serviceProviderMap.get(service); if (providers == null) { providers = new HashSet(); } providers.add(Provider.getProvider(instance.getProvider())); - serviceProviderMap.put(service, providers); + serviceProviderMap.put(Service.getService(service), providers); } return serviceProviderMap; diff --git a/server/src/com/cloud/template/TemplateManagerImpl.java b/server/src/com/cloud/template/TemplateManagerImpl.java index 33ba667b988..681be4af65b 100755 --- a/server/src/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/com/cloud/template/TemplateManagerImpl.java @@ -425,7 +425,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, if (isISO) { desc = Upload.Type.ISO.toString(); } - eventId = (eventId == null ? 0 : eventId); + eventId = eventId == null ? 0 : eventId; if (!_accountMgr.isRootAdmin(caller.getType()) && _disableExtraction) { throw new PermissionDeniedException("Extraction has been disabled by admin"); @@ -1801,7 +1801,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } // Don't allow to modify system template - if (Long.valueOf(1).equals(id)) { + if (id == Long.valueOf(1)) { InvalidParameterValueException ex = new InvalidParameterValueException("Unable to update template/iso of specified id"); ex.addProxyObject(String.valueOf(id), "templateId"); throw ex; From 901cf8ab32e8077e3b64204c89418181fd5ce0b6 Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Fri, 31 Jan 2014 08:53:14 +0100 Subject: [PATCH 065/169] findbugs get proper object from map --- server/src/com/cloud/network/vpc/VpcManagerImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/com/cloud/network/vpc/VpcManagerImpl.java b/server/src/com/cloud/network/vpc/VpcManagerImpl.java index b9dd729a6b1..f7c38d4a460 100644 --- a/server/src/com/cloud/network/vpc/VpcManagerImpl.java +++ b/server/src/com/cloud/network/vpc/VpcManagerImpl.java @@ -432,14 +432,14 @@ public class VpcManagerImpl extends ManagerBase implements VpcManager, VpcProvis List map = _vpcOffSvcMapDao.listByVpcOffId(vpcOffId); for (VpcOfferingServiceMapVO instance : map) { - String service = instance.getService(); + Service service = Service.getService(instance.getService()); Set providers; providers = serviceProviderMap.get(service); if (providers == null) { providers = new HashSet(); } providers.add(Provider.getProvider(instance.getProvider())); - serviceProviderMap.put(Service.getService(service), providers); + serviceProviderMap.put(service, providers); } return serviceProviderMap; From cce4a055ae9bdda1e3f5ae2c95efc9dace75814b Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Fri, 31 Jan 2014 08:58:50 +0100 Subject: [PATCH 066/169] findbugs explicit toString on enum for call get(Object) --- server/src/com/cloud/network/guru/ControlNetworkGuru.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/com/cloud/network/guru/ControlNetworkGuru.java b/server/src/com/cloud/network/guru/ControlNetworkGuru.java index 26caa61e09a..2461b1687e0 100755 --- a/server/src/com/cloud/network/guru/ControlNetworkGuru.java +++ b/server/src/com/cloud/network/guru/ControlNetworkGuru.java @@ -218,12 +218,12 @@ public class ControlNetworkGuru extends PodBasedNetworkGuru implements NetworkGu Map dbParams = _configDao.getConfiguration(params); - _cidr = dbParams.get(Config.ControlCidr); + _cidr = dbParams.get(Config.ControlCidr.toString()); if (_cidr == null) { _cidr = "169.254.0.0/16"; } - _gateway = dbParams.get(Config.ControlGateway); + _gateway = dbParams.get(Config.ControlGateway.toString()); if (_gateway == null) { _gateway = NetUtils.getLinkLocalGateway(); } From ba5b302ae6cf5b1c38f226f5ded0bbaa5fe85248 Mon Sep 17 00:00:00 2001 From: Likitha Shetty Date: Fri, 31 Jan 2014 13:08:53 +0530 Subject: [PATCH 067/169] CLOUDSTACK-6002. [vCenter 5.5] restoreVM fails with ROOT-x-000001.vmdk not found. --- .../src/com/cloud/storage/resource/VmwareStorageProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageProcessor.java index 24cfe537b95..fe568cd8965 100644 --- a/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -363,7 +363,7 @@ public class VmwareStorageProcessor implements StorageProcessor { // restoreVM - move the new ROOT disk into corresponding VM folder String vmInternalCSName = volume.getVmName(); if (dsMo.folderExists(String.format("[%s]", dsMo.getName()), vmInternalCSName)) { - VmwareStorageLayoutHelper.syncVolumeToVmDefaultFolder(dcMo, vmInternalCSName, dsMo, vmdkName); + VmwareStorageLayoutHelper.syncVolumeToVmDefaultFolder(dcMo, vmInternalCSName, dsMo, vmdkFileBaseName); } VolumeObjectTO newVol = new VolumeObjectTO(); From a4f2f0b39fadf3510e15c8955a50a3a4a5d7c694 Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Fri, 31 Jan 2014 09:06:11 +0100 Subject: [PATCH 068/169] findbugs get from map with proper key-type --- server/src/com/cloud/network/NetworkModelImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/com/cloud/network/NetworkModelImpl.java b/server/src/com/cloud/network/NetworkModelImpl.java index 27b200633e5..f64c68a4f38 100755 --- a/server/src/com/cloud/network/NetworkModelImpl.java +++ b/server/src/com/cloud/network/NetworkModelImpl.java @@ -1024,14 +1024,14 @@ public class NetworkModelImpl extends ManagerBase implements NetworkModel { List map = _ntwkOfferingSrvcDao.listByNetworkOfferingId(networkOfferingId); for (NetworkOfferingServiceMapVO instance : map) { - String service = instance.getService(); + Service service = Network.Service.getService(instance.getService()); Set providers; providers = serviceProviderMap.get(service); if (providers == null) { providers = new HashSet(); } providers.add(Provider.getProvider(instance.getProvider())); - serviceProviderMap.put(Service.getService(service), providers); + serviceProviderMap.put(service, providers); } return serviceProviderMap; From f4db8df66fbcb5df3cd9bbf607ede9a4d514fe4a Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Fri, 31 Jan 2014 09:07:55 +0100 Subject: [PATCH 069/169] findbugs Long comparison == -> equals() --- server/src/com/cloud/template/TemplateManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/com/cloud/template/TemplateManagerImpl.java b/server/src/com/cloud/template/TemplateManagerImpl.java index 681be4af65b..3bfe5c8b0cd 100755 --- a/server/src/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/com/cloud/template/TemplateManagerImpl.java @@ -1801,7 +1801,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } // Don't allow to modify system template - if (id == Long.valueOf(1)) { + if (Long.valueOf(1).equals(id)) { InvalidParameterValueException ex = new InvalidParameterValueException("Unable to update template/iso of specified id"); ex.addProxyObject(String.valueOf(id), "templateId"); throw ex; From 62a259ffa9b4daabcbe4f9afe9a78eadebbb7bff Mon Sep 17 00:00:00 2001 From: David Grizzanti Date: Fri, 31 Jan 2014 11:25:59 +0100 Subject: [PATCH 070/169] CLOUDSTACK-5062: Deleting Load Balancing Rule fails when generating usage events are enabled Signed-off-by: Daan Hoogland --- .../cloud/network/lb/LoadBalancingRulesManagerImpl.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/server/src/com/cloud/network/lb/LoadBalancingRulesManagerImpl.java b/server/src/com/cloud/network/lb/LoadBalancingRulesManagerImpl.java index bd58e398473..b830910eba1 100755 --- a/server/src/com/cloud/network/lb/LoadBalancingRulesManagerImpl.java +++ b/server/src/com/cloud/network/lb/LoadBalancingRulesManagerImpl.java @@ -1405,7 +1405,8 @@ public class LoadBalancingRulesManagerImpl extends ManagerBase implements if (generateUsageEvent) { // Generate usage event right after all rules were marked for revoke - UsageEventUtils.publishUsageEvent(EventTypes.EVENT_LOAD_BALANCER_DELETE, lb.getAccountId(), 0, lb.getId(), + Network network = _networkModel.getNetwork(lb.getNetworkId()); + UsageEventUtils.publishUsageEvent(EventTypes.EVENT_LOAD_BALANCER_DELETE, lb.getAccountId(), network.getDataCenterId(), lb.getId(), null, LoadBalancingRule.class.getName(), lb.getUuid()); } @@ -2052,6 +2053,11 @@ public class LoadBalancingRulesManagerImpl extends ManagerBase implements List serviceStates = new ArrayList(); List vmLoadBalancerMappings = null; vmLoadBalancerMappings = _lb2VmMapDao.listByLoadBalancerId(loadBalancerId); + if(vmLoadBalancerMappings == null) { + String msg = "no VM Loadbalancer Mapping found"; + s_logger.error(msg); + throw new CloudRuntimeException(msg); + } Map vmServiceState = new HashMap(vmLoadBalancerMappings.size()); List appliedInstanceIdList = new ArrayList(); From b811bc0945f9d028d814356b98ccd58c05f71793 Mon Sep 17 00:00:00 2001 From: Sanjay Tripathi Date: Fri, 31 Jan 2014 17:03:02 +0530 Subject: [PATCH 071/169] CLOUDSTACK-5566: "Entity com.cloud.event.EventVO$$EnhancerByCGLIB$$9e317420@aeb3d75 and entity com.cloud.event.EventVO$$EnhancerByCGLIB$$9e317420@5a0fd1ec belong to different accounts" message while deleting multiple events . --- server/src/com/cloud/server/ManagementServerImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/com/cloud/server/ManagementServerImpl.java b/server/src/com/cloud/server/ManagementServerImpl.java index 048e1839292..3744cb3e3f0 100755 --- a/server/src/com/cloud/server/ManagementServerImpl.java +++ b/server/src/com/cloud/server/ManagementServerImpl.java @@ -908,7 +908,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe List events = _eventDao.listToArchiveOrDeleteEvents(ids, cmd.getType(), cmd.getStartDate(), cmd.getEndDate(), permittedAccountIds); ControlledEntity[] sameOwnerEvents = events.toArray(new ControlledEntity[events.size()]); - _accountMgr.checkAccess(CallContext.current().getCallingAccount(), null, true, sameOwnerEvents); + _accountMgr.checkAccess(CallContext.current().getCallingAccount(), null, false, sameOwnerEvents); if (ids != null && events.size() < ids.size()) { result = false; @@ -935,7 +935,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe List events = _eventDao.listToArchiveOrDeleteEvents(ids, cmd.getType(), cmd.getStartDate(), cmd.getEndDate(), permittedAccountIds); ControlledEntity[] sameOwnerEvents = events.toArray(new ControlledEntity[events.size()]); - _accountMgr.checkAccess(CallContext.current().getCallingAccount(), null, true, sameOwnerEvents); + _accountMgr.checkAccess(CallContext.current().getCallingAccount(), null, false, sameOwnerEvents); if (ids != null && events.size() < ids.size()) { result = false; From 92abb9439e7a4f18d79e2a48e72a0ee94d84f477 Mon Sep 17 00:00:00 2001 From: Likitha Shetty Date: Mon, 3 Feb 2014 11:34:54 +0530 Subject: [PATCH 072/169] CLOUDSTACK-6007. Restore VM command fails with NPE. If a VM instance is deployed with startVm=false, then calling restoreVm on the instance fails with NPE because CS tries to expunge a volume that has not been created in primary store. --- server/src/com/cloud/vm/UserVmManagerImpl.java | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/server/src/com/cloud/vm/UserVmManagerImpl.java b/server/src/com/cloud/vm/UserVmManagerImpl.java index d9923d238e1..81be19fd876 100755 --- a/server/src/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/com/cloud/vm/UserVmManagerImpl.java @@ -71,6 +71,7 @@ import org.apache.cloudstack.engine.service.api.OrchestrationService; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService.VolumeApiResult; import org.apache.cloudstack.framework.async.AsyncCallFuture; @@ -5113,14 +5114,17 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir _volsDao.detachVolume(root.getId()); volumeMgr.destroyVolume(root); - // For VMware hypervisor since the old root volume is replaced by the new root volume in storage, force expunge old root volume + // For VMware hypervisor since the old root volume is replaced by the new root volume, force expunge old root volume if it has been created in storage if (vm.getHypervisorType() == HypervisorType.VMware) { - s_logger.info("Expunging volume " + root.getId() + " from primary data store"); - AsyncCallFuture future = _volService.expungeVolumeAsync(volFactory.getVolume(root.getId())); - try { - future.get(); - } catch (Exception e) { - s_logger.debug("Failed to expunge volume:" + root.getId(), e); + VolumeInfo volumeInStorage = volFactory.getVolume(root.getId()); + if (volumeInStorage != null) { + s_logger.info("Expunging volume " + root.getId() + " from primary data store"); + AsyncCallFuture future = _volService.expungeVolumeAsync(volFactory.getVolume(root.getId())); + try { + future.get(); + } catch (Exception e) { + s_logger.debug("Failed to expunge volume:" + root.getId(), e); + } } } From 8e96111c5512e67b4fca2315bb73ed457791ce18 Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Mon, 3 Feb 2014 10:47:19 +0100 Subject: [PATCH 073/169] Revert "findbugs replaced byte[].toString() with Arrays.toString(byte[])(cherry picked from commit ce7c6d23575d455a8273b77702a56b7bb419b37b)" This reverts commit 55ec8f3a2d8ae73ae1e20a806a8f8d0ee29fa274. --- .../api/command/LdapCreateAccountCmd.java | 3 +-- .../api/command/LdapImportUsersCmd.java | 17 +++-------------- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/command/LdapCreateAccountCmd.java b/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/command/LdapCreateAccountCmd.java index 3afa22b3b5c..981e72e64e1 100644 --- a/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/command/LdapCreateAccountCmd.java +++ b/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/command/LdapCreateAccountCmd.java @@ -18,7 +18,6 @@ package org.apache.cloudstack.api.command; import java.security.NoSuchAlgorithmException; import java.security.SecureRandom; -import java.util.Arrays; import java.util.Map; import javax.inject.Inject; @@ -126,7 +125,7 @@ public class LdapCreateAccountCmd extends BaseCmd { final SecureRandom randomGen = SecureRandom.getInstance("SHA1PRNG"); final byte bytes[] = new byte[20]; randomGen.nextBytes(bytes); - return Arrays.toString(Base64.encode(bytes)); + return Base64.encode(bytes).toString(); } catch (final NoSuchAlgorithmException e) { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to generate random password"); diff --git a/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/command/LdapImportUsersCmd.java b/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/command/LdapImportUsersCmd.java index 41eacf0e37e..1855d5d41c5 100644 --- a/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/command/LdapImportUsersCmd.java +++ b/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/command/LdapImportUsersCmd.java @@ -19,19 +19,13 @@ package org.apache.cloudstack.api.command; import java.security.NoSuchAlgorithmException; import java.security.SecureRandom; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.UUID; import javax.inject.Inject; -import org.apache.cloudstack.api.APICommand; -import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.ApiErrorCode; -import org.apache.cloudstack.api.BaseListCmd; -import org.apache.cloudstack.api.Parameter; -import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.*; import org.apache.cloudstack.api.response.DomainResponse; import org.apache.cloudstack.api.response.LdapUserResponse; import org.apache.cloudstack.api.response.ListResponse; @@ -43,12 +37,7 @@ import org.apache.log4j.Logger; import org.bouncycastle.util.encoders.Base64; import com.cloud.domain.Domain; -import com.cloud.exception.ConcurrentOperationException; -import com.cloud.exception.InsufficientCapacityException; -import com.cloud.exception.InvalidParameterValueException; -import com.cloud.exception.NetworkRuleConflictException; -import com.cloud.exception.ResourceAllocationException; -import com.cloud.exception.ResourceUnavailableException; +import com.cloud.exception.*; import com.cloud.user.AccountService; import com.cloud.user.DomainService; @@ -186,7 +175,7 @@ public class LdapImportUsersCmd extends BaseListCmd { final SecureRandom randomGen = SecureRandom.getInstance("SHA1PRNG"); final byte bytes[] = new byte[20]; randomGen.nextBytes(bytes); - return Arrays.toString(Base64.encode(bytes)); + return Base64.encode(bytes).toString(); } catch (final NoSuchAlgorithmException e) { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to generate random password"); } From 2513d6b91e29356e1eeee31e96eca5e00ed09894 Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Sun, 2 Feb 2014 19:50:58 +0100 Subject: [PATCH 074/169] findbugs replaced byte[].toString() with new String(byte[]) --- .../org/apache/cloudstack/api/command/LdapCreateAccountCmd.java | 2 +- .../org/apache/cloudstack/api/command/LdapImportUsersCmd.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/command/LdapCreateAccountCmd.java b/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/command/LdapCreateAccountCmd.java index 981e72e64e1..08784102b2f 100644 --- a/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/command/LdapCreateAccountCmd.java +++ b/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/command/LdapCreateAccountCmd.java @@ -125,7 +125,7 @@ public class LdapCreateAccountCmd extends BaseCmd { final SecureRandom randomGen = SecureRandom.getInstance("SHA1PRNG"); final byte bytes[] = new byte[20]; randomGen.nextBytes(bytes); - return Base64.encode(bytes).toString(); + return new String(Base64.encode(bytes)); } catch (final NoSuchAlgorithmException e) { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to generate random password"); diff --git a/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/command/LdapImportUsersCmd.java b/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/command/LdapImportUsersCmd.java index 1855d5d41c5..5e724e8fa5c 100644 --- a/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/command/LdapImportUsersCmd.java +++ b/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/command/LdapImportUsersCmd.java @@ -175,7 +175,7 @@ public class LdapImportUsersCmd extends BaseListCmd { final SecureRandom randomGen = SecureRandom.getInstance("SHA1PRNG"); final byte bytes[] = new byte[20]; randomGen.nextBytes(bytes); - return Base64.encode(bytes).toString(); + return new String(Base64.encode(bytes)); } catch (final NoSuchAlgorithmException e) { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to generate random password"); } From 402254337c8f29e5bdef4996953284d36043f8ea Mon Sep 17 00:00:00 2001 From: Min Chen Date: Mon, 3 Feb 2014 12:16:19 -0800 Subject: [PATCH 075/169] CLOUDSTACK-6017: NPE while creating snapshot multiple times on same Root volume with S3 as secondary storage. --- .../apache/cloudstack/storage/snapshot/SnapshotObject.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotObject.java b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotObject.java index 314ce8ee928..a69b2d77f02 100644 --- a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotObject.java +++ b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotObject.java @@ -288,7 +288,10 @@ public class SnapshotObject implements SnapshotInfo { } else if (answer instanceof CopyCmdAnswer) { SnapshotObjectTO snapshotTO = (SnapshotObjectTO) ((CopyCmdAnswer) answer).getNewData(); snapshotStore.setInstallPath(snapshotTO.getPath()); - snapshotStore.setSize(snapshotTO.getPhysicalSize()); + if (snapshotTO.getPhysicalSize() != null) { + // For S3 delta snapshot, physical size is currently not set + snapshotStore.setSize(snapshotTO.getPhysicalSize()); + } if (snapshotTO.getParentSnapshotPath() == null) { snapshotStore.setParentSnapshotId(0L); } From 50234ea72b29b3e192a3d80b8cb8d69e4309d141 Mon Sep 17 00:00:00 2001 From: Sheng Yang Date: Mon, 3 Feb 2014 14:49:31 -0800 Subject: [PATCH 076/169] CLOUDSTACK-6018: Increase dnsmasq dhcp-lease-max to 2100 Would able to accomodate one /21 network then. --- systemvm/patches/debian/config/etc/dnsmasq.conf.tmpl | 2 +- systemvm/patches/debian/config/etc/vpcdnsmasq.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/systemvm/patches/debian/config/etc/dnsmasq.conf.tmpl b/systemvm/patches/debian/config/etc/dnsmasq.conf.tmpl index a3e0bc84856..07c5902e118 100644 --- a/systemvm/patches/debian/config/etc/dnsmasq.conf.tmpl +++ b/systemvm/patches/debian/config/etc/dnsmasq.conf.tmpl @@ -501,7 +501,7 @@ dhcp-option=vendor:MSFT,2,1i #dhcp-boot=/var/ftpd/pxelinux.0,boothost,tftp_server_name # Set the limit on DHCP leases, the default is 150 -#dhcp-lease-max=150 +dhcp-lease-max=2100 # The DHCP server needs somewhere on disk to keep its lease database. # This defaults to a sane location, but if you want to change it, use diff --git a/systemvm/patches/debian/config/etc/vpcdnsmasq.conf b/systemvm/patches/debian/config/etc/vpcdnsmasq.conf index 3622d0e7916..d46d6236950 100644 --- a/systemvm/patches/debian/config/etc/vpcdnsmasq.conf +++ b/systemvm/patches/debian/config/etc/vpcdnsmasq.conf @@ -334,7 +334,7 @@ dhcp-option=vendor:MSFT,2,1i #dhcp-boot=/var/ftpd/pxelinux.0,boothost,192.168.0.3 # Set the limit on DHCP leases, the default is 150 -#dhcp-lease-max=150 +dhcp-lease-max=2100 # The DHCP server needs somewhere on disk to keep its lease database. # This defaults to a sane location, but if you want to change it, use From b11c13d4a120a6361231ee415082576dac6fe385 Mon Sep 17 00:00:00 2001 From: Min Chen Date: Mon, 3 Feb 2014 16:26:40 -0800 Subject: [PATCH 077/169] CLOUDSTACK-6019:CreateVolume from snapshot in NFS is throwing Warning "sync snapshot from cache to region wide store failed for image store test". --- .../resource/VmwareStorageSubsystemCommandHandler.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageSubsystemCommandHandler.java b/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageSubsystemCommandHandler.java index d1b8be8ba70..5eccc3cd314 100644 --- a/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageSubsystemCommandHandler.java +++ b/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageSubsystemCommandHandler.java @@ -95,6 +95,15 @@ public class VmwareStorageSubsystemCommandHandler extends StorageSubsystemComman } else if (srcData.getObjectType() == DataObjectType.TEMPLATE) { // sync template from NFS cache to S3 in NFS migration to S3 case storageManager.createOvaForTemplate((TemplateObjectTO)srcData); + } else if (srcData.getObjectType() == DataObjectType.SNAPSHOT && destData.getObjectType() == DataObjectType.SNAPSHOT) { + // sync snapshot from NFS cache to S3 in NFS migration to S3 case + String parentPath = storageResource.getRootDir(srcDataStore.getUrl()); + SnapshotObjectTO snap = (SnapshotObjectTO)srcData; + String path = snap.getPath(); + int index = path.lastIndexOf(File.separator); + String name = path.substring(index + 1); + String snapDir = path.substring(0, index); + storageManager.createOva(parentPath + File.separator + snapDir, name); } else if (srcData.getObjectType() == DataObjectType.SNAPSHOT && destData.getObjectType() == DataObjectType.TEMPLATE) { //create template from snapshot on src at first, then copy it to s3 TemplateObjectTO cacheTemplate = (TemplateObjectTO)destData; From a3caea7e0aca15c74362bbe8469070228177c96f Mon Sep 17 00:00:00 2001 From: Min Chen Date: Tue, 4 Feb 2014 10:46:30 -0800 Subject: [PATCH 078/169] CLOUDSTACK-6025:CreateTemplate from snapshot in NFS is failing , after migarting to S3 datastore. --- .../storage/snapshot/SnapshotServiceImpl.java | 6 +-- .../VmwareStorageSubsystemCommandHandler.java | 44 ++++++++++--------- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java index 3b456cef069..e967b06d20e 100644 --- a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java +++ b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java @@ -463,9 +463,6 @@ public class SnapshotServiceImpl implements SnapshotService { // push one individual snapshots currently on cache store to region store if it is not there already private void syncSnapshotToRegionStore(long snapshotId, DataStore store){ - if (s_logger.isDebugEnabled()) { - s_logger.debug("sync snapshot " + snapshotId + " from cache to object store..."); - } // if snapshot is already on region wide object store, check if it is really downloaded there (by checking install_path). Sync snapshot to region // wide store if it is not there physically. SnapshotInfo snapOnStore = _snapshotFactory.getSnapshot(snapshotId, store); @@ -473,6 +470,9 @@ public class SnapshotServiceImpl implements SnapshotService { throw new CloudRuntimeException("Cannot find an entry in snapshot_store_ref for snapshot " + snapshotId + " on region store: " + store.getName()); } if (snapOnStore.getPath() == null || snapOnStore.getPath().length() == 0) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("sync snapshot " + snapshotId + " from cache to object store..."); + } // snapshot is not on region store yet, sync to region store SnapshotInfo srcSnapshot = _snapshotFactory.getReadySnapshotOnCache(snapshotId); if (srcSnapshot == null) { diff --git a/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageSubsystemCommandHandler.java b/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageSubsystemCommandHandler.java index 5eccc3cd314..a708add44f2 100644 --- a/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageSubsystemCommandHandler.java +++ b/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageSubsystemCommandHandler.java @@ -95,7 +95,8 @@ public class VmwareStorageSubsystemCommandHandler extends StorageSubsystemComman } else if (srcData.getObjectType() == DataObjectType.TEMPLATE) { // sync template from NFS cache to S3 in NFS migration to S3 case storageManager.createOvaForTemplate((TemplateObjectTO)srcData); - } else if (srcData.getObjectType() == DataObjectType.SNAPSHOT && destData.getObjectType() == DataObjectType.SNAPSHOT) { + } else if (srcData.getObjectType() == DataObjectType.SNAPSHOT) { + // pack ova first // sync snapshot from NFS cache to S3 in NFS migration to S3 case String parentPath = storageResource.getRootDir(srcDataStore.getUrl()); SnapshotObjectTO snap = (SnapshotObjectTO)srcData; @@ -104,27 +105,28 @@ public class VmwareStorageSubsystemCommandHandler extends StorageSubsystemComman String name = path.substring(index + 1); String snapDir = path.substring(0, index); storageManager.createOva(parentPath + File.separator + snapDir, name); - } else if (srcData.getObjectType() == DataObjectType.SNAPSHOT && destData.getObjectType() == DataObjectType.TEMPLATE) { - //create template from snapshot on src at first, then copy it to s3 - TemplateObjectTO cacheTemplate = (TemplateObjectTO)destData; - cacheTemplate.setDataStore(srcDataStore); - CopyCmdAnswer answer = (CopyCmdAnswer)processor.createTemplateFromSnapshot(cmd); - if (!answer.getResult()) { - return answer; + if (destData.getObjectType() == DataObjectType.TEMPLATE) { + //create template from snapshot on src at first, then copy it to s3 + TemplateObjectTO cacheTemplate = (TemplateObjectTO)destData; + cacheTemplate.setDataStore(srcDataStore); + CopyCmdAnswer answer = (CopyCmdAnswer)processor.createTemplateFromSnapshot(cmd); + if (!answer.getResult()) { + return answer; + } + cacheTemplate.setDataStore(destDataStore); + TemplateObjectTO template = (TemplateObjectTO)answer.getNewData(); + template.setDataStore(srcDataStore); + CopyCommand newCmd = new CopyCommand(template, destData, cmd.getWait(), cmd.executeInSequence()); + Answer result = storageResource.defaultAction(newCmd); + //clean up template data on staging area + try { + DeleteCommand deleteCommand = new DeleteCommand(template); + storageResource.defaultAction(deleteCommand); + } catch (Exception e) { + s_logger.debug("Failed to clean up staging area:", e); + } + return result; } - cacheTemplate.setDataStore(destDataStore); - TemplateObjectTO template = (TemplateObjectTO)answer.getNewData(); - template.setDataStore(srcDataStore); - CopyCommand newCmd = new CopyCommand(template, destData, cmd.getWait(), cmd.executeInSequence()); - Answer result = storageResource.defaultAction(newCmd); - //clean up template data on staging area - try { - DeleteCommand deleteCommand = new DeleteCommand(template); - storageResource.defaultAction(deleteCommand); - } catch (Exception e) { - s_logger.debug("Failed to clean up staging area:", e); - } - return result; } needDelegation = true; } From d7ec9ca815609d95c29b9db09ee1720bc7df4816 Mon Sep 17 00:00:00 2001 From: Min Chen Date: Tue, 4 Feb 2014 15:04:11 -0800 Subject: [PATCH 079/169] CLOUDSTACK-6027:CreateVolume from snapshot is failing with S3 on Vmware. --- .../com/cloud/storage/resource/VmwareStorageProcessor.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageProcessor.java index fe568cd8965..6f3f2ef8651 100644 --- a/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -1705,6 +1705,12 @@ public class VmwareStorageProcessor implements StorageProcessor { throw new Exception(msg); } + // strip off the extension since restoreVolumeFromSecStorage internally will append suffix there. + if (backedUpSnapshotUuid.endsWith(".ova")){ + backedUpSnapshotUuid = backedUpSnapshotUuid.replace(".ova", ""); + } else if (backedUpSnapshotUuid.endsWith(".ovf")){ + backedUpSnapshotUuid = backedUpSnapshotUuid.replace(".ovf", ""); + } DatastoreMO primaryDsMo = new DatastoreMO(hyperHost.getContext(), morPrimaryDs); restoreVolumeFromSecStorage(hyperHost, primaryDsMo, newVolumeName, secondaryStorageUrl, backupPath, backedUpSnapshotUuid); From e7ec79b5f34f9f5cbe06653ec1948c7064c5b784 Mon Sep 17 00:00:00 2001 From: Jessica Wang Date: Tue, 4 Feb 2014 16:16:31 -0800 Subject: [PATCH 080/169] CLOUDSTACK-6026: UI > create network offering > pass serviceofferingid parameter instead of systemOfferingForRouter parameter to API. --- ui/scripts/configuration.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ui/scripts/configuration.js b/ui/scripts/configuration.js index 42ae5eede2d..420cb7b2de4 100644 --- a/ui/scripts/configuration.js +++ b/ui/scripts/configuration.js @@ -1716,7 +1716,7 @@ preFilter: function(args) { var $availability = args.$form.find('.form-item[rel=availability]'); var $lbType = args.$form.find('.form-item[rel=lbType]'); - var $systemOfferingForRouter = args.$form.find('.form-item[rel=systemOfferingForRouter]'); + var $serviceofferingid = args.$form.find('.form-item[rel=serviceofferingid]'); var $conservemode = args.$form.find('.form-item[rel=conservemode]'); var $serviceSourceNatRedundantRouterCapabilityCheckbox = args.$form.find('.form-item[rel="service.SourceNat.redundantRouterCapabilityCheckbox"]'); var hasAdvancedZones = false; @@ -1858,9 +1858,9 @@ } }); if (havingVirtualRouterForAtLeastOneService == true) { - $systemOfferingForRouter.css('display', 'inline-block'); + $serviceofferingid.css('display', 'inline-block'); } else { - $systemOfferingForRouter.hide(); + $serviceofferingid.hide(); } @@ -2281,7 +2281,7 @@ }, //show or hide upon checked services and selected providers above (begin) - systemOfferingForRouter: { + serviceofferingid: { label: 'System Offering for Router', isHidden: true, docID: 'helpNetworkOfferingSystemOffering', @@ -2574,8 +2574,8 @@ delete inputData.egresspolicy; } - if (args.$form.find('.form-item[rel=systemOfferingForRouter]').css("display") == "none") - delete inputData.systemOfferingForRouter; + if (args.$form.find('.form-item[rel=serviceofferingid]').css("display") == "none") + delete inputData.serviceofferingid; inputData['traffictype'] = 'GUEST'; //traffic type dropdown has been removed since it has only one option ('Guest'). Hardcode traffic type value here. From 09e27fd7bf92a7ff723e36c8ca485e54ee00e8c7 Mon Sep 17 00:00:00 2001 From: Hiroaki KAWAI Date: Wed, 5 Feb 2014 16:35:18 +0900 Subject: [PATCH 081/169] stratosphere-ssp plugin: Fix HttpClient4 connection leak Replaced HttpClient#execute(HttpUriRequest) with HttpClient#execute(HttpUriRequest,ResponseHandler). The former requires extra EntityUtils#consume(HttpEntity). --- .../cloudstack/network/element/SspClient.java | 110 +++++------------- .../network/element/SspClientTest.java | 20 +--- 2 files changed, 36 insertions(+), 94 deletions(-) diff --git a/plugins/network-elements/stratosphere-ssp/src/org/apache/cloudstack/network/element/SspClient.java b/plugins/network-elements/stratosphere-ssp/src/org/apache/cloudstack/network/element/SspClient.java index c0db92ccab0..30630a35d69 100644 --- a/plugins/network-elements/stratosphere-ssp/src/org/apache/cloudstack/network/element/SspClient.java +++ b/plugins/network-elements/stratosphere-ssp/src/org/apache/cloudstack/network/element/SspClient.java @@ -17,15 +17,15 @@ package org.apache.cloudstack.network.element; import java.io.IOException; -import java.io.InputStreamReader; import java.io.UnsupportedEncodingException; import java.net.URI; import java.net.URISyntaxException; import java.util.Arrays; -import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; +import org.apache.http.client.ClientProtocolException; import org.apache.http.client.HttpClient; +import org.apache.http.client.HttpResponseException; import org.apache.http.client.entity.UrlEncodedFormEntity; import org.apache.http.client.methods.HttpDelete; import org.apache.http.client.methods.HttpPost; @@ -35,6 +35,7 @@ import org.apache.http.client.params.ClientPNames; import org.apache.http.client.params.CookiePolicy; import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.BasicResponseHandler; import org.apache.http.impl.client.DefaultHttpClient; import org.apache.http.impl.conn.PoolingClientConnectionManager; import org.apache.http.message.BasicNameValuePair; @@ -42,8 +43,6 @@ import org.apache.http.params.CoreConnectionPNames; import org.apache.log4j.Logger; import com.google.gson.Gson; -import com.google.gson.JsonIOException; -import com.google.gson.JsonSyntaxException; import com.google.gson.annotations.SerializedName; /** @@ -74,7 +73,7 @@ public class SspClient { return s_client; } - private HttpResponse innerExecuteMethod(HttpRequestBase req, String path) { + private String executeMethod(HttpRequestBase req, String path) { try { URI base = new URI(apiUrl); req.setURI(new URI(base.getScheme(), base.getUserInfo(), base.getHost(), @@ -83,23 +82,26 @@ public class SspClient { s_logger.error("invalid API URL " + apiUrl + " path " + path, e); return null; } - HttpResponse res = null; try { - res = getHttpClient().execute(req); - s_logger.info("ssp api call:" + req + " status=" + res.getStatusLine()); + String content = null; + try { + content = getHttpClient().execute(req, new BasicResponseHandler()); + s_logger.info("ssp api call: " + req); + } catch (HttpResponseException e) { + s_logger.info("ssp api call failed: " + req, e); + if (e.getStatusCode() == HttpStatus.SC_UNAUTHORIZED && login()) { + req.reset(); + content = getHttpClient().execute(req, new BasicResponseHandler()); + s_logger.info("ssp api retry call: " + req); + } + } + return content; + } catch (ClientProtocolException e) { // includes HttpResponseException + s_logger.error("ssp api call failed: " + req, e); } catch (IOException e) { s_logger.error("ssp api call failed: " + req, e); } - return res; - } - - private HttpResponse executeMethod(HttpRequestBase req, String path) { - HttpResponse res = innerExecuteMethod(req, path); - if (res.getStatusLine().getStatusCode() == HttpStatus.SC_UNAUTHORIZED && login()) { - req.reset(); - res = innerExecuteMethod(req, path); - } - return res; + return null; } public boolean login() { @@ -112,9 +114,7 @@ public class SspClient { s_logger.error("invalid username or password", e); return false; } - - HttpResponse res = this.innerExecuteMethod(method, "/ws.v1/login"); - if (res != null && res.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { + if (executeMethod(method, "/ws.v1/login") != null) { return true; } return false; @@ -134,29 +134,14 @@ public class SspClient { HttpPost method = new HttpPost(); method.setEntity(new StringEntity(new Gson().toJson(req), ContentType.APPLICATION_JSON)); - HttpResponse res = executeMethod(method, "/ssp.v1/tenant-networks"); - if (res == null || res.getStatusLine().getStatusCode() != HttpStatus.SC_CREATED) { - return null; - } - try { - return new Gson().fromJson(new InputStreamReader(res.getEntity().getContent()), - TenantNetwork.class); - } catch (JsonSyntaxException e) { - s_logger.error("reading response body failed", e); - } catch (JsonIOException e) { - s_logger.error("reading response body failed", e); - } catch (IllegalStateException e) { - s_logger.error("reading response body failed", e); - } catch (IOException e) { - s_logger.error("reading response body failed", e); - } - return null; + return new Gson().fromJson( + executeMethod(method, "/ssp.v1/tenant-networks"), + TenantNetwork.class); } public boolean deleteTenantNetwork(String tenantNetworkUuid) { HttpDelete method = new HttpDelete(); - HttpResponse res = executeMethod(method, "/ssp.v1/tenant-networks/" + tenantNetworkUuid); - if (res != null && res.getStatusLine().getStatusCode() == HttpStatus.SC_NO_CONTENT) { + if (executeMethod(method, "/ssp.v1/tenant-networks/" + tenantNetworkUuid) != null) { return true; } return false; @@ -182,31 +167,14 @@ public class SspClient { HttpPost method = new HttpPost(); method.setEntity(new StringEntity(new Gson().toJson(req), ContentType.APPLICATION_JSON)); - HttpResponse res = executeMethod(method, "/ssp.v1/tenant-ports"); - - if (res == null || res.getStatusLine().getStatusCode() != HttpStatus.SC_CREATED) { - return null; - } - try { - return new Gson().fromJson(new InputStreamReader(res.getEntity().getContent()), - TenantPort.class); - } catch (JsonSyntaxException e) { - s_logger.error("reading response body failed", e); - } catch (JsonIOException e) { - s_logger.error("reading response body failed", e); - } catch (IllegalStateException e) { - s_logger.error("reading response body failed", e); - } catch (IOException e) { - s_logger.error("reading response body failed", e); - } - return null; + return new Gson().fromJson( + executeMethod(method, "/ssp.v1/tenant-ports"), + TenantPort.class); } public boolean deleteTenantPort(String tenantPortUuid) { HttpDelete method = new HttpDelete(); - HttpResponse res = executeMethod(method, "/ssp.v1/tenant-ports/" + tenantPortUuid); - - if (res != null && res.getStatusLine().getStatusCode() == HttpStatus.SC_NO_CONTENT) { + if (executeMethod(method, "/ssp.v1/tenant-ports/" + tenantPortUuid) != null) { return true; } return false; @@ -223,22 +191,8 @@ public class SspClient { HttpPut method = new HttpPut(); method.setEntity(new StringEntity(new Gson().toJson(req), ContentType.APPLICATION_JSON)); - HttpResponse res = executeMethod(method, "/ssp.v1/tenant-ports/" + portUuid); - if (res == null || res.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { - return null; - } - try { - return new Gson().fromJson(new InputStreamReader(res.getEntity().getContent()), - TenantPort.class); - } catch (JsonSyntaxException e) { - s_logger.error("reading response body failed", e); - } catch (JsonIOException e) { - s_logger.error("reading response body failed", e); - } catch (IllegalStateException e) { - s_logger.error("reading response body failed", e); - } catch (IOException e) { - s_logger.error("reading response body failed", e); - } - return null; + return new Gson().fromJson( + executeMethod(method, "/ssp.v1/tenant-ports/" + portUuid), + TenantPort.class); } } diff --git a/plugins/network-elements/stratosphere-ssp/test/org/apache/cloudstack/network/element/SspClientTest.java b/plugins/network-elements/stratosphere-ssp/test/org/apache/cloudstack/network/element/SspClientTest.java index 627cc878f59..a6b723ccda1 100644 --- a/plugins/network-elements/stratosphere-ssp/test/org/apache/cloudstack/network/element/SspClientTest.java +++ b/plugins/network-elements/stratosphere-ssp/test/org/apache/cloudstack/network/element/SspClientTest.java @@ -23,15 +23,11 @@ import static org.mockito.Mockito.when; import static org.mockito.Mockito.any; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.RETURNS_DEEP_STUBS; - -import java.io.ByteArrayInputStream; import java.util.UUID; -import org.apache.commons.httpclient.HttpStatus; -import org.apache.http.HttpResponse; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.impl.client.BasicResponseHandler; import org.junit.Test; public class SspClientTest { @@ -46,10 +42,8 @@ public class SspClientTest { SspClient sspClient = spy(new SspClient(apiUrl, username, password)); HttpClient client = mock(HttpClient.class); - HttpResponse res = mock(HttpResponse.class, RETURNS_DEEP_STUBS); doReturn(client).when(sspClient).getHttpClient(); - when(client.execute(any(HttpUriRequest.class))).thenReturn(res); - when(res.getStatusLine().getStatusCode()).thenReturn(HttpStatus.SC_OK); + when(client.execute(any(HttpUriRequest.class), any(BasicResponseHandler.class))).thenReturn(""); assertTrue(sspClient.login()); assertTrue(sspClient.login()); @@ -63,14 +57,10 @@ public class SspClientTest { SspClient sspClient = spy(new SspClient(apiUrl, username, password)); HttpClient client = mock(HttpClient.class); - HttpResponse res = mock(HttpResponse.class, RETURNS_DEEP_STUBS); doReturn(client).when(sspClient).getHttpClient(); - when(client.execute(any(HttpUriRequest.class))).thenReturn(res); - when(res.getStatusLine().getStatusCode()).thenReturn(HttpStatus.SC_CREATED); String body = "{\"uuid\":\"" + tenant_net_uuid + "\",\"name\":\"" + networkName + "\",\"tenant_uuid\":\"" + uuid + "\"}"; - when(res.getEntity().getContent()).thenReturn( - new ByteArrayInputStream(body.getBytes("UTF-8"))); + when(client.execute(any(HttpUriRequest.class), any(BasicResponseHandler.class))).thenReturn(body); SspClient.TenantNetwork tnet = sspClient.createTenantNetwork(uuid, networkName); assertEquals(tnet.name, networkName); @@ -84,10 +74,8 @@ public class SspClientTest { SspClient sspClient = spy(new SspClient(apiUrl, username, password)); HttpClient client = mock(HttpClient.class); - HttpResponse res = mock(HttpResponse.class, RETURNS_DEEP_STUBS); doReturn(client).when(sspClient).getHttpClient(); - when(client.execute(any(HttpUriRequest.class))).thenReturn(res); - when(res.getStatusLine().getStatusCode()).thenReturn(HttpStatus.SC_NO_CONTENT); + when(client.execute(any(HttpUriRequest.class), any(BasicResponseHandler.class))).thenReturn(""); sspClient.deleteTenantNetwork(tenant_net_uuid); } From 2e004878b1da0f7fb5ec12e77babbb626e96c1ef Mon Sep 17 00:00:00 2001 From: Murali Reddy Date: Wed, 5 Feb 2014 17:12:58 +0530 Subject: [PATCH 082/169] CLOUDSTACK-5967: GRE tunnel creation is failing after network orchestrator refactor, only network elements providingg services as defined by network offering, are invloved network design and imlement phase. So OVS network element need to be enables as 'Connectivity' service provider to make GRE tunnels work. This fix introduced 'Ovs' provider as Connectivity service provider. --- api/src/com/cloud/network/Network.java | 1 + .../hypervisor/xen/resource/CitrixResourceBase.java | 7 ++++++- .../ovs/src/com/cloud/network/element/OvsElement.java | 7 +++++-- server/src/com/cloud/network/NetworkServiceImpl.java | 10 ++++++++++ 4 files changed, 22 insertions(+), 3 deletions(-) diff --git a/api/src/com/cloud/network/Network.java b/api/src/com/cloud/network/Network.java index a7701dacf5b..b5e8173c99b 100644 --- a/api/src/com/cloud/network/Network.java +++ b/api/src/com/cloud/network/Network.java @@ -130,6 +130,7 @@ public interface Network extends ControlledEntity, StateObject, I public static final Provider NiciraNvp = new Provider("NiciraNvp", false); public static final Provider InternalLbVm = new Provider("InternalLbVm", false); public static final Provider CiscoVnmc = new Provider("CiscoVnmc", true); + public static final Provider Ovs = new Provider("Ovs", false); private final String name; private final boolean isExternal; diff --git a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java index 43c7d8b683f..aa1ef838104 100644 --- a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java +++ b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java @@ -870,7 +870,12 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe dom0vif = VIF.create(conn, vifr); } // At this stage we surely have a VIF - dom0vif.plug(conn); + try { + dom0vif.plug(conn); + } catch (Exception e) { + // though wierd exception is thrown, VIF actually gets plugged-in to dom0, so just ignore exception + s_logger.info("Ignoring the benign error thrown while plugging VIF to dom0"); + } dom0vif.unplug(conn); synchronized(_tmpDom0Vif) { _tmpDom0Vif.add(dom0vif); diff --git a/plugins/network-elements/ovs/src/com/cloud/network/element/OvsElement.java b/plugins/network-elements/ovs/src/com/cloud/network/element/OvsElement.java index 0366143a7e7..6ebc0709e9a 100644 --- a/plugins/network-elements/ovs/src/com/cloud/network/element/OvsElement.java +++ b/plugins/network-elements/ovs/src/com/cloud/network/element/OvsElement.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.network.element; +import java.util.HashMap; import java.util.Map; import java.util.Set; @@ -53,12 +54,14 @@ public class OvsElement extends AdapterBase implements NetworkElement { @Override public Map> getCapabilities() { - return null; + Map> capabilities = new HashMap>(); + capabilities.put(Service.Connectivity, null); + return capabilities; } @Override public Provider getProvider() { - return null; + return Network.Provider.Ovs; } @Override diff --git a/server/src/com/cloud/network/NetworkServiceImpl.java b/server/src/com/cloud/network/NetworkServiceImpl.java index d4310d452cc..270ff7b02cc 100755 --- a/server/src/com/cloud/network/NetworkServiceImpl.java +++ b/server/src/com/cloud/network/NetworkServiceImpl.java @@ -2502,6 +2502,11 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService { //Add Internal Load Balancer element as a default network service provider addDefaultInternalLbProviderToPhysicalNetwork(pNetwork.getId()); + // Add OVS provider as default network service provider + if (pNetwork.getIsolationMethods().contains("GRE")) { + addDefaultOvsToPhysicalNetwork(pNetwork.getId()); + } + return pNetwork; } }); @@ -3796,6 +3801,11 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService { return null; } + private PhysicalNetworkServiceProvider addDefaultOvsToPhysicalNetwork(long physicalNetworkId) { + PhysicalNetworkServiceProvider nsp = addProviderToPhysicalNetwork(physicalNetworkId, Network.Provider.Ovs.getName(), null, null); + return nsp; + } + protected boolean isNetworkSystem(Network network) { NetworkOffering no = _networkOfferingDao.findByIdIncludingRemoved(network.getNetworkOfferingId()); if (no.isSystemOnly()) { From 7df148c762510f669b60edff9d8d690c800b1f95 Mon Sep 17 00:00:00 2001 From: Rajesh Battala Date: Wed, 5 Feb 2014 23:44:14 +0530 Subject: [PATCH 083/169] Fixed agent code to handle untagged public vlan while parsing the vlan id --- .../DotNet/ServerResource/HypervResource/WmiCallsV2.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/WmiCallsV2.cs b/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/WmiCallsV2.cs index 2e3aca59675..f5a04dce847 100644 --- a/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/WmiCallsV2.cs +++ b/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/WmiCallsV2.cs @@ -391,7 +391,7 @@ namespace HypervResource string vlan = null; string isolationUri = nic.isolationUri; string broadcastUri = nic.broadcastUri; - if ( (broadcastUri != null ) || (isolationUri != null && isolationUri.StartsWith("vlan://")) && !isolationUri.Equals("vlan://untagged")) + if ( (broadcastUri != null ) || (isolationUri != null && isolationUri.StartsWith("vlan://"))) { if (broadcastUri != null && broadcastUri.StartsWith("storage")) { @@ -402,7 +402,11 @@ namespace HypervResource vlan = isolationUri.Substring("vlan://".Length); } int tmp; - if (!int.TryParse(vlan, out tmp)) + if (vlan.Equals("untagged", StringComparison.CurrentCultureIgnoreCase) ) { + // recevied vlan is untagged, don't parse for the vlan in the isolation uri + vlan = null; + } + else if (!int.TryParse(vlan, out tmp)) { // TODO: double check exception type errMsg = string.Format("Invalid VLAN value {0} for on vm {1} for nic uuid {2}", isolationUri, vmName, nic.uuid); From e0dfe0ab15ee71c2cdf8fe7cce46b73258120a8e Mon Sep 17 00:00:00 2001 From: Jessica Wang Date: Wed, 5 Feb 2014 14:09:27 -0800 Subject: [PATCH 084/169] CLOUDSTACK-6031: UI > infrastructure > count > pass listAll=true to all listXXXXXXX API for counting resource. --- ui/scripts/system.js | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/ui/scripts/system.js b/ui/scripts/system.js index 6b89182ed7e..743427b2984 100644 --- a/ui/scripts/system.js +++ b/ui/scripts/system.js @@ -203,6 +203,11 @@ zoneCount: function(data) { $.ajax({ url: createURL('listZones'), + data: { + listAll: true, + page: 1, + pagesize: 1 //specifying pagesize as 1 because we don't need any embedded objects to be returned here. The only thing we need from API response is "count" property. + }, success: function(json) { dataFns.podCount($.extend(data, { zoneCount: json.listzonesresponse.count ? json.listzonesresponse.count : 0, @@ -216,6 +221,7 @@ $.ajax({ url: createURL('listPods'), data: { + listAll: true, page: 1, pagesize: 1 //specifying pagesize as 1 because we don't need any embedded objects to be returned here. The only thing we need from API response is "count" property. }, @@ -231,6 +237,7 @@ $.ajax({ url: createURL('listClusters'), data: { + listAll: true, page: 1, pagesize: 1 //specifying pagesize as 1 because we don't need any embedded objects to be returned here. The only thing we need from API response is "count" property. }, @@ -254,6 +261,7 @@ hostCount: function(data) { var data2 = { type: 'routing', + listAll: true, page: 1, pagesize: 1 //specifying pagesize as 1 because we don't need any embedded objects to be returned here. The only thing we need from API response is "count" property. }; @@ -270,6 +278,7 @@ primaryStorageCount: function(data) { var data2 = { + listAll: true, page: 1, pagesize: 1 //specifying pagesize as 1 because we don't need any embedded objects to be returned here. The only thing we need from API response is "count" property. }; @@ -287,6 +296,7 @@ secondaryStorageCount: function(data) { var data2 = { type: 'SecondaryStorage', + listAll: true, page: 1, pagesize: 1 //specifying pagesize as 1 because we don't need any embedded objects to be returned here. The only thing we need from API response is "count" property. }; @@ -305,6 +315,7 @@ $.ajax({ url: createURL('listSystemVms'), data: { + listAll: true, page: 1, pagesize: 1 //specifying pagesize as 1 because we don't need any embedded objects to be returned here. The only thing we need from API response is "count" property. }, @@ -318,6 +329,7 @@ virtualRouterCount: function(data) { var data2 = { + listAll: true, page: 1, pagesize: 1 //specifying pagesize as 1 because we don't need any embedded objects to be returned here. The only thing we need from API response is "count" property. }; @@ -336,7 +348,7 @@ if (cloudStack.context && cloudStack.context.projects == null) { //non-project view var data3 = { listAll: true, - projectid: -1, + projectid: -1, page: 1, pagesize: 1 //specifying pagesize as 1 because we don't need any embedded objects to be returned here. The only thing we need from API response is "count" property. }; @@ -350,11 +362,11 @@ }); } - dataFns.capacity($.extend(data, { - virtualRouterCount: (total1 + total2) - })); - } - }); + dataFns.capacity($.extend(data, { + virtualRouterCount: (total1 + total2) + })); + } + }); }, capacity: function(data) { From 96d8e3c945b24af4d27429c4bd62b459e807e2a1 Mon Sep 17 00:00:00 2001 From: Devdeep Singh Date: Wed, 5 Feb 2014 14:58:12 +0530 Subject: [PATCH 085/169] CLOUDSTACK-6030: Encrypt the primary and secondary smb storage password when it is stored in the db. --- api/src/com/cloud/storage/Storage.java | 5 +- .../storage/datastore/db/ImageStoreVO.java | 10 ++- .../storage/datastore/db/StoragePoolVO.java | 17 ++++-- .../datastore/PrimaryDataStoreHelper.java | 2 +- .../HypervResource/CloudStackTypes.cs | 8 ++- .../HypervResourceController.cs | 8 ++- ...oudStackPrimaryDataStoreLifeCycleImpl.java | 9 ++- utils/src/com/cloud/utils/UriUtils.java | 61 +++++++++++++++++++ 8 files changed, 102 insertions(+), 18 deletions(-) diff --git a/api/src/com/cloud/storage/Storage.java b/api/src/com/cloud/storage/Storage.java index 07b6667e246..54c22c8db81 100755 --- a/api/src/com/cloud/storage/Storage.java +++ b/api/src/com/cloud/storage/Storage.java @@ -95,7 +95,7 @@ public class Storage { public static enum StoragePoolType { Filesystem(false), // local directory - NetworkFilesystem(true), // NFS or CIFS + NetworkFilesystem(true), // NFS IscsiLUN(true), // shared LUN, with a clusterfs overlay Iscsi(true), // for e.g., ZFS Comstar ISO(false), // for iso image @@ -106,7 +106,8 @@ public class Storage { VMFS(true), // VMware VMFS storage PreSetup(true), // for XenServer, Storage Pool is set up by customers. EXT(false), // XenServer local EXT SR - OCFS2(true); + OCFS2(true), + SMB(true); boolean shared; diff --git a/engine/schema/src/org/apache/cloudstack/storage/datastore/db/ImageStoreVO.java b/engine/schema/src/org/apache/cloudstack/storage/datastore/db/ImageStoreVO.java index 5ed48a3781c..97ae9c8c70e 100644 --- a/engine/schema/src/org/apache/cloudstack/storage/datastore/db/ImageStoreVO.java +++ b/engine/schema/src/org/apache/cloudstack/storage/datastore/db/ImageStoreVO.java @@ -31,6 +31,7 @@ import javax.persistence.TableGenerator; import com.cloud.storage.DataStoreRole; import com.cloud.storage.ImageStore; import com.cloud.storage.ScopeType; +import com.cloud.utils.UriUtils; import com.cloud.utils.db.GenericDao; @Entity @@ -144,11 +145,18 @@ public class ImageStoreVO implements ImageStore { } public String getUrl() { - return url; + String updatedUrl = url; + if ("cifs".equalsIgnoreCase(this.protocol)) { + updatedUrl = UriUtils.getUpdateUri(updatedUrl, false); + } + return updatedUrl; } public void setUrl(String url) { this.url = url; + if ("cifs".equalsIgnoreCase(this.protocol)) { + this.url = UriUtils.getUpdateUri(url, true); + } } public Date getCreated() { diff --git a/engine/schema/src/org/apache/cloudstack/storage/datastore/db/StoragePoolVO.java b/engine/schema/src/org/apache/cloudstack/storage/datastore/db/StoragePoolVO.java index 557c96456ab..7d52858cbbd 100644 --- a/engine/schema/src/org/apache/cloudstack/storage/datastore/db/StoragePoolVO.java +++ b/engine/schema/src/org/apache/cloudstack/storage/datastore/db/StoragePoolVO.java @@ -30,10 +30,11 @@ import javax.persistence.Temporal; import javax.persistence.TemporalType; import com.cloud.hypervisor.Hypervisor.HypervisorType; -import com.cloud.storage.Storage.StoragePoolType; import com.cloud.storage.ScopeType; +import com.cloud.storage.Storage.StoragePoolType; import com.cloud.storage.StoragePool; import com.cloud.storage.StoragePoolStatus; +import com.cloud.utils.UriUtils; import com.cloud.utils.db.GenericDao; @Entity @@ -135,10 +136,10 @@ public class StoragePoolVO implements StoragePool { this.usedBytes = availableBytes; this.capacityBytes = capacityBytes; this.hostAddress = hostAddress; - this.path = hostPath; this.port = port; this.podId = podId; this.setStatus(StoragePoolStatus.Initial); + this.setPath(hostPath); } public StoragePoolVO(StoragePoolVO that) { @@ -150,9 +151,9 @@ public class StoragePoolVO implements StoragePool { this.poolType = type; this.hostAddress = hostAddress; this.port = port; - this.path = path; this.setStatus(StoragePoolStatus.Initial); this.uuid = UUID.randomUUID().toString(); + this.setPath(path); } public String getName() { @@ -244,7 +245,12 @@ public class StoragePoolVO implements StoragePool { } public String getPath() { - return path; + String updatedPath = path; + if (this.poolType == StoragePoolType.SMB) { + updatedPath = UriUtils.getUpdateUri(updatedPath, false); + } + + return updatedPath; } public String getUserInfo() { @@ -273,6 +279,9 @@ public class StoragePoolVO implements StoragePool { public void setPath(String path) { this.path = path; + if (this.poolType == StoragePoolType.SMB) { + this.path = UriUtils.getUpdateUri(this.path, true); + } } public void setUserInfo(String userInfo) { diff --git a/engine/storage/src/org/apache/cloudstack/storage/volume/datastore/PrimaryDataStoreHelper.java b/engine/storage/src/org/apache/cloudstack/storage/volume/datastore/PrimaryDataStoreHelper.java index 90ad17aa6d8..95dc40b324d 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/volume/datastore/PrimaryDataStoreHelper.java +++ b/engine/storage/src/org/apache/cloudstack/storage/volume/datastore/PrimaryDataStoreHelper.java @@ -69,8 +69,8 @@ public class PrimaryDataStoreHelper { dataStoreVO = new StoragePoolVO(); dataStoreVO.setStorageProviderName(params.getProviderName()); dataStoreVO.setHostAddress(params.getHost()); - dataStoreVO.setPath(params.getPath()); dataStoreVO.setPoolType(params.getType()); + dataStoreVO.setPath(params.getPath()); dataStoreVO.setPort(params.getPort()); dataStoreVO.setName(params.getName()); dataStoreVO.setUuid(params.getUuid()); diff --git a/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/CloudStackTypes.cs b/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/CloudStackTypes.cs index d54295ccdaa..96379944d9f 100644 --- a/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/CloudStackTypes.cs +++ b/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/CloudStackTypes.cs @@ -61,7 +61,7 @@ namespace HypervResource get { string uncPath = null; - if (uri != null && (uri.Scheme.Equals("cifs") || uri.Scheme.Equals("networkfilesystem"))) + if (uri != null && (uri.Scheme.Equals("cifs") || uri.Scheme.Equals("networkfilesystem") || uri.Scheme.Equals("smb"))) { uncPath = @"\\" + uri.Host + uri.LocalPath; } @@ -584,7 +584,11 @@ namespace HypervResource /// /// /// - OCFS2 + OCFS2, + /// + /// for hyper-v + /// + SMB } public enum StorageResourceType diff --git a/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/HypervResourceController.cs b/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/HypervResourceController.cs index 81ab42a7393..a1326064694 100644 --- a/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/HypervResourceController.cs +++ b/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/HypervResourceController.cs @@ -916,7 +916,8 @@ namespace HypervResource GetCapacityForLocalPath(localPath, out capacityBytes, out availableBytes); hostPath = localPath; } - else if (poolType == StoragePoolType.NetworkFilesystem) + else if (poolType == StoragePoolType.NetworkFilesystem || + poolType == StoragePoolType.SMB) { NFSTO share = new NFSTO(); String uriStr = "cifs://" + (string)cmd.pool.host + (string)cmd.pool.path; @@ -972,7 +973,8 @@ namespace HypervResource } if (poolType != StoragePoolType.Filesystem && - poolType != StoragePoolType.NetworkFilesystem) + poolType != StoragePoolType.NetworkFilesystem && + poolType != StoragePoolType.SMB) { details = "Request to create / modify unsupported pool type: " + (poolTypeStr == null ? "NULL" : poolTypeStr) + "in cmd " + JsonConvert.SerializeObject(cmd); logger.Error(details); @@ -1715,7 +1717,7 @@ namespace HypervResource used = capacity - available; result = true; } - else if (poolType == StoragePoolType.NetworkFilesystem) + else if (poolType == StoragePoolType.NetworkFilesystem || poolType == StoragePoolType.SMB) { string sharePath = config.getPrimaryStorage((string)cmd.id); if (sharePath != null) diff --git a/plugins/storage/volume/default/src/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java b/plugins/storage/volume/default/src/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java index 393391575f6..7a7257a8be7 100644 --- a/plugins/storage/volume/default/src/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java +++ b/plugins/storage/volume/default/src/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java @@ -28,7 +28,6 @@ import java.util.UUID; import javax.inject.Inject; import org.apache.log4j.Logger; - import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; @@ -206,12 +205,11 @@ public class CloudStackPrimaryDataStoreLifeCycleImpl implements PrimaryDataStore if (port == -1) { port = 445; } - parameters.setType(StoragePoolType.NetworkFilesystem); + + parameters.setType(StoragePoolType.SMB); parameters.setHost(storageHost); parameters.setPort(port); parameters.setPath(hostPath); - parameters.setUserInfo(uri.getQuery()); - } else if (scheme.equalsIgnoreCase("file")) { if (port == -1) { port = 0; @@ -356,7 +354,8 @@ public class CloudStackPrimaryDataStoreLifeCycleImpl implements PrimaryDataStore && pool.getPoolType() != StoragePoolType.IscsiLUN && pool.getPoolType() != StoragePoolType.Iscsi && pool.getPoolType() != StoragePoolType.VMFS && pool.getPoolType() != StoragePoolType.SharedMountPoint && pool.getPoolType() != StoragePoolType.PreSetup && pool.getPoolType() != StoragePoolType.OCFS2 - && pool.getPoolType() != StoragePoolType.RBD && pool.getPoolType() != StoragePoolType.CLVM) { + && pool.getPoolType() != StoragePoolType.RBD && pool.getPoolType() != StoragePoolType.CLVM + && pool.getPoolType() != StoragePoolType.SMB) { s_logger.warn(" Doesn't support storage pool type " + pool.getPoolType()); return false; } diff --git a/utils/src/com/cloud/utils/UriUtils.java b/utils/src/com/cloud/utils/UriUtils.java index bf967071910..9261b075a0c 100644 --- a/utils/src/com/cloud/utils/UriUtils.java +++ b/utils/src/com/cloud/utils/UriUtils.java @@ -26,7 +26,10 @@ import java.net.URI; import java.net.URISyntaxException; import java.net.URLEncoder; import java.net.UnknownHostException; +import java.util.ArrayList; import java.util.List; +import java.util.ListIterator; +import java.util.StringTokenizer; import javax.net.ssl.HttpsURLConnection; @@ -37,10 +40,14 @@ import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager; import org.apache.commons.httpclient.UsernamePasswordCredentials; import org.apache.commons.httpclient.auth.AuthScope; import org.apache.commons.httpclient.methods.GetMethod; +import org.apache.commons.httpclient.util.URIUtil; import org.apache.http.NameValuePair; +import org.apache.http.message.BasicNameValuePair; +import org.apache.http.client.utils.URIBuilder; import org.apache.http.client.utils.URLEncodedUtils; import org.apache.log4j.Logger; +import com.cloud.utils.crypt.DBEncryptionUtil; import com.cloud.utils.exception.CloudRuntimeException; public class UriUtils { @@ -143,6 +150,60 @@ public class UriUtils { } return (foundUser && foundPswd); } + + public static String getUpdateUri(String url, boolean encrypt) { + String updatedPath = null; + try { + String query = URIUtil.getQuery(url); + URIBuilder builder = new URIBuilder(url); + builder.removeQuery(); + + String updatedQuery = new String(); + List queryParams = getUserDetails(query); + ListIterator iterator = queryParams.listIterator(); + while (iterator.hasNext()) { + NameValuePair param = iterator.next(); + String value = null; + if ("password".equalsIgnoreCase(param.getName()) && + param.getValue() != null) { + value = encrypt ? DBEncryptionUtil.encrypt(param.getValue()) : DBEncryptionUtil.decrypt(param.getValue()); + } else { + value = param.getValue(); + } + + if (updatedQuery.isEmpty()) { + updatedQuery += (param.getName() + "=" + value); + } else { + updatedQuery += ("&" + param.getName() + "=" + value); + } + } + + String schemeAndHost = new String(); + URI newUri = builder.build(); + if (newUri.getScheme() != null) { + schemeAndHost = newUri.getScheme() + "://" + newUri.getHost(); + } + + updatedPath = schemeAndHost + newUri.getPath() + "?" + updatedQuery; + } catch (URISyntaxException e) { + throw new CloudRuntimeException("Couldn't generate an updated uri. " + e.getMessage()); + } + + return updatedPath; + } + + private static List getUserDetails(String query) { + List details = new ArrayList(); + StringTokenizer allParams = new StringTokenizer(query, "&"); + while (allParams.hasMoreTokens()) { + String param = allParams.nextToken(); + details.add(new BasicNameValuePair(param.substring(0, param.indexOf("=")), + param.substring(param.indexOf("=") + 1))); + } + + return details; + } + // Get the size of a file from URL response header. public static Long getRemoteSize(String url) { Long remoteSize = (long) 0; From fb87c85b2a313d75af7cd4b790118fea30a2dd1b Mon Sep 17 00:00:00 2001 From: Anshul Gangwar Date: Fri, 31 Jan 2014 13:04:36 +0530 Subject: [PATCH 086/169] CLOUDSTACK-6001: Fixed hyperv vm console not working for 3 minutes after migration. --- .../consoleproxy/ConsoleProxyRdpClient.java | 143 ++++++++++-------- 1 file changed, 76 insertions(+), 67 deletions(-) diff --git a/services/console-proxy/server/src/com/cloud/consoleproxy/ConsoleProxyRdpClient.java b/services/console-proxy/server/src/com/cloud/consoleproxy/ConsoleProxyRdpClient.java index d5a3fcd3639..7d49b19bfe3 100644 --- a/services/console-proxy/server/src/com/cloud/consoleproxy/ConsoleProxyRdpClient.java +++ b/services/console-proxy/server/src/com/cloud/consoleproxy/ConsoleProxyRdpClient.java @@ -19,7 +19,6 @@ package com.cloud.consoleproxy; import java.awt.event.InputEvent; import java.awt.event.KeyEvent; import java.awt.event.MouseEvent; -import java.io.IOException; import java.net.InetSocketAddress; import org.apache.log4j.Logger; @@ -55,7 +54,8 @@ public class ConsoleProxyRdpClient extends ConsoleProxyClientBase { private RdpBufferedImageCanvas _canvas = null; private Thread _worker; - private volatile boolean _workerDone = false; + private volatile boolean _workerDone = true; + private volatile long _threadStopTime; private AwtMouseEventSource _mouseEventSource = null; private AwtKeyEventSource _keyEventSource = null; @@ -231,77 +231,86 @@ public class ConsoleProxyRdpClient extends ConsoleProxyClientBase { @Override public void initClient(final ConsoleProxyClientParam param) { - _workerDone = false; - - int canvasWidth = 1024; - int canvasHeight = 768; - setClientParam(param); - - final String host = param.getHypervHost(); - final String password = param.getPassword(); - final String instanceId = param.getClientHostAddress(); - final int port = param.getClientHostPort(); - - _screen = new ScreenDescription(); - _canvas = new RdpBufferedImageCanvas(this, canvasWidth, canvasHeight); - onFramebufferSizeChange(canvasWidth, canvasHeight); - - _screen.addSizeChangeListener(new SizeChangeListener() { - @Override - public void sizeChanged(int width, int height) { - if (_canvas != null) { - _canvas.setCanvasSize(width, height); - } - } - }); - - final SSLState sslState = new SSLState(); - - final String username = param.getUsername(); - String name = null; - String domain = null; - if (username.contains("\\")) { - String[] tokens = username.split("\\\\"); - name = tokens[1]; - domain = tokens[0]; - } else { - name = username; - domain = "Workgroup"; + if ((System.currentTimeMillis() - _threadStopTime) < 1000) { + return; } - _client = new RdpClient("client", host, domain, name, password, instanceId, _screen, _canvas, - sslState); + try { + int canvasWidth = 1024; + int canvasHeight = 768; + setClientParam(param); - _mouseEventSource = _client.getMouseEventSource(); - _keyEventSource = _client.getKeyEventSource(); + final String host = param.getHypervHost(); + final String password = param.getPassword(); + final String instanceId = param.getClientHostAddress(); + final int port = param.getClientHostPort(); - _worker = new Thread(new Runnable() { - @Override - public void run() { - _socket = new AprSocketWrapperImpl("socket", sslState); - Pipeline pipeline = new PipelineImpl("Client"); - pipeline.add(_socket, _client); - pipeline.link("socket", _client.getId(), "socket"); - pipeline.validate(); + final SSLState sslState = new SSLState(); - InetSocketAddress address = new InetSocketAddress(host, port); - ConsoleProxy.ensureRoute(host); - - try { - // Connect socket to remote server and run main loop(s) - _socket.connect(address); - } catch (IOException e) { - e.printStackTrace(); - } finally { - shutdown(); - } - - s_logger.info("Receiver thread stopped."); - _workerDone = true; + final String username = param.getUsername(); + String name = null; + String domain = null; + if (username.contains("\\")) { + String[] tokens = username.split("\\\\"); + name = tokens[1]; + domain = tokens[0]; + } else { + name = username; + domain = "Workgroup"; } - }); - _worker.setDaemon(true); - _worker.start(); + + _screen = new ScreenDescription(); + _canvas = new RdpBufferedImageCanvas(this, canvasWidth, canvasHeight); + onFramebufferSizeChange(canvasWidth, canvasHeight); + + _screen.addSizeChangeListener(new SizeChangeListener() { + @Override + public void sizeChanged(int width, int height) { + if (_canvas != null) { + _canvas.setCanvasSize(width, height); + } + } + }); + + s_logger.info("connecting to instance " + instanceId + " on host " + host); + _client = new RdpClient("client", host, domain, name, password, instanceId, _screen, _canvas, sslState); + + _mouseEventSource = _client.getMouseEventSource(); + _keyEventSource = _client.getKeyEventSource(); + + _worker = new Thread(new Runnable() { + @Override + public void run() { + _socket = new AprSocketWrapperImpl("socket", sslState); + Pipeline pipeline = new PipelineImpl("Client"); + pipeline.add(_socket, _client); + pipeline.link("socket", _client.getId(), "socket"); + pipeline.validate(); + + InetSocketAddress address = new InetSocketAddress(host, port); + ConsoleProxy.ensureRoute(host); + + try { + _workerDone = false; + s_logger.info("Connecting socket to remote server and run main loop(s)"); + _socket.connect(address); + } catch (Exception e) { + s_logger.info(" error occurred in connecting to socket " + e.getMessage()); + } finally { + shutdown(); + } + + _threadStopTime = System.currentTimeMillis(); + s_logger.info("Receiver thread stopped."); + _workerDone = true; + } + }); + _worker.setDaemon(true); + _worker.start(); + } catch (Exception e) { + _workerDone = true; + s_logger.info("error occurred in initializing rdp client " + e.getMessage()); + } } @Override From bba6b77177a6404f1d27f7adc278b46fe00759f0 Mon Sep 17 00:00:00 2001 From: Sanjay Tripathi Date: Thu, 6 Feb 2014 17:42:14 +0530 Subject: [PATCH 087/169] CLOUDSTACK-5953: In hypervisor_capabilities, max_guests_limit are not correct for XS 6.2 or other specific version hypervisor. --- setup/db/db/schema-421to430.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup/db/db/schema-421to430.sql b/setup/db/db/schema-421to430.sql index 316cbe94613..43e94db1d80 100644 --- a/setup/db/db/schema-421to430.sql +++ b/setup/db/db/schema-421to430.sql @@ -828,6 +828,8 @@ CREATE TABLE `cloud`.`network_acl_item_details` ( ALTER TABLE `cloud`.`alert` ADD COLUMN `name` varchar(255) DEFAULT NULL COMMENT 'name of the alert'; +UPDATE `cloud`.`hypervisor_capabilities` set max_guests_limit='150' WHERE hypervisor_version='6.1.0'; +UPDATE `cloud`.`hypervisor_capabilities` set max_guests_limit='500' WHERE hypervisor_version='6.2.0'; UPDATE `cloud`.`hypervisor_capabilities` SET `max_data_volumes_limit`=13 WHERE `hypervisor_type`='Vmware'; INSERT IGNORE INTO `cloud`.`hypervisor_capabilities`(uuid, hypervisor_type, hypervisor_version, max_guests_limit, security_group_enabled, max_data_volumes_limit, storage_motion_supported) VALUES (UUID(), 'Hyperv', '6.2', 1024, 0, 64, 0); From 7a71cf33ce103392914ac51cd4689a6f5a340d0a Mon Sep 17 00:00:00 2001 From: Jayapal Date: Thu, 6 Feb 2014 17:58:26 +0530 Subject: [PATCH 088/169] CLOUDSTACK-6040: Updated the ip addr validation in create port forwarding --- .../api/command/user/firewall/CreatePortForwardingRuleCmd.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/src/org/apache/cloudstack/api/command/user/firewall/CreatePortForwardingRuleCmd.java b/api/src/org/apache/cloudstack/api/command/user/firewall/CreatePortForwardingRuleCmd.java index ff63d088e96..2e8f662ec76 100644 --- a/api/src/org/apache/cloudstack/api/command/user/firewall/CreatePortForwardingRuleCmd.java +++ b/api/src/org/apache/cloudstack/api/command/user/firewall/CreatePortForwardingRuleCmd.java @@ -18,6 +18,7 @@ package org.apache.cloudstack.api.command.user.firewall; import java.util.List; +import com.cloud.utils.net.NetUtils; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiCommandJobType; import org.apache.cloudstack.api.ApiConstants; @@ -317,7 +318,7 @@ public class CreatePortForwardingRuleCmd extends BaseAsyncCreateCmd implements P Ip privateIp = getVmSecondaryIp(); if (privateIp != null) { - if ( !privateIp.isIp4()) { + if (!NetUtils.isValidIp(privateIp.toString())) { throw new InvalidParameterValueException("Invalid vm ip address"); } } From 900c51103ab1e29b86ea5f76cb73c72b3de51db7 Mon Sep 17 00:00:00 2001 From: Min Chen Date: Thu, 6 Feb 2014 15:33:05 -0800 Subject: [PATCH 089/169] CLOUDSTACK-6046:CreateVolume from snapshot is failing with S3 as secondary storage and zone-wide primary storage. --- .../cloudstack/storage/motion/AncientDataMotionStrategy.java | 2 +- .../cloudstack/storage/endpoint/DefaultEndPointSelector.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java b/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java index 30edf68fb98..f2ed6679edd 100644 --- a/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java +++ b/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java @@ -249,7 +249,7 @@ AncientDataMotionStrategy implements DataMotionStrategy { if (srcData.getDataStore().getRole() == DataStoreRole.Primary) { ep = selector.select(volObj); } else { - ep = selector.select(snapObj, volObj); + ep = selector.select(srcData, volObj); } CopyCommand cmd = new CopyCommand(srcData.getTO(), volObj.getTO(), _createVolumeFromSnapshotWait, VirtualMachineManager.ExecuteInSequence.value()); diff --git a/engine/storage/src/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java b/engine/storage/src/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java index 967251f329c..39bc8399b80 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java +++ b/engine/storage/src/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java @@ -167,10 +167,11 @@ public class DefaultEndPointSelector implements EndPointSelector { poolId = destStore.getId(); } else { // if both are zone scope - selectedScope = srcScope; if (srcStore.getRole() == DataStoreRole.Primary) { + selectedScope = srcScope; poolId = srcStore.getId(); } else if (destStore.getRole() == DataStoreRole.Primary) { + selectedScope = destScope; poolId = destStore.getId(); } } From e00241f41d6542cf1c5d93216fed55609cd7746a Mon Sep 17 00:00:00 2001 From: Min Chen Date: Thu, 6 Feb 2014 17:11:36 -0800 Subject: [PATCH 090/169] CLOUDSTACK-6049: Give priority to cache stores where data object is already there instead of randomly picking one in case there are multiple cache stores in the scope. --- .../api/storage/StorageCacheManager.java | 2 + .../allocator/StorageCacheAllocator.java | 3 ++ .../StorageCacheRandomAllocator.java | 39 +++++++++++++++++++ .../manager/StorageCacheManagerImpl.java | 13 +++++++ .../motion/AncientDataMotionStrategy.java | 2 +- 5 files changed, 58 insertions(+), 1 deletion(-) diff --git a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/StorageCacheManager.java b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/StorageCacheManager.java index 92724c91889..92263a7e7c8 100644 --- a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/StorageCacheManager.java +++ b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/StorageCacheManager.java @@ -20,6 +20,8 @@ package org.apache.cloudstack.engine.subsystem.api.storage; public interface StorageCacheManager { DataStore getCacheStorage(Scope scope); + + DataStore getCacheStorage(DataObject data, Scope scope); DataObject createCacheObject(DataObject data, Scope scope); diff --git a/engine/storage/cache/src/org/apache/cloudstack/storage/cache/allocator/StorageCacheAllocator.java b/engine/storage/cache/src/org/apache/cloudstack/storage/cache/allocator/StorageCacheAllocator.java index 4259d9ed03f..9f0052f659c 100644 --- a/engine/storage/cache/src/org/apache/cloudstack/storage/cache/allocator/StorageCacheAllocator.java +++ b/engine/storage/cache/src/org/apache/cloudstack/storage/cache/allocator/StorageCacheAllocator.java @@ -18,9 +18,12 @@ */ package org.apache.cloudstack.storage.cache.allocator; +import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.Scope; public interface StorageCacheAllocator { DataStore getCacheStore(Scope scope); + + DataStore getCacheStore(DataObject data, Scope scope); } diff --git a/engine/storage/cache/src/org/apache/cloudstack/storage/cache/allocator/StorageCacheRandomAllocator.java b/engine/storage/cache/src/org/apache/cloudstack/storage/cache/allocator/StorageCacheRandomAllocator.java index 3b434d175fd..0054e3dcae1 100644 --- a/engine/storage/cache/src/org/apache/cloudstack/storage/cache/allocator/StorageCacheRandomAllocator.java +++ b/engine/storage/cache/src/org/apache/cloudstack/storage/cache/allocator/StorageCacheRandomAllocator.java @@ -23,9 +23,14 @@ import java.util.List; import javax.inject.Inject; +import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; +import org.apache.cloudstack.engine.subsystem.api.storage.DataObjectInStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; import org.apache.cloudstack.engine.subsystem.api.storage.Scope; +import org.apache.cloudstack.storage.datastore.ObjectInDataStoreManager; + import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -36,6 +41,8 @@ public class StorageCacheRandomAllocator implements StorageCacheAllocator { private static final Logger s_logger = Logger.getLogger(StorageCacheRandomAllocator.class); @Inject DataStoreManager dataStoreMgr; + @Inject + ObjectInDataStoreManager objectInStoreMgr; @Override public DataStore getCacheStore(Scope scope) { @@ -53,4 +60,36 @@ public class StorageCacheRandomAllocator implements StorageCacheAllocator { Collections.shuffle(cacheStores); return cacheStores.get(0); } + + @Override + public DataStore getCacheStore(DataObject data, Scope scope) { + if (scope.getScopeType() != ScopeType.ZONE) { + s_logger.debug("Can only support zone wide cache storage"); + return null; + } + + List cacheStores = dataStoreMgr.getImageCacheStores(scope); + if (cacheStores.size() <= 0) { + s_logger.debug("Can't find staging storage in zone: " + scope.getScopeId()); + return null; + } + + // if there are multiple cache stores, we give priority to the one where data is already there + if (cacheStores.size() > 1) { + for (DataStore store : cacheStores) { + DataObjectInStore obj = objectInStoreMgr.findObject(data, store); + if (obj != null && obj.getState() == ObjectInDataStoreStateMachine.State.Ready) { + s_logger.debug("pick the cache store " + store.getId() + " where data is already there"); + return store; + } + } + + // otherwise, just random pick one + Collections.shuffle(cacheStores); + } + return cacheStores.get(0); + + } + + } diff --git a/engine/storage/cache/src/org/apache/cloudstack/storage/cache/manager/StorageCacheManagerImpl.java b/engine/storage/cache/src/org/apache/cloudstack/storage/cache/manager/StorageCacheManagerImpl.java index bb8d67d321f..7c74729bc28 100644 --- a/engine/storage/cache/src/org/apache/cloudstack/storage/cache/manager/StorageCacheManagerImpl.java +++ b/engine/storage/cache/src/org/apache/cloudstack/storage/cache/manager/StorageCacheManagerImpl.java @@ -90,6 +90,19 @@ public class StorageCacheManagerImpl implements StorageCacheManager, Manager { return null; } + + @Override + public DataStore getCacheStorage(DataObject data, Scope scope) { + for (StorageCacheAllocator allocator : storageCacheAllocator) { + DataStore store = allocator.getCacheStore(data, scope); + if (store != null) { + return store; + } + } + return null; + } + + protected List getCacheStores() { QueryBuilder sc = QueryBuilder.create(ImageStoreVO.class); sc.and(sc.entity().getRole(), SearchCriteria.Op.EQ,DataStoreRole.ImageCache); diff --git a/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java b/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java index f2ed6679edd..d6759cb9537 100644 --- a/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java +++ b/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java @@ -201,7 +201,7 @@ AncientDataMotionStrategy implements DataMotionStrategy { protected DataObject cacheSnapshotChain(SnapshotInfo snapshot, Scope scope) { DataObject leafData = null; - DataStore store = cacheMgr.getCacheStorage(scope); + DataStore store = cacheMgr.getCacheStorage(snapshot, scope); while (snapshot != null) { DataObject cacheData = cacheMgr.createCacheObject(snapshot, store); if (leafData == null) { From f0b861fedeea9e752665a42231f910112c5b8b46 Mon Sep 17 00:00:00 2001 From: Devdeep Singh Date: Fri, 7 Feb 2014 17:46:03 +0530 Subject: [PATCH 091/169] CLOUDSTACK-6053: While adding a primary or secondary of type smb the password wasn't encoded. This cause createStoragePool or addImageStore command to fail if special characters were present. Updated the code to pass user, password and domain as part of details while adding primary or secondary. Also made changes on server side to handle it. --- .../storage/datastore/db/ImageStoreVO.java | 3 -- .../storage/datastore/db/StoragePoolVO.java | 3 -- .../image/datastore/ImageStoreHelper.java | 25 ++++++++++++++++ .../datastore/PrimaryDataStoreHelper.java | 29 +++++++++++++++++++ .../api/query/dao/ImageStoreJoinDaoImpl.java | 4 +-- ui/scripts/sharedFunctions.js | 2 +- ui/scripts/system.js | 15 ++++++++-- ui/scripts/zoneWizard.js | 15 ++++++++-- utils/src/com/cloud/utils/UriUtils.java | 25 ++++++---------- 9 files changed, 90 insertions(+), 31 deletions(-) diff --git a/engine/schema/src/org/apache/cloudstack/storage/datastore/db/ImageStoreVO.java b/engine/schema/src/org/apache/cloudstack/storage/datastore/db/ImageStoreVO.java index 97ae9c8c70e..387e096ff05 100644 --- a/engine/schema/src/org/apache/cloudstack/storage/datastore/db/ImageStoreVO.java +++ b/engine/schema/src/org/apache/cloudstack/storage/datastore/db/ImageStoreVO.java @@ -154,9 +154,6 @@ public class ImageStoreVO implements ImageStore { public void setUrl(String url) { this.url = url; - if ("cifs".equalsIgnoreCase(this.protocol)) { - this.url = UriUtils.getUpdateUri(url, true); - } } public Date getCreated() { diff --git a/engine/schema/src/org/apache/cloudstack/storage/datastore/db/StoragePoolVO.java b/engine/schema/src/org/apache/cloudstack/storage/datastore/db/StoragePoolVO.java index 7d52858cbbd..0d1e43db1c2 100644 --- a/engine/schema/src/org/apache/cloudstack/storage/datastore/db/StoragePoolVO.java +++ b/engine/schema/src/org/apache/cloudstack/storage/datastore/db/StoragePoolVO.java @@ -279,9 +279,6 @@ public class StoragePoolVO implements StoragePool { public void setPath(String path) { this.path = path; - if (this.poolType == StoragePoolType.SMB) { - this.path = UriUtils.getUpdateUri(this.path, true); - } } public void setUserInfo(String userInfo) { diff --git a/engine/storage/src/org/apache/cloudstack/storage/image/datastore/ImageStoreHelper.java b/engine/storage/src/org/apache/cloudstack/storage/image/datastore/ImageStoreHelper.java index 0f5b817b511..187d6e4defd 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/image/datastore/ImageStoreHelper.java +++ b/engine/storage/src/org/apache/cloudstack/storage/image/datastore/ImageStoreHelper.java @@ -18,6 +18,8 @@ */ package org.apache.cloudstack.storage.image.datastore; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; import java.util.Iterator; import java.util.Map; import java.util.UUID; @@ -34,6 +36,7 @@ import org.apache.cloudstack.storage.datastore.db.ImageStoreDetailsDao; import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; +import com.cloud.exception.InvalidParameterValueException; import com.cloud.storage.DataStoreRole; import com.cloud.storage.ScopeType; import com.cloud.utils.crypt.DBEncryptionUtil; @@ -96,6 +99,28 @@ public class ImageStoreHelper { store.setName(store.getUuid()); } store.setRole((DataStoreRole) params.get("role")); + + if ("cifs".equalsIgnoreCase((String) params.get("protocol")) && details != null) { + String user = details.get("user"); + String password = details.get("password"); + String domain = details.get("domain"); + String updatedPath = (String) params.get("url"); + + if (user == null || password == null) { + String errMsg = "Missing cifs user and password details. Add them as details parameter."; + throw new InvalidParameterValueException(errMsg); + } else { + try { + password = DBEncryptionUtil.encrypt(URLEncoder.encode(password, "UTF-8")); + details.put("password", password); + updatedPath += "?user=" + user + "&password=" + password + "&domain=" + domain; + } catch (UnsupportedEncodingException e) { + throw new CloudRuntimeException("Error while generating the cifs url. " + e.getMessage()); + } + store.setUrl(updatedPath); + } + } + store = imageStoreDao.persist(store); // persist details diff --git a/engine/storage/src/org/apache/cloudstack/storage/volume/datastore/PrimaryDataStoreHelper.java b/engine/storage/src/org/apache/cloudstack/storage/volume/datastore/PrimaryDataStoreHelper.java index 95dc40b324d..c2bb31f532f 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/volume/datastore/PrimaryDataStoreHelper.java +++ b/engine/storage/src/org/apache/cloudstack/storage/volume/datastore/PrimaryDataStoreHelper.java @@ -18,6 +18,8 @@ */ package org.apache.cloudstack.storage.volume.datastore; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; import java.util.List; import java.util.Map; @@ -36,13 +38,17 @@ import com.cloud.agent.api.StoragePoolInfo; import com.cloud.capacity.Capacity; import com.cloud.capacity.CapacityVO; import com.cloud.capacity.dao.CapacityDao; +import com.cloud.exception.InvalidParameterValueException; import com.cloud.storage.DataStoreRole; import com.cloud.storage.ScopeType; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.storage.StorageManager; import com.cloud.storage.StoragePoolHostVO; import com.cloud.storage.StoragePoolStatus; +import com.cloud.storage.Storage.StoragePoolType; import com.cloud.storage.dao.StoragePoolHostDao; +import com.cloud.utils.UriUtils; +import com.cloud.utils.crypt.DBEncryptionUtil; import com.cloud.utils.db.TransactionLegacy; import com.cloud.utils.exception.CloudRuntimeException; @@ -86,6 +92,29 @@ public class PrimaryDataStoreHelper { dataStoreVO.setHypervisor(params.getHypervisorType()); Map details = params.getDetails(); + if (params.getType() == StoragePoolType.SMB && details != null) { + String user = details.get("user"); + String password = details.get("password"); + String domain = details.get("domain"); + String updatedPath = params.getPath(); + + if (user == null || password == null) { + String errMsg = "Missing cifs user and password details. Add them as details parameter."; + s_logger.warn(errMsg); + throw new InvalidParameterValueException(errMsg); + } else { + try { + password = DBEncryptionUtil.encrypt(URLEncoder.encode(password, "UTF-8")); + details.put("password", password); + updatedPath += "?user=" + user + "&password=" + password + "&domain=" + domain; + } catch (UnsupportedEncodingException e) { + throw new CloudRuntimeException("Error while generating the cifs url. " + e.getMessage()); + } + } + + dataStoreVO.setPath(updatedPath); + } + String tags = params.getTags(); if (tags != null) { String[] tokens = tags.split(","); diff --git a/server/src/com/cloud/api/query/dao/ImageStoreJoinDaoImpl.java b/server/src/com/cloud/api/query/dao/ImageStoreJoinDaoImpl.java index 447a1ad4387..23cde0066eb 100644 --- a/server/src/com/cloud/api/query/dao/ImageStoreJoinDaoImpl.java +++ b/server/src/com/cloud/api/query/dao/ImageStoreJoinDaoImpl.java @@ -86,7 +86,7 @@ public class ImageStoreJoinDaoImpl extends GenericDaoBase 0 ){ + if ( detailName != null && detailName.length() > 0 && !detailName.equals(ApiConstants.PASSWORD)) { String detailValue = ids.getDetailValue(); if (detailName.equals(ApiConstants.KEY) || detailName.equals(ApiConstants.S3_SECRET_KEY)) { detailValue = DBEncryptionUtil.decrypt(detailValue); @@ -105,7 +105,7 @@ public class ImageStoreJoinDaoImpl extends GenericDaoBase 0 ){ + if ( detailName != null && detailName.length() > 0 && !detailName.equals(ApiConstants.PASSWORD)) { String detailValue = ids.getDetailValue(); if (detailName.equals(ApiConstants.KEY) || detailName.equals(ApiConstants.S3_SECRET_KEY)) { detailValue = DBEncryptionUtil.decrypt(detailValue); diff --git a/ui/scripts/sharedFunctions.js b/ui/scripts/sharedFunctions.js index b9dc2f37767..2a159675f31 100644 --- a/ui/scripts/sharedFunctions.js +++ b/ui/scripts/sharedFunctions.js @@ -1253,7 +1253,7 @@ var processPropertiesInImagestoreObject = function(jsonObj) { url += 'cifs://'; } - url += (server + path + '?user=' + smbUsername + '&password=' + smbPassword + '&domain=' + smbDomain); + url += (server + path); return url; } diff --git a/ui/scripts/system.js b/ui/scripts/system.js index 743427b2984..500a339ca00 100644 --- a/ui/scripts/system.js +++ b/ui/scripts/system.js @@ -14759,7 +14759,10 @@ var path = args.data.path; if (path.substring(0, 1) != "/") path = "/" + path; - url = smbURL(server, path, args.data.smbUsername, args.data.smbPassword, args.data.smbDomain); + url = smbURL(server, path); + array1.push("&details[0].user=" + args.data.smbUsername); + array1.push("&details[1].password=" + todb(args.data.smbPassword)); + array1.push("&details[2].domain=" + args.data.smbDomain); } else if (args.data.protocol == "PreSetup") { var path = args.data.path; if (path.substring(0, 1) != "/") @@ -16157,12 +16160,18 @@ var zoneid = args.data.zoneid; var nfs_server = args.data.nfsServer; var path = args.data.path; - var url = smbURL(nfs_server, path, args.data.smbUsername, args.data.smbPassword, args.data.smbDomain); + var url = smbURL(nfs_server, path); $.extend(data, { provider: args.data.provider, zoneid: zoneid, - url: url + url: url, + 'details[0].key': 'user', + 'details[0].value': args.data.smbUsername, + 'details[1].key': 'password', + 'details[1].value': args.data.smbPassword, + 'details[2].key': 'domain', + 'details[2].value': args.data.smbDomain }); $.ajax({ diff --git a/ui/scripts/zoneWizard.js b/ui/scripts/zoneWizard.js index b26c8bf7c9f..006bf7da51e 100755 --- a/ui/scripts/zoneWizard.js +++ b/ui/scripts/zoneWizard.js @@ -4128,7 +4128,10 @@ var path = args.data.primaryStorage.path; if (path.substring(0, 1) != "/") path = "/" + path; - url = smbURL(server, path, args.data.primaryStorage.smbUsername, args.data.primaryStorage.smbPassword, args.data.primaryStorage.smbDomain); + url = smbURL(server, path); + array1.push("&details[0].user=" + args.data.primaryStorage.smbUsername); + array1.push("&details[1].password=" + todb(args.data.primaryStorage.smbPassword)); + array1.push("&details[2].domain=" + args.data.primaryStorage.smbDomain); } else if (args.data.primaryStorage.protocol == "PreSetup") { var path = args.data.primaryStorage.path; if (path.substring(0, 1) != "/") @@ -4237,12 +4240,18 @@ } else if (args.data.secondaryStorage.provider == 'SMB') { var nfs_server = args.data.secondaryStorage.nfsServer; var path = args.data.secondaryStorage.path; - var url = smbURL(nfs_server, path, args.data.secondaryStorage.smbUsername, args.data.secondaryStorage.smbPassword, args.data.secondaryStorage.smbDomain); + var url = smbURL(nfs_server, path); $.extend(data, { provider: args.data.secondaryStorage.provider, zoneid: args.data.returnedZone.id, - url: url + url: url, + 'details[0].key': 'user', + 'details[0].value': args.data.secondaryStorage.smbUsername, + 'details[1].key': 'password', + 'details[1].value': args.data.secondaryStorage.smbPassword, + 'details[2].key': 'domain', + 'details[2].value': args.data.secondaryStorage.smbDomain }); $.ajax({ diff --git a/utils/src/com/cloud/utils/UriUtils.java b/utils/src/com/cloud/utils/UriUtils.java index 9261b075a0c..67b86b80da8 100644 --- a/utils/src/com/cloud/utils/UriUtils.java +++ b/utils/src/com/cloud/utils/UriUtils.java @@ -111,17 +111,8 @@ public class UriUtils { public static String getCifsUriParametersProblems(URI uri) { if (!UriUtils.hostAndPathPresent(uri)) { - String errMsg = "cifs URI missing host and/or path. " - + " Make sure it's of the format " - + "cifs://hostname/path?user=&password="; - s_logger.warn(errMsg); - return errMsg; - } - if (!UriUtils.cifsCredentialsPresent(uri)) - { - String errMsg = "cifs URI missing user and password details. " - + "Add them as query parameters, e.g. " - + "cifs://example.com/some_share?user=foo&password=bar"; + String errMsg = "cifs URI missing host and/or path. Make sure it is " + + " of the format cifs://hostname/path"; s_logger.warn(errMsg); return errMsg; } @@ -194,11 +185,13 @@ public class UriUtils { private static List getUserDetails(String query) { List details = new ArrayList(); - StringTokenizer allParams = new StringTokenizer(query, "&"); - while (allParams.hasMoreTokens()) { - String param = allParams.nextToken(); - details.add(new BasicNameValuePair(param.substring(0, param.indexOf("=")), - param.substring(param.indexOf("=") + 1))); + if (query != null && !query.isEmpty()) { + StringTokenizer allParams = new StringTokenizer(query, "&"); + while (allParams.hasMoreTokens()) { + String param = allParams.nextToken(); + details.add(new BasicNameValuePair(param.substring(0, param.indexOf("=")), + param.substring(param.indexOf("=") + 1))); + } } return details; From 95e41fdf0da50c165a9317847058fce4efeddbcf Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Fri, 7 Feb 2014 16:44:11 +0530 Subject: [PATCH 092/169] CLOUDSTACK-6023: Non windows instances are created on XenServer with a vcpu-max above supported xenserver limits Changed the VCPU max limit to 16 and provided a cluster level configuration parameter for this max limit named xen.vm.vcpu.max --- .../cloud/agent/api/to/VirtualMachineTO.java | 9 ++++++ .../com/cloud/hypervisor/XenServerGuru.java | 30 ++++++++++++++++++- .../xen/resource/CitrixResourceBase.java | 14 ++++++--- 3 files changed, 48 insertions(+), 5 deletions(-) diff --git a/api/src/com/cloud/agent/api/to/VirtualMachineTO.java b/api/src/com/cloud/agent/api/to/VirtualMachineTO.java index e6240ffda10..a5860d74979 100644 --- a/api/src/com/cloud/agent/api/to/VirtualMachineTO.java +++ b/api/src/com/cloud/agent/api/to/VirtualMachineTO.java @@ -60,6 +60,7 @@ public class VirtualMachineTO { DiskTO[] disks; NicTO[] nics; + Integer vcpuMaxLimit; public VirtualMachineTO(long id, String instanceName, VirtualMachine.Type type, int cpus, Integer speed, long minRam, long maxRam, BootloaderType bootloader, String os, boolean enableHA, boolean limitCpuUse, String vncPassword) { this.id = id; @@ -263,5 +264,13 @@ public class VirtualMachineTO { this.uuid = uuid; } + public Integer getVcpuMaxLimit() { + return vcpuMaxLimit; + } + + public void setVcpuMaxLimit(Integer vcpuMaxLimit) { + this.vcpuMaxLimit = vcpuMaxLimit; + } + } diff --git a/plugins/hypervisors/xen/src/com/cloud/hypervisor/XenServerGuru.java b/plugins/hypervisors/xen/src/com/cloud/hypervisor/XenServerGuru.java index 28bd724ed4e..47ab90a1e52 100644 --- a/plugins/hypervisors/xen/src/com/cloud/hypervisor/XenServerGuru.java +++ b/plugins/hypervisors/xen/src/com/cloud/hypervisor/XenServerGuru.java @@ -35,19 +35,24 @@ import com.cloud.storage.dao.GuestOSDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.template.VirtualMachineTemplate.BootloaderType; import com.cloud.utils.Pair; +import com.cloud.vm.UserVmVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineProfile; +import com.cloud.vm.dao.UserVmDao; import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; +import org.apache.cloudstack.framework.config.Configurable; import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.command.DettachCommand; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.framework.config.ConfigKey; + @Local(value=HypervisorGuru.class) -public class XenServerGuru extends HypervisorGuruBase implements HypervisorGuru { +public class XenServerGuru extends HypervisorGuruBase implements HypervisorGuru, Configurable{ @Inject GuestOSDao _guestOsDao; @Inject EndPointSelector endPointSelector; @@ -59,6 +64,11 @@ public class XenServerGuru extends HypervisorGuruBase implements HypervisorGuru PrimaryDataStoreDao _storagePoolDao; @Inject VolumeDataFactory _volFactory; + @Inject + UserVmDao _userVmDao; + + static final ConfigKey MaxNumberOfVCPUSPerVM = new ConfigKey("Advanced", Integer.class, "xen.vm.vcpu.max", "16", + "Maximum number of VCPUs that VM can get in XenServer.", true, ConfigKey.Scope.Cluster); protected XenServerGuru() { super(); @@ -76,6 +86,14 @@ public class XenServerGuru extends HypervisorGuruBase implements HypervisorGuru bt = vm.getBootLoaderType(); } VirtualMachineTO to = toVirtualMachineTO(vm); + UserVmVO userVmVO = _userVmDao.findById(vm.getId()); + if (userVmVO != null) { + HostVO host = hostDao.findById(userVmVO.getHostId()); + if (host != null) { + to.setVcpuMaxLimit(MaxNumberOfVCPUSPerVM.valueIn(host.getClusterId())); + } + } + to.setBootloader(bt); // Determine the VM's OS description @@ -151,4 +169,14 @@ public class XenServerGuru extends HypervisorGuruBase implements HypervisorGuru } return new Pair(Boolean.FALSE, new Long(hostId)); } + + @Override + public String getConfigComponentName() { + return XenServerGuru.class.getSimpleName(); + } + + @Override + public ConfigKey[] getConfigKeys() { + return new ConfigKey[] {MaxNumberOfVCPUSPerVM}; + } } diff --git a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java index aa1ef838104..b398ca517b7 100644 --- a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java +++ b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java @@ -1265,6 +1265,8 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe vmr.nameLabel = vmSpec.getName(); vmr.actionsAfterCrash = Types.OnCrashBehaviour.DESTROY; vmr.actionsAfterShutdown = Types.OnNormalExit.DESTROY; + vmr.VCPUsMax = (long) vmSpec.getCpus(); // FIX ME: In case of dynamic scaling this VCPU max should be the minumum of + // recommended value for that template and capacity remaining on host if (isDmcEnabled(conn, host) && vmSpec.isEnableDynamicallyScaleVm()) { //scaling is allowed @@ -1272,6 +1274,14 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe vmr.memoryStaticMax = getStaticMax(vmSpec.getOs(), vmSpec.getBootloader() == BootloaderType.CD, vmSpec.getMinRam(), vmSpec.getMaxRam()); vmr.memoryDynamicMin = vmSpec.getMinRam(); vmr.memoryDynamicMax = vmSpec.getMaxRam(); + + if (guestOsTypeName.toLowerCase().contains("windows")) { + vmr.VCPUsMax = (long) vmSpec.getCpus(); + } else { + if (vmSpec.getVcpuMaxLimit() != null) { + vmr.VCPUsMax = (long) vmSpec.getVcpuMaxLimit(); + } + } } else { //scaling disallowed, set static memory target if (vmSpec.isEnableDynamicallyScaleVm() && !isDmcEnabled(conn, host)) { @@ -1281,12 +1291,8 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe vmr.memoryStaticMax = vmSpec.getMaxRam(); vmr.memoryDynamicMin = vmSpec.getMinRam(); vmr.memoryDynamicMax = vmSpec.getMaxRam(); - } - if (guestOsTypeName.toLowerCase().contains("windows")) { vmr.VCPUsMax = (long) vmSpec.getCpus(); - } else { - vmr.VCPUsMax = 32L; } vmr.VCPUsAtStartup = (long) vmSpec.getCpus(); From 65e1e700313b3276714e32f4943b5efd42d38d84 Mon Sep 17 00:00:00 2001 From: Gaurav Aradhye Date: Mon, 10 Feb 2014 12:34:17 +0530 Subject: [PATCH 093/169] CLOUDSTACK-5959: Fixed cascading failures in portable IP test cases --- .../integration/component/test_portable_ip.py | 132 +++++++++++++----- tools/marvin/marvin/integration/lib/common.py | 24 +++- 2 files changed, 117 insertions(+), 39 deletions(-) diff --git a/test/integration/component/test_portable_ip.py b/test/integration/component/test_portable_ip.py index b09daf9aa13..c06f2ea340a 100644 --- a/test/integration/component/test_portable_ip.py +++ b/test/integration/component/test_portable_ip.py @@ -15,14 +15,31 @@ # specific language governing permissions and limitations # under the License. """ Tests for Portable public IP Ranges feature + + Test Plan: https://cwiki.apache.org/confluence/display/CLOUDSTACK/Portable+IP+Test+Execution + + Feature Specifications: https://cwiki.apache.org/confluence/display/CLOUDSTACK/portable+public+IP """ -from marvin.cloudstackTestCase import * -from marvin.cloudstackAPI import * -from marvin.cloudstackException import cloudstackAPIException -from marvin.integration.lib.utils import * -from marvin.integration.lib.base import * -from marvin.integration.lib.common import * -from netaddr import * +from marvin.cloudstackTestCase import cloudstackTestCase +from marvin.integration.lib.utils import cleanup_resources +from marvin.integration.lib.base import (Account, + NetworkOffering, + ServiceOffering, + Network, + VirtualMachine, + PublicIPAddress, + FireWallRule, + NATRule, + PortablePublicIpRange, + StaticNATRule) +from marvin.integration.lib.common import (get_zone, + get_pod, + get_domain, + get_region, + get_template, + get_portable_ip_range_services, + is_public_ip_in_correct_state) +from netaddr import IPAddress from marvin.sshClient import SshClient from nose.plugins.attrib import attr @@ -147,7 +164,7 @@ class TestCreatePortablePublicIpRanges(cloudstackTestCase): cls.services["domainid"] = cls.domain.id cls.services["zoneid"] = cls.zone.id cls.services["regionid"] = cls.region.id - + cls._cleanup = [] return @@ -255,7 +272,7 @@ class TestCreatePortablePublicIpRanges(cloudstackTestCase): portable_ip_range_services["regionid"] = -1 - #create new portable ip range + #create new portable ip range self.debug("Trying to create portable ip range with wrong region id") with self.assertRaises(Exception): @@ -655,7 +672,7 @@ class TestAssociatePublicIp(cloudstackTestCase): self.portable_ip_range.id)) self.cleanup.append(self.portable_ip_range) - + return def tearDown(self): @@ -715,7 +732,7 @@ class TestAssociatePublicIp(cloudstackTestCase): publicipaddressportable.delete(self.apiclient) return - + @attr(tags=["advanced"]) def test_associate_ip_address_invalid_zone(self): """ Test Associate IP with invalid zone id @@ -739,7 +756,6 @@ class TestAssociatePublicIp(cloudstackTestCase): self.debug("Associating ip address failed") return - @unittest.skip("SSH failing to portable ip, need to investigate the issue") @attr(tags=["advanced"]) def test_associate_ip_address_services_enable_disable(self): """ Test enabling and disabling NAT, Firewall services on portable ip @@ -785,9 +801,15 @@ class TestAssociatePublicIp(cloudstackTestCase): ) self.debug("created public ip address (portable): %s" % portableip.ipaddress.ipaddress) - # Open up firewall port for SSH - self.debug("Opening firewall on the portable public ip") - fw_rule = FireWallRule.create( + ipInCorrectState = is_public_ip_in_correct_state(self.apiclient, portableip.ipaddress.id, state="allocated") + if not ipInCorrectState: + portableip.delete(self.apiclient) + self.fail("Portable IP not in allocated state even after 10 mins") + + try: + # Open up firewall port for SSH + self.debug("Opening firewall on the portable public ip") + fw_rule = FireWallRule.create( self.apiclient, ipaddressid=portableip.ipaddress.id, protocol=self.services["natrule"]["protocol"], @@ -796,14 +818,17 @@ class TestAssociatePublicIp(cloudstackTestCase): endport=self.services["natrule"]["publicport"] ) - #Create NAT rule - self.debug("Creating NAT rule on the portable public ip") - nat_rule = NATRule.create( + #Create NAT rule + self.debug("Creating NAT rule on the portable public ip") + nat_rule = NATRule.create( self.apiclient, self.virtual_machine, self.services["natrule"], portableip.ipaddress.id ) + except Exception as e: + portableip.delete(self.apiclient) + self.fail("Error: %s" % e) try: @@ -991,7 +1016,7 @@ class TestDisassociatePublicIp(cloudstackTestCase): new_portable_ip_range.id)) self.cleanup.append(new_portable_ip_range) - + return def tearDown(self): @@ -1002,7 +1027,7 @@ class TestDisassociatePublicIp(cloudstackTestCase): except Exception as e: raise Exception("Warning: Exception during cleanup : %s" % e) return - + @attr(tags=["advanced"]) def test_disassociate_ip_address_no_services(self): """ Test disassociating portable ip @@ -1054,9 +1079,15 @@ class TestDisassociatePublicIp(cloudstackTestCase): ) self.debug("created public ip address (portable): %s" % portableip.ipaddress.ipaddress) - # Open up firewall port for SSH - self.debug("Opening firewall on the portable public ip") - FireWallRule.create( + ipInCorrectState = is_public_ip_in_correct_state(self.apiclient, portableip.ipaddress.id, state="allocated") + if not ipInCorrectState: + portableip.delete(self.apiclient) + self.fail("Portable IP not in allocated state even after 10 mins") + + try: + # Open up firewall port for SSH + self.debug("Opening firewall on the portable public ip") + FireWallRule.create( self.apiclient, ipaddressid=portableip.ipaddress.id, protocol=self.services["natrule"]["protocol"], @@ -1065,14 +1096,17 @@ class TestDisassociatePublicIp(cloudstackTestCase): endport=self.services["natrule"]["publicport"] ) - #Create NAT rule - self.debug("Creating NAT rule on the portable public ip") - NATRule.create( + #Create NAT rule + self.debug("Creating NAT rule on the portable public ip") + NATRule.create( self.apiclient, self.virtual_machine, self.services["natrule"], portableip.ipaddress.id ) + except Exception as e: + portableip.delete(self.apiclient) + self.fail("Error: %s" % e) try: self.debug("Disassociating portable ip: %s with id: %s" % @@ -1310,9 +1344,16 @@ class TestDeleteAccount(cloudstackTestCase): ) self.debug("created public ip address (portable): %s" % portableip.ipaddress.ipaddress) - # Open up firewall port for SSH - self.debug("Opening firewall on the portable public ip") - FireWallRule.create( + ipInCorrectState = is_public_ip_in_correct_state(self.apiclient, portableip.ipaddress.id, state="allocated") + if not ipInCorrectState: + portableip.delete(self.apiclient) + self.account.delete(self.apiclient) + self.fail("Portable IP not in allocated state even after 10 mins") + + try: + # Open up firewall port for SSH + self.debug("Opening firewall on the portable public ip") + FireWallRule.create( self.apiclient, ipaddressid=portableip.ipaddress.id, protocol=self.services["natrule"]["protocol"], @@ -1321,14 +1362,18 @@ class TestDeleteAccount(cloudstackTestCase): endport=self.services["natrule"]["publicport"] ) - #Create NAT rule - self.debug("Creating NAT rule on the portable public ip") - NATRule.create( + #Create NAT rule + self.debug("Creating NAT rule on the portable public ip") + NATRule.create( self.apiclient, self.virtual_machine, self.services["natrule"], portableip.ipaddress.id ) + except Exception as e: + portableip.delete(self.apiclient) + self.account.delete(self.apiclient) + self.fail("Error %s" % e) self.debug("Deleting account: %s :" % self.account.name) @@ -1494,19 +1539,28 @@ class TestPortableIpTransferAcrossNetworks(cloudstackTestCase): networkid=self.network1.id, isportable=True ) + + ipInCorrectState = is_public_ip_in_correct_state(self.apiclient, portableip.ipaddress.id, state="allocated") + if not ipInCorrectState: + portableip.delete(self.apiclient) + self.fail("Portable IP not in allocated state even after 10 mins") + self.debug("created public ip address (portable): %s" % portableip.ipaddress.ipaddress) #Create NAT rule self.debug("Creating NAT rule on the portable public ip") - # Enable Static NAT for VM - StaticNATRule.enable( + + try: + # Enable Static NAT for VM + StaticNATRule.enable( self.apiclient, portableip.ipaddress.id, self.virtual_machine2.id, networkid=self.network2.id ) - # Open up firewall port for SSH - self.debug("Opening firewall on the portable public ip") - fw_rule = FireWallRule.create( + + # Open up firewall port for SSH + self.debug("Opening firewall on the portable public ip") + FireWallRule.create( self.apiclient, ipaddressid=portableip.ipaddress.id, protocol=self.services["natrule"]["protocol"], @@ -1514,6 +1568,10 @@ class TestPortableIpTransferAcrossNetworks(cloudstackTestCase): startport=self.services["natrule"]["publicport"], endport=self.services["natrule"]["publicport"] ) + except Exception as e: + portableip.delete(self.apiclient) + self.fail("Error: %s" % e) + static_nat_list = PublicIPAddress.list( self.apiclient, associatednetworkid=self.network2.id, diff --git a/tools/marvin/marvin/integration/lib/common.py b/tools/marvin/marvin/integration/lib/common.py index 550de1aca77..f23b332190b 100644 --- a/tools/marvin/marvin/integration/lib/common.py +++ b/tools/marvin/marvin/integration/lib/common.py @@ -62,11 +62,14 @@ from marvin.integration.lib.base import (Configurations, Template, Resources, PhysicalNetwork, - Host) + Host, + PublicIPAddress) from marvin.integration.lib.utils import (get_process_status, - xsplit) + xsplit, + validateList) from marvin.sshClient import SshClient +from marvin.codes import PASS import random #Import System modules @@ -907,3 +910,20 @@ def setNonContiguousVlanIds(apiclient, zoneid): return None, None return physical_network, vlan + +def is_public_ip_in_correct_state(apiclient, ipaddressid, state): + """ Check if the given IP is in the correct state (given) + and return True/False accordingly""" + retriesCount = 10 + while True: + portableips = PublicIPAddress.list(apiclient, id=ipaddressid) + assert validateList(portableips)[0] == PASS, "IPs list validation failed" + if str(portableips[0].state).lower() == state: + break + elif retriesCount == 0: + return False + else: + retriesCount -= 1 + time.sleep(60) + continue + return True From 675414feb8d7e15bb4a3251b922a1a1d10639cb0 Mon Sep 17 00:00:00 2001 From: Ashutosh K Date: Mon, 10 Feb 2014 12:38:09 +0530 Subject: [PATCH 094/169] CLOUDSTACK-5978: Changes in get_template function to make it independent of ostype passed through test case --- tools/marvin/marvin/integration/lib/common.py | 20 +++++-------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/tools/marvin/marvin/integration/lib/common.py b/tools/marvin/marvin/integration/lib/common.py index f23b332190b..e202391e521 100644 --- a/tools/marvin/marvin/integration/lib/common.py +++ b/tools/marvin/marvin/integration/lib/common.py @@ -215,17 +215,7 @@ def get_pod(apiclient, zoneid, services=None): def get_template(apiclient, zoneid, ostype, services=None, templatefilter='featured', templatetype='BUILTIN'): - "Returns a template" - - cmd = listOsTypes.listOsTypesCmd() - cmd.description = ostype - ostypes = apiclient.listOsTypes(cmd) - - if isinstance(ostypes, list): - ostypeid = ostypes[0].id - else: - raise Exception( - "Failed to find OS type with description: %s" % ostype) + "Returns a featured built in template in given zone" cmd = listTemplates.listTemplatesCmd() cmd.templatefilter = templatefilter @@ -238,13 +228,13 @@ def get_template(apiclient, zoneid, ostype, services=None, list_templates = apiclient.listTemplates(cmd) if isinstance(list_templates, list): - assert len(list_templates) > 0, "received empty response on template of type %s"%ostype + assert len(list_templates) > 0, "received empty response on featured templates" for template in list_templates: - if template.ostypeid == ostypeid and template.isready and template.templatetype == templatetype: + if template.isready and template.templatetype == templatetype: return template - raise Exception("Exception: Failed to find template of type %s with OSTypeID and which is in " - "ready state: %s" %(templatetype, ostypeid)) + raise Exception("Exception: Failed to find built in template which is in " + "ready state: %s" % templatetype) return From aff22869cc18bcfb0845af2624fdd2afeabb3787 Mon Sep 17 00:00:00 2001 From: Alena Prokharchyk Date: Mon, 10 Feb 2014 10:21:34 -0800 Subject: [PATCH 095/169] CLOUDSTACK-6068: set display flag to true in service/disk_offering_details tables --- setup/db/db/schema-421to430.sql | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/setup/db/db/schema-421to430.sql b/setup/db/db/schema-421to430.sql index 43e94db1d80..dc6131dd06c 100644 --- a/setup/db/db/schema-421to430.sql +++ b/setup/db/db/schema-421to430.sql @@ -173,7 +173,7 @@ CREATE VIEW `cloud`.`disk_offering_view` AS DROP VIEW IF EXISTS `cloud`.`service_offering_view`; CREATE VIEW `cloud`.`service_offering_view` AS - select + select service_offering.id, disk_offering.uuid, disk_offering.name, @@ -212,10 +212,10 @@ CREATE VIEW `cloud`.`service_offering_view` AS `cloud`.`domain` ON disk_offering.domain_id = domain.id where disk_offering.state='Active'; - + DROP VIEW IF EXISTS `cloud`.`template_view`; CREATE VIEW `cloud`.`template_view` AS - select + select vm_template.id, vm_template.uuid, vm_template.unique_name, @@ -257,7 +257,7 @@ CREATE VIEW `cloud`.`template_view` AS domain.path domain_path, projects.id project_id, projects.uuid project_uuid, - projects.name project_name, + projects.name project_name, data_center.id data_center_id, data_center.uuid data_center_uuid, data_center.name data_center_name, @@ -287,23 +287,23 @@ CREATE VIEW `cloud`.`template_view` AS from `cloud`.`vm_template` inner join - `cloud`.`guest_os` ON guest_os.id = vm_template.guest_os_id + `cloud`.`guest_os` ON guest_os.id = vm_template.guest_os_id inner join `cloud`.`account` ON account.id = vm_template.account_id inner join `cloud`.`domain` ON domain.id = account.domain_id left join - `cloud`.`projects` ON projects.project_account_id = account.id + `cloud`.`projects` ON projects.project_account_id = account.id left join - `cloud`.`vm_template_details` ON vm_template_details.template_id = vm_template.id + `cloud`.`vm_template_details` ON vm_template_details.template_id = vm_template.id left join - `cloud`.`vm_template` source_template ON source_template.id = vm_template.source_template_id + `cloud`.`vm_template` source_template ON source_template.id = vm_template.source_template_id left join `cloud`.`template_store_ref` ON template_store_ref.template_id = vm_template.id and template_store_ref.store_role = 'Image' left join - `cloud`.`image_store` ON image_store.removed is NULL AND template_store_ref.store_id is not NULL AND image_store.id = template_store_ref.store_id + `cloud`.`image_store` ON image_store.removed is NULL AND template_store_ref.store_id is not NULL AND image_store.id = template_store_ref.store_id left join - `cloud`.`template_zone_ref` ON template_zone_ref.template_id = vm_template.id AND template_store_ref.store_id is NULL AND template_zone_ref.removed is null + `cloud`.`template_zone_ref` ON template_zone_ref.template_id = vm_template.id AND template_store_ref.store_id is NULL AND template_zone_ref.removed is null left join `cloud`.`data_center` ON (image_store.data_center_id = data_center.id OR template_zone_ref.zone_id = data_center.id) left join @@ -420,7 +420,7 @@ CREATE VIEW `cloud`.`volume_view` AS `cloud`.`async_job` ON async_job.instance_id = volumes.id and async_job.instance_type = 'Volume' and async_job.job_status = 0; - + DROP VIEW IF EXISTS `cloud`.`storage_pool_view`; CREATE VIEW `cloud`.`storage_pool_view` AS select @@ -1093,3 +1093,9 @@ CREATE VIEW `cloud`.`user_vm_view` AS INSERT IGNORE INTO `cloud`.`configuration`(category, instance, component, name, value, description, default_value) VALUES ('NetworkManager', 'DEFAULT', 'management-server', 'network.router.EnableServiceMonitoring', 'true', 'service monitoring in router enable/disable option, default true', 'true') ON DUPLICATE KEY UPDATE category='NetworkManager'; + +ALTER TABLE `cloud`.`service_offering_details` CHANGE `display` `display` tinyint(1) NOT NULL DEFAULT '1' COMMENT 'True if the detail can be displayed to the end user'; +UPDATE `cloud`.`service_offering_details` set `display`=1 where id> 0; + +ALTER TABLE `cloud`.`disk_offering_details` CHANGE `display` `display` tinyint(1) NOT NULL DEFAULT '1' COMMENT 'True if the detail can be displayed to the end user'; +UPDATE `cloud`.`disk_offering_details` set `display`=1 where id> 0; From 1cfbfab8162b82b30d797d4805112a75cfef3ce0 Mon Sep 17 00:00:00 2001 From: Alena Prokharchyk Date: Mon, 10 Feb 2014 17:15:50 -0800 Subject: [PATCH 096/169] CLOUDSTACK-6068: set display flag to true in service/disk_offering_details tables --- setup/db/db/schema-421to430.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/db/db/schema-421to430.sql b/setup/db/db/schema-421to430.sql index dc6131dd06c..b55e3881852 100644 --- a/setup/db/db/schema-421to430.sql +++ b/setup/db/db/schema-421to430.sql @@ -1097,5 +1097,5 @@ INSERT IGNORE INTO `cloud`.`configuration`(category, instance, component, name, ALTER TABLE `cloud`.`service_offering_details` CHANGE `display` `display` tinyint(1) NOT NULL DEFAULT '1' COMMENT 'True if the detail can be displayed to the end user'; UPDATE `cloud`.`service_offering_details` set `display`=1 where id> 0; -ALTER TABLE `cloud`.`disk_offering_details` CHANGE `display` `display` tinyint(1) NOT NULL DEFAULT '1' COMMENT 'True if the detail can be displayed to the end user'; -UPDATE `cloud`.`disk_offering_details` set `display`=1 where id> 0; +ALTER TABLE `cloud`.`disk_offering_details` CHANGE `display_detail` `display_detail` tinyint(1) NOT NULL DEFAULT '1' COMMENT 'True if the detail can be displayed to the end user'; +UPDATE `cloud`.`disk_offering_details` set `display_detail`=1 where id> 0; From eef65ac776de171d83f9f532ac2dda6902117178 Mon Sep 17 00:00:00 2001 From: Anshul Gangwar Date: Mon, 10 Feb 2014 09:41:41 -0800 Subject: [PATCH 097/169] CLOUDSTACK-6054: Changes for making vmsync work for hyper-v. Made changes to PingCommand and StartupCommand to return the state of all vms on the host. --- .../HypervResource/CloudStackTypes.cs | 1 + .../HypervResourceController.cs | 34 +++++++++++++ .../HypervResource/IWmiCallsV2.cs | 1 + .../HypervResource/WmiCallsV2.cs | 25 +++++++++ .../resource/HypervDirectConnectResource.java | 51 ++++++++++++++++++- 5 files changed, 110 insertions(+), 2 deletions(-) diff --git a/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/CloudStackTypes.cs b/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/CloudStackTypes.cs index 96379944d9f..889ed29ce74 100644 --- a/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/CloudStackTypes.cs +++ b/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/CloudStackTypes.cs @@ -900,5 +900,6 @@ namespace HypervResource public const string DeleteCommand = "org.apache.cloudstack.storage.command.DeleteCommand"; public const string DettachAnswer = "org.apache.cloudstack.storage.command.DettachAnswer"; public const string DettachCommand = "org.apache.cloudstack.storage.command.DettachCommand"; + public const string HostVmStateReportCommand = "org.apache.cloudstack.HostVmStateReportCommand"; } } diff --git a/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/HypervResourceController.cs b/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/HypervResourceController.cs index a1326064694..5db58e698f9 100644 --- a/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/HypervResourceController.cs +++ b/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/HypervResourceController.cs @@ -2042,6 +2042,40 @@ namespace HypervResource } } + // POST api/HypervResource/HostVmStateReportCommand + [HttpPost] + [ActionName(CloudStackTypes.HostVmStateReportCommand)] + public JContainer HostVmStateReportCommand([FromBody]dynamic cmd) + { + using (log4net.NDC.Push(Guid.NewGuid().ToString())) + { + logger.Info(CloudStackTypes.HostVmStateReportCommand + cmd.ToString()); + + string details = null; + Dictionary[] hostVmStateReport = null; + + try + { + var vmCollection = wmiCallsV2.GetComputerSystemCollection(); + hostVmStateReport = new Dictionary[vmCollection.Count]; + int i = 0; + foreach (ComputerSystem vm in vmCollection) + { + var dict = new Dictionary(); + dict.Add(vm.ElementName, EnabledState.ToCloudStackPowerState(vm.EnabledState)); + hostVmStateReport[i++] = dict; + } + } + catch (Exception sysEx) + { + details = CloudStackTypes.HostVmStateReportCommand + " failed due to " + sysEx.Message; + logger.Error(details, sysEx); + } + + return JArray.FromObject(hostVmStateReport); + } + } + public static System.Net.NetworkInformation.NetworkInterface GetNicInfoFromIpAddress(string ipAddress, out string subnet) { System.Net.NetworkInformation.NetworkInterface[] nics = System.Net.NetworkInformation.NetworkInterface.GetAllNetworkInterfaces(); diff --git a/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/IWmiCallsV2.cs b/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/IWmiCallsV2.cs index 5f814c59535..0fe6a754139 100644 --- a/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/IWmiCallsV2.cs +++ b/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/IWmiCallsV2.cs @@ -42,6 +42,7 @@ namespace HypervResource void MigrateVm(string vmName, string destination); void DetachDisk(string displayName, string diskFileName); ComputerSystem GetComputerSystem(string displayName); + ComputerSystem.ComputerSystemCollection GetComputerSystemCollection(); string GetDefaultDataRoot(); string GetDefaultVirtualDiskFolder(); ResourceAllocationSettingData GetDvdDriveSettings(VirtualSystemSettingData vmSettings); diff --git a/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/WmiCallsV2.cs b/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/WmiCallsV2.cs index f5a04dce847..991ef558272 100644 --- a/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/WmiCallsV2.cs +++ b/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/WmiCallsV2.cs @@ -1812,6 +1812,12 @@ namespace HypervResource return null; } + public ComputerSystem.ComputerSystemCollection GetComputerSystemCollection() + { + var wmiQuery = String.Format("Caption=\"Virtual Machine\""); + return ComputerSystem.GetInstances(wmiQuery); + } + public Dictionary GetVmSync(String privateIpAddress) { List vms = GetVmElementNames(); @@ -2412,5 +2418,24 @@ namespace HypervResource } return result; } + + public static string ToCloudStackPowerState(UInt16 value) + { + string result = "Unknown"; + switch (value) + { + case Enabled: result = "PowerOn"; break; + case Disabled: result = "PowerOff"; break; + case Paused: result = "PowerUnknown"; break; + case Suspended: result = "PowerUnknown"; break; + case Starting: result = "PowerOn"; break; + case Snapshotting: result = "PowerUnknown"; break; // NOT used + case Saving: result = "PowerOn"; break; + case Stopping: result = "PowerOff"; break; + case Pausing: result = "PowerUnknown"; break; + case Resuming: result = "PowerOn"; break; + } + return result; + } } } diff --git a/plugins/hypervisors/hyperv/src/com/cloud/hypervisor/hyperv/resource/HypervDirectConnectResource.java b/plugins/hypervisors/hyperv/src/com/cloud/hypervisor/hyperv/resource/HypervDirectConnectResource.java index a4dca6a3fa7..0904d5c5bda 100644 --- a/plugins/hypervisors/hyperv/src/com/cloud/hypervisor/hyperv/resource/HypervDirectConnectResource.java +++ b/plugins/hypervisors/hyperv/src/com/cloud/hypervisor/hyperv/resource/HypervDirectConnectResource.java @@ -62,6 +62,7 @@ import org.apache.http.util.EntityUtils; import org.apache.log4j.Logger; import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; import com.cloud.agent.api.Answer; import com.cloud.agent.api.CheckRouterAnswer; @@ -132,6 +133,7 @@ import com.cloud.utils.StringUtils; import com.cloud.utils.net.NetUtils; import com.cloud.utils.ssh.SshHelper; import com.cloud.vm.VirtualMachine; +import com.cloud.vm.VirtualMachine.PowerState; import com.cloud.vm.VirtualMachineName; /** * Implementation of dummy resource to be returned from discoverer. @@ -139,6 +141,7 @@ import com.cloud.vm.VirtualMachineName; @Local(value = ServerResource.class) public class HypervDirectConnectResource extends ServerResourceBase implements ServerResource { public static final int DEFAULT_AGENT_PORT = 8250; + public static final String HOST_VM_STATE_REPORT_COMMAND = "org.apache.cloudstack.HostVmStateReportCommand"; private static final Logger s_logger = Logger.getLogger(HypervDirectConnectResource.class.getName()); private static final Gson s_gson = GsonHelper.getGson(); @@ -206,6 +209,7 @@ public class HypervDirectConnectResource extends ServerResourceBase implements S defaultStartRoutCmd.setPrivateIpAddress(_agentIp); defaultStartRoutCmd.setStorageIpAddress(_agentIp); defaultStartRoutCmd.setPool(_clusterGuid); + defaultStartRoutCmd.setHostVmStateReport(getHostVmStateReport()); s_logger.debug("Generated StartupRoutingCommand for _agentIp \"" + _agentIp + "\""); @@ -328,8 +332,7 @@ public class HypervDirectConnectResource extends ServerResourceBase implements S @Override public final PingCommand getCurrentStatus(final long id) { - // TODO, need to report VM states on host - PingCommand pingCmd = new PingRoutingCommand(getType(), id, null, null); + PingCommand pingCmd = new PingRoutingCommand(getType(), id, null, getHostVmStateReport()); if (s_logger.isDebugEnabled()) { s_logger.debug("Ping host " + _name + " (IP " + _agentIp + ")"); @@ -345,6 +348,50 @@ public class HypervDirectConnectResource extends ServerResourceBase implements S return pingCmd; } + public final ArrayList> requestHostVmStateReport() { + URI agentUri = null; + try { + agentUri = new URI("https", null, _agentIp, _port, "/api/HypervResource/" + HOST_VM_STATE_REPORT_COMMAND, null, null); + } catch (URISyntaxException e) { + String errMsg = "Could not generate URI for Hyper-V agent"; + s_logger.error(errMsg, e); + return null; + } + String incomingCmd = postHttpRequest("{}", agentUri); + + if (incomingCmd == null) { + return null; + } + ArrayList> result = null; + try { + result = s_gson.fromJson(incomingCmd, new TypeToken>>() { + }.getType()); + } catch (Exception ex) { + String errMsg = "Failed to deserialize Command[] " + incomingCmd; + s_logger.error(errMsg, ex); + } + s_logger.debug("HostVmStateReportCommand received response " + + s_gson.toJson(result)); + if (!result.isEmpty()) { + return result; + } + return null; + } + + protected HashMap getHostVmStateReport() { + final HashMap vmStates = new HashMap(); + ArrayList> vmList = requestHostVmStateReport(); + if (vmList == null || vmList.isEmpty()) { + return null; + } + + for (Map vmMap : vmList) { + String name = (String)vmMap.keySet().toArray()[0]; + vmStates.put(name, new HostVmStateReportEntry(PowerState.valueOf(vmMap.get(name)), _guid, null)); + } + return vmStates; + } + // TODO: Is it valid to return NULL, or should we throw on error? // Returns StartupCommand with fields revised with values known only to the // host From 3f1bed64ebd46b09cee4b1dd7c25d86640880f5c Mon Sep 17 00:00:00 2001 From: Gaurav Aradhye Date: Wed, 12 Feb 2014 10:04:39 +0530 Subject: [PATCH 098/169] CLOUDSTACK-6066: Improved migration code to handle scenarios according to hypervisor type --- test/integration/smoke/test_vm_life_cycle.py | 84 ++++++++++++-------- 1 file changed, 52 insertions(+), 32 deletions(-) diff --git a/test/integration/smoke/test_vm_life_cycle.py b/test/integration/smoke/test_vm_life_cycle.py index ff37c997136..c164f59985e 100644 --- a/test/integration/smoke/test_vm_life_cycle.py +++ b/test/integration/smoke/test_vm_life_cycle.py @@ -5,9 +5,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -17,12 +17,27 @@ """ BVT tests for Virtual Machine Life Cycle """ #Import Local Modules -import marvin -from marvin.cloudstackTestCase import * -from marvin.cloudstackAPI import * -from marvin.integration.lib.utils import * -from marvin.integration.lib.base import * -from marvin.integration.lib.common import * +from marvin.cloudstackTestCase import cloudstackTestCase +from marvin.cloudstackAPI import (detachIso, + attachIso, + recoverVirtualMachine, + destroyVirtualMachine) +from marvin.integration.lib.utils import (cleanup_resources, + validateList, + get_hypervisor_type) +from marvin.integration.lib.base import (Account, + ServiceOffering, + VirtualMachine, + Iso, + Host) +from marvin.integration.lib.common import (get_domain, + get_zone, + get_template, + list_virtual_machines, + list_configurations, + list_routers, + list_isos) +from marvin.codes import PASS from nose.plugins.attrib import attr #Import System modules import time @@ -44,12 +59,12 @@ class Services: "firstname": "Test", "lastname": "User", "username": "test", - # Random characters are appended in create account to + # Random characters are appended in create account to # ensure unique username generated each time "password": "password", }, "small": - # Create a small virtual machine instance with disk offering + # Create a small virtual machine instance with disk offering { "displayname": "testserver", "username": "root", # VM creds for SSH @@ -60,7 +75,7 @@ class Services: "publicport": 22, "protocol": 'TCP', }, - "medium": # Create a medium virtual machine instance + "medium": # Create a medium virtual machine instance { "displayname": "testserver", "username": "root", @@ -83,7 +98,7 @@ class Services: }, "small": { - # Small service offering ID to for change VM + # Small service offering ID to for change VM # service offering from medium to small "name": "Small Instance", "displaytext": "Small Instance", @@ -109,7 +124,7 @@ class Services: "url": "http://people.apache.org/~tsp/dummy.iso", # Source URL where ISO is located "ostype": 'CentOS 5.3 (64-bit)', - "mode": 'HTTP_DOWNLOAD', # Downloading existing ISO + "mode": 'HTTP_DOWNLOAD', # Downloading existing ISO }, "template": { "displaytext": "Cent OS Template", @@ -126,7 +141,6 @@ class Services: # CentOS 5.3 (64-bit) } - class TestDeployVM(cloudstackTestCase): @classmethod @@ -378,7 +392,7 @@ class TestVMLifeCycle(cloudstackTestCase): cleanup_resources(self.apiclient, self.cleanup) return - + @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"]) def test_01_stop_vm(self): """Test Stop Virtual Machine @@ -575,31 +589,37 @@ class TestVMLifeCycle(cloudstackTestCase): # 2. DeployVM on suitable host (with another host in the cluster) # 3. Migrate the VM and assert migration successful + suitable_hosts = None + hosts = Host.list( self.apiclient, zoneid=self.zone.id, type='Routing' ) - self.assertEqual( - isinstance(hosts, list), - True, - "Check the number of hosts in the zone" - ) - self.assertGreaterEqual( - len(hosts), - 2, - "Atleast 2 hosts should be present for VM migration" - ) + self.assertEqual(validateList(hosts)[0], PASS, "hosts list validation failed") - #identify suitable host - clusters = [h.clusterid for h in hosts] - #find hosts withe same clusterid - clusters = [cluster for index, cluster in enumerate(clusters) if clusters.count(cluster) > 1] + if len(hosts) < 2: + self.skipTest("At least two hosts should be present in the zone for migration") - if len(clusters) <= 1: - self.skipTest("Migration needs a cluster with at least two hosts") + hypervisor = str(get_hypervisor_type(self.apiclient)).lower() + + # For KVM, two hosts used for migration should be present in same cluster + # For XenServer and VMware, migration is possible between hosts belonging to different clusters + # with the help of XenMotion and Vmotion respectively. + + if hypervisor == "kvm": + #identify suitable host + clusters = [h.clusterid for h in hosts] + #find hosts withe same clusterid + clusters = [cluster for index, cluster in enumerate(clusters) if clusters.count(cluster) > 1] + + if len(clusters) <= 1: + self.skipTest("In KVM, Live Migration needs two hosts within same cluster") + + suitable_hosts = [host for host in hosts if host.clusterid == clusters[0]] + else: + suitable_hosts = hosts - suitable_hosts = [host for host in hosts if host.clusterid == clusters[0]] target_host = suitable_hosts[0] migrate_host = suitable_hosts[1] From 5af163ff7f658c30c6d93b50ac17c9cc0c5eb8c1 Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Tue, 11 Feb 2014 14:12:00 +0100 Subject: [PATCH 099/169] pass isSourceNat to second ip on the same network --- server/src/com/cloud/network/vpc/VpcManagerImpl.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/com/cloud/network/vpc/VpcManagerImpl.java b/server/src/com/cloud/network/vpc/VpcManagerImpl.java index f7c38d4a460..40e910dca07 100644 --- a/server/src/com/cloud/network/vpc/VpcManagerImpl.java +++ b/server/src/com/cloud/network/vpc/VpcManagerImpl.java @@ -1444,7 +1444,8 @@ public class VpcManagerImpl extends ManagerBase implements VpcManager, VpcProvis Long nextMac = mac + 1; dc.setMacAddress(nextMac); - privateIp = new PrivateIpVO(ipAddress, privateNtwk.getId(), nextMac, vpcId, true); + s_logger.info("creating private ip adress for vpc (" + ipAddress + ", " + privateNtwk.getId() + ", " + nextMac + ", " + vpcId + ", " + isSourceNat + ")"); + privateIp = new PrivateIpVO(ipAddress, privateNtwk.getId(), nextMac, vpcId, isSourceNat); _privateIpDao.persist(privateIp); _dcDao.update(dc.getId(), dc); From 06ae23710ddc3784939bcd499b7437c13495bb88 Mon Sep 17 00:00:00 2001 From: Marcus Sorensen Date: Wed, 12 Feb 2014 16:07:34 -0700 Subject: [PATCH 100/169] CLOUDSTACK-6089: Implement equals() method for ResourceTagResponse so that the java Set can properly determine if a ResourceTagResponse is unique. This ensures we don't get duplicate resource tags showing up any time a UserVmResponse is crafted (which can be quite often due to the way the responses are crafted). --- .../api/response/ResourceTagResponse.java | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/api/src/org/apache/cloudstack/api/response/ResourceTagResponse.java b/api/src/org/apache/cloudstack/api/response/ResourceTagResponse.java index 47b06250a2c..8044376c863 100644 --- a/api/src/org/apache/cloudstack/api/response/ResourceTagResponse.java +++ b/api/src/org/apache/cloudstack/api/response/ResourceTagResponse.java @@ -101,4 +101,34 @@ public class ResourceTagResponse extends BaseResponse implements ControlledViewE public void setCustomer(String customer) { this.customer = customer; } + + public String getResourceId() { + return this.resourceId; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + String rId = this.getResourceId(); + result = prime * result + ((rId== null) ? 0 : rId.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (this.getClass() != obj.getClass()) + return false; + ResourceTagResponse other = (ResourceTagResponse) obj; + String rId = this.getResourceId(); + if (rId == null && other.getResourceId() != null) { + return false; + } else if (!rId.equals(other.getResourceId())) + return false; + return true; + } } From ed73e3e1b30c7c49c5dbfe8f2cac3d3dac85090e Mon Sep 17 00:00:00 2001 From: Marcus Sorensen Date: Wed, 12 Feb 2014 17:00:10 -0700 Subject: [PATCH 101/169] CLOUDSTACK-6089: Use resource tag's key to determine match in equals() method for ResourceTagResponse --- .../api/response/ResourceTagResponse.java | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/api/src/org/apache/cloudstack/api/response/ResourceTagResponse.java b/api/src/org/apache/cloudstack/api/response/ResourceTagResponse.java index 8044376c863..00db1b5f5fd 100644 --- a/api/src/org/apache/cloudstack/api/response/ResourceTagResponse.java +++ b/api/src/org/apache/cloudstack/api/response/ResourceTagResponse.java @@ -102,16 +102,16 @@ public class ResourceTagResponse extends BaseResponse implements ControlledViewE this.customer = customer; } - public String getResourceId() { - return this.resourceId; + public String getKey() { + return this.key; } @Override public int hashCode() { final int prime = 31; int result = 1; - String rId = this.getResourceId(); - result = prime * result + ((rId== null) ? 0 : rId.hashCode()); + String key = this.getKey(); + result = prime * result + ((key == null) ? 0 : key.hashCode()); return result; } @@ -121,13 +121,14 @@ public class ResourceTagResponse extends BaseResponse implements ControlledViewE return true; if (obj == null) return false; - if (this.getClass() != obj.getClass()) + if (getClass() != obj.getClass()) return false; ResourceTagResponse other = (ResourceTagResponse) obj; - String rId = this.getResourceId(); - if (rId == null && other.getResourceId() != null) { - return false; - } else if (!rId.equals(other.getResourceId())) + String key = this.getKey(); + if (key == null) { + if (other.getKey() != null) + return false; + } else if (!key.equals(other.getKey())) return false; return true; } From e8f93f28fc424c73156723d9b65b13c05dafc5a8 Mon Sep 17 00:00:00 2001 From: Jayapal Date: Thu, 13 Feb 2014 14:24:42 +0530 Subject: [PATCH 102/169] CLOUDSTACK-6083 corrected firewall rule cidr load issue --- .../src/com/cloud/network/dao/FirewallRulesCidrsDaoImpl.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/engine/schema/src/com/cloud/network/dao/FirewallRulesCidrsDaoImpl.java b/engine/schema/src/com/cloud/network/dao/FirewallRulesCidrsDaoImpl.java index 52d52790f07..aa3b82e580b 100644 --- a/engine/schema/src/com/cloud/network/dao/FirewallRulesCidrsDaoImpl.java +++ b/engine/schema/src/com/cloud/network/dao/FirewallRulesCidrsDaoImpl.java @@ -39,8 +39,7 @@ public class FirewallRulesCidrsDaoImpl extends GenericDaoBase Date: Thu, 13 Feb 2014 14:34:06 -0800 Subject: [PATCH 103/169] CLOUDSTACK-6065: Fix NPE problem caused by the lack of context setup in threads from agent manager thread pool --- .../framework/jobs/AsyncJobExecutionContext.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/AsyncJobExecutionContext.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/AsyncJobExecutionContext.java index 966e638e3e5..4343a4d60a5 100644 --- a/framework/jobs/src/org/apache/cloudstack/framework/jobs/AsyncJobExecutionContext.java +++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/AsyncJobExecutionContext.java @@ -29,6 +29,8 @@ import org.apache.cloudstack.managed.threadlocal.ManagedThreadLocal; import com.cloud.exception.ConcurrentOperationException; import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.ResourceUnavailableException; +import com.cloud.user.Account; +import com.cloud.user.User; public class AsyncJobExecutionContext { private static final Logger s_logger = Logger.getLogger(AsyncJobExecutionContext.class); @@ -167,10 +169,15 @@ public class AsyncJobExecutionContext { public static AsyncJobExecutionContext getCurrentExecutionContext() { AsyncJobExecutionContext context = s_currentExectionContext.get(); if (context == null) { - // TODO, this has security implicitions + // TODO, this has security implications, operations carried from API layer should always + // set its context, otherwise, the fall-back here will use system security context + // s_logger.warn("Job is executed without a context, setup psudo job for the executing thread"); - context = registerPseudoExecutionContext(CallContext.current().getCallingAccountId(), - CallContext.current().getCallingUserId()); + if (CallContext.current() != null) + context = registerPseudoExecutionContext(CallContext.current().getCallingAccountId(), + CallContext.current().getCallingUserId()); + else + context = registerPseudoExecutionContext(Account.ACCOUNT_ID_SYSTEM, User.UID_SYSTEM); } return context; } From 40d069a6e4ee058515f625f6e76d56d986a4a9d9 Mon Sep 17 00:00:00 2001 From: Brian Federle Date: Fri, 24 Jan 2014 10:47:26 -0800 Subject: [PATCH 104/169] Disable root disk size field -- not supported in backend --- ui/scripts/instanceWizard.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ui/scripts/instanceWizard.js b/ui/scripts/instanceWizard.js index 646ffbf5b19..4a740c6685e 100644 --- a/ui/scripts/instanceWizard.js +++ b/ui/scripts/instanceWizard.js @@ -239,7 +239,11 @@ templates: templatesObj, hypervisors: hypervisorObjs }, - customHidden: function(args) { + customHidden: function(args) { + //// + return true; // Disabled -- not supported in backend right now + //// + if (selectedTemplateOrIso == 'select-template') { return false; //show Root Disk Size field } else { //selectedTemplateOrIso == 'select-iso' From 76e019f717ca27f5124aebc9db3c5d807bb72868 Mon Sep 17 00:00:00 2001 From: Marcus Sorensen Date: Fri, 14 Feb 2014 00:43:23 -0700 Subject: [PATCH 105/169] CLOUDSTACK-6103: Pass VM iso information along with PrepareForMigrationCommand, so that destination hypervisor can mount pool. This further exposed an issue for KVM where iso was not getting cleaned up upon successful migration, fixed as well. --- .../com/cloud/template/TemplateManager.java | 9 ++++- .../orchestration/VolumeOrchestrator.java | 10 +++--- .../resource/LibvirtComputingResource.java | 6 +++- .../cloud/template/TemplateManagerImpl.java | 35 +++++++++++++++++++ .../src/com/cloud/vm/UserVmManagerImpl.java | 31 ++-------------- 5 files changed, 56 insertions(+), 35 deletions(-) diff --git a/engine/components-api/src/com/cloud/template/TemplateManager.java b/engine/components-api/src/com/cloud/template/TemplateManager.java index 3adce4f1504..13b0cdfd3fd 100755 --- a/engine/components-api/src/com/cloud/template/TemplateManager.java +++ b/engine/components-api/src/com/cloud/template/TemplateManager.java @@ -32,6 +32,7 @@ import com.cloud.storage.VMTemplateHostVO; import com.cloud.storage.VMTemplateStoragePoolVO; import com.cloud.storage.VMTemplateVO; import com.cloud.utils.Pair; +import com.cloud.vm.VirtualMachineProfile; /** * TemplateManager manages the templates stored on secondary storage. It is responsible for creating private/public templates. @@ -115,7 +116,13 @@ public interface TemplateManager { TemplateInfo prepareIso(long isoId, long dcId); - + + /** + * Adds ISO definition to given vm profile + * + * @param VirtualMachineProfile + */ + void prepareIsoForVmProfile(VirtualMachineProfile profile); } diff --git a/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java index a3ab5a4164a..2f181ab0640 100644 --- a/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java +++ b/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java @@ -1017,10 +1017,12 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati vm.addDisk(disk); } - if (vm.getType() == VirtualMachine.Type.User && vm.getTemplate().getFormat() == ImageFormat.ISO) { - DataTO dataTO = tmplFactory.getTemplate(vm.getTemplate().getId(), DataStoreRole.Image, vm.getVirtualMachine().getDataCenterId()).getTO(); - DiskTO iso = new DiskTO(dataTO, 3L, null, Volume.Type.ISO); - vm.addDisk(iso); + //if (vm.getType() == VirtualMachine.Type.User && vm.getTemplate().getFormat() == ImageFormat.ISO) { + if (vm.getType() == VirtualMachine.Type.User) { + _tmpltMgr.prepareIsoForVmProfile(vm); + //DataTO dataTO = tmplFactory.getTemplate(vm.getTemplate().getId(), DataStoreRole.Image, vm.getVirtualMachine().getDataCenterId()).getTO(); + //DiskTO iso = new DiskTO(dataTO, 3L, null, Volume.Type.ISO); + //vm.addDisk(iso); } } diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 6e0c6fa1a85..34770e4164d 100755 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -2956,6 +2956,7 @@ ServerResource { } List ifaces = null; + List disks = null; Domain dm = null; Connect dconn = null; @@ -2965,6 +2966,7 @@ ServerResource { try { conn = LibvirtConnection.getConnectionByVmName(cmd.getVmName()); ifaces = getInterfaces(conn, vmName); + disks = getDisks(conn, vmName); dm = conn.domainLookupByName(vmName); /* We replace the private IP address with the address of the destination host. @@ -2990,7 +2992,9 @@ ServerResource { destDomain = dm.migrate(dconn, (1 << 0) | (1 << 3), xmlDesc, vmName, "tcp:" + cmd.getDestinationIp(), _migrateSpeed); - _storagePoolMgr.disconnectPhysicalDisksViaVmSpec(cmd.getVirtualMachine()); + for (DiskDef disk : disks) { + cleanupDisk(disk); + } } catch (LibvirtException e) { s_logger.debug("Can't migrate domain: " + e.getMessage()); result = e.getMessage(); diff --git a/server/src/com/cloud/template/TemplateManagerImpl.java b/server/src/com/cloud/template/TemplateManagerImpl.java index 3bfe5c8b0cd..b8a7a5f7790 100755 --- a/server/src/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/com/cloud/template/TemplateManagerImpl.java @@ -83,6 +83,7 @@ import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; +import org.apache.cloudstack.storage.to.TemplateObjectTO; import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; @@ -159,6 +160,7 @@ import com.cloud.storage.download.DownloadMonitor; import com.cloud.storage.secondary.SecondaryStorageVmManager; import com.cloud.storage.upload.UploadMonitor; import com.cloud.template.TemplateAdapter.TemplateAdapterType; +import com.cloud.template.VirtualMachineTemplate.BootloaderType; import com.cloud.user.Account; import com.cloud.user.AccountManager; import com.cloud.user.AccountService; @@ -184,6 +186,7 @@ import com.cloud.vm.UserVmManager; import com.cloud.vm.UserVmVO; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine.State; +import com.cloud.vm.VirtualMachineProfile; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.UserVmDetailsDao; import com.cloud.vm.dao.VMInstanceDao; @@ -492,6 +495,38 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, return tmpltStore.createEntityExtractUrl(templateObject.getInstallPath(), template.getFormat(), templateObject); } + @Override + public void prepareIsoForVmProfile(VirtualMachineProfile profile) { + UserVmVO vm = _userVmDao.findById(profile.getId()); + if (vm.getIsoId() != null) { + TemplateInfo template = prepareIso(vm.getIsoId(), vm.getDataCenterId()); + if (template == null){ + s_logger.error("Failed to prepare ISO on secondary or cache storage"); + throw new CloudRuntimeException("Failed to prepare ISO on secondary or cache storage"); + } + if (template.isBootable()) { + profile.setBootLoaderType(BootloaderType.CD); + } + + GuestOSVO guestOS = _guestOSDao.findById(template.getGuestOSId()); + String displayName = null; + if (guestOS != null) { + displayName = guestOS.getDisplayName(); + } + + TemplateObjectTO iso = (TemplateObjectTO)template.getTO(); + iso.setGuestOsType(displayName); + DiskTO disk = new DiskTO(iso, 3L, null, Volume.Type.ISO); + profile.addDisk(disk); + } else { + TemplateObjectTO iso = new TemplateObjectTO(); + iso.setFormat(ImageFormat.ISO); + DiskTO disk = new DiskTO(iso, 3L, null, Volume.Type.ISO); + profile.addDisk(disk); + } + + } + public void prepareTemplateInAllStoragePools(final VMTemplateVO template, long zoneId) { List pools = _poolDao.listByStatus(StoragePoolStatus.Up); for (final StoragePoolVO pool : pools) { diff --git a/server/src/com/cloud/vm/UserVmManagerImpl.java b/server/src/com/cloud/vm/UserVmManagerImpl.java index 81be19fd876..7cf4a24e36e 100755 --- a/server/src/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/com/cloud/vm/UserVmManagerImpl.java @@ -412,7 +412,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir @Inject VpcManager _vpcMgr; @Inject - TemplateManager templateMgr; + TemplateManager _templateMgr; @Inject protected GuestOSCategoryDao _guestOSCategoryDao; @Inject @@ -3182,34 +3182,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir UserVmVO vm = _vmDao.findById(profile.getId()); Map details = _vmDetailsDao.listDetailsKeyPairs(vm.getId()); vm.setDetails(details); - - if (vm.getIsoId() != null) { - TemplateInfo template = templateMgr.prepareIso(vm.getIsoId(), vm.getDataCenterId()); - if (template == null){ - s_logger.error("Failed to prepare ISO on secondary or cache storage"); - throw new CloudRuntimeException("Failed to prepare ISO on secondary or cache storage"); - } - if (template.isBootable()) { - profile.setBootLoaderType(BootloaderType.CD); - } - - GuestOSVO guestOS = _guestOSDao.findById(template.getGuestOSId()); - String displayName = null; - if (guestOS != null) { - displayName = guestOS.getDisplayName(); - } - - TemplateObjectTO iso = (TemplateObjectTO)template.getTO(); - iso.setGuestOsType(displayName); - DiskTO disk = new DiskTO(iso, 3L, null, Volume.Type.ISO); - profile.addDisk(disk); - } else { - TemplateObjectTO iso = new TemplateObjectTO(); - iso.setFormat(ImageFormat.ISO); - DiskTO disk = new DiskTO(iso, 3L, null, Volume.Type.ISO); - profile.addDisk(disk); - } - + _templateMgr.prepareIsoForVmProfile(profile); return true; } From ccc168c7044017ce8916cff923b9b18fbb18088c Mon Sep 17 00:00:00 2001 From: Rajesh Battala Date: Fri, 14 Feb 2014 18:23:13 +0530 Subject: [PATCH 106/169] Revert "CLOUDSTACK-6054: Changes for making vmsync work for hyper-v. Made changes to PingCommand and" This reverts commit eef65ac776de171d83f9f532ac2dda6902117178. This commit should not be in 4.3-forward and its already present in master --- .../HypervResource/CloudStackTypes.cs | 1 - .../HypervResourceController.cs | 34 ------------- .../HypervResource/IWmiCallsV2.cs | 1 - .../HypervResource/WmiCallsV2.cs | 25 --------- .../resource/HypervDirectConnectResource.java | 51 +------------------ 5 files changed, 2 insertions(+), 110 deletions(-) diff --git a/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/CloudStackTypes.cs b/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/CloudStackTypes.cs index 889ed29ce74..96379944d9f 100644 --- a/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/CloudStackTypes.cs +++ b/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/CloudStackTypes.cs @@ -900,6 +900,5 @@ namespace HypervResource public const string DeleteCommand = "org.apache.cloudstack.storage.command.DeleteCommand"; public const string DettachAnswer = "org.apache.cloudstack.storage.command.DettachAnswer"; public const string DettachCommand = "org.apache.cloudstack.storage.command.DettachCommand"; - public const string HostVmStateReportCommand = "org.apache.cloudstack.HostVmStateReportCommand"; } } diff --git a/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/HypervResourceController.cs b/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/HypervResourceController.cs index 5db58e698f9..a1326064694 100644 --- a/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/HypervResourceController.cs +++ b/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/HypervResourceController.cs @@ -2042,40 +2042,6 @@ namespace HypervResource } } - // POST api/HypervResource/HostVmStateReportCommand - [HttpPost] - [ActionName(CloudStackTypes.HostVmStateReportCommand)] - public JContainer HostVmStateReportCommand([FromBody]dynamic cmd) - { - using (log4net.NDC.Push(Guid.NewGuid().ToString())) - { - logger.Info(CloudStackTypes.HostVmStateReportCommand + cmd.ToString()); - - string details = null; - Dictionary[] hostVmStateReport = null; - - try - { - var vmCollection = wmiCallsV2.GetComputerSystemCollection(); - hostVmStateReport = new Dictionary[vmCollection.Count]; - int i = 0; - foreach (ComputerSystem vm in vmCollection) - { - var dict = new Dictionary(); - dict.Add(vm.ElementName, EnabledState.ToCloudStackPowerState(vm.EnabledState)); - hostVmStateReport[i++] = dict; - } - } - catch (Exception sysEx) - { - details = CloudStackTypes.HostVmStateReportCommand + " failed due to " + sysEx.Message; - logger.Error(details, sysEx); - } - - return JArray.FromObject(hostVmStateReport); - } - } - public static System.Net.NetworkInformation.NetworkInterface GetNicInfoFromIpAddress(string ipAddress, out string subnet) { System.Net.NetworkInformation.NetworkInterface[] nics = System.Net.NetworkInformation.NetworkInterface.GetAllNetworkInterfaces(); diff --git a/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/IWmiCallsV2.cs b/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/IWmiCallsV2.cs index 0fe6a754139..5f814c59535 100644 --- a/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/IWmiCallsV2.cs +++ b/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/IWmiCallsV2.cs @@ -42,7 +42,6 @@ namespace HypervResource void MigrateVm(string vmName, string destination); void DetachDisk(string displayName, string diskFileName); ComputerSystem GetComputerSystem(string displayName); - ComputerSystem.ComputerSystemCollection GetComputerSystemCollection(); string GetDefaultDataRoot(); string GetDefaultVirtualDiskFolder(); ResourceAllocationSettingData GetDvdDriveSettings(VirtualSystemSettingData vmSettings); diff --git a/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/WmiCallsV2.cs b/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/WmiCallsV2.cs index 991ef558272..f5a04dce847 100644 --- a/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/WmiCallsV2.cs +++ b/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/WmiCallsV2.cs @@ -1812,12 +1812,6 @@ namespace HypervResource return null; } - public ComputerSystem.ComputerSystemCollection GetComputerSystemCollection() - { - var wmiQuery = String.Format("Caption=\"Virtual Machine\""); - return ComputerSystem.GetInstances(wmiQuery); - } - public Dictionary GetVmSync(String privateIpAddress) { List vms = GetVmElementNames(); @@ -2418,24 +2412,5 @@ namespace HypervResource } return result; } - - public static string ToCloudStackPowerState(UInt16 value) - { - string result = "Unknown"; - switch (value) - { - case Enabled: result = "PowerOn"; break; - case Disabled: result = "PowerOff"; break; - case Paused: result = "PowerUnknown"; break; - case Suspended: result = "PowerUnknown"; break; - case Starting: result = "PowerOn"; break; - case Snapshotting: result = "PowerUnknown"; break; // NOT used - case Saving: result = "PowerOn"; break; - case Stopping: result = "PowerOff"; break; - case Pausing: result = "PowerUnknown"; break; - case Resuming: result = "PowerOn"; break; - } - return result; - } } } diff --git a/plugins/hypervisors/hyperv/src/com/cloud/hypervisor/hyperv/resource/HypervDirectConnectResource.java b/plugins/hypervisors/hyperv/src/com/cloud/hypervisor/hyperv/resource/HypervDirectConnectResource.java index 0904d5c5bda..a4dca6a3fa7 100644 --- a/plugins/hypervisors/hyperv/src/com/cloud/hypervisor/hyperv/resource/HypervDirectConnectResource.java +++ b/plugins/hypervisors/hyperv/src/com/cloud/hypervisor/hyperv/resource/HypervDirectConnectResource.java @@ -62,7 +62,6 @@ import org.apache.http.util.EntityUtils; import org.apache.log4j.Logger; import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; import com.cloud.agent.api.Answer; import com.cloud.agent.api.CheckRouterAnswer; @@ -133,7 +132,6 @@ import com.cloud.utils.StringUtils; import com.cloud.utils.net.NetUtils; import com.cloud.utils.ssh.SshHelper; import com.cloud.vm.VirtualMachine; -import com.cloud.vm.VirtualMachine.PowerState; import com.cloud.vm.VirtualMachineName; /** * Implementation of dummy resource to be returned from discoverer. @@ -141,7 +139,6 @@ import com.cloud.vm.VirtualMachineName; @Local(value = ServerResource.class) public class HypervDirectConnectResource extends ServerResourceBase implements ServerResource { public static final int DEFAULT_AGENT_PORT = 8250; - public static final String HOST_VM_STATE_REPORT_COMMAND = "org.apache.cloudstack.HostVmStateReportCommand"; private static final Logger s_logger = Logger.getLogger(HypervDirectConnectResource.class.getName()); private static final Gson s_gson = GsonHelper.getGson(); @@ -209,7 +206,6 @@ public class HypervDirectConnectResource extends ServerResourceBase implements S defaultStartRoutCmd.setPrivateIpAddress(_agentIp); defaultStartRoutCmd.setStorageIpAddress(_agentIp); defaultStartRoutCmd.setPool(_clusterGuid); - defaultStartRoutCmd.setHostVmStateReport(getHostVmStateReport()); s_logger.debug("Generated StartupRoutingCommand for _agentIp \"" + _agentIp + "\""); @@ -332,7 +328,8 @@ public class HypervDirectConnectResource extends ServerResourceBase implements S @Override public final PingCommand getCurrentStatus(final long id) { - PingCommand pingCmd = new PingRoutingCommand(getType(), id, null, getHostVmStateReport()); + // TODO, need to report VM states on host + PingCommand pingCmd = new PingRoutingCommand(getType(), id, null, null); if (s_logger.isDebugEnabled()) { s_logger.debug("Ping host " + _name + " (IP " + _agentIp + ")"); @@ -348,50 +345,6 @@ public class HypervDirectConnectResource extends ServerResourceBase implements S return pingCmd; } - public final ArrayList> requestHostVmStateReport() { - URI agentUri = null; - try { - agentUri = new URI("https", null, _agentIp, _port, "/api/HypervResource/" + HOST_VM_STATE_REPORT_COMMAND, null, null); - } catch (URISyntaxException e) { - String errMsg = "Could not generate URI for Hyper-V agent"; - s_logger.error(errMsg, e); - return null; - } - String incomingCmd = postHttpRequest("{}", agentUri); - - if (incomingCmd == null) { - return null; - } - ArrayList> result = null; - try { - result = s_gson.fromJson(incomingCmd, new TypeToken>>() { - }.getType()); - } catch (Exception ex) { - String errMsg = "Failed to deserialize Command[] " + incomingCmd; - s_logger.error(errMsg, ex); - } - s_logger.debug("HostVmStateReportCommand received response " - + s_gson.toJson(result)); - if (!result.isEmpty()) { - return result; - } - return null; - } - - protected HashMap getHostVmStateReport() { - final HashMap vmStates = new HashMap(); - ArrayList> vmList = requestHostVmStateReport(); - if (vmList == null || vmList.isEmpty()) { - return null; - } - - for (Map vmMap : vmList) { - String name = (String)vmMap.keySet().toArray()[0]; - vmStates.put(name, new HostVmStateReportEntry(PowerState.valueOf(vmMap.get(name)), _guid, null)); - } - return vmStates; - } - // TODO: Is it valid to return NULL, or should we throw on error? // Returns StartupCommand with fields revised with values known only to the // host From c8422b3b50769b4064e7a497b0b6cd09a736fd26 Mon Sep 17 00:00:00 2001 From: Ashutosh K Date: Tue, 18 Feb 2014 15:51:57 +0530 Subject: [PATCH 107/169] CLOUDSTACK-4840: Adding first set of test cases for Multiple IPs per NIC feature --- tools/marvin/marvin/codes.py | 4 ++ tools/marvin/marvin/integration/lib/base.py | 29 ++++++++++ tools/marvin/marvin/integration/lib/common.py | 56 ++++++++++++++++++- 3 files changed, 87 insertions(+), 2 deletions(-) diff --git a/tools/marvin/marvin/codes.py b/tools/marvin/marvin/codes.py index 74fb05da35c..3882f0d59b8 100644 --- a/tools/marvin/marvin/codes.py +++ b/tools/marvin/marvin/codes.py @@ -47,3 +47,7 @@ YES = "yes" FAILED = "FAILED" UNKNOWN_ERROR = "Unknown Error" EXCEPTION = "EXCEPTION" +BASIC_ZONE = "basic" +ISOLATED_NETWORK = "ISOLATED" +SHARED_NETWORK = "SHARED" +VPC_NETWORK = "VPC" diff --git a/tools/marvin/marvin/integration/lib/base.py b/tools/marvin/marvin/integration/lib/base.py index 409530c2282..f76f07e24cb 100755 --- a/tools/marvin/marvin/integration/lib/base.py +++ b/tools/marvin/marvin/integration/lib/base.py @@ -3626,3 +3626,32 @@ class Resources: cmd = updateResourceCount.updateResourceCountCmd() [setattr(cmd, k, v) for k, v in kwargs.items()] return(apiclient.updateResourceCount(cmd)) + +class NIC: + """NIC related API""" + def __init__(self, items): + self.__dict__.update(items) + + @classmethod + def addIp(cls, apiclient, id, ipaddress=None): + """Add Ip (secondary) to NIC""" + cmd = addIpToNic.addIpToNicCmd() + cmd.nicid = id + if ipaddress: + cmd.ipaddress = ipaddress + return(apiclient.addIpToNic(cmd)) + + @classmethod + def removeIp(cls,apiclient,ipaddressid): + """Remove secondary Ip from NIC""" + cmd = removeIpFromNic.removeIpFromNicCmd() + cmd.id = ipaddressid + return(apiclient.addIpToNic(cmd)) + + @classmethod + def list(cls, apiclient, **kwargs): + """List NICs belonging to a virtual machine""" + + cmd = listNics.listNicsCmd() + [setattr(cmd, k, v) for k, v in kwargs.items()] + return(apiclient.listNics(cmd)) diff --git a/tools/marvin/marvin/integration/lib/common.py b/tools/marvin/marvin/integration/lib/common.py index e202391e521..5b56c733563 100644 --- a/tools/marvin/marvin/integration/lib/common.py +++ b/tools/marvin/marvin/integration/lib/common.py @@ -63,13 +63,14 @@ from marvin.integration.lib.base import (Configurations, Resources, PhysicalNetwork, Host, - PublicIPAddress) + PublicIPAddress, + NetworkOffering) from marvin.integration.lib.utils import (get_process_status, xsplit, validateList) from marvin.sshClient import SshClient -from marvin.codes import PASS +from marvin.codes import PASS, ISOLATED_NETWORK, VPC_NETWORK, BASIC_ZONE, FAIL import random #Import System modules @@ -917,3 +918,54 @@ def is_public_ip_in_correct_state(apiclient, ipaddressid, state): time.sleep(60) continue return True + +def setSharedNetworkParams(networkServices, range=20): + """Fill up the services dictionary for shared network using random subnet""" + + # @range: range decides the endip. Pass the range as "x" if you want the difference between the startip + # and endip as "x" + # Set the subnet number of shared networks randomly prior to execution + # of each test case to avoid overlapping of ip addresses + shared_network_subnet_number = random.randrange(1,254) + + networkServices["gateway"] = "172.16."+str(shared_network_subnet_number)+".1" + networkServices["startip"] = "172.16."+str(shared_network_subnet_number)+".2" + networkServices["endip"] = "172.16."+str(shared_network_subnet_number)+"."+str(range+1) + networkServices["netmask"] = "255.255.255.0" + return networkServices + +def createEnabledNetworkOffering(apiclient, networkServices): + """Create and enable network offering according to the type + + @output: List, containing [ Result,Network Offering,Reason ] + Ist Argument('Result') : FAIL : If exception or assertion error occurs + PASS : If network offering + is created and enabled successfully + IInd Argument(Net Off) : Enabled network offering + In case of exception or + assertion error, it will be None + IIIrd Argument(Reason) : Reason for failure, + default to None + """ + try: + resultSet = [FAIL, None, None] + # Create network offering + network_offering = NetworkOffering.create(apiclient, networkServices, conservemode=False) + + # Update network offering state from disabled to enabled. + NetworkOffering.update(network_offering, apiclient, id=network_offering.id, + state="enabled") + except Exception as e: + resultSet[2] = e + return resultSet + return [PASS, network_offering, None] + +def shouldTestBeSkipped(networkType, zoneType): + """Decide which test to skip, according to type of network and zone type""" + + # If network type is isolated or vpc and zone type is basic, then test should be skipped + skipIt = False + if ((networkType.lower() == str(ISOLATED_NETWORK).lower() or networkType.lower() == str(VPC_NETWORK).lower()) + and (zoneType.lower() == BASIC_ZONE)): + skipIt = True + return skipIt From 6db3cb26184e0209fde41cbbc3aba6ac9950823f Mon Sep 17 00:00:00 2001 From: Girish Shilamkar Date: Wed, 19 Feb 2014 11:15:10 +0530 Subject: [PATCH 108/169] CLOUDSTACK-4840: Add missed testcase in previous commit --- .../component/test_multiple_ips_per_nic.py | 452 ++++++++++++++++++ 1 file changed, 452 insertions(+) create mode 100644 test/integration/component/test_multiple_ips_per_nic.py diff --git a/test/integration/component/test_multiple_ips_per_nic.py b/test/integration/component/test_multiple_ips_per_nic.py new file mode 100644 index 00000000000..7d7e9f1ade2 --- /dev/null +++ b/test/integration/component/test_multiple_ips_per_nic.py @@ -0,0 +1,452 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" Tests for Multiple IPs per NIC feature + + Test Plan: https://cwiki.apache.org/confluence/display/CLOUDSTACK/Multiple+IPs+per+NIC+Test+Plan + + Issue Link: https://issues.apache.org/jira/browse/CLOUDSTACK-4840 + + Design Document: https://cwiki.apache.org/confluence/display/CLOUDSTACK/Multiple+IP+address+per+NIC +""" +from marvin.cloudstackTestCase import cloudstackTestCase +from marvin.integration.lib.utils import (cleanup_resources, + validateList, + random_gen) +from marvin.integration.lib.base import (Account, + ServiceOffering, + Network, + VirtualMachine, + VpcOffering, + VPC, + NIC, + Domain, + PublicIPAddress) +from marvin.integration.lib.common import (get_domain, + get_zone, + get_template, + get_free_vlan, + setSharedNetworkParams, + createEnabledNetworkOffering, + shouldTestBeSkipped) + +from nose.plugins.attrib import attr +from marvin.codes import PASS, ISOLATED_NETWORK, VPC_NETWORK, SHARED_NETWORK +from ddt import ddt, data + +def createNetwork(self, networkType): + """Create a network of given type (isolated/shared/isolated in VPC)""" + + network = None + + if networkType == ISOLATED_NETWORK: + try: + network = Network.create(self.apiclient,self.services["isolated_network"], + networkofferingid=self.isolated_network_offering.id, + accountid=self.account.name,domainid=self.account.domainid, + zoneid=self.zone.id) + except Exception as e: + self.fail("Isolated network creation failed because: %s" % e) + + elif networkType == SHARED_NETWORK: + physical_network, vlan = get_free_vlan(self.api_client, self.zone.id) + + #create network using the shared network offering created + self.services["shared_network"]["acltype"] = "domain" + self.services["shared_network"]["vlan"] = vlan + self.services["shared_network"]["networkofferingid"] = self.shared_network_offering.id + self.services["shared_network"]["physicalnetworkid"] = physical_network.id + + self.services["shared_network"] = setSharedNetworkParams(self.services["shared_network"]) + + try: + network = Network.create(self.api_client, self.services["shared_network"], + networkofferingid=self.shared_network_offering.id, zoneid=self.zone.id) + self.cleanup.append(network) + except Exception as e: + self.fail("Shared Network creation failed because: %s" % e) + + elif networkType == VPC_NETWORK: + self.services["vpc"]["cidr"] = "10.1.1.1/16" + self.debug("creating a VPC network in the account: %s" % + self.account.name) + vpc = VPC.create(self.apiclient, self.services["vpc"], + vpcofferingid=self.vpc_off.id, zoneid=self.zone.id, + account=self.account.name, domainid=self.account.domainid) + vpcs = VPC.list(self.apiclient, id=vpc.id) + self.assertEqual(validateList(vpcs)[0], PASS, "VPC list validation failed, vpc list is %s" % vpcs) + + network = Network.create(self.api_client,self.services["isolated_network"], + networkofferingid=self.isolated_network_offering_vpc.id, + accountid=self.account.name,domainid=self.account.domainid, + zoneid=self.zone.id, vpcid=vpc.id, gateway="10.1.1.1", netmask="255.255.255.0") + return network + +def CreateEnabledNetworkOffering(apiclient, networkServices): + """Create network offering of given services and enable it""" + + result = createEnabledNetworkOffering(apiclient, networkServices) + assert result[0] == PASS, "Network offering creation/enabling failed due to %s" % result[2] + return result[1] + +@ddt +class TestBasicOperations(cloudstackTestCase): + """Test Basic operations (add/remove/list) IP to/from NIC + """ + + @classmethod + def setUpClass(cls): + cloudstackTestClient = super(TestBasicOperations,cls).getClsTestClient() + cls.api_client = cloudstackTestClient.getApiClient() + + # Fill services from the external config file + cls.services = cloudstackTestClient.getConfigParser().parsedDict + + # Get Zone, Domain and templates + cls.domain = get_domain(cls.api_client, cls.services) + cls.zone = get_zone(cls.api_client, cls.services) + cls.mode = str(cls.zone.networktype).lower() + cls.template = get_template( + cls.api_client, + cls.zone.id, + cls.services["ostype"] + ) + cls.services["virtual_machine"]["zoneid"] = cls.zone.id + cls.services["virtual_machine"]["template"] = cls.template.id + cls.service_offering = ServiceOffering.create( + cls.api_client, + cls.services["service_offering"] + ) + cls._cleanup = [cls.service_offering] + cls.services["shared_network_offering"]["specifyVlan"] = "True" + cls.services["shared_network_offering"]["specifyIpRanges"] = "True" + + cls.shared_network_offering = CreateEnabledNetworkOffering(cls.api_client, + cls.services["shared_network_offering"]) + cls._cleanup.append(cls.shared_network_offering) + + if cls.mode == "advanced": + cls.isolated_network_offering = CreateEnabledNetworkOffering(cls.api_client, + cls.services["isolated_network_offering"]) + cls._cleanup.append(cls.isolated_network_offering) + cls.isolated_network_offering_vpc = CreateEnabledNetworkOffering(cls.api_client, + cls.services["nw_offering_isolated_vpc"]) + cls._cleanup.append(cls.isolated_network_offering_vpc) + cls.vpc_off = VpcOffering.create(cls.api_client, cls.services["vpc_offering"]) + cls.vpc_off.update(cls.api_client, state='Enabled') + cls._cleanup.append(cls.vpc_off) + return + + @classmethod + def tearDownClass(cls): + try: + # Cleanup resources used + cleanup_resources(cls.api_client, cls._cleanup) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + return + + def setUp(self): + self.apiclient = self.testClient.getApiClient() + self.dbclient = self.testClient.getDbConnection() + self.cleanup = [ ] + return + + def tearDown(self): + try: + # Clean up, terminate the resources created + cleanup_resources(self.apiclient, self.cleanup) + self.cleanup[:] = [] + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + return + + def VerifyStaticNatForPublicIp(self, ipaddressid, natrulestatus): + """ List public IP and verify that NAT rule status for the IP is as desired """ + + publiciplist = PublicIPAddress.list(self.apiclient, id=ipaddressid, listall=True) + self.assertEqual(validateList(publiciplist)[0], PASS, "Public IP list validation failed") + self.assertEqual(publiciplist[0].isstaticnat, natrulestatus, "isstaticnat should be %s, it is %s" % + (natrulestatus, publiciplist[0].isstaticnat)) + + return + + @data(ISOLATED_NETWORK, SHARED_NETWORK, VPC_NETWORK) + @attr(tags=["basic","advanced"]) + def test_add_ip_to_nic(self, value): + """ Add secondary IP to NIC of a VM""" + + # Steps: + # 1. Create Account and create network in it (isoalted/ shared/ vpc) + # 2. Deploy a VM in this network and account + # 3. Add secondary IP to the default nic of VM + # 4. Try to add the same IP again + # 5. Try to add secondary IP providing wrong virtual machine id + # 6. Try to add secondary IP with correct virtual machine id but wrong IP address + + # Validations: + # 1. Step 3 should succeed + # 2. Step 4 should fail + # 3. Step 5 should should fail + # 4. Step 6 should fail + + self.account = Account.create(self.apiclient,self.services["account"],domainid=self.domain.id) + self.cleanup.append(self.account) + + if(shouldTestBeSkipped(networkType=value, zoneType=self.mode)): + self.skipTest("Skipping test as %s network is not supported in basic zone" % value) + + network = createNetwork(self, value) + + try: + virtual_machine = VirtualMachine.create(self.apiclient,self.services["virtual_machine"], + networkids=[network.id],serviceofferingid=self.service_offering.id, + accountid=self.account.name,domainid=self.account.domainid) + except Exception as e: + self.fail("vm creation failed: %s" % e) + + try: + ipaddress_1 = NIC.addIp(self.apiclient, id=virtual_machine.nic[0].id) + except Exception as e: + self.fail("Failed while adding secondary IP to NIC of vm %s" % virtual_machine.id) + + try: + NIC.addIp(self.apiclient, id=virtual_machine.nic[0].id, ipaddress=ipaddress_1.ipaddress) + self.debug("Adding already added secondary IP %s to NIC of vm %s succeeded, should have failed" % + (ipaddress_1.ipaddress, virtual_machine.id)) + except Exception as e: + self.debug("Failed while adding already added secondary IP to NIC of vm %s" % virtual_machine.id) + + try: + NIC.addIp(self.apiclient, id=(virtual_machine.nic[0].id + random_gen())) + self.fail("Adding secondary IP with wrong NIC id succeded, it shoud have failed") + except Exception as e: + self.debug("Failed while adding secondary IP to wrong NIC") + + try: + NIC.addIp(self.apiclient, id=virtual_machine.nic[0].id, ipaddress = "255.255.255.300") + self.fail("Adding secondary IP with wrong ipaddress succeded, it should have failed") + except Exception as e: + self.debug("Failed while adding wrong secondary IP to NIC of VM %s" % virtual_machine.id) + return + + @data(ISOLATED_NETWORK, SHARED_NETWORK, VPC_NETWORK) + @attr(tags=["basic","advanced"]) + def test_remove_ip_from_nic(self, value): + """ Remove secondary IP from NIC of a VM""" + + # Steps: + # 1. Create Account and create network in it (isoalted/ shared/ vpc) + # 2. Deploy a VM in this network and account + # 3. Add secondary IP to the default nic of VM + # 4. Remove the secondary IP + # 5. Try to remove secondary ip by giving incorrect ipaddress id + + # Validations: + # 1. Step 4 should succeed + # 2. Step 5 should fail + + self.account = Account.create(self.apiclient,self.services["account"],domainid=self.domain.id) + self.cleanup.append(self.account) + + if(shouldTestBeSkipped(networkType=value, zoneType=self.mode)): + self.skipTest("Skipping test as %s network is not supported in basic zone" % value) + + network = createNetwork(self, value) + + try: + virtual_machine = VirtualMachine.create(self.apiclient,self.services["virtual_machine"], + networkids=[network.id],serviceofferingid=self.service_offering.id, + accountid=self.account.name,domainid=self.account.domainid) + except Exception as e: + self.fail("vm creation failed: %s" % e) + + try: + ipaddress_1 = NIC.addIp(self.apiclient, id=virtual_machine.nic[0].id) + except Exception as e: + self.fail("Failed while adding secondary IP to NIC of vm %s" % virtual_machine.id) + + try: + NIC.removeIp(self.apiclient, ipaddressid=ipaddress_1.id) + except Exception as e: + self.fail("Removing seondary IP %s from NIC failed as expected with Exception %s" % (ipaddress_1.id,e)) + + try: + NIC.removeIp(self.apiclient, ipaddressid=(ipaddress_1.id + random_gen())) + self.fail("Removing invalid IP address, it should have failed") + except Exception as e: + self.debug("Removing invalid IP failed as expected with Exception %s" % e) + return + + @attr(tags=["basic","advanced"]) + def test_remove_invalid_ip(self): + """ Remove invalid ip""" + + # Steps: + # 1. Try to remove secondary ip without passing ip address id + + # Validations: + # 1. Step 1 should fail + + try: + NIC.removeIp(self.apiclient, ipaddressid="") + self.fail("Removing IP address without passing IP succeeded, it should have failed") + except Exception as e: + self.debug("Removing IP from NIC without passing ipaddressid failed as expected with Exception %s" % e) + return + + @data(ISOLATED_NETWORK, SHARED_NETWORK, VPC_NETWORK) + @attr(tags=["basic","advanced"]) + def test_list_nics(self, value): + """Test listing nics associated with the ip address""" + + # Steps: + # 1. Create Account and create network in it (isoalted/ shared/ vpc) + # 2. Deploy a VM in this network and account + # 3. Add secondary IP to the default nic of VM + # 4. Try to list the secondary ips without passing vm id + # 5. Try to list secondary IPs by passing correct vm id + # 6. Try to list secondary IPs by passing correct vm id and its nic id + # 7. Try to list secondary IPs by passing incorrect vm id and correct nic id + # 8. Try to list secondary IPs by passing correct vm id and incorrect nic id + # 9. Try to list secondary IPs by passing incorrect vm id and incorrect nic id + + # Validations: + # 1. Step 4 should fail + # 2. Step 5 should succeed + # 3. Step 6 should succeed + # 4. Step 7 should fail + # 5. Step 8 should fail + # 6. Step 9 should fail + + + self.account = Account.create(self.apiclient,self.services["account"],domainid=self.domain.id) + self.cleanup.append(self.account) + + if(shouldTestBeSkipped(networkType=value, zoneType=self.mode)): + self.skipTest("Skipping test as %s network is not supported in basic zone" % value) + + network = createNetwork(self, value) + + try: + virtual_machine = VirtualMachine.create(self.apiclient,self.services["virtual_machine"], + networkids=[network.id],serviceofferingid=self.service_offering.id, + accountid=self.account.name,domainid=self.account.domainid) + except Exception as e: + self.fail("vm creation failed: %s" % e) + + try: + NIC.addIp(self.apiclient, id=virtual_machine.nic[0].id) + except Exception as e: + self.fail("Failed while adding secondary IP to NIC of vm %s" % virtual_machine.id) + + try: + nics = NIC.list(self.apiclient) + self.fail("Listing NICs without passign VM id succeeded, it should have failed, list is %s" % nics) + except Exception as e: + self.debug("Listing NICs without passing virtual machine id failed as expected") + + try: + NIC.list(self.apiclient, virtualmachineid=virtual_machine.id) + except Exception as e: + self.fail("Listing NICs for virtual machine %s failed with Exception %s" % (virtual_machine.id, e)) + + try: + NIC.list(self.apiclient, virtualmachineid=virtual_machine.id, nicid=virtual_machine.nic[0].id) + except Exception as e: + self.fail("Listing NICs for virtual machine %s and nic id %s failed with Exception %s" % + (virtual_machine.id, virtual_machine.nic[0].id, e)) + + try: + nics = NIC.list(self.apiclient, virtualmachineid=(virtual_machine.id + random_gen()), nicid=virtual_machine.nic[0].id) + self.fail("Listing NICs with wrong virtual machine id and right nic id succeeded, should have failed") + except Exception as e: + self.debug("Listing NICs with wrong virtual machine id and right nic failed as expected with Exception %s" % e) + + try: + nics = NIC.list(self.apiclient, virtualmachineid=virtual_machine.id, nicid=(virtual_machine.nic[0].id + random_gen())) + self.fail("Listing NICs with correct virtual machine id but wrong nic id succeeded, should have failed") + except Exception as e: + self.debug("Listing NICs with correct virtual machine id but wrong nic id failed as expected with Exception %s" % e) + + try: + nics = NIC.list(self.apiclient, virtualmachineid=(virtual_machine.id+random_gen()), nicid=(virtual_machine.nic[0].id + random_gen())) + self.fail("Listing NICs with wrong virtual machine id and wrong nic id succeeded, should have failed") + except Exception as e: + self.debug("Listing NICs with wrong virtual machine id and wrong nic id failed as expected with Exception %s" % e) + + return + + @data(ISOLATED_NETWORK, SHARED_NETWORK, VPC_NETWORK) + @attr(tags=["basic","advanced"]) + def test_operations_non_root_admin_api_client(self, value): + """Test basic operations using non root admin apii client""" + + # Steps: + # 1. Create Domain and Account in it + # 2. Create network in it (isoalted/ shared/ vpc) + # 3. Create User API client of this account + # 4. Deploy a VM in this network and account + # 5. Add secondary IP to the default nic of VM using non root admin api client + # 6. List secondary IPs using non root admin api client + # 7. Remove secondary IP using non root admin api client + + # Validations: + # 1. All the operations should be successful + + child_domain = Domain.create(self.apiclient,services=self.services["domain"], + parentdomainid=self.domain.id) + + self.account = Account.create(self.apiclient,self.services["account"],domainid=child_domain.id) + self.cleanup.append(self.account) + self.cleanup.append(child_domain) + + apiclient = self.testClient.createUserApiClient(UserName=self.account.name, DomainName=self.account.domain) + + if(shouldTestBeSkipped(networkType=value, zoneType=self.mode)): + self.skipTest("Skipping test as %s network is not supported in basic zone" % value) + + network = createNetwork(self, value) + + try: + virtual_machine = VirtualMachine.create(self.apiclient,self.services["virtual_machine"], + networkids=[network.id],serviceofferingid=self.service_offering.id, + accountid=self.account.name,domainid=self.account.domainid) + except Exception as e: + self.fail("vm creation failed: %s" % e) + + try: + ipaddress_1 = NIC.addIp(apiclient, id=virtual_machine.nic[0].id) + except Exception as e: + self.fail("Failed while adding secondary IP to NIC of vm %s" % virtual_machine.id) + + try: + NIC.list(apiclient, virtualmachineid=virtual_machine.id) + except Exception as e: + self.fail("Listing NICs for virtual machine %s failed with Exception %s" % (virtual_machine.id, e)) + + try: + NIC.list(apiclient, virtualmachineid=virtual_machine.id, nicid=virtual_machine.nic[0].id) + except Exception as e: + self.fail("Listing NICs for virtual machine %s and nic id %s failed with Exception %s" % + (virtual_machine.id, virtual_machine.nic[0].id, e)) + + try: + NIC.removeIp(apiclient, ipaddressid=ipaddress_1.id) + except Exception as e: + self.fail("Removing seondary IP %s from NIC failed as expected with Exception %s" % (ipaddress_1.id,e)) + + return From 3787579d2910f4882c4b2763d610293fcbc03e61 Mon Sep 17 00:00:00 2001 From: "Frank.Zhang" Date: Thu, 20 Feb 2014 14:23:41 -0800 Subject: [PATCH 109/169] BUG-ID: CS-19152 [upgrade] Baremetal 4.2.0 -> 4.3. After upgrade, destroy baremetal instance fail BUG-ID: CS-19150 [upgrade] Baremetal 4.2.0 -> 4.3. After upgrade, unable to start baremetal instance BUG-ID: CS-19149 [upgrade] Baremetal 4.2.0 -> 4.3. After upgrade, baremetal host remain in Disconnected state. But instance associated with baremetal host is running. BUG-ID: CS-19151 [upgrade] Baremetal 4.2.0 -> 4.3. After upgrade, unable to reboot baremetal instance BUG-ID: CS-19156 [upgrade] Baremetal 4.2.0 -> 4.3. After upgrade, unable to stop baremetal instance reviewed-by: edison --- .../manager/BareMetalDiscoverer.java | 4 ++++ .../networkservice/BareMetalResourceBase.java | 10 ++++---- .../BaremetalPxeManagerImpl.java | 4 ++-- .../deploy/DeploymentPlanningManagerImpl.java | 24 +++++++++++++++---- 4 files changed, 30 insertions(+), 12 deletions(-) diff --git a/plugins/hypervisors/baremetal/src/com/cloud/baremetal/manager/BareMetalDiscoverer.java b/plugins/hypervisors/baremetal/src/com/cloud/baremetal/manager/BareMetalDiscoverer.java index 23eecd11c78..1120688fa7c 100755 --- a/plugins/hypervisors/baremetal/src/com/cloud/baremetal/manager/BareMetalDiscoverer.java +++ b/plugins/hypervisors/baremetal/src/com/cloud/baremetal/manager/BareMetalDiscoverer.java @@ -170,6 +170,8 @@ public class BareMetalDiscoverer extends DiscovererBase implements Discoverer, R params.put(ApiConstants.PRIVATE_IP, ipmiIp); params.put(ApiConstants.USERNAME, username); params.put(ApiConstants.PASSWORD, password); + params.put("vmDao", _vmDao); + params.put("configDao", _configDao); String resourceClassName = _configDao.getValue(Config.ExternalBaremetalResourceClassName.key()); BareMetalResourceBase resource = null; @@ -284,6 +286,8 @@ public class BareMetalDiscoverer extends DiscovererBase implements Discoverer, R HashMap params = super.buildConfigParams(host); params.put("hostId", host.getId()); params.put("ipaddress", host.getPrivateIpAddress()); + params.put("vmDao", _vmDao); + params.put("configDao", _configDao); return params; } diff --git a/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BareMetalResourceBase.java b/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BareMetalResourceBase.java index 9dcfa135d3c..4fe40a274bc 100755 --- a/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BareMetalResourceBase.java +++ b/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BareMetalResourceBase.java @@ -34,7 +34,6 @@ import com.cloud.host.Host.Type; import com.cloud.hypervisor.Hypervisor; import com.cloud.resource.ServerResource; import com.cloud.utils.Pair; -import com.cloud.utils.component.ComponentContext; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.OutputInterpreter; @@ -86,9 +85,10 @@ public class BareMetalResourceBase extends ManagerBase implements ServerResource protected Script2 _forcePowerOffCommand; protected Script2 _bootOrRebootCommand; protected String _vmName; - protected VMInstanceDao vmDao; protected int ipmiRetryTimes = 5; + protected ConfigurationDao configDao; + protected VMInstanceDao vmDao; private void changeVmState(String vmName, VirtualMachine.State state) { synchronized (_vms) { @@ -126,6 +126,8 @@ public class BareMetalResourceBase extends ManagerBase implements ServerResource _password = (String) params.get(ApiConstants.PASSWORD); _vmName = (String) params.get("vmName"); String echoScAgent = (String) params.get(BaremetalManager.EchoSecurityGroupAgent); + vmDao = (VMInstanceDao) params.get("vmDao"); + configDao = (ConfigurationDao) params.get("configDao"); if (_pod == null) { throw new ConfigurationException("Unable to get the pod"); @@ -156,7 +158,6 @@ public class BareMetalResourceBase extends ManagerBase implements ServerResource _isEchoScAgent = Boolean.valueOf(echoScAgent); } - ConfigurationDao configDao = ComponentContext.getComponent(ConfigurationDao.class); String ipmiIface = "default"; try { ipmiIface = configDao.getValue(Config.BaremetalIpmiLanInterface.key()); @@ -330,7 +331,6 @@ public class BareMetalResourceBase extends ManagerBase implements ServerResource protected Map fullSync() { Map states = new HashMap(); if (hostId != null) { - vmDao = ComponentContext.getComponent(VMInstanceDao.class); final List vms = vmDao.listByHostId(hostId); for (VMInstanceVO vm : vms) { states.put(vm.getInstanceName(), vm.getState()); @@ -349,7 +349,6 @@ public class BareMetalResourceBase extends ManagerBase implements ServerResource protected Map getHostVmStateReport() { Map states = new HashMap(); if (hostId != null) { - vmDao = ComponentContext.getComponent(VMInstanceDao.class); final List vms = vmDao.listByHostId(hostId); for (VMInstanceVO vm : vms) { states.put( @@ -412,7 +411,6 @@ public class BareMetalResourceBase extends ManagerBase implements ServerResource } if (hostId != null) { - vmDao = ComponentContext.getComponent(VMInstanceDao.class); final List vms = vmDao.listByHostId(hostId); if (vms.isEmpty()) { return new PingRoutingCommand(getType(), id, deltaSync(), getHostVmStateReport()); diff --git a/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalPxeManagerImpl.java b/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalPxeManagerImpl.java index 9618e6bb47f..e82eeceff5a 100755 --- a/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalPxeManagerImpl.java +++ b/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalPxeManagerImpl.java @@ -184,9 +184,9 @@ public class BaremetalPxeManagerImpl extends ManagerBase implements BaremetalPxe @Override public boolean addUserData(NicProfile nic, VirtualMachineProfile profile) { - UserVmVO vm = (UserVmVO) profile.getVirtualMachine(); + UserVmVO vm = _vmDao.findById(profile.getVirtualMachine().getId()); _vmDao.loadDetails(vm); - + String serviceOffering = _serviceOfferingDao.findByIdIncludingRemoved(vm.getId(), vm.getServiceOfferingId()).getDisplayText(); String zoneName = _dcDao.findById(vm.getDataCenterId()).getName(); NicVO nvo = _nicDao.findById(nic.getId()); diff --git a/server/src/com/cloud/deploy/DeploymentPlanningManagerImpl.java b/server/src/com/cloud/deploy/DeploymentPlanningManagerImpl.java index 0a2580542e1..d9e1ecc2291 100644 --- a/server/src/com/cloud/deploy/DeploymentPlanningManagerImpl.java +++ b/server/src/com/cloud/deploy/DeploymentPlanningManagerImpl.java @@ -299,6 +299,15 @@ public class DeploymentPlanningManagerImpl extends ManagerBase implements Deploy + ", pod: " + host.getPodId() + ", cluster: " + host.getClusterId()); } + Pod pod = _podDao.findById(host.getPodId()); + Cluster cluster = _clusterDao.findById(host.getClusterId()); + + if (vm.getHypervisorType() == HypervisorType.BareMetal) { + DeployDestination dest = new DeployDestination(dc, pod, cluster, host, new HashMap()); + s_logger.debug("Returning Deployment Destination: " + dest); + return dest; + } + // search for storage under the zone, pod, cluster of the host. DataCenterDeployment lastPlan = new DataCenterDeployment(host.getDataCenterId(), host.getPodId(), host.getClusterId(), hostIdSpecified, plan.getPoolId(), null, plan.getReservationContext()); @@ -315,8 +324,6 @@ public class DeploymentPlanningManagerImpl extends ManagerBase implements Deploy Pair> potentialResources = findPotentialDeploymentResources( suitableHosts, suitableVolumeStoragePools, avoids, getPlannerUsage(planner,vmProfile, plan ,avoids), readyAndReusedVolumes); if (potentialResources != null) { - Pod pod = _podDao.findById(host.getPodId()); - Cluster cluster = _clusterDao.findById(host.getClusterId()); Map storageVolMap = potentialResources.second(); // remove the reused vol<->pool from destination, since // we don't have to prepare this volume. @@ -362,6 +369,17 @@ public class DeploymentPlanningManagerImpl extends ManagerBase implements Deploy + ", pod: " + host.getPodId() + ", cluster: " + host.getClusterId()); // search for storage under the zone, pod, cluster of // the last host. + + + Pod pod = _podDao.findById(host.getPodId()); + Cluster cluster = _clusterDao.findById(host.getClusterId()); + + if (vm.getHypervisorType() == HypervisorType.BareMetal) { + DeployDestination dest = new DeployDestination(dc, pod, cluster, host, new HashMap()); + s_logger.debug("Returning Deployment Destination: " + dest); + return dest; + } + DataCenterDeployment lastPlan = new DataCenterDeployment(host.getDataCenterId(), host.getPodId(), host.getClusterId(), host.getId(), plan.getPoolId(), null); Pair>, List> result = findSuitablePoolsForVolumes( @@ -376,8 +394,6 @@ public class DeploymentPlanningManagerImpl extends ManagerBase implements Deploy Pair> potentialResources = findPotentialDeploymentResources( suitableHosts, suitableVolumeStoragePools, avoids, getPlannerUsage(planner,vmProfile, plan ,avoids), readyAndReusedVolumes); if (potentialResources != null) { - Pod pod = _podDao.findById(host.getPodId()); - Cluster cluster = _clusterDao.findById(host.getClusterId()); Map storageVolMap = potentialResources.second(); // remove the reused vol<->pool from // destination, since we don't have to prepare From c04fb68fc7022c9d3ef6206d314194c3ce2a1872 Mon Sep 17 00:00:00 2001 From: Marcus Sorensen Date: Thu, 20 Feb 2014 23:22:57 -0700 Subject: [PATCH 110/169] CLOUDSTACK-6072: Release VNIs when guest networks are destroyed submitted-by: mail@ynojima.net --- server/src/com/cloud/network/guru/GuestNetworkGuru.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/com/cloud/network/guru/GuestNetworkGuru.java b/server/src/com/cloud/network/guru/GuestNetworkGuru.java index 85728c22fbb..d1c065688f7 100755 --- a/server/src/com/cloud/network/guru/GuestNetworkGuru.java +++ b/server/src/com/cloud/network/guru/GuestNetworkGuru.java @@ -414,13 +414,13 @@ public abstract class GuestNetworkGuru extends AdapterBase implements NetworkGur @Override public void shutdown(NetworkProfile profile, NetworkOffering offering) { - if (profile.getBroadcastDomainType() == BroadcastDomainType.Vlan && + if ((profile.getBroadcastDomainType() == BroadcastDomainType.Vlan || profile.getBroadcastDomainType() == BroadcastDomainType.Vxlan) && profile.getBroadcastUri() != null && !offering.getSpecifyVlan()) { s_logger.debug("Releasing vnet for the network id=" + profile.getId()); _dcDao.releaseVnet(BroadcastDomainType.getValue(profile.getBroadcastUri()), profile.getDataCenterId(), profile.getPhysicalNetworkId(), profile.getAccountId(), profile.getReservationId()); ActionEventUtils.onCompletedActionEvent(CallContext.current().getCallingUserId(), profile.getAccountId(), - EventVO.LEVEL_INFO, EventTypes.EVENT_ZONE_VLAN_RELEASE, "Released Zone Vlan: " + EventVO.LEVEL_INFO, EventTypes.EVENT_ZONE_VLAN_RELEASE, "Released Zone vnet: " + BroadcastDomainType.getValue(profile.getBroadcastUri()) + " for Network: " + profile.getId(), 0); } profile.setBroadcastUri(null); From 96355dca4b5230e96a64c38d29d727cebd3a10fb Mon Sep 17 00:00:00 2001 From: Marcus Sorensen Date: Thu, 20 Feb 2014 23:23:58 -0700 Subject: [PATCH 111/169] CLOUDSTACK-6079: fixed issue: if public traffic is on a physical network isolated by VLAN, and guest traffic is on an another physical network isolated by VXLAN, public traffic's broadcast type is updated to VXLAN by mistake. submitted-by: mail@ynojima.net --- .../com/cloud/network/NetworkServiceImpl.java | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/server/src/com/cloud/network/NetworkServiceImpl.java b/server/src/com/cloud/network/NetworkServiceImpl.java index 270ff7b02cc..34d674eefb9 100755 --- a/server/src/com/cloud/network/NetworkServiceImpl.java +++ b/server/src/com/cloud/network/NetworkServiceImpl.java @@ -3587,15 +3587,17 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService { // For public traffic, get isolation method of physical network and update the public network accordingly // each broadcast type will individually need to be qualified for support of public traffic - List isolationMethods = network.getIsolationMethods(); - if ((isolationMethods.size() == 1 && isolationMethods.get(0).toLowerCase().equals("vxlan")) - || (isolationMethod != null && isolationMethods.contains(isolationMethod) && isolationMethod.toLowerCase().equals("vxlan"))) { - // find row in networks table that is defined as 'Public', created when zone was deployed - NetworkVO publicNetwork = _networksDao.listByZoneAndTrafficType(network.getDataCenterId(),TrafficType.Public).get(0); - if (publicNetwork != null) { - s_logger.debug("setting public network " + publicNetwork + " to broadcast type vxlan"); - publicNetwork.setBroadcastDomainType(BroadcastDomainType.Vxlan); - _networksDao.persist(publicNetwork); + if (TrafficType.Public.equals(trafficType)){ + List isolationMethods = network.getIsolationMethods(); + if ((isolationMethods.size() == 1 && isolationMethods.get(0).toLowerCase().equals("vxlan")) + || (isolationMethod != null && isolationMethods.contains(isolationMethod) && isolationMethod.toLowerCase().equals("vxlan"))) { + // find row in networks table that is defined as 'Public', created when zone was deployed + NetworkVO publicNetwork = _networksDao.listByZoneAndTrafficType(network.getDataCenterId(),TrafficType.Public).get(0); + if (publicNetwork != null) { + s_logger.debug("setting public network " + publicNetwork + " to broadcast type vxlan"); + publicNetwork.setBroadcastDomainType(BroadcastDomainType.Vxlan); + _networksDao.persist(publicNetwork); + } } } From c6a8659ac2c2c85c6f0e9fb8af11606aaa2f9a37 Mon Sep 17 00:00:00 2001 From: Koushik Das Date: Fri, 21 Feb 2014 16:16:52 +0530 Subject: [PATCH 112/169] CLOUDSTACK-6124: During MS maintenance unfinished work items are not cleaned up resulting in them getting repeated for every subsequent maintenance Updating the op_it_work table entry appropriately in db once the unfinished work item is completed. --- .../src/com/cloud/vm/VirtualMachineManagerImpl.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java index 3405d4449d4..bc6b65d36c2 100755 --- a/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java @@ -2129,10 +2129,12 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac if (work.getType() == State.Starting) { _haMgr.scheduleRestart(vm, true); work.setManagementServerId(_nodeId); + work.setStep(Step.Done); _workDao.update(work.getId(), work); } else if (work.getType() == State.Stopping) { _haMgr.scheduleStop(vm, vm.getHostId(), WorkType.CheckStop); work.setManagementServerId(_nodeId); + work.setStep(Step.Done); _workDao.update(work.getId(), work); } else if (work.getType() == State.Migrating) { _haMgr.scheduleMigration(vm); From c03fd70ed22d4b0b2ea41fdb923f4432664219d1 Mon Sep 17 00:00:00 2001 From: Ashutosh K Date: Fri, 21 Feb 2014 17:46:14 +0530 Subject: [PATCH 113/169] CLOUDSTACK-2232: Adding automation test cases for Persistent Networks feature --- .../component/test_persistent_networks.py | 2047 +++++++++++++++-- tools/marvin/marvin/config/config.cfg | 233 ++ tools/marvin/marvin/integration/lib/base.py | 7 + tools/marvin/marvin/integration/lib/common.py | 13 +- 4 files changed, 2132 insertions(+), 168 deletions(-) diff --git a/test/integration/component/test_persistent_networks.py b/test/integration/component/test_persistent_networks.py index f61ccaae360..8c1d54d1928 100644 --- a/test/integration/component/test_persistent_networks.py +++ b/test/integration/component/test_persistent_networks.py @@ -15,122 +15,60 @@ # specific language governing permissions and limitations # under the License. """ Tests for Persistent Networks without running VMs feature + + Test Plan: https://cwiki.apache.org/confluence/display/CLOUDSTACK/Persistent+Networks+without+a+running+VM + + Issue Link: https://issues.apache.org/jira/browse/CLOUDSTACK-2232 + + Feature Specifications: https://cwiki.apache.org/confluence/display/CLOUDSTACK/FS+-+Persistent+Networks """ -import marvin -from marvin.cloudstackTestCase import * -from marvin.cloudstackAPI import * -from marvin.cloudstackException import cloudstackAPIException -from marvin.integration.lib.utils import * -from marvin.integration.lib.base import * -from marvin.integration.lib.common import * -import netaddr +from marvin.cloudstackTestCase import cloudstackTestCase, unittest +from marvin.integration.lib.utils import (cleanup_resources, + validateList, + get_hypervisor_type) +from marvin.integration.lib.base import (Account, + ServiceOffering, + NetworkOffering, + Network, + VirtualMachine, + PublicIPAddress, + FireWallRule, + Router, + Host, + NATRule, + Project, + LoadBalancerRule, + VpcOffering, + VPC, + Domain, + StaticNATRule, + NetworkACL) +from marvin.integration.lib.common import (get_domain, + get_zone, + get_template, + wait_for_cleanup, + add_netscaler, + verifyNetworkState) from nose.plugins.attrib import attr +from marvin.codes import PASS, FAIL +from marvin.sshClient import SshClient +from ddt import ddt, data +import time -class Services(object): - """Test Persistent Networks without running VMs - """ - def __init__(self): - self.services = { - "account": { - "email": "test@test.com", - "firstname": "Test", - "lastname": "User", - "username": "test", - # Random characters are appended for unique - # username - "password": "password", - }, - "service_offering": { - "name": "Tiny Instance ", - "displaytext": "Tiny Instance", - "cpunumber": 1, - "cpuspeed": 200, # in MHz - "memory": 256, # In MBs - }, - "shared_persistent_network_offering": { - "name": 'Network offering for Shared Persistent Network', - "displaytext": 'Network offering-DA services', - "guestiptype": 'Shared', - "supportedservices": 'Dhcp,Dns,SourceNat,PortForwarding,Vpn,Firewall,Lb,UserData,StaticNat', - "traffictype": 'GUEST', - "availability": 'Optional', - "ispersistent": 'True', - "serviceProviderList": { - "Dhcp": 'VirtualRouter', - "Dns": 'VirtualRouter', - "SourceNat": 'VirtualRouter', - "PortForwarding": 'VirtualRouter', - "Vpn": 'VirtualRouter', - "Firewall": 'VirtualRouter', - "Lb": 'VirtualRouter', - "UserData": 'VirtualRouter', - "StaticNat": 'VirtualRouter', - }, - }, - "isolated_persistent_network_offering": { - "name": 'Network offering for Isolated Persistent Network', - "displaytext": 'Network offering-DA services', - "guestiptype": 'Isolated', - "supportedservices": 'Dhcp,Dns,SourceNat,PortForwarding,Vpn,Firewall,Lb,UserData,StaticNat', - "traffictype": 'GUEST', - "availability": 'Optional', - "ispersistent": 'True', - "serviceProviderList": { - "Dhcp": 'VirtualRouter', - "Dns": 'VirtualRouter', - "SourceNat": 'VirtualRouter', - "PortForwarding": 'VirtualRouter', - "Vpn": 'VirtualRouter', - "Firewall": 'VirtualRouter', - "Lb": 'VirtualRouter', - "UserData": 'VirtualRouter', - "StaticNat": 'VirtualRouter', - }, - }, - "isolated_network_offering": { - "name": 'Network offering for Isolated Persistent Network', - "displaytext": 'Network offering-DA services', - "guestiptype": 'Isolated', - "supportedservices": 'Dhcp,Dns,SourceNat,PortForwarding,Vpn,Firewall,Lb,UserData,StaticNat', - "traffictype": 'GUEST', - "availability": 'Optional', - "serviceProviderList": { - "Dhcp": 'VirtualRouter', - "Dns": 'VirtualRouter', - "SourceNat": 'VirtualRouter', - "PortForwarding": 'VirtualRouter', - "Vpn": 'VirtualRouter', - "Firewall": 'VirtualRouter', - "Lb": 'VirtualRouter', - "UserData": 'VirtualRouter', - "StaticNat": 'VirtualRouter', - }, - }, - "isolated_network": { - "name": "Isolated Network", - "displaytext": "Isolated Network", - }, - "virtual_machine": { - "displayname": "Test VM", - }, - "ostype": 'CentOS 5.3 (64-bit)', - # Cent OS 5.3 (64 bit) - "sleep": 90, - "timeout": 10, - "mode": 'advanced' - } - - - +@ddt class TestPersistentNetworks(cloudstackTestCase): """Test Persistent Networks without running VMs """ @classmethod def setUpClass(cls): - cls.api_client = super(TestPersistentNetworks, cls).getClsTestClient().getApiClient() - cls.services = Services().services + cloudstackTestClient = super(TestPersistentNetworks,cls).getClsTestClient() + cls.api_client = cloudstackTestClient.getApiClient() + + # Fill services from the external config file + cls.services = cloudstackTestClient.getConfigParser().parsedDict + # Get Zone, Domain and templates cls.domain = get_domain(cls.api_client, cls.services) cls.zone = get_zone(cls.api_client, cls.services) @@ -139,28 +77,39 @@ class TestPersistentNetworks(cloudstackTestCase): cls.zone.id, cls.services["ostype"] ) - cls.services["domainid"] = cls.domain.id - cls.services["zoneid"] = cls.zone.id cls.account = Account.create( cls.api_client, cls.services["account"], domainid=cls.domain.id ) - cls.services["account"] = cls.account.name cls.services["virtual_machine"]["zoneid"] = cls.zone.id cls.services["virtual_machine"]["template"] = cls.template.id cls.service_offering = ServiceOffering.create( cls.api_client, cls.services["service_offering"] ) - cls.isolated_persistent_network_offering = cls.create_network_offering("isolated_persistent_network_offering") - cls.isolated_network = cls.create_isolated_network(cls.isolated_persistent_network_offering.id) - cls.isolated_network_offering = cls.create_network_offering("isolated_network_offering") + cls.isolated_persistent_network_offering = cls.createNetworkOffering("nw_off_isolated_persistent") + cls.isolated_persistent_network_offering_netscaler = cls.createNetworkOffering("nw_off_isolated_persistent_netscaler") + cls.isolated_persistent_network_offering_RVR = cls.createNetworkOffering("nw_off_persistent_RVR") + cls.isolated_network_offering = cls.createNetworkOffering("isolated_network_offering") + cls.isolated_network_offering_netscaler = cls.createNetworkOffering("nw_off_isolated_netscaler") + + # Configure Netscaler device + # If configuration succeeds, set ns_configured to True so that Netscaler tests are executed + cls.ns_configured = False + try: + cls.netscaler = add_netscaler(cls.api_client, cls.zone.id, cls.services["netscaler_VPX"]) + cls._cleanup.append(cls.netscaler) + cls.ns_configured = True + except Exception: + cls.ns_configured = False # network will be deleted as part of account cleanup cls._cleanup = [ - cls.account, cls.service_offering, cls.isolated_persistent_network_offering, cls.isolated_network_offering + cls.account, cls.service_offering, cls.isolated_persistent_network_offering, cls.isolated_network_offering, + cls.isolated_persistent_network_offering_RVR, cls.isolated_persistent_network_offering_netscaler, + cls.isolated_network_offering_netscaler ] return @@ -174,33 +123,76 @@ class TestPersistentNetworks(cloudstackTestCase): return @classmethod - def create_network_offering(cls, network_offering_type): + def createNetworkOffering(cls, network_offering_type): network_offering = NetworkOffering.create( cls.api_client, cls.services[network_offering_type], conservemode=False ) # Update network offering state from disabled to enabled. - network_offering_update_response = NetworkOffering.update( - network_offering, - cls.api_client, - id=network_offering.id, - state="enabled" - ) + NetworkOffering.update(network_offering, cls.api_client, id=network_offering.id, + state="enabled") return network_offering - @classmethod - def create_isolated_network(cls, network_offering_id): - isolated_network = Network.create( - cls.api_client, - cls.services["isolated_network"], - networkofferingid=network_offering_id, - accountid=cls.account.name, - domainid=cls.domain.id, - zoneid=cls.zone.id + def checkRouterAccessibility(self, router): + """Check if router is accessible through its linklocalip""" + + hypervisor = str(get_hypervisor_type(self.api_client)) + + if hypervisor.lower() == 'vmware': + #SSH is done via management server for Vmware + sourceip = self.api_client.connection.mgtSvr + else: + #For others, we will have to get the ipaddress of host connected to vm + hosts = Host.list(self.api_client,id=router.hostid) + self.assertEqual(validateList(hosts)[0], PASS, "hosts list validation failed, list is %s" % hosts) + host = hosts[0] + sourceip = host.ipaddress + # end else + + try: + sshClient = SshClient(host=sourceip, port=22, user='root',passwd=self.services["host_password"]) + res = sshClient.execute("ping -c 1 %s" % ( + router.linklocalip + )) + self.debug("SSH result: %s" % res) + except Exception as e: + self.fail("SSH Access failed for %s: %s" % \ + (sourceip, e) + ) + result = str(res) + self.assertEqual( + result.count("1 received"), + 1, + "Ping to router should be successful" ) - cls.debug("persistent isolated network is created: " + isolated_network.id) - return isolated_network + return + + def verifyVmExpungement(self, virtual_machine): + """verify if vm is expunged""" + isVmExpunged = False + try: + virtual_machine.delete(self.apiclient) + except Exception as e: + self.fail("Failed to delete VM: %s with error %s" % (virtual_machine.id, e)) + + # Verify if it is expunged + retriesCount = 20 + while True: + vms = VirtualMachine.list(self.api_client, id=virtual_machine.id) + # When vm is expunged, list will be None + if vms is None: + isVmExpunged = True + break + elif retriesCount == 0: + break + time.sleep(60) + retriesCount -= 1 + # end while + + if not isVmExpunged: + self.fail("Failed to expunge vm even after 20 minutes") + return def setUp(self): self.apiclient = self.testClient.getApiClient() @@ -212,54 +204,49 @@ class TestPersistentNetworks(cloudstackTestCase): try: # Clean up, terminate the resources created cleanup_resources(self.apiclient, self.cleanup) + self.cleanup[:] = [] except Exception as e: raise Exception("Warning: Exception during cleanup : %s" % e) return - def create_virtual_machine(self, network_id=None): - virtual_machine = VirtualMachine.create(self.apiclient, - self.services["virtual_machine"], - networkids=network_id, - serviceofferingid=self.service_offering.id, - accountid=self.account.name, - domainid=self.domain.id - ) - self.debug("Virtual Machine is created: " + virtual_machine.id) - return virtual_machine - @attr(tags=["advanced"]) def test_network_state_after_destroying_vms(self): # steps - # 1. create virtual machine in network - # 2. destroy created virtual machine + # 1. Create an isolated persistent network + # 2. Deploy virtual machine in network + # 3. destroy created virtual machine # # validation # 1. Persistent network state should be implemented before VM creation and have some vlan assigned # 2. virtual machine should be created successfully # 3. Network state should be implemented even after destroying all vms in network - self.assertEquals(self.isolated_network.state, u"Implemented", "network state of persistent is not implemented") - self.assertIsNotNone(self.isolated_network.vlan, "vlan must not be null for persistent network") + + # Creating isolated persistent network + network = Network.create(self.apiclient,self.services["isolated_network"], + networkofferingid=self.isolated_persistent_network_offering.id, + accountid=self.account.name,domainid=self.domain.id, + zoneid=self.zone.id) + self.cleanup.append(network) + verifyNetworkState(self.apiclient, network.id, "implemented") + self.assertIsNotNone(network.vlan, "vlan must not be null for persistent network") try: - virtual_machine = self.create_virtual_machine(network_id=self.isolated_network.id) + virtual_machine = VirtualMachine.create(self.apiclient,self.services["virtual_machine"], + networkids=[network.id],serviceofferingid=self.service_offering.id, + accountid=self.account.name,domainid=self.domain.id) virtual_machine.delete(self.apiclient) except Exception as e: - self.skipTest("vm creation/deletion fails") + self.fail("vm creation failed: %s" % e) + + # Verify VM is expunged + self.verifyVmExpungement(virtual_machine) # wait for time such that, network is cleaned up # assuming that it will change its state to allocated after this much period wait_for_cleanup(self.api_client, ["network.gc.interval", "network.gc.wait"]) - - networks = Network.list(self.apiclient, id=self.isolated_network.id) - self.assertEqual( - isinstance(networks, list), - True, - "list Networks should return valid response" - ) - - - self.assertEquals(networks[0].state, u"Implemented", "network state of persistent network after all vms are destroyed is not implemented") + verifyNetworkState(self.api_client, network.id, "implemented") + return @attr(tags=["advanced"]) def test_shared_network_offering_with_persistent(self): @@ -269,22 +256,1748 @@ class TestPersistentNetworks(cloudstackTestCase): # validation # 1. network offering should throw an exception try: - shared_persistent_network_offering = self.create_network_offering("shared_persistent_network_offering") + shared_persistent_network_offering = self.createNetworkOffering("nw_offering_shared_persistent") shared_persistent_network_offering.delete(self.apiclient) - self.fail("For shared network ispersistent must be False") - except Exception as e: + self.fail("Shared network got created with ispersistent flag True in the offering, it should have failed") + except Exception: pass @attr(tags=["advanced"]) - def test_upgrade_network_offering_to_persistent(self): + def test_persistent_network_offering_with_VPCVR_services(self): + # steps + # 1. create network offering with persistent field enabled and all the services through VpcVirtualRouter + # + # validation + # 1. network offering should be created successfully + try: + persistent_network_offering_VPCVR = self.createNetworkOffering("nw_off_persistent_VPCVR_LB") + persistent_network_offering_VPCVR.delete(self.apiclient) + except Exception as e: + self.fail("Failed creating persistent network offering with VPCVR services: %s" % e) + + @attr(tags=["advanced"]) + def test_list_persistent_network_offering(self): + # steps + # 1. create isolated network offering with ispersistent True + # 2. List network offerings by id and check ispersistent flag + # + # validation + # 1. ispersistent flag should list as True + network_offering = self.createNetworkOffering("nw_off_isolated_persistent") + self.cleanup.append(network_offering) + nw_offering_list = NetworkOffering.list(self.api_client, id=network_offering.id) + self.assertEqual(validateList(nw_offering_list)[0], PASS, "network offerings' list validation failed, list is %s" % + nw_offering_list) + self.assertEqual(nw_offering_list[0].ispersistent, True, "ispersistent flag should be true for the network offering") + return + + @data("LB-VR","LB-NS") + @attr(tags=["advanced","advancedns"]) + def test_upgrade_to_persistent_services_VR(self, value): + + # This test is run against two networks (one with LB as virtual router and other one is LB with Netscaler) + # All other services through VR + # steps # 1. create isolated network with network offering which has ispersistent field disabled # 2. upgrade isolated network offering to network offering which has ispersistent field enabled + # 3. Deploy VM ,acquire IP, create Firewall, NAT rules + # 4. Verify the working of NAT, Firewall rules + # 5. Delete VM + # 6. Verify network state after network cleanup interval # # validation # 1. update of network should happen successfully - # 2. network state should be implemented and have some vlan assigned - isolated_network = self.create_isolated_network(self.isolated_network_offering.id) - isolated_network_response = isolated_network.update(self.apiclient, networkofferingid=self.isolated_persistent_network_offering.id) - self.assertEquals(self.isolated_network.state, u"Implemented", "network state of isolated network upgraded to persistent is not implemented") - self.assertIsNotNone(self.isolated_network.vlan, "vlan must not be null isolated network upgraded to for persistent network") + # 2. NAT and Firewall rule should work as expected + # 3. After network clenaup interval, network state should be implemented and have some vlan assigned + + # Set network offering as required (either LB through VR or LB through Netscaler) + networkOffering = self.isolated_network_offering + + changecidr = False # This will be true in case of Netscaler case, you have to change cidr while updating network + + # In case Netscaler is used for LB + if value == "LB-NS": + if self.ns_configured: + networkOffering = self.isolated_network_offering_netscaler + changecidr = True + else: + self.skipTest("Skipping - this test required netscaler configured in the network") + + # Create Account + account = Account.create(self.api_client, self.services["account"],domainid=self.domain.id) + self.cleanup.append(account) + + # create network with the appropriate offering (not persistent) + isolated_network = Network.create(self.api_client,self.services["isolated_network"], + networkofferingid=networkOffering.id, + accountid=account.name,domainid=self.domain.id, + zoneid=self.zone.id) + verifyNetworkState(self.api_client, isolated_network.id, "allocated") + + # Update the network with persistent network offering + isolated_network.update(self.apiclient, networkofferingid=self.isolated_persistent_network_offering.id, changecidr=changecidr) + + try: + virtual_machine = VirtualMachine.create(self.apiclient,self.services["virtual_machine"], + networkids=[isolated_network.id], + serviceofferingid=self.service_offering.id, + accountid=account.name,domainid=self.domain.id) + except Exception as e: + self.fail("vm creation failed: %s" % e) + + # Acquire public ip and open firewall for it + self.debug("Associating public IP for network: %s" % isolated_network.id) + ipaddress = PublicIPAddress.create(self.api_client,accountid=account.name, + zoneid=self.zone.id,domainid=account.domainid, + networkid=isolated_network.id) + + FireWallRule.create(self.apiclient,ipaddressid=ipaddress.ipaddress.id, + protocol='TCP', cidrlist=[self.services["fwrule"]["cidr"]], + startport=self.services["fwrule"]["startport"],endport=self.services["fwrule"]["endport"]) + + # Create NAT rule + NATRule.create(self.api_client, virtual_machine, + self.services["natrule"],ipaddressid=ipaddress.ipaddress.id, + networkid=isolated_network.id) + + # Check if SSH works + try: + virtual_machine.get_ssh_client(ipaddress=ipaddress.ipaddress.ipaddress) + except Exception as e: + self.fail("Exception while SSHing to VM %s with IP %s" % (virtual_machine.id, ipaddress.ipaddress.ipaddress)) + + # Delete VM + virtual_machine.delete(self.api_client) + + # Verify VM is expunged + self.verifyVmExpungement(virtual_machine) + + # wait for time such that, network is cleaned up + wait_for_cleanup(self.api_client, ["network.gc.interval", "network.gc.wait"]) + + # Check network state now, this will bolster that network updation has taken effect + verifyNetworkState(self.api_client, isolated_network.id, "implemented") + return + + @attr(tags=["advanced"]) + def test_upgrade_network_VR_to_PersistentRVR(self): + # steps + # 1. create isolated network with network offering which has ispersistent field disabled (services through VR) + # 2. upgrade isolated network offering to network offering which has ispersistent field enabled (with RVR) + # 3. Deploy VM ,acquire IP, create Firewall, NAT rules + # 4. Verify the working of NAT, Firewall rules + # 5. Delete VM + # 6. Verify network state after network cleanup interval + # + # validation + # 1. update of network should happen successfully + # 2. NAT and Firewall rule should work as expected + # 3. After network clenaup interval, network state should be implemented and have some vlan assigned + + # Create Account and isolated network in it + account = Account.create(self.api_client, self.services["account"],domainid=self.domain.id) + self.cleanup.append(account) + isolated_network = Network.create(self.api_client,self.services["isolated_network"], + networkofferingid=self.isolated_network_offering.id, + accountid=account.name,domainid=self.domain.id, + zoneid=self.zone.id) + verifyNetworkState(self.api_client, isolated_network.id, "allocated") + + # Update network with network offering which has RVR + isolated_network.update(self.apiclient, networkofferingid=self.isolated_persistent_network_offering_RVR.id) + + # Deploy VM + try: + virtual_machine = VirtualMachine.create(self.apiclient,self.services["virtual_machine"], + networkids=[isolated_network.id], + serviceofferingid=self.service_offering.id, + accountid=account.name,domainid=self.domain.id) + except Exception as e: + self.fail("vm creation failed: %s" % e) + + # Acquire public IP and open firewall rule, create NAT rule + self.debug("Associating public IP for network: %s" % isolated_network.id) + ipaddress = PublicIPAddress.create(self.api_client,accountid=account.name, + zoneid=self.zone.id,domainid=account.domainid, + networkid=isolated_network.id) + + FireWallRule.create(self.apiclient,ipaddressid=ipaddress.ipaddress.id, + protocol='TCP', cidrlist=[self.services["fwrule"]["cidr"]], + startport=self.services["fwrule"]["startport"],endport=self.services["fwrule"]["endport"]) + + NATRule.create(self.api_client, virtual_machine, + self.services["natrule"],ipaddressid=ipaddress.ipaddress.id, + networkid=isolated_network.id) + + try: + virtual_machine.get_ssh_client(ipaddress=ipaddress.ipaddress.ipaddress) + except Exception as e: + self.fail("Exception while SSHing to VM %s with IP %s" % (virtual_machine.id, ipaddress.ipaddress.ipaddress)) + + virtual_machine.delete(self.api_client) + + # Verify VM is expunged + self.verifyVmExpungement(virtual_machine) + + # wait for time such that, network is cleaned up + wait_for_cleanup(self.api_client, ["network.gc.interval", "network.gc.wait"]) + + # Check network state now, this will bolster that network updation has taken effect + verifyNetworkState(self.api_client, isolated_network.id, "implemented") + return + + @attr(tags=["advanced", "advancedns"]) + def test_upgrade_network_NS_to_persistent_NS(self): + # steps + # 1. create isolated network with network offering which has ispersistent field disabled + # and LB service through Netscaler + # 2. upgrade isolated network offering to network offering which has ispersistent field enabled + # and LB service through Netscaler + # 3. Deploy VM ,acquire IP, create Firewall, NAT rules + # 4. Verify the working of NAT, Firewall rules + # 5. Delete VM + # 6. Verify network state after network cleanup interval + # + # validation + # 1. update of network should happen successfully + # 2. NAT and Firewall rule should work as expected + # 3. After network clenaup interval, network state should be implemented and have some vlan assigned + + # Skip test if Netscaler is not configured + if not self.ns_configured: + self.skipTest("Skipping - this test required netscaler configured in the network") + + # Create Account and create isolated network in it + account = Account.create(self.api_client, self.services["account"],domainid=self.domain.id) + self.cleanup.append(account) + isolated_network = Network.create(self.api_client,self.services["isolated_network"], + networkofferingid=self.isolated_network_offering_netscaler.id, + accountid=account.name,domainid=self.domain.id, + zoneid=self.zone.id) + verifyNetworkState(self.api_client, isolated_network.id, "allocated") + + isolated_network.update(self.apiclient, networkofferingid=self.isolated_persistent_network_offering_netscaler.id, changecidr=True) + + # Deploy VM + try: + virtual_machine = VirtualMachine.create(self.apiclient,self.services["virtual_machine"], + networkids=[isolated_network.id], + serviceofferingid=self.service_offering.id, + accountid=account.name,domainid=self.domain.id) + except Exception as e: + self.fail("vm creation failed: %s" % e) + + self.debug("Associating public IP for network: %s" % isolated_network.id) + ipaddress = PublicIPAddress.create(self.api_client,accountid=account.name, + zoneid=self.zone.id,domainid=account.domainid, + networkid=isolated_network.id) + + FireWallRule.create(self.apiclient,ipaddressid=ipaddress.ipaddress.id, + protocol='TCP', cidrlist=[self.services["fwrule"]["cidr"]], + startport=self.services["fwrule"]["startport"],endport=self.services["fwrule"]["endport"]) + + NATRule.create(self.api_client, virtual_machine, + self.services["natrule"],ipaddressid=ipaddress.ipaddress.id, + networkid=isolated_network.id) + + try: + virtual_machine.get_ssh_client(ipaddress=ipaddress.ipaddress.ipaddress) + except Exception as e: + self.fail("Exception while SSHing to VM %s with IP %s" % (virtual_machine.id, ipaddress.ipaddress.ipaddress)) + + virtual_machine.delete(self.api_client) + + # Verify VM is expunged + self.verifyVmExpungement(virtual_machine) + + # wait for time such that, network is cleaned up + wait_for_cleanup(self.api_client, ["network.gc.interval", "network.gc.wait"]) + + # Check network state now, this will bolster that network updation has taken effect + verifyNetworkState(self.api_client, isolated_network.id, "implemented") + return + + @data("LB-VR","LB-Netscaler") + @attr(tags=["advanced", "advancedns"]) + def test_pf_nat_rule_persistent_network(self, value): + + # This test shall run with two scenarios, one with LB services through VR and other through LB service + # through Netscaler""" + + # steps + # 1. create isolated network with network offering which has ispersistent field enabled + # and LB service through VR or Netscaler + # 2. Check routers belonging to network and verify that router is reachable through host using linklocalip + # 3. Deploy VM ,acquire IP, create Firewall, NAT rules + # 4. Verify the working of NAT, Firewall rules + # + # validation + # 1. Router should be reachable + # 2. NAT and Firewall rule should work as expected + + # Set network offering according to data passed to test case + networkOffering = self.isolated_persistent_network_offering + if value == "LB-Netscaler": + if self.ns_configured: + networkOffering = self.isolated_persistent_network_offering_netscaler + else: + self.skipTest("Skipping - this test required netscaler configured in the network") + + # Create account and network in it + account = Account.create(self.api_client, self.services["account"],domainid=self.domain.id) + self.cleanup.append(account) + isolated_persistent_network = Network.create(self.api_client,self.services["isolated_network"], + networkofferingid=networkOffering.id, + accountid=account.name,domainid=self.domain.id, + zoneid=self.zone.id) + self.assertEqual(str(isolated_persistent_network.state).lower(), "implemented", "network state should be implemented, it is %s" \ + % isolated_persistent_network.state) + self.assertIsNotNone(isolated_persistent_network.vlan, "vlan must not be null for persistent network") + + # Check if router is assigned to the persistent network + routers = Router.list(self.api_client, account=account.name, + domainid=account.domainid, + networkid=isolated_persistent_network.id) + + self.assertEqual(validateList(routers)[0], PASS, "Routers list validation failed, list is %s" % routers) + router = routers[0] + + # Check if router if reachable from the host + self.checkRouterAccessibility(router) + + # Deploy VM in the network + try: + virtual_machine = VirtualMachine.create(self.apiclient,self.services["virtual_machine"], + networkids=[isolated_persistent_network.id],serviceofferingid=self.service_offering.id, + accountid=account.name,domainid=self.domain.id) + except Exception as e: + self.fail("vm creation failed: %s" % e) + + # Acquire IP address, create Firewall, NAT rule + self.debug("Associating public IP for network: %s" % isolated_persistent_network.id) + ipaddress = PublicIPAddress.create(self.api_client,accountid=account.name, + zoneid=self.zone.id,domainid=account.domainid, + networkid=isolated_persistent_network.id) + + FireWallRule.create(self.apiclient,ipaddressid=ipaddress.ipaddress.id, + protocol='TCP', cidrlist=[self.services["fwrule"]["cidr"]], + startport=self.services["fwrule"]["startport"],endport=self.services["fwrule"]["endport"]) + NATRule.create(self.api_client, virtual_machine, + self.services["natrule"],ipaddressid=ipaddress.ipaddress.id, + networkid=isolated_persistent_network.id) + + # Check working of PF, NAT rules + try: + virtual_machine.get_ssh_client(ipaddress=ipaddress.ipaddress.ipaddress) + except Exception as e: + self.fail("Exception while SSHing to VM %s with IP %s" % (virtual_machine.id, ipaddress.ipaddress.ipaddress)) + return + + @attr(tags=["advanced"]) + def test_persistent_network_with_RVR(self): + # steps + # 1. create account and isolated network with network offering which has ispersistent field enabled + # and supporting Redundant Virtual Router in it + # 2. Check the Master and Backup Routers are present + # 3. Deploy VM ,acquire IP, create Firewall, NAT rules + # 4. Verify the working of NAT, Firewall rules + # + # validation + # 1. Two routers should belong to the network amd they should be reachable from host + # 2. NAT and Firewall rule should work as expected + + # Create account and create isolated persistent network with RVR in it + account = Account.create(self.api_client, self.services["account"],domainid=self.domain.id) + self.cleanup.append(account) + + # Create isolated persistent network with RVR + isolated_persistent_network_RVR = Network.create(self.api_client,self.services["isolated_network"], + networkofferingid=self.isolated_persistent_network_offering_RVR.id, + accountid=account.name,domainid=self.domain.id, + zoneid=self.zone.id) + self.assertEqual(str(isolated_persistent_network_RVR.state).lower(), "implemented", "network state should be implemented, it is %s" \ + % isolated_persistent_network_RVR.state) + self.assertIsNotNone(isolated_persistent_network_RVR.vlan, "vlan must not be null for persistent network") + + # Check if two routers belong to the network + self.debug("Listing routers for network: %s" % isolated_persistent_network_RVR.name) + routers = Router.list(self.api_client, listall=True, + networkid=isolated_persistent_network_RVR.id) + + self.assertEqual(validateList(routers)[0], PASS, "Routers list validation failed, list is %s" % routers) + self.assertEqual(len(routers), 2, "Length of the list router should be 2 (Backup & master)") + + # Check if routers are reachable from the host + for router in routers: + self.checkRouterAccessibility(router) + + # Deploy VM + try: + virtual_machine = VirtualMachine.create(self.apiclient,self.services["virtual_machine"], + networkids=[isolated_persistent_network_RVR.id],serviceofferingid=self.service_offering.id, + accountid=account.name,domainid=self.domain.id) + except Exception as e: + self.fail("vm creation failed: %s" % e) + + # Acquire public ip, create Firewall, NAT rule + self.debug("Associating public IP for network: %s" % isolated_persistent_network_RVR.id) + ipaddress = PublicIPAddress.create(self.api_client,accountid=account.name, + zoneid=self.zone.id,domainid=account.domainid, + networkid=isolated_persistent_network_RVR.id) + + FireWallRule.create(self.apiclient,ipaddressid=ipaddress.ipaddress.id, + protocol='TCP', cidrlist=[self.services["fwrule"]["cidr"]], + startport=self.services["fwrule"]["startport"],endport=self.services["fwrule"]["endport"]) + NATRule.create(self.api_client, virtual_machine, + self.services["natrule"],ipaddressid=ipaddress.ipaddress.id, + networkid=isolated_persistent_network_RVR.id) + + # Check if Firewall, NAT rule work as expected + try: + virtual_machine.get_ssh_client(ipaddress=ipaddress.ipaddress.ipaddress) + except Exception as e: + self.fail("Exception while SSHing to VM %s with IP %s" % (virtual_machine.id, ipaddress.ipaddress.ipaddress)) + return + + @attr(tags=["advanced"]) + def test_vm_deployment_two_persistent_networks(self): + # steps + # 1. Deploy VM in two persistent networks + # 2. Check working of NAT, Firewall rules in both the networks + # + # validation + # 1. VM should be deployed successfully in two networks + # 2. All network rules should work as expected + + account = Account.create(self.api_client, self.services["account"],domainid=self.domain.id) + self.cleanup.append(account) + isolated_persistent_network_1 = Network.create(self.api_client,self.services["isolated_network"], + networkofferingid=self.isolated_persistent_network_offering.id, + accountid=account.name,domainid=self.domain.id, + zoneid=self.zone.id) + self.assertEqual(str(isolated_persistent_network_1.state).lower(), "implemented", "network state should be implemented, it is %s" \ + % isolated_persistent_network_1.state) + self.assertIsNotNone(isolated_persistent_network_1.vlan, "vlan must not be null for persistent network") + + isolated_persistent_network_2 = Network.create(self.api_client,self.services["isolated_network"], + networkofferingid=self.isolated_persistent_network_offering.id, + accountid=account.name,domainid=self.domain.id, + zoneid=self.zone.id) + self.assertEqual(str(isolated_persistent_network_2.state).lower(), "implemented", "network state should be implemented, it is %s" \ + % isolated_persistent_network_2.state) + self.assertIsNotNone(isolated_persistent_network_2.vlan, "vlan must not be null for persistent network") + + self.debug("Listing routers for network: %s" % isolated_persistent_network_1.name) + routers_nw_1 = Router.list(self.api_client, listall=True, + networkid=isolated_persistent_network_1.id) + + self.assertEqual(validateList(routers_nw_1)[0], PASS, "Routers list validation failed, list is %s" % routers_nw_1) + + # Check if router is reachable from the host + for router in routers_nw_1: + self.checkRouterAccessibility(router) + + self.debug("Listing routers for network: %s" % isolated_persistent_network_2.name) + routers_nw_2 = Router.list(self.api_client, listall=True, + networkid=isolated_persistent_network_2.id) + + self.assertEqual(validateList(routers_nw_2)[0], PASS, "Routers list validation failed, list is %s" % routers_nw_2) + + # Check if router is reachable from the host + for router in routers_nw_2: + self.checkRouterAccessibility(router) + try: + virtual_machine = VirtualMachine.create(self.apiclient,self.services["virtual_machine"], + networkids=[isolated_persistent_network_1.id, isolated_persistent_network_2.id], + serviceofferingid=self.service_offering.id, + accountid=account.name,domainid=self.domain.id) + except Exception as e: + self.fail("vm creation failed: %s" % e) + + self.debug("Associating public IP for network: %s" % isolated_persistent_network_1.id) + ipaddress_nw_1 = PublicIPAddress.create(self.api_client,accountid=account.name, + zoneid=self.zone.id,domainid=account.domainid, + networkid=isolated_persistent_network_1.id) + + FireWallRule.create(self.apiclient,ipaddressid=ipaddress_nw_1.ipaddress.id, + protocol='TCP', cidrlist=[self.services["fwrule"]["cidr"]], + startport=self.services["fwrule"]["startport"],endport=self.services["fwrule"]["endport"]) + + self.debug("Associating public IP for network: %s" % isolated_persistent_network_2.id) + ipaddress_nw_2 = PublicIPAddress.create(self.api_client,accountid=account.name, + zoneid=self.zone.id,domainid=account.domainid, + networkid=isolated_persistent_network_2.id) + + FireWallRule.create(self.apiclient,ipaddressid=ipaddress_nw_2.ipaddress.id, + protocol='TCP', cidrlist=[self.services["fwrule"]["cidr"]], + startport=self.services["fwrule"]["startport"],endport=self.services["fwrule"]["endport"]) + NATRule.create(self.api_client, virtual_machine, + self.services["natrule"],ipaddressid=ipaddress_nw_1.ipaddress.id, + networkid=isolated_persistent_network_1.id) + + try: + virtual_machine.get_ssh_client(ipaddress=ipaddress_nw_1.ipaddress.ipaddress) + except Exception as e: + self.fail("Exception while SSHing to VM %s with IP %s" % (virtual_machine.id, ipaddress_nw_1.ipaddress.ipaddress)) + + NATRule.create(self.api_client, virtual_machine, + self.services["natrule"],ipaddressid=ipaddress_nw_2.ipaddress.id, + networkid=isolated_persistent_network_2.id) + + try: + virtual_machine.get_ssh_client(ipaddress=ipaddress_nw_2.ipaddress.ipaddress) + except Exception as e: + self.fail("Exception while SSHing to VM %s with IP %s" % (virtual_machine.id, ipaddress_nw_2.ipaddress.ipaddress)) + return + + @attr(tags=["advanced"]) + def test_vm_deployment_persistent_and_non_persistent_networks(self): + # steps + # 1. create account and create two networks in it (persistent and non persistent) + # 2. Deploy virtual machine in these two networks + # 3. Associate ip address with both the accounts and create firewall,port forwarding rule + # 4. Try to SSH to VM through both the IPs + # + # validation + # 1. Both persistent and non persistent networks should be created + # 2. virtual machine should be created successfully + # 3. SSH should be successful through both the IPs + + account = Account.create(self.api_client, self.services["account"],domainid=self.domain.id) + self.cleanup.append(account) + network_1 = Network.create(self.api_client,self.services["isolated_network"], + networkofferingid=self.isolated_persistent_network_offering.id, + accountid=account.name,domainid=self.domain.id, + zoneid=self.zone.id) + self.assertEqual(str(network_1.state).lower(), "implemented", "network state should be implemented, it is %s" \ + % network_1.state) + self.assertIsNotNone(network_1.vlan, "vlan must not be null for persistent network") + + network_2 = Network.create(self.api_client,self.services["isolated_network"], + networkofferingid=self.isolated_network_offering.id, + accountid=account.name,domainid=self.domain.id, + zoneid=self.zone.id) + + try: + virtual_machine = VirtualMachine.create(self.apiclient,self.services["virtual_machine"], + networkids=[network_1.id, network_2.id], + serviceofferingid=self.service_offering.id, + accountid=account.name,domainid=self.domain.id) + except Exception as e: + self.fail("vm creation failed: %s" % e) + + self.debug("Associating public IP for network: %s" % network_1.id) + ipaddress_nw_1 = PublicIPAddress.create(self.api_client,accountid=account.name, + zoneid=self.zone.id,domainid=account.domainid, + networkid=network_1.id) + + FireWallRule.create(self.apiclient,ipaddressid=ipaddress_nw_1.ipaddress.id, + protocol='TCP', cidrlist=[self.services["fwrule"]["cidr"]], + startport=self.services["fwrule"]["startport"],endport=self.services["fwrule"]["endport"]) + NATRule.create(self.api_client, virtual_machine, + self.services["natrule"],ipaddressid=ipaddress_nw_1.ipaddress.id, + networkid=network_1.id) + + try: + virtual_machine.get_ssh_client(ipaddress=ipaddress_nw_1.ipaddress.ipaddress) + except Exception as e: + self.fail("Exception while SSHing to VM %s with IP %s" % (virtual_machine.id, ipaddress_nw_1.ipaddress.ipaddress)) + + self.debug("Associating public IP for network: %s" % network_2.id) + ipaddress_nw_2 = PublicIPAddress.create(self.api_client,accountid=account.name, + zoneid=self.zone.id,domainid=account.domainid, + networkid=network_2.id) + + FireWallRule.create(self.apiclient,ipaddressid=ipaddress_nw_2.ipaddress.id, + protocol='TCP', cidrlist=[self.services["fwrule"]["cidr"]], + startport=self.services["fwrule"]["startport"],endport=self.services["fwrule"]["endport"]) + NATRule.create(self.api_client, virtual_machine, + self.services["natrule"],ipaddressid=ipaddress_nw_2.ipaddress.id, + networkid=network_2.id) + + try: + virtual_machine.get_ssh_client(ipaddress=ipaddress_nw_2.ipaddress.ipaddress) + except Exception as e: + self.fail("Exception while SSHing to VM %s with IP %s" % (virtual_machine.id, ipaddress_nw_2.ipaddress.ipaddress)) + return + + @attr(tags=["advanced"]) + def test_change_persistent_network_to_non_persistent(self): + # steps + # 1. Create a persistent network and deploy VM in it + # 2. Update network with non persistent network offering + # 3. Acquire IP, create NAT, firewall rules + # 4. Test NAT, Firewall rules + # 5. Delete VM + # 6. Check the network state after network clenaup interval + # + # validation + # 1. Network updation should be successful + # 2. Network rules should work as expected + # 3. Network state should be allocated after network cleanup interval + + # Create account and create persistent network in it + account = Account.create(self.api_client, self.services["account"],domainid=self.domain.id) + self.cleanup.append(account) + network = Network.create(self.api_client,self.services["isolated_network"], + networkofferingid=self.isolated_persistent_network_offering.id, + accountid=account.name,domainid=self.domain.id, + zoneid=self.zone.id) + self.assertEqual(str(network.state).lower(), "implemented", "network state should be implemented, it is %s" \ + % network.state) + self.assertIsNotNone(network.vlan, "vlan must not be null for persistent network") + + # Update network with non persistent network offering + network.update(self.apiclient, networkofferingid=self.isolated_network_offering.id) + + # Deploy VM, acquire IP, create NAT, firewall rules + try: + virtual_machine = VirtualMachine.create(self.apiclient,self.services["virtual_machine"], + networkids=[network.id], + serviceofferingid=self.service_offering.id, + accountid=account.name,domainid=self.domain.id) + except Exception as e: + self.fail("vm creation failed: %s" % e) + + self.debug("Associating public IP for network: %s" % network.id) + ipaddress = PublicIPAddress.create(self.api_client,accountid=account.name, + zoneid=self.zone.id,domainid=account.domainid, + networkid=network.id) + + FireWallRule.create(self.apiclient,ipaddressid=ipaddress.ipaddress.id, + protocol='TCP', cidrlist=[self.services["fwrule"]["cidr"]], + startport=self.services["fwrule"]["startport"],endport=self.services["fwrule"]["endport"]) + NATRule.create(self.api_client, virtual_machine, + self.services["natrule"],ipaddressid=ipaddress.ipaddress.id, + networkid=network.id) + + # Verify working of network rules + try: + virtual_machine.get_ssh_client(ipaddress=ipaddress.ipaddress.ipaddress) + except Exception as e: + self.fail("Exception while SSHing to VM %s with IP %s" % (virtual_machine.id, ipaddress.ipaddress.ipaddress)) + + # Delete VM + virtual_machine.delete(self.api_client) + + # Verify VM is expunged + self.verifyVmExpungement(virtual_machine) + + # wait for time such that, network is cleaned up + wait_for_cleanup(self.api_client, ["network.gc.interval", "network.gc.wait"]) + + # Check network state now, this will bolster that network updation has taken effect + verifyNetworkState(self.api_client, network.id, "allocated") + return + + @attr(tags=["advanced"]) + def test_delete_account(self): + # steps + # 1. create persistent network and deploy VM in it + # 2. Deploy VM in it and acquire IP address, create NAT, firewall rules + # 3. Delete the account in which network is created + # + # validation + # 1. Persistent network state should be implemented before VM creation and have some vlan assigned + # 2. Network rules should work as expected + # 3. Network and IPs should be freed after account is deleted + + account = Account.create(self.api_client, self.services["account"],domainid=self.domain.id) + network = Network.create(self.api_client,self.services["isolated_network"], + networkofferingid=self.isolated_persistent_network_offering.id, + accountid=account.name,domainid=self.domain.id, + zoneid=self.zone.id) + + try: + virtual_machine = VirtualMachine.create(self.apiclient,self.services["virtual_machine"], + networkids=[network.id],serviceofferingid=self.service_offering.id, + accountid=account.name,domainid=self.domain.id) + except Exception as e: + self.fail("vm creation failed: %s" % e) + self.debug("Associating public IP for network: %s" % network.id) + ipaddress = PublicIPAddress.create(self.api_client,accountid=account.name, + zoneid=self.zone.id,domainid=account.domainid, + networkid=network.id) + + FireWallRule.create(self.apiclient,ipaddressid=ipaddress.ipaddress.id, + protocol='TCP', cidrlist=[self.services["fwrule"]["cidr"]], + startport=self.services["fwrule"]["startport"],endport=self.services["fwrule"]["endport"]) + NATRule.create(self.api_client, virtual_machine, + self.services["natrule"],ipaddressid=ipaddress.ipaddress.id, + networkid=network.id) + + try: + virtual_machine.get_ssh_client(ipaddress=ipaddress.ipaddress.ipaddress) + except Exception as e: + self.fail("Exception while SSHing to VM %s with IP %s" % (virtual_machine.id, ipaddress.ipaddress.ipaddress)) + + # Delete the account + account.delete(self.api_client) + + # Verify the resources belonging to account have been cleaned up + networks = Network.list(self.apiclient, id=network.id) + self.assertEqual(validateList(networks)[0], FAIL, "network list should be enmpty, it is %s" % networks) + + public_ips = PublicIPAddress.list(self.apiclient, id=ipaddress.ipaddress.id) + self.assertEqual(validateList(public_ips)[0], FAIL, "Public Ip list be empty, it is %s" % public_ips) + + return + +@ddt +class TestAssignVirtualMachine(cloudstackTestCase): + """Test Persistent Network creation with assigning VM to different account/domain + """ + + @classmethod + def setUpClass(cls): + cloudstackTestClient = super(TestAssignVirtualMachine,cls).getClsTestClient() + cls.api_client = cloudstackTestClient.getApiClient() + + # Fill services from the external config file + cls.services = cloudstackTestClient.getConfigParser().parsedDict + + # Get Zone, Domain and templates + cls.domain = get_domain(cls.api_client, cls.services) + cls.zone = get_zone(cls.api_client, cls.services) + cls.template = get_template( + cls.api_client, + cls.zone.id, + cls.services["ostype"] + ) + cls.services["virtual_machine"]["zoneid"] = cls.zone.id + cls.services["virtual_machine"]["template"] = cls.template.id + cls.service_offering = ServiceOffering.create( + cls.api_client, + cls.services["service_offering"] + ) + cls.isolated_persistent_network_offering = cls.createNetworkOffering("nw_off_isolated_persistent") + cls.isolated_persistent_network_offering_RVR = cls.createNetworkOffering("nw_off_persistent_RVR") + cls.persistent_network_offering_netscaler = cls.createNetworkOffering("nw_off_isolated_persistent_netscaler") + + # Configure Netscaler device + # If configuration succeeds, set ns_configured to True so that Netscaler tests are executed + cls.ns_configured = False + try: + cls.netscaler = add_netscaler(cls.api_client, cls.zone.id, cls.services["netscaler_VPX"]) + cls._cleanup.append(cls.netscaler) + cls.ns_configured = True + except Exception: + cls.ns_configured = False + + + # network will be deleted as part of account cleanup + cls._cleanup = [ + cls.service_offering, cls.isolated_persistent_network_offering, + cls.isolated_persistent_network_offering_RVR, + cls.persistent_network_offering_netscaler + ] + return + + @classmethod + def tearDownClass(cls): + try: + # Cleanup resources used + cleanup_resources(cls.api_client, cls._cleanup) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + return + + @classmethod + def createNetworkOffering(cls, network_offering_type): + network_offering = NetworkOffering.create( + cls.api_client, + cls.services[network_offering_type], + conservemode=False + ) + # Update network offering state from disabled to enabled. + NetworkOffering.update(network_offering, cls.api_client, id=network_offering.id, + state="enabled") + return network_offering + + def setUp(self): + self.apiclient = self.testClient.getApiClient() + self.dbclient = self.testClient.getDbConnection() + self.cleanup = [ ] + return + + def tearDown(self): + try: + # Clean up, terminate the resources created + cleanup_resources(self.apiclient, self.cleanup) + self.cleanup[:] = [] + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + return + + @data("VR","RVR","LB-NS") + @attr(tags=["advanced", "advancedns"]) + def test_assign_vm_different_account_VR(self, value): + + # This test shall be run with three types of persistent networks + # a) All services through VR + # b) LB service through Netscaler + # c) with Redundant Virtual Router facility + + # steps + # 1. create two accounts (account1 and account2) + # 2. Create a persistent network (with VR/RVR/Netscaler-LB with VR services) in account1 and deploy VM in it with this network + # 3. Stop the VM and assign the VM to account2 + # + # validation + # 1. Assign VM operation should be successful + # 2. New network should be created in the other account + + # Create Accounts + account_1 = Account.create(self.apiclient,self.services["account"],domainid=self.domain.id) + self.cleanup.append(account_1) + + account_2 = Account.create(self.apiclient,self.services["account"],domainid=self.domain.id) + self.cleanup.append(account_2) + + # Set the network offering according to the test scenario (data passed to the test case + if value == "VR": + network_offering = self.isolated_persistent_network_offering + elif value == "RVR": + network_offering = self.isolated_persistent_network_offering_RVR + elif value == "LB-NS": + if self.ns_configured: + network_offering = self.persistent_network_offering_netscaler + else: + self.skipTest("This test requires netscaler to be configured in the network") + + network = Network.create(self.api_client,self.services["isolated_network"], + networkofferingid=network_offering.id, + accountid=account_1.name,domainid=self.domain.id, + zoneid=self.zone.id) + verifyNetworkState(self.api_client, network.id, "implemented") + self.assertIsNotNone(network.vlan, "vlan must not be null for persistent network") + + try: + virtual_machine = VirtualMachine.create(self.apiclient,self.services["virtual_machine"], + networkids=[network.id], + serviceofferingid=self.service_offering.id, + accountid=account_1.name,domainid=self.domain.id) + except Exception as e: + self.fail("vm creation failed: %s" % e) + + virtual_machine.stop(self.apiclient) + + vms = VirtualMachine.list(self.apiclient, id=virtual_machine.id) + self.assertEqual(validateList(vms)[0], PASS, "vm list validation failed, vm list is %s" % vms) + self.assertEqual(str(vms[0].state).lower(), "stopped", "vm state should be stopped, it is %s" % vms[0].state) + + # Assign virtual machine to different account + virtual_machine.assign_virtual_machine(self.apiclient, account=account_2.name, domainid=self.domain.id) + + # Start VM + virtual_machine.start(self.apiclient) + + # Verify that new network is created in other account + networks = Network.list(self.apiclient, account=account_2.name, domainid = account_2.domainid) + self.assertEqual(validateList(networks)[0], PASS, "networks list validation failed, list is %s" % networks) + return + +@ddt +class TestProjectAccountOperations(cloudstackTestCase): + """Test suspend/disable/lock account/project operations when they have persistent network + """ + + @classmethod + def setUpClass(cls): + cloudstackTestClient = super(TestProjectAccountOperations,cls).getClsTestClient() + cls.api_client = cloudstackTestClient.getApiClient() + + # Fill services from the external config file + cls.services = cloudstackTestClient.getConfigParser().parsedDict + + # Get Zone, Domain and templates + cls.domain = get_domain(cls.api_client, cls.services) + cls.zone = get_zone(cls.api_client, cls.services) + cls.template = get_template( + cls.api_client, + cls.zone.id, + cls.services["ostype"] + ) + cls.services["virtual_machine"]["zoneid"] = cls.zone.id + cls.services["virtual_machine"]["template"] = cls.template.id + cls.service_offering = ServiceOffering.create( + cls.api_client, + cls.services["service_offering"] + ) + cls.isolated_persistent_network_offering = NetworkOffering.create( + cls.api_client, + cls.services["nw_off_isolated_persistent"], + conservemode=False + ) + # Update network offering state from disabled to enabled. + cls.isolated_persistent_network_offering.update(cls.api_client, state="enabled") + + # network will be deleted as part of account cleanup + cls._cleanup = [ + cls.service_offering, cls.isolated_persistent_network_offering + ] + return + + @classmethod + def tearDownClass(cls): + try: + # Cleanup resources used + cleanup_resources(cls.api_client, cls._cleanup) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + return + + def setUp(self): + self.apiclient = self.testClient.getApiClient() + self.dbclient = self.testClient.getDbConnection() + self.cleanup = [ ] + return + + def tearDown(self): + try: + # Clean up, terminate the resources created + cleanup_resources(self.apiclient, self.cleanup) + self.cleanup[:] = [] + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + return + + @data("locked","disabled") + @attr(tags=["advanced"]) + def test_account_operations(self, value): + # steps + # 1. create account and create persistent network in it + # 2. Disable/lock the account + # + # validation + # 3. Wait for network cleanup interval and verify that network is not cleaned up and it is still in + # implemented state + + account = Account.create(self.apiclient,self.services["account"],domainid=self.domain.id) + self.cleanup.append(account) + + network = Network.create(self.api_client,self.services["isolated_network"], + networkofferingid=self.isolated_persistent_network_offering.id, + accountid=account.name,domainid=self.domain.id, + zoneid=self.zone.id) + verifyNetworkState(self.api_client, network.id, "implemented") + self.assertIsNotNone(network.vlan, "vlan must not be null for persistent network") + + if value == "disabled": + account.disable(self.apiclient) + elif value == "locked": + account.disable(self.apiclient, lock=True) + + accounts = Account.list(self.apiclient, id=account.id) + self.assertEqual(validateList(accounts)[0], PASS, "accounts list validation failed, list id %s" % accounts) + self.assertEqual(str(accounts[0].state).lower(), value, "account state should be %s, it is %s" % (value, accounts[0].state)) + + # Wait for network cleanup interval + wait_for_cleanup(self.api_client, ["network.gc.interval", "network.gc.wait"]) + + networks = Network.list(self.apiclient, account=account.name, domainid = account.domainid) + self.assertEqual(validateList(networks)[0], PASS, "networks list validation failed, list is %s" % networks) + + verifyNetworkState(self.api_client, networks[0].id, "implemented") + self.assertIsNotNone(networks[0].vlan, "vlan must not be null for persistent network") + return + + @attr(tags=["advanced"]) + def test_project_operations(self): + # steps + # 1. create account and create persistent network in it + # 2. Add account to project + # 3. Suspend the project + # + # validation + # 1. Verify that account has been added to the project + # 2. Wait for network cleanup interval and verify that network is not cleaned up and it is still in + # implemented state + + # Create Account + account = Account.create(self.apiclient,self.services["account"],domainid=self.domain.id) + self.cleanup.append(account) + + # Create Project + project = Project.create(self.apiclient, self.services["project"]) + self.cleanup.append(project) + + network = Network.create(self.api_client,self.services["isolated_network"], + networkofferingid=self.isolated_persistent_network_offering.id, + accountid=account.name,domainid=self.domain.id, + zoneid=self.zone.id) + + # Add account to the project + project.addAccount(self.apiclient, account = account.name) + + # Verify the account name in the list of accounts belonging to the project + projectAccounts = Project.listAccounts(self.apiclient, projectid = project.id) + self.assertEqual(validateList(projectAccounts)[0], PASS, "project accounts list validation failed, list is %s" % projectAccounts) + + accountNames = [projectAccount.account for projectAccount in projectAccounts] + self.assertTrue(account.name in accountNames, "account %s is not present in account list %s of project %s" % + (account.name, accountNames, project.id)) + + # Suspend Project + project.suspend(self.apiclient) + + # Verify the project is suspended + projects = Project.list(self.apiclient, id=project.id) + self.assertEqual(validateList(projects)[0], PASS, "projects list validation failed, list is %s" % projects) + self.assertEqual(str(projects[0].state).lower(), "suspended", "project state should be suspended, it is %s" % projects[0].state) + + # Wait for network cleanup interval + wait_for_cleanup(self.api_client, ["network.gc.interval", "network.gc.wait"]) + + verifyNetworkState(self.apiclient, network.id, "implemented") + return + +@ddt +class TestRestartPersistentNetwork(cloudstackTestCase): + """Test restart persistent network with cleanup parameter true and false + """ + + @classmethod + def setUpClass(cls): + cloudstackTestClient = super(TestRestartPersistentNetwork,cls).getClsTestClient() + cls.api_client = cloudstackTestClient.getApiClient() + + # Fill services from the external config file + cls.services = cloudstackTestClient.getConfigParser().parsedDict + + # Get Zone, Domain and templates + cls.domain = get_domain(cls.api_client, cls.services) + cls.zone = get_zone(cls.api_client, cls.services) + cls.template = get_template( + cls.api_client, + cls.zone.id, + cls.services["ostype"] + ) + cls.services["virtual_machine"]["zoneid"] = cls.zone.id + cls.services["virtual_machine"]["template"] = cls.template.id + cls.service_offering = ServiceOffering.create( + cls.api_client, + cls.services["service_offering"] + ) + cls.isolated_persistent_network_offering = NetworkOffering.create(cls.api_client, + cls.services["nw_off_isolated_persistent"], + conservemode=False) + + cls.isolated_persistent_network_offering_netscaler = NetworkOffering.create(cls.api_client, + cls.services["nw_off_isolated_persistent_netscaler"], + conservemode=False) + # Update network offering state from disabled to enabled. + cls.isolated_persistent_network_offering.update(cls.api_client, state="enabled") + cls.isolated_persistent_network_offering_netscaler.update(cls.api_client, state="enabled") + + + # Configure Netscaler device + # If configuration succeeds, set ns_configured to True so that Netscaler tests are executed + cls.ns_configured = False + try: + cls.netscaler = add_netscaler(cls.api_client, cls.zone.id, cls.services["netscaler_VPX"]) + cls._cleanup.append(cls.netscaler) + cls.ns_configured = True + except Exception: + cls.ns_configured = False + + # network will be deleted as part of account cleanup + cls._cleanup = [ + cls.service_offering, cls.isolated_persistent_network_offering, + cls.isolated_persistent_network_offering_netscaler + ] + return + + @classmethod + def tearDownClass(cls): + try: + # Cleanup resources used + cleanup_resources(cls.api_client, cls._cleanup) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + return + + def setUp(self): + self.apiclient = self.testClient.getApiClient() + self.dbclient = self.testClient.getDbConnection() + self.cleanup = [ ] + return + + def tearDown(self): + try: + # Clean up, terminate the resources created + cleanup_resources(self.apiclient, self.cleanup) + self.cleanup[:] = [] + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + return + + def checkRouterAccessibility(self, router): + """Check if router is accessible through its linklocalip""" + + hypervisor = str(get_hypervisor_type(self.api_client)) + + if hypervisor.lower() == 'vmware': + #SSH is done via management server for Vmware + sourceip = self.api_client.connection.mgtSvr + else: + #For others, we will have to get the ipaddress of host connected to vm + hosts = Host.list(self.api_client,id=router.hostid) + self.assertEqual(validateList(hosts)[0], PASS, "hosts list validation failed, list is %s" % hosts) + host = hosts[0] + sourceip = host.ipaddress + # end else + + try: + sshClient = SshClient(host=sourceip, port=22, user='root',passwd=self.services["host_password"]) + res = sshClient.execute("ping -c 1 %s" % ( + router.linklocalip + )) + except Exception as e: + self.fail("SSH Access failed for %s: %s" % \ + (sourceip, e) + ) + result = str(res) + self.assertEqual( + result.count("1 received"), + 1, + "ping to router should be successful" + ) + return + + @data("true","false") + @attr(tags=["advanced"]) + def test_cleanup_persistent_network(self, value): + # steps + # 1. Create account and create persistent network in it + # 2. Verify that router is reachable from the host + # 3. Acquire public IP, open firewall and create LB rule + # 4. Restart the network with clenup parameter true/false + # 5. Check network state after restart, it should be implemented + # 6. Deploy VM, assign LB rule to it, and verify the LB rule + + account = Account.create(self.apiclient,self.services["account"],domainid=self.domain.id) + self.cleanup.append(account) + + isolated_persistent_network = Network.create(self.api_client,self.services["isolated_network"], + networkofferingid=self.isolated_persistent_network_offering.id, + accountid=account.name,domainid=self.domain.id, + zoneid=self.zone.id) + verifyNetworkState(self.apiclient, isolated_persistent_network.id, "implemented") + self.assertIsNotNone(isolated_persistent_network.vlan, "vlan must not be null for persistent network") + + self.debug("Listing routers for network: %s" % isolated_persistent_network.name) + routers = Router.list(self.api_client, listall=True, + networkid=isolated_persistent_network.id) + + self.assertEqual(validateList(routers)[0], PASS, "Routers list validation failed, list is %s" % routers) + + # Check if router is reachable from the host + for router in routers: + self.checkRouterAccessibility(router) + + self.debug("Associating public IP for network: %s" % isolated_persistent_network.id) + ipaddress = PublicIPAddress.create(self.api_client,accountid=account.name, + zoneid=self.zone.id,domainid=account.domainid, + networkid=isolated_persistent_network.id) + + FireWallRule.create(self.apiclient,ipaddressid=ipaddress.ipaddress.id, + protocol='TCP', cidrlist=[self.services["fwrule"]["cidr"]], + startport=self.services["fwrule"]["startport"],endport=self.services["fwrule"]["endport"]) + + # Create LB Rule + lb_rule = LoadBalancerRule.create(self.apiclient,self.services["lbrule"], + ipaddressid=ipaddress.ipaddress.id, accountid=account.name, + networkid=isolated_persistent_network.id, domainid=account.domainid) + + # Restart Network + isolated_persistent_network.restart(self.apiclient, cleanup=value) + + # List networks + networks = Network.list(self.apiclient, account=account.name, domainid = account.domainid) + self.assertEqual(validateList(networks)[0], PASS, "networks list validation failed, list is %s" % networks) + + verifyNetworkState(self.apiclient, networks[0].id, "implemented") + self.assertIsNotNone(networks[0].vlan, "vlan must not be null for persistent network") + + # Deploy VM + try: + virtual_machine = VirtualMachine.create(self.apiclient,self.services["virtual_machine"], + networkids=[isolated_persistent_network.id],serviceofferingid=self.service_offering.id, + accountid=account.name,domainid=self.domain.id) + except Exception as e: + self.fail("vm creation failed: %s" % e) + + lb_rule.assign(self.api_client, [virtual_machine]) + + try: + virtual_machine.get_ssh_client(ipaddress=ipaddress.ipaddress.ipaddress) + except Exception as e: + self.fail("Exception while SSHing to VM %s with IP %s" % (virtual_machine.id, ipaddress.ipaddress.ipaddress)) + + return + + @data("true","false") + @attr(tags=["advanced", "advancedns"]) + def test_cleanup_persistent_network_lb_netscaler(self, value): + # steps + # 1. Create account and create persistent network in it with LB service provided by netscaler + # 2. Verify that router is reachable from the host + # 3. Acquire public IP, open firewall and create LB rule + # 4. Restart the network with clenup parameter true/false + # 5. Check network state after restart, it should be implemented + # 6. Deploy VM, assign LB rule to it, and verify the LB rule + + if not self.ns_configured: + self.skipTest("This test required netscaler to be configured in the network") + + account = Account.create(self.apiclient,self.services["account"],domainid=self.domain.id) + self.cleanup.append(account) + + isolated_persistent_network = Network.create(self.api_client,self.services["isolated_network"], + networkofferingid=self.isolated_persistent_network_offering_netscaler.id, + accountid=account.name,domainid=self.domain.id, + zoneid=self.zone.id) + verifyNetworkState(self.apiclient, isolated_persistent_network.id, "implemented") + self.assertIsNotNone(isolated_persistent_network.vlan, "vlan must not be null for persistent network") + + self.debug("Listing routers for network: %s" % isolated_persistent_network.name) + routers = Router.list(self.api_client, listall=True, + networkid=isolated_persistent_network.id) + + self.assertEqual(validateList(routers)[0], PASS, "Routers list validation failed, list is %s" % routers) + + # Check if router is reachable from the host + for router in routers: + self.checkRouterAccessibility(router) + + self.debug("Associating public IP for network: %s" % isolated_persistent_network.id) + ipaddress = PublicIPAddress.create(self.api_client,accountid=account.name, + zoneid=self.zone.id,domainid=account.domainid, + networkid=isolated_persistent_network.id) + + # Create LB Rule + lb_rule = LoadBalancerRule.create(self.apiclient,self.services["lbrule"], + ipaddressid=ipaddress.ipaddress.id, accountid=account.name, + networkid=isolated_persistent_network.id, domainid=account.domainid) + + # Restart Network + isolated_persistent_network.restart(self.apiclient, cleanup=value) + + # List networks + networks = Network.list(self.apiclient, account=account.name, domainid = account.domainid) + self.assertEqual(validateList(networks)[0], PASS, "networks list validation failed, list is %s" % networks) + + verifyNetworkState(self.apiclient, networks[0].id, "implemented") + self.assertIsNotNone(networks[0].vlan, "vlan must not be null for persistent network") + + # Deploy VM + try: + virtual_machine = VirtualMachine.create(self.apiclient,self.services["virtual_machine"], + networkids=[isolated_persistent_network.id],serviceofferingid=self.service_offering.id, + accountid=account.name,domainid=self.domain.id) + except Exception as e: + self.fail("vm creation failed: %s" % e) + + lb_rule.assign(self.api_client, [virtual_machine]) + + try: + virtual_machine.get_ssh_client(ipaddress=ipaddress.ipaddress.ipaddress) + except Exception as e: + self.fail("Exception while SSHing to VM %s with IP %s" % (virtual_machine.id, ipaddress.ipaddress.ipaddress)) + + return + +@ddt +class TestVPCNetworkOperations(cloudstackTestCase): + """Test VPC network operations consisting persistent networks + """ + + @classmethod + def setUpClass(cls): + cloudstackTestClient = super(TestVPCNetworkOperations,cls).getClsTestClient() + cls.api_client = cloudstackTestClient.getApiClient() + + # Fill services from the external config file + cls.services = cloudstackTestClient.getConfigParser().parsedDict + + # Get Zone, Domain and templates + cls.domain = get_domain(cls.api_client, cls.services) + cls.zone = get_zone(cls.api_client, cls.services) + cls.template = get_template( + cls.api_client, + cls.zone.id, + cls.services["ostype"] + ) + cls.services["virtual_machine"]["zoneid"] = cls.zone.id + cls.services["virtual_machine"]["template"] = cls.template.id + cls.service_offering = ServiceOffering.create( + cls.api_client, + cls.services["service_offering"] + ) + cls.persistent_network_offering_NoLB = NetworkOffering.create(cls.api_client, cls.services["nw_off_persistent_VPCVR_NoLB"], + conservemode=False) + # Update network offering state from disabled to enabled. + cls.persistent_network_offering_NoLB.update(cls.api_client, state="enabled") + + cls.persistent_network_offering_LB = NetworkOffering.create(cls.api_client, cls.services["nw_off_persistent_VPCVR_LB"], + conservemode=False) + # Update network offering state from disabled to enabled. + cls.persistent_network_offering_LB.update(cls.api_client, state="enabled") + + cls.vpc_off = VpcOffering.create(cls.api_client, cls.services["vpc_offering"]) + cls.vpc_off.update(cls.api_client, state='Enabled') + + # network will be deleted as part of account cleanup + cls._cleanup = [ + cls.service_offering, cls.persistent_network_offering_NoLB, cls.vpc_off, + cls.persistent_network_offering_LB + ] + return + + @classmethod + def tearDownClass(cls): + try: + # Cleanup resources used + cleanup_resources(cls.api_client, cls._cleanup) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + return + + def setUp(self): + self.apiclient = self.testClient.getApiClient() + self.dbclient = self.testClient.getDbConnection() + self.cleanup = [] + return + + def tearDown(self): + try: + # Clean up, terminate the resources created + cleanup_resources(self.apiclient, self.cleanup) + self.cleanup[:] = [] + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + return + + def GetAssociatedIpForNetwork(self, networkid, vpcid, account): + """ Associate IP address with the network and open firewall for it + return associated IPaddress""" + + ipaddress = PublicIPAddress.create(self.api_client,zoneid=self.zone.id,networkid=networkid, vpcid=vpcid, + accountid=account.name, domainid=account.domainid) + + return ipaddress + + def CreateIngressEgressNetworkACLForNetwork(self, networkid): + + try: + ingressAcl = NetworkACL.create(self.apiclient, networkid=networkid, services=self.services["natrule"], traffictype='Ingress') + egressAcl = NetworkACL.create(self.apiclient, networkid=networkid, services=self.services["icmprule"], traffictype='Egress') + except Exception as e: + self.fail("Failed while creating Network ACL rule for network %s with error %s" % (networkid,e)) + return ingressAcl,egressAcl + + def CheckIngressEgressConnectivityofVM(self, virtual_machine, ipaddress): + try: + ssh = SshClient( + ipaddress, + 22, + virtual_machine.username, + virtual_machine.password + ) + # Ping to outsite world + res = ssh.execute("ping -c 1 www.google.com") + # res = 64 bytes from maa03s17-in-f20.1e100.net (74.125.236.212): + # icmp_req=1 ttl=57 time=25.9 ms + # --- www.l.google.com ping statistics --- + # 1 packets transmitted, 1 received, 0% packet loss, time 0ms + # rtt min/avg/max/mdev = 25.970/25.970/25.970/0.000 ms + except Exception as e: + self.fail("SSH Access failed for vm %s with IP address %s: %s" % \ + (virtual_machine.id,ipaddress, e)) + result = str(res) + self.assertEqual(result.count("1 received"),1, "Ping to outside world from VM should be successful") + return + + def VerifyNetworkCleanup(self, networkid): + """Verify that network is cleaned up""" + + networks=Network.list(self.apiclient,id=networkid) + self.assertEqual(validateList(networks)[0], FAIL, "networks list should be empty, it is %s" % networks) + return + + def VerifyDomainCleanup(self, domainid): + """Verify that domain is cleaned up""" + + domains=Domain.list(self.apiclient,id=domainid) + self.assertEqual(validateList(domains)[0], FAIL, "domains list should be empty, it is %s" % domains) + return + + def VerifyVpcCleanup(self, vpcid): + """Verify that VPC is cleaned up""" + vpcs = VPC.list(self.apiclient,id=vpcid) + self.assertEqual(validateList(vpcs)[0], FAIL, "VPC list should be empty, it is %s" % vpcs) + return + + def VerifyVirtualMachineCleanup(self, vmid): + """Verify that virtual machine is cleaned up""" + vms=VirtualMachine.list(self.apiclient,id=vmid) + self.assertEqual(validateList(vms)[0], FAIL, "vms list should be empty, it is %s" % vms) + return + + def VerifyAclRuleCleanup(self, aclRuleId): + """Verify that network ACL rule is cleaned up""" + networkAcls=NetworkACL.list(self.apiclient,id=aclRuleId) + self.assertEqual(validateList(networkAcls)[0], FAIL, "networkAcls list should be empty, it is %s" % networkAcls) + return + + @data("delete", "restart") + @attr(tags=["advanced"]) + def test_vpc_network_life_cycle(self, value): + # steps + # 1. Create account and create VPC network in the account + # 2. Create two persistent networks within this VPC + # 3. Restart/delete VPC network + + # Validations + # 1. In case of Restart operation, restart should be successful and persistent networks should be back in persistent state + # 2. In case of Delete operation, VR servicing the VPC should get destroyed and sourceNAT ip should get released + + account = Account.create(self.apiclient,self.services["account"],domainid=self.domain.id) + self.cleanup.append(account) + + self.services["vpc"]["cidr"] = "10.1.1.1/16" + self.debug("creating a VPC network in the account: %s" % + account.name) + vpc = VPC.create(self.apiclient, self.services["vpc"], + vpcofferingid=self.vpc_off.id, zoneid=self.zone.id, + account=account.name, domainid=account.domainid) + vpcs = VPC.list(self.apiclient, id=vpc.id) + self.assertEqual(validateList(vpcs)[0], PASS, "VPC list validation failed, vpc list is %s" % vpcs) + + + VpcRouters = Router.list(self.apiclient, vpcid = vpc.id, listall=True) + self.assertEqual(validateList(VpcRouters)[0], PASS, "VpcRouters list validation failed, list is %s" % VpcRouters) + vpcrouter = VpcRouters[0] + + publicipaddresses = PublicIPAddress.list(self.apiclient, vpcid=vpc.id, listall=True) + self.assertEqual(validateList(publicipaddresses)[0], PASS, "Public IP Addresses list validation failed, list is %s" % publicipaddresses) + + persistent_network_1 = Network.create(self.api_client,self.services["isolated_network"], + networkofferingid=self.persistent_network_offering_NoLB.id, + accountid=account.name,domainid=self.domain.id, + zoneid=self.zone.id, vpcid=vpc.id, gateway="10.1.1.1", netmask="255.255.255.0") + verifyNetworkState(self.apiclient, persistent_network_1.id, "implemented") + self.assertIsNotNone(persistent_network_1.vlan, "vlan must not be null for persistent network %s" % persistent_network_1.id) + + persistent_network_2 = Network.create(self.api_client,self.services["isolated_network"], + networkofferingid=self.persistent_network_offering_NoLB.id, + accountid=account.name,domainid=self.domain.id, + zoneid=self.zone.id, vpcid=vpc.id, gateway="10.1.2.1", netmask="255.255.255.0") + verifyNetworkState(self.apiclient, persistent_network_2.id, "implemented") + self.assertIsNotNone(persistent_network_2.vlan, "vlan must not be null for persistent network: %s" % persistent_network_2.id) + + if value == "restart": + # Restart VPC + vpc.restart(self.apiclient) + verifyNetworkState(self.apiclient, persistent_network_1.id, "implemented") + verifyNetworkState(self.apiclient, persistent_network_2.id, "implemented") + + elif value == "delete": + vpc.delete(self.apiclient) + vpcs = VPC.list(self.apiclient, id=vpc.id) + self.assertEqual(validateList(vpcs)[0], FAIL, "vpc list should be empty, list is %s" % vpcs) + + # Check if router is deleted or not + routers = Router.list(self.apiclient, id=vpcrouter.id) + self.assertEqual(validateList(routers)[0], FAIL, "routers list should be empty, it is %s" % routers) + + # Check if source nat IP address is released + ipaddresses = PublicIPAddress.list(self.apiclient, id=publicipaddresses[0].id) + self.assertEqual(validateList(ipaddresses)[0], FAIL, "public ip addresses list should be empty, list is %s" % ipaddresses) + return + + @attr(tags=["advanced"]) + def test_vpc_force_delete_domain(self): + # steps + # 1. Create account and create VPC network in the account + # 2. Create two persistent networks within this VPC + # 3. Restart/delete VPC network + + # Validations + # 1. In case of Restart operation, restart should be successful and persistent networks should be back in persistent state + # 2. In case of Delete operation, VR servicing the VPC should get destroyed and sourceNAT ip should get released + + child_domain = Domain.create(self.apiclient,services=self.services["domain"], + parentdomainid=self.domain.id) + + account_1 = Account.create(self.apiclient,self.services["account"],domainid=child_domain.id) + account_2 = Account.create(self.apiclient,self.services["account"],domainid=child_domain.id) + + self.services["vpc"]["cidr"] = "10.1.1.1/16" + vpc_1 = VPC.create(self.apiclient, self.services["vpc"], + vpcofferingid=self.vpc_off.id, zoneid=self.zone.id, + account=account_1.name, domainid=account_1.domainid) + vpcs = VPC.list(self.apiclient, id=vpc_1.id) + self.assertEqual(validateList(vpcs)[0], PASS, "VPC list validation failed, vpc list is %s" % vpcs) + + vpc_2 = VPC.create(self.apiclient, self.services["vpc"], + vpcofferingid=self.vpc_off.id, zoneid=self.zone.id, + account=account_2.name, domainid=account_2.domainid) + vpcs = VPC.list(self.apiclient, id=vpc_2.id) + self.assertEqual(validateList(vpcs)[0], PASS, "VPC list validation failed, vpc list is %s" % vpcs) + + persistent_network_1 = Network.create(self.api_client,self.services["isolated_network"], + networkofferingid=self.persistent_network_offering_NoLB.id, + accountid=account_1.name,domainid=account_1.domainid, + zoneid=self.zone.id, vpcid=vpc_1.id, gateway="10.1.1.1", netmask="255.255.255.0") + verifyNetworkState(self.apiclient, persistent_network_1.id, "implemented") + self.assertIsNotNone(persistent_network_1.vlan, "vlan must not be null for persistent network %s" % persistent_network_1.id) + + persistent_network_2 = Network.create(self.api_client,self.services["isolated_network"], + networkofferingid=self.persistent_network_offering_NoLB.id, + accountid=account_2.name,domainid=account_2.domainid, + zoneid=self.zone.id, vpcid=vpc_2.id, gateway="10.1.1.1", netmask="255.255.255.0") + verifyNetworkState(self.apiclient, persistent_network_2.id, "implemented") + self.assertIsNotNone(persistent_network_2.vlan, "vlan must not be null for persistent network: %s" % persistent_network_2.id) + + # Force delete domain + try: + child_domain.delete(self.apiclient, cleanup=True) + except Exception as e: + self.fail("Failed to delete domain: %s" % e) + + self.debug("Waiting for account.cleanup.interval" + + " to cleanup any remaining resouces") + # Sleep 3*account.gc to ensure that all resources are deleted + wait_for_cleanup(self.apiclient, ["account.cleanup.interval"]*3) + self.VerifyDomainCleanup(child_domain.id) + + with self.assertRaises(Exception) as e: + Account.list(self.apiclient,name=account_1.name, domainid=account_1.domainid,listall=True) + + with self.assertRaises(Exception) as e: + Account.list(self.apiclient,name=account_2.name, domainid=account_2.domainid,listall=True) + + self.VerifyVpcCleanup(vpc_1.id) + self.VerifyVpcCleanup(vpc_2.id) + self.VerifyNetworkClenaup(persistent_network_1.id) + self.VerifyNetworkCleanup(persistent_network_2.id) + return + + @attr(tags=["advanced"]) + def test_vpc_delete_account(self): + # steps + # 1. Create account and create VPC network in the account + # 2. Create two persistent networks within this VPC + # 3. Restart/delete VPC network + + # Validations + # 1. In case of Restart operation, restart should be successful and persistent networks should be back in persistent state + # 2. In case of Delete operation, VR servicing the VPC should get destroyed and sourceNAT ip should get released + + # Create Account + account = Account.create(self.apiclient,self.services["account"],domainid=self.domain.id) + + # Create VPC + self.services["vpc"]["cidr"] = "10.1.1.1/16" + vpc = VPC.create(self.apiclient, self.services["vpc"], + vpcofferingid=self.vpc_off.id, zoneid=self.zone.id, + account=account.name, domainid=account.domainid) + vpcs = VPC.list(self.apiclient, id=vpc.id) + self.assertEqual(validateList(vpcs)[0], PASS, "VPC list validation failed, vpc list is %s" % vpcs) + + # Create Persistent Networks as tiers of VPC + persistent_network_1 = Network.create(self.api_client,self.services["isolated_network"], + networkofferingid=self.persistent_network_offering_NoLB.id, + accountid=account.name,domainid=account.domainid, + zoneid=self.zone.id, vpcid=vpc.id, gateway="10.1.1.1", netmask="255.255.255.0") + verifyNetworkState(self.apiclient, persistent_network_1.id, "implemented") + self.assertIsNotNone(persistent_network_1.vlan, "vlan must not be null for persistent network %s" % persistent_network_1.id) + + persistent_network_2 = Network.create(self.api_client,self.services["isolated_network"], + networkofferingid=self.persistent_network_offering_LB.id, + accountid=account.name,domainid=account.domainid, + zoneid=self.zone.id, vpcid=vpc.id, gateway="10.1.2.1", netmask="255.255.255.0") + verifyNetworkState(self.apiclient, persistent_network_2.id, "implemented") + self.assertIsNotNone(persistent_network_2.vlan, "vlan must not be null for persistent network: %s" % persistent_network_2.id) + + # Deploy VMs in above networks (VM1, VM2 in network1 and VM3, VM4 in network2) + try: + virtual_machine_1 = VirtualMachine.create(self.apiclient,self.services["virtual_machine"], + networkids=[persistent_network_1.id],serviceofferingid=self.service_offering.id, + accountid=account.name,domainid=self.domain.id) + + virtual_machine_2 = VirtualMachine.create(self.apiclient,self.services["virtual_machine"], + networkids=[persistent_network_1.id],serviceofferingid=self.service_offering.id, + accountid=account.name,domainid=self.domain.id) + + virtual_machine_3 = VirtualMachine.create(self.apiclient,self.services["virtual_machine"], + networkids=[persistent_network_2.id],serviceofferingid=self.service_offering.id, + accountid=account.name,domainid=self.domain.id) + + virtual_machine_4 = VirtualMachine.create(self.apiclient,self.services["virtual_machine"], + networkids=[persistent_network_2.id],serviceofferingid=self.service_offering.id, + accountid=account.name,domainid=self.domain.id) + except Exception as e: + self.fail("vm creation failed: %s" % e) + + # Associate IP addresses to persistent networks + ipaddress_1 = self.GetAssociatedIpForNetwork(persistent_network_1.id, vpcid=vpc.id, account=account) + ipaddress_2 = self.GetAssociatedIpForNetwork(persistent_network_1.id, vpcid=vpc.id, account=account) + ipaddress_3 = self.GetAssociatedIpForNetwork(persistent_network_2.id, vpcid=vpc.id, account=account) + + # Create NAT rule for VM 1 + NATRule.create(self.api_client, virtual_machine_1, + self.services["natrule"],ipaddressid=ipaddress_1.ipaddress.id, + networkid=persistent_network_1.id) + + # Create Static NAT rule for VM 2 + StaticNATRule.enable(self.apiclient, ipaddressid=ipaddress_2.ipaddress.id, + virtualmachineid=virtual_machine_2.id, + networkid=persistent_network_1.id) + + # Create load balancer rule for ipaddress3 and assign to VM3 and VM4 + lb_rule = LoadBalancerRule.create(self.apiclient,self.services["lbrule"], + ipaddressid=ipaddress_3.ipaddress.id, accountid=account.name, + networkid=persistent_network_2.id, domainid=account.domainid) + lb_rule.assign(self.api_client, [virtual_machine_3, virtual_machine_4]) + + # Create network ACL for both ther persistent networks (tiers of VPC) + ingressAclNetwork1, egressAclNetwork1 = self.CreateIngressEgressNetworkACLForNetwork(persistent_network_1.id) + ingressAclNetwork2, egressAclNetwork2 = self.CreateIngressEgressNetworkACLForNetwork(persistent_network_2.id) + + self.CheckIngressEgressConnectivityofVM(virtual_machine_1, ipaddress_1.ipaddress.ipaddress) + self.CheckIngressEgressConnectivityofVM(virtual_machine_2, ipaddress_2.ipaddress.ipaddress) + self.CheckIngressEgressConnectivityofVM(virtual_machine_3, ipaddress_3.ipaddress.ipaddress) + self.CheckIngressEgressConnectivityofVM(virtual_machine_4, ipaddress_3.ipaddress.ipaddress) + + # Delete account + account.delete(self.apiclient) + + # Verify all the resources owned by the account are deleted + self.debug("Waiting for account.cleanup.interval" + + " to cleanup any remaining resouces") + # Sleep 3*account.gc to ensure that all resources are deleted + wait_for_cleanup(self.apiclient, ["account.cleanup.interval"]*3) + + self.VerifyVpcCleanup(vpc.id) + self.VerifyNetworkCleanup(persistent_network_1.id) + self.VerifyNetworkCleanup(persistent_network_2.id) + self.VerifyVirtualMachineCleanup(virtual_machine_1.id) + self.VerifyVirtualMachineCleanup(virtual_machine_2.id) + self.VerifyVirtualMachineCleanup(virtual_machine_3.id) + self.VerifyVirtualMachineCleanup(virtual_machine_4.id) + self.VerifyAclRuleCleanup(ingressAclNetwork1.id) + self.VerifyAclRuleCleanup(egressAclNetwork1.id) + self.VerifyAclRuleCleanup(ingressAclNetwork2.id) + self.VerifyAclRuleCleanup(egressAclNetwork2.id) + return + + @unittest.skip("WIP") + @attr(tags=["advanced"]) + def test_vpc_private_gateway_static_route(self): + # steps + # 1. Create account and create VPC network in the account + # 2. Create two persistent networks within this VPC + # 3. Restart/delete VPC network + + # Validations + # 1. In case of Restart operation, restart should be successful and persistent networks should be back in persistent state + # 2. In case of Delete operation, VR servicing the VPC should get destroyed and sourceNAT ip should get released + + # Create Account + account = Account.create(self.apiclient,self.services["account"],domainid=self.domain.id) + self.cleanup.append(account) + + # Create VPC + self.services["vpc"]["cidr"] = "10.1.1.1/16" + vpc = VPC.create(self.apiclient, self.services["vpc"], + vpcofferingid=self.vpc_off.id, zoneid=self.zone.id, + account=account.name, domainid=account.domainid) + vpcs = VPC.list(self.apiclient, id=vpc.id) + self.assertEqual(validateList(vpcs)[0], PASS, "VPC list validation failed, vpc list is %s" % vpcs) + + # Create Persistent Networks as tiers of VPC + persistent_network_1 = Network.create(self.api_client,self.services["isolated_network"], + networkofferingid=self.persistent_network_offering_NoLB.id, + accountid=account.name,domainid=account.domainid, + zoneid=self.zone.id, vpcid=vpc.id, gateway="10.1.1.1", netmask="255.255.255.0") + verifyNetworkState(self.apiclient, persistent_network_1.id, "implemented") + self.assertIsNotNone(persistent_network_1.vlan, "vlan must not be null for persistent network %s" % persistent_network_1.id) + + persistent_network_2 = Network.create(self.api_client,self.services["isolated_network"], + networkofferingid=self.persistent_network_offering_LB.id, + accountid=account.name,domainid=account.domainid, + zoneid=self.zone.id, vpcid=vpc.id, gateway="10.1.2.1", netmask="255.255.255.0") + verifyNetworkState(self.apiclient, persistent_network_2.id, "implemented") + self.assertIsNotNone(persistent_network_2.vlan, "vlan must not be null for persistent network: %s" % persistent_network_2.id) + + # Deploy VMs in above networks (VM1, VM2 in network1 and VM3, VM4 in network2) + try: + virtual_machine_1 = VirtualMachine.create(self.apiclient,self.services["virtual_machine"], + networkids=[persistent_network_1.id],serviceofferingid=self.service_offering.id, + accountid=account.name,domainid=self.domain.id) + + virtual_machine_2 = VirtualMachine.create(self.apiclient,self.services["virtual_machine"], + networkids=[persistent_network_1.id],serviceofferingid=self.service_offering.id, + accountid=account.name,domainid=self.domain.id) + + virtual_machine_3 = VirtualMachine.create(self.apiclient,self.services["virtual_machine"], + networkids=[persistent_network_2.id],serviceofferingid=self.service_offering.id, + accountid=account.name,domainid=self.domain.id) + + virtual_machine_4 = VirtualMachine.create(self.apiclient,self.services["virtual_machine"], + networkids=[persistent_network_2.id],serviceofferingid=self.service_offering.id, + accountid=account.name,domainid=self.domain.id) + except Exception as e: + self.fail("vm creation failed: %s" % e) + + # Associate IP addresses to persistent networks + ipaddress_1 = self.GetAssociatedIpForNetwork(persistent_network_1.id, vpcid=vpc.id, account=account) + ipaddress_2 = self.GetAssociatedIpForNetwork(persistent_network_1.id, vpcid=vpc.id, account=account) + ipaddress_3 = self.GetAssociatedIpForNetwork(persistent_network_2.id, vpcid=vpc.id, account=account) + + # Create NAT rule for VM 1 + NATRule.create(self.api_client, virtual_machine_1, + self.services["natrule"],ipaddressid=ipaddress_1.ipaddress.id, + networkid=persistent_network_1.id) + + # Create Static NAT rule for VM 2 + StaticNATRule.enable(self.apiclient, ipaddressid=ipaddress_2.ipaddress.id, + virtualmachineid=virtual_machine_2.id, + networkid=persistent_network_1.id) + + # Create load balancer rule for ipaddress3 and assign to VM3 and VM4 + lb_rule = LoadBalancerRule.create(self.apiclient,self.services["lbrule"], + ipaddressid=ipaddress_3.ipaddress.id, accountid=account.name, + networkid=persistent_network_2.id, domainid=account.domainid) + lb_rule.assign(self.api_client, [virtual_machine_3, virtual_machine_4]) + + # Create network ACL for both ther persistent networks (tiers of VPC) + ingressAclNetwork1, egressAclNetwork1 = self.CreateIngressEgressNetworkACLForNetwork(persistent_network_1.id) + ingressAclNetwork2, egressAclNetwork2 = self.CreateIngressEgressNetworkACLForNetwork(persistent_network_2.id) + + """private_gateway = PrivateGateway.create(self.apiclient,gateway='10.1.4.1',ipaddress='10.1.4.100', + netmask='255.255.255.0',vlan=679,vpcid=vpc.id) + + gateways = PrivateGateway.list(self.apiclient,id=private_gateway.id, listall=True) + self.assertEqual(validateList(gateways)[0], PASS, "gateways list validation failed, list is %s" % gateways) + + static_route = StaticRoute.create(self.apiclient, cidr='11.1.1.1/24',gatewayid=private_gateway.id) + static_routes = StaticRoute.list(self.apiclient,id=static_route.id,listall=True) + self.assertEqual(validateList(static_routes)[0], PASS, "static routes list validation failed, list is %s" % static_routes)""" + + self.CheckIngressEgressConnectivityofVM(virtual_machine_1, ipaddress_1.ipaddress.ipaddress) + self.CheckIngressEgressConnectivityofVM(virtual_machine_2, ipaddress_2.ipaddress.ipaddress) + self.CheckIngressEgressConnectivityofVM(virtual_machine_3, ipaddress_3.ipaddress.ipaddress) + """self.CheckIngressEgressConnectivityofVM(virtual_machine_4, ipaddress_3.ipaddress.ipaddress)""" + + vpc.restart(self.apiclient) + + self.CheckIngressEgressConnectivityofVM(virtual_machine_1, ipaddress_1.ipaddress.ipaddress) + self.CheckIngressEgressConnectivityofVM(virtual_machine_2, ipaddress_2.ipaddress.ipaddress) + self.CheckIngressEgressConnectivityofVM(virtual_machine_3, ipaddress_3.ipaddress.ipaddress) + """self.CheckIngressEgressConnectivityofVM(virtual_machine_4, ipaddress_3.ipaddress.ipaddress)""" + return diff --git a/tools/marvin/marvin/config/config.cfg b/tools/marvin/marvin/config/config.cfg index 5849fe88afd..ec41c10ccd7 100644 --- a/tools/marvin/marvin/config/config.cfg +++ b/tools/marvin/marvin/config/config.cfg @@ -146,6 +146,239 @@ "ostype": "CentOS 5.3 (64-bit)", "sleep": 90, "timeout": 10, + "netscaler_VPX": { + "ipaddress": "10.223.240.174", + "username": "nsroot", + "password": "nsroot", + "networkdevicetype": "NetscalerVPXLoadBalancer", + "publicinterface": "1/1", + "privateinterface": "1/2", + "numretries": 2, + "lbdevicededicated": "True", + "lbdevicecapacity": 2, + "port": 22 + }, + "nw_offering_shared_persistent": { + "name": "Network offering for Shared Persistent Network", + "displaytext": "Network offering-DA services", + "guestiptype": "Shared", + "supportedservices": "Dhcp,Dns,SourceNat,PortForwarding,Vpn,Firewall,Lb,UserData,StaticNat", + "traffictype": "GUEST", + "availability": "Optional", + "ispersistent": "True", + "serviceProviderList": { + "Dhcp": "VirtualRouter", + "Dns": "VirtualRouter", + "SourceNat": "VirtualRouter", + "PortForwarding": "VirtualRouter", + "Vpn": "VirtualRouter", + "Firewall": "VirtualRouter", + "Lb": "VirtualRouter", + "UserData": "VirtualRouter", + "StaticNat": "VirtualRouter" + } + }, + "nw_off_isolated_persistent": { + "name": "Network offering for Isolated Persistent Network", + "displaytext": "Network Offering for Isolated Persistent Network", + "guestiptype": "Isolated", + "supportedservices": "Dhcp,Dns,SourceNat,PortForwarding,Vpn,Firewall,Lb,UserData,StaticNat", + "traffictype":"GUEST", + "availability":"Optional", + "ispersistent": "True", + "serviceProviderList": { + "Dhcp": "VirtualRouter", + "Dns": "VirtualRouter", + "SourceNat": "VirtualRouter", + "PortForwarding": "VirtualRouter", + "Vpn": "VirtualRouter", + "Firewall": "VirtualRouter", + "Lb": "VirtualRouter", + "UserData": "VirtualRouter", + "StaticNat": "VirtualRouter" + } + }, + "nw_off_isolated_persistent_netscaler": { + "name": "Network offering for Isolated Persistent Network", + "displaytext": "Network Offering for Isolated Persistent Network", + "guestiptype": "Isolated", + "supportedservices": "Dhcp,Dns,SourceNat,PortForwarding,Vpn,Firewall,Lb,UserData,StaticNat", + "traffictype": "GUEST", + "availability": "Optional", + "ispersistent": "True", + "serviceProviderList": { + "Dhcp": "VirtualRouter", + "Dns": "VirtualRouter", + "SourceNat": "VirtualRouter", + "PortForwarding": "VirtualRouter", + "Vpn": "VirtualRouter", + "Firewall": "VirtualRouter", + "Lb": "Netscaler", + "UserData": "VirtualRouter", + "StaticNat": "VirtualRouter" + } + }, + "nw_offering_isolated": { + "name": "Network offering for Isolated Persistent Network", + "displaytext": "Network offering-DA services", + "guestiptype": "Isolated", + "supportedservices": "Dhcp,Dns,SourceNat,PortForwarding,Vpn,Firewall,Lb,UserData,StaticNat", + "traffictype": "GUEST", + "availability": "Optional", + "serviceProviderList": { + "Dhcp": "VirtualRouter", + "Dns": "VirtualRouter", + "SourceNat": "VirtualRouter", + "PortForwarding": "VirtualRouter", + "Vpn": "VirtualRouter", + "Firewall": "VirtualRouter", + "Lb": "VirtualRouter", + "UserData": "VirtualRouter", + "StaticNat": "VirtualRouter" + } + }, + "nw_off_isolated_netscaler": { + "name": "Network offering for Isolated Persistent Network", + "displaytext": "Network offering-DA services", + "guestiptype": "Isolated", + "supportedservices": "Dhcp,Dns,SourceNat,PortForwarding,Vpn,Firewall,Lb,UserData,StaticNat", + "traffictype": "GUEST", + "availability": "Optional", + "serviceProviderList": { + "Dhcp": "VirtualRouter", + "Dns": "VirtualRouter", + "SourceNat": "VirtualRouter", + "PortForwarding": "VirtualRouter", + "Vpn": "VirtualRouter", + "Firewall": "VirtualRouter", + "Lb": "Netscaler", + "UserData": "VirtualRouter", + "StaticNat": "VirtualRouter" + } + }, + "nw_off_persistent_RVR": { + "name": "Network offering-RVR services", + "displaytext": "Network off-RVR services", + "guestiptype": "Isolated", + "supportedservices": "Vpn,Dhcp,Dns,SourceNat,PortForwarding,Firewall,Lb,UserData,StaticNat", + "traffictype": "GUEST", + "availability": "Optional", + "ispersistent": "True", + "serviceProviderList": { + "Vpn": "VirtualRouter", + "Dhcp": "VirtualRouter", + "Dns": "VirtualRouter", + "SourceNat": "VirtualRouter", + "PortForwarding": "VirtualRouter", + "Firewall": "VirtualRouter", + "Lb": "VirtualRouter", + "UserData": "VirtualRouter", + "StaticNat": "VirtualRouter" + }, + "serviceCapabilityList": { + "SourceNat": { + "SupportedSourceNatTypes": "peraccount", + "RedundantRouter": "true" + }, + "lb": { + "SupportedLbIsolation": "dedicated" + } + } + }, + "nw_off_persistent_VPCVR_NoLB": { + "name": "Persistent Network VPC No LB", + "displaytext": "Persistent Network VPC No LB", + "guestiptype": "Isolated", + "supportedservices": "Dhcp,Dns,SourceNat,PortForwarding,Vpn,UserData,StaticNat,NetworkACL", + "traffictype": "GUEST", + "availability": "Optional", + "ispersistent": "False", + "useVpc": "on", + "serviceProviderList": { + "Dhcp": "VpcVirtualRouter", + "Dns": "VpcVirtualRouter", + "SourceNat": "VpcVirtualRouter", + "PortForwarding": "VpcVirtualRouter", + "Vpn": "VpcVirtualRouter", + "UserData": "VpcVirtualRouter", + "StaticNat": "VpcVirtualRouter", + "NetworkACL": "VpcVirtualRouter" + } + }, + "nw_off_persistent_VPCVR_LB": { + "name": "Persistent Network VPC with LB", + "displaytext": "Persistent Network VPC No LB", + "guestiptype": "Isolated", + "supportedservices": "Dhcp,Dns,SourceNat,PortForwarding,Vpn,Lb,UserData,StaticNat,NetworkACL", + "traffictype": "GUEST", + "availability": "Optional", + "ispersistent": "False", + "useVpc": "on", + "serviceProviderList": { + "Dhcp": "VpcVirtualRouter", + "Dns": "VpcVirtualRouter", + "SourceNat": "VpcVirtualRouter", + "PortForwarding": "VpcVirtualRouter", + "Vpn": "VpcVirtualRouter", + "Lb": "VpcVirtualRouter", + "UserData": "VpcVirtualRouter", + "StaticNat": "VpcVirtualRouter", + "NetworkACL": "VpcVirtualRouter" + } + }, + "nw_offering_isolated_vpc": { + "name": "Isolated Network for VPC", + "displaytext": "Isolated Network for VPC", + "guestiptype": "Isolated", + "supportedservices": "Dhcp,Dns,SourceNat,PortForwarding,Vpn,UserData,StaticNat,NetworkACL", + "traffictype": "GUEST", + "availability": "Optional", + "ispersistent": "False", + "useVpc": "on", + "serviceProviderList": { + "Dhcp": "VpcVirtualRouter", + "Dns": "VpcVirtualRouter", + "SourceNat": "VpcVirtualRouter", + "PortForwarding": "VpcVirtualRouter", + "Vpn": "VpcVirtualRouter", + "UserData": "VpcVirtualRouter", + "StaticNat": "VpcVirtualRouter", + "NetworkACL": "VpcVirtualRouter" + } + }, + "natrule": { + "privateport": 22, + "publicport": 22, + "startport": 22, + "endport": 22, + "protocol": "TCP", + "cidrlist": "0.0.0.0/0" + }, + "fwrule": { + "startport": 22, + "endport": 22, + "cidr": "0.0.0.0/0", + "protocol": "TCP" + }, + "lbrule": { + "name": "SSH", + "alg": "leastconn", + # Algorithm used for load balancing + "privateport": 22, + "publicport": 22, + "openfirewall": "False", + "startport": 22, + "endport": 22, + "protocol": "TCP", + "cidrlist": "0.0.0.0/0" + }, + "icmprule": { + "icmptype": -1, + "icmpcode": -1, + "cidrlist": "0.0.0.0/0", + "protocol": "ICMP" + }, + "host_password": "password", "advanced_sg": { "zone": { "name": "", diff --git a/tools/marvin/marvin/integration/lib/base.py b/tools/marvin/marvin/integration/lib/base.py index f76f07e24cb..d25b6f7a7a8 100755 --- a/tools/marvin/marvin/integration/lib/base.py +++ b/tools/marvin/marvin/integration/lib/base.py @@ -131,6 +131,13 @@ class Account: [setattr(cmd, k, v) for k, v in kwargs.items()] return(apiclient.listAccounts(cmd)) + def disable(self, apiclient, lock=False): + """Disable an account""" + cmd = disableAccount.disableAccountCmd() + cmd.id = self.id + cmd.lock = lock + apiclient.disableAccount(cmd) + class User: """ User Life Cycle """ diff --git a/tools/marvin/marvin/integration/lib/common.py b/tools/marvin/marvin/integration/lib/common.py index 5b56c733563..b2da3ffef24 100644 --- a/tools/marvin/marvin/integration/lib/common.py +++ b/tools/marvin/marvin/integration/lib/common.py @@ -64,7 +64,8 @@ from marvin.integration.lib.base import (Configurations, PhysicalNetwork, Host, PublicIPAddress, - NetworkOffering) + NetworkOffering, + Network) from marvin.integration.lib.utils import (get_process_status, xsplit, validateList) @@ -969,3 +970,13 @@ def shouldTestBeSkipped(networkType, zoneType): and (zoneType.lower() == BASIC_ZONE)): skipIt = True return skipIt + +def verifyNetworkState(apiclient, networkid, state): + """List networks and check if the network state matches the given state""" + try: + networks = Network.list(apiclient, id=networkid) + except Exception as e: + raise Exception("Failed while fetching network list with error: %s" % e) + assert validateList(networks)[0] == PASS, "Networks list validation failed, list is %s" % networks + assert str(networks[0].state).lower() == state, "network state should be %s, it is %s" % (state, networks[0].state) + return From c1806bd873fe661756aeee94b25339c25f9724a5 Mon Sep 17 00:00:00 2001 From: "Frank.Zhang" Date: Fri, 21 Feb 2014 13:22:55 -0800 Subject: [PATCH 114/169] CLOUDSTACK-6155 Baremetal agent RPM missing in spec file --- setup/bindir/cloud-setup-baremetal.in | 232 ++++++++++++++++++++++++++ 1 file changed, 232 insertions(+) create mode 100644 setup/bindir/cloud-setup-baremetal.in diff --git a/setup/bindir/cloud-setup-baremetal.in b/setup/bindir/cloud-setup-baremetal.in new file mode 100644 index 00000000000..367e38943e2 --- /dev/null +++ b/setup/bindir/cloud-setup-baremetal.in @@ -0,0 +1,232 @@ +#!/usr/bin/python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import sys, os +from subprocess import PIPE, Popen +import logging +import traceback +from os.path import exists, join +from signal import alarm, signal, SIGALRM, SIGKILL + +class CloudRuntimeException(Exception): + def __init__(self, errMsg): + self.errMsg = errMsg + def __str__(self): + return self.errMsg +def formatExceptionInfo(maxTBlevel=5): + cla, exc, trbk = sys.exc_info() + excTb = traceback.format_tb(trbk, maxTBlevel) + msg = str(exc) + "\n" + for tb in excTb: + msg += tb + return msg + +class bash: + def __init__(self, args, timeout=600): + self.args = args + logging.debug("execute:%s"%args) + self.timeout = timeout + self.process = None + self.success = False + self.run() + + def run(self): + class Alarm(Exception): + pass + def alarm_handler(signum, frame): + raise Alarm + + try: + self.process = Popen(self.args, shell=True, stdout=PIPE, stderr=PIPE) + if self.timeout != -1: + signal(SIGALRM, alarm_handler) + alarm(self.timeout) + + try: + self.stdout, self.stderr = self.process.communicate() + if self.timeout != -1: + alarm(0) + except Alarm: + os.kill(self.process.pid, SIGKILL) + raise CloudRuntimeException("Timeout during command execution") + + self.success = self.process.returncode == 0 + except: + raise CloudRuntimeException(formatExceptionInfo()) + +# if not self.success: +# raise CloudRuntimeException(self.getStderr()) + + def isSuccess(self): + return self.success + + def getStdout(self): + return self.stdout.strip("\n") + + def getLines(self): + return self.stdout.split("\n") + + def getStderr(self): + return self.stderr.strip("\n") + + +def initLoging(logFile=None): + try: + if logFile is None: + logging.basicConfig(level=logging.DEBUG) + else: + logging.basicConfig(filename=logFile, level=logging.DEBUG) + except: + logging.basicConfig(level=logging.DEBUG) + +def writeProgressBar(msg, result=None): + if msg is not None: + output = "%-80s"%msg + elif result is True: + output = "[ \033[92m%-2s\033[0m ]\n"%"OK" + elif result is False: + output = "[ \033[91m%-6s\033[0m ]\n"%"FAILED" + sys.stdout.write(output) + sys.stdout.flush() + +def printError(msg): + sys.stderr.write(msg) + sys.stderr.write("\n") + sys.stderr.flush() + +def printMsg(msg): + sys.stdout.write(msg+"\n") + sys.stdout.flush() + +def checkRpm(pkgName): + chkPkg = bash("rpm -q %s"%pkgName) + writeProgressBar("Checking %s"%pkgName, None) + if not chkPkg.isSuccess(): + writeProgressBar(None, False) + printError("%s is not found, please make sure it is installed. You may try 'yum install %s'\n"%(pkgName, pkgName)) + return False + else: + writeProgressBar(None, True) + return True + +def checkEnv(): + writeProgressBar("Checking is root") + ret = bash("whoami") + if ret.getStdout() != "root": + writeProgressBar(None, False) + printError("This script must run as root") + return False + else: + writeProgressBar(None, True) + + pkgList = ['tftp-server', 'syslinux', 'xinetd', 'chkconfig', 'dhcp'] + for pkg in pkgList: + if not checkRpm(pkg): + return False + return True + +def exitIfFail(ret): + if not ret: sys.exit(1) + +def bashWithResult(cmd): + writeProgressBar("Executing '%s'"%cmd) + ret = bash(cmd) + if not ret.isSuccess(): + writeProgressBar(None, False) + writeProgressBar(ret.getStderr() + '\n') + return False + else: + writeProgressBar(None, True) + return True + +def configurePxeStuff(): + stuff = ['tftp', 'xinetd', 'dhcpd'] + cmds = ['chkconfig --level 345 %s on' % i for i in stuff] + cmds.append('/etc/init.d/xinetd restart') + + for cmd in cmds: + if not bashWithResult(cmd): return False + + chkIptable = bash('chkconfig --list iptables') + if 'on' in chkIptable.getStdout(): + printMsg("Detected iptables is running, need to open tftp port 69") + if not bashWithResult('iptables -I INPUT 1 -p udp --dport 69 -j ACCEPT'): return False + if not bashWithResult('/etc/init.d/iptables save'): return False + + return True + +def getTftpRootDir(tftpRootDirList): + tftpRoot = bash("cat /etc/xinetd.d/tftp | grep server_args") + if not tftpRoot.isSuccess(): + printError("Cannot get tftp root directory from /etc/xinetd.d/tftp, here may be something wrong with your tftp-server, try reinstall it\n") + return False + tftpRootDir = tftpRoot.getStdout() + index = tftpRootDir.find("/") + if index == -1: + printError("Wrong server_arg in /etc/xinetd.d/tftp (%s)"%tftpRootDir) + return False + tftpRootDir = tftpRootDir[index:] + tftpRootDirList.append(tftpRootDir) + return True + +def preparePING(tftpRootDir): + pingFiles = ['boot.msg', 'initrd.gz', 'kernel', 'pxelinux.0'] + pingDir = "/usr/share/PING" + + for f in pingFiles: + path = join(pingDir, f) + if not exists(path): + printError("Cannot find %s, please make sure PING-3.01 is installed"%path) + return False + if not bashWithResult("cp -f %s %s"%(path, tftpRootDir)): return False + + if not bashWithResult("mkdir -p %s/pxelinux.cfg"%tftpRootDir): return False + + return True + +def prepareSyslinux(tftpRootDir): + pkg = bash('rpm -ql syslinux | grep "/pxelinux.0$"') + if not pkg.isSuccess(): + printError('cannot find pxelinux.0 on system. is syslinux installed?') + return False + + pkg = pkg.getStdout() + cp = "cp -f %s %s" % (pkg, tftpRootDir) + if not bashWithResult(cp): + return False + + return True + + + +if __name__ == "__main__": + initLoging("/tmp/cloud-setup-baremetal.log") + tftpRootDirList = [] + + exitIfFail(checkEnv()) + exitIfFail(configurePxeStuff()) + exitIfFail(getTftpRootDir(tftpRootDirList)) + + tftpRootDir = tftpRootDirList[0].strip() + #exitIfFail(preparePING(tftpRootDir)) + exitIfFail(prepareSyslinux(tftpRootDir)) + printMsg("") + printMsg("Setup BareMetal PXE server successfully") + printMsg("TFTP root directory is: %s\n"%tftpRootDir) + sys.exit(0) + From d1440217e9a44b6c12b588cbdca52daac6335d0c Mon Sep 17 00:00:00 2001 From: Jessica Wang Date: Fri, 21 Feb 2014 14:01:52 -0800 Subject: [PATCH 115/169] CLOUDSTACK-6154: UI > compute offerings > memory field > check if value is undefined before converting it. --- ui/scripts/configuration.js | 6 ++++++ ui/scripts/sharedFunctions.js | 3 +++ 2 files changed, 9 insertions(+) diff --git a/ui/scripts/configuration.js b/ui/scripts/configuration.js index 420cb7b2de4..e52f51998fd 100644 --- a/ui/scripts/configuration.js +++ b/ui/scripts/configuration.js @@ -522,6 +522,9 @@ memory: { label: 'label.memory.mb', converter: function(args) { + if (args == undefined) + return ''; + else return cloudStack.converters.convertBytes(args * 1024 * 1024); } }, @@ -1038,6 +1041,9 @@ memory: { label: 'label.memory.mb', converter: function(args) { + if (args == undefined) + return ''; + else return cloudStack.converters.convertBytes(args * 1024 * 1024); } }, diff --git a/ui/scripts/sharedFunctions.js b/ui/scripts/sharedFunctions.js index 2a159675f31..9fa70e3bda4 100644 --- a/ui/scripts/sharedFunctions.js +++ b/ui/scripts/sharedFunctions.js @@ -900,6 +900,9 @@ var roleTypeDomainAdmin = "2"; cloudStack.converters = { convertBytes: function(bytes) { + if (bytes == undefined) + return ''; + if (bytes < 1024 * 1024) { return (bytes / 1024).toFixed(2) + " KB"; } else if (bytes < 1024 * 1024 * 1024) { From b44901c059e50ae34f55a56809cda74d4c0c08f9 Mon Sep 17 00:00:00 2001 From: Rajesh Battala Date: Fri, 21 Feb 2014 15:47:26 +0530 Subject: [PATCH 116/169] CLOUDSTACK-6159 Fixed creating volumes(Root, DataDisk) with uuid in primary store --- .../HypervResource/CloudStackTypes.cs | 16 ++++++++-------- .../HypervResource/HypervResourceController.cs | 10 ++++++---- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/CloudStackTypes.cs b/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/CloudStackTypes.cs index 96379944d9f..541c2053172 100644 --- a/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/CloudStackTypes.cs +++ b/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/CloudStackTypes.cs @@ -173,11 +173,11 @@ namespace HypervResource PrimaryDataStoreTO store = this.primaryDataStore; if (store.isLocal) { - fileName = Path.Combine(store.Path, this.name); + fileName = Path.Combine(store.Path, this.uuid); } else { - fileName = @"\\" + store.uri.Host + store.uri.LocalPath + @"\" + this.name; + fileName = @"\\" + store.uri.Host + store.uri.LocalPath + @"\" + this.uuid; fileName = Utils.NormalizePath(fileName); } } @@ -198,7 +198,7 @@ namespace HypervResource fileName = Utils.NormalizePath(fileName); if (Directory.Exists(fileName)) { - fileName = Utils.NormalizePath(fileName + @"\" + this.name); + fileName = Utils.NormalizePath(fileName + @"\" + this.uuid); } } } @@ -300,11 +300,11 @@ namespace HypervResource path = Utils.NormalizePath(path); if (Directory.Exists(path)) { - string[] choices = choices = Directory.GetFiles(path, volInfo.name + ".vhd*"); + string[] choices = choices = Directory.GetFiles(path, volInfo.uuid + ".vhd*"); if (choices.Length != 1) { String errMsg = "Tried to guess file extension, but cannot find file corresponding to " + - Path.Combine(volInfo.primaryDataStore.Path, volInfo.name); + Path.Combine(volInfo.primaryDataStore.Path, volInfo.uuid); logger.Debug(errMsg); } else @@ -334,18 +334,18 @@ namespace HypervResource PrimaryDataStoreTO store = this.primaryDataStore; if (store.isLocal) { - fileName = Path.Combine(store.Path, this.name); + fileName = Path.Combine(store.Path, this.uuid); } else { - fileName = @"\\" + store.uri.Host + store.uri.LocalPath + @"\" + this.name; + fileName = @"\\" + store.uri.Host + store.uri.LocalPath + @"\" + this.uuid; } fileName = fileName + '.' + this.format.ToLowerInvariant(); } else if (this.nfsDataStoreTO != null) { NFSTO store = this.nfsDataStoreTO; - fileName = store.UncPath + @"\" + this.path + @"\" + this.name; + fileName = store.UncPath + @"\" + this.path + @"\" + this.uuid; if (!this.format.Equals("RAW")) { fileName = fileName + '.' + this.format.ToLowerInvariant(); diff --git a/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/HypervResourceController.cs b/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/HypervResourceController.cs index a1326064694..8fd15f4c258 100644 --- a/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/HypervResourceController.cs +++ b/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/HypervResourceController.cs @@ -1150,13 +1150,13 @@ namespace HypervResource bool result = false; string details = null; - + object newData = null; try { VolumeObjectTO volume = VolumeObjectTO.ParseJson(cmd.data); PrimaryDataStoreTO primary = volume.primaryDataStore; ulong volumeSize = volume.size; - string volumeName = volume.name + ".vhd"; + string volumeName = volume.uuid + ".vhd"; string volumePath = null; if (primary.isLocal) @@ -1169,11 +1169,13 @@ namespace HypervResource volumePath = Utils.NormalizePath(volumePath); Utils.ConnectToRemote(primary.UncPath, primary.Domain, primary.User, primary.Password); } - + volume.path = volumePath; wmiCallsV2.CreateDynamicVirtualHardDisk(volumeSize, volumePath); if (File.Exists(volumePath)) { result = true; + JObject ansObj = Utils.CreateCloudStackObject(CloudStackTypes.VolumeObjectTO, volume); + newData = ansObj; } else { @@ -1191,7 +1193,7 @@ namespace HypervResource { result = result, details = details, - data = cmd.data, + data = newData, contextMap = contextMap }; From 20efa0c8a6b83c6590eab48fefac81b72b2082a9 Mon Sep 17 00:00:00 2001 From: "Frank.Zhang" Date: Mon, 24 Feb 2014 13:30:24 -0800 Subject: [PATCH 117/169] CLOUDSTACK-6155 Baremetal agent RPM missing in spec file --- packaging/centos63/cloud.spec | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/packaging/centos63/cloud.spec b/packaging/centos63/cloud.spec index 8029b61cc31..b363a2ea5fd 100644 --- a/packaging/centos63/cloud.spec +++ b/packaging/centos63/cloud.spec @@ -136,6 +136,18 @@ Group: System Environment/Libraries %description agent The CloudStack agent for KVM hypervisors +%package baremetal-agent +Summary: CloudStack baremetal agent +Requires: tftp-server +Requires: xinetd +Requires: syslinux +Requires: chkconfig +Requires: dhcp +Requires: httpd +Group: System Environment/Libraries +%description baremetal-agent +The CloudStack baremetal agent + %package usage Summary: CloudStack Usage calculation server Requires: java >= 1.6.0 @@ -244,6 +256,7 @@ install -D client/target/utilities/bin/cloud-set-guest-sshkey ${RPM_BUILD_ROOT}% install -D client/target/utilities/bin/cloud-setup-databases ${RPM_BUILD_ROOT}%{_bindir}/%{name}-setup-databases install -D client/target/utilities/bin/cloud-setup-encryption ${RPM_BUILD_ROOT}%{_bindir}/%{name}-setup-encryption install -D client/target/utilities/bin/cloud-setup-management ${RPM_BUILD_ROOT}%{_bindir}/%{name}-setup-management +install -D client/target/utilities/bin/cloud-setup-baremetal ${RPM_BUILD_ROOT}%{_bindir}/%{name}-setup-baremetal install -D client/target/utilities/bin/cloud-sysvmadm ${RPM_BUILD_ROOT}%{_bindir}/%{name}-sysvmadm install -D client/target/utilities/bin/cloud-update-xenserver-licenses ${RPM_BUILD_ROOT}%{_bindir}/%{name}-update-xenserver-licenses @@ -633,6 +646,9 @@ fi %{_defaultdocdir}/%{name}-awsapi-%{version}/LICENSE %{_defaultdocdir}/%{name}-awsapi-%{version}/NOTICE +%files baremetal-agent +%attr(0755,root,root) %{_bindir}/cloudstack-setup-baremetal + %changelog * Fri Oct 03 2012 Hugo Trippaers 4.1.0 From c652ff45dffaafa1cb0de29103e90fe936382028 Mon Sep 17 00:00:00 2001 From: Likitha Shetty Date: Mon, 24 Feb 2014 10:12:58 +0530 Subject: [PATCH 118/169] CLOUDSTACK-6146. [VMware] [ESXi 5.5] Live storage migration of an already migrated volume fails In vCenter 5.5, once a volume is migrated the VMDKs are renamed to match the name of the VM. If a volume has been renamed upon migration update its volumePath to that of the new disk filename. --- .../vmware/resource/VmwareResource.java | 17 +++++++++++++---- .../hypervisor/vmware/mo/VirtualMachineMO.java | 18 ++++++++++++------ 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java index d1fea92c0bd..d27d81f61f3 100755 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -4517,12 +4517,12 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa throw new Exception(msg); } - VmwareStorageLayoutHelper.getVmwareDatastorePathFromVmdkFileName( - new DatastoreMO(srcHyperHost.getContext(), morDs), vmName, - volumePath + ".vmdk"); + DatastoreMO targetDsMo = new DatastoreMO(srcHyperHost.getContext(), morDs); + String fullVolumePath = VmwareStorageLayoutHelper.getVmwareDatastorePathFromVmdkFileName(targetDsMo, vmName, volumePath + ".vmdk"); + int diskId = getVirtualDiskInfo(vmMo, volumePath + ".vmdk"); diskLocator = new VirtualMachineRelocateSpecDiskLocator(); diskLocator.setDatastore(morDs); - diskLocator.setDiskId(getVirtualDiskInfo(vmMo, volumePath + ".vmdk")); + diskLocator.setDiskId(diskId); diskLocators.add(diskLocator); relocateSpec.getDisk().add(diskLocator); @@ -4534,6 +4534,15 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa s_logger.debug("Successfully migrated volume " + volumePath + " to target datastore " + tgtDsName); } + // Update and return volume path because that could have changed after migration + if (!targetDsMo.fileExists(fullVolumePath)) { + VirtualDisk[] disks = vmMo.getAllDiskDevice(); + for (VirtualDisk disk : disks) + if (disk.getKey() == diskId) { + volumePath = vmMo.getVmdkFileBaseName(disk); + } + } + return new MigrateVolumeAnswer(cmd, true, null, volumePath); } catch (Exception e) { String msg = "Catch Exception " + e.getClass().getName() + " due to " + e.toString(); diff --git a/vmware-base/src/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java b/vmware-base/src/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java index e4e2712ec23..69a7e50f439 100644 --- a/vmware-base/src/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java +++ b/vmware-base/src/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java @@ -1783,17 +1783,23 @@ public class VirtualMachineMO extends BaseMO { VirtualDevice[] devices = getAllDiskDevice(); for(VirtualDevice device : devices) { if(device instanceof VirtualDisk) { - VirtualDeviceBackingInfo backingInfo = ((VirtualDisk)device).getBacking(); - if(backingInfo instanceof VirtualDiskFlatVer2BackingInfo) { - VirtualDiskFlatVer2BackingInfo diskBackingInfo = (VirtualDiskFlatVer2BackingInfo)backingInfo; - DatastoreFile dsBackingFile = new DatastoreFile(diskBackingInfo.getFileName()); - vmdkFileBaseNames.add(dsBackingFile.getFileBaseName()); - } + vmdkFileBaseNames.add(getVmdkFileBaseName((VirtualDisk)device)); } } return vmdkFileBaseNames; } + public String getVmdkFileBaseName(VirtualDisk disk) throws Exception { + String vmdkFileBaseName = null; + VirtualDeviceBackingInfo backingInfo = disk.getBacking(); + if(backingInfo instanceof VirtualDiskFlatVer2BackingInfo) { + VirtualDiskFlatVer2BackingInfo diskBackingInfo = (VirtualDiskFlatVer2BackingInfo)backingInfo; + DatastoreFile dsBackingFile = new DatastoreFile(diskBackingInfo.getFileName()); + vmdkFileBaseName = dsBackingFile.getFileBaseName(); + } + return vmdkFileBaseName; + } + // this method relies on un-offical VMware API @Deprecated public void moveAllVmDiskFiles(DatastoreMO destDsMo, String destDsDir, boolean followDiskChain) throws Exception { From 8cb03ddb2387c7aebf20dc0aa011e41f227e9f68 Mon Sep 17 00:00:00 2001 From: Likitha Shetty Date: Tue, 25 Feb 2014 18:10:52 +0530 Subject: [PATCH 119/169] CLOUDSTACK-6146. [VMware] [ESXi 5.5] Live VM migration of an already migrated VM (with storage) across clusters fails In vCenter 5.5, once a volume is migrated the VMDKs are renamed to match the name of the VM. Update volume path for every volume belonging to the VM to the corresponding new disk filename. --- .../vmware/resource/VmwareResource.java | 25 +++++++++++-- .../motion/VmwareStorageMotionStrategy.java | 36 ++++++++++++------- 2 files changed, 45 insertions(+), 16 deletions(-) diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java index d27d81f61f3..9f5cf045866 100755 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -4338,6 +4338,8 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa VolumeTO volume; StorageFilerTO filerTo; Set mountedDatastoresAtSource = new HashSet(); + List volumeToList = new ArrayList(); + Map volumeDeviceKey = new HashMap(); Map volToFiler = cmd.getVolumeToFiler(); String tgtHost = cmd.getTargetHost(); @@ -4402,9 +4404,11 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa volume.getPath() + ".vmdk"); diskLocator = new VirtualMachineRelocateSpecDiskLocator(); diskLocator.setDatastore(morDsAtSource); - diskLocator.setDiskId(getVirtualDiskInfo(vmMo, volume.getPath() + ".vmdk")); + int diskId = getVirtualDiskInfo(vmMo, volume.getPath() + ".vmdk"); + diskLocator.setDiskId(diskId); diskLocators.add(diskLocator); + volumeDeviceKey.put(volume.getId(), diskId); } relocateSpec.getDisk().addAll(diskLocators); @@ -4436,6 +4440,22 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa s_logger.debug("Successfully migrated storage of VM " + vmName + " to target datastore(s)"); } + // Update and return volume path for every disk because that could have changed after migration + for (Entry entry : volToFiler.entrySet()) { + volume = entry.getKey(); + long volumeId = volume.getId(); + VirtualDisk[] disks = vmMo.getAllDiskDevice(); + for (VirtualDisk disk : disks) { + if (volumeDeviceKey.get(volumeId) == disk.getKey()) { + VolumeObjectTO newVol = new VolumeObjectTO(); + newVol.setId(volumeId); + newVol.setPath(vmMo.getVmdkFileBaseName(disk)); + volumeToList.add(newVol); + break; + } + } + } + // Change host ManagedObjectReference morPool = tgtHyperHost.getHyperHostOwnerResourcePool(); if (!vmMo.migrate(morPool, tgtHyperHost.getMor())) { @@ -4445,7 +4465,6 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } state = State.Stopping; - List volumeToList = null; return new MigrateWithStorageAnswer(cmd, volumeToList); } catch (Throwable e) { if (e instanceof RemoteException) { @@ -4552,7 +4571,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } private int getVirtualDiskInfo(VirtualMachineMO vmMo, String srcDiskName) throws Exception { - Pair deviceInfo = vmMo.getDiskDevice(srcDiskName, false); + Pair deviceInfo = vmMo.getDiskDevice(srcDiskName, true); if(deviceInfo == null) { throw new Exception("No such disk device: " + srcDiskName); } diff --git a/plugins/hypervisors/vmware/src/org/apache/cloudstack/storage/motion/VmwareStorageMotionStrategy.java b/plugins/hypervisors/vmware/src/org/apache/cloudstack/storage/motion/VmwareStorageMotionStrategy.java index 446093f99a4..3c4d704138b 100644 --- a/plugins/hypervisors/vmware/src/org/apache/cloudstack/storage/motion/VmwareStorageMotionStrategy.java +++ b/plugins/hypervisors/vmware/src/org/apache/cloudstack/storage/motion/VmwareStorageMotionStrategy.java @@ -20,6 +20,7 @@ package org.apache.cloudstack.storage.motion; import java.util.HashMap; +import java.util.List; import java.util.Map; import javax.inject.Inject; @@ -33,6 +34,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; import org.apache.cloudstack.framework.async.AsyncCompletionCallback; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.to.VolumeObjectTO; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -143,7 +145,7 @@ public class VmwareStorageMotionStrategy implements DataMotionStrategy { ". " + migrateWithStorageAnswer.getDetails()); } else { // Update the volume details after migration. - updateVolumesAfterMigration(volumeToPool); + updateVolumesAfterMigration(volumeToPool, migrateWithStorageAnswer.getVolumeTos()); } s_logger.debug("Storage migration of VM " + vm.getInstanceName() + " completed successfully. Migrated to host " + destHost.getName()); @@ -178,7 +180,7 @@ public class VmwareStorageMotionStrategy implements DataMotionStrategy { ". " + answer.getDetails()); } else { // Update the volume details after migration. - updateVolumesAfterMigration(volumeToPool); + updateVolumesAfterMigration(volumeToPool, answer.getVolumeTos()); } return answer; @@ -188,20 +190,28 @@ public class VmwareStorageMotionStrategy implements DataMotionStrategy { } } - private void updateVolumesAfterMigration(Map volumeToPool) { + private void updateVolumesAfterMigration(Map volumeToPool, List volumeTos) { for (Map.Entry entry : volumeToPool.entrySet()) { + boolean updated = false; VolumeInfo volume = entry.getKey(); StoragePool pool = (StoragePool)entry.getValue(); - - VolumeVO volumeVO = volDao.findById(volume.getId()); - Long oldPoolId = volumeVO.getPoolId(); - volumeVO.setLastPoolId(oldPoolId); - volumeVO.setFolder(pool.getPath()); - volumeVO.setPodId(pool.getPodId()); - volumeVO.setPoolId(pool.getId()); - - volDao.update(volume.getId(), volumeVO); - s_logger.debug("Volume path was successfully updated for volume " + volume.getName() + " after it was migrated."); + for (VolumeObjectTO volumeTo : volumeTos) { + if (volume.getId() == volumeTo.getId()) { + VolumeVO volumeVO = volDao.findById(volume.getId()); + Long oldPoolId = volumeVO.getPoolId(); + volumeVO.setPath(volumeTo.getPath()); + volumeVO.setLastPoolId(oldPoolId); + volumeVO.setFolder(pool.getPath()); + volumeVO.setPodId(pool.getPodId()); + volumeVO.setPoolId(pool.getId()); + volDao.update(volume.getId(), volumeVO); + updated = true; + break; + } + } + if (!updated) { + s_logger.error("Volume path wasn't updated for volume " + volume + " after it was migrated."); + } } } } From f5aeac845df8b73b1525442461a8a811e61439d0 Mon Sep 17 00:00:00 2001 From: John Kinsella Date: Wed, 26 Feb 2014 09:04:56 -0800 Subject: [PATCH 120/169] Fix the developer database simulator deploy procedure so it includes the mysql database driver --- developer/pom.xml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/developer/pom.xml b/developer/pom.xml index a0b60c68c39..cf89fd6dbe8 100644 --- a/developer/pom.xml +++ b/developer/pom.xml @@ -182,6 +182,12 @@ org.codehaus.mojo exec-maven-plugin + + + mysql + mysql-connector-java + ${cs.mysql.version} + 1.2.1 @@ -194,6 +200,7 @@ com.cloud.upgrade.DatabaseCreator + true ${basedir}/../utils/conf/db.properties From 54db315214f8dd518eaf1f1615d423d9f4129d31 Mon Sep 17 00:00:00 2001 From: John Kinsella Date: Wed, 26 Feb 2014 09:33:58 -0800 Subject: [PATCH 121/169] Last commit broke developer/pom.xml. fixing that. --- developer/pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/developer/pom.xml b/developer/pom.xml index cf89fd6dbe8..7ed6335fe16 100644 --- a/developer/pom.xml +++ b/developer/pom.xml @@ -188,6 +188,7 @@ mysql-connector-java ${cs.mysql.version} + 1.2.1 From 4c83f80c809674cb8ff9561d546a6ade25d983f8 Mon Sep 17 00:00:00 2001 From: Min Chen Date: Wed, 26 Feb 2014 11:24:01 -0800 Subject: [PATCH 122/169] CLOUDSTACK-6174: Provide separate option for Windows Server 2012 R2 as an OS type when registering template. --- .../hypervisor/xen/resource/CitrixHelper.java | 345 +++++++++++++++++- .../xen/resource/Xenserver625Resource.java | 15 +- setup/db/db/schema-421to430.sql | 4 + setup/db/templates.sql | 3 + .../vmware/util/VmwareGuestOsMapper.java | 5 +- 5 files changed, 358 insertions(+), 14 deletions(-) diff --git a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixHelper.java b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixHelper.java index ed890efffa9..c7e39382272 100644 --- a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixHelper.java +++ b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixHelper.java @@ -11,16 +11,16 @@ // Unless required by applicable law or agreed to in writing, // software distributed under the License is distributed on an // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the +// KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. package com.cloud.hypervisor.xen.resource; -import org.apache.log4j.Logger; - import java.util.ArrayList; import java.util.HashMap; +import org.apache.log4j.Logger; + /** * Reduce bloat inside CitrixResourceBase * @@ -31,12 +31,14 @@ public class CitrixHelper { private static final HashMap _xcp160GuestOsMap = new HashMap(70); private static final HashMap _xenServerGuestOsMap = new HashMap(70); private static final HashMap _xenServer56FP1GuestOsMap = new HashMap(70); + private static final HashMap _xenServer602GuestOsMap = new HashMap(70); private static final HashMap _xenServer56FP2GuestOsMap = new HashMap(70); private static final HashMap _xenServer600GuestOsMap = new HashMap(70); - private static final HashMap _xenServer602GuestOsMap = new HashMap(70); private static final HashMap _xenServer610GuestOsMap = new HashMap(70); private static final HashMap _xenServer620GuestOsMap = new HashMap(70); + private static final HashMap _xenServer625GuestOsMap = new HashMap(70); private static final HashMap _xenServer620GuestOsMemoryMap = new HashMap(70); + private static final HashMap _xenServer625GuestOsMemoryMap = new HashMap(70); private static final HashMap _xenServer610GuestOsMemoryMap = new HashMap(70); private static final HashMap _xenServer602GuestOsMemoryMap = new HashMap(70); private static final HashMap _xenServer600GuestOsMemoryMap = new HashMap(70); @@ -1532,6 +1534,168 @@ public class CitrixHelper { _xenServer620GuestOsMap.put("Other PV (64-bit)", "CentOS 5 (64-bit)"); } + static { + _xenServer625GuestOsMap.put("CentOS 4.5 (32-bit)", "CentOS 4.5 (32-bit)"); + _xenServer625GuestOsMap.put("CentOS 4.6 (32-bit)", "CentOS 4.6 (32-bit)"); + _xenServer625GuestOsMap.put("CentOS 4.7 (32-bit)", "CentOS 4.7 (32-bit)"); + _xenServer625GuestOsMap.put("CentOS 4.8 (32-bit)", "CentOS 4.8 (32-bit)"); + _xenServer625GuestOsMap.put("CentOS 5.0 (32-bit)", "CentOS 5 (32-bit)"); + _xenServer625GuestOsMap.put("CentOS 5.0 (64-bit)", "CentOS 5 (64-bit)"); + _xenServer625GuestOsMap.put("CentOS 5.1 (32-bit)", "CentOS 5 (32-bit)"); + _xenServer625GuestOsMap.put("CentOS 5.1 (64-bit)", "CentOS 5 (64-bit)"); + _xenServer625GuestOsMap.put("CentOS 5.2 (32-bit)", "CentOS 5 (32-bit)"); + _xenServer625GuestOsMap.put("CentOS 5.2 (64-bit)", "CentOS 5 (64-bit)"); + _xenServer625GuestOsMap.put("CentOS 5.3 (32-bit)", "CentOS 5 (32-bit)"); + _xenServer625GuestOsMap.put("CentOS 5.3 (64-bit)", "CentOS 5 (64-bit)"); + _xenServer625GuestOsMap.put("CentOS 5.4 (32-bit)", "CentOS 5 (32-bit)"); + _xenServer625GuestOsMap.put("CentOS 5.4 (64-bit)", "CentOS 5 (64-bit)"); + _xenServer625GuestOsMap.put("CentOS 5.5 (32-bit)", "CentOS 5 (32-bit)"); + _xenServer625GuestOsMap.put("CentOS 5.5 (64-bit)", "CentOS 5 (64-bit)"); + _xenServer625GuestOsMap.put("CentOS 5.6 (32-bit)", "CentOS 5 (32-bit)"); + _xenServer625GuestOsMap.put("CentOS 5.6 (64-bit)", "CentOS 5 (64-bit)"); + _xenServer625GuestOsMap.put("CentOS 5.7 (32-bit)", "CentOS 5 (32-bit)"); + _xenServer625GuestOsMap.put("CentOS 5.7 (64-bit)", "CentOS 5 (64-bit)"); + _xenServer625GuestOsMap.put("CentOS 5.8 (32-bit)", "CentOS 5 (32-bit)"); + _xenServer625GuestOsMap.put("CentOS 5.8 (64-bit)", "CentOS 5 (64-bit)"); + _xenServer625GuestOsMap.put("CentOS 5.9 (32-bit)", "CentOS 5 (32-bit)"); + _xenServer625GuestOsMap.put("CentOS 5.9 (64-bit)", "CentOS 5 (64-bit)"); + _xenServer625GuestOsMap.put("CentOS 6.0 (32-bit)", "CentOS 6 (32-bit)"); + _xenServer625GuestOsMap.put("CentOS 6.0 (64-bit)", "CentOS 6 (64-bit)"); + _xenServer625GuestOsMap.put("CentOS 6.1 (32-bit)", "CentOS 6 (32-bit)"); + _xenServer625GuestOsMap.put("CentOS 6.1 (64-bit)", "CentOS 6 (64-bit)"); + _xenServer625GuestOsMap.put("CentOS 6.2 (32-bit)", "CentOS 6 (32-bit)"); + _xenServer625GuestOsMap.put("CentOS 6.2 (64-bit)", "CentOS 6 (64-bit)"); + _xenServer625GuestOsMap.put("CentOS 6.3 (32-bit)", "CentOS 6 (32-bit)"); + _xenServer625GuestOsMap.put("CentOS 6.3 (64-bit)", "CentOS 6 (64-bit)"); + _xenServer625GuestOsMap.put("CentOS 6.4 (32-bit)", "CentOS 6 (32-bit)"); + _xenServer625GuestOsMap.put("CentOS 6.4 (64-bit)", "CentOS 6 (64-bit)"); + _xenServer625GuestOsMap.put("Debian GNU/Linux 6(32-bit)", "Debian Squeeze 6.0 (32-bit)"); + _xenServer625GuestOsMap.put("Debian GNU/Linux 6(64-bit)", "Debian Squeeze 6.0 (64-bit)"); + _xenServer625GuestOsMap.put("Debian GNU/Linux 7(32-bit)", "Debian Wheezy 7.0 (32-bit)"); + _xenServer625GuestOsMap.put("Debian GNU/Linux 7(64-bit)", "Debian Wheezy 7.0 (64-bit)"); + _xenServer625GuestOsMap.put("Oracle Enterprise Linux 5.0 (32-bit)", "Oracle Enterprise Linux 5 (32-bit)"); + _xenServer625GuestOsMap.put("Oracle Enterprise Linux 5.0 (64-bit)", "Oracle Enterprise Linux 5 (64-bit)"); + _xenServer625GuestOsMap.put("Oracle Enterprise Linux 5.1 (32-bit)", "Oracle Enterprise Linux 5 (32-bit)"); + _xenServer625GuestOsMap.put("Oracle Enterprise Linux 5.1 (64-bit)", "Oracle Enterprise Linux 5 (64-bit)"); + _xenServer625GuestOsMap.put("Oracle Enterprise Linux 5.2 (32-bit)", "Oracle Enterprise Linux 5 (32-bit)"); + _xenServer625GuestOsMap.put("Oracle Enterprise Linux 5.2 (64-bit)", "Oracle Enterprise Linux 5 (64-bit)"); + _xenServer625GuestOsMap.put("Oracle Enterprise Linux 5.3 (32-bit)", "Oracle Enterprise Linux 5 (32-bit)"); + _xenServer625GuestOsMap.put("Oracle Enterprise Linux 5.3 (64-bit)", "Oracle Enterprise Linux 5 (64-bit)"); + _xenServer625GuestOsMap.put("Oracle Enterprise Linux 5.4 (32-bit)", "Oracle Enterprise Linux 5 (32-bit)"); + _xenServer625GuestOsMap.put("Oracle Enterprise Linux 5.4 (64-bit)", "Oracle Enterprise Linux 5 (64-bit)"); + _xenServer625GuestOsMap.put("Oracle Enterprise Linux 5.5 (32-bit)", "Oracle Enterprise Linux 5 (32-bit)"); + _xenServer625GuestOsMap.put("Oracle Enterprise Linux 5.5 (64-bit)", "Oracle Enterprise Linux 5 (64-bit)"); + _xenServer625GuestOsMap.put("Oracle Enterprise Linux 5.6 (32-bit)", "Oracle Enterprise Linux 5 (32-bit)"); + _xenServer625GuestOsMap.put("Oracle Enterprise Linux 5.6 (64-bit)", "Oracle Enterprise Linux 5 (64-bit)"); + _xenServer625GuestOsMap.put("Oracle Enterprise Linux 5.7 (32-bit)", "Oracle Enterprise Linux 5 (32-bit)"); + _xenServer625GuestOsMap.put("Oracle Enterprise Linux 5.7 (64-bit)", "Oracle Enterprise Linux 5 (64-bit)"); + _xenServer625GuestOsMap.put("Oracle Enterprise Linux 5.8 (32-bit)", "Oracle Enterprise Linux 5 (32-bit)"); + _xenServer625GuestOsMap.put("Oracle Enterprise Linux 5.8 (64-bit)", "Oracle Enterprise Linux 5 (64-bit)"); + _xenServer625GuestOsMap.put("Oracle Enterprise Linux 5.9 (32-bit)", "Oracle Enterprise Linux 5 (32-bit)"); + _xenServer625GuestOsMap.put("Oracle Enterprise Linux 5.9 (64-bit)", "Oracle Enterprise Linux 5 (64-bit)"); + _xenServer625GuestOsMap.put("Oracle Enterprise Linux 6.0 (32-bit)", "Oracle Enterprise Linux 6 (32-bit)"); + _xenServer625GuestOsMap.put("Oracle Enterprise Linux 6.0 (64-bit)", "Oracle Enterprise Linux 6 (64-bit)"); + _xenServer625GuestOsMap.put("Oracle Enterprise Linux 6.1 (32-bit)", "Oracle Enterprise Linux 6 (32-bit)"); + _xenServer625GuestOsMap.put("Oracle Enterprise Linux 6.1 (64-bit)", "Oracle Enterprise Linux 6 (64-bit)"); + _xenServer625GuestOsMap.put("Oracle Enterprise Linux 6.2 (32-bit)", "Oracle Enterprise Linux 6 (32-bit)"); + _xenServer625GuestOsMap.put("Oracle Enterprise Linux 6.2 (64-bit)", "Oracle Enterprise Linux 6 (64-bit)"); + _xenServer625GuestOsMap.put("Oracle Enterprise Linux 6.3 (32-bit)", "Oracle Enterprise Linux 6 (32-bit)"); + _xenServer625GuestOsMap.put("Oracle Enterprise Linux 6.3 (64-bit)", "Oracle Enterprise Linux 6 (64-bit)"); + _xenServer625GuestOsMap.put("Oracle Enterprise Linux 6.4 (32-bit)", "Oracle Enterprise Linux 6 (32-bit)"); + _xenServer625GuestOsMap.put("Oracle Enterprise Linux 6.4 (64-bit)", "Oracle Enterprise Linux 6 (64-bit)"); + _xenServer625GuestOsMap.put("Red Hat Enterprise Linux 4.5 (32-bit)", "Red Hat Enterprise Linux 4.5 (32-bit)"); + _xenServer625GuestOsMap.put("Red Hat Enterprise Linux 4.6 (32-bit)", "Red Hat Enterprise Linux 4.6 (32-bit)"); + _xenServer625GuestOsMap.put("Red Hat Enterprise Linux 4.7 (32-bit)", "Red Hat Enterprise Linux 4.7 (32-bit)"); + _xenServer625GuestOsMap.put("Red Hat Enterprise Linux 4.8 (32-bit)", "Red Hat Enterprise Linux 4.8 (32-bit)"); + _xenServer625GuestOsMap.put("Red Hat Enterprise Linux 5.0 (32-bit)", "Red Hat Enterprise Linux 5 (32-bit)"); + _xenServer625GuestOsMap.put("Red Hat Enterprise Linux 5.0 (64-bit)", "Red Hat Enterprise Linux 5 (64-bit)"); + _xenServer625GuestOsMap.put("Red Hat Enterprise Linux 5.1 (32-bit)", "Red Hat Enterprise Linux 5 (32-bit)"); + _xenServer625GuestOsMap.put("Red Hat Enterprise Linux 5.1 (64-bit)", "Red Hat Enterprise Linux 5 (64-bit)"); + _xenServer625GuestOsMap.put("Red Hat Enterprise Linux 5.2 (32-bit)", "Red Hat Enterprise Linux 5 (32-bit)"); + _xenServer625GuestOsMap.put("Red Hat Enterprise Linux 5.2 (64-bit)", "Red Hat Enterprise Linux 5 (64-bit)"); + _xenServer625GuestOsMap.put("Red Hat Enterprise Linux 5.3 (32-bit)", "Red Hat Enterprise Linux 5 (32-bit)"); + _xenServer625GuestOsMap.put("Red Hat Enterprise Linux 5.3 (64-bit)", "Red Hat Enterprise Linux 5 (64-bit)"); + _xenServer625GuestOsMap.put("Red Hat Enterprise Linux 5.4 (32-bit)", "Red Hat Enterprise Linux 5 (32-bit)"); + _xenServer625GuestOsMap.put("Red Hat Enterprise Linux 5.4 (64-bit)", "Red Hat Enterprise Linux 5 (64-bit)"); + _xenServer625GuestOsMap.put("Red Hat Enterprise Linux 5.5 (32-bit)", "Red Hat Enterprise Linux 5 (32-bit)"); + _xenServer625GuestOsMap.put("Red Hat Enterprise Linux 5.5 (64-bit)", "Red Hat Enterprise Linux 5 (64-bit)"); + _xenServer625GuestOsMap.put("Red Hat Enterprise Linux 5.6 (32-bit)", "Red Hat Enterprise Linux 5 (32-bit)"); + _xenServer625GuestOsMap.put("Red Hat Enterprise Linux 5.6 (64-bit)", "Red Hat Enterprise Linux 5 (64-bit)"); + _xenServer625GuestOsMap.put("Red Hat Enterprise Linux 5.7 (32-bit)", "Red Hat Enterprise Linux 5 (32-bit)"); + _xenServer625GuestOsMap.put("Red Hat Enterprise Linux 5.7 (64-bit)", "Red Hat Enterprise Linux 5 (64-bit)"); + _xenServer625GuestOsMap.put("Red Hat Enterprise Linux 5.8 (32-bit)", "Red Hat Enterprise Linux 5 (32-bit)"); + _xenServer625GuestOsMap.put("Red Hat Enterprise Linux 5.8 (64-bit)", "Red Hat Enterprise Linux 5 (64-bit)"); + _xenServer625GuestOsMap.put("Red Hat Enterprise Linux 5.9 (32-bit)", "Red Hat Enterprise Linux 5 (32-bit)"); + _xenServer625GuestOsMap.put("Red Hat Enterprise Linux 5.9 (64-bit)", "Red Hat Enterprise Linux 5 (64-bit)"); + _xenServer625GuestOsMap.put("Red Hat Enterprise Linux 6.0 (32-bit)", "Red Hat Enterprise Linux 6 (32-bit)"); + _xenServer625GuestOsMap.put("Red Hat Enterprise Linux 6.0 (64-bit)", "Red Hat Enterprise Linux 6 (64-bit)"); + _xenServer625GuestOsMap.put("Red Hat Enterprise Linux 6.1 (32-bit)", "Red Hat Enterprise Linux 6 (32-bit)"); + _xenServer625GuestOsMap.put("Red Hat Enterprise Linux 6.1 (64-bit)", "Red Hat Enterprise Linux 6 (64-bit)"); + _xenServer625GuestOsMap.put("Red Hat Enterprise Linux 6.2 (32-bit)", "Red Hat Enterprise Linux 6 (32-bit)"); + _xenServer625GuestOsMap.put("Red Hat Enterprise Linux 6.2 (64-bit)", "Red Hat Enterprise Linux 6 (64-bit)"); + _xenServer625GuestOsMap.put("Red Hat Enterprise Linux 6.3 (32-bit)", "Red Hat Enterprise Linux 6 (32-bit)"); + _xenServer625GuestOsMap.put("Red Hat Enterprise Linux 6.3 (64-bit)", "Red Hat Enterprise Linux 6 (64-bit)"); + _xenServer625GuestOsMap.put("Red Hat Enterprise Linux 6.4 (32-bit)", "Red Hat Enterprise Linux 6 (32-bit)"); + _xenServer625GuestOsMap.put("Red Hat Enterprise Linux 6.4 (64-bit)", "Red Hat Enterprise Linux 6 (64-bit)"); + _xenServer625GuestOsMap.put("SUSE Linux Enterprise Server 10 SP1 (32-bit)", "SUSE Linux Enterprise Server 10 SP1 (32-bit)"); + _xenServer625GuestOsMap.put("SUSE Linux Enterprise Server 10 SP1 (64-bit)", "SUSE Linux Enterprise Server 10 SP1 (64-bit)"); + _xenServer625GuestOsMap.put("SUSE Linux Enterprise Server 10 SP2 (32-bit)", "SUSE Linux Enterprise Server 10 SP2 (32-bit)"); + _xenServer625GuestOsMap.put("SUSE Linux Enterprise Server 10 SP2 (64-bit)", "SUSE Linux Enterprise Server 10 SP2 (64-bit)"); + _xenServer625GuestOsMap.put("SUSE Linux Enterprise Server 10 SP3 (32-bit)", "SUSE Linux Enterprise Server 10 SP3 (32-bit)"); + _xenServer625GuestOsMap.put("SUSE Linux Enterprise Server 10 SP3 (64-bit)", "SUSE Linux Enterprise Server 10 SP3 (64-bit)"); + _xenServer625GuestOsMap.put("SUSE Linux Enterprise Server 10 SP4 (32-bit)", "SUSE Linux Enterprise Server 10 SP4 (32-bit)"); + _xenServer625GuestOsMap.put("SUSE Linux Enterprise Server 10 SP4 (64-bit)", "SUSE Linux Enterprise Server 10 SP4 (64-bit)"); + _xenServer625GuestOsMap.put("SUSE Linux Enterprise Server 11 (32-bit)", "SUSE Linux Enterprise Server 11 (32-bit)"); + _xenServer625GuestOsMap.put("SUSE Linux Enterprise Server 11 (64-bit)", "SUSE Linux Enterprise Server 11 (64-bit)"); + _xenServer625GuestOsMap.put("SUSE Linux Enterprise Server 11 SP1 (32-bit)", "SUSE Linux Enterprise Server 11 SP1 (32-bit)"); + _xenServer625GuestOsMap.put("SUSE Linux Enterprise Server 11 SP1 (64-bit)", "SUSE Linux Enterprise Server 11 SP1 (64-bit)"); + _xenServer625GuestOsMap.put("SUSE Linux Enterprise Server 11 SP2 (32-bit)", "SUSE Linux Enterprise Server 11 SP2 (32-bit)"); + _xenServer625GuestOsMap.put("SUSE Linux Enterprise Server 11 SP2 (64-bit)", "SUSE Linux Enterprise Server 11 SP2 (64-bit)"); + _xenServer625GuestOsMap.put("Windows 7 (32-bit)", "Windows 7 (32-bit)"); + _xenServer625GuestOsMap.put("Windows 7 (64-bit)", "Windows 7 (64-bit)"); + _xenServer625GuestOsMap.put("Windows 7 SP1 (32-bit)", "Windows 7 (32-bit)"); + _xenServer625GuestOsMap.put("Windows 7 SP1 (64-bit)", "Windows 7 (64-bit)"); + _xenServer625GuestOsMap.put("Windows 8 (32-bit)", "Windows 8 (32-bit)"); + _xenServer625GuestOsMap.put("Windows 8 (64-bit)", "Windows 8 (64-bit)"); + _xenServer625GuestOsMap.put("Windows Server 2003 SP2 (32-bit)", "Windows Server 2003 (32-bit)"); + _xenServer625GuestOsMap.put("Windows Server 2003 SP2 (64-bit)", "Windows Server 2003 (64-bit)"); + _xenServer625GuestOsMap.put("Windows Server 2003 PAE (32-bit)", "Windows Server 2003 (32-bit)"); + _xenServer625GuestOsMap.put("Windows Server 2003 Enterprise Edition(32-bit)", "Windows Server 2003 (32-bit)"); + _xenServer625GuestOsMap.put("Windows Server 2003 Enterprise Edition(64-bit)", "Windows Server 2003 (64-bit)"); + _xenServer625GuestOsMap.put("Windows Server 2003 DataCenter Edition(32-bit)", "Windows Server 2003 (32-bit)"); + _xenServer625GuestOsMap.put("Windows Server 2003 DataCenter Edition(64-bit)", "Windows Server 2003 (64-bit)"); + _xenServer625GuestOsMap.put("Windows Server 2003 Standard Edition(32-bit)", "Windows Server 2003 (32-bit)"); + _xenServer625GuestOsMap.put("Windows Server 2003 Standard Edition(64-bit)", "Windows Server 2003 (64-bit)"); + _xenServer625GuestOsMap.put("Windows Server 2008 SP2 (32-bit)", "Windows Server 2008 (34-bit)"); + _xenServer625GuestOsMap.put("Windows Server 2008 SP2 (64-bit)", "Windows Server 2008 (64-bit)"); + _xenServer625GuestOsMap.put("Windows Server 2008 R2 (64-bit)", "Windows Server 2008 R2 (64-bit)"); + _xenServer625GuestOsMap.put("Windows Server 2008 R2 SP1 (64-bit)", "Windows Server 2008 R2 (64-bit)"); + _xenServer625GuestOsMap.put("Windows Server 2012 (64-bit)", "Windows Server 2012 (64-bit)"); + _xenServer625GuestOsMap.put("Windows Server 2012 R2 (64-bit)", "Windows Server 2012 (64-bit)"); + _xenServer625GuestOsMap.put("Windows Vista SP2 (32-bit)", "Windows Vista (32-bit)"); + _xenServer625GuestOsMap.put("Windows XP SP3 (32-bit)", "Windows XP SP3 (32-bit)"); + _xenServer625GuestOsMap.put("Ubuntu 10.04 (32-bit)", "Ubuntu Lucid Lynx 10.04 (32-bit)"); + _xenServer625GuestOsMap.put("Ubuntu 10.04 (64-bit)", "Ubuntu Lucid Lynx 10.04 (64-bit)"); + _xenServer625GuestOsMap.put("Ubuntu 10.10 (32-bit)", "Ubuntu Maverick Meerkat 10.10 (32-bit) (experimental)"); + _xenServer625GuestOsMap.put("Ubuntu 10.10 (64-bit)", "Ubuntu Maverick Meerkat 10.10 (64-bit) (experimental)"); + _xenServer625GuestOsMap.put("Ubuntu 12.04 (32-bit)", "Ubuntu Precise Pangolin 12.04 (32-bit)"); + _xenServer625GuestOsMap.put("Ubuntu 12.04 (64-bit)", "Ubuntu Precise Pangolin 12.04 (64-bit)"); + _xenServer625GuestOsMap.put("Ubuntu 11.04 (32-bit)", "Other install media"); + _xenServer625GuestOsMap.put("Ubuntu 11.04 (64-bit)", "Other install media"); + _xenServer625GuestOsMap.put("Other Linux (32-bit)", "Other install media"); + _xenServer625GuestOsMap.put("Other Linux (64-bit)", "Other install media"); + _xenServer625GuestOsMap.put("Other (32-bit)", "Other install media"); + _xenServer625GuestOsMap.put("Other (64-bit)", "Other install media"); + _xenServer625GuestOsMap.put("Other CentOS (32-bit)", "Other install media"); + _xenServer625GuestOsMap.put("Other CentOS (64-bit)", "Other install media"); + _xenServer625GuestOsMap.put("Other Ubuntu (32-bit)", "Other install media"); + _xenServer625GuestOsMap.put("Other Ubuntu (64-bit)", "Other install media"); + _xenServer625GuestOsMap.put("Other SUSE Linux(32-bit)", "Other install media"); + _xenServer625GuestOsMap.put("Other SUSE Linux(64-bit)", "Other install media"); + _xenServer625GuestOsMap.put("Other PV (32-bit)", "CentOS 5 (32-bit)"); + _xenServer625GuestOsMap.put("Other PV (64-bit)", "CentOS 5 (64-bit)"); + } + public static class MemoryValues { long max; long min; @@ -1823,6 +1987,147 @@ public class CitrixHelper { _xenServer620GuestOsMemoryMap.put("Ubuntu 12.04 (64-bit)", new MemoryValues(128l, 128*1024l)); } + static { + _xenServer625GuestOsMemoryMap.put("CentOS 4.5 (32-bit)", new MemoryValues(256l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("CentOS 4.6 (32-bit)", new MemoryValues(256l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("CentOS 4.7 (32-bit)", new MemoryValues(256l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("CentOS 4.8 (32-bit)", new MemoryValues(256l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("CentOS 5.0 (32-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("CentOS 5.0 (64-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("CentOS 5.1 (32-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("CentOS 5.1 (64-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("CentOS 5.2 (32-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("CentOS 5.2 (64-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("CentOS 5.3 (32-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("CentOS 5.3 (64-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("CentOS 5.4 (32-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("CentOS 5.4 (64-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("CentOS 5.5 (32-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("CentOS 5.5 (64-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("CentOS 5.6 (32-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("CentOS 5.6 (64-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("CentOS 5.7 (32-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("CentOS 5.7 (64-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("CentOS 5.8 (32-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("CentOS 5.8 (64-bit)", new MemoryValues(512l, 128 * 1024l)); + _xenServer625GuestOsMemoryMap.put("CentOS 5.9 (32-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("CentOS 5.9 (64-bit)", new MemoryValues(512l, 128 * 1024l)); + _xenServer625GuestOsMemoryMap.put("CentOS 6.0 (32-bit)", new MemoryValues(512l, 8 * 1024l)); + _xenServer625GuestOsMemoryMap.put("CentOS 6.0 (64-bit)", new MemoryValues(512l, 32 * 1024l)); + _xenServer625GuestOsMemoryMap.put("CentOS 6.1 (32-bit)", new MemoryValues(512l, 8 * 1024l)); + _xenServer625GuestOsMemoryMap.put("CentOS 6.1 (64-bit)", new MemoryValues(512l, 32 * 1024l)); + _xenServer625GuestOsMemoryMap.put("CentOS 6.2 (32-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("CentOS 6.2 (64-bit)", new MemoryValues(512l, 128 * 1024l)); + _xenServer625GuestOsMemoryMap.put("CentOS 6.3 (32-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("CentOS 6.3 (64-bit)", new MemoryValues(512l, 128 * 1024l)); + _xenServer625GuestOsMemoryMap.put("CentOS 6.4 (32-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("CentOS 6.4 (64-bit)", new MemoryValues(512l, 128 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Debian GNU/Linux 6(32-bit)", new MemoryValues(128l, 32 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Debian GNU/Linux 6(64-bit)", new MemoryValues(128l, 32 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Debian GNU/Linux 7(32-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Debian GNU/Linux 7(64-bit)", new MemoryValues(512l, 128 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Oracle Enterprise Linux 5.0 (32-bit)", new MemoryValues(512l, 64 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Oracle Enterprise Linux 5.0 (64-bit)", new MemoryValues(512l, 128 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Oracle Enterprise Linux 5.1 (32-bit)", new MemoryValues(512l, 64 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Oracle Enterprise Linux 5.1 (64-bit)", new MemoryValues(512l, 128 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Oracle Enterprise Linux 5.2 (32-bit)", new MemoryValues(512l, 64 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Oracle Enterprise Linux 5.2 (64-bit)", new MemoryValues(512l, 128 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Oracle Enterprise Linux 5.3 (32-bit)", new MemoryValues(512l, 64 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Oracle Enterprise Linux 5.3 (64-bit)", new MemoryValues(512l, 128 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Oracle Enterprise Linux 5.4 (32-bit)", new MemoryValues(512l, 64 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Oracle Enterprise Linux 5.4 (64-bit)", new MemoryValues(512l, 128 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Oracle Enterprise Linux 5.5 (32-bit)", new MemoryValues(512l, 64 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Oracle Enterprise Linux 5.5 (64-bit)", new MemoryValues(512l, 128 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Oracle Enterprise Linux 5.6 (32-bit)", new MemoryValues(512l, 64 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Oracle Enterprise Linux 5.6 (64-bit)", new MemoryValues(512l, 128 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Oracle Enterprise Linux 5.7 (32-bit)", new MemoryValues(512l, 64 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Oracle Enterprise Linux 5.7 (64-bit)", new MemoryValues(512l, 128 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Oracle Enterprise Linux 5.8 (32-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Oracle Enterprise Linux 5.8 (64-bit)", new MemoryValues(512l, 128 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Oracle Enterprise Linux 5.9 (32-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Oracle Enterprise Linux 5.9 (64-bit)", new MemoryValues(512l, 128 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Oracle Enterprise Linux 6.0 (32-bit)", new MemoryValues(512l, 8 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Oracle Enterprise Linux 6.0 (64-bit)", new MemoryValues(512l, 32 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Oracle Enterprise Linux 6.1 (32-bit)", new MemoryValues(512l, 8 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Oracle Enterprise Linux 6.1 (64-bit)", new MemoryValues(512l, 32 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Oracle Enterprise Linux 6.2 (32-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Oracle Enterprise Linux 6.2 (64-bit)", new MemoryValues(512l, 128 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Oracle Enterprise Linux 6.3 (32-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Oracle Enterprise Linux 6.3 (64-bit)", new MemoryValues(512l, 128 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Oracle Enterprise Linux 6.4 (32-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Oracle Enterprise Linux 6.4 (64-bit)", new MemoryValues(512l, 128 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Red Hat Enterprise Linux 4.5 (32-bit)", new MemoryValues(256l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Red Hat Enterprise Linux 4.6 (32-bit)", new MemoryValues(256l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Red Hat Enterprise Linux 4.7 (32-bit)", new MemoryValues(256l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Red Hat Enterprise Linux 4.8 (32-bit)", new MemoryValues(256l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Red Hat Enterprise Linux 5.0 (32-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Red Hat Enterprise Linux 5.0 (64-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Red Hat Enterprise Linux 5.1 (32-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Red Hat Enterprise Linux 5.1 (64-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Red Hat Enterprise Linux 5.2 (32-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Red Hat Enterprise Linux 5.2 (64-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Red Hat Enterprise Linux 5.3 (32-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Red Hat Enterprise Linux 5.3 (64-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Red Hat Enterprise Linux 5.4 (32-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Red Hat Enterprise Linux 5.4 (64-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Red Hat Enterprise Linux 5.5 (32-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Red Hat Enterprise Linux 5.5 (64-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Red Hat Enterprise Linux 5.6 (32-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Red Hat Enterprise Linux 5.6 (64-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Red Hat Enterprise Linux 5.7 (32-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Red Hat Enterprise Linux 5.7 (64-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Red Hat Enterprise Linux 5.8 (32-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Red Hat Enterprise Linux 5.8 (64-bit)", new MemoryValues(512l, 128 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Red Hat Enterprise Linux 5.9 (32-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Red Hat Enterprise Linux 5.9 (64-bit)", new MemoryValues(512l, 128 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Red Hat Enterprise Linux 6.0 (32-bit)", new MemoryValues(512l, 8 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Red Hat Enterprise Linux 6.0 (64-bit)", new MemoryValues(512l, 32 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Red Hat Enterprise Linux 6.1 (32-bit)", new MemoryValues(512l, 8 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Red Hat Enterprise Linux 6.1 (64-bit)", new MemoryValues(512l, 32 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Red Hat Enterprise Linux 6.2 (32-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Red Hat Enterprise Linux 6.2 (64-bit)", new MemoryValues(512l, 128 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Red Hat Enterprise Linux 6.3 (32-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Red Hat Enterprise Linux 6.3 (64-bit)", new MemoryValues(512l, 128 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Red Hat Enterprise Linux 6.4 (32-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Red Hat Enterprise Linux 6.4 (64-bit)", new MemoryValues(512l, 128 * 1024l)); + _xenServer625GuestOsMemoryMap.put("SUSE Linux Enterprise Server 10 SP1 (32-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("SUSE Linux Enterprise Server 10 SP1 (64-bit)", new MemoryValues(512l, 128 * 1024l)); + _xenServer625GuestOsMemoryMap.put("SUSE Linux Enterprise Server 10 SP2 (32-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("SUSE Linux Enterprise Server 10 SP2 (64-bit)", new MemoryValues(512l, 128 * 1024l)); + _xenServer625GuestOsMemoryMap.put("SUSE Linux Enterprise Server 10 SP3 (32-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("SUSE Linux Enterprise Server 10 SP3 (64-bit)", new MemoryValues(512l, 128 * 1024l)); + _xenServer625GuestOsMemoryMap.put("SUSE Linux Enterprise Server 10 SP4 (32-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("SUSE Linux Enterprise Server 10 SP4 (64-bit)", new MemoryValues(512l, 128 * 1024l)); + _xenServer625GuestOsMemoryMap.put("SUSE Linux Enterprise Server 11 (32-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("SUSE Linux Enterprise Server 11 (64-bit)", new MemoryValues(512l, 128 * 1024l)); + _xenServer625GuestOsMemoryMap.put("SUSE Linux Enterprise Server 11 SP1 (32-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("SUSE Linux Enterprise Server 11 SP1 (64-bit)", new MemoryValues(512l, 128 * 1024l)); + _xenServer625GuestOsMemoryMap.put("SUSE Linux Enterprise Server 11 SP2 (32-bit)", new MemoryValues(512l, 16 * 1024l)); + _xenServer625GuestOsMemoryMap.put("SUSE Linux Enterprise Server 11 SP2 (64-bit)", new MemoryValues(512l, 128 * 1024l)); + + _xenServer625GuestOsMemoryMap.put("Windows 7 (32-bit)", new MemoryValues(1024l, 4 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Windows 7 (64-bit)", new MemoryValues(2 * 1024l, 128 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Windows 7 SP1 (32-bit)", new MemoryValues(1024l, 4 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Windows 7 SP1 (64-bit)", new MemoryValues(2 * 1024l, 128 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Windows 8 (32-bit)", new MemoryValues(1024l, 4 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Windows 8 (64-bit)", new MemoryValues(2 * 1024l, 128 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Windows Server 2003 SP2 (32-bit)", new MemoryValues(256l, 64 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Windows Server 2003 SP2 (64-bit)", new MemoryValues(256l, 128 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Windows Server 2008 SP2 (32-bit)", new MemoryValues(512l, 64 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Windows Server 2008 SP2 (64-bit)", new MemoryValues(512l, 128 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Windows Server 2008 R2 (64-bit)", new MemoryValues(512l, 128 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Windows Server 2008 R2 SP1 (64-bit)", new MemoryValues(512l, 128 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Windows Server 2012 (64-bit)", new MemoryValues(512l, 128 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Windows Server 2012 R2 (64-bit)", new MemoryValues(512l, 128 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Windows Vista SP2 (32-bit)", new MemoryValues(1024l, 4 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Windows XP SP3 (32-bit)", new MemoryValues(256l, 4 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Ubuntu 10.04 (32-bit)", new MemoryValues(128l, 512l)); + _xenServer625GuestOsMemoryMap.put("Ubuntu 10.04 (64-bit)", new MemoryValues(128l, 32 * 1024l)); + //_xenServer620GuestOsMemoryMap.put("Ubuntu 10.10 (32-bit)", new MemoryValues(512l, 16*1024l));//? + //_xenServer620GuestOsMemoryMap.put("Ubuntu 10.10 (64-bit)", new MemoryValues(512l, 16*1024l)); //? + _xenServer625GuestOsMemoryMap.put("Ubuntu 12.04 (32-bit)", new MemoryValues(128l, 32 * 1024l)); + _xenServer625GuestOsMemoryMap.put("Ubuntu 12.04 (64-bit)", new MemoryValues(128l, 128 * 1024l)); + } public static String getXcpGuestOsType(String stdType) { String guestOS = _xcp100GuestOsMap.get(stdType); @@ -1857,7 +2162,7 @@ public class CitrixHelper { return guestOS; } - public static String getXenServer56FP1GuestOsType(String stdType, boolean bootFromCD) { + public static String getXenServer56FP1GuestOsType(String stdType, boolean bootFromCD) { String guestOS = _xenServer56FP1GuestOsMap.get(stdType); if (guestOS == null) { if ( !bootFromCD ) { @@ -2023,6 +2328,20 @@ public class CitrixHelper { return guestOS; } + public static String getXenServer625GuestOsType(String stdType, boolean bootFromCD) { + String guestOS = _xenServer625GuestOsMap.get(stdType); + if (guestOS == null) { + if (!bootFromCD) { + s_logger.debug("Can't find the guest os: " + stdType + " mapping into XenServer 6.2.5 guestOS type, start it as HVM guest"); + guestOS = "Other install media"; + } else { + String msg = "XenServer 6.2.5 DOES NOT support Guest OS type " + stdType; + s_logger.warn(msg); + } + } + return guestOS; + } + public static long getXenServer620StaticMax(String stdType, boolean bootFromCD) { MemoryValues recommendedMaxMinMemory = _xenServer620GuestOsMemoryMap.get(stdType); if (recommendedMaxMinMemory == null) { @@ -2038,4 +2357,20 @@ public class CitrixHelper { } return recommendedMaxMinMemory.getMin(); } + + public static long getXenServer625StaticMax(String stdType, boolean bootFromCD) { + MemoryValues recommendedMaxMinMemory = _xenServer625GuestOsMemoryMap.get(stdType); + if (recommendedMaxMinMemory == null) { + return 0l; + } + return recommendedMaxMinMemory.getMax(); + } + + public static long getXenServer625StaticMin(String stdType, boolean bootFromCD) { + MemoryValues recommendedMaxMinMemory = _xenServer625GuestOsMemoryMap.get(stdType); + if (recommendedMaxMinMemory == null) { + return 0l; + } + return recommendedMaxMinMemory.getMin(); + } } \ No newline at end of file diff --git a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/Xenserver625Resource.java b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/Xenserver625Resource.java index 7d2a2a13ecb..a085e2de000 100644 --- a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/Xenserver625Resource.java +++ b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/Xenserver625Resource.java @@ -25,20 +25,22 @@ import java.util.Map; import javax.ejb.Local; -import org.apache.cloudstack.hypervisor.xenserver.XenServerResourceNewBase; import org.apache.log4j.Logger; +import com.xensource.xenapi.Connection; + +import org.apache.cloudstack.hypervisor.xenserver.XenServerResourceNewBase; + import com.cloud.agent.api.StartupRoutingCommand; import com.cloud.resource.ServerResource; import com.cloud.storage.resource.StorageSubsystemCommandHandler; import com.cloud.storage.resource.StorageSubsystemCommandHandlerBase; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.Script; -import com.xensource.xenapi.Connection; @Local(value=ServerResource.class) public class Xenserver625Resource extends XenServerResourceNewBase { - private static final Logger s_logger = Logger.getLogger(XenServer620Resource.class); + private static final Logger s_logger = Logger.getLogger(Xenserver625Resource.class); public Xenserver625Resource() { super(); @@ -53,7 +55,7 @@ public class Xenserver625Resource extends XenServerResourceNewBase { @Override protected String getGuestOsType(String stdType, boolean bootFromCD) { - return CitrixHelper.getXenServer620GuestOsType(stdType, bootFromCD); + return CitrixHelper.getXenServer625GuestOsType(stdType, bootFromCD); } @Override @@ -71,7 +73,7 @@ public class Xenserver625Resource extends XenServerResourceNewBase { @Override public long getStaticMax(String os, boolean b, long dynamicMinRam, long dynamicMaxRam){ - long recommendedValue = CitrixHelper.getXenServer620StaticMax(os, b); + long recommendedValue = CitrixHelper.getXenServer625StaticMax(os, b); if(recommendedValue == 0){ s_logger.warn("No recommended value found for dynamic max, setting static max and dynamic max equal"); return dynamicMaxRam; @@ -86,7 +88,7 @@ public class Xenserver625Resource extends XenServerResourceNewBase { @Override public long getStaticMin(String os, boolean b, long dynamicMinRam, long dynamicMaxRam){ - long recommendedValue = CitrixHelper.getXenServer620StaticMin(os, b); + long recommendedValue = CitrixHelper.getXenServer625StaticMin(os, b); if(recommendedValue == 0){ s_logger.warn("No recommended value found for dynamic min"); return dynamicMinRam; @@ -108,5 +110,4 @@ public class Xenserver625Resource extends XenServerResourceNewBase { protected void umountSnapshotDir(Connection conn, Long dcId) { } - } \ No newline at end of file diff --git a/setup/db/db/schema-421to430.sql b/setup/db/db/schema-421to430.sql index b55e3881852..3811a5a5e05 100644 --- a/setup/db/db/schema-421to430.sql +++ b/setup/db/db/schema-421to430.sql @@ -1099,3 +1099,7 @@ UPDATE `cloud`.`service_offering_details` set `display`=1 where id> 0; ALTER TABLE `cloud`.`disk_offering_details` CHANGE `display_detail` `display_detail` tinyint(1) NOT NULL DEFAULT '1' COMMENT 'True if the detail can be displayed to the end user'; UPDATE `cloud`.`disk_offering_details` set `display_detail`=1 where id> 0; + +INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name) VALUES (168, UUID(), 6, 'Windows Server 2012 R2 (64-bit)'); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (hypervisor_type, guest_os_name, guest_os_id) VALUES ("XenServer", 'Windows Server 2012 R2 (64-bit)', 168); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (hypervisor_type, guest_os_name, guest_os_id) VALUES ("VmWare", 'Windows Server 2012 R2 (64-bit)', 168); diff --git a/setup/db/templates.sql b/setup/db/templates.sql index 266dc60a74a..929345d885d 100755 --- a/setup/db/templates.sql +++ b/setup/db/templates.sql @@ -220,6 +220,7 @@ INSERT INTO `cloud`.`guest_os` (id, uuid, category_id, display_name) VALUES (164 INSERT INTO `cloud`.`guest_os` (id, uuid, category_id, display_name) VALUES (165, UUID(), 6, 'Windows 8 (32-bit)'); INSERT INTO `cloud`.`guest_os` (id, uuid, category_id, display_name) VALUES (166, UUID(), 6, 'Windows 8 (64-bit)'); INSERT INTO `cloud`.`guest_os` (id, uuid, category_id, display_name) VALUES (167, UUID(), 6, 'Windows Server 2012 (64-bit)'); +INSERT INTO `cloud`.`guest_os` (id, uuid, category_id, display_name) VALUES (168, UUID(), 6, 'Windows Server 2012 R2 (64-bit)'); INSERT INTO `cloud`.`guest_os` (id, uuid, category_id, display_name) VALUES (169, UUID(), 10, 'Ubuntu 11.04 (32-bit)'); INSERT INTO `cloud`.`guest_os` (id, uuid, category_id, display_name) VALUES (170, UUID(), 10, 'Ubuntu 11.04 (64-bit)'); INSERT INTO `cloud`.`guest_os` (id, uuid, category_id, display_name) VALUES (171, UUID(), 1, 'CentOS 6.3 (32-bit)'); @@ -375,6 +376,7 @@ INSERT INTO `cloud`.`guest_os_hypervisor` (hypervisor_type, guest_os_name, guest INSERT INTO `cloud`.`guest_os_hypervisor` (hypervisor_type, guest_os_name, guest_os_id) VALUES ("XenServer", 'Windows 8 (32-bit)', 165); INSERT INTO `cloud`.`guest_os_hypervisor` (hypervisor_type, guest_os_name, guest_os_id) VALUES ("XenServer", 'Windows 8 (64-bit)', 166); INSERT INTO `cloud`.`guest_os_hypervisor` (hypervisor_type, guest_os_name, guest_os_id) VALUES ("XenServer", 'Windows Server 2012 (64-bit)', 167); +INSERT INTO `cloud`.`guest_os_hypervisor` (hypervisor_type, guest_os_name, guest_os_id) VALUES ("XenServer", 'Windows Server 2012 R2 (64-bit)', 168); INSERT INTO `cloud`.`guest_os_hypervisor` (hypervisor_type, guest_os_name, guest_os_id) VALUES ("VmWare", 'Microsoft Windows 7(32-bit)', 48); INSERT INTO `cloud`.`guest_os_hypervisor` (hypervisor_type, guest_os_name, guest_os_id) VALUES ("VmWare", 'Microsoft Windows 7(64-bit)', 49); @@ -406,6 +408,7 @@ INSERT INTO `cloud`.`guest_os_hypervisor` (hypervisor_type, guest_os_name, guest INSERT INTO `cloud`.`guest_os_hypervisor` (hypervisor_type, guest_os_name, guest_os_id) VALUES ("VmWare", 'Windows 8 (32-bit)', 165); INSERT INTO `cloud`.`guest_os_hypervisor` (hypervisor_type, guest_os_name, guest_os_id) VALUES ("VmWare", 'Windows 8 (64-bit)', 166); INSERT INTO `cloud`.`guest_os_hypervisor` (hypervisor_type, guest_os_name, guest_os_id) VALUES ("VmWare", 'Windows Server 2012 (64-bit)', 167); +INSERT INTO `cloud`.`guest_os_hypervisor` (hypervisor_type, guest_os_name, guest_os_id) VALUES ("VmWare", 'Windows Server 2012 R2 (64-bit)', 168); INSERT INTO `cloud`.`guest_os_hypervisor` (hypervisor_type, guest_os_name, guest_os_id) VALUES ("VmWare", 'Red Hat Enterprise Linux 5.0(32-bit)', 30); INSERT INTO `cloud`.`guest_os_hypervisor` (hypervisor_type, guest_os_name, guest_os_id) VALUES ("VmWare", 'Red Hat Enterprise Linux 5.1(32-bit)', 32); diff --git a/vmware-base/src/com/cloud/hypervisor/vmware/util/VmwareGuestOsMapper.java b/vmware-base/src/com/cloud/hypervisor/vmware/util/VmwareGuestOsMapper.java index 325e6a4f575..4197941e908 100755 --- a/vmware-base/src/com/cloud/hypervisor/vmware/util/VmwareGuestOsMapper.java +++ b/vmware-base/src/com/cloud/hypervisor/vmware/util/VmwareGuestOsMapper.java @@ -61,6 +61,7 @@ public class VmwareGuestOsMapper { s_mapper.put("Windows 8 (32-bit)", VirtualMachineGuestOsIdentifier.WINDOWS_8_GUEST); s_mapper.put("Windows 8 (64-bit)", VirtualMachineGuestOsIdentifier.WINDOWS_8_64_GUEST); s_mapper.put("Windows Server 2012 (64-bit)", VirtualMachineGuestOsIdentifier.WINDOWS_8_SERVER_64_GUEST); + s_mapper.put("Windows Server 2012 R2 (64-bit)", VirtualMachineGuestOsIdentifier.WINDOWS_8_SERVER_64_GUEST); s_mapper.put("Apple Mac OS X 10.6 (32-bit)", VirtualMachineGuestOsIdentifier.DARWIN_10_GUEST); s_mapper.put("Apple Mac OS X 10.6 (64-bit)", VirtualMachineGuestOsIdentifier.DARWIN_10_64_GUEST); @@ -157,7 +158,7 @@ public class VmwareGuestOsMapper { s_mapper.put("Red Hat Enterprise Linux 5.4 (32-bit)", VirtualMachineGuestOsIdentifier.RHEL_5_GUEST); s_mapper.put("Red Hat Enterprise Linux 5.4 (64-bit)", VirtualMachineGuestOsIdentifier.RHEL_5_64_GUEST); s_mapper.put("Red Hat Enterprise Linux 6.0 (32-bit)", VirtualMachineGuestOsIdentifier.RHEL_6_GUEST); - s_mapper.put("Red Hat Enterprise Linux 6.0 (64-bit)", VirtualMachineGuestOsIdentifier.RHEL_6_64_GUEST); + s_mapper.put("Red Hat Enterprise Linux 6.0 (64-bit)", VirtualMachineGuestOsIdentifier.RHEL_6_64_GUEST); s_mapper.put("Ubuntu 8.04 (32-bit)", VirtualMachineGuestOsIdentifier.UBUNTU_GUEST); s_mapper.put("Ubuntu 8.04 (64-bit)", VirtualMachineGuestOsIdentifier.UBUNTU_64_GUEST); @@ -172,7 +173,7 @@ public class VmwareGuestOsMapper { s_mapper.put("Ubuntu 10.10 (32-bit)", VirtualMachineGuestOsIdentifier.UBUNTU_GUEST); s_mapper.put("Ubuntu 10.10 (64-bit)", VirtualMachineGuestOsIdentifier.UBUNTU_64_GUEST); s_mapper.put("Ubuntu 12.04 (32-bit)", VirtualMachineGuestOsIdentifier.UBUNTU_GUEST); - s_mapper.put("Ubuntu 12.04 (64-bit)", VirtualMachineGuestOsIdentifier.UBUNTU_64_GUEST); + s_mapper.put("Ubuntu 12.04 (64-bit)", VirtualMachineGuestOsIdentifier.UBUNTU_64_GUEST); s_mapper.put("Other Ubuntu (32-bit)", VirtualMachineGuestOsIdentifier.UBUNTU_GUEST); s_mapper.put("Other Ubuntu (64-bit)", VirtualMachineGuestOsIdentifier.UBUNTU_64_GUEST); From 94257c0fc85c7178e6ca3146662de08feee2c290 Mon Sep 17 00:00:00 2001 From: Min Chen Date: Wed, 26 Feb 2014 11:51:24 -0800 Subject: [PATCH 123/169] CLOUDSTACK-6175:s3.singleupload.max.size option not applicable for backup snapshot on KVM. --- .../kvm/storage/KVMStorageProcessor.java | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index cd576435fe9..8d237cc0c4d 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -18,6 +18,7 @@ */ package com.cloud.hypervisor.kvm.storage; +import static com.cloud.utils.S3Utils.mputFile; import static com.cloud.utils.S3Utils.putFile; import java.io.BufferedOutputStream; @@ -566,13 +567,20 @@ public class KVMStorageProcessor implements StorageProcessor { public Answer createTemplateFromSnapshot(CopyCommand cmd) { return null; //To change body of implemented methods use File | Settings | File Templates. } - protected String copyToS3(File srcFile, S3TO destStore, String destPath) { + + protected String copyToS3(File srcFile, S3TO destStore, String destPath) throws InterruptedException { final String bucket = destStore.getBucketName(); + long srcSize = srcFile.length(); String key = destPath + S3Utils.SEPARATOR + srcFile.getName(); - putFile(destStore, srcFile, bucket, key); + if (!destStore.getSingleUpload(srcSize)) { + mputFile(destStore, srcFile, bucket, key); + } else { + putFile(destStore, srcFile, bucket, key); + } return key; } + protected Answer copyToObjectStore(CopyCommand cmd) { DataTO srcData = cmd.getSrcTO(); DataTO destData = cmd.getDestTO(); @@ -603,6 +611,9 @@ public class KVMStorageProcessor implements StorageProcessor { SnapshotObjectTO newSnapshot = new SnapshotObjectTO(); newSnapshot.setPath(destPath); return new CopyCmdAnswer(newSnapshot); + } catch (Exception e) { + s_logger.error("failed to upload" + srcPath, e); + return new CopyCmdAnswer("failed to upload" + srcPath + e.toString()); } finally { try { if (srcFile != null) { From 82121a214926fe3ace33183da9e76e0c13c5e203 Mon Sep 17 00:00:00 2001 From: Jessica Wang Date: Wed, 26 Feb 2014 15:56:30 -0800 Subject: [PATCH 124/169] BUG-ID: CLOUDSTACK-6162: UI > zone > physical network > service provider > add OVS. Reviewed-by: Brian --- ui/scripts/system.js | 133 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) diff --git a/ui/scripts/system.js b/ui/scripts/system.js index 500a339ca00..9ba2597a964 100644 --- a/ui/scripts/system.js +++ b/ui/scripts/system.js @@ -192,6 +192,16 @@ return allowedActions; }; + function ovsProviderActionFilter(args) { + var allowedActions = []; + var jsonObj = args.context.item; //args.context.item == nspMap["virtualRouter"] + if (jsonObj.state == "Enabled") + allowedActions.push("disable"); + else if (jsonObj.state == "Disabled") + allowedActions.push("enable"); + return allowedActions; + }; + cloudStack.sections.system = { title: 'label.menu.infrastructure', id: 'system', @@ -3858,6 +3868,120 @@ } }, + OVS: { + id: "OVS", + label: "OVS", + isMaximized: true, + type: 'detailView', + fields: { + name: { + label: 'label.name' + }, + state: { + label: 'label.status', + indicator: { + 'Enabled': 'on' + } + } + }, + tabs: { + network: { + title: 'label.network', + fields: [{ + name: { + label: 'label.name' + } + }, { + state: { + label: 'label.state' + }, + supportedServices: { + label: 'label.supported.services' + }, + id: { + label: 'label.id' + }, + physicalnetworkid: { + label: 'label.physical.network.ID' + } + }], + dataProvider: function(args) { + refreshNspData("OVS"); + args.response.success({ + actionFilter: ovsProviderActionFilter, + data: $.extend(nspMap["OVS"], { + supportedServices: nspMap["OVS"] == undefined? "": nspMap["OVS"].servicelist.join(', ') + }) + }); + } + }, + }, + actions: { + enable: { + label: 'label.enable.provider', + action: function(args) { + $.ajax({ + url: createURL("updateNetworkServiceProvider&id=" + nspMap["OVS"].id + "&state=Enabled"), + dataType: "json", + success: function(json) { + var jid = json.updatenetworkserviceproviderresponse.jobid; + args.response.success({ + _custom: { + jobId: jid, + getUpdatedItem: function(json) { + $(window).trigger('cloudStack.fullRefresh'); + } + } + }); + } + }); + }, + messages: { + confirm: function(args) { + return 'message.confirm.enable.provider'; + }, + notification: function() { + return 'label.enable.provider'; + } + }, + notification: { + poll: pollAsyncJobResult + } + }, + disable: { + label: 'label.disable.provider', + action: function(args) { + $.ajax({ + url: createURL("updateNetworkServiceProvider&id=" + nspMap["OVS"].id + "&state=Disabled"), + dataType: "json", + success: function(json) { + var jid = json.updatenetworkserviceproviderresponse.jobid; + args.response.success({ + _custom: { + jobId: jid, + getUpdatedItem: function(json) { + $(window).trigger('cloudStack.fullRefresh'); + } + } + }); + } + }); + }, + messages: { + confirm: function(args) { + return 'message.confirm.disable.provider'; + }, + notification: function() { + return 'label.disable.provider'; + } + }, + notification: { + poll: pollAsyncJobResult + } + } + } + }, + // NetScaler provider detail view netscaler: { type: 'detailView', @@ -17999,6 +18123,9 @@ case "VpcVirtualRouter": nspMap["vpcVirtualRouter"] = items[i]; break; + case "OVS": + nspMap["OVS"] = items[i]; + break; case "Netscaler": nspMap["netscaler"] = items[i]; break; @@ -18107,6 +18234,12 @@ state: nspMap.pa ? nspMap.pa.state : 'Disabled' }); } + + nspHardcodingArray.push({ + id: 'OVS', + name: 'OVS', + state: nspMap.OVS ? nspMap.OVS.state : 'Disabled' + }); }; cloudStack.actionFilter.physicalNetwork = function(args) { From 2c3cd90b5604cde8123968b15ef6c53bad43b5e2 Mon Sep 17 00:00:00 2001 From: Hugo Trippaers Date: Tue, 25 Feb 2014 11:58:02 +0100 Subject: [PATCH 125/169] Fix the developer database deploy procedure so it includes the mysql database driver --- developer/pom.xml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/developer/pom.xml b/developer/pom.xml index 7ed6335fe16..5996de79977 100644 --- a/developer/pom.xml +++ b/developer/pom.xml @@ -114,6 +114,14 @@ org.codehaus.mojo exec-maven-plugin 1.2.1 + + + + mysql + mysql-connector-java + ${cs.mysql.version} + + process-resources @@ -125,6 +133,7 @@ com.cloud.upgrade.DatabaseCreator + true ${basedir}/../utils/conf/db.properties From 24dcf2948c2d4cdd98fcda0f766d82f40eee8be1 Mon Sep 17 00:00:00 2001 From: Hugo Trippaers Date: Tue, 25 Feb 2014 12:13:29 +0100 Subject: [PATCH 126/169] Fix jetty:run startup to include the mysql driver as a dependency for developers --- client/pom.xml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/client/pom.xml b/client/pom.xml index 84df72d86ca..253b3585f4c 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -350,6 +350,14 @@ org.mortbay.jetty maven-jetty-plugin 6.1.26 + + + + mysql + mysql-connector-java + ${cs.mysql.version} + + 0 9966 From b8c4ed435abc26950e36e21f9d482639b6691206 Mon Sep 17 00:00:00 2001 From: Murali Reddy Date: Thu, 27 Feb 2014 17:27:07 +0530 Subject: [PATCH 127/169] CLOUDSTACK-6162:support OVS as connectivity service provider adds 'Ovs' as network service provider in the physical network --- server/src/com/cloud/network/NetworkServiceImpl.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/server/src/com/cloud/network/NetworkServiceImpl.java b/server/src/com/cloud/network/NetworkServiceImpl.java index 34d674eefb9..8dc63bb1637 100755 --- a/server/src/com/cloud/network/NetworkServiceImpl.java +++ b/server/src/com/cloud/network/NetworkServiceImpl.java @@ -2503,9 +2503,8 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService { addDefaultInternalLbProviderToPhysicalNetwork(pNetwork.getId()); // Add OVS provider as default network service provider - if (pNetwork.getIsolationMethods().contains("GRE")) { - addDefaultOvsToPhysicalNetwork(pNetwork.getId()); - } + addDefaultOvsToPhysicalNetwork(pNetwork.getId()); + return pNetwork; } From 531a4ecc457ee3abfdc3f3ef1073895e482e5433 Mon Sep 17 00:00:00 2001 From: Alena Prokharchyk Date: Thu, 27 Feb 2014 10:47:38 -0800 Subject: [PATCH 128/169] CLOUDSTACK-6184: for userVm/ServiceOffering/StoragePool details set display=true by default in the VO objects --- .../schema/src/com/cloud/service/ServiceOfferingDetailsVO.java | 2 +- engine/schema/src/com/cloud/vm/UserVmDetailVO.java | 2 +- .../cloudstack/storage/datastore/db/StoragePoolDetailVO.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/engine/schema/src/com/cloud/service/ServiceOfferingDetailsVO.java b/engine/schema/src/com/cloud/service/ServiceOfferingDetailsVO.java index 4ab313438f2..e04473826ef 100644 --- a/engine/schema/src/com/cloud/service/ServiceOfferingDetailsVO.java +++ b/engine/schema/src/com/cloud/service/ServiceOfferingDetailsVO.java @@ -43,7 +43,7 @@ public class ServiceOfferingDetailsVO implements ResourceDetail { private String value; @Column(name="display") - boolean display; + boolean display = true; protected ServiceOfferingDetailsVO() { } diff --git a/engine/schema/src/com/cloud/vm/UserVmDetailVO.java b/engine/schema/src/com/cloud/vm/UserVmDetailVO.java index 82bf32d49a6..84a11f68687 100644 --- a/engine/schema/src/com/cloud/vm/UserVmDetailVO.java +++ b/engine/schema/src/com/cloud/vm/UserVmDetailVO.java @@ -43,7 +43,7 @@ public class UserVmDetailVO implements ResourceDetail { private String value; @Column(name="display") - private boolean display; + private boolean display = true; public UserVmDetailVO() {} diff --git a/engine/schema/src/org/apache/cloudstack/storage/datastore/db/StoragePoolDetailVO.java b/engine/schema/src/org/apache/cloudstack/storage/datastore/db/StoragePoolDetailVO.java index 5968fd28539..ee22d74d6ae 100644 --- a/engine/schema/src/org/apache/cloudstack/storage/datastore/db/StoragePoolDetailVO.java +++ b/engine/schema/src/org/apache/cloudstack/storage/datastore/db/StoragePoolDetailVO.java @@ -43,7 +43,7 @@ public class StoragePoolDetailVO implements ResourceDetail { String value; @Column(name="display") - private boolean display; + private boolean display = true; public StoragePoolDetailVO(long poolId, String name, String value) { this.resourceId = poolId; From 930acdde3d9c9a6ef1db90ae2f6ba3a9d8f15959 Mon Sep 17 00:00:00 2001 From: Jessica Wang Date: Thu, 27 Feb 2014 11:40:15 -0800 Subject: [PATCH 129/169] CLOUDSTACK-6162: UI > zone > physical network > service provider > OVS > match provider name "Ovs" in listNetworkServiceProviders API response. --- ui/scripts/system.js | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/ui/scripts/system.js b/ui/scripts/system.js index 9ba2597a964..0c56baf3603 100644 --- a/ui/scripts/system.js +++ b/ui/scripts/system.js @@ -3868,9 +3868,9 @@ } }, - OVS: { - id: "OVS", - label: "OVS", + Ovs: { + id: "Ovs", + label: "Ovs", isMaximized: true, type: 'detailView', fields: { @@ -3906,11 +3906,11 @@ } }], dataProvider: function(args) { - refreshNspData("OVS"); + refreshNspData("Ovs"); args.response.success({ actionFilter: ovsProviderActionFilter, - data: $.extend(nspMap["OVS"], { - supportedServices: nspMap["OVS"] == undefined? "": nspMap["OVS"].servicelist.join(', ') + data: $.extend(nspMap["Ovs"], { + supportedServices: nspMap["Ovs"] == undefined? "": nspMap["Ovs"].servicelist.join(', ') }) }); } @@ -3921,7 +3921,7 @@ label: 'label.enable.provider', action: function(args) { $.ajax({ - url: createURL("updateNetworkServiceProvider&id=" + nspMap["OVS"].id + "&state=Enabled"), + url: createURL("updateNetworkServiceProvider&id=" + nspMap["Ovs"].id + "&state=Enabled"), dataType: "json", success: function(json) { var jid = json.updatenetworkserviceproviderresponse.jobid; @@ -3952,7 +3952,7 @@ label: 'label.disable.provider', action: function(args) { $.ajax({ - url: createURL("updateNetworkServiceProvider&id=" + nspMap["OVS"].id + "&state=Disabled"), + url: createURL("updateNetworkServiceProvider&id=" + nspMap["Ovs"].id + "&state=Disabled"), dataType: "json", success: function(json) { var jid = json.updatenetworkserviceproviderresponse.jobid; @@ -18123,8 +18123,8 @@ case "VpcVirtualRouter": nspMap["vpcVirtualRouter"] = items[i]; break; - case "OVS": - nspMap["OVS"] = items[i]; + case "Ovs": + nspMap["Ovs"] = items[i]; break; case "Netscaler": nspMap["netscaler"] = items[i]; @@ -18236,9 +18236,9 @@ } nspHardcodingArray.push({ - id: 'OVS', - name: 'OVS', - state: nspMap.OVS ? nspMap.OVS.state : 'Disabled' + id: "Ovs", + name: "Ovs", + state: nspMap.Ovs ? nspMap.Ovs.state : 'Disabled' }); }; From 793a6a7177e7b2737b01c328951c86da1ba1808b Mon Sep 17 00:00:00 2001 From: frank Date: Thu, 27 Feb 2014 15:17:38 -0800 Subject: [PATCH 130/169] CS-19377 [Baremetal] After deploy VM on Baremetal host, BM host fail to be PXE booted --- .../networkservice/BaremetalDhcpElement.java | 14 +++++++++++--- .../networkservice/BaremetalDhcpManagerImpl.java | 7 +++---- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalDhcpElement.java b/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalDhcpElement.java index 8057cd42f91..b3fdab5b030 100755 --- a/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalDhcpElement.java +++ b/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalDhcpElement.java @@ -25,6 +25,8 @@ import java.util.Set; import javax.ejb.Local; import javax.inject.Inject; +import com.cloud.network.dao.PhysicalNetworkDao; +import com.cloud.network.dao.PhysicalNetworkVO; import org.apache.log4j.Logger; import com.cloud.baremetal.database.BaremetalDhcpVO; @@ -64,12 +66,14 @@ public class BaremetalDhcpElement extends AdapterBase implements DhcpServiceProv @Inject NicDao _nicDao; @Inject BaremetalDhcpManager _dhcpMgr; + @Inject + PhysicalNetworkDao phyNwDao; static { Capability cap = new Capability(BaremetalDhcpManager.BAREMETAL_DHCP_SERVICE_CAPABITLITY); Map baremetalCaps = new HashMap(); - baremetalCaps.put(cap, null); - baremetalCaps.put(Capability.DhcpAccrossMultipleSubnets, Boolean.TRUE.toString()); + baremetalCaps.put(cap, null); + baremetalCaps.put(Capability.DhcpAccrossMultipleSubnets, Boolean.TRUE.toString()); capabilities = new HashMap>(); capabilities.put(Service.Dhcp, baremetalCaps); } @@ -87,8 +91,12 @@ public class BaremetalDhcpElement extends AdapterBase implements DhcpServiceProv private boolean canHandle(DeployDestination dest, TrafficType trafficType, GuestType networkType) { Pod pod = dest.getPod(); if (pod != null && dest.getDataCenter().getNetworkType() == NetworkType.Basic && trafficType == TrafficType.Guest) { + QueryBuilder phyq = QueryBuilder.create(PhysicalNetworkVO.class); + phyq.and(phyq.entity().getDataCenterId(), Op.EQ, dest.getDataCenter().getId()); + PhysicalNetworkVO phynw = phyq.find(); + QueryBuilder sc = QueryBuilder.create(BaremetalDhcpVO.class); - sc.and(sc.entity().getPodId(), Op.EQ,pod.getId()); + sc.and(sc.entity().getPhysicalNetworkId(), Op.EQ, phynw.getId()); return sc.find() != null; } diff --git a/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalDhcpManagerImpl.java b/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalDhcpManagerImpl.java index 82397f5b31b..e507b4d8716 100755 --- a/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalDhcpManagerImpl.java +++ b/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalDhcpManagerImpl.java @@ -133,14 +133,13 @@ public class BaremetalDhcpManagerImpl extends ManagerBase implements BaremetalDh public boolean addVirtualMachineIntoNetwork(Network network, NicProfile nic, VirtualMachineProfile profile, DeployDestination dest, ReservationContext context) throws ResourceUnavailableException { Long zoneId = profile.getVirtualMachine().getDataCenterId(); - Long podId = profile.getVirtualMachine().getPodIdToDeployIn(); - List hosts = _resourceMgr.listAllUpAndEnabledHosts(Type.BaremetalDhcp, null, podId, zoneId); + List hosts = _resourceMgr.listAllUpAndEnabledHosts(Type.BaremetalDhcp, null, null, zoneId); if (hosts.size() == 0) { - throw new CloudRuntimeException("No external Dhcp found in zone " + zoneId + " pod " + podId); + throw new CloudRuntimeException("No external Dhcp found in zone " + zoneId); } if (hosts.size() > 1) { - throw new CloudRuntimeException("Something wrong, more than 1 external Dhcp found in zone " + zoneId + " pod " + podId); + throw new CloudRuntimeException("Something wrong, more than 1 external Dhcp found in zone " + zoneId); } HostVO h = hosts.get(0); From 7700a1b71622c1d45b2b54aa7444be5adc4fb8ab Mon Sep 17 00:00:00 2001 From: Jayapal Date: Thu, 27 Feb 2014 18:39:37 +0530 Subject: [PATCH 131/169] CLOUDSTACK-6183: unplug the nic when all the ips of public vlan range is removed --- .../resource/LibvirtComputingResource.java | 31 +++++++++++++++++++ .../xen/resource/CitrixResourceBase.java | 15 +++++++-- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 34770e4164d..ab7b9b54551 100755 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -1789,6 +1789,23 @@ ServerResource { return new Answer(cmd, true, result); } + private void vifHotUnPlug (Connect conn, String vmName, String macAddr) throws InternalErrorException, LibvirtException { + + Domain vm = null; + vm = getDomain(conn, vmName); + List pluggedNics = getInterfaces(conn, vmName); + for (InterfaceDef pluggedNic : pluggedNics) { + if (pluggedNic.getMacAddress().equalsIgnoreCase(macAddr)) { + vm.detachDevice(pluggedNic.toString()); + // We don't know which "traffic type" is associated with + // each interface at this point, so inform all vif drivers + for (VifDriver vifDriver : getAllVifDrivers()) { + vifDriver.unplug(pluggedNic); + } + } + } + } + private void VifHotPlug(Connect conn, String vmName, String broadcastUri, String macAddr) throws InternalErrorException, LibvirtException { NicTO nicTO = new NicTO(); @@ -2103,6 +2120,12 @@ ServerResource { String result = null; int nicNum = 0; boolean newNic = false; + int numOfIps = 0; + + if (ips != null) { + numOfIps = ips.length; + } + for (IpAddressTO ip : ips) { if (!broadcastUriAllocatedToVM.containsKey(ip.getBroadcastUri())) { /* plug a vif into router */ @@ -2121,6 +2144,14 @@ ServerResource { if (result == null) { results[i++] = ip.getPublicIp() + " - success"; } + + //there is only only ip in public subnet and it is deleted so unplug the vif + if (numOfIps == 1 && !ip.isAdd()) { + // There are no ips on the vm so delete the vif + networkUsage(routerIp, "deleteVif", "eth" + nicNum); + vifHotUnPlug(conn, routerName, ip.getVifMacAddress()); + } + } return new IpAssocAnswer(cmd, results); } catch (LibvirtException e) { diff --git a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java index b398ca517b7..7199d287ac0 100644 --- a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java +++ b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java @@ -2372,7 +2372,8 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe protected void assignPublicIpAddress(Connection conn, String vmName, String privateIpAddress, String publicIpAddress, boolean add, boolean firstIP, - boolean sourceNat, String vlanId, String vlanGateway, String vlanNetmask, String vifMacAddress, Integer networkRate, TrafficType trafficType, String name) throws InternalErrorException { + boolean sourceNat, String vlanId, String vlanGateway, String vlanNetmask, String vifMacAddress, Integer networkRate, TrafficType trafficType, String name, + int numOfips) throws InternalErrorException { try { VM router = getVM(conn, vmName); @@ -2463,6 +2464,11 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe throw new InternalErrorException("Xen plugin \"ipassoc\" failed."); } + //there is only only ip in public subnet and it is deleted so unplug the vif + if (numOfips == 1 && !add) { + removeVif = true; + } + if (removeVif) { network = correctVif.getNetwork(conn); @@ -2564,14 +2570,19 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe Connection conn = getConnection(); String[] results = new String[cmd.getIpAddresses().length]; int i = 0; + int numOfIps = 0; String routerName = cmd.getAccessDetail(NetworkElementCommand.ROUTER_NAME); String routerIp = cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP); try { IpAddressTO[] ips = cmd.getIpAddresses(); + if (ips != null) { + numOfIps = ips.length; + } + for (IpAddressTO ip : ips) { assignPublicIpAddress(conn, routerName, routerIp, ip.getPublicIp(), ip.isAdd(), ip.isFirstIP(), ip.isSourceNat(), ip.getBroadcastUri(), - ip.getVlanGateway(), ip.getVlanNetmask(), ip.getVifMacAddress(), ip.getNetworkRate(), ip.getTrafficType(), ip.getNetworkName()); + ip.getVlanGateway(), ip.getVlanNetmask(), ip.getVifMacAddress(), ip.getNetworkRate(), ip.getTrafficType(), ip.getNetworkName(), numOfIps); results[i++] = ip.getPublicIp() + " - success"; } } catch (InternalErrorException e) { From 37ce4e52c28081a0c718ca87fdbff79935074b75 Mon Sep 17 00:00:00 2001 From: Jayapal Date: Fri, 28 Feb 2014 11:01:43 +0530 Subject: [PATCH 132/169] CLOUDSTACK-6187: Fixed MigrateSystemVMCmd response for router --- server/src/com/cloud/api/ApiResponseHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/com/cloud/api/ApiResponseHelper.java b/server/src/com/cloud/api/ApiResponseHelper.java index 5bd8241f2ab..e543d1c37e2 100755 --- a/server/src/com/cloud/api/ApiResponseHelper.java +++ b/server/src/com/cloud/api/ApiResponseHelper.java @@ -1127,7 +1127,7 @@ public class ApiResponseHelper implements ResponseGenerator { @Override public SystemVmResponse createSystemVmResponse(VirtualMachine vm) { SystemVmResponse vmResponse = new SystemVmResponse(); - if (vm.getType() == Type.SecondaryStorageVm || vm.getType() == Type.ConsoleProxy) { + if (vm.getType() == Type.SecondaryStorageVm || vm.getType() == Type.ConsoleProxy || vm.getType() == Type.DomainRouter) { // SystemVm vm = (SystemVm) systemVM; vmResponse.setId(vm.getUuid()); // vmResponse.setObjectId(vm.getId()); From 258d8e2a0494ec8772d9238511fe5f8c6bf0ea01 Mon Sep 17 00:00:00 2001 From: Nitin Mehta Date: Fri, 28 Feb 2014 10:42:24 -0800 Subject: [PATCH 133/169] CLOUDSTACK-6186: Unable to register private template across all Zones - crosszones flag no longer works. This was because the code skipped the loop after the first download initiation. Introduced a set so that if it is already downloaded to the same zone then skip it for private template. Reviewed-by: Min --- .../template/HypervisorTemplateAdapter.java | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/server/src/com/cloud/template/HypervisorTemplateAdapter.java b/server/src/com/cloud/template/HypervisorTemplateAdapter.java index c6bb6175a3d..258528a54c8 100755 --- a/server/src/com/cloud/template/HypervisorTemplateAdapter.java +++ b/server/src/com/cloud/template/HypervisorTemplateAdapter.java @@ -19,7 +19,9 @@ package com.cloud.template; import java.net.MalformedURLException; import java.net.URL; import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.concurrent.ExecutionException; import javax.ejb.Local; @@ -187,7 +189,8 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { throw new CloudRuntimeException("Unable to find image store to download template "+ profile.getTemplate()); } - Collections.shuffle(imageStores);// For private templates choose a random store. TODO - Have a better algorithm based on size, no. of objects, load etc. + Set zoneSet = new HashSet(); + Collections.shuffle(imageStores); // For private templates choose a random store. TODO - Have a better algorithm based on size, no. of objects, load etc. for (DataStore imageStore : imageStores) { // skip data stores for a disabled zone Long zoneId = imageStore.getScope().getScopeId(); @@ -203,6 +206,14 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { s_logger.info("Zone " + zoneId + " is disabled, so skip downloading template to its image store " + imageStore.getId()); continue; } + + // We want to download private template to one of the image store in a zone + if(isPrivateTemplate(template) && zoneSet.contains(zoneId)){ + continue; + }else { + zoneSet.add(zoneId); + } + } TemplateInfo tmpl = imageFactory.getTemplate(template.getId(), imageStore); @@ -211,15 +222,21 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { caller.setCallback(caller.getTarget().createTemplateAsyncCallBack(null, null)); caller.setContext(context); imageService.createTemplateAsync(tmpl, imageStore, caller); - if( !(profile.getIsPublic() || profile.getFeatured()) ){ // If private template then break - break; - } } _resourceLimitMgr.incrementResourceCount(profile.getAccountId(), ResourceType.template); return template; } + private boolean isPrivateTemplate(VMTemplateVO template){ + + // if public OR featured OR system template + if(template.isPublicTemplate() || template.isFeatured() || template.getTemplateType() == TemplateType.SYSTEM) + return false; + else + return true; + } + private class CreateTemplateContext extends AsyncRpcContext { final TemplateInfo template; public CreateTemplateContext(AsyncCompletionCallback callback, TemplateInfo template) { From eb61f8146d0af899d8e39b7b1e0f6ea47b04c14e Mon Sep 17 00:00:00 2001 From: Alena Prokharchyk Date: Tue, 4 Mar 2014 13:33:38 -0800 Subject: [PATCH 134/169] CLOUDSTACK-5872: use List DS for storing NicProfiles as public network can have more than one nic --- .../com/cloud/vm/VirtualMachineManager.java | 5 +- .../service/NetworkOrchestrationService.java | 2 +- .../cloud/vm/VirtualMachineManagerImpl.java | 4 +- .../orchestration/CloudOrchestrator.java | 12 +- .../orchestration/NetworkOrchestrator.java | 117 +++++---- .../lb/ElasticLoadBalancerManagerImpl.java | 15 +- .../lb/InternalLoadBalancerVMManagerImpl.java | 17 +- .../management/ServiceManagerImpl.java | 32 +-- .../consoleproxy/ConsoleProxyManagerImpl.java | 16 +- .../VirtualNetworkApplianceManagerImpl.java | 231 ++++++++++++++---- ...VpcVirtualNetworkApplianceManagerImpl.java | 31 ++- .../SecondaryStorageManagerImpl.java | 11 +- .../src/com/cloud/vm/UserVmManagerImpl.java | 18 +- .../com/cloud/vpc/MockNetworkManagerImpl.java | 2 +- 14 files changed, 338 insertions(+), 175 deletions(-) diff --git a/engine/api/src/com/cloud/vm/VirtualMachineManager.java b/engine/api/src/com/cloud/vm/VirtualMachineManager.java index 5e57b85db6e..58e9280a1df 100644 --- a/engine/api/src/com/cloud/vm/VirtualMachineManager.java +++ b/engine/api/src/com/cloud/vm/VirtualMachineManager.java @@ -18,6 +18,7 @@ package com.cloud.vm; import java.net.URI; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import org.apache.cloudstack.framework.config.ConfigKey; @@ -81,14 +82,14 @@ public interface VirtualMachineManager extends Manager { ServiceOffering serviceOffering, Pair rootDiskOffering, LinkedHashMap dataDiskOfferings, - LinkedHashMap auxiliaryNetworks, + LinkedHashMap> auxiliaryNetworks, DeploymentPlan plan, HypervisorType hyperType) throws InsufficientCapacityException; void allocate(String vmInstanceName, VirtualMachineTemplate template, ServiceOffering serviceOffering, - LinkedHashMap networkProfiles, + LinkedHashMap> networkProfiles, DeploymentPlan plan, HypervisorType hyperType) throws InsufficientCapacityException; diff --git a/engine/api/src/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java b/engine/api/src/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java index 25591672eae..e3dbeeac143 100755 --- a/engine/api/src/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java +++ b/engine/api/src/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java @@ -76,7 +76,7 @@ public interface NetworkOrchestrationService { List setupNetwork(Account owner, NetworkOffering offering, Network predefined, DeploymentPlan plan, String name, String displayText, boolean errorIfAlreadySetup, Long domainId, ACLType aclType, Boolean subdomainAccess, Long vpcId, Boolean isDisplayNetworkEnabled) throws ConcurrentOperationException; - void allocate(VirtualMachineProfile vm, LinkedHashMap networks) throws InsufficientCapacityException, ConcurrentOperationException; + void allocate(VirtualMachineProfile vm, LinkedHashMap> networks) throws InsufficientCapacityException, ConcurrentOperationException; void prepare(VirtualMachineProfile profile, DeployDestination dest, ReservationContext context) throws InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException; diff --git a/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java index bc6b65d36c2..be05cd96dcc 100755 --- a/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java @@ -364,7 +364,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac @Override @DB public void allocate(String vmInstanceName, final VirtualMachineTemplate template, ServiceOffering serviceOffering, final Pair rootDiskOffering, - LinkedHashMap dataDiskOfferings, final LinkedHashMap auxiliaryNetworks, DeploymentPlan plan, + LinkedHashMap dataDiskOfferings, final LinkedHashMap> auxiliaryNetworks, DeploymentPlan plan, HypervisorType hyperType) throws InsufficientCapacityException { VMInstanceVO vm = _vmDao.findVMByInstanceName(vmInstanceName); @@ -423,7 +423,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } @Override - public void allocate(String vmInstanceName, VirtualMachineTemplate template, ServiceOffering serviceOffering, LinkedHashMap networks, + public void allocate(String vmInstanceName, VirtualMachineTemplate template, ServiceOffering serviceOffering, LinkedHashMap> networks, DeploymentPlan plan, HypervisorType hyperType) throws InsufficientCapacityException { allocate(vmInstanceName, template, serviceOffering, new Pair(serviceOffering, null), null, networks, plan, hyperType); } diff --git a/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/CloudOrchestrator.java b/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/CloudOrchestrator.java index 594b38810a2..af75b35b34d 100755 --- a/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/CloudOrchestrator.java +++ b/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/CloudOrchestrator.java @@ -20,14 +20,13 @@ package org.apache.cloudstack.engine.orchestration; import java.net.URL; import java.util.ArrayList; +import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import javax.inject.Inject; -import org.springframework.stereotype.Component; - import org.apache.cloudstack.engine.cloud.entity.api.NetworkEntity; import org.apache.cloudstack.engine.cloud.entity.api.TemplateEntity; import org.apache.cloudstack.engine.cloud.entity.api.VMEntityManager; @@ -36,6 +35,7 @@ import org.apache.cloudstack.engine.cloud.entity.api.VirtualMachineEntityImpl; import org.apache.cloudstack.engine.cloud.entity.api.VolumeEntity; import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; import org.apache.cloudstack.engine.service.api.OrchestrationService; +import org.springframework.stereotype.Component; import com.cloud.deploy.DeploymentPlan; import com.cloud.exception.InsufficientCapacityException; @@ -168,11 +168,11 @@ public class CloudOrchestrator implements OrchestrationService { // VirtualMachineEntityImpl vmEntity = new VirtualMachineEntityImpl(id, owner, hostName, displayName, cpu, speed, memory, computeTags, rootDiskTags, networks, vmEntityManager); - LinkedHashMap networkIpMap = new LinkedHashMap(); + LinkedHashMap> networkIpMap = new LinkedHashMap>(); for (String uuid : networkNicMap.keySet()) { NetworkVO network = _networkDao.findByUuid(uuid); if(network != null){ - networkIpMap.put(network, networkNicMap.get(uuid)); + networkIpMap.put(network, new ArrayList(Arrays.asList(networkNicMap.get(uuid)))); } } @@ -263,11 +263,11 @@ public class CloudOrchestrator implements OrchestrationService { rootDiskOffering.first(diskOffering); rootDiskOffering.second(size); - LinkedHashMap networkIpMap = new LinkedHashMap(); + LinkedHashMap> networkIpMap = new LinkedHashMap>(); for (String uuid : networkNicMap.keySet()) { NetworkVO network = _networkDao.findByUuid(uuid); if(network != null){ - networkIpMap.put(network, networkNicMap.get(uuid)); + networkIpMap.put(network, new ArrayList(Arrays.asList(networkNicMap.get(uuid)))); } } diff --git a/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java b/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java index 3cf93cdaffe..9e869c436c4 100755 --- a/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java +++ b/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java @@ -685,67 +685,84 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra @Override @DB - public void allocate(final VirtualMachineProfile vm, final LinkedHashMap networks) throws InsufficientCapacityException, + public void allocate(final VirtualMachineProfile vm, final LinkedHashMap> networks) throws InsufficientCapacityException, ConcurrentOperationException { Transaction.execute(new TransactionCallbackWithExceptionNoReturn() { @Override public void doInTransactionWithoutResult(TransactionStatus status) throws InsufficientCapacityException { int deviceId = 0; - - boolean[] deviceIds = new boolean[networks.size()]; + + int size = 0; + for (Network ntwk : networks.keySet()) { + List profiles = networks.get(ntwk); + if (profiles != null && !profiles.isEmpty()) { + size = size + profiles.size(); + } else { + size = size + 1; + } + } + + boolean[] deviceIds = new boolean[size]; Arrays.fill(deviceIds, false); - List nics = new ArrayList(networks.size()); + List nics = new ArrayList(size); NicProfile defaultNic = null; - for (Map.Entry network : networks.entrySet()) { + for (Map.Entry> network : networks.entrySet()) { Network config = network.getKey(); - NicProfile requested = network.getValue(); - - Boolean isDefaultNic = false; - if (vm != null && (requested != null && requested.isDefaultNic())) { - isDefaultNic = true; + List requestedProfiles = network.getValue(); + if (requestedProfiles == null) { + requestedProfiles = new ArrayList(); } - - while (deviceIds[deviceId] && deviceId < deviceIds.length) { - deviceId++; + if (requestedProfiles.isEmpty()) { + requestedProfiles.add(null); } - - Pair vmNicPair = allocateNic(requested, config, isDefaultNic, deviceId, vm); - - NicProfile vmNic = vmNicPair.first(); - if (vmNic == null) { - continue; - } - - deviceId = vmNicPair.second(); - - int devId = vmNic.getDeviceId(); - if (devId > deviceIds.length) { - throw new IllegalArgumentException("Device id for nic is too large: " + vmNic); - } - if (deviceIds[devId]) { - throw new IllegalArgumentException("Conflicting device id for two different nics: " + vmNic); - } - - deviceIds[devId] = true; - - if (vmNic.isDefaultNic()) { - if (defaultNic != null) { - throw new IllegalArgumentException("You cannot specify two nics as default nics: nic 1 = " + defaultNic + "; nic 2 = " + vmNic); + + for (NicProfile requested : requestedProfiles) { + Boolean isDefaultNic = false; + if (vm != null && (requested != null && requested.isDefaultNic())) { + isDefaultNic = true; } - defaultNic = vmNic; + + while (deviceIds[deviceId] && deviceId < deviceIds.length) { + deviceId++; + } + + Pair vmNicPair = allocateNic(requested, config, isDefaultNic, deviceId, vm); + + NicProfile vmNic = vmNicPair.first(); + if (vmNic == null) { + continue; + } + + deviceId = vmNicPair.second(); + + int devId = vmNic.getDeviceId(); + if (devId > deviceIds.length) { + throw new IllegalArgumentException("Device id for nic is too large: " + vmNic); + } + if (deviceIds[devId]) { + throw new IllegalArgumentException("Conflicting device id for two different nics: " + vmNic); + } + + deviceIds[devId] = true; + + if (vmNic.isDefaultNic()) { + if (defaultNic != null) { + throw new IllegalArgumentException("You cannot specify two nics as default nics: nic 1 = " + defaultNic + "; nic 2 = " + vmNic); + } + defaultNic = vmNic; + } + + nics.add(vmNic); + vm.addNic(vmNic); } - - nics.add(vmNic); - vm.addNic(vmNic); - } - if (nics.size() != networks.size()) { - s_logger.warn("Number of nics " + nics.size() + " doesn't match number of requested networks " + networks.size()); - throw new CloudRuntimeException("Number of nics " + nics.size() + " doesn't match number of requested networks " + networks.size()); + if (nics.size() != size) { + s_logger.warn("Number of nics " + nics.size() + " doesn't match number of requested nics " + size); + throw new CloudRuntimeException("Number of nics " + nics.size() + " doesn't match number of requested networks " + size); } if (nics.size() == 1) { @@ -768,14 +785,14 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra requested.setMode(network.getMode()); } NicProfile profile = guru.allocate(network, requested, vm); - if (isDefaultNic != null) { - profile.setDefaultNic(isDefaultNic); - } - if (profile == null) { return null; } + if (isDefaultNic != null) { + profile.setDefaultNic(isDefaultNic); + } + if (requested != null && requested.getMode() == null) { profile.setMode(requested.getMode()); } else { @@ -2476,8 +2493,8 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra if (dc.getNetworkType() == NetworkType.Basic) { List nics = _nicDao.listByVmId(vmInstance.getId()); NetworkVO network = _networksDao.findById(nics.get(0).getNetworkId()); - final LinkedHashMap profiles = new LinkedHashMap(); - profiles.put(network, null); + final LinkedHashMap> profiles = new LinkedHashMap>(); + profiles.put(network, new ArrayList()); Transaction.execute(new TransactionCallbackWithExceptionNoReturn() { @Override diff --git a/plugins/network-elements/elastic-loadbalancer/src/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java b/plugins/network-elements/elastic-loadbalancer/src/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java index e049cece058..c60b3ca7756 100644 --- a/plugins/network-elements/elastic-loadbalancer/src/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java +++ b/plugins/network-elements/elastic-loadbalancer/src/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java @@ -17,6 +17,7 @@ package com.cloud.network.lb; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -34,13 +35,13 @@ import javax.ejb.Local; import javax.inject.Inject; import javax.naming.ConfigurationException; -import org.apache.log4j.Logger; -import org.springframework.stereotype.Component; import org.apache.cloudstack.api.command.user.loadbalancer.CreateLoadBalancerRuleCmd; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; @@ -97,8 +98,8 @@ import com.cloud.network.dao.PhysicalNetworkServiceProviderDao; import com.cloud.network.dao.VirtualRouterProviderDao; import com.cloud.network.lb.LoadBalancingRule.LbDestination; import com.cloud.network.lb.LoadBalancingRule.LbHealthCheckPolicy; -import com.cloud.network.lb.LoadBalancingRule.LbStickinessPolicy; import com.cloud.network.lb.LoadBalancingRule.LbSslCert; +import com.cloud.network.lb.LoadBalancingRule.LbStickinessPolicy; import com.cloud.network.lb.dao.ElasticLbVmMapDao; import com.cloud.network.router.VirtualRouter; import com.cloud.network.router.VirtualRouter.RedundantState; @@ -124,8 +125,6 @@ import com.cloud.utils.db.DB; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.Transaction; -import com.cloud.utils.db.TransactionCallback; -import com.cloud.utils.db.TransactionCallbackNoReturn; import com.cloud.utils.db.TransactionCallbackWithException; import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; @@ -500,11 +499,11 @@ public class ElasticLoadBalancerManagerImpl extends ManagerBase implements Elast NetworkOffering controlOffering = offerings.get(0); Network controlConfig = _networkMgr.setupNetwork(_systemAcct, controlOffering, plan, null, null, false).get(0); - LinkedHashMap networks = new LinkedHashMap(2); + LinkedHashMap> networks = new LinkedHashMap>(2); NicProfile guestNic = new NicProfile(); guestNic.setDefaultNic(true); - networks.put(controlConfig, null); - networks.put(guestNetwork, guestNic); + networks.put(controlConfig, new ArrayList()); + networks.put(guestNetwork, new ArrayList(Arrays.asList(guestNic))); VMTemplateVO template = _templateDao.findSystemVMTemplate(dcId); diff --git a/plugins/network-elements/internal-loadbalancer/src/org/apache/cloudstack/network/lb/InternalLoadBalancerVMManagerImpl.java b/plugins/network-elements/internal-loadbalancer/src/org/apache/cloudstack/network/lb/InternalLoadBalancerVMManagerImpl.java index b9482b5a490..02e872b3fac 100644 --- a/plugins/network-elements/internal-loadbalancer/src/org/apache/cloudstack/network/lb/InternalLoadBalancerVMManagerImpl.java +++ b/plugins/network-elements/internal-loadbalancer/src/org/apache/cloudstack/network/lb/InternalLoadBalancerVMManagerImpl.java @@ -17,6 +17,7 @@ package org.apache.cloudstack.network.lb; import java.util.ArrayList; +import java.util.Arrays; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; @@ -26,12 +27,11 @@ import javax.ejb.Local; import javax.inject.Inject; import javax.naming.ConfigurationException; -import org.apache.log4j.Logger; - import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.lb.ApplicationLoadBalancerRuleVO; import org.apache.cloudstack.lb.dao.ApplicationLoadBalancerRuleDao; +import org.apache.log4j.Logger; import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; @@ -77,7 +77,6 @@ import com.cloud.network.lb.LoadBalancingRule; import com.cloud.network.lb.LoadBalancingRule.LbDestination; import com.cloud.network.lb.LoadBalancingRule.LbHealthCheckPolicy; import com.cloud.network.lb.LoadBalancingRule.LbStickinessPolicy; -import com.cloud.network.lb.LoadBalancingRule.LbSslCert; import com.cloud.network.lb.LoadBalancingRulesManager; import com.cloud.network.router.VirtualNetworkApplianceManager; import com.cloud.network.router.VirtualRouter; @@ -610,7 +609,7 @@ public class InternalLoadBalancerVMManagerImpl extends ManagerBase implements In return internalLbVms; } - LinkedHashMap networks = createInternalLbVmNetworks(guestNetwork, plan, requestedGuestIp); + LinkedHashMap> networks = createInternalLbVmNetworks(guestNetwork, plan, requestedGuestIp); //Pass startVm=false as we are holding the network lock that needs to be released at the end of vm allocation DomainRouterVO internalLbVm = deployInternalLbVm(owner, dest, plan, params, internalLbProviderId, _internalLbVmOfferingId, guestNetwork.getVpcId(), networks, false); @@ -646,11 +645,11 @@ public class InternalLoadBalancerVMManagerImpl extends ManagerBase implements In return internalLbProvider.getId(); } - protected LinkedHashMap createInternalLbVmNetworks(Network guestNetwork, DeploymentPlan plan, Ip guestIp) throws ConcurrentOperationException, + protected LinkedHashMap> createInternalLbVmNetworks(Network guestNetwork, DeploymentPlan plan, Ip guestIp) throws ConcurrentOperationException, InsufficientAddressCapacityException { //Form networks - LinkedHashMap networks = new LinkedHashMap(3); + LinkedHashMap> networks = new LinkedHashMap>(3); //1) Guest network - default if (guestNetwork != null) { @@ -669,7 +668,7 @@ public class InternalLoadBalancerVMManagerImpl extends ManagerBase implements In String gatewayCidr = guestNetwork.getCidr(); guestNic.setNetmask(NetUtils.getCidrNetmask(gatewayCidr)); guestNic.setDefaultNic(true); - networks.put(guestNetwork, guestNic); + networks.put(guestNetwork, new ArrayList(Arrays.asList(guestNic))); } //2) Control network @@ -677,7 +676,7 @@ public class InternalLoadBalancerVMManagerImpl extends ManagerBase implements In List offerings = _ntwkModel.getSystemAccountNetworkOfferings(NetworkOffering.SystemControlNetwork); NetworkOffering controlOffering = offerings.get(0); Network controlConfig = _ntwkMgr.setupNetwork(_accountMgr.getSystemAccount(), controlOffering, plan, null, null, false).get(0); - networks.put(controlConfig, null); + networks.put(controlConfig, new ArrayList()); return networks; } @@ -711,7 +710,7 @@ public class InternalLoadBalancerVMManagerImpl extends ManagerBase implements In protected DomainRouterVO deployInternalLbVm(Account owner, DeployDestination dest, DeploymentPlan plan, Map params, long internalLbProviderId, long svcOffId, Long vpcId, - LinkedHashMap networks, + LinkedHashMap> networks, boolean startVm) throws ConcurrentOperationException, InsufficientAddressCapacityException, InsufficientServerCapacityException, InsufficientCapacityException, StorageUnavailableException, ResourceUnavailableException { diff --git a/plugins/network-elements/juniper-contrail/src/org/apache/cloudstack/network/contrail/management/ServiceManagerImpl.java b/plugins/network-elements/juniper-contrail/src/org/apache/cloudstack/network/contrail/management/ServiceManagerImpl.java index 82f2cc4935d..71dad298869 100644 --- a/plugins/network-elements/juniper-contrail/src/org/apache/cloudstack/network/contrail/management/ServiceManagerImpl.java +++ b/plugins/network-elements/juniper-contrail/src/org/apache/cloudstack/network/contrail/management/ServiceManagerImpl.java @@ -27,6 +27,14 @@ import java.util.Map; import javax.ejb.Local; import javax.inject.Inject; +import net.juniper.contrail.api.ApiConnector; +import net.juniper.contrail.api.types.ServiceInstance; + +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.network.contrail.api.response.ServiceInstanceResponse; +import org.apache.cloudstack.network.contrail.model.ServiceInstanceModel; +import org.apache.cloudstack.network.contrail.model.VirtualMachineModel; +import org.apache.cloudstack.network.contrail.model.VirtualNetworkModel; import org.apache.log4j.Logger; import com.cloud.api.ApiDBUtils; @@ -36,22 +44,16 @@ import com.cloud.event.ActionEvent; import com.cloud.event.EventTypes; import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.InvalidParameterValueException; -import com.cloud.exception.ResourceUnavailableException; import com.cloud.network.Network; import com.cloud.network.NetworkModel; import com.cloud.network.Networks.TrafficType; import com.cloud.network.dao.NetworkVO; import com.cloud.offering.ServiceOffering; import com.cloud.projects.Project; -import com.cloud.service.ServiceOfferingVO; -import com.cloud.storage.VMTemplateVO; import com.cloud.template.VirtualMachineTemplate; import com.cloud.user.Account; import com.cloud.user.AccountService; -import com.cloud.user.User; -import com.cloud.user.UserVO; import com.cloud.user.dao.UserDao; -import com.cloud.utils.Pair; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.NicProfile; import com.cloud.vm.UserVmVO; @@ -60,16 +62,6 @@ import com.cloud.vm.VirtualMachineName; import com.cloud.vm.dao.UserVmDao; import com.google.gson.Gson; -import org.apache.cloudstack.context.CallContext; -import org.apache.cloudstack.network.contrail.api.response.ServiceInstanceResponse; -import org.apache.cloudstack.network.contrail.model.ServiceInstanceModel; -import org.apache.cloudstack.network.contrail.model.VirtualMachineModel; -import org.apache.cloudstack.network.contrail.model.VirtualNetworkModel; - -import net.juniper.contrail.api.ApiConnector; -import net.juniper.contrail.api.types.ServiceInstance; -import net.juniper.contrail.api.types.VirtualNetwork; - @Local(value = {ServiceManager.class}) public class ServiceManagerImpl implements ServiceManager { private static final Logger s_logger = Logger.getLogger(ServiceManager.class); @@ -106,12 +98,12 @@ public class ServiceManagerImpl implements ServiceManager { DataCenterDeployment plan = new DataCenterDeployment(zone.getId()); - LinkedHashMap networks = new LinkedHashMap(); + LinkedHashMap> networks = new LinkedHashMap>(); NetworkVO linklocal = (NetworkVO) _networkModel.getSystemNetworkByZoneAndTrafficType(zone.getId(), TrafficType.Management); - networks.put(linklocal, null); - networks.put((NetworkVO) left, null); - networks.put((NetworkVO) right, null); + networks.put(linklocal, new ArrayList()); + networks.put((NetworkVO)left, new ArrayList()); + networks.put((NetworkVO)right, new ArrayList()); String instanceName = VirtualMachineName.getVmName(id, owner.getId(), "SRV"); ServiceVirtualMachine svm = new ServiceVirtualMachine(id, instanceName, name, template.getId(), diff --git a/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java b/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java index 3572127f20c..c8769d43349 100755 --- a/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java +++ b/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java @@ -17,6 +17,7 @@ package com.cloud.consoleproxy; import java.nio.charset.Charset; +import java.util.ArrayList; import java.util.Arrays; import java.util.Date; import java.util.HashMap; @@ -29,11 +30,6 @@ import javax.ejb.Local; import javax.inject.Inject; import javax.naming.ConfigurationException; -import org.apache.log4j.Logger; - -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; - import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; @@ -41,6 +37,7 @@ import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; +import org.apache.log4j.Logger; import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; @@ -142,6 +139,8 @@ import com.cloud.vm.VirtualMachineProfile; import com.cloud.vm.dao.ConsoleProxyDao; import com.cloud.vm.dao.UserVmDetailsDao; import com.cloud.vm.dao.VMInstanceDao; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; // // Possible console proxy state transition cases @@ -722,15 +721,16 @@ VirtualMachineGuru, SystemVmLoadScanHandler, ResourceStateAdapter { } List offerings = _networkModel.getSystemAccountNetworkOfferings(NetworkOffering.SystemControlNetwork, NetworkOffering.SystemManagementNetwork); - LinkedHashMap networks = new LinkedHashMap(offerings.size() + 1); + LinkedHashMap> networks = new LinkedHashMap>(offerings.size() + 1); NicProfile defaultNic = new NicProfile(); defaultNic.setDefaultNic(true); defaultNic.setDeviceId(2); - networks.put(_networkMgr.setupNetwork(systemAcct, _networkOfferingDao.findById(defaultNetwork.getNetworkOfferingId()), plan, null, null, false).get(0), defaultNic); + networks.put(_networkMgr.setupNetwork(systemAcct, _networkOfferingDao.findById(defaultNetwork.getNetworkOfferingId()), plan, null, null, false).get(0), + new ArrayList(Arrays.asList(defaultNic))); for (NetworkOffering offering : offerings) { - networks.put(_networkMgr.setupNetwork(systemAcct, offering, plan, null, null, false).get(0), null); + networks.put(_networkMgr.setupNetwork(systemAcct, offering, plan, null, null, false).get(0), new ArrayList()); } ConsoleProxyVO proxy = new ConsoleProxyVO(id, _serviceOffering.getId(), name, template.getId(), template.getHypervisorType(), template.getGuestOSId(), dataCenterId, systemAcct.getDomainId(), diff --git a/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java b/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java index c372e7b3678..466b8610bf7 100755 --- a/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java +++ b/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java @@ -17,13 +17,91 @@ package com.cloud.network.router; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Calendar; +import java.util.Collections; +import java.util.Comparator; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TimeZone; +import java.util.UUID; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import javax.ejb.Local; +import javax.inject.Inject; +import javax.naming.ConfigurationException; + +import org.apache.cloudstack.api.command.admin.router.RebootRouterCmd; +import org.apache.cloudstack.api.command.admin.router.UpgradeRouterCmd; +import org.apache.cloudstack.api.command.admin.router.UpgradeRouterTemplateCmd; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; +import org.apache.cloudstack.framework.config.ConfigDepot; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.framework.jobs.AsyncJobManager; +import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO; +import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.cloudstack.utils.identity.ManagementServerNode; +import org.apache.log4j.Logger; + import com.cloud.agent.AgentManager; import com.cloud.agent.Listener; -import com.cloud.agent.api.*; +import com.cloud.agent.api.AgentControlAnswer; +import com.cloud.agent.api.AgentControlCommand; +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.BumpUpPriorityCommand; +import com.cloud.agent.api.CheckRouterAnswer; +import com.cloud.agent.api.CheckRouterCommand; +import com.cloud.agent.api.CheckS2SVpnConnectionsAnswer; +import com.cloud.agent.api.CheckS2SVpnConnectionsCommand; +import com.cloud.agent.api.Command; +import com.cloud.agent.api.GetDomRVersionAnswer; +import com.cloud.agent.api.GetDomRVersionCmd; +import com.cloud.agent.api.ModifySshKeysCommand; +import com.cloud.agent.api.NetworkUsageAnswer; +import com.cloud.agent.api.NetworkUsageCommand; +import com.cloud.agent.api.PvlanSetupCommand; +import com.cloud.agent.api.StartupCommand; import com.cloud.agent.api.check.CheckSshAnswer; import com.cloud.agent.api.check.CheckSshCommand; -import com.cloud.agent.api.routing.*; -import com.cloud.agent.api.to.*; +import com.cloud.agent.api.routing.CreateIpAliasCommand; +import com.cloud.agent.api.routing.DeleteIpAliasCommand; +import com.cloud.agent.api.routing.DhcpEntryCommand; +import com.cloud.agent.api.routing.DnsMasqConfigCommand; +import com.cloud.agent.api.routing.IpAliasTO; +import com.cloud.agent.api.routing.IpAssocCommand; +import com.cloud.agent.api.routing.LoadBalancerConfigCommand; +import com.cloud.agent.api.routing.NetworkElementCommand; +import com.cloud.agent.api.routing.RemoteAccessVpnCfgCommand; +import com.cloud.agent.api.routing.SavePasswordCommand; +import com.cloud.agent.api.routing.SetFirewallRulesCommand; +import com.cloud.agent.api.routing.SetMonitorServiceCommand; +import com.cloud.agent.api.routing.SetPortForwardingRulesCommand; +import com.cloud.agent.api.routing.SetPortForwardingRulesVpcCommand; +import com.cloud.agent.api.routing.SetStaticNatRulesCommand; +import com.cloud.agent.api.routing.VmDataCommand; +import com.cloud.agent.api.routing.VpnUsersCfgCommand; +import com.cloud.agent.api.to.DhcpTO; +import com.cloud.agent.api.to.FirewallRuleTO; +import com.cloud.agent.api.to.IpAddressTO; +import com.cloud.agent.api.to.LoadBalancerTO; +import com.cloud.agent.api.to.MonitorServiceTO; +import com.cloud.agent.api.to.PortForwardingRuleTO; +import com.cloud.agent.api.to.StaticNatRuleTO; import com.cloud.agent.manager.Commands; import com.cloud.alert.AlertManager; import com.cloud.api.ApiAsyncJobDispatcher; @@ -35,8 +113,14 @@ import com.cloud.cluster.dao.ManagementServerHostDao; import com.cloud.configuration.Config; import com.cloud.configuration.ConfigurationManager; import com.cloud.configuration.ZoneConfig; -import com.cloud.dc.*; +import com.cloud.dc.ClusterVO; +import com.cloud.dc.DataCenter; import com.cloud.dc.DataCenter.NetworkType; +import com.cloud.dc.DataCenterVO; +import com.cloud.dc.HostPodVO; +import com.cloud.dc.Pod; +import com.cloud.dc.Vlan; +import com.cloud.dc.VlanVO; import com.cloud.dc.dao.ClusterDao; import com.cloud.dc.dao.DataCenterDao; import com.cloud.dc.dao.HostPodDao; @@ -47,23 +131,66 @@ import com.cloud.deploy.DeploymentPlan; import com.cloud.deploy.DeploymentPlanner.ExcludeList; import com.cloud.event.ActionEvent; import com.cloud.event.EventTypes; -import com.cloud.exception.*; +import com.cloud.exception.AgentUnavailableException; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.ConnectionException; +import com.cloud.exception.InsufficientAddressCapacityException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.InsufficientServerCapacityException; +import com.cloud.exception.InsufficientVirtualNetworkCapcityException; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.OperationTimedoutException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.exception.StorageUnavailableException; import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.host.Status; import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.maint.Version; -import com.cloud.network.*; +import com.cloud.network.IpAddress; +import com.cloud.network.IpAddressManager; +import com.cloud.network.MonitoringService; +import com.cloud.network.Network; import com.cloud.network.Network.GuestType; import com.cloud.network.Network.Provider; import com.cloud.network.Network.Service; +import com.cloud.network.NetworkModel; +import com.cloud.network.NetworkService; import com.cloud.network.Networks.BroadcastDomainType; import com.cloud.network.Networks.IsolationType; import com.cloud.network.Networks.TrafficType; +import com.cloud.network.PhysicalNetworkServiceProvider; +import com.cloud.network.PublicIpAddress; +import com.cloud.network.RemoteAccessVpn; +import com.cloud.network.Site2SiteCustomerGateway; +import com.cloud.network.Site2SiteVpnConnection; +import com.cloud.network.SshKeysDistriMonitor; +import com.cloud.network.VirtualNetworkApplianceService; +import com.cloud.network.VirtualRouterProvider; import com.cloud.network.VirtualRouterProvider.Type; +import com.cloud.network.VpnUser; +import com.cloud.network.VpnUserVO; import com.cloud.network.addr.PublicIp; -import com.cloud.network.dao.*; +import com.cloud.network.dao.FirewallRulesDao; +import com.cloud.network.dao.IPAddressDao; +import com.cloud.network.dao.IPAddressVO; +import com.cloud.network.dao.LoadBalancerDao; +import com.cloud.network.dao.LoadBalancerVMMapDao; +import com.cloud.network.dao.LoadBalancerVO; +import com.cloud.network.dao.MonitoringServiceDao; +import com.cloud.network.dao.MonitoringServiceVO; +import com.cloud.network.dao.NetworkDao; +import com.cloud.network.dao.NetworkVO; +import com.cloud.network.dao.PhysicalNetworkServiceProviderDao; +import com.cloud.network.dao.RemoteAccessVpnDao; +import com.cloud.network.dao.Site2SiteCustomerGatewayDao; +import com.cloud.network.dao.Site2SiteVpnConnectionDao; +import com.cloud.network.dao.Site2SiteVpnConnectionVO; +import com.cloud.network.dao.Site2SiteVpnGatewayDao; +import com.cloud.network.dao.UserIpv6AddressDao; +import com.cloud.network.dao.VirtualRouterProviderDao; +import com.cloud.network.dao.VpnUserDao; import com.cloud.network.lb.LoadBalancingRule; import com.cloud.network.lb.LoadBalancingRule.LbDestination; import com.cloud.network.lb.LoadBalancingRule.LbHealthCheckPolicy; @@ -72,9 +199,15 @@ import com.cloud.network.lb.LoadBalancingRule.LbStickinessPolicy; import com.cloud.network.lb.LoadBalancingRulesManager; import com.cloud.network.router.VirtualRouter.RedundantState; import com.cloud.network.router.VirtualRouter.Role; -import com.cloud.network.rules.*; +import com.cloud.network.rules.FirewallRule; import com.cloud.network.rules.FirewallRule.Purpose; +import com.cloud.network.rules.FirewallRuleVO; import com.cloud.network.rules.LoadBalancerContainer.Scheme; +import com.cloud.network.rules.PortForwardingRule; +import com.cloud.network.rules.RulesManager; +import com.cloud.network.rules.StaticNat; +import com.cloud.network.rules.StaticNatImpl; +import com.cloud.network.rules.StaticNatRule; import com.cloud.network.rules.dao.PortForwardingRulesDao; import com.cloud.network.vpn.Site2SiteVpnManager; import com.cloud.offering.NetworkOffering; @@ -91,7 +224,12 @@ import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.GuestOSDao; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VolumeDao; -import com.cloud.user.*; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; +import com.cloud.user.User; +import com.cloud.user.UserStatisticsVO; +import com.cloud.user.UserStatsLogVO; +import com.cloud.user.UserVO; import com.cloud.user.dao.UserDao; import com.cloud.user.dao.UserStatisticsDao; import com.cloud.user.dao.UserStatsLogDao; @@ -103,35 +241,43 @@ import com.cloud.utils.StringUtils; import com.cloud.utils.component.ComponentContext; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.concurrency.NamedThreadFactory; -import com.cloud.utils.db.*; +import com.cloud.utils.db.DB; +import com.cloud.utils.db.EntityManager; +import com.cloud.utils.db.Filter; +import com.cloud.utils.db.GlobalLock; +import com.cloud.utils.db.JoinBuilder; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallbackNoReturn; +import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.net.Ip; import com.cloud.utils.net.MacAddress; import com.cloud.utils.net.NetUtils; -import com.cloud.vm.*; +import com.cloud.vm.DomainRouterVO; +import com.cloud.vm.Nic; +import com.cloud.vm.NicIpAlias; +import com.cloud.vm.NicProfile; +import com.cloud.vm.NicVO; +import com.cloud.vm.ReservationContext; +import com.cloud.vm.ReservationContextImpl; +import com.cloud.vm.UserVmVO; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachine.State; +import com.cloud.vm.VirtualMachineGuru; +import com.cloud.vm.VirtualMachineManager; +import com.cloud.vm.VirtualMachineName; +import com.cloud.vm.VirtualMachineProfile; import com.cloud.vm.VirtualMachineProfile.Param; -import com.cloud.vm.dao.*; -import org.apache.cloudstack.api.command.admin.router.RebootRouterCmd; -import org.apache.cloudstack.api.command.admin.router.UpgradeRouterCmd; -import org.apache.cloudstack.api.command.admin.router.UpgradeRouterTemplateCmd; -import org.apache.cloudstack.context.CallContext; -import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; -import org.apache.cloudstack.framework.config.ConfigDepot; -import org.apache.cloudstack.framework.config.ConfigKey; -import org.apache.cloudstack.framework.config.Configurable; -import org.apache.cloudstack.framework.config.dao.ConfigurationDao; -import org.apache.cloudstack.framework.jobs.AsyncJobManager; -import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO; -import org.apache.cloudstack.managed.context.ManagedContextRunnable; -import org.apache.cloudstack.utils.identity.ManagementServerNode; -import org.apache.log4j.Logger; - -import javax.ejb.Local; -import javax.inject.Inject; -import javax.naming.ConfigurationException; -import java.util.*; -import java.util.concurrent.*; +import com.cloud.vm.dao.DomainRouterDao; +import com.cloud.vm.dao.NicDao; +import com.cloud.vm.dao.NicIpAliasDao; +import com.cloud.vm.dao.NicIpAliasVO; +import com.cloud.vm.dao.UserVmDao; +import com.cloud.vm.dao.UserVmDetailsDao; +import com.cloud.vm.dao.VMInstanceDao; /** * VirtualNetworkApplianceManagerImpl manages the different types of virtual network appliances available in the Cloud Stack. @@ -1427,18 +1573,17 @@ public class VirtualNetworkApplianceManagerImpl extends ManagerBase implements V int count = routerCount - routers.size(); DeploymentPlan plan = planAndRouters.first(); for (int i = 0; i < count; i++) { - LinkedHashMap networks = createRouterNetworks(owner, isRedundant, plan, guestNetwork, - new Pair(publicNetwork, sourceNatIp)); + LinkedHashMap> networks = createRouterNetworks(owner, isRedundant, plan, guestNetwork, new Pair( + publicNetwork, sourceNatIp)); //don't start the router as we are holding the network lock that needs to be released at the end of router allocation - DomainRouterVO router = deployRouter(owner, destination, plan, params, isRedundant, vrProvider, offeringId, - null, networks, false, null); + DomainRouterVO router = deployRouter(owner, destination, plan, params, isRedundant, vrProvider, offeringId, null, networks, false, null); if (router != null) { _routerDao.addRouterToGuestNetwork(router, guestNetwork); routers.add(router); - } } } + } } finally { if (lock != null) { _networkDao.releaseFromLockTable(lock.getId()); @@ -1478,7 +1623,7 @@ public class VirtualNetworkApplianceManagerImpl extends ManagerBase implements V VirtualRouterProvider vrProvider, long svcOffId, Long vpcId, - LinkedHashMap networks, + LinkedHashMap> networks, boolean startRouter, List supportedHypervisors) throws ConcurrentOperationException, InsufficientAddressCapacityException, @@ -1626,7 +1771,7 @@ public class VirtualNetworkApplianceManagerImpl extends ManagerBase implements V return hypervisors; } - protected LinkedHashMap createRouterNetworks(Account owner, + protected LinkedHashMap> createRouterNetworks(Account owner, boolean isRedundant, DeploymentPlan plan, Network guestNetwork, Pair publicNetwork) throws ConcurrentOperationException, InsufficientAddressCapacityException { @@ -1638,7 +1783,7 @@ public class VirtualNetworkApplianceManagerImpl extends ManagerBase implements V } //Form networks - LinkedHashMap networks = new LinkedHashMap(3); + LinkedHashMap> networks = new LinkedHashMap>(3); //1) Guest network boolean hasGuestNetwork = false; @@ -1695,7 +1840,7 @@ public class VirtualNetworkApplianceManagerImpl extends ManagerBase implements V gatewayNic.setDefaultNic(true); } - networks.put(guestNetwork, gatewayNic); + networks.put(guestNetwork, new ArrayList(Arrays.asList(gatewayNic))); hasGuestNetwork = true; } @@ -1704,7 +1849,7 @@ public class VirtualNetworkApplianceManagerImpl extends ManagerBase implements V List offerings = _networkModel.getSystemAccountNetworkOfferings(NetworkOffering.SystemControlNetwork); NetworkOffering controlOffering = offerings.get(0); Network controlConfig = _networkMgr.setupNetwork(_systemAcct, controlOffering, plan, null, null, false).get(0); - networks.put(controlConfig, null); + networks.put(controlConfig, new ArrayList()); //3) Public network @@ -1741,7 +1886,7 @@ public class VirtualNetworkApplianceManagerImpl extends ManagerBase implements V s_logger.info("Use same MAC as previous RvR, the MAC is " + peerNic.getMacAddress()); defaultNic.setMacAddress(peerNic.getMacAddress()); } - networks.put(publicNetworks.get(0), defaultNic); + networks.put(publicNetworks.get(0), new ArrayList(Arrays.asList(defaultNic))); } return networks; diff --git a/server/src/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java b/server/src/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java index f65c8b0d122..620add383f2 100644 --- a/server/src/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java +++ b/server/src/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java @@ -18,6 +18,7 @@ package com.cloud.network.router; import java.net.URI; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashMap; @@ -40,12 +41,10 @@ import com.cloud.agent.api.SetupGuestNetworkAnswer; import com.cloud.agent.api.SetupGuestNetworkCommand; import com.cloud.agent.api.routing.IpAssocVpcCommand; import com.cloud.agent.api.routing.NetworkElementCommand; -import com.cloud.agent.api.routing.RemoteAccessVpnCfgCommand; import com.cloud.agent.api.routing.SetNetworkACLCommand; import com.cloud.agent.api.routing.SetSourceNatCommand; import com.cloud.agent.api.routing.SetStaticRouteCommand; import com.cloud.agent.api.routing.Site2SiteVpnCfgCommand; -import com.cloud.agent.api.routing.VpnUsersCfgCommand; import com.cloud.agent.api.to.IpAddressTO; import com.cloud.agent.api.to.NetworkACLTO; import com.cloud.agent.api.to.NicTO; @@ -333,7 +332,7 @@ public class VpcVirtualNetworkApplianceManagerImpl extends VirtualNetworkApplian InsufficientAddressCapacityException, InsufficientServerCapacityException, InsufficientCapacityException, StorageUnavailableException, ResourceUnavailableException { - LinkedHashMap networks = createVpcRouterNetworks(owner, isRedundant, plan, new Pair(true, sourceNatIp), + LinkedHashMap> networks = createVpcRouterNetworks(owner, isRedundant, plan, new Pair(true, sourceNatIp), vpcId); DomainRouterVO router = super.deployRouter(owner, dest, plan, params, isRedundant, vrProvider, svcOffId, vpcId, networks, true, @@ -1185,12 +1184,12 @@ public class VpcVirtualNetworkApplianceManagerImpl extends VirtualNetworkApplian } - protected LinkedHashMap + protected LinkedHashMap> createVpcRouterNetworks(Account owner, boolean isRedundant, DeploymentPlan plan, Pair sourceNatIp, long vpcId) throws ConcurrentOperationException, InsufficientAddressCapacityException { - LinkedHashMap networks = new LinkedHashMap(4); + LinkedHashMap> networks = new LinkedHashMap>(4); TreeSet publicVlans = new TreeSet(); publicVlans.add(sourceNatIp.second().getVlanTag()); @@ -1204,7 +1203,7 @@ public class VpcVirtualNetworkApplianceManagerImpl extends VirtualNetworkApplian for (PrivateGateway privateGateway : privateGateways) { NicProfile privateNic = createPrivateNicProfileForGateway(privateGateway); Network privateNetwork = _networkModel.getNetwork(privateGateway.getNetworkId()); - networks.put(privateNetwork, privateNic); + networks.put(privateNetwork, new ArrayList(Arrays.asList(privateNic))); } } @@ -1213,12 +1212,14 @@ public class VpcVirtualNetworkApplianceManagerImpl extends VirtualNetworkApplian for (Network guestNetwork : guestNetworks) { if (guestNetwork.getState() == Network.State.Implemented) { NicProfile guestNic = createGuestNicProfileForVpcRouter(guestNetwork); - networks.put(guestNetwork, guestNic); + networks.put(guestNetwork, new ArrayList(Arrays.asList(guestNic))); } } //4) allocate nic for additional public network(s) List ips = _ipAddressDao.listByAssociatedVpc(vpcId, false); + List publicNics = new ArrayList(); + Network publicNetwork = null; for (IPAddressVO ip : ips) { PublicIp publicIp = PublicIp.createFromAddrAndVlan(ip, _vlanDao.findById(ip.getVlanId())); if ((ip.getState() == IpAddress.State.Allocated || ip.getState() == IpAddress.State.Allocating) @@ -1234,11 +1235,23 @@ public class VpcVirtualNetworkApplianceManagerImpl extends VirtualNetworkApplian publicNic.setBroadcastUri(BroadcastDomainType.Vlan.toUri(publicIp.getVlanTag())); publicNic.setIsolationUri(IsolationType.Vlan.toUri(publicIp.getVlanTag())); NetworkOffering publicOffering = _networkModel.getSystemAccountNetworkOfferings(NetworkOffering.SystemPublicNetwork).get(0); - List publicNetworks = _networkMgr.setupNetwork(_systemAcct, publicOffering, plan, null, null, false); - networks.put(publicNetworks.get(0), publicNic); + if (publicNetwork == null) { + List publicNetworks = _networkMgr.setupNetwork(_systemAcct, publicOffering, plan, null, null, false); + publicNetwork = publicNetworks.get(0); + } + publicNics.add(publicNic); publicVlans.add(publicIp.getVlanTag()); } } + if (publicNetwork != null) { + if (networks.get(publicNetwork) != null) { + List publicNicProfiles = (List)networks.get(publicNetwork); + publicNicProfiles.addAll(publicNics); + networks.put(publicNetwork, publicNicProfiles); + } else { + networks.put(publicNetwork, publicNics); + } + } return networks; } diff --git a/server/src/com/cloud/storage/secondary/SecondaryStorageManagerImpl.java b/server/src/com/cloud/storage/secondary/SecondaryStorageManagerImpl.java index a3f837b7086..611550e9083 100755 --- a/server/src/com/cloud/storage/secondary/SecondaryStorageManagerImpl.java +++ b/server/src/com/cloud/storage/secondary/SecondaryStorageManagerImpl.java @@ -19,6 +19,7 @@ package com.cloud.storage.secondary; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Date; import java.util.HashMap; @@ -30,8 +31,6 @@ import javax.ejb.Local; import javax.inject.Inject; import javax.naming.ConfigurationException; -import org.apache.log4j.Logger; - import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; @@ -42,6 +41,7 @@ import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.utils.identity.ManagementServerNode; +import org.apache.log4j.Logger; import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; @@ -553,14 +553,15 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements Secondar } List offerings = _networkModel.getSystemAccountNetworkOfferings(NetworkOfferingVO.SystemControlNetwork, NetworkOfferingVO.SystemManagementNetwork, NetworkOfferingVO.SystemStorageNetwork); - LinkedHashMap networks = new LinkedHashMap(offerings.size() + 1); + LinkedHashMap> networks = new LinkedHashMap>(offerings.size() + 1); NicProfile defaultNic = new NicProfile(); defaultNic.setDefaultNic(true); defaultNic.setDeviceId(2); try { - networks.put(_networkMgr.setupNetwork(systemAcct, _networkOfferingDao.findById(defaultNetwork.getNetworkOfferingId()), plan, null, null, false).get(0), defaultNic); + networks.put(_networkMgr.setupNetwork(systemAcct, _networkOfferingDao.findById(defaultNetwork.getNetworkOfferingId()), plan, null, null, false).get(0), + new ArrayList(Arrays.asList(defaultNic))); for (NetworkOffering offering : offerings) { - networks.put(_networkMgr.setupNetwork(systemAcct, offering, plan, null, null, false).get(0), null); + networks.put(_networkMgr.setupNetwork(systemAcct, offering, plan, null, null, false).get(0), new ArrayList()); } } catch (ConcurrentOperationException e) { s_logger.info("Unable to setup due to concurrent operation. " + e); diff --git a/server/src/com/cloud/vm/UserVmManagerImpl.java b/server/src/com/cloud/vm/UserVmManagerImpl.java index 7cf4a24e36e..267a21e1715 100755 --- a/server/src/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/com/cloud/vm/UserVmManagerImpl.java @@ -17,6 +17,7 @@ package com.cloud.vm; import java.util.ArrayList; +import java.util.Arrays; import java.util.Date; import java.util.HashMap; import java.util.HashSet; @@ -34,9 +35,6 @@ import javax.ejb.Local; import javax.inject.Inject; import javax.naming.ConfigurationException; -import org.apache.commons.codec.binary.Base64; -import org.apache.log4j.Logger; - import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.acl.SecurityChecker.AccessType; import org.apache.cloudstack.affinity.AffinityGroupService; @@ -69,7 +67,6 @@ import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationSe import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; import org.apache.cloudstack.engine.service.api.OrchestrationService; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory; -import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService; @@ -82,7 +79,8 @@ import org.apache.cloudstack.framework.jobs.AsyncJobManager; import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; -import org.apache.cloudstack.storage.to.TemplateObjectTO; +import org.apache.commons.codec.binary.Base64; +import org.apache.log4j.Logger; import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; @@ -94,7 +92,6 @@ import com.cloud.agent.api.PvlanSetupCommand; import com.cloud.agent.api.StartAnswer; import com.cloud.agent.api.VmDiskStatsEntry; import com.cloud.agent.api.VmStatsEntry; -import com.cloud.agent.api.to.DiskTO; import com.cloud.agent.api.to.NicTO; import com.cloud.agent.api.to.VirtualMachineTO; import com.cloud.agent.manager.Commands; @@ -219,7 +216,6 @@ import com.cloud.storage.snapshot.SnapshotManager; import com.cloud.tags.dao.ResourceTagDao; import com.cloud.template.TemplateManager; import com.cloud.template.VirtualMachineTemplate; -import com.cloud.template.VirtualMachineTemplate.BootloaderType; import com.cloud.user.Account; import com.cloud.user.AccountManager; import com.cloud.user.AccountService; @@ -4800,10 +4796,10 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir } } - LinkedHashMap networks = new LinkedHashMap(); + LinkedHashMap> networks = new LinkedHashMap>(); NicProfile profile = new NicProfile(); profile.setDefaultNic(true); - networks.put(networkList.get(0), profile); + networks.put(networkList.get(0), new ArrayList(Arrays.asList(profile))); VirtualMachine vmi = _itMgr.findById(vm.getId()); VirtualMachineProfileImpl vmProfile = new VirtualMachineProfileImpl(vmi); @@ -4925,7 +4921,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir } // add the new nics - LinkedHashMap networks = new LinkedHashMap(); + LinkedHashMap> networks = new LinkedHashMap>(); int toggle = 0; for (NetworkVO appNet : applicableNetworks) { NicProfile defaultNic = new NicProfile(); @@ -4933,7 +4929,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir defaultNic.setDefaultNic(true); toggle++; } - networks.put(appNet, defaultNic); + networks.put(appNet, new ArrayList(Arrays.asList(defaultNic))); } VirtualMachine vmi = _itMgr.findById(vm.getId()); VirtualMachineProfileImpl vmProfile = new VirtualMachineProfileImpl(vmi); diff --git a/server/test/com/cloud/vpc/MockNetworkManagerImpl.java b/server/test/com/cloud/vpc/MockNetworkManagerImpl.java index 130a8aecf4a..aae0a3de11c 100644 --- a/server/test/com/cloud/vpc/MockNetworkManagerImpl.java +++ b/server/test/com/cloud/vpc/MockNetworkManagerImpl.java @@ -682,7 +682,7 @@ public class MockNetworkManagerImpl extends ManagerBase implements NetworkOrches * @see com.cloud.network.NetworkManager#allocate(com.cloud.vm.VirtualMachineProfile, java.util.List) */ @Override - public void allocate(VirtualMachineProfile vm, LinkedHashMap networks) + public void allocate(VirtualMachineProfile vm, LinkedHashMap> networks) throws InsufficientCapacityException, ConcurrentOperationException { // TODO Auto-generated method stub From 135247afd114c1dfc6dde22184430237e79aafc5 Mon Sep 17 00:00:00 2001 From: Marcus Sorensen Date: Tue, 4 Mar 2014 22:19:08 -0700 Subject: [PATCH 135/169] CLOUDSTACK-6194: Failed to increase Shared network IP Range Submitted-by:Saksham Srivastava --- .../src/com/cloud/configuration/ConfigurationManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/com/cloud/configuration/ConfigurationManagerImpl.java index d2e5aaa39a1..2e2c39faf3c 100755 --- a/server/src/com/cloud/configuration/ConfigurationManagerImpl.java +++ b/server/src/com/cloud/configuration/ConfigurationManagerImpl.java @@ -2666,7 +2666,7 @@ ConfigurationManagerImpl extends ManagerBase implements ConfigurationManager, Co List vlans = _vlanDao.listVlansByNetworkId(network.getId()); if (vlans != null && vlans.size() > 0) { VlanVO vlan = vlans.get(0); - if (vlanId == null) { + if (vlanId == null || vlanId.contains(Vlan.UNTAGGED)) { vlanId = vlan.getVlanTag(); } else if (!NetUtils.isSameIsolationId(vlan.getVlanTag(), vlanId)) { throw new InvalidParameterValueException("there is already one vlan " + vlan.getVlanTag() From 7fb2b8c68e7374d1969ce5ae9af890948dfdd9ce Mon Sep 17 00:00:00 2001 From: Saksham Srivastava Date: Thu, 20 Feb 2014 11:25:25 +0530 Subject: [PATCH 136/169] CLOUDSTACK-6151: Local data disk with tag goes to the wrong local storage pool Signed-off-by: Koushik Das --- ...spring-engine-schema-core-daos-context.xml | 2 +- .../datastore/db/PrimaryDataStoreDao.java | 2 + .../datastore/db/PrimaryDataStoreDaoImpl.java | 48 +++++++++++++++++-- .../allocator/LocalStoragePoolAllocator.java | 8 ++-- .../test/resources/createNetworkOffering.xml | 2 + 5 files changed, 53 insertions(+), 9 deletions(-) diff --git a/engine/schema/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml b/engine/schema/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml index ea0bad98868..699fd6bd63f 100644 --- a/engine/schema/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml +++ b/engine/schema/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml @@ -222,6 +222,7 @@ + @@ -259,7 +260,6 @@ - diff --git a/engine/schema/src/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java b/engine/schema/src/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java index 59c338e2803..a3cb76fe123 100644 --- a/engine/schema/src/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java +++ b/engine/schema/src/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java @@ -114,4 +114,6 @@ public interface PrimaryDataStoreDao extends GenericDao { List findZoneWideStoragePoolsByTags(long dcId, String[] tags); List findZoneWideStoragePoolsByHypervisor(long dataCenterId, HypervisorType hypervisorType); + + List findLocalStoragePoolsByHostAndTags(long hostId, String[] tags); } diff --git a/engine/schema/src/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java b/engine/schema/src/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java index d35aa440f74..eea07628af8 100644 --- a/engine/schema/src/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java +++ b/engine/schema/src/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java @@ -24,19 +24,21 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import javax.annotation.PostConstruct; import javax.ejb.Local; import javax.inject.Inject; import javax.naming.ConfigurationException; -import org.springframework.stereotype.Component; - import com.cloud.host.Status; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.storage.ScopeType; +import com.cloud.storage.StoragePoolHostVO; import com.cloud.storage.StoragePoolStatus; +import com.cloud.storage.dao.StoragePoolHostDao; import com.cloud.utils.db.DB; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.GenericSearchBuilder; +import com.cloud.utils.db.JoinBuilder; import com.cloud.utils.db.QueryBuilder; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; @@ -45,7 +47,6 @@ import com.cloud.utils.db.SearchCriteria.Op; import com.cloud.utils.db.TransactionLegacy; import com.cloud.utils.exception.CloudRuntimeException; - @Local(value = { PrimaryDataStoreDao.class }) @DB() public class PrimaryDataStoreDaoImpl extends GenericDaoBase implements PrimaryDataStoreDao { @@ -54,8 +55,12 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase protected final SearchBuilder DcPodAnyClusterSearch; protected final SearchBuilder DeleteLvmSearch; protected final GenericSearchBuilder StatusCountSearch; + protected SearchBuilder HostSearch; + protected SearchBuilder HostPoolSearch; + protected SearchBuilder TagPoolSearch; @Inject protected StoragePoolDetailsDao _detailsDao; + @Inject protected StoragePoolHostDao _hostDao; private final String DetailsSqlPrefix = "SELECT storage_pool.* from storage_pool LEFT JOIN storage_pool_details ON storage_pool.id = storage_pool_details.pool_id WHERE storage_pool.removed is null and storage_pool.status = 'Up' and storage_pool.data_center_id = ? and (storage_pool.pod_id = ? or storage_pool.pod_id is null) and storage_pool.scope = ? and ("; private final String DetailsSqlSuffix = ") GROUP BY storage_pool_details.pool_id HAVING COUNT(storage_pool_details.name) >= ?"; @@ -112,6 +117,26 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase } + @PostConstruct + void init() { + HostSearch = createSearchBuilder(); + TagPoolSearch = _detailsDao.createSearchBuilder(); + HostPoolSearch = _hostDao.createSearchBuilder(); + // Search for pools on the host + HostPoolSearch.and("hostId", HostPoolSearch.entity().getHostId(), Op.EQ); + // Set criteria for pools + HostSearch.and("scope", HostSearch.entity().getScope(), Op.EQ); + HostSearch.and("removed", HostSearch.entity().getRemoved(), Op.NULL); + HostSearch.and("status", HostSearch.entity().getStatus(), Op.EQ); + HostSearch.join("hostJoin", HostPoolSearch, HostSearch.entity().getId(), HostPoolSearch.entity().getPoolId(), JoinBuilder.JoinType.INNER); + // Set criteria for tags + TagPoolSearch.and("name", TagPoolSearch.entity().getName(), Op.EQ); + TagPoolSearch.and("value", TagPoolSearch.entity().getValue(), Op.EQ); + + HostSearch.join("tagJoin", TagPoolSearch, HostSearch.entity().getId(), TagPoolSearch.entity().getResourceId(), JoinBuilder.JoinType.INNER); + HostSearch.done(); + } + @Override public List findPoolByName(String name) { SearchCriteria sc = AllFieldSearch.create(); @@ -353,6 +378,23 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase } } + @Override + public List findLocalStoragePoolsByHostAndTags(long hostId, String[] tags) { + + SearchCriteria sc = HostSearch.create(); + sc.setJoinParameters("hostJoin", "hostId", hostId ); + sc.setParameters("scope", ScopeType.HOST.toString()); + sc.setParameters("status", Status.Up.toString()); + if (!(tags == null || tags.length == 0 )) { + Map details = tagsToDetails(tags); + for (Map.Entry detail : details.entrySet()) { + sc.setJoinParameters("tagJoin","name", detail.getKey()); + sc.setJoinParameters("tagJoin", "value", detail.getValue()); + } + } + return listBy(sc); + } + @Override @DB public List searchForStoragePoolDetails(long poolId, String value) { diff --git a/engine/storage/src/org/apache/cloudstack/storage/allocator/LocalStoragePoolAllocator.java b/engine/storage/src/org/apache/cloudstack/storage/allocator/LocalStoragePoolAllocator.java index 1f61e8b948d..51ab0feb047 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/allocator/LocalStoragePoolAllocator.java +++ b/engine/storage/src/org/apache/cloudstack/storage/allocator/LocalStoragePoolAllocator.java @@ -28,7 +28,6 @@ import javax.naming.ConfigurationException; import org.apache.cloudstack.engine.subsystem.api.storage.StoragePoolAllocator; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; - import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -73,14 +72,13 @@ public class LocalStoragePoolAllocator extends AbstractStoragePoolAllocator { if (!dskCh.useLocalStorage()) { return null; } - List suitablePools = new ArrayList(); // data disk and host identified from deploying vm (attach volume case) if (dskCh.getType() == Volume.Type.DATADISK && plan.getHostId() != null) { - List hostPools = _poolHostDao.listByHostId(plan.getHostId()); - for (StoragePoolHostVO hostPool : hostPools) { - StoragePoolVO pool = _storagePoolDao.findById(hostPool.getPoolId()); + List hostTagsPools = null; + hostTagsPools =_storagePoolDao.findLocalStoragePoolsByHostAndTags(plan.getHostId(), dskCh.getTags()); + for (StoragePoolVO pool : hostTagsPools) { if (pool != null && pool.isLocal()) { StoragePool storagePool = (StoragePool) this.dataStoreMgr.getPrimaryDataStore(pool.getId()); if (filter(avoid, storagePool, dskCh, plan)) { diff --git a/server/test/resources/createNetworkOffering.xml b/server/test/resources/createNetworkOffering.xml index c6228dab707..6ae739309b9 100644 --- a/server/test/resources/createNetworkOffering.xml +++ b/server/test/resources/createNetworkOffering.xml @@ -46,4 +46,6 @@ + + From dadc29920c29b8b9168e709621837d7373540718 Mon Sep 17 00:00:00 2001 From: Rajani Karuturi Date: Mon, 3 Mar 2014 16:47:41 +0530 Subject: [PATCH 137/169] added mock handlers for VpnUsersCfgCommand and RemoteAccessVpnCfgCommand in the simulator Signed-off-by: Koushik Das --- .../src/com/cloud/agent/manager/SimulatorManagerImpl.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/plugins/hypervisors/simulator/src/com/cloud/agent/manager/SimulatorManagerImpl.java b/plugins/hypervisors/simulator/src/com/cloud/agent/manager/SimulatorManagerImpl.java index f30bd3661d0..0801287379b 100644 --- a/plugins/hypervisors/simulator/src/com/cloud/agent/manager/SimulatorManagerImpl.java +++ b/plugins/hypervisors/simulator/src/com/cloud/agent/manager/SimulatorManagerImpl.java @@ -70,6 +70,7 @@ import com.cloud.agent.api.routing.DhcpEntryCommand; import com.cloud.agent.api.routing.IpAssocCommand; import com.cloud.agent.api.routing.IpAssocVpcCommand; import com.cloud.agent.api.routing.LoadBalancerConfigCommand; +import com.cloud.agent.api.routing.RemoteAccessVpnCfgCommand; import com.cloud.agent.api.routing.SavePasswordCommand; import com.cloud.agent.api.routing.SetFirewallRulesCommand; import com.cloud.agent.api.routing.SetNetworkACLCommand; @@ -80,6 +81,7 @@ import com.cloud.agent.api.routing.SetStaticNatRulesCommand; import com.cloud.agent.api.routing.SetStaticRouteCommand; import com.cloud.agent.api.routing.Site2SiteVpnCfgCommand; import com.cloud.agent.api.routing.VmDataCommand; +import com.cloud.agent.api.routing.VpnUsersCfgCommand; import com.cloud.agent.api.storage.CopyVolumeCommand; import com.cloud.agent.api.storage.CreateCommand; import com.cloud.agent.api.storage.DestroyCommand; @@ -362,7 +364,9 @@ public class SimulatorManagerImpl extends ManagerBase implements SimulatorManage } else if (cmd instanceof PvlanSetupCommand) { return _mockNetworkMgr.setupPVLAN((PvlanSetupCommand) cmd); } else if (cmd instanceof StorageSubSystemCommand) { - return this.storageHandler.handleStorageCommands((StorageSubSystemCommand)cmd); + return this.storageHandler.handleStorageCommands((StorageSubSystemCommand) cmd); + } else if (cmd instanceof VpnUsersCfgCommand || cmd instanceof RemoteAccessVpnCfgCommand) { + return new Answer(cmd); } else { s_logger.error("Simulator does not implement command of type "+cmd.toString()); return Answer.createUnsupportedCommandAnswer(cmd); From badff180210aec0b22c36a90daad03842849912f Mon Sep 17 00:00:00 2001 From: Rajani Karuturi Date: Wed, 26 Feb 2014 14:56:16 +0530 Subject: [PATCH 138/169] added delete iso api to the simulator in the smoke tests, this will fix test_03_delete_iso when run against a simulator Signed-off-by: Koushik Das --- .../datastore/driver/SimulatorImageStoreDriverImpl.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/plugins/hypervisors/simulator/src/org/apache/cloudstack/storage/datastore/driver/SimulatorImageStoreDriverImpl.java b/plugins/hypervisors/simulator/src/org/apache/cloudstack/storage/datastore/driver/SimulatorImageStoreDriverImpl.java index 26f8a40e1d5..854e7d1abbc 100644 --- a/plugins/hypervisors/simulator/src/org/apache/cloudstack/storage/datastore/driver/SimulatorImageStoreDriverImpl.java +++ b/plugins/hypervisors/simulator/src/org/apache/cloudstack/storage/datastore/driver/SimulatorImageStoreDriverImpl.java @@ -24,6 +24,7 @@ import java.util.UUID; import javax.inject.Inject; +import org.apache.cloudstack.storage.command.CommandResult; import org.apache.log4j.Logger; import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult; @@ -80,6 +81,11 @@ public class SimulatorImageStoreDriverImpl extends BaseImageStoreDriverImpl { } } + @Override + public void deleteAsync(DataStore dataStore, DataObject data, AsyncCompletionCallback callback) { + callback.complete(new CommandResult()); + } + protected void createTemplate(DataObject data, AsyncCompletionCallback callback) { CreateContext context = new CreateContext(callback, data); AsyncCallbackDispatcher caller = AsyncCallbackDispatcher From 1e5415d9612d9226b89bd4906563986feb3a175c Mon Sep 17 00:00:00 2001 From: David Grizzanti Date: Thu, 7 Nov 2013 15:27:59 -0500 Subject: [PATCH 139/169] CLOUDSTACK-5080: Hypervisor Capabilities table missing entry for Simulator Signed-off-by: Prasanna Santhanam --- developer/pom.xml | 1 + .../db/hypervisor_capabilities.simulator.sql | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100755 setup/db/hypervisor_capabilities.simulator.sql diff --git a/developer/pom.xml b/developer/pom.xml index 5996de79977..f2f5f771dfc 100644 --- a/developer/pom.xml +++ b/developer/pom.xml @@ -218,6 +218,7 @@ ${basedir}/target/db/create-schema-simulator.sql ${basedir}/target/db/templates.simulator.sql + ${basedir}/target/db/hypervisor_capabilities.simulator.sql com.cloud.upgrade.DatabaseUpgradeChecker --database=simulator diff --git a/setup/db/hypervisor_capabilities.simulator.sql b/setup/db/hypervisor_capabilities.simulator.sql new file mode 100755 index 00000000000..32f9aaa7351 --- /dev/null +++ b/setup/db/hypervisor_capabilities.simulator.sql @@ -0,0 +1,19 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + + +INSERT INTO `cloud`.`hypervisor_capabilities` (uuid, hypervisor_type, hypervisor_version, max_guests_limit, security_group_enabled, max_data_volumes_limit, max_hosts_per_cluster, storage_motion_supported, vm_snapshot_enabled) values (UUID(), 'Simulator', 'default', 50, 1, 6, NULL, 0, 1); From 8fdf941cc369284029f15b1b4cd36416eb57c16d Mon Sep 17 00:00:00 2001 From: Alena Prokharchyk Date: Wed, 5 Mar 2014 15:38:45 -0800 Subject: [PATCH 140/169] CLOUDSTACK-6205: VPC VR start - when create guest nics for the VR, consider networks not only in Implemented state, but in Setup state as well --- .../network/router/VpcVirtualNetworkApplianceManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java b/server/src/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java index 620add383f2..c2c0e5f7015 100644 --- a/server/src/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java +++ b/server/src/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java @@ -1210,7 +1210,7 @@ public class VpcVirtualNetworkApplianceManagerImpl extends VirtualNetworkApplian //3) allocate nic for guest gateway if needed List guestNetworks = _vpcMgr.getVpcNetworks(vpcId); for (Network guestNetwork : guestNetworks) { - if (guestNetwork.getState() == Network.State.Implemented) { + if (guestNetwork.getState() == Network.State.Implemented || guestNetwork.getState() == Network.State.Setup) { NicProfile guestNic = createGuestNicProfileForVpcRouter(guestNetwork); networks.put(guestNetwork, new ArrayList(Arrays.asList(guestNic))); } From f731781f49328a48a0476c3e766a814a48b3f77e Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Fri, 7 Mar 2014 00:12:20 +0100 Subject: [PATCH 141/169] fix apidocs build failure --- tools/apidoc/gen_toc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/apidoc/gen_toc.py b/tools/apidoc/gen_toc.py index 9e4cd31236e..29ff3bf441b 100644 --- a/tools/apidoc/gen_toc.py +++ b/tools/apidoc/gen_toc.py @@ -160,6 +160,7 @@ known_categories = { 'Ucs' : 'UCS', 'CacheStores' : 'Cache Stores', 'CacheStore' : 'Cache Store', + 'OvsElement' : 'Ovs Element', 'StratosphereSsp' : ' Stratosphere SSP' } From e789245f07a78d4c7b28264a13e8fc110ed9ce9a Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Fri, 7 Mar 2014 00:23:40 +0100 Subject: [PATCH 142/169] Revert "fix apidocs build failure" This reverts commit f731781f49328a48a0476c3e766a814a48b3f77e. --- tools/apidoc/gen_toc.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/apidoc/gen_toc.py b/tools/apidoc/gen_toc.py index 29ff3bf441b..9e4cd31236e 100644 --- a/tools/apidoc/gen_toc.py +++ b/tools/apidoc/gen_toc.py @@ -160,7 +160,6 @@ known_categories = { 'Ucs' : 'UCS', 'CacheStores' : 'Cache Stores', 'CacheStore' : 'Cache Store', - 'OvsElement' : 'Ovs Element', 'StratosphereSsp' : ' Stratosphere SSP' } From 164ea3e84f6f282006e66725f22cd2246f0be8f8 Mon Sep 17 00:00:00 2001 From: Alena Prokharchyk Date: Fri, 7 Mar 2014 15:06:35 -0800 Subject: [PATCH 143/169] CLOUDSTACK-6214: apply network rules when plug new guest nic to router for the network in Setup state Conflicts: server/src/com/cloud/network/router/VpcVirtualNetworkApplianceManager.java server/src/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java --- .../VpcVirtualNetworkApplianceService.java | 6 +++- .../element/VpcVirtualRouterElement.java | 14 +++++++-- .../VpcVirtualNetworkApplianceManager.java | 5 +++ ...VpcVirtualNetworkApplianceManagerImpl.java | 31 +++++++++++++++++-- ...MockVpcVirtualNetworkApplianceManager.java | 2 +- 5 files changed, 52 insertions(+), 6 deletions(-) diff --git a/api/src/com/cloud/network/VpcVirtualNetworkApplianceService.java b/api/src/com/cloud/network/VpcVirtualNetworkApplianceService.java index 6e8fd38ba66..661ac64cb4c 100644 --- a/api/src/com/cloud/network/VpcVirtualNetworkApplianceService.java +++ b/api/src/com/cloud/network/VpcVirtualNetworkApplianceService.java @@ -16,10 +16,13 @@ // under the License. package com.cloud.network; +import java.util.Map; + import com.cloud.exception.ConcurrentOperationException; import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.ResourceUnavailableException; import com.cloud.network.router.VirtualRouter; +import com.cloud.vm.VirtualMachineProfile; public interface VpcVirtualNetworkApplianceService extends VirtualNetworkApplianceService{ @@ -27,12 +30,13 @@ public interface VpcVirtualNetworkApplianceService extends VirtualNetworkApplian * @param router * @param network * @param isRedundant + * @param params TODO * @return * @throws ConcurrentOperationException * @throws ResourceUnavailableException * @throws InsufficientCapacityException */ - boolean addVpcRouterToGuestNetwork(VirtualRouter router, Network network, boolean isRedundant) + boolean addVpcRouterToGuestNetwork(VirtualRouter router, Network network, boolean isRedundant, Map params) throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException; /** diff --git a/server/src/com/cloud/network/element/VpcVirtualRouterElement.java b/server/src/com/cloud/network/element/VpcVirtualRouterElement.java index f4d16e1feb7..8b213b59d75 100644 --- a/server/src/com/cloud/network/element/VpcVirtualRouterElement.java +++ b/server/src/com/cloud/network/element/VpcVirtualRouterElement.java @@ -37,6 +37,7 @@ import com.cloud.network.Network; import com.cloud.network.Network.Capability; import com.cloud.network.Network.Provider; import com.cloud.network.Network.Service; +import com.cloud.network.Network.State; import com.cloud.network.NetworkModel; import com.cloud.network.PublicIpAddress; import com.cloud.network.RemoteAccessVpn; @@ -185,7 +186,11 @@ public class VpcVirtualRouterElement extends VirtualRouterElement implements Vpc DomainRouterVO router = routers.get(0); //Add router to guest network if needed if (!_networkMgr.isVmPartOfNetwork(router.getId(), network.getId())) { - if (!_vpcRouterMgr.addVpcRouterToGuestNetwork(router, network, false)) { + Map paramsForRouter = new HashMap(1); + if (network.getState() == State.Setup) { + paramsForRouter.put(VirtualMachineProfile.Param.ReProgramGuestNetworks, true); + } + if (!_vpcRouterMgr.addVpcRouterToGuestNetwork(router, network, false, paramsForRouter)) { throw new CloudRuntimeException("Failed to add VPC router " + router + " to guest network " + network); } else { s_logger.debug("Successfully added VPC router " + router + " to guest network " + network); @@ -229,7 +234,12 @@ public class VpcVirtualRouterElement extends VirtualRouterElement implements Vpc DomainRouterVO router = routers.get(0); //Add router to guest network if needed if (!_networkMgr.isVmPartOfNetwork(router.getId(), network.getId())) { - if (!_vpcRouterMgr.addVpcRouterToGuestNetwork(router, network, false)) { + Map paramsForRouter = new HashMap(1); + //need to reprogram guest network if it comes in a setup state + if (network.getState() == State.Setup) { + paramsForRouter.put(VirtualMachineProfile.Param.ReProgramGuestNetworks, true); + } + if (!_vpcRouterMgr.addVpcRouterToGuestNetwork(router, network, false, paramsForRouter)) { throw new CloudRuntimeException("Failed to add VPC router " + router + " to guest network " + network); } else { s_logger.debug("Successfully added VPC router " + router + " to guest network " + network); diff --git a/server/src/com/cloud/network/router/VpcVirtualNetworkApplianceManager.java b/server/src/com/cloud/network/router/VpcVirtualNetworkApplianceManager.java index fa34b0dd70e..0ecae428320 100644 --- a/server/src/com/cloud/network/router/VpcVirtualNetworkApplianceManager.java +++ b/server/src/com/cloud/network/router/VpcVirtualNetworkApplianceManager.java @@ -18,6 +18,7 @@ package com.cloud.network.router; import java.util.List; import java.util.Map; + import com.cloud.deploy.DeployDestination; import com.cloud.exception.ConcurrentOperationException; import com.cloud.exception.InsufficientCapacityException; @@ -28,6 +29,10 @@ import com.cloud.network.Site2SiteVpnConnection; import com.cloud.network.VpcVirtualNetworkApplianceService; import com.cloud.network.VpnUser; import com.cloud.network.vpc.*; +import com.cloud.network.vpc.NetworkACLItem; +import com.cloud.network.vpc.PrivateGateway; +import com.cloud.network.vpc.StaticRouteProfile; +import com.cloud.network.vpc.Vpc; import com.cloud.user.Account; import com.cloud.vm.DomainRouterVO; import com.cloud.vm.VirtualMachineProfile.Param; diff --git a/server/src/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java b/server/src/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java index c2c0e5f7015..c540eb7a35d 100644 --- a/server/src/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java +++ b/server/src/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java @@ -35,6 +35,7 @@ import org.springframework.stereotype.Component; import com.cloud.agent.api.Answer; import com.cloud.agent.api.Command; +import com.cloud.agent.api.Command.OnError; import com.cloud.agent.api.NetworkUsageCommand; import com.cloud.agent.api.PlugNicCommand; import com.cloud.agent.api.SetupGuestNetworkAnswer; @@ -253,7 +254,7 @@ public class VpcVirtualNetworkApplianceManagerImpl extends VirtualNetworkApplian @Override - public boolean addVpcRouterToGuestNetwork(VirtualRouter router, Network network, boolean isRedundant) + public boolean addVpcRouterToGuestNetwork(VirtualRouter router, Network network, boolean isRedundant, Map params) throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException { if (network.getTrafficType() != TrafficType.Guest) { @@ -264,16 +265,24 @@ public class VpcVirtualNetworkApplianceManagerImpl extends VirtualNetworkApplian //Add router to the Guest network boolean result = true; try { + + //1) add nic to the router _routerDao.addRouterToGuestNetwork(router, network); NicProfile guestNic = _itMgr.addVmToNetwork(router, network, null); - //setup guest network + //2) setup guest network if (guestNic != null) { result = setupVpcGuestNetwork(network, router, true, guestNic); } else { s_logger.warn("Failed to add router " + router + " to guest network " + network); result = false; } + //3) apply networking rules + if (result && params.get(Param.ReProgramGuestNetworks) != null + && (Boolean) params.get(Param.ReProgramGuestNetworks) == true) { + sendNetworkRulesToRouter(router.getId(), network.getId()); + + } } catch (Exception ex) { s_logger.warn("Failed to add router " + router + " to network " + network + " due to ", ex); result = false; @@ -906,6 +915,24 @@ public class VpcVirtualNetworkApplianceManagerImpl extends VirtualNetworkApplian } } } + + protected boolean sendNetworkRulesToRouter(long routerId, long networkId) + throws ResourceUnavailableException { + DomainRouterVO router = _routerDao.findById(routerId); + Commands cmds = new Commands(OnError.Continue); + + VirtualRouterProvider vrProvider = _vrProviderDao.findById(router.getElementId()); + if (vrProvider == null) { + throw new CloudRuntimeException("Cannot find related virtual router provider of router: " + router.getHostName()); + } + Provider provider = Network.Provider.getProvider(vrProvider.getType().toString()); + if (provider == null) { + throw new CloudRuntimeException("Cannot find related provider of virtual router provider: " + vrProvider.getType().toString()); + } + + finalizeNetworkRulesForNetwork(cmds, router, provider, networkId); + return sendCommandsToRouter(router, cmds); + } @Override public boolean setupPrivateGateway(PrivateGateway gateway, VirtualRouter router) throws ConcurrentOperationException, ResourceUnavailableException { diff --git a/server/test/com/cloud/vpc/MockVpcVirtualNetworkApplianceManager.java b/server/test/com/cloud/vpc/MockVpcVirtualNetworkApplianceManager.java index 5c216c8f19f..4c06e799595 100644 --- a/server/test/com/cloud/vpc/MockVpcVirtualNetworkApplianceManager.java +++ b/server/test/com/cloud/vpc/MockVpcVirtualNetworkApplianceManager.java @@ -320,7 +320,7 @@ VpcVirtualNetworkApplianceService { * @see com.cloud.network.VpcVirtualNetworkApplianceService#addVpcRouterToGuestNetwork(com.cloud.network.router.VirtualRouter, com.cloud.network.Network, boolean) */ @Override - public boolean addVpcRouterToGuestNetwork(VirtualRouter router, Network network, boolean isRedundant) + public boolean addVpcRouterToGuestNetwork(VirtualRouter router, Network network, boolean isRedundant, Map params) throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException { // TODO Auto-generated method stub return false; From 2fe7aeea23ddef25224e3e248f0a91513a14811f Mon Sep 17 00:00:00 2001 From: John Kinsella Date: Sun, 9 Mar 2014 14:04:41 -0700 Subject: [PATCH 144/169] CLOUDSTACK-6204: Removing realhostip.com dependency For more info, see https://cwiki.apache.org/confluence/display/CLOUDSTACK/Realhost+IP+changes Author: Amogh Vasekar Signed-off-by: John Kinsella 1394399081 -0700 --- .../storage/image/TemplateServiceImpl.java | 13 +++++++++++-- .../driver/CloudStackImageStoreDriverImpl.java | 12 ++++++++---- server/src/com/cloud/configuration/Config.java | 4 ++-- .../cloud/consoleproxy/ConsoleProxyManagerImpl.java | 11 ++++++++--- .../secondary/SecondaryStorageManagerImpl.java | 7 +++++++ setup/db/db/schema-421to430.sql | 4 ++++ systemvm/conf/consoleproxy.properties | 2 +- 7 files changed, 41 insertions(+), 12 deletions(-) diff --git a/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java b/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java index a649bb72123..1dd288185aa 100644 --- a/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java +++ b/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java @@ -793,12 +793,21 @@ public class TemplateServiceImpl implements TemplateService { String scheme = "http"; boolean _sslCopy = false; String sslCfg = _configDao.getValue(Config.SecStorageEncryptCopy.toString()); + String _ssvmUrlDomain = _configDao.getValue("secstorage.ssl.cert.domain"); if ( sslCfg != null ){ _sslCopy = Boolean.parseBoolean(sslCfg); } + if(_sslCopy && (_ssvmUrlDomain == null || _ssvmUrlDomain.isEmpty())){ + s_logger.warn("Empty secondary storage url domain, ignoring SSL"); + _sslCopy = false; + } if (_sslCopy) { - hostname = ipAddress.replace(".", "-"); - hostname = hostname + ".realhostip.com"; + if(_ssvmUrlDomain.startsWith("*")) { + hostname = ipAddress.replace(".", "-"); + hostname = hostname + _ssvmUrlDomain.substring(1); + } else { + hostname = _ssvmUrlDomain; + } scheme = "https"; } return scheme + "://" + hostname + "/copy/SecStorage/" + dir + "/" + path; diff --git a/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java b/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java index 52cad3bc7af..1af0eba993f 100644 --- a/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java +++ b/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java @@ -92,12 +92,16 @@ public class CloudStackImageStoreDriverImpl extends BaseImageStoreDriverImpl { if ( sslCfg != null ){ _sslCopy = Boolean.parseBoolean(sslCfg); } + if(_sslCopy && (_ssvmUrlDomain == null || _ssvmUrlDomain.isEmpty())){ + s_logger.warn("Empty secondary storage url domain, ignoring SSL"); + _sslCopy = false; + } if (_sslCopy) { - hostname = ipAddress.replace(".", "-"); - if(_ssvmUrlDomain != null && _ssvmUrlDomain.length() > 0){ - hostname = hostname + "." + _ssvmUrlDomain; + if(_ssvmUrlDomain.startsWith("*")) { + hostname = ipAddress.replace(".", "-"); + hostname = hostname + _ssvmUrlDomain.substring(1); } else { - hostname = hostname + ".realhostip.com"; + hostname = _ssvmUrlDomain; } scheme = "https"; } diff --git a/server/src/com/cloud/configuration/Config.java b/server/src/com/cloud/configuration/Config.java index 2d55f4634e0..6ebb7e63a74 100755 --- a/server/src/com/cloud/configuration/Config.java +++ b/server/src/com/cloud/configuration/Config.java @@ -118,7 +118,7 @@ public enum Config { ConsoleProxyCapacityScanInterval("Console Proxy", AgentManager.class, String.class, "consoleproxy.capacityscan.interval", "30000", "The time interval(in millisecond) to scan whether or not system needs more console proxy to ensure minimal standby capacity", null), ConsoleProxyCmdPort("Console Proxy", AgentManager.class, Integer.class, "consoleproxy.cmd.port", "8001", "Console proxy command port that is used to communicate with management server", null), ConsoleProxyRestart("Console Proxy", AgentManager.class, Boolean.class, "consoleproxy.restart", "true", "Console proxy restart flag, defaulted to true", null), - ConsoleProxyUrlDomain("Console Proxy", AgentManager.class, String.class, "consoleproxy.url.domain", "realhostip.com", "Console proxy url domain", null), + ConsoleProxyUrlDomain("Console Proxy", AgentManager.class, String.class, "consoleproxy.url.domain", "", "Console proxy url domain", null), ConsoleProxyLoadscanInterval("Console Proxy", AgentManager.class, String.class, "consoleproxy.loadscan.interval", "10000", "The time interval(in milliseconds) to scan console proxy working-load info", null), ConsoleProxySessionMax("Console Proxy", AgentManager.class, Integer.class, "consoleproxy.session.max", String.valueOf(ConsoleProxyManager.DEFAULT_PROXY_CAPACITY), "The max number of viewer sessions console proxy is configured to serve for", null), ConsoleProxySessionTimeout("Console Proxy", AgentManager.class, Integer.class, "consoleproxy.session.timeout", "300000", "Timeout(in milliseconds) that console proxy tries to maintain a viewer session before it times out the session for no activity", null), @@ -185,7 +185,7 @@ public enum Config { MaxTemplateAndIsoSize("Advanced", ManagementServer.class, Long.class, "max.template.iso.size", "50", "The maximum size for a downloaded template or ISO (in GB).", null), SecStorageAllowedInternalDownloadSites("Advanced", ManagementServer.class, String.class, "secstorage.allowed.internal.sites", null, "Comma separated list of cidrs internal to the datacenter that can host template download servers, please note 0.0.0.0 is not a valid site", null), SecStorageEncryptCopy("Advanced", ManagementServer.class, Boolean.class, "secstorage.encrypt.copy", "false", "Use SSL method used to encrypt copy traffic between zones", "true,false"), - SecStorageSecureCopyCert("Advanced", ManagementServer.class, String.class, "secstorage.ssl.cert.domain", "realhostip.com", "SSL certificate used to encrypt copy traffic between zones", null), + SecStorageSecureCopyCert("Advanced", ManagementServer.class, String.class, "secstorage.ssl.cert.domain", "", "SSL certificate used to encrypt copy traffic between zones", null), SecStorageCapacityStandby("Advanced", AgentManager.class, Integer.class, "secstorage.capacity.standby", "10", "The minimal number of command execution sessions that system is able to serve immediately(standby capacity)", null), SecStorageSessionMax("Advanced", AgentManager.class, Integer.class, "secstorage.session.max", "50", "The max number of command execution sessions that a SSVM can handle", null), SecStorageCmdExecutionTimeMax("Advanced", AgentManager.class, Integer.class, "secstorage.cmd.execution.time.max", "30", "The max command execution time in minute", null), diff --git a/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java b/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java index c8769d43349..fa5d682a617 100755 --- a/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java +++ b/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java @@ -249,6 +249,7 @@ VirtualMachineGuru, SystemVmLoadScanHandler, ResourceStateAdapter { private int _proxySessionTimeoutValue = DEFAULT_PROXY_SESSION_TIMEOUT; private boolean _sslEnabled = true; + private String _consoleProxyUrlDomain; // global load picture at zone basis private SystemVmLoadScanner _loadScanner; @@ -402,9 +403,9 @@ VirtualMachineGuru, SystemVmLoadScanHandler, ResourceStateAdapter { assert (ksVo != null); if (_staticPublicIp == null) { - return new ConsoleProxyInfo(proxy.isSslEnabled(), proxy.getPublicIpAddress(), _consoleProxyPort, proxy.getPort(), ksVo.getDomainSuffix()); + return new ConsoleProxyInfo(proxy.isSslEnabled(), proxy.getPublicIpAddress(), _consoleProxyPort, proxy.getPort(), _consoleProxyUrlDomain); } else { - return new ConsoleProxyInfo(proxy.isSslEnabled(), _staticPublicIp, _consoleProxyPort, _staticPort, ksVo.getDomainSuffix()); + return new ConsoleProxyInfo(proxy.isSslEnabled(), _staticPublicIp, _consoleProxyPort, _staticPort, _consoleProxyUrlDomain); } } @@ -1220,7 +1221,11 @@ VirtualMachineGuru, SystemVmLoadScanHandler, ResourceStateAdapter { if (value != null && value.equalsIgnoreCase("true")) { _sslEnabled = true; } - + _consoleProxyUrlDomain = configs.get(Config.ConsoleProxyUrlDomain.key()); + if( _sslEnabled && (_consoleProxyUrlDomain == null || _consoleProxyUrlDomain.isEmpty())) { + s_logger.warn("Empty console proxy domain, explicitly disabling SSL"); + _sslEnabled = false; + } value = configs.get(Config.ConsoleProxyCapacityScanInterval.key()); _capacityScanInterval = NumbersUtil.parseLong(value, DEFAULT_CAPACITY_SCAN_INTERVAL); diff --git a/server/src/com/cloud/storage/secondary/SecondaryStorageManagerImpl.java b/server/src/com/cloud/storage/secondary/SecondaryStorageManagerImpl.java index 611550e9083..88b3908f720 100755 --- a/server/src/com/cloud/storage/secondary/SecondaryStorageManagerImpl.java +++ b/server/src/com/cloud/storage/secondary/SecondaryStorageManagerImpl.java @@ -812,6 +812,13 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements Secondar _useSSlCopy = true; } + //default to HTTP in case of missing domain + String ssvmUrlDomain = _configDao.getValue("secstorage.ssl.cert.domain"); + if(_useSSlCopy && (ssvmUrlDomain == null || ssvmUrlDomain.isEmpty())){ + s_logger.warn("Empty secondary storage url domain, explicitly disabling SSL"); + _useSSlCopy = false; + } + _allowedInternalSites = _configDao.getValue("secstorage.allowed.internal.sites"); String value = configs.get("secstorage.capacityscan.interval"); diff --git a/setup/db/db/schema-421to430.sql b/setup/db/db/schema-421to430.sql index 3811a5a5e05..6a8ec494880 100644 --- a/setup/db/db/schema-421to430.sql +++ b/setup/db/db/schema-421to430.sql @@ -110,6 +110,10 @@ CREATE TABLE `cloud`.`async_job_join_map` ( INDEX `i_async_job_join_map__expiration`(`expiration`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; +#realhostip changes, before changing table and adding default value +UPDATE `cloud`.`configuration` SET value = CONCAT("*.",(SELECT `temptable`.`value` FROM (SELECT * FROM `cloud`.`configuration` WHERE `name`="consoleproxy.url.domain") AS `temptable` WHERE `temptable`.`name`="consoleproxy.url.domain")) WHERE `name`="consoleproxy.url.domain"; +UPDATE `cloud`.`configuration` SET `value` = CONCAT("*.",(SELECT `temptable`.`value` FROM (SELECT * FROM `cloud`.`configuration` WHERE `name`="secstorage.ssl.cert.domain") AS `temptable` WHERE `temptable`.`name`="secstorage.ssl.cert.domain")) WHERE `name`="secstorage.ssl.cert.domain"; + ALTER TABLE `cloud`.`configuration` ADD COLUMN `default_value` VARCHAR(4095) COMMENT 'Default value for a configuration parameter'; ALTER TABLE `cloud`.`configuration` ADD COLUMN `updated` datetime COMMENT 'Time this was updated by the server. null means this row is obsolete.'; ALTER TABLE `cloud`.`configuration` ADD COLUMN `scope` VARCHAR(255) DEFAULT NULL COMMENT 'Can this parameter be scoped'; diff --git a/systemvm/conf/consoleproxy.properties b/systemvm/conf/consoleproxy.properties index bb452f5823c..a3cddbcab96 100644 --- a/systemvm/conf/consoleproxy.properties +++ b/systemvm/conf/consoleproxy.properties @@ -16,7 +16,7 @@ # under the License. consoleproxy.tcpListenPort=0 -consoleproxy.httpListenPort=8088 +consoleproxy.httpListenPort=80 consoleproxy.httpCmdListenPort=8001 consoleproxy.jarDir=./applet/ consoleproxy.viewerLinger=180 From a253ff24682a7d405bfa8428a0833423bf19f9b9 Mon Sep 17 00:00:00 2001 From: Sheng Yang Date: Tue, 11 Mar 2014 16:13:30 -0700 Subject: [PATCH 145/169] CLOUDSTACK-5986: Fix dnsmasq lease for VPC --- .../debian/config/etc/init.d/cloud-early-config | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/systemvm/patches/debian/config/etc/init.d/cloud-early-config b/systemvm/patches/debian/config/etc/init.d/cloud-early-config index 77e0f17361f..9152df24cc7 100755 --- a/systemvm/patches/debian/config/etc/init.d/cloud-early-config +++ b/systemvm/patches/debian/config/etc/init.d/cloud-early-config @@ -1042,6 +1042,16 @@ EOF sed -r -i s/^[#]?domain=.*$/domain=$DOMAIN/ /etc/dnsmasq.conf #answer all local domain queries sed -i -e "s/^[#]*local=.*$/local=\/$DOMAIN\//" /etc/dnsmasq.conf + + command -v dhcp_release > /dev/null 2>&1 + no_dhcp_release=$? + if [ $no_dhcp_release -eq 0 ] + then + echo 1 > /var/cache/cloud/dnsmasq_managed_lease + sed -i -e "/^leasefile-ro/d" /etc/dnsmasq.conf + else + echo 0 > /var/cache/cloud/dnsmasq_managed_lease + fi } From 454cac448d83b973d8cd337cf214b17e31828b93 Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Wed, 12 Mar 2014 15:50:43 +0100 Subject: [PATCH 146/169] CLOUDSTACK-6231 allow for cidr list entry of more than 256 chars --- .../src/com/cloud/network/vpc/NetworkACLItemVO.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/engine/schema/src/com/cloud/network/vpc/NetworkACLItemVO.java b/engine/schema/src/com/cloud/network/vpc/NetworkACLItemVO.java index 46f84c9f617..f4b65b53085 100644 --- a/engine/schema/src/com/cloud/network/vpc/NetworkACLItemVO.java +++ b/engine/schema/src/com/cloud/network/vpc/NetworkACLItemVO.java @@ -61,7 +61,7 @@ public class NetworkACLItemVO implements NetworkACLItem { @Enumerated(value=EnumType.STRING) TrafficType trafficType; - @Column(name="cidr") + @Column(name = "cidr", length = 2048) String sourceCidrs; @Column(name="uuid") @@ -75,7 +75,7 @@ public class NetworkACLItemVO implements NetworkACLItem { Action action; public NetworkACLItemVO() { - this.uuid = UUID.randomUUID().toString(); + uuid = UUID.randomUUID().toString(); } public NetworkACLItemVO(Integer portStart, Integer portEnd, String protocol, @@ -85,11 +85,11 @@ public class NetworkACLItemVO implements NetworkACLItem { this.sourcePortEnd = portEnd; this.protocol = protocol; this.aclId = aclId; - this.state = State.Staged; + state = State.Staged; this.icmpCode = icmpCode; this.icmpType = icmpType; setSourceCidrList(sourceCidrs); - this.uuid = UUID.randomUUID().toString(); + uuid = UUID.randomUUID().toString(); this.trafficType = trafficType; this.action = action; this.number = number; @@ -181,7 +181,7 @@ public class NetworkACLItemVO implements NetworkACLItem { @Override public String getUuid() { - return this.uuid; + return uuid; } @Override From f78e7ae51f55da17ed5ba239f99457ee8bc7c4d8 Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Wed, 5 Mar 2014 20:16:55 +0100 Subject: [PATCH 147/169] [CLOUDSTACK-6232] bridging allowed in isolated networks --- server/src/com/cloud/network/NetworkServiceImpl.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/server/src/com/cloud/network/NetworkServiceImpl.java b/server/src/com/cloud/network/NetworkServiceImpl.java index 8dc63bb1637..d31bb1df5d8 100755 --- a/server/src/com/cloud/network/NetworkServiceImpl.java +++ b/server/src/com/cloud/network/NetworkServiceImpl.java @@ -40,6 +40,8 @@ import javax.ejb.Local; import javax.inject.Inject; import javax.naming.ConfigurationException; +import org.apache.log4j.Logger; + import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.acl.SecurityChecker.AccessType; import org.apache.cloudstack.api.command.admin.network.DedicateGuestVlanRangeCmd; @@ -53,7 +55,6 @@ import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.network.element.InternalLoadBalancerElementService; -import org.apache.log4j.Logger; import com.cloud.api.ApiDBUtils; import com.cloud.configuration.Config; @@ -1237,7 +1238,10 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService { // Vlan is created in 1 cases - works in Advance zone only: // 1) GuestType is Shared - boolean createVlan = (startIP != null && endIP != null && zone.getNetworkType() == NetworkType.Advanced && (ntwkOff.getGuestType() == Network.GuestType.Shared)); + boolean createVlan = (startIP != null && endIP != null && zone.getNetworkType() == NetworkType.Advanced + && ((ntwkOff.getGuestType() == Network.GuestType.Shared) + || (ntwkOff.getGuestType() == GuestType.Isolated && + !areServicesSupportedByNetworkOffering(ntwkOff.getId(), Service.SourceNat)))); if (!createVlan) { // Only support advance shared network in IPv6, which means createVlan is a must @@ -4005,7 +4009,8 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService { @Inject public void setNetworkGurus(List networkGurus) { - this._networkGurus = networkGurus; + _networkGurus = networkGurus; + } } From 0379dbb489b2502b8cf89583afd1a1f3baec93a3 Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Wed, 12 Mar 2014 18:09:36 +0100 Subject: [PATCH 148/169] check-style: tabs to spaces --- server/src/com/cloud/network/NetworkServiceImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/com/cloud/network/NetworkServiceImpl.java b/server/src/com/cloud/network/NetworkServiceImpl.java index d31bb1df5d8..8bae5842803 100755 --- a/server/src/com/cloud/network/NetworkServiceImpl.java +++ b/server/src/com/cloud/network/NetworkServiceImpl.java @@ -1240,8 +1240,8 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService { // 1) GuestType is Shared boolean createVlan = (startIP != null && endIP != null && zone.getNetworkType() == NetworkType.Advanced && ((ntwkOff.getGuestType() == Network.GuestType.Shared) - || (ntwkOff.getGuestType() == GuestType.Isolated && - !areServicesSupportedByNetworkOffering(ntwkOff.getId(), Service.SourceNat)))); + || (ntwkOff.getGuestType() == GuestType.Isolated && + !areServicesSupportedByNetworkOffering(ntwkOff.getId(), Service.SourceNat)))); if (!createVlan) { // Only support advance shared network in IPv6, which means createVlan is a must From edf97ac86c94741ae8964b640bdfc0234929c1e4 Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Wed, 12 Mar 2014 19:36:11 +0100 Subject: [PATCH 149/169] CLOUDSTACK-6232 allow expansion of ip on isolated networks as well --- .../com/cloud/configuration/ConfigurationManagerImpl.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/server/src/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/com/cloud/configuration/ConfigurationManagerImpl.java index 2e2c39faf3c..662eaae930d 100755 --- a/server/src/com/cloud/configuration/ConfigurationManagerImpl.java +++ b/server/src/com/cloud/configuration/ConfigurationManagerImpl.java @@ -2643,9 +2643,10 @@ ConfigurationManagerImpl extends ManagerBase implements ConfigurationManager, Co networkId = network.getId(); zoneId = network.getDataCenterId(); } - } else if (network.getGuestType() == null || network.getGuestType() == Network.GuestType.Isolated) { - throw new InvalidParameterValueException("Can't create direct vlan for network id=" + networkId - + " with type: " + network.getGuestType()); + } else if (network.getGuestType() == null || + (network.getGuestType() == Network.GuestType.Isolated + && _ntwkOffServiceMapDao.areServicesSupportedByNetworkOffering(network.getNetworkOfferingId(), Service.SourceNat))) { + throw new InvalidParameterValueException("Can't create direct vlan for network id=" + networkId + " with type: " + network.getGuestType()); } } From 15bf144f1a89ee024f087427a3c72cf233e49356 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Fri, 7 Mar 2014 15:36:26 +0530 Subject: [PATCH 150/169] CLOUDSTACK-6211: Xenserver - HA - SSVM fails to start due to running out of management Ip ranges when testing host down scenarios Signed-off-by: Koushik Das --- .../cloud/vm/VirtualMachineManagerImpl.java | 95 ++++++++++--------- 1 file changed, 49 insertions(+), 46 deletions(-) diff --git a/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java index be05cd96dcc..b30fc16c646 100755 --- a/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java @@ -1233,62 +1233,65 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac VirtualMachine vm = profile.getVirtualMachine(); State state = vm.getState(); s_logger.debug("Cleaning up resources for the vm " + vm + " in " + state + " state"); - if (state == State.Starting) { - Step step = work.getStep(); - if (step == Step.Starting && !cleanUpEvenIfUnableToStop) { - s_logger.warn("Unable to cleanup vm " + vm + "; work state is incorrect: " + step); - return false; - } + try { + if (state == State.Starting) { + Step step = work.getStep(); + if (step == Step.Starting && !cleanUpEvenIfUnableToStop) { + s_logger.warn("Unable to cleanup vm " + vm + "; work state is incorrect: " + step); + return false; + } - if (step == Step.Started || step == Step.Starting || step == Step.Release) { + if (step == Step.Started || step == Step.Starting || step == Step.Release) { + if (vm.getHostId() != null) { + if (!sendStop(guru, profile, cleanUpEvenIfUnableToStop)) { + s_logger.warn("Failed to stop vm " + vm + " in " + State.Starting + " state as a part of cleanup process"); + return false; + } + } + } + + if (step != Step.Release && step != Step.Prepare && step != Step.Started && step != Step.Starting) { + s_logger.debug("Cleanup is not needed for vm " + vm + "; work state is incorrect: " + step); + return true; + } + } else if (state == State.Stopping) { if (vm.getHostId() != null) { if (!sendStop(guru, profile, cleanUpEvenIfUnableToStop)) { - s_logger.warn("Failed to stop vm " + vm + " in " + State.Starting + " state as a part of cleanup process"); + s_logger.warn("Failed to stop vm " + vm + " in " + State.Stopping + " state as a part of cleanup process"); return false; } } + } else if (state == State.Migrating) { + if (vm.getHostId() != null) { + if (!sendStop(guru, profile, cleanUpEvenIfUnableToStop)) { + s_logger.warn("Failed to stop vm " + vm + " in " + State.Migrating + " state as a part of cleanup process"); + return false; + } + } + if (vm.getLastHostId() != null) { + if (!sendStop(guru, profile, cleanUpEvenIfUnableToStop)) { + s_logger.warn("Failed to stop vm " + vm + " in " + State.Migrating + " state as a part of cleanup process"); + return false; + } + } + } else if (state == State.Running) { + if (!sendStop(guru, profile, cleanUpEvenIfUnableToStop)) { + s_logger.warn("Failed to stop vm " + vm + " in " + State.Running + " state as a part of cleanup process"); + return false; + } + } + } finally { + try { + _networkMgr.release(profile, cleanUpEvenIfUnableToStop); + s_logger.debug("Successfully released network resources for the vm " + vm); + } catch (Exception e) { + s_logger.warn("Unable to release some network resources.", e); } - if (step != Step.Release && step != Step.Prepare && step != Step.Started && step != Step.Starting) { - s_logger.debug("Cleanup is not needed for vm " + vm + "; work state is incorrect: " + step); - return true; - } - } else if (state == State.Stopping) { - if (vm.getHostId() != null) { - if (!sendStop(guru, profile, cleanUpEvenIfUnableToStop)) { - s_logger.warn("Failed to stop vm " + vm + " in " + State.Stopping + " state as a part of cleanup process"); - return false; - } - } - } else if (state == State.Migrating) { - if (vm.getHostId() != null) { - if (!sendStop(guru, profile, cleanUpEvenIfUnableToStop)) { - s_logger.warn("Failed to stop vm " + vm + " in " + State.Migrating + " state as a part of cleanup process"); - return false; - } - } - if (vm.getLastHostId() != null) { - if (!sendStop(guru, profile, cleanUpEvenIfUnableToStop)) { - s_logger.warn("Failed to stop vm " + vm + " in " + State.Migrating + " state as a part of cleanup process"); - return false; - } - } - } else if (state == State.Running) { - if (!sendStop(guru, profile, cleanUpEvenIfUnableToStop)) { - s_logger.warn("Failed to stop vm " + vm + " in " + State.Running + " state as a part of cleanup process"); - return false; - } + volumeMgr.release(profile); + s_logger.debug("Successfully cleanued up resources for the vm " + vm + " in " + state + " state"); } - try { - _networkMgr.release(profile, cleanUpEvenIfUnableToStop); - s_logger.debug("Successfully released network resources for the vm " + vm); - } catch (Exception e) { - s_logger.warn("Unable to release some network resources.", e); - } - - volumeMgr.release(profile); - s_logger.debug("Successfully cleanued up resources for the vm " + vm + " in " + state + " state"); return true; } From 94817c1b4b89cfa69f2db50a7060dcb7d7c39dbb Mon Sep 17 00:00:00 2001 From: Alena Prokharchyk Date: Fri, 14 Mar 2014 11:43:40 -0700 Subject: [PATCH 151/169] CLOUDSTACK-6205: VPC VR start - skip private gateway when create Guest nics as the nic for the private gateway is being created separately --- api/src/com/cloud/network/NetworkModel.java | 4 ++-- server/src/com/cloud/network/NetworkModelImpl.java | 4 ++-- .../router/VpcVirtualNetworkApplianceManagerImpl.java | 5 ++++- server/test/com/cloud/network/MockNetworkModelImpl.java | 2 +- server/test/com/cloud/vpc/MockNetworkModelImpl.java | 2 +- 5 files changed, 10 insertions(+), 7 deletions(-) diff --git a/api/src/com/cloud/network/NetworkModel.java b/api/src/com/cloud/network/NetworkModel.java index 99ce1f5bd64..bf6ac9a591e 100644 --- a/api/src/com/cloud/network/NetworkModel.java +++ b/api/src/com/cloud/network/NetworkModel.java @@ -201,10 +201,10 @@ public interface NetworkModel { List getPhysicalNtwksSupportingTrafficType(long zoneId, TrafficType trafficType); /** - * @param guestNic + * @param ntwkId * @return */ - boolean isPrivateGateway(Nic guestNic); + boolean isPrivateGateway(long ntwkId); Map> getNetworkCapabilities(long networkId); diff --git a/server/src/com/cloud/network/NetworkModelImpl.java b/server/src/com/cloud/network/NetworkModelImpl.java index f64c68a4f38..1843d1cfcfa 100755 --- a/server/src/com/cloud/network/NetworkModelImpl.java +++ b/server/src/com/cloud/network/NetworkModelImpl.java @@ -1656,8 +1656,8 @@ public class NetworkModelImpl extends ManagerBase implements NetworkModel { } @Override - public boolean isPrivateGateway(Nic guestNic) { - Network network = getNetwork(guestNic.getNetworkId()); + public boolean isPrivateGateway(long ntwkId) { + Network network = getNetwork(ntwkId); if (network.getTrafficType() != TrafficType.Guest || network.getNetworkOfferingId() != _privateOfferingId.longValue()) { return false; } diff --git a/server/src/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java b/server/src/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java index c540eb7a35d..033d52c0f05 100644 --- a/server/src/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java +++ b/server/src/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java @@ -805,7 +805,7 @@ public class VpcVirtualNetworkApplianceManagerImpl extends VirtualNetworkApplian PlugNicCommand plugNicCmd = new PlugNicCommand(getNicTO(router, guestNic.getNetworkId(), null), router.getInstanceName(), router.getType()); cmds.addCommand(plugNicCmd); - if (!_networkModel.isPrivateGateway(guestNic)) { + if (!_networkModel.isPrivateGateway(guestNic.getNetworkId())) { //set guest network VirtualMachine vm = _vmDao.findById(router.getId()); NicProfile nicProfile = _networkModel.getNicProfile(vm, guestNic.getNetworkId(), null); @@ -1237,6 +1237,9 @@ public class VpcVirtualNetworkApplianceManagerImpl extends VirtualNetworkApplian //3) allocate nic for guest gateway if needed List guestNetworks = _vpcMgr.getVpcNetworks(vpcId); for (Network guestNetwork : guestNetworks) { + if (_networkModel.isPrivateGateway(guestNetwork.getId())) { + continue; + } if (guestNetwork.getState() == Network.State.Implemented || guestNetwork.getState() == Network.State.Setup) { NicProfile guestNic = createGuestNicProfileForVpcRouter(guestNetwork); networks.put(guestNetwork, new ArrayList(Arrays.asList(guestNic))); diff --git a/server/test/com/cloud/network/MockNetworkModelImpl.java b/server/test/com/cloud/network/MockNetworkModelImpl.java index 04c50c0ab65..74e0da867dc 100644 --- a/server/test/com/cloud/network/MockNetworkModelImpl.java +++ b/server/test/com/cloud/network/MockNetworkModelImpl.java @@ -596,7 +596,7 @@ public class MockNetworkModelImpl extends ManagerBase implements NetworkModel { * @see com.cloud.network.NetworkModel#isPrivateGateway(com.cloud.vm.Nic) */ @Override - public boolean isPrivateGateway(Nic guestNic) { + public boolean isPrivateGateway(long ntwkId) { // TODO Auto-generated method stub return false; } diff --git a/server/test/com/cloud/vpc/MockNetworkModelImpl.java b/server/test/com/cloud/vpc/MockNetworkModelImpl.java index e98253b534b..7361e498eab 100644 --- a/server/test/com/cloud/vpc/MockNetworkModelImpl.java +++ b/server/test/com/cloud/vpc/MockNetworkModelImpl.java @@ -610,7 +610,7 @@ public class MockNetworkModelImpl extends ManagerBase implements NetworkModel { * @see com.cloud.network.NetworkModel#isPrivateGateway(com.cloud.vm.Nic) */ @Override - public boolean isPrivateGateway(Nic guestNic) { + public boolean isPrivateGateway(long ntwkId) { // TODO Auto-generated method stub return false; } From 657d042953e40e3220c5b0b86511f3bf6a1e186f Mon Sep 17 00:00:00 2001 From: Sheng Yang Date: Wed, 12 Mar 2014 18:02:31 -0700 Subject: [PATCH 152/169] CLOUDSTACK-5986: Fix dnsmasq lease for VPC --- systemvm/patches/debian/config/root/edithosts.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/systemvm/patches/debian/config/root/edithosts.sh b/systemvm/patches/debian/config/root/edithosts.sh index 8e7ddac8fd8..d6ff983ad18 100755 --- a/systemvm/patches/debian/config/root/edithosts.sh +++ b/systemvm/patches/debian/config/root/edithosts.sh @@ -95,7 +95,7 @@ wait_for_dnsmasq () { return 1 } -if [ $dnsmasq_managed_lease ] +if [ $dnsmasq_managed_lease -eq 1 ] then #release previous dhcp lease if present logger -t cloud "edithosts: releasing $ipv4" @@ -212,7 +212,7 @@ pid=$(pidof dnsmasq) if [ "$pid" != "" ] then # use SIGHUP to avoid service outage if dhcp_release is available. - if [ $dnsmasq_managed_lease ] + if [ $dnsmasq_managed_lease -eq 1 ] then kill -HUP $pid else From e406adc0810b22ded0964080946f019be1337b52 Mon Sep 17 00:00:00 2001 From: Sheng Yang Date: Fri, 14 Mar 2014 13:36:43 -0700 Subject: [PATCH 153/169] CLOUDSTACK-6241: Fix default route for IPv6 network when IPv4 not used --- systemvm/patches/debian/config/root/edithosts.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/systemvm/patches/debian/config/root/edithosts.sh b/systemvm/patches/debian/config/root/edithosts.sh index d6ff983ad18..57ee7dc1d64 100755 --- a/systemvm/patches/debian/config/root/edithosts.sh +++ b/systemvm/patches/debian/config/root/edithosts.sh @@ -184,7 +184,7 @@ then echo "$ipv6 $host" >> $HOSTS fi -if [ "$dflt" != "" ] +if [ "$dflt" != "" -a "$ipv4" != "" ] then #make sure dnsmasq looks into options file sed -i /dhcp-optsfile/d /etc/dnsmasq.conf From e5c391fcf3852e50ebd99d4a72fd51d1753b05eb Mon Sep 17 00:00:00 2001 From: Edison Su Date: Fri, 14 Mar 2014 14:40:48 -0700 Subject: [PATCH 154/169] KVM security bug: no forwarding rule applied --- scripts/vm/network/security_group.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/vm/network/security_group.py b/scripts/vm/network/security_group.py index a7c64b0984e..f1ed76e631d 100755 --- a/scripts/vm/network/security_group.py +++ b/scripts/vm/network/security_group.py @@ -978,7 +978,7 @@ def addFWFramework(brname): execute("iptables -N " + brfwin) try: - refs = execute("""iptables -n -L " + brfw + " | awk '/%s(.*)references/ {gsub(/\(/, "") ;print $3}'""" % brfw).strip() + refs = execute("""iptables -n -L %s | awk '/%s(.*)references/ {gsub(/\(/, "") ;print $3}'""" % (brfw,brfw)).strip() if refs == "0": execute("iptables -I FORWARD -i " + brname + " -j DROP") execute("iptables -I FORWARD -o " + brname + " -j DROP") From d4fdc184fe9c6717d2ed4e4fe4c39d9759a90608 Mon Sep 17 00:00:00 2001 From: Edison Su Date: Mon, 17 Mar 2014 14:34:53 -0700 Subject: [PATCH 155/169] CLOUDSTACK-6245: the security group rule is lagging behind the rules in DB, due to there is a worker thread launched inside a transaction Reviewed-by: Alex --- .../security/SecurityGroupManagerImpl.java | 33 +++++++++++++------ 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/server/src/com/cloud/network/security/SecurityGroupManagerImpl.java b/server/src/com/cloud/network/security/SecurityGroupManagerImpl.java index e48bb344ecd..51c93b7f178 100755 --- a/server/src/com/cloud/network/security/SecurityGroupManagerImpl.java +++ b/server/src/com/cloud/network/security/SecurityGroupManagerImpl.java @@ -713,7 +713,7 @@ public class SecurityGroupManagerImpl extends ManagerBase implements SecurityGro final Integer startPortOrTypeFinal = startPortOrType; final Integer endPortOrCodeFinal = endPortOrCode; final String protocolFinal = protocol; - return Transaction.execute(new TransactionCallback>() { + List newRules = Transaction.execute(new TransactionCallback>() { @Override public List doInTransaction(TransactionStatus status) { // Prevents other threads/management servers from creating duplicate security rules @@ -756,9 +756,6 @@ public class SecurityGroupManagerImpl extends ManagerBase implements SecurityGro if (s_logger.isDebugEnabled()) { s_logger.debug("Added " + newRules.size() + " rules to security group " + securityGroup.getName()); } - final ArrayList affectedVms = new ArrayList(); - affectedVms.addAll(_securityGroupVMMapDao.listVmIdsBySecurityGroup(securityGroup.getId())); - scheduleRulesetUpdateToHosts(affectedVms, true, null); return newRules; } catch (Exception e) { s_logger.warn("Exception caught when adding security group rules ", e); @@ -771,6 +768,15 @@ public class SecurityGroupManagerImpl extends ManagerBase implements SecurityGro } }); + try { + final ArrayList affectedVms = new ArrayList(); + affectedVms.addAll(_securityGroupVMMapDao.listVmIdsBySecurityGroup(securityGroup.getId())); + scheduleRulesetUpdateToHosts(affectedVms, true, null); + } catch (Exception e) { + s_logger.debug("can't update rules on host, ignore", e); + } + + return newRules; } @Override @@ -810,7 +816,8 @@ public class SecurityGroupManagerImpl extends ManagerBase implements SecurityGro SecurityGroup securityGroup = _securityGroupDao.findById(rule.getSecurityGroupId()); _accountMgr.checkAccess(caller, null, true, securityGroup); - return Transaction.execute(new TransactionCallback() { + long securityGroupId = rule.getSecurityGroupId(); + Boolean result = Transaction.execute(new TransactionCallback() { @Override public Boolean doInTransaction(TransactionStatus status) { SecurityGroupVO groupHandle = null; @@ -825,11 +832,7 @@ public class SecurityGroupManagerImpl extends ManagerBase implements SecurityGro _securityGroupRuleDao.remove(id); s_logger.debug("revokeSecurityGroupRule succeeded for security rule id: " + id); - - final ArrayList affectedVms = new ArrayList(); - affectedVms.addAll(_securityGroupVMMapDao.listVmIdsBySecurityGroup(groupHandle.getId())); - scheduleRulesetUpdateToHosts(affectedVms, true, null); - + return true; } catch (Exception e) { s_logger.warn("Exception caught when deleting security rules ", e); @@ -841,6 +844,16 @@ public class SecurityGroupManagerImpl extends ManagerBase implements SecurityGro } } }); + + try { + final ArrayList affectedVms = new ArrayList(); + affectedVms.addAll(_securityGroupVMMapDao.listVmIdsBySecurityGroup(securityGroupId)); + scheduleRulesetUpdateToHosts(affectedVms, true, null); + } catch (Exception e) { + s_logger.debug("Can't update rules for host, ignore", e); + } + + return result; } @Override From 33476b87351d10252911a4191986d5a5342d5597 Mon Sep 17 00:00:00 2001 From: Jessica Wang Date: Mon, 17 Mar 2014 15:41:48 -0700 Subject: [PATCH 156/169] CLOUDSTACK-6246: UI > Infrastructure > SSL Certificate > update (1) mouse-over hint of SSL Certificate button. (2) description in SSL Certificate dialog. --- client/WEB-INF/classes/resources/messages.properties | 2 +- ui/index.jsp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/client/WEB-INF/classes/resources/messages.properties b/client/WEB-INF/classes/resources/messages.properties index 403225397c1..bccf71d2b25 100644 --- a/client/WEB-INF/classes/resources/messages.properties +++ b/client/WEB-INF/classes/resources/messages.properties @@ -1567,7 +1567,7 @@ message.tooltip.reserved.system.netmask=The network prefix that defines the pod message.tooltip.zone.name=A name for the zone. message.update.os.preference=Please choose a OS preference for this host. All virtual instances with similar preferences will be first allocated to this host before choosing another. message.update.resource.count=Please confirm that you want to update resource counts for this account. -message.update.ssl=Please submit a new X.509 compliant SSL certificate to be updated to each console proxy virtual instance\: +message.update.ssl=Please submit a new X.509 compliant SSL certificate to be updated to each console proxy and secondary storage virtual instance\: message.validate.instance.name=Instance name can not be longer than 63 characters. Only ASCII letters a~z, A~Z, digits 0~9, hyphen are allowed. Must start with a letter and end with a letter or a digit. message.virtual.network.desc=A dedicated virtualized network for your account. The broadcast domain is contained within a VLAN and all public network access is routed out by a virtual router. message.vm.create.template.confirm=Create Template will reboot the VM automatically. diff --git a/ui/index.jsp b/ui/index.jsp index 096747b10ac..0cd5dc5a7f8 100644 --- a/ui/index.jsp +++ b/ui/index.jsp @@ -1044,7 +1044,7 @@
-
+
 
From d417ce0d70c9337748ba6598332d04c7581942d2 Mon Sep 17 00:00:00 2001 From: Devdeep Singh Date: Tue, 4 Mar 2014 15:41:55 +0530 Subject: [PATCH 157/169] CLOUDSTACK-5743. The link generated for downlading a volume or a template was giving a permission error on hyper-v. Fixed the path with which a softlink is created for a downloaded volume or template. Also addressed the issue at other places where the path field wasn't set correctly when a volume was migrated or a template was created from a volume. --- .../storage/volume/VolumeServiceImpl.java | 10 +++- .../HypervResource/CloudStackTypes.cs | 47 ++++++++++++------- .../HypervResourceController.cs | 29 ++++++++++-- 3 files changed, 63 insertions(+), 23 deletions(-) diff --git a/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java b/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java index 7796565dadd..d221fa970ce 100644 --- a/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java +++ b/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java @@ -75,6 +75,7 @@ import com.cloud.exception.ResourceAllocationException; import com.cloud.host.Host; import com.cloud.storage.DataStoreRole; import com.cloud.storage.ScopeType; +import com.cloud.storage.Storage.StoragePoolType; import com.cloud.storage.StoragePool; import com.cloud.storage.VMTemplateStoragePoolVO; import com.cloud.storage.VMTemplateStorageResourceAssoc; @@ -679,9 +680,16 @@ public class VolumeServiceImpl implements VolumeService { protected VolumeVO duplicateVolumeOnAnotherStorage(Volume volume, StoragePool pool) { Long lastPoolId = volume.getPoolId(); + String folder = pool.getPath(); + // For SMB, pool credentials are also stored in the uri query string. We trim the query string + // part here to make sure the credentials do not get stored in the db unencrypted. + if (pool.getPoolType() == StoragePoolType.SMB && folder != null && folder.contains("?")) { + folder = folder.substring(0, folder.indexOf("?")); + } + VolumeVO newVol = new VolumeVO(volume); newVol.setPoolId(pool.getId()); - newVol.setFolder(pool.getPath()); + newVol.setFolder(folder); newVol.setPodId(pool.getPodId()); newVol.setPoolId(pool.getId()); newVol.setLastPoolId(lastPoolId); diff --git a/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/CloudStackTypes.cs b/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/CloudStackTypes.cs index 541c2053172..b6989a97702 100644 --- a/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/CloudStackTypes.cs +++ b/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/CloudStackTypes.cs @@ -173,33 +173,35 @@ namespace HypervResource PrimaryDataStoreTO store = this.primaryDataStore; if (store.isLocal) { - fileName = Path.Combine(store.Path, this.uuid); + String volume = this.path; + if (String.IsNullOrEmpty(volume)) + { + volume = this.uuid; + } + fileName = Path.Combine(store.Path, volume); } else { - fileName = @"\\" + store.uri.Host + store.uri.LocalPath + @"\" + this.uuid; + String volume = this.path; + if (String.IsNullOrEmpty(volume)) + { + volume = this.uuid; + } + fileName = @"\\" + store.uri.Host + store.uri.LocalPath + @"\" + volume; fileName = Utils.NormalizePath(fileName); } } else if (this.nfsDataStore != null) { - if (this.path != null && File.Exists(this.path)) + fileName = this.nfsDataStore.UncPath; + if (this.path != null) { - fileName = this.path; + fileName = Utils.NormalizePath(fileName + @"\" + this.path); } - else - { - fileName = this.nfsDataStore.UncPath; - if (this.path != null) - { - fileName += @"\" + this.path; - } - fileName = Utils.NormalizePath(fileName); - if (Directory.Exists(fileName)) - { - fileName = Utils.NormalizePath(fileName + @"\" + this.uuid); - } + if (fileName != null && !File.Exists(fileName)) + { + fileName = Utils.NormalizePath(fileName + @"\" + this.uuid); } } else @@ -344,8 +346,17 @@ namespace HypervResource } else if (this.nfsDataStoreTO != null) { - NFSTO store = this.nfsDataStoreTO; - fileName = store.UncPath + @"\" + this.path + @"\" + this.uuid; + fileName = this.nfsDataStoreTO.UncPath; + if (this.path != null) + { + fileName = Utils.NormalizePath(fileName + @"\" + this.path); + } + + if (fileName != null && !File.Exists(fileName)) + { + fileName = Utils.NormalizePath(fileName + @"\" + this.uuid); + } + if (!this.format.Equals("RAW")) { fileName = fileName + '.' + this.format.ToLowerInvariant(); diff --git a/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/HypervResourceController.cs b/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/HypervResourceController.cs index 8fd15f4c258..5d9f2afc810 100644 --- a/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/HypervResourceController.cs +++ b/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/HypervResourceController.cs @@ -1169,7 +1169,7 @@ namespace HypervResource volumePath = Utils.NormalizePath(volumePath); Utils.ConnectToRemote(primary.UncPath, primary.Domain, primary.User, primary.Password); } - volume.path = volumePath; + volume.path = volume.uuid; wmiCallsV2.CreateDynamicVirtualHardDisk(volumeSize, volumePath); if (File.Exists(volumePath)) { @@ -1478,7 +1478,7 @@ namespace HypervResource { // TODO: thin provision instead of copying the full file. File.Copy(srcFile, destFile); - destVolumeObjectTO.path = destFile; + destVolumeObjectTO.path = destVolumeObjectTO.uuid; JObject ansObj = Utils.CreateCloudStackObject(CloudStackTypes.VolumeObjectTO, destVolumeObjectTO); newData = ansObj; result = true; @@ -1512,8 +1512,25 @@ namespace HypervResource // doesn't do anything if the directory is already present. Directory.CreateDirectory(Path.GetDirectoryName(destFile)); File.Copy(srcFile, destFile); + + if (srcVolumeObjectTO.nfsDataStore != null && srcVolumeObjectTO.primaryDataStore == null) + { + logger.Info("Copied volume from secondary data store to primary. Path: " + destVolumeObjectTO.path); + } + else if (srcVolumeObjectTO.primaryDataStore != null && srcVolumeObjectTO.nfsDataStore == null) + { + destVolumeObjectTO.path = destVolumeObjectTO.path + "/" + destVolumeObjectTO.uuid; + if (destVolumeObjectTO.format != null) + { + destVolumeObjectTO.path += "." + destVolumeObjectTO.format.ToLower(); + } + } + else + { + logger.Error("Destination volume path wasn't set. Unsupported source volume data store."); + } + // Create volumeto object deserialize and send it - destVolumeObjectTO.path = destFile; JObject ansObj = Utils.CreateCloudStackObject(CloudStackTypes.VolumeObjectTO, destVolumeObjectTO); newData = ansObj; result = true; @@ -1556,7 +1573,11 @@ namespace HypervResource TemplateObjectTO destTemplateObject = new TemplateObjectTO(); destTemplateObject.size = srcVolumeObjectTO.size.ToString(); destTemplateObject.format = srcVolumeObjectTO.format; - destTemplateObject.path = destFile; + destTemplateObject.path = destTemplateObjectTO.path + "/" + destTemplateObjectTO.uuid; + if (destTemplateObject.format != null) + { + destTemplateObject.path += "." + destTemplateObject.format.ToLower(); + } destTemplateObject.nfsDataStoreTO = destTemplateObjectTO.nfsDataStoreTO; destTemplateObject.checksum = destTemplateObjectTO.checksum; newData = destTemplateObject; From a6e33143c215aad668bc4ba625dfe4c840cdb53b Mon Sep 17 00:00:00 2001 From: Sheng Yang Date: Tue, 25 Mar 2014 15:55:41 -0700 Subject: [PATCH 158/169] CLOUDSTACK-6285: Fix savepassword.sh script for clear out old entries --- systemvm/patches/debian/config/opt/cloud/bin/savepassword.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/savepassword.sh b/systemvm/patches/debian/config/opt/cloud/bin/savepassword.sh index 1ea27e5b702..21fa09d5b7e 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/savepassword.sh +++ b/systemvm/patches/debian/config/opt/cloud/bin/savepassword.sh @@ -52,7 +52,7 @@ done [ -f $PASSWD_FILE ] || touch $PASSWD_FILE -sed -i /$VM_IP/d $PASSWD_FILE +sed -i /$VM_IP=/d $PASSWD_FILE ps aux | grep serve_password.sh |grep -v grep 2>&1 > /dev/null if [ $? -eq 0 ] From d042a8183760d85b0f05432267bef4a8b5e2e2d0 Mon Sep 17 00:00:00 2001 From: Anshul Gangwar Date: Wed, 2 Apr 2014 01:07:32 -0700 Subject: [PATCH 159/169] CLOUDSTACK-6325: [hyper-v] fixed cleaning of bin and obj directories when building with mono, they were not cleaning up and resulting in use of stale dlls in some cases --- .../hyperv/DotNet/ServerResource/AgentShell/AgentShell.csproj | 4 ++++ .../ServerResource/HypervResource/HypervResource.csproj | 4 ++++ .../ServerResource.Tests/ServerResource.Tests.csproj | 4 ++++ .../DotNet/ServerResource/WmiWrappers/WmiWrappers.csproj | 4 ++++ 4 files changed, 16 insertions(+) diff --git a/plugins/hypervisors/hyperv/DotNet/ServerResource/AgentShell/AgentShell.csproj b/plugins/hypervisors/hyperv/DotNet/ServerResource/AgentShell/AgentShell.csproj index a4c6b1fc073..f804ef6f670 100644 --- a/plugins/hypervisors/hyperv/DotNet/ServerResource/AgentShell/AgentShell.csproj +++ b/plugins/hypervisors/hyperv/DotNet/ServerResource/AgentShell/AgentShell.csproj @@ -137,6 +137,10 @@ + + + +