diff --git a/CHANGES b/CHANGES index 054e7b06615..624ef906622 100644 --- a/CHANGES +++ b/CHANGES @@ -7,7 +7,7 @@ http://cloudstack.apache.org/docs Version 4.2.0 ------------------------ -In progress +Please check the release notes for details diff --git a/LICENSE b/LICENSE index 2094d029e90..c970ff13924 100644 --- a/LICENSE +++ b/LICENSE @@ -306,7 +306,7 @@ Within the scripts/vm/hypervisor/xenserver directory from OpenStack, LLC http://www.openstack.org swift -Within the tools/appliance/definitions/{devcloud,systemvmtemplate,systemvmtemplate64} directories +Within the tools/appliance/definitions/{devcloud,systemvmtemplate,systemvmtemplate64} directory licensed under the MIT License http://www.opensource.org/licenses/mit-license.php (as follows) Copyright (c) 2010-2012 Patrick Debois @@ -460,7 +460,7 @@ Within the ui/lib directory licensed under the MIT License http://www.opensource.org/licenses/mit-license.php (as follows) - Copyright (c) 2006 - 2011 Jörn Zaefferer + Copyright (c) 2006 - 2011 Jörn Zaefferer Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the @@ -655,7 +655,7 @@ Within the ui/lib/jquery-ui directory Within the ui/lib/qunit directory licensed under the MIT License http://www.opensource.org/licenses/mit-license.php (as follows) - Copyright (c) 2012 John Resig, Jörn Zaefferer + Copyright (c) 2012 John Resig, Jörn Zaefferer Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the @@ -686,3 +686,10 @@ Within the utils/src/com/cloud/utils/db directory from Clinton Begin http://code.google.com/p/mybatis/ ScriptRunner.java from http://code.google.com/p/mybatis/ +Within the utils/src/org/apache/commons/httpclient/contrib/ssl directory + licensed under the Apache License, Version 2 http://www.apache.org/licenses/LICENSE-2.0.txt (as above) + Copyright (c) 2007 The Apache Software Foundation + from The Apache Software Foundation http://www.apache.org/ + EasySSLProtocolSocketFactory.java + EasyX509TrustManager.java + diff --git a/agent-simulator/tomcatconf/commands-simulator.properties.in b/agent-simulator/tomcatconf/commands-simulator.properties.in index a0c13013c44..ba19e33dc5f 100644 --- a/agent-simulator/tomcatconf/commands-simulator.properties.in +++ b/agent-simulator/tomcatconf/commands-simulator.properties.in @@ -16,4 +16,4 @@ # under the License. -configureSimulator=com.cloud.api.commands.ConfigureSimulator;1 +configureSimulator=com.cloud.api.commands.ConfigureSimulatorCmd;1 diff --git a/agent/bindir/cloudstack-agent-upgrade.in b/agent/bindir/cloudstack-agent-upgrade.in new file mode 100644 index 00000000000..4972d3901fe --- /dev/null +++ b/agent/bindir/cloudstack-agent-upgrade.in @@ -0,0 +1,50 @@ +#!/usr/bin/python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from cloudutils.networkConfig import networkConfig +from cloudutils.utilities import bash +def isOldStyleBridge(brName): + if brName.find("cloudVirBr") == 0: + return True + else: + return False +def upgradeBridgeName(brName, enslavedDev): + print("upgrade bridge: %s, %s"%(brName, enslavedDev)) + vlanId = brName.replace("cloudVirBr", "") + print("find vlan Id: %s"%vlanId) + phyDev = enslavedDev.split(".")[0] + print("find physical device %s"%phyDev) + newBrName = "br" + phyDev + "-" + vlanId + print("new bridge name %s"%newBrName) + bash("ip link set %s down"%brName) + bash("ip link set %s name %s"%(brName, newBrName)) + bash("ip link set %s up" %newBrName) +if __name__ == '__main__': + netlib = networkConfig() + bridges = netlib.listNetworks() + bridges = filter(isOldStyleBridge, bridges) + for br in bridges: + enslavedDev = netlib.getEnslavedDev(br, 1) + if enslavedDev is not None: + upgradeBridgeName(br, enslavedDev) + + bridges = netlib.listNetworks() + bridges = filter(isOldStyleBridge, bridges) + if len(bridges) > 0: + print("Warning: upgrade is not finished, still some bridges have the old style name:" + str(bridges)) + else: + print("Upgrade succeed") diff --git a/agent/bindir/libvirtqemuhook.in b/agent/bindir/libvirtqemuhook.in new file mode 100755 index 00000000000..7bf9634fdf5 --- /dev/null +++ b/agent/bindir/libvirtqemuhook.in @@ -0,0 +1,53 @@ +#!/usr/bin/python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import sys +from xml.dom.minidom import parse +from cloudutils.configFileOps import configFileOps +from cloudutils.networkConfig import networkConfig +def isOldStyleBridge(brName): + if brName.find("cloudVirBr") == 0: + return True + else: + return False +def getGuestNetworkDevice(): + netlib = networkConfig() + cfo = configFileOps("/etc/cloudstack/agent/agent.properties") + guestDev = cfo.getEntry("guest.network.device") + enslavedDev = netlib.getEnslavedDev(guestDev, 1) + return enslavedDev +def handleMigrateBegin(): + try: + domain = parse(sys.stdin) + for interface in domain.getElementsByTagName("interface"): + source = interface.getElementsByTagName("source")[0] + bridge = source.getAttribute("bridge") + if not isOldStyleBridge(bridge): + continue + vlanId = bridge.replace("cloudVirBr","") + phyDev = getGuestNetworkDevice() + newBrName="br" + phyDev + "-" + vlanId + source.setAttribute("bridge", newBrName) + print(domain.toxml()) + except: + pass +if __name__ == '__main__': + if len(sys.argv) != 5: + sys.exit(0) + + if sys.argv[2] == "migrate" and sys.argv[3] == "begin": + handleMigrateBegin() diff --git a/agent/conf/agent.properties b/agent/conf/agent.properties index 60030ae4f11..5f5f3682afd 100644 --- a/agent/conf/agent.properties +++ b/agent/conf/agent.properties @@ -94,3 +94,23 @@ domr.scripts.dir=scripts/network/domr/kvm # libvirt.vif.driver=com.cloud.hypervisor.kvm.resource.DirectVifDriver # network.direct.source.mode=private # network.direct.device=eth0 + +# setting to enable the cpu model to kvm guest globally. +# three option:custom,host-model and host-passthrough. +# custom - user custom the CPU model which specified by guest.cpu.model. +# host-model - identify the named CPU model which most closely matches the host, +# and then request additional CPU flags to complete the match. This should give +# close to maximum functionality/performance, which maintaining good +# reliability/compatibility if the guest is migrated to another host with slightly different host CPUs. +# host-passthrough - tell KVM to passthrough the host CPU with no modifications. +# The difference to host-model, instead of just matching feature flags, +# every last detail of the host CPU is matched. This gives absolutely best performance, +# and can be important to some apps which check low level CPU details, +# but it comes at a cost wrt migration. The guest can only be migrated to +# an exactly matching host CPU. +# +# guest.cpu.mode=custom|host-model|host-passthrough +# This param is only valid if guest.cpu.mode=custom, +# for examples:"Conroe" "Penryn", "Nehalem", "Westmere", "pentiumpro" and so +# on,run virsh capabilities for more details. +# guest.cpu.model= diff --git a/agent/pom.xml b/agent/pom.xml index c2b1502728f..62dab0329cb 100644 --- a/agent/pom.xml +++ b/agent/pom.xml @@ -23,7 +23,7 @@ org.apache.cloudstack cloudstack - 4.2.0-SNAPSHOT + 4.2.0 diff --git a/api/pom.xml b/api/pom.xml index 8ca258f12e3..2d13a63651c 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -23,7 +23,7 @@ org.apache.cloudstack cloudstack - 4.2.0-SNAPSHOT + 4.2.0 diff --git a/api/src/com/cloud/agent/api/to/VolumeTO.java b/api/src/com/cloud/agent/api/to/VolumeTO.java index a5681a003dd..fb08384187e 100644 --- a/api/src/com/cloud/agent/api/to/VolumeTO.java +++ b/api/src/com/cloud/agent/api/to/VolumeTO.java @@ -126,6 +126,10 @@ public class VolumeTO implements InternalIdentity { public String getChainInfo() { return chainInfo; } + + public void setChainInfo(String chainInfo) { + this.chainInfo = chainInfo; + } public String getOsType() { return guestOsType; diff --git a/api/src/com/cloud/dc/DedicatedResources.java b/api/src/com/cloud/dc/DedicatedResources.java index e8e5ab3dffc..b3aea50b893 100755 --- a/api/src/com/cloud/dc/DedicatedResources.java +++ b/api/src/com/cloud/dc/DedicatedResources.java @@ -29,5 +29,5 @@ public interface DedicatedResources extends InfrastructureEntity, InternalIdenti Long getDomainId(); Long getAccountId(); String getUuid(); - + long getAffinityGroupId(); } diff --git a/api/src/com/cloud/deploy/DeploymentPlanner.java b/api/src/com/cloud/deploy/DeploymentPlanner.java index 769da39f3ff..32e4f9efce3 100644 --- a/api/src/com/cloud/deploy/DeploymentPlanner.java +++ b/api/src/com/cloud/deploy/DeploymentPlanner.java @@ -192,6 +192,13 @@ public interface DeploymentPlanner extends Adapter { _podIds.add(podId); } + public void addPodList(Collection podList) { + if (_podIds == null) { + _podIds = new HashSet(); + } + _podIds.addAll(podList); + } + public void addCluster(long clusterId) { if (_clusterIds == null) { _clusterIds = new HashSet(); diff --git a/api/src/com/cloud/network/NetworkService.java b/api/src/com/cloud/network/NetworkService.java index 59ccdbf754d..87fecb0f873 100755 --- a/api/src/com/cloud/network/NetworkService.java +++ b/api/src/com/cloud/network/NetworkService.java @@ -87,7 +87,7 @@ public interface NetworkService { Long startIndex, Long pageSize, String name); PhysicalNetwork updatePhysicalNetwork(Long id, String networkSpeed, List tags, - String newVnetRangeString, String state, String removeVlan); + String newVnetRangeString, String state); boolean deletePhysicalNetwork(Long id); diff --git a/api/src/com/cloud/network/vpc/VpcGateway.java b/api/src/com/cloud/network/vpc/VpcGateway.java index 5d278e952ed..9652b4b467e 100644 --- a/api/src/com/cloud/network/vpc/VpcGateway.java +++ b/api/src/com/cloud/network/vpc/VpcGateway.java @@ -56,7 +56,7 @@ public interface VpcGateway extends Identity, ControlledEntity, InternalIdentity /** * @return */ - Long getNetworkId(); + long getNetworkId(); /** * @return diff --git a/api/src/com/cloud/storage/S3.java b/api/src/com/cloud/storage/S3.java deleted file mode 100644 index 0c58a902923..00000000000 --- a/api/src/com/cloud/storage/S3.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package com.cloud.storage; - -import java.util.Date; - -import org.apache.cloudstack.api.Identity; -import org.apache.cloudstack.api.InternalIdentity; - -import com.cloud.agent.api.to.S3TO; - -public interface S3 extends InternalIdentity, Identity { - - String getAccessKey(); - - String getSecretKey(); - - String getEndPoint(); - - String getBucketName(); - - Integer getHttpsFlag(); - - Integer getConnectionTimeout(); - - Integer getMaxErrorRetry(); - - Integer getSocketTimeout(); - - Date getCreated(); - - S3TO toS3TO(); - -} diff --git a/api/src/com/cloud/storage/VolumeApiService.java b/api/src/com/cloud/storage/VolumeApiService.java index 95f962df374..0194c817cac 100644 --- a/api/src/com/cloud/storage/VolumeApiService.java +++ b/api/src/com/cloud/storage/VolumeApiService.java @@ -20,6 +20,7 @@ package com.cloud.storage; import java.net.URISyntaxException; +import com.cloud.exception.StorageUnavailableException; import org.apache.cloudstack.api.command.user.volume.*; import com.cloud.exception.ConcurrentOperationException; diff --git a/api/src/org/apache/cloudstack/affinity/AffinityGroup.java b/api/src/org/apache/cloudstack/affinity/AffinityGroup.java index ac2eb613370..c1ad11dbdd0 100644 --- a/api/src/org/apache/cloudstack/affinity/AffinityGroup.java +++ b/api/src/org/apache/cloudstack/affinity/AffinityGroup.java @@ -28,4 +28,6 @@ public interface AffinityGroup extends ControlledEntity, InternalIdentity, Ident String getType(); + ACLType getAclType(); + } diff --git a/api/src/org/apache/cloudstack/affinity/AffinityGroupProcessor.java b/api/src/org/apache/cloudstack/affinity/AffinityGroupProcessor.java index e3a9b62c6ce..c93fff9ec57 100644 --- a/api/src/org/apache/cloudstack/affinity/AffinityGroupProcessor.java +++ b/api/src/org/apache/cloudstack/affinity/AffinityGroupProcessor.java @@ -59,4 +59,34 @@ public interface AffinityGroupProcessor extends Adapter { */ boolean check(VirtualMachineProfile vm, DeployDestination plannedDestination) throws AffinityConflictException; + + /** + * isAdminControlledGroup() should return true if the affinity/anti-affinity + * group can only be operated on[create/delete/modify] by the Admin + * + * @return boolean true/false + */ + boolean isAdminControlledGroup(); + + + /** + * canBeSharedDomainWide() should return true if the affinity/anti-affinity + * group can be created for a domain and shared by all accounts under the + * domain. + * + * @return boolean true/false + */ + boolean canBeSharedDomainWide(); + + /** + * subDomainAccess() should return true if the affinity/anti-affinity group + * can be created for a domain and used by the sub-domains. If true, all + * accounts under the sub-domains can see this group and use it. + * + * @return boolean true/false + */ + boolean subDomainAccess(); + + void handleDeleteGroup(AffinityGroup group); + } \ No newline at end of file diff --git a/api/src/org/apache/cloudstack/affinity/AffinityGroupService.java b/api/src/org/apache/cloudstack/affinity/AffinityGroupService.java index 1b30e58d92f..0c4374c6877 100644 --- a/api/src/org/apache/cloudstack/affinity/AffinityGroupService.java +++ b/api/src/org/apache/cloudstack/affinity/AffinityGroupService.java @@ -75,4 +75,11 @@ public interface AffinityGroupService { boolean isAffinityGroupProcessorAvailable(String affinityGroupType); + boolean isAdminControlledGroup(AffinityGroup group); + + boolean isAffinityGroupAvailableInDomain(long affinityGroupId, long domainId); + + AffinityGroup createAffinityGroupInternal(String account, Long domainId, String affinityGroupName, + String affinityGroupType, String description); + } diff --git a/api/src/org/apache/cloudstack/affinity/AffinityProcessorBase.java b/api/src/org/apache/cloudstack/affinity/AffinityProcessorBase.java index 325ab80e047..8ffdc9e872a 100644 --- a/api/src/org/apache/cloudstack/affinity/AffinityProcessorBase.java +++ b/api/src/org/apache/cloudstack/affinity/AffinityProcessorBase.java @@ -48,4 +48,25 @@ public class AffinityProcessorBase extends AdapterBase implements AffinityGroupP throws AffinityConflictException { return true; } + + @Override + public boolean isAdminControlledGroup() { + return false; + } + + @Override + public boolean canBeSharedDomainWide() { + return false; + } + + @Override + public void handleDeleteGroup(AffinityGroup group) { + // TODO Auto-generated method stub + return; + } + + @Override + public boolean subDomainAccess() { + return false; + } } diff --git a/api/src/org/apache/cloudstack/api/ResponseGenerator.java b/api/src/org/apache/cloudstack/api/ResponseGenerator.java index d8d07cb56fb..3e7d4b53076 100644 --- a/api/src/org/apache/cloudstack/api/ResponseGenerator.java +++ b/api/src/org/apache/cloudstack/api/ResponseGenerator.java @@ -73,10 +73,8 @@ import com.cloud.projects.ProjectInvitation; import com.cloud.region.ha.GlobalLoadBalancerRule; import com.cloud.server.ResourceTag; import com.cloud.storage.GuestOS; -import com.cloud.storage.S3; import com.cloud.storage.Snapshot; import com.cloud.storage.StoragePool; -import com.cloud.storage.Swift; import com.cloud.storage.Volume; import com.cloud.storage.snapshot.SnapshotPolicy; import com.cloud.storage.snapshot.SnapshotSchedule; @@ -151,7 +149,6 @@ import org.apache.cloudstack.api.response.RemoteAccessVpnResponse; import org.apache.cloudstack.api.response.ResourceCountResponse; import org.apache.cloudstack.api.response.ResourceLimitResponse; import org.apache.cloudstack.api.response.ResourceTagResponse; -import org.apache.cloudstack.api.response.S3Response; import org.apache.cloudstack.api.response.SecurityGroupResponse; import org.apache.cloudstack.api.response.ServiceOfferingResponse; import org.apache.cloudstack.api.response.ServiceResponse; @@ -164,7 +161,6 @@ import org.apache.cloudstack.api.response.SnapshotScheduleResponse; import org.apache.cloudstack.api.response.StaticRouteResponse; import org.apache.cloudstack.api.response.StorageNetworkIpRangeResponse; import org.apache.cloudstack.api.response.StoragePoolResponse; -import org.apache.cloudstack.api.response.SwiftResponse; import org.apache.cloudstack.api.response.SystemVmInstanceResponse; import org.apache.cloudstack.api.response.SystemVmResponse; import org.apache.cloudstack.api.response.TemplatePermissionsResponse; @@ -349,10 +345,6 @@ public interface ResponseGenerator { SystemVmInstanceResponse createSystemVmInstanceResponse(VirtualMachine systemVM); - SwiftResponse createSwiftResponse(Swift swift); - - S3Response createS3Response(S3 result); - PhysicalNetworkResponse createPhysicalNetworkResponse(PhysicalNetwork result); ServiceResponse createNetworkServiceResponse(Service service); diff --git a/api/src/org/apache/cloudstack/api/command/admin/cluster/AddClusterCmd.java b/api/src/org/apache/cloudstack/api/command/admin/cluster/AddClusterCmd.java index c8b012a4824..e8cb10b770a 100644 --- a/api/src/org/apache/cloudstack/api/command/admin/cluster/AddClusterCmd.java +++ b/api/src/org/apache/cloudstack/api/command/admin/cluster/AddClusterCmd.java @@ -17,11 +17,9 @@ package org.apache.cloudstack.api.command.admin.cluster; -import com.cloud.exception.DiscoveryException; -import com.cloud.exception.InvalidParameterValueException; -import com.cloud.exception.ResourceInUseException; -import com.cloud.org.Cluster; -import com.cloud.user.Account; +import java.util.ArrayList; +import java.util.List; + import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; @@ -34,8 +32,10 @@ import org.apache.cloudstack.api.response.PodResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.log4j.Logger; -import java.util.ArrayList; -import java.util.List; +import com.cloud.exception.DiscoveryException; +import com.cloud.exception.ResourceInUseException; +import com.cloud.org.Cluster; +import com.cloud.user.Account; @APICommand(name = "addCluster", description="Adds a new cluster", responseObject=ClusterResponse.class) public class AddClusterCmd extends BaseCmd { @@ -87,10 +87,10 @@ public class AddClusterCmd extends BaseCmd { @Parameter(name = ApiConstants.VSWITCH_TYPE_PUBLIC_TRAFFIC, type = CommandType.STRING, required = false, description = "Type of virtual switch used for public traffic in the cluster. Allowed values are, vmwaresvs (for VMware standard vSwitch) and vmwaredvs (for VMware distributed vSwitch)") private String vSwitchTypePublicTraffic; - @Parameter(name = ApiConstants.VSWITCH_TYPE_GUEST_TRAFFIC, type = CommandType.STRING, required = false, description = "Name of virtual switch used for guest traffic in the cluster. This would override zone wide traffic label setting.") + @Parameter(name = ApiConstants.VSWITCH_NAME_GUEST_TRAFFIC, type = CommandType.STRING, required = false, description = "Name of virtual switch used for guest traffic in the cluster. This would override zone wide traffic label setting.") private String vSwitchNameGuestTraffic; - @Parameter(name = ApiConstants.VSWITCH_TYPE_PUBLIC_TRAFFIC, type = CommandType.STRING, required = false, description = "Name of virtual switch used for public traffic in the cluster. This would override zone wide traffic label setting.") + @Parameter(name = ApiConstants.VSWITCH_NAME_PUBLIC_TRAFFIC, type = CommandType.STRING, required = false, description = "Name of virtual switch used for public traffic in the cluster. This would override zone wide traffic label setting.") private String vSwitchNamePublicTraffic; public String getVSwitchTypeGuestTraffic() { diff --git a/api/src/org/apache/cloudstack/api/command/admin/ldap/LDAPConfigCmd.java b/api/src/org/apache/cloudstack/api/command/admin/ldap/LDAPConfigCmd.java index 6f8b7b1705b..38f58eca5f0 100644 --- a/api/src/org/apache/cloudstack/api/command/admin/ldap/LDAPConfigCmd.java +++ b/api/src/org/apache/cloudstack/api/command/admin/ldap/LDAPConfigCmd.java @@ -29,7 +29,6 @@ import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.response.LDAPConfigResponse; import org.apache.cloudstack.api.response.ListResponse; -import org.apache.commons.lang.StringEscapeUtils; import org.apache.log4j.Logger; import com.cloud.exception.ConcurrentOperationException; @@ -103,7 +102,7 @@ public class LDAPConfigCmd extends BaseCmd { } public void setQueryFilter(String queryFilter) { - this.queryFilter=StringEscapeUtils.unescapeHtml(queryFilter); + this.queryFilter=queryFilter; } public String getSearchBase() { return searchBase; diff --git a/api/src/org/apache/cloudstack/api/command/admin/network/UpdatePhysicalNetworkCmd.java b/api/src/org/apache/cloudstack/api/command/admin/network/UpdatePhysicalNetworkCmd.java index 6d37dd8a49b..06cf38dba3f 100644 --- a/api/src/org/apache/cloudstack/api/command/admin/network/UpdatePhysicalNetworkCmd.java +++ b/api/src/org/apache/cloudstack/api/command/admin/network/UpdatePhysicalNetworkCmd.java @@ -54,8 +54,6 @@ public class UpdatePhysicalNetworkCmd extends BaseAsyncCmd { @Parameter(name=ApiConstants.VLAN, type=CommandType.STRING, description="the VLAN for the physical network") private String vlan; - @Parameter(name=ApiConstants.REMOVE_VLAN, type = CommandType.STRING, description ="The vlan range we want to remove") - private String removevlan; ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// @@ -81,10 +79,6 @@ public class UpdatePhysicalNetworkCmd extends BaseAsyncCmd { return vlan; } - public String getRemoveVlan(){ - return removevlan; - } - ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @@ -101,7 +95,7 @@ public class UpdatePhysicalNetworkCmd extends BaseAsyncCmd { @Override public void execute(){ - PhysicalNetwork result = _networkService.updatePhysicalNetwork(getId(),getNetworkSpeed(), getTags(), getVlan(), getState(), getRemoveVlan()); + PhysicalNetwork result = _networkService.updatePhysicalNetwork(getId(),getNetworkSpeed(), getTags(), getVlan(), getState()); PhysicalNetworkResponse response = _responseGenerator.createPhysicalNetworkResponse(result); response.setResponseName(getCommandName()); this.setResponseObject(response); diff --git a/api/src/org/apache/cloudstack/api/command/admin/region/ListPortableIpRangesCmd.java b/api/src/org/apache/cloudstack/api/command/admin/region/ListPortableIpRangesCmd.java index 1f77c2c5164..bf0e34bf5a0 100644 --- a/api/src/org/apache/cloudstack/api/command/admin/region/ListPortableIpRangesCmd.java +++ b/api/src/org/apache/cloudstack/api/command/admin/region/ListPortableIpRangesCmd.java @@ -96,8 +96,6 @@ public class ListPortableIpRangesCmd extends BaseListCmd { } rangeResponse.setPortableIpResponses(portableIpResponses); } - - rangeResponse.setObjectName("portableiprange"); responses.add(rangeResponse); } } diff --git a/api/src/org/apache/cloudstack/api/command/admin/storage/AddImageStoreCmd.java b/api/src/org/apache/cloudstack/api/command/admin/storage/AddImageStoreCmd.java index 7e4409895c4..1552e0520a8 100644 --- a/api/src/org/apache/cloudstack/api/command/admin/storage/AddImageStoreCmd.java +++ b/api/src/org/apache/cloudstack/api/command/admin/storage/AddImageStoreCmd.java @@ -60,7 +60,7 @@ public class AddImageStoreCmd extends BaseCmd { @Parameter(name=ApiConstants.DETAILS, type=CommandType.MAP, description="the details for the image store. Example: details[0].key=accesskey&details[0].value=s389ddssaa&details[1].key=secretkey&details[1].value=8dshfsss") - private Map details; + private Map details; @@ -81,19 +81,19 @@ public class AddImageStoreCmd extends BaseCmd { return zoneId; } - public Map getDetails() { - Map detailsMap = null; - if (details != null && !details.isEmpty()) { - detailsMap = new HashMap(); - Collection props = details.values(); - Iterator iter = props.iterator(); - while (iter.hasNext()) { - HashMap detail = (HashMap) iter.next(); - String key = detail.get("key"); - String value = detail.get("value"); - detailsMap.put(key, value); - } - } + public Map getDetails() { + Map detailsMap = null; + if (details != null && !details.isEmpty()) { + detailsMap = new HashMap(); + Collection props = details.values(); + Iterator iter = props.iterator(); + while (iter.hasNext()) { + HashMap detail = (HashMap) iter.next(); + String key = detail.get("key"); + String value = detail.get("value"); + detailsMap.put(key, value); + } + } return detailsMap; } @@ -139,10 +139,10 @@ public class AddImageStoreCmd extends BaseCmd { ImageStore result = _storageService.discoverImageStore(this); ImageStoreResponse storeResponse = null; if (result != null ) { - storeResponse = _responseGenerator.createImageStoreResponse(result); - storeResponse.setResponseName(getCommandName()); - storeResponse.setObjectName("imagestore"); - this.setResponseObject(storeResponse); + storeResponse = _responseGenerator.createImageStoreResponse(result); + storeResponse.setResponseName(getCommandName()); + storeResponse.setObjectName("imagestore"); + this.setResponseObject(storeResponse); } else { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to add secondary storage"); } diff --git a/api/src/org/apache/cloudstack/api/command/admin/storage/AddS3Cmd.java b/api/src/org/apache/cloudstack/api/command/admin/storage/AddS3Cmd.java index 3ad84fd5a51..0af1a85051f 100644 --- a/api/src/org/apache/cloudstack/api/command/admin/storage/AddS3Cmd.java +++ b/api/src/org/apache/cloudstack/api/command/admin/storage/AddS3Cmd.java @@ -91,31 +91,44 @@ public final class AddS3Cmd extends BaseCmd { @Override public void execute() throws ResourceUnavailableException, InsufficientCapacityException, - ServerApiException, ConcurrentOperationException, ResourceAllocationException, - NetworkRuleConflictException { + ServerApiException, ConcurrentOperationException, ResourceAllocationException, + NetworkRuleConflictException { - AddImageStoreCmd cmd = new AddImageStoreCmd(); + AddImageStoreCmd cmd = new AddImageStoreCmd() { + @Override + public Map getDetails() { + Map dm = new HashMap(); + dm.put(ApiConstants.S3_ACCESS_KEY, getAccessKey()); + dm.put(ApiConstants.S3_SECRET_KEY, getSecretKey()); + dm.put(ApiConstants.S3_END_POINT, getEndPoint()); + dm.put(ApiConstants.S3_BUCKET_NAME, getBucketName()); + if (getHttpsFlag() != null) { + dm.put(ApiConstants.S3_HTTPS_FLAG, getHttpsFlag().toString()); + } + if (getConnectionTimeout() != null) { + dm.put(ApiConstants.S3_CONNECTION_TIMEOUT, getConnectionTimeout().toString()); + } + if (getMaxErrorRetry() != null) { + dm.put(ApiConstants.S3_MAX_ERROR_RETRY, getMaxErrorRetry().toString()); + } + if (getSocketTimeout() != null) { + dm.put(ApiConstants.S3_SOCKET_TIMEOUT, getSocketTimeout().toString()); + } + return dm; + } + }; cmd.setProviderName("S3"); - Map details = new HashMap(); - details.put(ApiConstants.S3_ACCESS_KEY, this.getAccessKey()); - details.put(ApiConstants.S3_SECRET_KEY, this.getSecretKey()); - details.put(ApiConstants.S3_END_POINT, this.getEndPoint()); - details.put(ApiConstants.S3_BUCKET_NAME, this.getBucketName()); - details.put(ApiConstants.S3_HTTPS_FLAG, this.getHttpsFlag().toString()); - details.put(ApiConstants.S3_CONNECTION_TIMEOUT, this.getConnectionTimeout().toString()); - details.put(ApiConstants.S3_MAX_ERROR_RETRY, this.getMaxErrorRetry().toString()); - details.put(ApiConstants.S3_SOCKET_TIMEOUT, this.getSocketTimeout().toString()); try{ ImageStore result = _storageService.discoverImageStore(cmd); ImageStoreResponse storeResponse = null; if (result != null ) { - storeResponse = _responseGenerator.createImageStoreResponse(result); - storeResponse.setResponseName(getCommandName()); - storeResponse.setObjectName("secondarystorage"); - this.setResponseObject(storeResponse); + storeResponse = _responseGenerator.createImageStoreResponse(result); + storeResponse.setResponseName(getCommandName()); + storeResponse.setObjectName("secondarystorage"); + this.setResponseObject(storeResponse); } else { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to add secondary storage"); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to add S3 secondary storage"); } } catch (DiscoveryException ex) { s_logger.warn("Exception: ", ex); diff --git a/api/src/org/apache/cloudstack/api/command/admin/storage/UpdateStoragePoolCmd.java b/api/src/org/apache/cloudstack/api/command/admin/storage/UpdateStoragePoolCmd.java index 2ecb90f69c7..f04ecbc402a 100644 --- a/api/src/org/apache/cloudstack/api/command/admin/storage/UpdateStoragePoolCmd.java +++ b/api/src/org/apache/cloudstack/api/command/admin/storage/UpdateStoragePoolCmd.java @@ -47,6 +47,13 @@ public class UpdateStoragePoolCmd extends BaseCmd { @Parameter(name=ApiConstants.TAGS, type=CommandType.LIST, collectionType=CommandType.STRING, description="comma-separated list of tags for the storage pool") private List tags; + @Parameter(name=ApiConstants.CAPACITY_IOPS, type=CommandType.LONG, + required=false, description="IOPS CloudStack can provision from this storage pool") + private Long capacityIops; + + @Parameter(name=ApiConstants.CAPACITY_BYTES, type=CommandType.LONG, + required=false, description="bytes CloudStack can provision from this storage pool") + private Long capacityBytes; ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// @@ -60,6 +67,14 @@ public class UpdateStoragePoolCmd extends BaseCmd { return tags; } + public Long getCapacityIops() { + return capacityIops; + } + + public Long getCapacityBytes() { + return capacityBytes; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/org/apache/cloudstack/api/command/admin/swift/AddSwiftCmd.java b/api/src/org/apache/cloudstack/api/command/admin/swift/AddSwiftCmd.java new file mode 100644 index 00000000000..ea22429f093 --- /dev/null +++ b/api/src/org/apache/cloudstack/api/command/admin/swift/AddSwiftCmd.java @@ -0,0 +1,122 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.swift; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.command.admin.storage.AddImageStoreCmd; +import org.apache.cloudstack.api.response.ImageStoreResponse; +import org.apache.log4j.Logger; + +import com.cloud.exception.DiscoveryException; +import com.cloud.storage.ImageStore; +import com.cloud.user.Account; + +@APICommand(name = "addSwift", description = "Adds Swift.", responseObject = ImageStoreResponse.class, since="3.0.0") +public class AddSwiftCmd extends BaseCmd { + public static final Logger s_logger = Logger.getLogger(AddSwiftCmd.class.getName()); + private static final String s_name = "addswiftresponse"; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.URL, type = CommandType.STRING, required = true, description = "the URL for swift") + private String url; + + @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, description = "the account for swift") + private String account; + + @Parameter(name = ApiConstants.USERNAME, type = CommandType.STRING, description = "the username for swift") + private String username; + + @Parameter(name = ApiConstants.KEY, type = CommandType.STRING, description = " key for the user for swift") + private String key; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public String getUrl() { + return url; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + public String getAccount() { + return account; + } + + public String getUsername() { + return username; + } + + public String getKey() { + return key; + } + + @Override + public String getCommandName() { + return s_name; + } + + @Override + public long getEntityOwnerId() { + return Account.ACCOUNT_ID_SYSTEM; + } + + @Override + public void execute(){ + AddImageStoreCmd cmd = new AddImageStoreCmd() { + @Override + public Map getDetails() { + Map dm = new HashMap(); + dm.put(ApiConstants.ACCOUNT, getAccount()); + dm.put(ApiConstants.USERNAME, getUsername()); + dm.put(ApiConstants.KEY, getKey()); + return dm; + } + }; + cmd.setProviderName("Swift"); + cmd.setUrl(this.getUrl()); + + try{ + ImageStore result = _storageService.discoverImageStore(cmd); + ImageStoreResponse storeResponse = null; + if (result != null ) { + storeResponse = _responseGenerator.createImageStoreResponse(result); + storeResponse.setResponseName(getCommandName()); + storeResponse.setObjectName("secondarystorage"); + this.setResponseObject(storeResponse); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to add Swift secondary storage"); + } + } catch (DiscoveryException ex) { + s_logger.warn("Exception: ", ex); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage()); + } + } +} diff --git a/api/src/org/apache/cloudstack/api/command/admin/swift/ListSwiftsCmd.java b/api/src/org/apache/cloudstack/api/command/admin/swift/ListSwiftsCmd.java new file mode 100644 index 00000000000..b0408f43792 --- /dev/null +++ b/api/src/org/apache/cloudstack/api/command/admin/swift/ListSwiftsCmd.java @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.swift; + +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseListCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.command.admin.storage.ListImageStoresCmd; +import org.apache.cloudstack.api.response.ImageStoreResponse; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.log4j.Logger; + +import com.cloud.user.Account; + +@APICommand(name = "listSwifts", description = "List Swift.", responseObject = ImageStoreResponse.class, since="3.0.0") +public class ListSwiftsCmd extends BaseListCmd { + public static final Logger s_logger = Logger.getLogger(ListSwiftsCmd.class.getName()); + private static final String s_name = "listswiftsresponse"; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.ID, type = CommandType.LONG, description = "the id of the swift") + private Long id; + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + public Long getId() { + return id; + } + + + @Override + public String getCommandName() { + return s_name; + } + + @Override + public long getEntityOwnerId() { + return Account.ACCOUNT_ID_SYSTEM; + } + + @Override + public void execute(){ + + ListImageStoresCmd cmd = new ListImageStoresCmd(); + cmd.setProvider("Swift"); + ListResponse response = _queryService.searchForImageStores(cmd); + response.setResponseName(getCommandName()); + this.setResponseObject(response); + } +} diff --git a/api/src/org/apache/cloudstack/api/command/admin/vm/AssignVMCmd.java b/api/src/org/apache/cloudstack/api/command/admin/vm/AssignVMCmd.java index 152dd4e14c2..2a60e192ca3 100644 --- a/api/src/org/apache/cloudstack/api/command/admin/vm/AssignVMCmd.java +++ b/api/src/org/apache/cloudstack/api/command/admin/vm/AssignVMCmd.java @@ -35,7 +35,7 @@ import org.apache.log4j.Logger; import com.cloud.user.Account; import com.cloud.uservm.UserVm; -@APICommand(name = "assignVirtualMachine", description="Assign a VM from one account to another under the same domain. This API is available for Basic zones with security groups and Advance zones with guest networks. The VM is restricted to move between accounts under same domain.", responseObject=UserVmResponse.class, since="3.0.0") +@APICommand(name = "assignVirtualMachine", description="Change ownership of a VM from one account to another. This API is available for Basic zones with security groups and Advanced zones with guest networks. A root administrator can reassign a VM from any account to any other account in any domain. A domain administrator can reassign a VM to any account in the same domain.", responseObject=UserVmResponse.class, since="3.0.0") public class AssignVMCmd extends BaseCmd { public static final Logger s_logger = Logger.getLogger(AssignVMCmd.class.getName()); diff --git a/api/src/org/apache/cloudstack/api/command/user/config/ListCapabilitiesCmd.java b/api/src/org/apache/cloudstack/api/command/user/config/ListCapabilitiesCmd.java index a30e26cfd8b..6f4845b9245 100644 --- a/api/src/org/apache/cloudstack/api/command/user/config/ListCapabilitiesCmd.java +++ b/api/src/org/apache/cloudstack/api/command/user/config/ListCapabilitiesCmd.java @@ -52,6 +52,9 @@ public class ListCapabilitiesCmd extends BaseCmd { response.setProjectInviteRequired((Boolean)capabilities.get("projectInviteRequired")); response.setAllowUsersCreateProjects((Boolean)capabilities.get("allowusercreateprojects")); response.setDiskOffMaxSize((Long)capabilities.get("customDiskOffMaxSize")); + response.setKVMSnapshotEnabled((Boolean)capabilities.get("KVMSnapshotEnabled")); + response.setRegionSecondaryEnabled((Boolean)capabilities.get("regionSecondaryEnabled")); + if (capabilities.containsKey("apiLimitInterval")) { response.setApiLimitInterval((Integer) capabilities.get("apiLimitInterval")); } diff --git a/api/src/org/apache/cloudstack/api/command/user/firewall/UpdatePortForwardingRuleCmd.java b/api/src/org/apache/cloudstack/api/command/user/firewall/UpdatePortForwardingRuleCmd.java index 2a8b9003fa8..947c209f228 100644 --- a/api/src/org/apache/cloudstack/api/command/user/firewall/UpdatePortForwardingRuleCmd.java +++ b/api/src/org/apache/cloudstack/api/command/user/firewall/UpdatePortForwardingRuleCmd.java @@ -26,6 +26,7 @@ import org.apache.cloudstack.api.response.UserVmResponse; import org.apache.log4j.Logger; import com.cloud.event.EventTypes; +import com.cloud.exception.InvalidParameterValueException; import com.cloud.network.IpAddress; import com.cloud.user.Account; @@ -127,4 +128,22 @@ public class UpdatePortForwardingRuleCmd extends BaseAsyncCmd { // throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to update port forwarding rule"); // } } + + @Override + public String getSyncObjType() { + return BaseAsyncCmd.networkSyncObject; + } + + @Override + public Long getSyncObjId() { + return getIp().getAssociatedWithNetworkId(); + } + + private IpAddress getIp() { + IpAddress ip = _networkService.getIp(publicIpId); + if (ip == null) { + throw new InvalidParameterValueException("Unable to find ip address by id " + publicIpId); + } + return ip; + } } diff --git a/api/src/org/apache/cloudstack/api/command/user/loadbalancer/CreateLBStickinessPolicyCmd.java b/api/src/org/apache/cloudstack/api/command/user/loadbalancer/CreateLBStickinessPolicyCmd.java index 02b253a7c0c..7a2283e4762 100644 --- a/api/src/org/apache/cloudstack/api/command/user/loadbalancer/CreateLBStickinessPolicyCmd.java +++ b/api/src/org/apache/cloudstack/api/command/user/loadbalancer/CreateLBStickinessPolicyCmd.java @@ -22,6 +22,7 @@ import java.util.Map; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCmd; import org.apache.cloudstack.api.BaseAsyncCreateCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ServerApiException; @@ -30,6 +31,7 @@ import org.apache.cloudstack.api.response.LBStickinessResponse; import org.apache.log4j.Logger; import com.cloud.event.EventTypes; +import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.NetworkRuleConflictException; import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; @@ -156,5 +158,18 @@ public class CreateLBStickinessPolicyCmd extends BaseAsyncCreateCmd { return "creating a Load Balancer Stickiness policy: " + getLBStickinessPolicyName(); } + @Override + public String getSyncObjType() { + return BaseAsyncCmd.networkSyncObject; + } + + @Override + public Long getSyncObjId() { + LoadBalancer lb = _lbService.findById(getLbRuleId()); + if (lb == null) { + throw new InvalidParameterValueException("Unable to find load balancer rule " + getLbRuleId() + " to create stickiness rule"); + } + return lb.getNetworkId(); + } } diff --git a/api/src/org/apache/cloudstack/api/command/user/loadbalancer/CreateLoadBalancerRuleCmd.java b/api/src/org/apache/cloudstack/api/command/user/loadbalancer/CreateLoadBalancerRuleCmd.java index f6cc1f130bd..e398380c757 100644 --- a/api/src/org/apache/cloudstack/api/command/user/loadbalancer/CreateLoadBalancerRuleCmd.java +++ b/api/src/org/apache/cloudstack/api/command/user/loadbalancer/CreateLoadBalancerRuleCmd.java @@ -21,6 +21,7 @@ import java.util.List; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCmd; import org.apache.cloudstack.api.BaseAsyncCreateCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ServerApiException; @@ -380,5 +381,14 @@ public class CreateLoadBalancerRuleCmd extends BaseAsyncCreateCmd /*implements return AsyncJob.Type.FirewallRule; } + @Override + public String getSyncObjType() { + return BaseAsyncCmd.networkSyncObject; + } + + @Override + public Long getSyncObjId() { + return getNetworkId(); + } } diff --git a/api/src/org/apache/cloudstack/api/command/user/loadbalancer/UpdateLoadBalancerRuleCmd.java b/api/src/org/apache/cloudstack/api/command/user/loadbalancer/UpdateLoadBalancerRuleCmd.java index c2960579977..9a29c121bb6 100644 --- a/api/src/org/apache/cloudstack/api/command/user/loadbalancer/UpdateLoadBalancerRuleCmd.java +++ b/api/src/org/apache/cloudstack/api/command/user/loadbalancer/UpdateLoadBalancerRuleCmd.java @@ -27,6 +27,7 @@ import org.apache.cloudstack.api.response.LoadBalancerResponse; import org.apache.log4j.Logger; import com.cloud.event.EventTypes; +import com.cloud.exception.InvalidParameterValueException; import com.cloud.network.rules.LoadBalancer; import com.cloud.user.Account; import com.cloud.user.UserContext; @@ -113,4 +114,18 @@ public class UpdateLoadBalancerRuleCmd extends BaseAsyncCmd { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to update load balancer rule"); } } + + @Override + public String getSyncObjType() { + return BaseAsyncCmd.networkSyncObject; + } + + @Override + public Long getSyncObjId() { + LoadBalancer lb = _lbService.findById(getId()); + if (lb == null) { + throw new InvalidParameterValueException("Unable to find load balancer rule " + getId()); + } + return lb.getNetworkId(); + } } diff --git a/api/src/org/apache/cloudstack/api/command/user/network/UpdateNetworkCmd.java b/api/src/org/apache/cloudstack/api/command/user/network/UpdateNetworkCmd.java index fe381246b28..ae9f35a3e55 100644 --- a/api/src/org/apache/cloudstack/api/command/user/network/UpdateNetworkCmd.java +++ b/api/src/org/apache/cloudstack/api/command/user/network/UpdateNetworkCmd.java @@ -158,4 +158,14 @@ public class UpdateNetworkCmd extends BaseAsyncCmd { public String getEventType() { return EventTypes.EVENT_NETWORK_UPDATE; } + + @Override + public String getSyncObjType() { + return BaseAsyncCmd.networkSyncObject; + } + + @Override + public Long getSyncObjId() { + return id; + } } diff --git a/api/src/org/apache/cloudstack/api/command/user/securitygroup/AuthorizeSecurityGroupIngressCmd.java b/api/src/org/apache/cloudstack/api/command/user/securitygroup/AuthorizeSecurityGroupIngressCmd.java index f8cd8c5d955..13d21338ae7 100644 --- a/api/src/org/apache/cloudstack/api/command/user/securitygroup/AuthorizeSecurityGroupIngressCmd.java +++ b/api/src/org/apache/cloudstack/api/command/user/securitygroup/AuthorizeSecurityGroupIngressCmd.java @@ -207,13 +207,6 @@ public class AuthorizeSecurityGroupIngressCmd extends BaseAsyncCmd { @Override public void execute() { - if(cidrList != null){ - for(String cidr : cidrList ){ - if (!NetUtils.isValidCIDR(cidr)){ - throw new ServerApiException(ApiErrorCode.PARAM_ERROR, cidr + " is an Invalid CIDR "); - } - } - } List ingressRules = _securityGroupService.authorizeSecurityGroupIngress(this); if (ingressRules != null && !ingressRules.isEmpty()) { SecurityGroupResponse response = _responseGenerator.createSecurityGroupResponseFromSecurityGroupRule(ingressRules); diff --git a/api/src/org/apache/cloudstack/api/command/user/securitygroup/RevokeSecurityGroupEgressCmd.java b/api/src/org/apache/cloudstack/api/command/user/securitygroup/RevokeSecurityGroupEgressCmd.java index c03d3e4ba5b..b30ba1c8d84 100644 --- a/api/src/org/apache/cloudstack/api/command/user/securitygroup/RevokeSecurityGroupEgressCmd.java +++ b/api/src/org/apache/cloudstack/api/command/user/securitygroup/RevokeSecurityGroupEgressCmd.java @@ -29,6 +29,7 @@ import org.apache.log4j.Logger; import com.cloud.async.AsyncJob; import com.cloud.event.EventTypes; import com.cloud.network.security.SecurityGroup; +import com.cloud.network.security.SecurityRule; import com.cloud.user.Account; @APICommand(name = "revokeSecurityGroupEgress", responseObject = SuccessResponse.class, description = "Deletes a particular egress rule from this security group", since="3.0.0") @@ -67,9 +68,12 @@ public class RevokeSecurityGroupEgressCmd extends BaseAsyncCmd { @Override public long getEntityOwnerId() { - SecurityGroup group = _entityMgr.findById(SecurityGroup.class, getId()); - if (group != null) { - return group.getAccountId(); + SecurityRule rule = _entityMgr.findById(SecurityRule.class, getId()); + if (rule != null) { + SecurityGroup group = _entityMgr.findById(SecurityGroup.class, rule.getSecurityGroupId()); + if (group != null) { + return group.getAccountId(); + } } return Account.ACCOUNT_ID_SYSTEM; // no account info given, parent this command to SYSTEM so ERROR events are tracked diff --git a/api/src/org/apache/cloudstack/api/command/user/securitygroup/RevokeSecurityGroupIngressCmd.java b/api/src/org/apache/cloudstack/api/command/user/securitygroup/RevokeSecurityGroupIngressCmd.java index c2fdb8b000f..a547fb02898 100644 --- a/api/src/org/apache/cloudstack/api/command/user/securitygroup/RevokeSecurityGroupIngressCmd.java +++ b/api/src/org/apache/cloudstack/api/command/user/securitygroup/RevokeSecurityGroupIngressCmd.java @@ -29,6 +29,7 @@ import org.apache.log4j.Logger; import com.cloud.async.AsyncJob; import com.cloud.event.EventTypes; import com.cloud.network.security.SecurityGroup; +import com.cloud.network.security.SecurityRule; import com.cloud.user.Account; @APICommand(name = "revokeSecurityGroupIngress", responseObject = SuccessResponse.class, description = "Deletes a particular ingress rule from this security group") @@ -67,9 +68,12 @@ public class RevokeSecurityGroupIngressCmd extends BaseAsyncCmd { @Override public long getEntityOwnerId() { - SecurityGroup group = _entityMgr.findById(SecurityGroup.class, getId()); - if (group != null) { - return group.getAccountId(); + SecurityRule rule = _entityMgr.findById(SecurityRule.class, getId()); + if (rule != null) { + SecurityGroup group = _entityMgr.findById(SecurityGroup.class, rule.getSecurityGroupId()); + if (group != null) { + return group.getAccountId(); + } } return Account.ACCOUNT_ID_SYSTEM; // no account info given, parent this command to SYSTEM so ERROR events are tracked diff --git a/api/src/org/apache/cloudstack/api/command/user/vm/ListVMsCmd.java b/api/src/org/apache/cloudstack/api/command/user/vm/ListVMsCmd.java index d3b29db2801..f9258cbbdf8 100644 --- a/api/src/org/apache/cloudstack/api/command/user/vm/ListVMsCmd.java +++ b/api/src/org/apache/cloudstack/api/command/user/vm/ListVMsCmd.java @@ -66,7 +66,7 @@ public class ListVMsCmd extends BaseListTaggedResourcesCmd { private Long id; @Parameter(name=ApiConstants.NAME, type=CommandType.STRING, description="name of the virtual machine") - private String instanceName; + private String name; @Parameter(name=ApiConstants.POD_ID, type=CommandType.UUID, entityType=PodResponse.class, description="the pod ID") @@ -130,8 +130,8 @@ public class ListVMsCmd extends BaseListTaggedResourcesCmd { return id; } - public String getInstanceName() { - return instanceName; + public String getName() { + return name; } public Long getPodId() { diff --git a/api/src/org/apache/cloudstack/api/command/user/volume/MigrateVolumeCmd.java b/api/src/org/apache/cloudstack/api/command/user/volume/MigrateVolumeCmd.java index 40e6123d0ec..61863976930 100644 --- a/api/src/org/apache/cloudstack/api/command/user/volume/MigrateVolumeCmd.java +++ b/api/src/org/apache/cloudstack/api/command/user/volume/MigrateVolumeCmd.java @@ -99,6 +99,7 @@ public class MigrateVolumeCmd extends BaseAsyncCmd { @Override public void execute(){ Volume result; + result = _volumeService.migrateVolume(this); if (result != null) { VolumeResponse response = _responseGenerator.createVolumeResponse(result); diff --git a/api/src/org/apache/cloudstack/api/command/user/vpc/DeleteVPCCmd.java b/api/src/org/apache/cloudstack/api/command/user/vpc/DeleteVPCCmd.java index 18866beb06d..f62ec5d6cfc 100644 --- a/api/src/org/apache/cloudstack/api/command/user/vpc/DeleteVPCCmd.java +++ b/api/src/org/apache/cloudstack/api/command/user/vpc/DeleteVPCCmd.java @@ -101,4 +101,13 @@ public class DeleteVPCCmd extends BaseAsyncCmd{ return Account.ACCOUNT_ID_SYSTEM; // no account info given, parent this command to SYSTEM so ERROR events are tracked } + @Override + public String getSyncObjType() { + return BaseAsyncCmd.vpcSyncObject; + } + + @Override + public Long getSyncObjId() { + return getId(); + } } diff --git a/api/src/org/apache/cloudstack/api/command/user/vpc/RestartVPCCmd.java b/api/src/org/apache/cloudstack/api/command/user/vpc/RestartVPCCmd.java index 714e9e79926..8d7f24dc1de 100644 --- a/api/src/org/apache/cloudstack/api/command/user/vpc/RestartVPCCmd.java +++ b/api/src/org/apache/cloudstack/api/command/user/vpc/RestartVPCCmd.java @@ -106,4 +106,14 @@ public class RestartVPCCmd extends BaseAsyncCmd{ public String getEventDescription() { return "restarting VPC id=" + getId(); } + + @Override + public String getSyncObjType() { + return BaseAsyncCmd.vpcSyncObject; + } + + @Override + public Long getSyncObjId() { + return getId(); + } } diff --git a/api/src/org/apache/cloudstack/api/command/user/vpc/UpdateVPCCmd.java b/api/src/org/apache/cloudstack/api/command/user/vpc/UpdateVPCCmd.java index a6410214cc3..588333efff0 100644 --- a/api/src/org/apache/cloudstack/api/command/user/vpc/UpdateVPCCmd.java +++ b/api/src/org/apache/cloudstack/api/command/user/vpc/UpdateVPCCmd.java @@ -105,4 +105,14 @@ public class UpdateVPCCmd extends BaseAsyncCmd{ public String getEventDescription() { return "updating VPC id=" + getId(); } + + @Override + public String getSyncObjType() { + return BaseAsyncCmd.vpcSyncObject; + } + + @Override + public Long getSyncObjId() { + return getId(); + } } diff --git a/api/src/org/apache/cloudstack/api/response/AccountResponse.java b/api/src/org/apache/cloudstack/api/response/AccountResponse.java index 0d4f2391f57..957936bb9e3 100644 --- a/api/src/org/apache/cloudstack/api/response/AccountResponse.java +++ b/api/src/org/apache/cloudstack/api/response/AccountResponse.java @@ -19,13 +19,14 @@ package org.apache.cloudstack.api.response; import java.util.List; import java.util.Map; +import com.google.gson.annotations.SerializedName; + import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseResponse; import org.apache.cloudstack.api.EntityReference; import com.cloud.serializer.Param; import com.cloud.user.Account; -import com.google.gson.annotations.SerializedName; @SuppressWarnings("unused") @EntityReference(value = Account.class) @@ -189,7 +190,7 @@ public class AccountResponse extends BaseResponse implements ResourceLimitAndCou @Override public String getObjectId() { - return this.id; + return id; } public void setId(String id) { @@ -351,7 +352,22 @@ public class AccountResponse extends BaseResponse implements ResourceLimitAndCou public void setNetworkAvailable(String networkAvailable) { this.networkAvailable = networkAvailable; } + + @Override + public void setVpcLimit(String vpcLimit) { + this.vpcLimit = networkLimit; + } + + @Override + public void setVpcTotal(Long vpcTotal) { + this.vpcTotal = vpcTotal; + } + @Override + public void setVpcAvailable(String vpcAvailable) { + this.vpcAvailable = vpcAvailable; + } + @Override public void setCpuLimit(String cpuLimit) { this.cpuLimit = cpuLimit; diff --git a/api/src/org/apache/cloudstack/api/response/CapabilitiesResponse.java b/api/src/org/apache/cloudstack/api/response/CapabilitiesResponse.java index c2996f0aa0a..0142e1c2e8d 100644 --- a/api/src/org/apache/cloudstack/api/response/CapabilitiesResponse.java +++ b/api/src/org/apache/cloudstack/api/response/CapabilitiesResponse.java @@ -46,6 +46,12 @@ public class CapabilitiesResponse extends BaseResponse { "create disk from disk offering with custom size") private Long diskOffMaxSize; + @SerializedName("KVMsnapshotenabled") @Param(description = "true if snapshot is supported for KVM host, false otherwise") + private boolean KVMSnapshotEnabled; + + @SerializedName("regionsecondaryenabled") @Param(description = "true if region wide secondary is enabled, false otherwise") + private boolean regionSecondaryEnabled; + @SerializedName("apilimitinterval") @Param(description="time interval (in seconds) to reset api count") private Integer apiLimitInterval; @@ -81,6 +87,14 @@ public class CapabilitiesResponse extends BaseResponse { this.diskOffMaxSize = diskOffMaxSize; } + public void setKVMSnapshotEnabled(boolean KVMSnapshotEnabled) { + this.KVMSnapshotEnabled = KVMSnapshotEnabled; + } + + public void setRegionSecondaryEnabled(boolean regionSecondaryEnabled){ + this.regionSecondaryEnabled = regionSecondaryEnabled; + } + public void setApiLimitInterval(Integer apiLimitInterval) { this.apiLimitInterval = apiLimitInterval; } diff --git a/api/src/org/apache/cloudstack/api/response/CreateCmdResponse.java b/api/src/org/apache/cloudstack/api/response/CreateCmdResponse.java index e4c6c60c5ba..0602962d39f 100644 --- a/api/src/org/apache/cloudstack/api/response/CreateCmdResponse.java +++ b/api/src/org/apache/cloudstack/api/response/CreateCmdResponse.java @@ -16,9 +16,14 @@ // under the License. package org.apache.cloudstack.api.response; +import com.google.gson.annotations.SerializedName; + +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseResponse; public class CreateCmdResponse extends BaseResponse { + + @SerializedName(ApiConstants.ID) private String id; public String getId() { diff --git a/api/src/org/apache/cloudstack/api/response/NetworkOfferingResponse.java b/api/src/org/apache/cloudstack/api/response/NetworkOfferingResponse.java index 6b35d7b4eaf..a593ae6bc3f 100644 --- a/api/src/org/apache/cloudstack/api/response/NetworkOfferingResponse.java +++ b/api/src/org/apache/cloudstack/api/response/NetworkOfferingResponse.java @@ -91,6 +91,8 @@ public class NetworkOfferingResponse extends BaseResponse { @SerializedName(ApiConstants.EGRESS_DEFAULT_POLICY) @Param(description="true if network offering supports persistent networks, false otherwise") private Boolean egressDefaultPolicy; + @SerializedName(ApiConstants.MAX_CONNECTIONS) @Param(description = "maximum number of concurrents connections to be handled by lb") + private Integer concurrentConnections; public void setId(String id) { this.id = id; @@ -173,4 +175,8 @@ public class NetworkOfferingResponse extends BaseResponse { this.egressDefaultPolicy = egressDefaultPolicy; } + public void setConcurrentConnections(Integer concurrentConnections) { + this.concurrentConnections = concurrentConnections; + } + } diff --git a/api/src/org/apache/cloudstack/api/response/NetworkResponse.java b/api/src/org/apache/cloudstack/api/response/NetworkResponse.java index 31024330dcd..e38daf2894e 100644 --- a/api/src/org/apache/cloudstack/api/response/NetworkResponse.java +++ b/api/src/org/apache/cloudstack/api/response/NetworkResponse.java @@ -76,6 +76,9 @@ public class NetworkResponse extends BaseResponse implements ControlledEntityRes @SerializedName("networkofferingdisplaytext") @Param(description="display text of the network offering the network is created from") private String networkOfferingDisplayText; + @SerializedName("networkofferingconservemode") @Param(description="true if network offering is ip conserve mode enabled") + private Boolean networkOfferingConserveMode; + @SerializedName("networkofferingavailability") @Param(description="availability of the network offering the network is created from") private String networkOfferingAvailability; @@ -247,7 +250,11 @@ public class NetworkResponse extends BaseResponse implements ControlledEntityRes public void setNetworkOfferingDisplayText(String networkOfferingDisplayText) { this.networkOfferingDisplayText = networkOfferingDisplayText; } - + + public void setNetworkOfferingConserveMode(Boolean networkOfferingConserveMode) { + this.networkOfferingConserveMode = networkOfferingConserveMode; + } + public void setDisplaytext(String displaytext) { this.displaytext = displaytext; } diff --git a/api/src/org/apache/cloudstack/api/response/ProjectResponse.java b/api/src/org/apache/cloudstack/api/response/ProjectResponse.java index 4fdd1374495..81b51c8b34d 100644 --- a/api/src/org/apache/cloudstack/api/response/ProjectResponse.java +++ b/api/src/org/apache/cloudstack/api/response/ProjectResponse.java @@ -19,13 +19,14 @@ package org.apache.cloudstack.api.response; import java.util.ArrayList; import java.util.List; +import com.google.gson.annotations.SerializedName; + import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseResponse; import org.apache.cloudstack.api.EntityReference; import com.cloud.projects.Project; import com.cloud.serializer.Param; -import com.google.gson.annotations.SerializedName; @EntityReference(value=Project.class) @SuppressWarnings("unused") @@ -182,7 +183,7 @@ public class ProjectResponse extends BaseResponse implements ResourceLimitAndCou } public void setOwner(String owner) { - this.ownerName = owner; + ownerName = owner; } public void setState(String state) { @@ -194,7 +195,7 @@ public class ProjectResponse extends BaseResponse implements ResourceLimitAndCou } public void addTag(ResourceTagResponse tag){ - this.tags.add(tag); + tags.add(tag); } @Override @@ -296,7 +297,22 @@ public class ProjectResponse extends BaseResponse implements ResourceLimitAndCou public void setNetworkAvailable(String networkAvailable) { this.networkAvailable = networkAvailable; } + + @Override + public void setVpcLimit(String vpcLimit) { + this.vpcLimit = networkLimit; + } + + @Override + public void setVpcTotal(Long vpcTotal) { + this.vpcTotal = vpcTotal; + } + @Override + public void setVpcAvailable(String vpcAvailable) { + this.vpcAvailable = vpcAvailable; + } + @Override public void setCpuLimit(String cpuLimit) { this.cpuLimit = cpuLimit; diff --git a/api/src/org/apache/cloudstack/api/response/RegionResponse.java b/api/src/org/apache/cloudstack/api/response/RegionResponse.java index f8bfe53aae3..acaa272f512 100644 --- a/api/src/org/apache/cloudstack/api/response/RegionResponse.java +++ b/api/src/org/apache/cloudstack/api/response/RegionResponse.java @@ -35,6 +35,12 @@ public class RegionResponse extends BaseResponse { @SerializedName(ApiConstants.END_POINT) @Param(description="the end point of the region") private String endPoint; + @SerializedName("gslbserviceenabled") @Param(description="true if GSLB service is enabled in the region, false otherwise") + private boolean gslbServiceEnabled; + + @SerializedName("portableipserviceenabled") @Param(description="true if security groups support is enabled, false otherwise") + private boolean portableipServiceEnabled; + public Integer getId() { return id; } @@ -59,4 +65,11 @@ public class RegionResponse extends BaseResponse { this.endPoint = endPoint; } - } + public void setGslbServiceEnabled(boolean gslbServiceEnabled) { + this.gslbServiceEnabled = gslbServiceEnabled; + } + + public void setPortableipServiceEnabled(boolean portableipServiceEnabled) { + this.portableipServiceEnabled = portableipServiceEnabled; + } +} diff --git a/api/src/org/apache/cloudstack/api/response/ResourceLimitAndCountResponse.java b/api/src/org/apache/cloudstack/api/response/ResourceLimitAndCountResponse.java index 57aabdd2c08..49bb4e02c07 100644 --- a/api/src/org/apache/cloudstack/api/response/ResourceLimitAndCountResponse.java +++ b/api/src/org/apache/cloudstack/api/response/ResourceLimitAndCountResponse.java @@ -29,6 +29,12 @@ public interface ResourceLimitAndCountResponse { public void setNetworkAvailable(String networkAvailable); + public void setVpcLimit(String vpcLimit); + + public void setVpcTotal(Long vpcTotal); + + public void setVpcAvailable(String vpcAvailable); + public void setCpuLimit(String cpuLimit); public void setCpuTotal(Long cpuTotal); diff --git a/api/src/org/apache/cloudstack/api/response/S3Response.java b/api/src/org/apache/cloudstack/api/response/S3Response.java deleted file mode 100644 index 259a3088c1e..00000000000 --- a/api/src/org/apache/cloudstack/api/response/S3Response.java +++ /dev/null @@ -1,218 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.cloudstack.api.response; - -import static org.apache.cloudstack.api.ApiConstants.ID; -import static org.apache.cloudstack.api.ApiConstants.S3_ACCESS_KEY; -import static org.apache.cloudstack.api.ApiConstants.S3_BUCKET_NAME; -import static org.apache.cloudstack.api.ApiConstants.S3_CONNECTION_TIMEOUT; -import static org.apache.cloudstack.api.ApiConstants.S3_END_POINT; -import static org.apache.cloudstack.api.ApiConstants.S3_HTTPS_FLAG; -import static org.apache.cloudstack.api.ApiConstants.S3_MAX_ERROR_RETRY; -import static org.apache.cloudstack.api.ApiConstants.S3_SECRET_KEY; -import static org.apache.cloudstack.api.ApiConstants.S3_SOCKET_TIMEOUT; - -import org.apache.cloudstack.api.BaseResponse; - -import com.cloud.serializer.Param; -import com.google.gson.annotations.SerializedName; - -public class S3Response extends BaseResponse { - - @SerializedName(ID) - @Param(description = "The ID of the S3 configuration") - private String id; - - @SerializedName(S3_ACCESS_KEY) - @Param(description = "The S3 access key") - private String accessKey; - - @SerializedName(S3_SECRET_KEY) - @Param(description = "The S3 secret key") - private String secretKey; - - @SerializedName(S3_END_POINT) - @Param(description = "The S3 end point") - private String endPoint; - - @SerializedName(S3_BUCKET_NAME) - @Param(description = "The name of the template storage bucket") - private String bucketName; - - @SerializedName(S3_HTTPS_FLAG) - @Param(description = "Connect to S3 using HTTPS?") - private Integer httpsFlag; - - @SerializedName(S3_CONNECTION_TIMEOUT) - @Param(description = "The connection timeout (milliseconds)") - private Integer connectionTimeout; - - @SerializedName(S3_MAX_ERROR_RETRY) - @Param(description = "The maximum number of time to retry a connection on error.") - private Integer maxErrorRetry; - - @SerializedName(S3_SOCKET_TIMEOUT) - @Param(description = "The connection socket (milliseconds)") - private Integer socketTimeout; - - @Override - public boolean equals(final Object thatObject) { - - if (this == thatObject) { - return true; - } - - if (thatObject == null || this.getClass() != thatObject.getClass()) { - return false; - } - - final S3Response thatS3Response = (S3Response) thatObject; - - if (this.httpsFlag != null ? !this.httpsFlag.equals(thatS3Response.httpsFlag) : thatS3Response.httpsFlag != null) { - return false; - } - - if (this.accessKey != null ? !this.accessKey.equals(thatS3Response.accessKey) : thatS3Response.accessKey != null) { - return false; - } - - if (this.connectionTimeout != null ? !this.connectionTimeout.equals(thatS3Response.connectionTimeout) : thatS3Response.connectionTimeout != null) { - return false; - } - - if (this.endPoint != null ? !this.endPoint.equals(thatS3Response.endPoint) : thatS3Response.endPoint != null) { - return false; - } - - if (this.id != null ? !this.id.equals(thatS3Response.id) : thatS3Response.id != null) { - return false; - } - - if (this.maxErrorRetry != null ? !this.maxErrorRetry.equals(thatS3Response.maxErrorRetry) : thatS3Response.maxErrorRetry != null) { - return false; - } - - if (this.secretKey != null ? !this.secretKey.equals(thatS3Response.secretKey) : thatS3Response.secretKey != null) { - return false; - } - - if (this.socketTimeout != null ? !this.socketTimeout.equals(thatS3Response.socketTimeout) : thatS3Response.socketTimeout != null) { - return false; - } - - if (this.bucketName != null ? !this.bucketName.equals(thatS3Response.bucketName) : thatS3Response.bucketName != null) { - return false; - } - - return true; - - } - - @Override - public int hashCode() { - - int result = this.id != null ? this.id.hashCode() : 0; - result = 31 * result + (this.accessKey != null ? this.accessKey.hashCode() : 0); - result = 31 * result + (this.secretKey != null ? this.secretKey.hashCode() : 0); - result = 31 * result + (this.endPoint != null ? this.endPoint.hashCode() : 0); - result = 31 * result + (this.bucketName != null ? this.bucketName.hashCode() : 0); - result = 31 * result + (this.httpsFlag != null ? this.httpsFlag : 0); - result = 31 * result + (this.connectionTimeout != null ? this.connectionTimeout.hashCode() : 0); - result = 31 * result + (this.maxErrorRetry != null ? this.maxErrorRetry.hashCode() : 0); - result = 31 * result + (this.socketTimeout != null ? this.socketTimeout.hashCode() : 0); - - return result; - - } - - @Override - public String getObjectId() { - return this.id; - } - - public void setObjectId(String id) { - this.id = id; - } - - public String getAccessKey() { - return this.accessKey; - } - - public void setAccessKey(final String accessKey) { - this.accessKey = accessKey; - } - - public String getSecretKey() { - return this.secretKey; - } - - public void setSecretKey(final String secretKey) { - this.secretKey = secretKey; - } - - public String getEndPoint() { - return this.endPoint; - } - - public void setEndPoint(final String endPoint) { - this.endPoint = endPoint; - } - - - public String getTemplateBucketName() { - return this.bucketName; - } - - public void setTemplateBucketName(final String templateBucketName) { - this.bucketName = templateBucketName; - } - - public Integer getHttpsFlag() { - return this.httpsFlag; - } - - public void setHttpsFlag(final Integer httpsFlag) { - this.httpsFlag = httpsFlag; - } - - public Integer getConnectionTimeout() { - return this.connectionTimeout; - } - - public void setConnectionTimeout(final Integer connectionTimeout) { - this.connectionTimeout = connectionTimeout; - } - - public Integer getMaxErrorRetry() { - return this.maxErrorRetry; - } - - public void setMaxErrorRetry(final Integer maxErrorRetry) { - this.maxErrorRetry = maxErrorRetry; - } - - public Integer getSocketTimeout() { - return this.socketTimeout; - } - - public void setSocketTimeout(final Integer socketTimeout) { - this.socketTimeout = socketTimeout; - } - -} diff --git a/api/src/org/apache/cloudstack/api/response/SecurityGroupRuleResponse.java b/api/src/org/apache/cloudstack/api/response/SecurityGroupRuleResponse.java index 5aeee6f0611..798b1237c94 100644 --- a/api/src/org/apache/cloudstack/api/response/SecurityGroupRuleResponse.java +++ b/api/src/org/apache/cloudstack/api/response/SecurityGroupRuleResponse.java @@ -20,11 +20,11 @@ import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseResponse; import org.apache.cloudstack.api.EntityReference; -import com.cloud.network.security.SecurityGroupRules; +import com.cloud.network.security.SecurityRule; import com.cloud.serializer.Param; import com.google.gson.annotations.SerializedName; -@EntityReference(value = SecurityGroupRules.class) +@EntityReference(value = SecurityRule.class) public class SecurityGroupRuleResponse extends BaseResponse { @SerializedName("ruleid") @Param(description="the id of the security group rule") private String ruleId; diff --git a/api/src/org/apache/cloudstack/api/response/SwiftResponse.java b/api/src/org/apache/cloudstack/api/response/SwiftResponse.java deleted file mode 100644 index 08b260943ef..00000000000 --- a/api/src/org/apache/cloudstack/api/response/SwiftResponse.java +++ /dev/null @@ -1,86 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package org.apache.cloudstack.api.response; - -import java.util.Date; - -import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.BaseResponse; - -import com.cloud.serializer.Param; -import com.google.gson.annotations.SerializedName; - -public class SwiftResponse extends BaseResponse { - @SerializedName(ApiConstants.ID) - @Param(description = "the ID of swift") - private String id; - - @SerializedName(ApiConstants.URL) - @Param(description = "url for swift") - private String url; - - @SerializedName(ApiConstants.CREATED) - @Param(description = "the date and time the host was created") - private Date created; - - @SerializedName(ApiConstants.ACCOUNT) - @Param(description = "the account for swift") - private String account; - - @SerializedName(ApiConstants.ACCOUNT) - @Param(description = "the username for swift") - private String username; - - - - public void setId(String id) { - this.id = id; - } - - public String getUrl() { - return url; - } - - public void setUrl(String url) { - this.url = url; - } - - public Date getCreated() { - return created; - } - - public void setCreated(Date created) { - this.created = created; - } - - public String getAccount() { - return account; - } - - public void setAccount(String account) { - this.account = account; - } - - public String getUsername() { - return username; - } - - public void setUsername(String username) { - this.username = username; - } - -} diff --git a/api/src/org/apache/cloudstack/api/response/ZoneResponse.java b/api/src/org/apache/cloudstack/api/response/ZoneResponse.java index 2ebb15a1ecf..2b3e4bec062 100644 --- a/api/src/org/apache/cloudstack/api/response/ZoneResponse.java +++ b/api/src/org/apache/cloudstack/api/response/ZoneResponse.java @@ -99,6 +99,10 @@ public class ZoneResponse extends BaseResponse { @SerializedName(ApiConstants.LOCAL_STORAGE_ENABLED) @Param(description="true if local storage offering enabled, false otherwise") private boolean localStorageEnabled; + @SerializedName(ApiConstants.AFFINITY_GROUP_ID) + @Param(description = "the UUID of the affinity group associated, null for public zones") + private String affinityGroupId; + public void setId(String id) { this.id = id; } @@ -198,4 +202,8 @@ public class ZoneResponse extends BaseResponse { public void setIp6Dns2(String ip6Dns2) { this.ip6Dns2 = ip6Dns2; } + + public void setAffinityGroupId(String affinityGroupId) { + this.affinityGroupId = affinityGroupId; + } } diff --git a/api/src/org/apache/cloudstack/region/Region.java b/api/src/org/apache/cloudstack/region/Region.java index 7119f1b2c14..c696fb24c13 100644 --- a/api/src/org/apache/cloudstack/region/Region.java +++ b/api/src/org/apache/cloudstack/region/Region.java @@ -31,10 +31,11 @@ public interface Region { public void setName(String name); public String getEndPoint(); - public boolean checkIfServiceEnabled(Service service); + public void enableService(Service service); + /** * A region level service, is a service that constitute services across one or more zones in the region or a service * made available to all the zones in the region. @@ -45,6 +46,7 @@ public interface Region { private static List regionServices = new ArrayList(); public static final Service Gslb = new Service("Gslb"); + public static final Service PortableIp = new Service("PortableIp"); public Service(String name ) { this.name = name; diff --git a/api/test/org/apache/cloudstack/api/command/test/ScaleVMCmdTest.java b/api/test/org/apache/cloudstack/api/command/test/ScaleVMCmdTest.java index bb022986e2d..1e71739fe10 100644 --- a/api/test/org/apache/cloudstack/api/command/test/ScaleVMCmdTest.java +++ b/api/test/org/apache/cloudstack/api/command/test/ScaleVMCmdTest.java @@ -24,16 +24,12 @@ import org.apache.cloudstack.api.ResponseGenerator; import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.command.user.vm.ScaleVMCmd; -import org.apache.cloudstack.api.response.SwiftResponse; import org.apache.cloudstack.api.response.UserVmResponse; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.mockito.Mockito; -import static org.mockito.Matchers.anyInt; - - import java.util.LinkedList; import java.util.List; @@ -45,6 +41,7 @@ public class ScaleVMCmdTest extends TestCase{ @Rule public ExpectedException expectedException = ExpectedException.none(); + @Override @Before public void setUp() { diff --git a/awsapi/conf/applicationContext.xml.in b/awsapi/conf/applicationContext.xml.in index 8b3a0222262..fd9c871d127 100644 --- a/awsapi/conf/applicationContext.xml.in +++ b/awsapi/conf/applicationContext.xml.in @@ -32,7 +32,7 @@ - + diff --git a/docs/en-US/Book_Info_Release_Notes_4-0.xml b/docs/en-US/Book_Info_Release_Notes_4.xml similarity index 87% rename from docs/en-US/Book_Info_Release_Notes_4-0.xml rename to docs/en-US/Book_Info_Release_Notes_4.xml index 9655986cc99..e1c270f3e14 100644 --- a/docs/en-US/Book_Info_Release_Notes_4-0.xml +++ b/docs/en-US/Book_Info_Release_Notes_4.xml @@ -18,13 +18,13 @@ specific language governing permissions and limitations under the License. --> - - Version 4.1.0 Release Notes - Apache CloudStack + + Version 4.2.0 Release Notes + Apache &PRODUCT; - Release notes for the Apache CloudStack 4.1.0 release. + Release notes for the Apache &PRODUCT; 4.2.0 release. diff --git a/docs/en-US/Developers_Guide.xml b/docs/en-US/Developers_Guide.xml index 87dc8a6675a..7452e29ecf2 100644 --- a/docs/en-US/Developers_Guide.xml +++ b/docs/en-US/Developers_Guide.xml @@ -50,6 +50,8 @@ + + diff --git a/docs/en-US/Release_Notes.xml b/docs/en-US/Release_Notes.xml index a4fcd471692..0ef3b07e5f9 100644 --- a/docs/en-US/Release_Notes.xml +++ b/docs/en-US/Release_Notes.xml @@ -19,19 +19,35 @@ specific language governing permissions and limitations under the License. --> - + - - Welcome to &PRODUCT; 4.1 - Welcome to the 4.1.0 release of &PRODUCT;, the first major release from the Apache - CloudStack project since its graduation from the Apache Incubator. + + Welcome to &PRODUCT; 4.2 + Welcome to the 4.2.0 release of &PRODUCT;, the second major release from the Apache + CloudStack project since its graduation from the Apache Incubator. &PRODUCT; 4.2 includes more + than 50 new features and enhancements. The focus of the release is on three major + areas: + + + Improved support for both legacy-style and cloud-style workloads + + + New third-party plug-in architecture + + + Networking enhancements + + + In addition to these major new areas of functionality, &PRODUCT; 4.2 provides many + additional enhancements in a variety of product areas. All of the new features are summarized + later in this Release Note. This document contains information specific to this release of &PRODUCT;, including upgrade instructions from prior releases, new features added to &PRODUCT;, API changes, and issues fixed in the release. For installation instructions, please see the Installation Guide. For usage and administration instructions, please see the &PRODUCT; Administrator's Guide. Developers and users who wish to work with the API will find instruction in the If you find any errors or problems in this guide, please see . We hope you enjoy working with &PRODUCT;! + + Version 4.2.0 +
+ What’s New in 4.2 + Apache CloudStack 4.2.0 includes many new features. This section covers the most + prominent new features and changes. +
+ Windows 8 and Windows Server as VM Guest OS + Supported on XenServer, VMware, and KVM. + Windows 8 and Windows Server 2012 can now be used as OS types on guest virtual + machines. The OS would be made available the same as any other, by uploading an ISO or a + template. The instructions for uploading ISOs and templates are given in the + Administrator's Guide. + + Limitation: When used with VMware hosts, this + feature works only for the following versions: vSphere ESXi 5.1 and ESXi 5.0 Patch + 4. + +
+
+ Portable IPs + CLOUDSTACK-3236:Portable IPs in &PRODUCT; are nothing but elastic IPs that can + be transferred across geographically separated zones. As an administrator, you can + provision a pool of portable IPs at region level and are available for user consumption. + The users can acquire portable IPs if admin has provisioned portable public IPs at the + region level they are part of. These IPs can be used for any service within an advanced + zone. You can also use portable IPs for EIP service in Basic zones. Additionally, a + portable IP can be transferred from one network to another network. +
+
+ N-Tier Applications + CLOUDSTACK-770:In &PRODUCT; 3.0.6, a functionality was added to allow users to + create a multi-tier application connected to a single instance of a Virtual Router that + supports inter-VLAN routing. Such a multi-tier application is called a virtual private + cloud (VPC). Users were also able to connect their multi-tier applications to a private + Gateway or a Site-to-Site VPN tunnel and route certain traffic to those gateways. For + &PRODUCT; 4.2, additional features are implemented to enhance VPC applications. + + + Internal Load Balancing between VPC tiers + + + Source NAT and ACL support on private gateways + + + Multiple private gateway support + + + Support for ACL deny rules + + + ACL support on all layer 4 protocols + + + Support up to 8 VPN Gateways + + + Support for blacklisting routes + + + NetScaler support for VPC load balancing + + + Support for KVM hypervisor + + + Support for the ability to simultaneously deploy an instance on a VPC Tier and one + or more Shared Networks + + +
+
+ Cisco VNMC Support + CLOUDSTACK-742:&PRODUCT; supports Cisco Virtual Network Management Center + (VNMC) on Cisco Nexus 1000v dvSwich-enabled VMware hypervisors. &PRODUCT; supports Cisco + ASA 1000v as an external Firewall provider when integrated with Cisco VNMC. + When Cisco VNMC is integrated with ASA 1000v Cloud Firewall and Cisco Nexus 1000v + dvSwitch in &PRODUCT; you will be able to: + + + Configure Cisco ASA 1000v Firewalls + + + Create and apply security profiles that contain ACL policy sets for both ingress + and egress traffic, connection timeout, NAT policy sets, and TCP intercept + + + Consider the following use cases before using this feature: + + + A Cloud administrator adds VNMC as a network element by using the admin API + addCiscoVnmcResource after specifying the credentials + + + A Cloud administrator adds ASA 1000v appliances by using the admin API + addCiscoAsa1000vResource. You can configure one per guest network. + + + A Cloud administrator creates an Isolated guest network offering by using ASA + 1000v as the service provider for Firewall, Source NAT, Port Forwarding, and Static + NAT. + + +
+
+ VMware vNetwork Distributed vSwitch + CLOUDSTACK-772:&PRODUCT; 4.2 supports VMware vSphere Distributed Switch (VDS) + for virtual network configuration in a VMware vSphere environment. Each vCenter server + instance can support up to 128 VDSs and each VDS can manage up to 500 VMware hosts. +
+ About VMware Distributed Virtual Switch + VMware VDS is an aggregation of host-level virtual switches on a VMware vCenter + server. VDS abstracts the configuration of individual virtual switches that span across + a large number of hosts, and enables centralized provisioning, administration, and + monitoring for your entire datacenter from a centralized interface. VDS is controlled as + a single distributed switch at the datacenter level. So there needed a component to + ensure that the network configurations on the source and the destination virtual switch + are consistent and will allow the VM to operate without breaking connectivity or network + policies. Particularly during migration of VM across hosts, the sync up among peers need + to be taken care. However in case of distributed vSwitch during VMotion, the vCenter + server, would update the vSwitch modules on the hosts in cluster accordingly. +
+
+ Enabling Virtual Distributed Switch in &PRODUCT; + To make a &PRODUCT; deployment VDS enabled, set the vmware.use.dvswitch parameter to + true by using the Global Settings page in the &PRODUCT; UI and restart the Management + Server. Unless you enable the vmware.use.dvswitch parameter, you cannot see any UI + options specific to VDS, and &PRODUCT; ignores the VDS-specific parameters specified in + the AddCluster API call. Additionally, &PRODUCT; uses VDS for virtual network + infrastructure if the value of vmware.use.dvswitch parameter is true and the value of + vmware.use.nexus.dvswitch parameter is false. + &PRODUCT; supports configuring virtual networks in a deployment with a mix of + Virtual Distributed Switch, Standard Virtual Switch and Nexus 1000v Virtual Switch. + +
+
+
+ Health Checks for Load Balanced Instances + + CLOUDSTACK-4243: This feature is supported only on NetScaler version 10.0 and + beyond. The Nitro API is not compatible with NetScaler 9.3 and therefore this version is + not supported for this feature. + + CLOUDSTACK-816:(NetScaler load balancer only) A load balancer rule distributes + requests among a pool of services (a service in this context means an application running + on a virtual machine). When creating a load balancer rule, you can specify a health check + which will ensure that the rule forwards requests only to services that are healthy + (running and available). This is in addition to specifying the stickiness policy, + algorithm, and other load balancer rule options. You can configure one health check policy + per load balancer rule. + When a health check is in effect, the load balancer will stop forwarding requests to + any resources that it has found to be unhealthy. If the resource later becomes available + again, the periodic health check (periodicity is configurable) will discover it and the + resource will once again be added to the pool of resources that can receive requests from + the load balancer. + You can delete or modify existing health check policies. + To configure how often the health check is performed by default, use the global + configuration setting healthcheck.update.interval. This default applies to all the health + check policies in the cloud. You can override this value for an individual health check + policy. +
+
+ Snaphotting, backups, cloning and System VMs for RBD Primary Storage + + These new RBD features require at least librbd 0.61.7 (Cuttlefish) and libvirt + 0.9.14 on the KVM hypervisors. + + CLOUDSTACK-1191: + With this release &PRODUCT; will leverage the features of RBD format 2. This allows + snapshotting and backing up those snapshots. + Backups of snapshots to Secondary Storage are full copies of the RBD snapshot, they + are not RBD diffs. This because when restoring a backup of a snapshot it is not mandatory + that this backup is deployed on RBD again, it could also be a NFS Primary Storage. + Another key feature of RBD format 2 is cloning and with this release templates will be + copied to Primary Storage once and using the cloning mechanism new disks will be cloned + from this parent template. This saves space and decreases deployment time for Instances + dramatically. + Cloning will however only work with new templates and when they are freshly downloaded + to primary storage. Templates currently stored on RBD Primary Storage are in RBD format 1 + which does not support cloning. Loglevel debug on the Agent will show if cloning is used + when deploying a template or not. + Before this release a NFS Primary Storage was still required for running the System + VMs from. The reason behind this was a so called 'patch disk' which was generated by the + hypervisor which contained metadata for the System VM. The scripts generating this disk + didn't support RBD and thus System VMs had to be deployed from NFS. With 4.2 instead of + the patch disk a VirtIO serial console is used to pass meta information to System VMs. + This enabled the deployment of System VMs on RBD Primary Storage. +
+
+ Disk I/O polling and throttling + CLOUDSTACK-1192: + On KVM hypervisors polling and throttling of disk I/Os is supported. Per disk disk attached to + an Instance the usage server will record the amount of IOps. + Per disk offering you are able to specify the number of Read and Write I/Os. Trottling is + done by Qemu/KVM. + Both polling and throttling only works with KVM and with all types of Primary Storage. +
+
+
+ Issues Fixed in 4.2.0 + Apache CloudStack uses Jira to track its issues. All new features and bugs for 4.2.0 have been tracked + in Jira, and have a standard naming convention of "CLOUDSTACK-NNNN" where "NNNN" is the + issue number. + This section includes a summary of known issues against 4.0.0 that were fixed in 4.2.0. + Approximately 470 bugs were resolved or closed in the 4.2.0 cycle. + + + + + + + + Defect + + + Description + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ Known Issues in 4.2.0 + + + + + + + + Issue ID + + + Description + + + + + + CLOUDSTACK-2709 + + VM Migration across VMware clusters which are added with different switches + (Standard Swith,Vmware DVS, Cisco Nexus 1000v) is not supported.. + + + + CLOUDSTACK-4207 + + The following exception is observed when the Management Server is started + after upgrade from any older versions to &PRODUCT; 4.2. + jsonParseException: The JsonDeserializer + com.cloud.agent.transport.ArrayTypeAdaptor@2426e26f failed to deserialize json + object + Ignore this exception, this would stop after you upgrade the System VM. + However, if you want to prevent this, stop system VM from the hypervisor before + upgrade. + + + + CLOUDSTACK-2709 + + Egress rules are are not supported on shared networks. + + + + CLOUDSTACK-1747 + mvn deploydb only creates 4.0 DB, not 4.2 + Due to tooling changes between 4.2 and 4.2, CloudStack's database is created + using the 4.0 schema and updated to the 4.2 schema when the management server + starts for the first time. It's OK to see the same schema if the management server + has not started yet. + + + + + CLOUDSTACK-1306 + + + Better Error message when trying to deploy Vm by passing static Ipv4 addresses + that are assigned to another VM/IP4 address is outside the iprange. + + + + + CLOUDSTACK-1236 + + + Warning while adding Xen 6.1 host [Unable to create local link network] + + + + + CLOUDSTACK-969 + + + api: zone response lists vlan in it as "vlan range of zone" but the + vlan belongs to physical network + + + + + CLOUDSTACK-963 + + + [cloud.utils.AnnotationHelper] class java.lang.Stringdoes not have a Table + annotation + + + + + CLOUDSTACK-458 + + + xen:snapshots:Storage gc fail to clean the failed snapshot images from + secondarystorage + + + + + CLOUDSTACK-315 + + + Infrastructure view does not show capacity values + + + + + CLOUDSTACK-300 + + + Creation of compute offering allow combination of local storage + HA + + + + + CLOUDSTACK-276 + + + SSVM ID is exposed in the Error Message thrown by AddTrafficType API + + + + + CLOUDSTACK-270 + + + Ui should not ask for a vlan range if the physical network isolation type is + not VLAN + + + + + CLOUDSTACK-245 + + + VPC ACLs are not stored and programmed consistently + + + + + CLOUDSTACK-231 + + + Tag creation using special charecters + + + + + CLOUDSTACK-124 + + + NetworkGarbageCollector not cleaning up networks + + + + + CLOUDSTACK-62 + + + console proxy does not support any keymaps besides us, jp + + + + + +
+ + + Upgrade Instructions + This section contains upgrade instructions from prior versions of CloudStack to Apache + CloudStack 4.2.0. We include instructions on upgrading to Apache CloudStack from pre-Apache + versions of Citrix CloudStack (last version prior to Apache is 3.0.2) and from the releases + made while CloudStack was in the Apache Incubator. + If you run into any issues during upgrades, please feel free to ask questions on + users@cloudstack.apache.org or dev@cloudstack.apache.org. +
+ Upgrade from 4.x.x to 4.2.0 + This section will guide you from &PRODUCT; 4.0.x versions to &PRODUCT; 4.2.0. + Any steps that are hypervisor-specific will be called out with a note. + + Package Structure Changes + The package structure for &PRODUCT; has changed significantly since the 4.0.x + releases. If you've compiled your own packages, you'll notice that the package names and + the number of packages has changed. This is not a bug. + However, this does mean that the procedure is not as simple as an apt-get + upgrade or yum update, so please follow this section + carefully. + + We recommend reading through this section once or twice before beginning your upgrade + procedure, and working through it on a test system before working on a production + system. + + + Most users of &PRODUCT; manage the installation and upgrades of &PRODUCT; with one + of Linux's predominant package systems, RPM or APT. This guide assumes you'll be using + RPM and Yum (for Red Hat Enterprise Linux or CentOS), or APT and Debian packages (for + Ubuntu). + + + Create RPM or Debian packages (as appropriate) and a repository from the 4.2.0 + source, or check the Apache CloudStack downloads page at http://cloudstack.apache.org/downloads.html for package repositories supplied + by community members. You will need them for step + or step . + Instructions for creating packages from the &PRODUCT; source are in the Installation + Guide. + + + Stop your management server or servers. Run this on all management server + hosts: + # service cloud-management stop + + + If you are running a usage server or usage servers, stop those as well: + # service cloud-usage stop + + + Make a backup of your MySQL database. If you run into any issues or need to roll + back the upgrade, this will assist in debugging or restoring your existing environment. + You'll be prompted for your password. + # mysqldump -u root -p cloud > cloudstack-backup.sql + + + If you have made changes to + /etc/cloud/management/components.xml, you'll need to carry these + over manually to the new file, + /etc/cloudstack/management/componentContext.xml. This is not done + automatically. (If you're unsure, we recommend making a backup of the original + components.xml to be on the safe side. + + + After upgrading to 4.2, API clients are expected to send plain text passwords for + login and user creation, instead of MD5 hash. If API client changes are not acceptable, + following changes are to be made for backward compatibility: + Modify componentsContext.xml, and make PlainTextUserAuthenticator as the default + authenticator (1st entry in the userAuthenticators adapter list is default) + +<!-- Security adapters --> +<bean id="userAuthenticators" class="com.cloud.utils.component.AdapterList"> + <property name="Adapters"> + <list> + <ref bean="PlainTextUserAuthenticator"/> + <ref bean="MD5UserAuthenticator"/> + <ref bean="LDAPUserAuthenticator"/> + </list> + </property> +</bean> + + PlainTextUserAuthenticator works the same way MD5UserAuthenticator worked prior to + 4.2. + + + If you are using Ubuntu, follow this procedure to upgrade your packages. If not, + skip to step . + + Community Packages + This section assumes you're using the community supplied packages for &PRODUCT;. + If you've created your own packages and APT repository, substitute your own URL for + the ones used in these examples. + + + + The first order of business will be to change the sources list for each system + with &PRODUCT; packages. This means all management servers, and any hosts that have + the KVM agent. (No changes should be necessary for hosts that are running VMware or + Xen.) + Start by opening /etc/apt/sources.list.d/cloudstack.list on + any systems that have &PRODUCT; packages installed. + This file should have one line, which contains: + deb http://cloudstack.apt-get.eu/ubuntu precise 4.0 + We'll change it to point to the new package repository: + deb http://cloudstack.apt-get.eu/ubuntu precise 4.2 + If you're using your own package repository, change this line to read as + appropriate for your 4.2.0 repository. + + + Now update your apt package list: + $ sudo apt-get update + + + Now that you have the repository configured, it's time to install the + cloudstack-management package. This will pull in any other + dependencies you need. + $ sudo apt-get install cloudstack-management + + + You will need to manually install the cloudstack-agent + package: + $ sudo apt-get install cloudstack-agent + During the installation of cloudstack-agent, APT will copy + your agent.properties, log4j-cloud.xml, + and environment.properties from + /etc/cloud/agent to + /etc/cloudstack/agent. + When prompted whether you wish to keep your configuration, say Yes. + + + Verify that the file + /etc/cloudstack/agent/environment.properties has a line that + reads: + paths.script=/usr/share/cloudstack-common + If not, add the line. + + + Restart the agent: + +service cloud-agent stop +killall jsvc +service cloudstack-agent start + + + + During the upgrade, log4j-cloud.xml was simply copied over, + so the logs will continue to be added to + /var/log/cloud/agent/agent.log. There's nothing + wrong with this, but if you prefer to be consistent, you can + change this by copying over the sample configuration file: + +cd /etc/cloudstack/agent +mv log4j-cloud.xml.dpkg-dist log4j-cloud.xml +service cloudstack-agent restart + + + + Once the agent is running, you can uninstall the old cloud-* packages from your + system: + sudo dpkg --purge cloud-agent + + + + + (VMware only) Additional steps are required for each VMware cluster. These steps + will not affect running guests in the cloud. These steps are required only for clouds + using VMware clusters: + + + Stop the Management Server: + service cloudstack-management stop + + + Generate the encrypted equivalent of your vCenter password: + java -classpath /usr/share/cloudstack-common/lib/jasypt-1.9.0.jar org.jasypt.intf.cli.JasyptPBEStringEncryptionCLI encrypt.sh input="_your_vCenter_password_" password="`cat /etc/cloudstack/management/key`" verbose=false + Store the output from this step, we need to add this in cluster_details table + and vmware_data_center tables in place of the plain text password + + + Find the ID of the row of cluster_details table that you have to update: + mysql -u <username> -p<password> + select * from cloud.cluster_details; + + + Update the plain text password with the encrypted one + update cloud.cluster_details set value = '_ciphertext_from_step_1_' where id = _id_from_step_2_; + + + Confirm that the table is updated: + select * from cloud.cluster_details; + + + Find the ID of the correct row of vmware_data_center that you want to + update + select * from cloud.vmware_data_center; + + + update the plain text password with the encrypted one: + update cloud.vmware_data_center set password = '_ciphertext_from_step_1_' where id = _id_from_step_5_; + + + Confirm that the table is updated: + select * from cloud.vmware_data_center; + + + Start the &PRODUCT; Management server + service cloudstack-management start + + + + + (KVM only) Additional steps are required for each KVM host. These steps will not + affect running guests in the cloud. These steps are required only for clouds using KVM + as hosts and only on the KVM hosts. + + + Manually clean up /var/cache/cloudstack. + + + Copy the 4.2 tar file to the host, untar it, and change directory to the + resulting directory. + + + Stop the running agent. + # service cloud-agent stop + + + Update the agent software. + # ./install.sh + + + Choose "U" to update the packages. + + + Start the agent. + # service cloudstack-agent start + + + + + If you are using CentOS or RHEL, follow this procedure to upgrade your packages. If + not, skip to step . + + Community Packages + This section assumes you're using the community supplied packages for &PRODUCT;. + If you've created your own packages and yum repository, substitute your own URL for + the ones used in these examples. + + + + The first order of business will be to change the yum repository for each system + with &PRODUCT; packages. This means all management servers, and any hosts that have + the KVM agent. + (No changes should be necessary for hosts that are running VMware or + Xen.) + Start by opening /etc/yum.repos.d/cloudstack.repo on any + systems that have &PRODUCT; packages installed. + This file should have content similar to the following: + +[apache-cloudstack] +name=Apache CloudStack +baseurl=http://cloudstack.apt-get.eu/rhel/4.0/ +enabled=1 +gpgcheck=0 + + If you are using the community provided package repository, change the base url + to http://cloudstack.apt-get.eu/rhel/4.2/ + If you're using your own package repository, change this line to read as + appropriate for your 4.2.0 repository. + + + Now that you have the repository configured, it's time to install the + cloudstack-management package by upgrading the older + cloud-client package. + $ sudo yum upgrade cloud-client + + + For KVM hosts, you will need to upgrade the cloud-agent + package, similarly installing the new version as + cloudstack-agent. + $ sudo yum upgrade cloud-agent + During the installation of cloudstack-agent, the RPM will + copy your agent.properties, + log4j-cloud.xml, and + environment.properties from + /etc/cloud/agent to + /etc/cloudstack/agent. + + + For CentOS 5.5, perform the following: + + + Run the following command: + rpm -Uvh http://download.cloud.com/support/jsvc/jakarta-commons-daemon-jsvc-1.0.1-8.9.el6.x86_64.rpm + + + Upgrade the Usage server. + sudo yum upgrade cloud-usage + + After upgrade, if the usage server fails to restart then copy + db.properties from /etc/cloudstack/management to /etc/cloudstack/usage. Then start the + Usage Server. + + + + + + Verify that the file + /etc/cloudstack/agent/environment.properties has a line that + reads: + paths.script=/usr/share/cloudstack-common + If not, add the line. + + + Restart the agent: + +service cloud-agent stop +killall jsvc +service cloudstack-agent start + + + + + + Once you've upgraded the packages on your management servers, you'll need to restart + the system VMs. Make sure port 8096 is open in your local host firewall to do + this. + There is a script that will do this for you, all you need to do is run the script + and supply the IP address for your MySQL instance and your MySQL credentials: + # nohup cloudstack-sysvmadm -d IP address -u cloud -p -a > sysvm.log 2>&1 & + You can monitor the log for progress. The process of restarting the system VMs can + take an hour or more. + # tail -f sysvm.log + The output to sysvm.log will look something like this: + +Stopping and starting 1 secondary storage vm(s)... +Done stopping and starting secondary storage vm(s) +Stopping and starting 1 console proxy vm(s)... +Done stopping and starting console proxy vm(s). +Stopping and starting 4 running routing vm(s)... +Done restarting router(s). + + + + + For Xen Hosts: Copy vhd-utils + This step is only for CloudStack installs that are using Xen hosts. + + Copy the file vhd-utils to + /usr/share/cloudstack-common/scripts/vm/hypervisor/xenserver. + + +
+
+ Upgrade from 3.0.2 to 4.2.0 + This section will guide you from Citrix CloudStack 3.0.2 to Apache CloudStack 4.2.0. + Sections that are hypervisor-specific will be called out with a note. + + + + The following upgrade instructions apply only if you're using VMware hosts. If + you're not using VMware hosts, skip this step and move on to . + + In each zone that includes VMware hosts, you need to add a new system VM template. + + + While running the existing 3.0.2 system, log in to the UI as root + administrator. + + + In the left navigation bar, click Templates. + + + In Select view, click Templates. + + + Click Register template. + The Register template dialog box is displayed. + + + In the Register template dialog box, specify the following values (do not change + these): + + + + + + + Field + Value + + + + + Name + systemvm-vmware-4.2 + + + Description + systemvm-vmware-4.2 + + + URL + http://download.cloud.com/templates/burbank/burbank-systemvm-08012012.ova + + + Zone + Choose the zone where this hypervisor is used + + + Hypervisor + VMware + + + Format + OVA + + + OS Type + Debian GNU/Linux 5.0 (32-bit) + + + Extractable + no + + + Password Enabled + no + + + Public + no + + + Featured + no + + + + + + + Watch the screen to be sure that the template downloads successfully and enters + the READY state. Do not proceed until this is successful. + + + + + Stop all Usage Servers if running. Run this on all Usage Server hosts. + # service cloud-usage stop + + + Stop the Management Servers. Run this on all Management Server hosts. + # service cloud-management stop + + + On the MySQL master, take a backup of the MySQL databases. We recommend performing + this step even in test upgrades. If there is an issue, this will assist with + debugging. + In the following commands, it is assumed that you have set the root password on the + database, which is a CloudStack recommended best practice. Substitute your own MySQL + root password. + # mysqldump -u root -pmysql_password cloud > cloud-backup.dmp + # mysqldump -u root -pmysql_password cloud_usage > cloud-usage-backup.dmp + + + Either build RPM/DEB packages as detailed in the Installation Guide, or use one of + the community provided yum/apt repositories to gain access to the &PRODUCT; + binaries. + + + If you are using Ubuntu, follow this procedure to upgrade your packages. If not, + skip to step . + + Community Packages + This section assumes you're using the community supplied packages for &PRODUCT;. + If you've created your own packages and APT repository, substitute your own URL for + the ones used in these examples. + + + + The first order of business will be to change the sources list for each system + with &PRODUCT; packages. This means all management servers, and any hosts that have + the KVM agent. (No changes should be necessary for hosts that are running VMware or + Xen.) + Start by opening /etc/apt/sources.list.d/cloudstack.list on + any systems that have &PRODUCT; packages installed. + This file should have one line, which contains: + deb http://cloudstack.apt-get.eu/ubuntu precise 4.0 + We'll change it to point to the new package repository: + deb http://cloudstack.apt-get.eu/ubuntu precise 4.2 + If you're using your own package repository, change this line to read as + appropriate for your 4.2.0 repository. + + + Now update your apt package list: + $ sudo apt-get update + + + Now that you have the repository configured, it's time to install the + cloudstack-management package. This will pull in any other + dependencies you need. + $ sudo apt-get install cloudstack-management + + + You will need to manually install the cloudstack-agent + package: + $ sudo apt-get install cloudstack-agent + During the installation of cloudstack-agent, APT will copy + your agent.properties, log4j-cloud.xml, + and environment.properties from + /etc/cloud/agent to + /etc/cloudstack/agent. + When prompted whether you wish to keep your configuration, say Yes. + + + Verify that the file + /etc/cloudstack/agent/environment.properties has a line that + reads: + paths.script=/usr/share/cloudstack-common + If not, add the line. + + + Restart the agent: + +service cloud-agent stop +killall jsvc +service cloudstack-agent start + + + + During the upgrade, log4j-cloud.xml was simply copied over, + so the logs will continue to be added to + /var/log/cloud/agent/agent.log. There's nothing + wrong with this, but if you prefer to be consistent, you can + change this by copying over the sample configuration file: + +cd /etc/cloudstack/agent +mv log4j-cloud.xml.dpkg-dist log4j-cloud.xml +service cloudstack-agent restart + + + + Once the agent is running, you can uninstall the old cloud-* packages from your + system: + sudo dpkg --purge cloud-agent + + + + + (KVM only) Additional steps are required for each KVM host. These steps will not + affect running guests in the cloud. These steps are required only for clouds using KVM + as hosts and only on the KVM hosts. + + + Copy the CloudPlatform 4.2 tar file to the host, untar it, and change directory + to the resulting directory. + + + Stop the running agent. + # service cloud-agent stop + + + Update the agent software. + # ./install.sh + + + Choose "U" to update the packages. + + + Start the agent. + # service cloudstack-agent start + + + + + If you are using CentOS or RHEL, follow this procedure to upgrade your packages. If + not, skip to step . + + Community Packages + This section assumes you're using the community supplied packages for &PRODUCT;. + If you've created your own packages and yum repository, substitute your own URL for + the ones used in these examples. + + + + The first order of business will be to change the yum repository for each system + with &PRODUCT; packages. This means all management servers, and any hosts that have + the KVM agent. (No changes should be necessary for hosts that are running VMware or + Xen.) + Start by opening /etc/yum.repos.d/cloudstack.repo on any + systems that have &PRODUCT; packages installed. + This file should have content similar to the following: + +[apache-cloudstack] +name=Apache CloudStack +baseurl=http://cloudstack.apt-get.eu/rhel/4.0/ +enabled=1 +gpgcheck=0 + + If you are using the community provided package repository, change the baseurl + to http://cloudstack.apt-get.eu/rhel/4.2/ + If you're using your own package repository, change this line to read as + appropriate for your 4.2.0 repository. + + + Now that you have the repository configured, it's time to install the + cloudstack-management package by upgrading the older + cloud-client package. + $ sudo yum upgrade cloud-client + + + For KVM hosts, you will need to upgrade the cloud-agent + package, similarly installing the new version as + cloudstack-agent. + $ sudo yum upgrade cloud-agent + During the installation of cloudstack-agent, the RPM will + copy your agent.properties, + log4j-cloud.xml, and + environment.properties from + /etc/cloud/agent to + /etc/cloudstack/agent. + + + Verify that the file + /etc/cloudstack/agent/environment.properties has a line that + reads: + paths.script=/usr/share/cloudstack-common + If not, add the line. + + + Restart the agent: + +service cloud-agent stop +killall jsvc +service cloudstack-agent start + + + + + + If you have made changes to your copy of + /etc/cloud/management/components.xml the changes will be + preserved in the upgrade. However, you need to do the following steps to place these + changes in a new version of the file which is compatible with version 4.2.0. + + + Make a backup copy of /etc/cloud/management/components.xml. + For example: + # mv /etc/cloud/management/components.xml /etc/cloud/management/components.xml-backup + + + Copy /etc/cloud/management/components.xml.rpmnew to create + a new /etc/cloud/management/components.xml: + # cp -ap /etc/cloud/management/components.xml.rpmnew /etc/cloud/management/components.xml + + + Merge your changes from the backup file into the new + components.xml. + # vi /etc/cloudstack/management/components.xml + + + + If you have more than one management server node, repeat the upgrade steps on each + node. + + + + After upgrading to 4.2, API clients are expected to send plain text passwords for + login and user creation, instead of MD5 hash. Incase, api client changes are not + acceptable, following changes are to be made for backward compatibility: + Modify componentsContext.xml, and make PlainTextUserAuthenticator as the default + authenticator (1st entry in the userAuthenticators adapter list is default) + +<!-- Security adapters --> +<bean id="userAuthenticators" class="com.cloud.utils.component.AdapterList"> + <property name="Adapters"> + <list> + <ref bean="PlainTextUserAuthenticator"/> + <ref bean="MD5UserAuthenticator"/> + <ref bean="LDAPUserAuthenticator"/> + </list> + </property> +</bean> + + PlainTextUserAuthenticator works the same way MD5UserAuthenticator worked prior to + 4.2. + + + Start the first Management Server. Do not start any other Management Server nodes + yet. + # service cloudstack-management start + Wait until the databases are upgraded. Ensure that the database upgrade is complete. + After confirmation, start the other Management Servers one at a time by running the same + command on each node. + + Failing to restart the Management Server indicates a problem in the upgrade. + Having the Management Server restarted without any issues indicates that the upgrade + is successfully completed. + + + + Start all Usage Servers (if they were running on your previous version). Perform + this on each Usage Server host. + # service cloudstack-usage start + + After upgrade, if the usage server fails to restart then copy + db.properties from /etc/cloudstack/management to /etc/cloudstack/usage. Then start the + Usage Server. + + + + Additional steps are required for each KVM host. These steps will not affect running + guests in the cloud. These steps are required only for clouds using KVM as hosts and + only on the KVM hosts. + + + Configure a yum or apt repository containing the &PRODUCT; packages as outlined + in the Installation Guide. + + + Stop the running agent. + # service cloud-agent stop + + + Update the agent software with one of the following command sets as appropriate + for your environment. + # yum update cloud-* + # apt-get update + # apt-get upgrade cloud-* + + + Edit /etc/cloudstack/agent/agent.properties to change the + resource parameter from + "com.cloud.agent.resource.computing.LibvirtComputingResource" to + "com.cloud.hypervisor.kvm.resource.LibvirtComputingResource". + + + Start the cloud agent and cloud management services. + # service cloudstack-agent start + + + When the Management Server is up and running, log in to the CloudStack UI and + restart the virtual router for proper functioning of all the features. + + + + + Log in to the CloudStack UI as administrator, and check the status of the hosts. All + hosts should come to Up state (except those that you know to be offline). You may need + to wait 20 or 30 minutes, depending on the number of hosts. + + Troubleshooting: If login fails, clear your browser cache and reload the + page. + + Do not proceed to the next step until the hosts show in Up state. + + + If you are upgrading from 3.0.2, perform the following: + + + Ensure that the admin port is set to 8096 by using the "integration.api.port" + global parameter. + This port is used by the cloud-sysvmadm script at the end of the upgrade + procedure. For information about how to set this parameter, see "Setting Global + Configuration Parameters" in the Installation Guide. + + + Restart the Management Server. + + If you don't want the admin port to remain open, you can set it to null after + the upgrade is done and restart the management server. + + + + + + Run the cloud-sysvmadm script to stop, then start, all Secondary + Storage VMs, Console Proxy VMs, and virtual routers. Run the script once on each + management server. Substitute your own IP address of the MySQL instance, the MySQL user + to connect as, and the password to use for that user. In addition to those parameters, + provide the -c and -r arguments. For + example: + # nohup cloud-sysvmadm -d 192.168.1.5 -u cloud -p password -c -r > + sysvm.log 2>&1 & + # tail -f sysvm.log + This might take up to an hour or more to run, depending on the number of accounts in + the system. + + + If needed, upgrade all Citrix XenServer hypervisor hosts in your cloud to a version + supported by CloudStack 4.2.0. The supported versions are XenServer 5.6 SP2 and 6.0.2. + Instructions for upgrade can be found in the CloudStack 4.2.0 Installation Guide under + "Upgrading XenServer Versions." + + + Now apply the XenServer hotfix XS602E003 (and any other needed hotfixes) to + XenServer v6.0.2 hypervisor hosts. + + + Disconnect the XenServer cluster from CloudStack. + In the left navigation bar of the CloudStack UI, select Infrastructure. Under + Clusters, click View All. Select the XenServer cluster and click Actions - + Unmanage. + This may fail if there are hosts not in one of the states Up, Down, + Disconnected, or Alert. You may need to fix that before unmanaging this + cluster. + Wait until the status of the cluster has reached Unmanaged. Use the CloudStack + UI to check on the status. When the cluster is in the unmanaged state, there is no + connection to the hosts in the cluster. + + + To clean up the VLAN, log in to one XenServer host and run: + /opt/xensource/bin/cloud-clean-vlan.sh + + + Now prepare the upgrade by running the following on one XenServer host: + /opt/xensource/bin/cloud-prepare-upgrade.sh + If you see a message like "can't eject CD", log in to the VM and unmount the CD, + then run this script again. + + + Upload the hotfix to the XenServer hosts. Always start with the Xen pool master, + then the slaves. Using your favorite file copy utility (e.g. WinSCP), copy the + hotfixes to the host. Place them in a temporary folder such as /tmp. + On the Xen pool master, upload the hotfix with this command: + xe patch-upload file-name=XS602E003.xsupdate + Make a note of the output from this command, which is a UUID for the hotfix + file. You'll need it in another step later. + + (Optional) If you are applying other hotfixes as well, you can repeat the + commands in this section with the appropriate hotfix number. For example, + XS602E004.xsupdate. + + + + Manually live migrate all VMs on this host to another host. First, get a list of + the VMs on this host: + # xe vm-list + Then use this command to migrate each VM. Replace the example host name and VM + name with your own: + # xe vm-migrate live=true host=host-name + vm=VM-name + + Troubleshooting + If you see a message like "You attempted an operation on a VM which requires + PV drivers to be installed but the drivers were not detected," run: + /opt/xensource/bin/make_migratable.sh + b6cf79c8-02ee-050b-922f-49583d9f1a14. + + + + Apply the hotfix. First, get the UUID of this host: + # xe host-list + Then use the following command to apply the hotfix. Replace the example host + UUID with the current host ID, and replace the hotfix UUID with the output from the + patch-upload command you ran on this machine earlier. You can also get the hotfix + UUID by running xe patch-list. + xe patch-apply host-uuid=host-uuid uuid=hotfix-uuid + + + Copy the following files from the CloudStack Management Server to the + host. + + + + + + + Copy from here... + ...to here + + + + + /usr/lib64/cloud/common/scripts/vm/hypervisor/xenserver/xenserver60/NFSSR.py + /opt/xensource/sm/NFSSR.py + + + /usr/lib64/cloud/common/scripts/vm/hypervisor/xenserver/setupxenserver.sh + /opt/xensource/bin/setupxenserver.sh + + + /usr/lib64/cloud/common/scripts/vm/hypervisor/xenserver/make_migratable.sh + /opt/xensource/bin/make_migratable.sh + + + + + + + (Only for hotfixes XS602E005 and XS602E007) You need to apply a new Cloud + Support Pack. + + + Download the CSP software onto the XenServer host from one of the following + links: + For hotfix XS602E005: http://coltrane.eng.hq.xensource.com/release/XenServer-6.x/XS-6.0.2/hotfixes/XS602E005/56710/xe-phase-2/xenserver-cloud-supp.tgz + For hotfix XS602E007: http://coltrane.eng.hq.xensource.com/release/XenServer-6.x/XS-6.0.2/hotfixes/XS602E007/57824/xe-phase-2/xenserver-cloud-supp.tgz + + + Extract the file: + # tar xf xenserver-cloud-supp.tgz + + + Run the following script: + # xe-install-supplemental-pack xenserver-cloud-supp.iso + + + If the XenServer host is part of a zone that uses basic networking, disable + Open vSwitch (OVS): + # xe-switch-network-backend bridge + + + + + Reboot this XenServer host. + + + Run the following: + /opt/xensource/bin/setupxenserver.sh + + If the message "mv: cannot stat `/etc/cron.daily/logrotate': No such file or + directory" appears, you can safely ignore it. + + + + Run the following: + for pbd in `xe pbd-list currently-attached=false| grep ^uuid | awk '{print $NF}'`; do xe pbd-plug uuid=$pbd ; + + + On each slave host in the Xen pool, repeat these steps, starting from "manually + live migrate VMs." + + + + + + Troubleshooting Tip + If passwords which you know to be valid appear not to work after upgrade, or other UI + issues are seen, try clearing your browser cache and reloading the UI page. + +
+
+ Upgrade from 2.2.14 to 4.2.0 + + + Ensure that you query your IPaddress usage records and process them; for example, + issue invoices for any usage that you have not yet billed users for. + Starting in 3.0.2, the usage record format for IP addresses is the same as the rest + of the usage types. Instead of a single record with the assignment and release dates, + separate records are generated per aggregation period with start and end dates. After + upgrading to 4.2.0, any existing IP address usage records in the old format will no + longer be available. + + + If you are using version 2.2.0 - 2.2.13, first upgrade to 2.2.14 by using the + instructions in the 2.2.14 + Release Notes. + + KVM Hosts + If KVM hypervisor is used in your cloud, be sure you completed the step to insert + a valid username and password into the host_details table on each KVM node as + described in the 2.2.14 Release Notes. This step is critical, as the database will be + encrypted after the upgrade to 4.2.0. + + + + While running the 2.2.14 system, log in to the UI as root administrator. + + + Using the UI, add a new System VM template for each hypervisor type that is used in + your cloud. In each zone, add a system VM template for each hypervisor used in that + zone + + + In the left navigation bar, click Templates. + + + In Select view, click Templates. + + + Click Register template. + The Register template dialog box is displayed. + + + In the Register template dialog box, specify the following values depending on + the hypervisor type (do not change these): + + + + + + + Hypervisor + Description + + + + + XenServer + Name: systemvm-xenserver-4.2.0 + Description: systemvm-xenserver-4.2.0 + URL:http://download.cloud.com/templates/4.2/systemvmtemplate-2013-07-12-master-xen.vhd.bz2 + Zone: Choose the zone where this hypervisor is used + Hypervisor: XenServer + Format: VHD + OS Type: Debian GNU/Linux 6.0 (32-bit) + Extractable: no + Password Enabled: no + Public: no + Featured: no + + + + KVM + Name: systemvm-kvm-4.2.0 + Description: systemvm-kvm-4.2.0 + URL: + http://download.cloud.com/templates/4.2/systemvmtemplate-2013-06-12-master-kvm.qcow2.bz2 + Zone: Choose the zone where this hypervisor is used + Hypervisor: KVM + Format: QCOW2 + OS Type: Debian GNU/Linux 5.0 (32-bit) + Extractable: no + Password Enabled: no + Public: no + Featured: no + + + + VMware + Name: systemvm-vmware-4.2.0 + Description: systemvm-vmware-4.2.0 + URL: + http://download.cloud.com/templates/4.2/systemvmtemplate-4.2-vh7.ova + Zone: Choose the zone where this hypervisor is used + Hypervisor: VMware + Format: OVA + OS Type: Debian GNU/Linux 5.0 (32-bit) + Extractable: no + Password Enabled: no + Public: no + Featured: no + + + + + + + + + + Watch the screen to be sure that the template downloads successfully and enters the + READY state. Do not proceed until this is successful + + + WARNING: If you use more than one type of + hypervisor in your cloud, be sure you have repeated these steps to download the system + VM template for each hypervisor type. Otherwise, the upgrade will fail. + + + Stop all Usage Servers if running. Run this on all Usage Server hosts. + # service cloud-usage stop + + + Stop the Management Servers. Run this on all Management Server hosts. + # service cloud-management stop + + + On the MySQL master, take a backup of the MySQL databases. We recommend performing + this step even in test upgrades. If there is an issue, this will assist with + debugging. + In the following commands, it is assumed that you have set the root password on the + database, which is a CloudStack recommended best practice. Substitute your own MySQL + root password. + # mysqldump -u root -pmysql_password cloud > cloud-backup.dmp + # mysqldump -u root -pmysql_password cloud_usage > cloud-usage-backup.dmp + + + + Either build RPM/DEB packages as detailed in the Installation Guide, or use one of + the community provided yum/apt repositories to gain access to the &PRODUCT; binaries. + + + + If you are using Ubuntu, follow this procedure to upgrade your packages. If not, + skip to step . + + Community Packages + This section assumes you're using the community supplied packages for &PRODUCT;. + If you've created your own packages and APT repository, substitute your own URL for + the ones used in these examples. + + + + The first order of business will be to change the sources list for each system + with &PRODUCT; packages. This means all management servers, and any hosts that have + the KVM agent. (No changes should be necessary for hosts that are running VMware or + Xen.) + Start by opening /etc/apt/sources.list.d/cloudstack.list on + any systems that have &PRODUCT; packages installed. + This file should have one line, which contains: + deb http://cloudstack.apt-get.eu/ubuntu precise 4.0 + We'll change it to point to the new package repository: + deb http://cloudstack.apt-get.eu/ubuntu precise 4.2 + If you're using your own package repository, change this line to read as + appropriate for your 4.2.0 repository. + + + Now update your apt package list: + $ sudo apt-get update + + + Now that you have the repository configured, it's time to install the + cloudstack-management package. This will pull in any other + dependencies you need. + $ sudo apt-get install cloudstack-management + + + On KVM hosts, you will need to manually install the + cloudstack-agent package: + $ sudo apt-get install cloudstack-agent + During the installation of cloudstack-agent, APT will copy + your agent.properties, log4j-cloud.xml, + and environment.properties from + /etc/cloud/agent to + /etc/cloudstack/agent. + When prompted whether you wish to keep your configuration, say Yes. + + + Verify that the file + /etc/cloudstack/agent/environment.properties has a line that + reads: + paths.script=/usr/share/cloudstack-common + If not, add the line. + + + Restart the agent: + +service cloud-agent stop +killall jsvc +service cloudstack-agent start + + + + During the upgrade, log4j-cloud.xml was simply copied over, + so the logs will continue to be added to + /var/log/cloud/agent/agent.log. There's nothing + wrong with this, but if you prefer to be consistent, you can + change this by copying over the sample configuration file: + +cd /etc/cloudstack/agent +mv log4j-cloud.xml.dpkg-dist log4j-cloud.xml +service cloudstack-agent restart + + + + Once the agent is running, you can uninstall the old cloud-* packages from your + system: + sudo dpkg --purge cloud-agent + + + + + If you are using CentOS or RHEL, follow this procedure to upgrade your packages. If + not, skip to step . + + Community Packages + This section assumes you're using the community supplied packages for &PRODUCT;. + If you've created your own packages and yum repository, substitute your own URL for + the ones used in these examples. + + + + The first order of business will be to change the yum repository for each system + with &PRODUCT; packages. This means all management servers, and any hosts that have + the KVM agent. (No changes should be necessary for hosts that are running VMware or + Xen.) + Start by opening /etc/yum.repos.d/cloudstack.repo on any + systems that have &PRODUCT; packages installed. + This file should have content similar to the following: + +[apache-cloudstack] +name=Apache CloudStack +baseurl=http://cloudstack.apt-get.eu/rhel/4.0/ +enabled=1 +gpgcheck=0 + + If you are using the community provided package repository, change the baseurl + to http://cloudstack.apt-get.eu/rhel/4.2/ + If you're using your own package repository, change this line to read as + appropriate for your 4.2.0 repository. + + + Now that you have the repository configured, it's time to install the + cloudstack-management package by upgrading the older + cloud-client package. + $ sudo yum upgrade cloud-client + + + For KVM hosts, you will need to upgrade the cloud-agent + package, similarly installing the new version as + cloudstack-agent. + $ sudo yum upgrade cloud-agent + During the installation of cloudstack-agent, the RPM will + copy your agent.properties, + log4j-cloud.xml, and + environment.properties from + /etc/cloud/agent to + /etc/cloudstack/agent. + + + Verify that the file + /etc/cloudstack/agent/environment.properties has a line that + reads: + paths.script=/usr/share/cloudstack-common + If not, add the line. + + + Restart the agent: + +service cloud-agent stop +killall jsvc +service cloudstack-agent start + + + + + + If you have made changes to your existing copy of the file components.xml in your + previous-version CloudStack installation, the changes will be preserved in the upgrade. + However, you need to do the following steps to place these changes in a new version of + the file which is compatible with version 4.0.0-incubating. + + How will you know whether you need to do this? If the upgrade output in the + previous step included a message like the following, then some custom content was + found in your old components.xml, and you need to merge the two files: + + warning: /etc/cloud/management/components.xml created as /etc/cloud/management/components.xml.rpmnew + + + Make a backup copy of your + /etc/cloud/management/components.xml file. For + example: + # mv /etc/cloud/management/components.xml /etc/cloud/management/components.xml-backup + + + Copy /etc/cloud/management/components.xml.rpmnew to create + a new /etc/cloud/management/components.xml: + # cp -ap /etc/cloud/management/components.xml.rpmnew /etc/cloud/management/components.xml + + + Merge your changes from the backup file into the new components.xml file. + # vi /etc/cloud/management/components.xml + + + + + + After upgrading to 4.2, API clients are expected to send plain text passwords for + login and user creation, instead of MD5 hash. If API client changes are not acceptable, + following changes are to be made for backward compatibility: + Modify componentsContext.xml, and make PlainTextUserAuthenticator as the default + authenticator (1st entry in the userAuthenticators adapter list is default) + +<!-- Security adapters --> +<bean id="userAuthenticators" class="com.cloud.utils.component.AdapterList"> + <property name="Adapters"> + <list> + <ref bean="PlainTextUserAuthenticator"/> + <ref bean="MD5UserAuthenticator"/> + <ref bean="LDAPUserAuthenticator"/> + </list> + </property> +</bean> + + PlainTextUserAuthenticator works the same way MD5UserAuthenticator worked prior to + 4.2. + + + If you have made changes to your existing copy of the + /etc/cloud/management/db.properties file in your previous-version + CloudStack installation, the changes will be preserved in the upgrade. However, you need + to do the following steps to place these changes in a new version of the file which is + compatible with version 4.0.0-incubating. + + + Make a backup copy of your file + /etc/cloud/management/db.properties. For example: + # mv /etc/cloud/management/db.properties /etc/cloud/management/db.properties-backup + + + Copy /etc/cloud/management/db.properties.rpmnew to create a + new /etc/cloud/management/db.properties: + # cp -ap /etc/cloud/management/db.properties.rpmnew etc/cloud/management/db.properties + + + Merge your changes from the backup file into the new db.properties file. + # vi /etc/cloud/management/db.properties + + + + + On the management server node, run the following command. It is recommended that you + use the command-line flags to provide your own encryption keys. See Password and Key + Encryption in the Installation Guide. + # cloudstack-setup-encryption -e encryption_type -m management_server_key -k database_key + When used without arguments, as in the following example, the default encryption + type and keys will be used: + + + (Optional) For encryption_type, use file or web to indicate the technique used + to pass in the database encryption password. Default: file. + + + (Optional) For management_server_key, substitute the default key that is used to + encrypt confidential parameters in the properties file. Default: password. It is + highly recommended that you replace this with a more secure value + + + (Optional) For database_key, substitute the default key that is used to encrypt + confidential parameters in the CloudStack database. Default: password. It is highly + recommended that you replace this with a more secure value. + + + + + Repeat steps 10 - 14 on every management server node. If you provided your own + encryption key in step 14, use the same key on all other management servers. + + + Start the first Management Server. Do not start any other Management Server nodes + yet. + # service cloudstack-management start + Wait until the databases are upgraded. Ensure that the database upgrade is complete. + You should see a message like "Complete! Done." After confirmation, start the other + Management Servers one at a time by running the same command on each node. + + + Start all Usage Servers (if they were running on your previous version). Perform + this on each Usage Server host. + # service cloudstack-usage start + + + (KVM only) Additional steps are required for each KVM host. These steps will not + affect running guests in the cloud. These steps are required only for clouds using KVM + as hosts and only on the KVM hosts. + + + Copy the CloudPlatform 4.2 tar file to the host, untar it, and change directory + to the resulting directory. + + + Stop the running agent. + # service cloud-agent stop + + + Update the agent software. + # ./install.sh + + + Choose "U" to update the packages. + + + Start the agent. + # service cloudstack-agent start + + + + + (KVM only) Perform the following additional steps on each KVM host. + These steps will not affect running guests in the cloud. These steps are required + only for clouds using KVM as hosts and only on the KVM hosts. + + + Configure your CloudStack package repositories as outlined in the Installation + Guide + + + Stop the running agent. + # service cloud-agent stop + + + Update the agent software with one of the following command sets as + appropriate. + # yum update cloud-* + + # apt-get update + # apt-get upgrade cloud-* + + + + Start the agent. + # service cloudstack-agent start + + + Copy the contents of the agent.properties file to the new + agent.properties file by using the following command + sed -i 's/com.cloud.agent.resource.computing.LibvirtComputingResource/com.cloud.hypervisor.kvm.resource.LibvirtComputingResource/g' /etc/cloud/agent/agent.properties + + + Start the cloud agent and cloud management services. + + + When the Management Server is up and running, log in to the CloudStack UI and + restart the virtual router for proper functioning of all the features. + + + + + Log in to the CloudStack UI as admin, and check the status of the hosts. All hosts + should come to Up state (except those that you know to be offline). You may need to wait + 20 or 30 minutes, depending on the number of hosts. + Do not proceed to the next step until the hosts show in the Up state. If the hosts + do not come to the Up state, contact support. + + + Run the following script to stop, then start, all Secondary Storage VMs, Console + Proxy VMs, and virtual routers. + + + Run the command once on one management server. Substitute your own IP address of + the MySQL instance, the MySQL user to connect as, and the password to use for that + user. In addition to those parameters, provide the "-c" and "-r" arguments. For + example: + # nohup cloud-sysvmadm -d 192.168.1.5 -u cloud -p password -c -r > sysvm.log 2>&1 & + # tail -f sysvm.log + This might take up to an hour or more to run, depending on the number of + accounts in the system. + + + After the script terminates, check the log to verify correct execution: + # tail -f sysvm.log + The content should be like the following: + + Stopping and starting 1 secondary storage vm(s)... + Done stopping and starting secondary storage vm(s) + Stopping and starting 1 console proxy vm(s)... + Done stopping and starting console proxy vm(s). + Stopping and starting 4 running routing vm(s)... + Done restarting router(s). + + + + + + If you would like additional confirmation that the new system VM templates were + correctly applied when these system VMs were rebooted, SSH into the System VM and check + the version. + Use one of the following techniques, depending on the hypervisor. + + XenServer or KVM: + SSH in by using the link local IP address of the system VM. For example, in the + command below, substitute your own path to the private key used to log in to the + system VM and your own link local IP. + + Run the following commands on the XenServer or KVM host on which the system VM is + present: + # ssh -i private-key-path link-local-ip -p 3922 + # cat /etc/cloudstack-release + The output should be like the following: + Cloudstack Release 4.0.0-incubating Mon Oct 9 15:10:04 PST 2012 + + ESXi + SSH in using the private IP address of the system VM. For example, in the command + below, substitute your own path to the private key used to log in to the system VM and + your own private IP. + + Run the following commands on the Management Server: + # ssh -i private-key-path private-ip -p 3922 + # cat /etc/cloudstack-release + + The output should be like the following: + Cloudstack Release 4.0.0-incubating Mon Oct 9 15:10:04 PST 2012 + + + If needed, upgrade all Citrix XenServer hypervisor hosts in your cloud to a version + supported by CloudStack 4.0.0-incubating. The supported versions are XenServer 5.6 SP2 + and 6.0.2. Instructions for upgrade can be found in the CloudStack 4.0.0-incubating + Installation Guide. + + + Apply the XenServer hotfix XS602E003 (and any other needed hotfixes) to XenServer + v6.0.2 hypervisor hosts. + + + Disconnect the XenServer cluster from CloudStack. + In the left navigation bar of the CloudStack UI, select Infrastructure. Under + Clusters, click View All. Select the XenServer cluster and click Actions - + Unmanage. + This may fail if there are hosts not in one of the states Up, Down, + Disconnected, or Alert. You may need to fix that before unmanaging this + cluster. + Wait until the status of the cluster has reached Unmanaged. Use the CloudStack + UI to check on the status. When the cluster is in the unmanaged state, there is no + connection to the hosts in the cluster. + + + To clean up the VLAN, log in to one XenServer host and run: + /opt/xensource/bin/cloud-clean-vlan.sh + + + Prepare the upgrade by running the following on one XenServer host: + /opt/xensource/bin/cloud-prepare-upgrade.sh + If you see a message like "can't eject CD", log in to the VM and umount the CD, + then run this script again. + + + Upload the hotfix to the XenServer hosts. Always start with the Xen pool master, + then the slaves. Using your favorite file copy utility (e.g. WinSCP), copy the + hotfixes to the host. Place them in a temporary folder such as /root or /tmp. + On the Xen pool master, upload the hotfix with this command: + xe patch-upload file-name=XS602E003.xsupdate + Make a note of the output from this command, which is a UUID for the hotfix + file. You'll need it in another step later. + + (Optional) If you are applying other hotfixes as well, you can repeat the + commands in this section with the appropriate hotfix number. For example, + XS602E004.xsupdate. + + + + Manually live migrate all VMs on this host to another host. First, get a list of + the VMs on this host: + # xe vm-list + Then use this command to migrate each VM. Replace the example host name and VM + name with your own: + # xe vm-migrate live=true host=host-name vm=VM-name + + Troubleshooting + If you see a message like "You attempted an operation on a VM which requires + PV drivers to be installed but the drivers were not detected," run: + /opt/xensource/bin/make_migratable.sh + b6cf79c8-02ee-050b-922f-49583d9f1a14. + + + + Apply the hotfix. First, get the UUID of this host: + # xe host-list + Then use the following command to apply the hotfix. Replace the example host + UUID with the current host ID, and replace the hotfix UUID with the output from the + patch-upload command you ran on this machine earlier. You can also get the hotfix + UUID by running xe patch-list. + xe patch-apply host-uuid=host-uuid + uuid=hotfix-uuid + + + Copy the following files from the CloudStack Management Server to the + host. + + + + + + + Copy from here... + ...to here + + + + + /usr/share/cloudstack-common/scripts/vm/hypervisor/xenserver/xenserver60/NFSSR.py + /opt/xensource/sm/NFSSR.py + + + /usr/share/cloudstack-common/scripts/vm/hypervisor/xenserver/setupxenserver.sh + /opt/xensource/bin/setupxenserver.sh + + + /usr/lib64/cloudstack-common/scripts/vm/hypervisor/xenserver/make_migratable.sh + /opt/xensource/bin/make_migratable.sh + + + + + + + (Only for hotfixes XS602E005 and XS602E007) You need to apply a new Cloud + Support Pack. + + + Download the CSP software onto the XenServer host from one of the following + links: + For hotfix XS602E005: http://coltrane.eng.hq.xensource.com/release/XenServer-6.x/XS-6.0.2/hotfixes/XS602E005/56710/xe-phase-2/xenserver-cloud-supp.tgz + For hotfix XS602E007: http://coltrane.eng.hq.xensource.com/release/XenServer-6.x/XS-6.0.2/hotfixes/XS602E007/57824/xe-phase-2/xenserver-cloud-supp.tgz + + + Extract the file: + # tar xf xenserver-cloud-supp.tgz + + + Run the following script: + # xe-install-supplemental-pack + xenserver-cloud-supp.iso + + + If the XenServer host is part of a zone that uses basic networking, disable + Open vSwitch (OVS): + # xe-switch-network-backend bridge + + + + + Reboot this XenServer host. + + + Run the following: + /opt/xensource/bin/setupxenserver.sh + + If the message "mv: cannot stat `/etc/cron.daily/logrotate': No such file or + directory" appears, you can safely ignore it. + + + + Run the following: + for pbd in `xe pbd-list currently-attached=false| grep ^uuid | awk + '{print $NF}'`; do xe pbd-plug uuid=$pbd ; + + + + On each slave host in the Xen pool, repeat these steps, starting from "manually + live migrate VMs." + + + + +
+ Version 4.1.0
@@ -4398,7 +6588,7 @@ under the License. CLOUDSTACK-2709 + >CLOUDSTACK-2709 Egress rules are are not supported on shared networks. @@ -4620,7 +6810,7 @@ under the License. source, or check the Apache CloudStack downloads page at http://cloudstack.apache.org/downloads.html for package repositories supplied - by community members. You will need them for step + by community members. You will need them for step or step . Instructions for creating packages from the &PRODUCT; source are in the Installation @@ -4670,7 +6860,7 @@ under the License. PlainTextUserAuthenticator works the same way MD5UserAuthenticator worked prior to 4.1. - + If you are using Ubuntu, follow this procedure to upgrade your packages. If not, skip to step . @@ -4679,7 +6869,7 @@ under the License. If you've created your own packages and APT repository, substitute your own URL for the ones used in these examples. - + The first order of business will be to change the sources list for each system with &PRODUCT; packages. This means all management servers, and any hosts that have @@ -4704,7 +6894,7 @@ under the License. dependencies you need. $ sudo apt-get install cloudstack-management - + You will need to manually install the cloudstack-agent package: $ sudo apt-get install cloudstack-agent @@ -4751,7 +6941,7 @@ service cloudstack-agent restart If you are using CentOS or RHEL, follow this procedure to upgrade your packages. If - not, skip to step . + not, skip to step . Community Packages This section assumes you're using the community supplied packages for &PRODUCT;. @@ -4785,7 +6975,7 @@ gpgcheck=0 cloud-client package. $ sudo yum upgrade cloud-client - + For KVM hosts, you will need to upgrade the cloud-agent package, similarly installing the new version as cloudstack-agent. @@ -4814,7 +7004,7 @@ service cloudstack-agent start - + Once you've upgraded the packages on your management servers, you'll need to restart the system VMs. Make sure port 8096 is open in your local host firewall to do this. @@ -4844,7 +7034,7 @@ Done restarting router(s).
-
+
Upgrade from 3.0.2 to 4.1.0 This section will guide you from Citrix CloudStack 3.0.2 to Apache CloudStack 4.1.0. Sections that are hypervisor-specific will be called out with a note. @@ -4853,7 +7043,7 @@ Done restarting router(s). The following upgrade instructions apply only if you're using VMware hosts. If you're not using VMware hosts, skip this step and move on to . + linkend="stopping-usageservers"/>. In each zone that includes VMware hosts, you need to add a new system VM template. @@ -4939,7 +7129,7 @@ Done restarting router(s). - + Stop all Usage Servers if running. Run this on all Usage Server hosts. # service cloud-usage stop @@ -4962,16 +7152,16 @@ Done restarting router(s). the community provided yum/apt repositories to gain access to the &PRODUCT; binaries. - + If you are using Ubuntu, follow this procedure to upgrade your packages. If not, - skip to step . + skip to step . Community Packages This section assumes you're using the community supplied packages for &PRODUCT;. If you've created your own packages and APT repository, substitute your own URL for the ones used in these examples. - + The first order of business will be to change the sources list for each system with &PRODUCT; packages. This means all management servers, and any hosts that have @@ -4990,13 +7180,13 @@ Done restarting router(s). Now update your apt package list: $ sudo apt-get update - + Now that you have the repository configured, it's time to install the cloudstack-management package. This will pull in any other dependencies you need. $ sudo apt-get install cloudstack-management - + You will need to manually install the cloudstack-agent package: $ sudo apt-get install cloudstack-agent @@ -5041,16 +7231,16 @@ service cloudstack-agent restart - + If you are using CentOS or RHEL, follow this procedure to upgrade your packages. If - not, skip to step . + not, skip to step . Community Packages This section assumes you're using the community supplied packages for &PRODUCT;. If you've created your own packages and yum repository, substitute your own URL for the ones used in these examples. - + The first order of business will be to change the yum repository for each system with &PRODUCT; packages. This means all management servers, and any hosts that have @@ -5071,13 +7261,13 @@ gpgcheck=0 If you're using your own package repository, change this line to read as appropriate for your 4.1.0 repository. - + Now that you have the repository configured, it's time to install the cloudstack-management package by upgrading the older cloud-client package. $ sudo yum upgrade cloud-client - + For KVM hosts, you will need to upgrade the cloud-agent package, similarly installing the new version as cloudstack-agent. @@ -5106,7 +7296,7 @@ service cloudstack-agent start - + If you have made changes to your copy of /etc/cloud/management/components.xml the changes will be preserved in the upgrade. However, you need to do the following steps to place these @@ -5558,16 +7748,16 @@ service cloudstack-agent start the community provided yum/apt repositories to gain access to the &PRODUCT; binaries. - + If you are using Ubuntu, follow this procedure to upgrade your packages. If not, - skip to step . + skip to step . Community Packages This section assumes you're using the community supplied packages for &PRODUCT;. If you've created your own packages and APT repository, substitute your own URL for the ones used in these examples. - + The first order of business will be to change the sources list for each system with &PRODUCT; packages. This means all management servers, and any hosts that have @@ -5586,15 +7776,15 @@ service cloudstack-agent start Now update your apt package list: $ sudo apt-get update - + Now that you have the repository configured, it's time to install the cloudstack-management package. This will pull in any other dependencies you need. $ sudo apt-get install cloudstack-management - - On KVM hosts, you will need to manually install the cloudstack-agent - package: + + On KVM hosts, you will need to manually install the + cloudstack-agent package: $ sudo apt-get install cloudstack-agent During the installation of cloudstack-agent, APT will copy your agent.properties, log4j-cloud.xml, @@ -5637,16 +7827,16 @@ service cloudstack-agent restart - + If you are using CentOS or RHEL, follow this procedure to upgrade your packages. If - not, skip to step . + not, skip to step . Community Packages This section assumes you're using the community supplied packages for &PRODUCT;. If you've created your own packages and yum repository, substitute your own URL for the ones used in these examples. - + The first order of business will be to change the yum repository for each system with &PRODUCT; packages. This means all management servers, and any hosts that have @@ -5667,13 +7857,13 @@ gpgcheck=0 If you're using your own package repository, change this line to read as appropriate for your 4.1.0 repository. - + Now that you have the repository configured, it's time to install the cloudstack-management package by upgrading the older cloud-client package. $ sudo yum upgrade cloud-client - + For KVM hosts, you will need to upgrade the cloud-agent package, similarly installing the new version as cloudstack-agent. @@ -5702,7 +7892,7 @@ service cloudstack-agent start - + If you have made changes to your existing copy of the file components.xml in your previous-version CloudStack installation, the changes will be preserved in the upgrade. However, you need to do the following steps to place these changes in a new version of diff --git a/docs/en-US/about-primary-storage.xml b/docs/en-US/about-primary-storage.xml index a9cf05486c6..9af9f2dae13 100644 --- a/docs/en-US/about-primary-storage.xml +++ b/docs/en-US/about-primary-storage.xml @@ -24,9 +24,12 @@
About Primary Storage - Primary storage is associated with a cluster, and it stores the disk volumes for all the VMs running on hosts in that cluster. You can add multiple primary storage servers to a cluster. At least one is required. It is typically located close to the hosts for increased performance. + Primary storage is associated with a cluster and/or a zone. It stores the disk volumes for all of the VMs running on hosts in that cluster. You can add multiple primary storage servers to a cluster or a zone (at least one is required at the cluster level). Primary storage is typically located close to the hosts for increased performance. &PRODUCT; manages the allocation of guest virtual disks to particular primary storage devices. + Primary storage uses the concept of a storage tag. A storage tag is a label that is used to identify the primary storage. Each primary storage can be associated with zero, one, or more storage tags. When a VM is spun up or a data disk attached to a VM for the first time, these tags, if supplied, are used to determine which primary storage can support the VM or data disk (ex. say you need to guarantee a certain number of IOPS to a particular volume). + Primary storage can be either static or dynamic. Static primary storage is what CloudStack has traditionally supported. In this model, the administrator must present CloudStack with a certain amount of preallocated storage (ex. a volume from a SAN) and CloudStack can place many of its volumes on this storage. In the newer, dynamic model, the administrator can present CloudStack with a storage system itself (ex. a SAN). CloudStack, working in concert with a plug-in developed for that storage system, can dynamically create volumes on the storage system. A valuable use for this ability is Quality of Service (QoS). If a volume created in CloudStack can be backed by a dedicated volume on a SAN (i.e. a one-to-one mapping between a SAN volume and a CloudStack volume) and the SAN provides QoS, then CloudStack can provide QoS. &PRODUCT; is designed to work with all standards-compliant iSCSI and NFS servers that are supported by the underlying hypervisor, including, for example: + SolidFire for iSCSI Dell EqualLogicâ„¢ for iSCSI Network Appliances filers for NFS and iSCSI Scale Computing for NFS diff --git a/docs/en-US/about-regions.xml b/docs/en-US/about-regions.xml index 432faeb6c5e..a12c183abd3 100644 --- a/docs/en-US/about-regions.xml +++ b/docs/en-US/about-regions.xml @@ -44,6 +44,7 @@ region-overview.png: Nested structure of a region. - Regions are visible to the end user. When a user starts a guest VM, the user must select a region for their guest. - Users might also be required to copy their private templates to additional regions to enable creation of guest VMs using their templates in those regions. + Regions are visible to the end user. When a user starts a guest VM on a particular &PRODUCT; Management Server, + the user is implicitly selecting that region for their guest. + Users might also be required to copy their private templates to additional regions to enable creation of guest VMs using their templates in those regions.
\ No newline at end of file diff --git a/docs/en-US/about-secondary-storage.xml b/docs/en-US/about-secondary-storage.xml index c5b4f5d5a2f..516ec0e6b78 100644 --- a/docs/en-US/about-secondary-storage.xml +++ b/docs/en-US/about-secondary-storage.xml @@ -24,12 +24,28 @@
About Secondary Storage - Secondary storage is associated with a zone, and it stores the following: + Secondary storage stores the following: Templates — OS images that can be used to boot VMs and can include additional configuration information, such as installed applications ISO images — disc images containing data or bootable media for operating systems Disk volume snapshots — saved copies of VM data which can be used for data recovery or to create new templates - The items in zone-based NFS secondary storage are available to all hosts in the zone. &PRODUCT; manages the allocation of guest virtual disks to particular primary storage devices. - To make items in secondary storage available to all hosts throughout the cloud, you can add OpenStack Object Storage (Swift, swift.openstack.org) in addition to the zone-based NFS secondary storage. When using Swift, you configure Swift storage for the entire &PRODUCT;, then set up NFS secondary storage for each zone as usual. The NFS storage in each zone acts as a staging area through which all templates and other secondary storage data pass before being forwarded to Swift. The Swift storage acts as a cloud-wide resource, making templates and other data available to any zone in the cloud. There is no hierarchy in the Swift storage, just one Swift container per storage object. Any secondary storage in the whole cloud can pull a container from Swift at need. It is not necessary to copy templates and snapshots from one zone to another, as would be required when using zone NFS alone. Everything is available everywhere. + The items in secondary storage are available to all hosts in the scope of + the secondary storage, which may be defined as per zone or per region. + To make items in secondary storage available to all hosts throughout the cloud, you can + add object storage in addition to the + zone-based NFS Secondary Staging Store. + It is not necessary to + copy templates and snapshots from one zone to another, as would be required when using zone + NFS alone. Everything is available everywhere. + &PRODUCT; provides plugins that enable both + OpenStack Object Storage (Swift, + swift.openstack.org) + and Amazon Simple Storage Service (S3) object storage. + When using one of these storage plugins, you configure Swift or S3 storage for + the entire &PRODUCT;, then set up the NFS Secondary Staging Store for each zone. The NFS + storage in each zone acts as a staging area through which all templates and other secondary + storage data pass before being forwarded to Swoft or S3. + The backing object storage acts as a cloud-wide + resource, making templates and other data available to any zone in the cloud.
diff --git a/docs/en-US/about-zones.xml b/docs/en-US/about-zones.xml index 8f6cd06e6d9..2a4eeb4659f 100644 --- a/docs/en-US/about-zones.xml +++ b/docs/en-US/about-zones.xml @@ -32,6 +32,7 @@ A zone consists of: One or more pods. Each pod contains one or more clusters of hosts and one or more primary storage servers. + A zone may contain one or more primary storage servers, which are shared by all the pods in the zone. Secondary storage, which is shared by all the pods in the zone. @@ -45,12 +46,29 @@ Hosts in the same zone are directly accessible to each other without having to go through a firewall. Hosts in different zones can access each other through statically configured VPN tunnels. For each zone, the administrator must decide the following. - How many pods to place in a zone. + How many pods to place in each zone. How many clusters to place in each pod. How many hosts to place in each cluster. - How many primary storage servers to place in each cluster and total capacity for the storage servers. + (Optional) How many primary storage servers to place in each zone and total capacity for these storage servers. + How many primary storage servers to place in each cluster and total capacity for these storage servers. How much secondary storage to deploy in a zone. When you add a new zone using the &PRODUCT; UI, you will be prompted to configure the zone’s physical network and add the first pod, cluster, host, primary storage, and secondary storage. + In order to support zone-wide functions for VMware, &PRODUCT; is aware of VMware Datacenters and can map each Datacenter to a + &PRODUCT; zone. To enable features like storage live migration and zone-wide + primary storage for VMware hosts, &PRODUCT; has to make sure that a zone + contains only a single VMware Datacenter. Therefore, when you are creating a new + &PRODUCT; zone, you can select a VMware Datacenter for the zone. If you + are provisioning multiple VMware Datacenters, each one will be set up as a single zone + in &PRODUCT;. + + If you are upgrading from a previous &PRODUCT; version, and your existing + deployment contains a zone with clusters from multiple VMware Datacenters, that zone + will not be forcibly migrated to the new model. It will continue to function as + before. However, any new zone-wide operations, such as zone-wide primary storage + and live storage migration, will + not be available in that zone. + +
diff --git a/docs/en-US/accessing-system-vms.xml b/docs/en-US/accessing-system-vms.xml new file mode 100755 index 00000000000..e1b6090d7af --- /dev/null +++ b/docs/en-US/accessing-system-vms.xml @@ -0,0 +1,66 @@ + + +%BOOK_ENTITIES; +]> + + + +
+ Accessing System VMs + It may sometimes be necessary to access System VMs for diagnostics of certain issues, for example if you are experiencing SSVM (Secondary Storage VM) connection issues. Use the steps below in order to connect to the SSH console of a running System VM. + + Accessing System VMs over the network requires the use of private keys and connecting to System VMs SSH Daemon on port 3922. + XenServer/KVM Hypervisors store this key at /root/.ssh/id_rsa.cloud on each &PRODUCT; agent. + To access System VMs running on ESXi, the key is stored on the management server at /var/lib/cloudstack/management/.ssh/id_rsa. + + + + Find the details of the System VM + + Log in with admin privileges to the &PRODUCT; UI. + Click Infrastructure, then System VMs, and then click the name of a running VM. + Take a note of the 'Host', 'Private IP Address' and 'Link Local IP Address' of the System VM you wish to access. + + + + + XenServer/KVM Hypervisors + + Connect to the Host of which the System VM is running. + SSH the 'Link Local IP Address' of the System VM from the Host on which the VM is running. + Format: ssh -i <path-to-private-key> <link-local-ip> -p 3922 + Example: root@faith:~# ssh -i /root/.ssh/id_rsa.cloud 169.254.3.93 -p 3922 + + + + ESXi Hypervisors + + Connect to your &PRODUCT; Management Server. + ESXi users should SSH to the private IP address of the System VM. + Format: ssh -i <path-to-private-key> <vm-private-ip> -p 3922 + Example: root@management:~# ssh -i /var/lib/cloudstack/management/.ssh/id_rsa 172.16.0.250 -p 3922 + + + + + + + +
diff --git a/docs/en-US/accounts-users-domains.xml b/docs/en-US/accounts-users-domains.xml index a3f5837db8e..3accbbe9b84 100644 --- a/docs/en-US/accounts-users-domains.xml +++ b/docs/en-US/accounts-users-domains.xml @@ -46,8 +46,88 @@ Root Administrator Root administrators have complete access to the system, including managing templates, service offerings, customer care administrators, and domains - The resources belong to the account, not individual users in that account. For example, - billing, resource limits, and so on are maintained by the account, not the users. A user can - operate on any resource in the account provided the user has privileges for that operation. - The privileges are determined by the role. + + Resource Ownership + Resources belong to the account, not individual users in that account. For example, + billing, resource limits, and so on are maintained by the account, not the users. A user + can operate on any resource in the account provided the user has privileges for that + operation. The privileges are determined by the role. A root administrator can change + the ownership of any virtual machine from one account to any other account by using the + assignVirtualMachine API. A domain or sub-domain administrator can do the same for VMs + within the domain from one account to any other account in the domain or any of its + sub-domains. + +
+ Dedicating Resources to Accounts and Domains + The root administrator can dedicate resources to a specific domain or account + that needs private infrastructure for additional security or performance guarantees. + A zone, pod, cluster, or host can be reserved by the root administrator for a specific domain or account. + Only users in that domain or its subdomain may use the infrastructure. + For example, only users in a given domain can create guests in a zone dedicated to that domain. + There are several types of dedication available: + + + Explicit dedication. A zone, pod, cluster, or host is dedicated to an account or + domain by the root administrator during initial deployment and + configuration. + Strict implicit dedication. A host will not be shared across multiple accounts. For example, + strict implicit dedication is useful for deployment of certain types of + applications, such as desktops, where no host can be shared + between different accounts without violating the desktop software's terms of license. + Preferred implicit dedication. The VM will be deployed in dedicated infrastructure if + possible. Otherwise, the VM can be deployed in shared + infrastructure. + +
+ How to Dedicate a Zone, Cluster, Pod, or Host to an Account or Domain + For explicit dedication: When deploying a new zone, pod, cluster, or host, the + root administrator can click the Dedicated checkbox, then choose a domain or account + to own the resource. + To explicitly dedicate an existing zone, pod, cluster, or host: log in as the root admin, + find the resource in the UI, and click the Dedicate button. + + + + + dedicate-resource-button.png: button to dedicate a zone, pod, cluster, or host + + + For implicit dedication: The administrator creates a compute service offering and + in the Deployment Planner field, chooses ImplicitDedicationPlanner. Then in Planner + Mode, the administrator specifies either Strict or Preferred, depending on whether + it is permissible to allow some use of shared resources when dedicated resources are + not available. Whenever a user creates a VM based on this service offering, it is + allocated on one of the dedicated hosts. +
+
+ How to Use Dedicated Hosts + To use an explicitly dedicated host, use the explicit-dedicated type of affinity + group (see ). For example, when creating a new VM, + an end user can choose to place it on dedicated infrastructure. This operation will + succeed only if some infrastructure has already been assigned as dedicated to the + user's account or domain. +
+
+ Behavior of Dedicated Hosts, Clusters, Pods, and Zones + The administrator can live migrate VMs away from dedicated hosts if desired, whether the destination + is a host reserved for a different account/domain or a host that is shared (not dedicated to any particular account or domain). + &PRODUCT; will generate an alert, but the operation is allowed. + Dedicated hosts can be used in conjunction with host tags. If both a host tag and dedication are requested, + the VM will be placed only on a host that meets both requirements. If there is no dedicated resource available + to that user that also has the host tag requested by the user, then the VM will not deploy. + If you delete an account or domain, any hosts, clusters, pods, and zones that were + dedicated to it are freed up. They will now be available to be shared by any account + or domain, or the administrator may choose to re-dedicate them to a different + account or domain. + System VMs and virtual routers affect the behavior of host dedication. + System VMs and virtual routers are owned by the &PRODUCT; system account, + and they can be deployed on any host. They do not adhere to explicit dedication. + The presence of system vms and virtual routers on a host makes it unsuitable for strict implicit dedication. + The host can not be used for strict implicit dedication, + because the host already has VMs of a specific account (the default system account). + However, a host with system VMs or virtual routers can be used + for preferred implicit dedication. + +
+
diff --git a/docs/en-US/add-gateway-vpc.xml b/docs/en-US/add-gateway-vpc.xml index 486cf84a824..403302df532 100644 --- a/docs/en-US/add-gateway-vpc.xml +++ b/docs/en-US/add-gateway-vpc.xml @@ -135,25 +135,85 @@ You might want to deploy multiple VPCs with the same super CIDR and guest tier CIDR. Therefore, multiple guest VMs from different VPCs can have the same IPs to reach a enterprise data center through the private gateway. In such cases, a NAT service need to be configured on - the private gateway. If Source NAT is enabled, the guest VMs in VPC reaches the enterprise - network via private gateway IP address by using the NAT service. + the private gateway to avoid IP conflicts. If Source NAT is enabled, the guest VMs in VPC + reaches the enterprise network via private gateway IP address by using the NAT service. The Source NAT service on a private gateway can be enabled while adding the private gateway. On deletion of a private gateway, source NAT rules specific to the private gateway are deleted. + To enable source NAT on existing private gateways, delete them and create afresh with + source NAT.
ACL on Private Gateway The traffic on the VPC private gateway is controlled by creating both ingress and egress - network ACL rules. The ACLs contains both allow and deny rules. In addition to the default ACL - rules, rules you might have created are also listed in the ACL drop-down list. As per the - rule, all the ingress traffic to the private gateway interface and all the egress traffic out - from the private gateway interface are blocked. You can change this default behaviour while - creating a private gateway. + network ACL rules. The ACLs contains both allow and deny rules. As per the rule, all the + ingress traffic to the private gateway interface and all the egress traffic out from the + private gateway interface are blocked. + You can change this default behaviour while creating a private gateway. Alternatively, you + can do the following: + + + In a VPC, identify the Private Gateway you want to work with. + + + In the Private Gateway page, do either of the following: + + + Use the Quickview. See . + + + Use the Details tab. See through . + + + + + In the Quickview of the selected Private Gateway, click Replace ACL, select the ACL + rule, then click OK + + + Click the IP address of the Private Gateway you want to work with. + + + In the Detail tab, click the Replace ACL button. + + + + + replace-acl-icon.png: button to replace the default ACL behaviour. + + + The Replace ACL dialog is displayed. + + + select the ACL rule, then click OK. + Wait for few seconds. You can see that the new ACL rule is displayed in the Details + page. + +
Creating a Static Route &PRODUCT; enables you to specify routing for the VPN connection you create. You can enter one or CIDR addresses to indicate which traffic is to be routed back to the gateway. + + + In a VPC, identify the Private Gateway you want to work with. + + + In the Private Gateway page, click the IP address of the Private Gateway you want to + work with. + + + Select the Static Routes tab. + + + Specify the CIDR of destination network. + + + Click Add. + Wait for few seconds until the new route is created. + +
Blacklisting Routes diff --git a/docs/en-US/add-ip-range.xml b/docs/en-US/add-ip-range.xml index 3912bc2815e..6da0668ec2b 100644 --- a/docs/en-US/add-ip-range.xml +++ b/docs/en-US/add-ip-range.xml @@ -19,85 +19,106 @@ under the License. -->
- Adding Multiple IP Ranges - - The feature can only be implemented on IPv4 addresses. - + Multiple Subnets in Shared Network &PRODUCT; provides you with the flexibility to add guest IP ranges from different subnets in Basic zones and security groups-enabled Advanced zones. For security groups-enabled Advanced zones, it implies multiple subnets can be added to the same VLAN. With the addition of this feature, you will be able to add IP address ranges from the same subnet or from a different one when IP address are exhausted. This would in turn allows you to employ higher number of subnets - and thus reduce the address management overhead. - Ensure that you manually configure the gateway of the new subnet before adding the IP range. - Note that &PRODUCT; supports only one gateway for a subnet; overlapping subnets are not - currently supported. - You can also delete IP ranges. This operation fails if an IP from the remove range is in - use. If the remove range contains the IP address on which the DHCP server is running, &PRODUCT; - acquires a new IP from the same subnet. If no IP is available in the subnet, the remove - operation fails. - This feature is supported on KVM, xenServer, and VMware hypervisors. - - - Log in to the &PRODUCT; UI as an administrator or end user. - - - In the left navigation, choose Infrastructure. - - - On Zones, click View More, then click the zone to which you want to work with.. - - - Click Physical Network. - - - In the Guest node of the diagram, click Configure. - - - Click Networks. - - - Select the networks you want to work with. - - - Click View IP Ranges. - - - Click Add IP Range. - The Add IP Range dialog is displayed, as follows: - - - - - - add-ip-range.png: adding an IP range to a network. - - - - - Specify the following: - All the fields are mandatory. - - - Gateway: The gateway for the tier you create. - Ensure that the gateway is within the Super CIDR range that you specified while creating - the VPC, and is not overlapped with the CIDR of any existing tier within the VPC. - - - Netmask: The netmask for the tier you create. - For example, if the VPC CIDR is 10.0.0.0/16 and the network tier CIDR is - 10.0.1.0/24, the gateway of the tier is 10.0.1.1, and the netmask of the tier is - 255.255.255.0. - - - Start IP/ End IP: A range of IP addresses that are - accessible from the Internet and will be allocated to guest VMs. Enter the first and - last IP addresses that define a range that &PRODUCT; can assign to guest VMs . - - - - - Click OK. - - + and thus reduce the address management overhead. You can delete the IP ranges you have + added. +
+ Prerequisites and Guidelines + + + This feature can only be implemented: + + + on IPv4 addresses + + + if virtual router is the DHCP provider + + + on KVM, xenServer, and VMware hypervisors + + + + + Manually configure the gateway of the new subnet before adding the IP range. + + + &PRODUCT; supports only one gateway for a subnet; overlapping subnets are not + currently supported + + +
+
+ Adding Multiple Subnets to a Shared Network + + + Log in to the &PRODUCT; UI as an administrator or end user. + + + In the left navigation, choose Infrastructure. + + + On Zones, click View More, then click the zone to which you want to work with.. + + + Click Physical Network. + + + In the Guest node of the diagram, click Configure. + + + Click Networks. + + + Select the networks you want to work with. + + + Click View IP Ranges. + + + Click Add IP Range. + The Add IP Range dialog is displayed, as follows: + + + + + + add-ip-range.png: adding an IP range to a network. + + + + + Specify the following: + All the fields are mandatory. + + + Gateway: The gateway for the tier you create. + Ensure that the gateway is within the Super CIDR range that you specified while + creating the VPC, and is not overlapped with the CIDR of any existing tier within the + VPC. + + + Netmask: The netmask for the tier you create. + For example, if the VPC CIDR is 10.0.0.0/16 and the network tier CIDR is + 10.0.1.0/24, the gateway of the tier is 10.0.1.1, and the netmask of the tier is + 255.255.255.0. + + + Start IP/ End IP: A range of IP addresses that + are accessible from the Internet and will be allocated to guest VMs. Enter the first + and last IP addresses that define a range that &PRODUCT; can assign to guest VMs + . + + + + + Click OK. + + +
diff --git a/docs/en-US/add-load-balancer-rule.xml b/docs/en-US/add-load-balancer-rule.xml index 2d911feaf75..01bf13d0014 100644 --- a/docs/en-US/add-load-balancer-rule.xml +++ b/docs/en-US/add-load-balancer-rule.xml @@ -74,6 +74,22 @@ AutoScale: Click Configure and complete the AutoScale configuration as explained in . + Health Check: (Optional; NetScaler load balancers only) + Click Configure and fill in the characteristics of the health check policy. + See . + + Ping path (Optional): Sequence of destinations to which to send health check queries. + Default: / (all). + Response time (Optional): How long to wait for a response from the health check (2 - 60 seconds). + Default: 5 seconds. + Interval time (Optional): Amount of time between health checks (1 second - 5 minutes). + Default value is set in the global configuration parameter lbrule_health check_time_interval. + Healthy threshold (Optional): Number of consecutive health check successes + that are required before declaring an instance healthy. + Default: 2. + Unhealthy threshold (Optional): Number of consecutive health check failures that are required before declaring an instance unhealthy. + Default: 10. + diff --git a/docs/en-US/add-loadbalancer-rule-vpc.xml b/docs/en-US/add-loadbalancer-rule-vpc.xml index 0f2a83dcbfd..90247b0a6f9 100644 --- a/docs/en-US/add-loadbalancer-rule-vpc.xml +++ b/docs/en-US/add-loadbalancer-rule-vpc.xml @@ -60,7 +60,7 @@
Creating a Network Offering for External LB - To have internal LB support on VPC, create a network offering as follows: + To have external LB support on VPC, create a network offering as follows: Log in to the &PRODUCT; UI as a user or admin. @@ -111,12 +111,16 @@ Indicate whether a VLAN should be specified when this offering is used. - Supported Services: Select Load Balancer. - Select InternalLbVM from the provider list. + Supported Services: Select Load Balancer. Use + Netscaler or VpcVirtualRouter. - Load Balancer Type: Select external LB from the - drop-down. Use Netscaler + Load Balancer Type: Select Public LB from the + drop-down. + + + LB Isolation: Select Dedicated if Netscaler is + used as the external LB provider. System Offering: Choose the system service @@ -274,6 +278,19 @@
+
+ Guidelines + + Internal LB and Public LB are mutually exclusive on a tier. If the tier has LB on the public + side, then it can't have the Internal LB. + Internal LB is supported just on VPC networks in &PRODUCT; 4.2 release. + Only Internal LB VM can act as the Internal LB provider in &PRODUCT; 4.2 release. + Network upgrade is not supported from the network offering with Internal LB to the network + offering with Public LB. + Multiple tiers can have internal LB support in a VPC. + Only one tier can have Public LB support in a VPC. + +
Enabling Internal LB on a VPC Tier @@ -288,7 +305,9 @@
Creating a Network Offering for Internal LB - To have internal LB support on VPC, create a network offering as follows: + To have internal LB support on VPC, either use the default offering, + DefaultIsolatedNetworkOfferingForVpcNetworksWithInternalLB, or create a network offering as + follows: Log in to the &PRODUCT; UI as a user or admin. @@ -364,6 +383,14 @@
Creating an Internal LB Rule + When you create the Internal LB rule and applies to a VM, an Internal LB VM, which is + responsible for load balancing, is created. + You can view the created Internal LB VM in the Instances page if you navigate to + Infrastructure > Zones > + <zone_ name> > <physical_network_name> > Network Service + Providers > Internal LB VM. You can manage the + Internal LB VMs as and when required from the location. Log in to the &PRODUCT; UI as an administrator or end user. @@ -397,9 +424,11 @@ that can be displayed to users. - Source IP Address: The source IP from which - traffic originates. Typically, this is the IP of an instance on another tier within - your VPC. + Source IP Address: (Optional) The source IP + from which traffic originates. The IP is acquired from the CIDR of that particular + tier on which you want to create the Internal LB rule. If not specified, the IP + address is automatically allocated from the network CIDR. + For every Source IP, a new Internal LB VM is created for load balancing. Source Port: The port associated with the diff --git a/docs/en-US/add-remove-nic-ui.xml b/docs/en-US/add-remove-nic-ui.xml index 688ba34cd08..a671329eb00 100644 --- a/docs/en-US/add-remove-nic-ui.xml +++ b/docs/en-US/add-remove-nic-ui.xml @@ -34,7 +34,7 @@ Adding a Network - Log in to the CloudPlatform UI as an administrator or end user. + Log in to the &PRODUCT; UI as an administrator or end user. In the left navigation, click Instances. @@ -87,7 +87,7 @@ Removing a Network - Log in to the CloudPlatform UI as an administrator or end user. + Log in to the &PRODUCT; UI as an administrator or end user. In the left navigation, click Instances. @@ -120,7 +120,7 @@ Selecting the Default Network - Log in to the CloudPlatform UI as an administrator or end user. + Log in to the &PRODUCT; UI as an administrator or end user. In the left navigation, click Instances. diff --git a/docs/en-US/add-tier.xml b/docs/en-US/add-tier.xml index 17e02be7b7b..94a8237c066 100644 --- a/docs/en-US/add-tier.xml +++ b/docs/en-US/add-tier.xml @@ -75,7 +75,8 @@ the VPC, and is not overlapped with the CIDR of any existing tier within the VPC. - VLAN: The VLAN ID for the tier you create. + VLAN: The VLAN ID for the tier that the root admin + creates. This option is only visible if the network offering you selected is VLAN-enabled. For more information, see the Assigning VLANs to Isolated diff --git a/docs/en-US/added-API-commands-4.2.xml b/docs/en-US/added-API-commands-4.2.xml index ddf80b19f13..14a5f64b8ee 100644 --- a/docs/en-US/added-API-commands-4.2.xml +++ b/docs/en-US/added-API-commands-4.2.xml @@ -21,6 +21,22 @@
Added API Commands in 4.2 + + addImageStore + Adds all types of secondary storage providers, S3/Swift/NFS. + + + createSecondaryStagingStore + Adds a staging secondary storage in each zone. + + + listImageStores + Lists all secondary storages, S3/Swift/NFS. + + + listSecondaryStagingStores + Lists all staging secondary storages. + addIpToNic Adds an IP address to the NIC from the guest subnet. The request parameters are: nicid, @@ -135,328 +151,404 @@ The request parameters are elastic ip id and region id. - createVMSnapshot (create a virtual machine snapshot) + createVMSnapshot + Creates a virtual machine snapshot. - deleteVMSnapshot (delete a virtual machine snapshot) + deleteVMSnapshot + Deletes a virtual machine snapshot. - listVMSnapshot (show a virtual machine snapshot) + listVMSnapshot + Shows a virtual machine snapshot. - revertToVMSnapshot (return a virtual machine to the state and data saved in a given - snapshot) + revertToVMSnapshot + Returns a virtual machine to the state and data saved in a given snapshot. - createLBHealthCheckPolicy (creates a new health check policy for a load balancer rule; - see ) + createLBHealthCheckPolicy + Creates a new health check policy for a load balancer rule. - deleteLBHealthCheckPolicy (deletes an existing health check policy from a load balancer - rule) + deleteLBHealthCheckPolicy + Deletes an existing health check policy from a load balancer rule. - listLBHealthCheckPolicies (displays the health check policy for a load balancer - rule) + listLBHealthCheckPolicies + Displays the health check policy for a load balancer rule. - createEgressFirewallRules (creates an egress firewall rule on the guest network; see - ) + createEgressFirewallRules + Creates an egress firewall rule on the guest network. - deleteEgressFirewallRules (deletes a egress firewall rule on the guest network.) + deleteEgressFirewallRules + Deletes a egress firewall rule on the guest network. - listEgressFirewallRules (lists the egress firewall rules configured for a guest - network.) + listEgressFirewallRules + Lists the egress firewall rules configured for a guest network. - resetSSHKeyForVirtualMachine (Resets the SSHkey for virtual machine.) + resetSSHKeyForVirtualMachine + Resets the SSHkey for virtual machine. - addBaremetalHost (Adds a new host. Technically, this API command was present in v3.0.6, - but its functionality was disabled. See ) + addBaremetalHost + Adds a new host. Technically, this API command was present in v3.0.6, but its + functionality was disabled. - addBaremetalDhcp (Adds a DHCP server for bare metal hosts) + addBaremetalDhcp + Adds a DHCP server for bare metal hosts. - addBaremetalPxePingServer (Adds a PXE PING server for bare metal hosts) + addBaremetalPxePingServer + Adds a PXE PING server for bare metal hosts. addBaremetalPxeKickStartServer (Adds a PXE server for bare metal hosts) - listBaremetalDhcp (Shows the DHCP servers currently defined for bare metal - hosts) + listBaremetalDhcp + Shows the DHCP servers currently defined for bare metal hosts. - listBaremetalPxePingServer (Shows the PXE PING servers currently defined for bare metal - hosts) + listBaremetalPxePingServer + Shows the PXE PING servers currently defined for bare metal hosts. - addNicToVirtualMachine (Adds a new NIC to the specified VM on a selected network; see - ) + addNicToVirtualMachine + Adds a new NIC to the specified VM on a selected network. - removeNicFromVirtualMachine (Removes the specified NIC from a selected VM.) + removeNicFromVirtualMachine + Removes the specified NIC from a selected VM. - updateDefaultNicForVirtualMachine (Updates the specified NIC to be the default one for a - selected VM.) + updateDefaultNicForVirtualMachine + Updates the specified NIC to be the default one for a selected VM. - addRegion (Registers a Region into another Region; see ) + addRegion + Registers a Region into another Region. - updateRegion (Updates Region details: ID, Name, Endpoint, User API Key, and User Secret - Key.) + updateRegion + Updates Region details: ID, Name, Endpoint, User API Key, and User Secret Key. - removeRegion (Removes a Region from current Region.) + removeRegion + Removes a Region from current Region. - listRegions (Get all the Regions. They can be filtered by using the ID or Name.) + listRegions + Get all the Regions. They can be filtered by using the ID or Name. - getUser (This API can only be used by the Admin. Get user account details by using the - API Key.) + getUser + This API can only be used by the Admin. Get user account details by using the API + Key. - getApiLimit (Show number of remaining APIs for the invoking user in current - window) + getApiLimit + Shows number of remaining APIs for the invoking user in current window. - resetApiLimit (For root admin, if account ID parameter is passed, it will reset count - for that particular account, otherwise it will reset all counters) + resetApiLimit + For root admin, if account ID parameter is passed, it will reset count for that + particular account, otherwise it will reset all counters. - lockAccount (Locks an account) + lockAccount + Locks an account. - lockUser (Locks a user account) + lockUser + Locks a user account. - scaleVirtualMachine (Scales the virtual machine to a new service offering.) + scaleVirtualMachine + Scales the virtual machine to a new service offering. - migrateVirtualMachineWithVolume (Attempts migrating VM with its volumes to a different - host.) + migrateVirtualMachineWithVolume + Attempts migrating VM with its volumes to a different host. - dedicatePublicIpRange (Dedicates a Public IP range to an account.) + dedicatePublicIpRange + Dedicates a Public IP range to an account. - releasePublicIpRange (Releases a Public IP range back to the system pool.) + releasePublicIpRange + Releases a Public IP range back to the system pool. - dedicateGuestVlanRange (Dedicates a guest VLAN range to an account.) + dedicateGuestVlanRange + Dedicates a guest VLAN range to an account. - releaseDedicatedGuestVlanRange (Releases a dedicated guest VLAN range to the system.) - + releaseDedicatedGuestVlanRange + Releases a dedicated guest VLAN range to the system. - listDedicatedGuestVlanRanges (Lists dedicated guest VLAN ranges.) + listDedicatedGuestVlanRanges + Lists dedicated guest VLAN ranges. - updatePortForwardingRule (Updates a port forwarding rule. Only the private port and the - VM can be updated.) + updatePortForwardingRule + Updates a port forwarding rule. Only the private port and the VM can be updated. - scaleSystemVm (Scale the service offering for a systemVM, console proxy, or secondary - storage. The system VM must be in Stopped state for this command to take effect.) + scaleSystemVm + Scales the service offering for a systemVM, console proxy, or secondary storage. - listDeploymentPlanners (Lists all the deployment planners available.) + listDeploymentPlanners + Lists all the deployment planners available. - addS3 (Adds a Amazon Simple Storage Service instance.) + addS3 + Adds a Amazon Simple Storage Service instance. - listS3s (Lists all the Amazon Simple Storage Service instances.) + listS3s + Lists all the Amazon Simple Storage Service instances. - findHostsForMigration (Find hosts suitable for migrating a VM to.) + findHostsForMigration + Finds hosts suitable for migrating a VM to. - releaseHostReservation (Releases host reservation.) + releaseHostReservation + Releases host reservation. - resizeVolume (Resizes a volume.) + resizeVolume + Resizes a volume. - updateVolume (Updates the volume.) + updateVolume + Updates the volume. - listStorageProviders (Lists storage providers.) + listStorageProviders + Lists storage providers. - findStoragePoolsForMigration (Lists storage pools available for migrating a volume.) - + findStoragePoolsForMigration + Lists storage pools available for migrating a volume. - createEgressFirewallRule (Creates a egress firewall rule for a given network. ) + createEgressFirewallRule + Creates a egress firewall rule for a given network. - deleteEgressFirewallRule (Deletes an egress firewall rule.) + deleteEgressFirewallRule + Deletes an egress firewall rule. - listEgressFirewallRules (Lists all egress firewall rules for network.) + listEgressFirewallRules + Lists all egress firewall rules for network. - updateNetworkACLItem (Updates ACL item with specified ID.) + updateNetworkACLItem + Updates ACL item with specified ID. - createNetworkACLList (Creates a Network ACL for the given VPC.) + createNetworkACLList + Creates a Network ACL for the given VPC. - deleteNetworkACLList (Deletes a Network ACL.) + deleteNetworkACLList + Deletes a Network ACL. - replaceNetworkACLList (Replaces ACL associated with a Network or private gateway.) - + replaceNetworkACLList + Replaces ACL associated with a Network or private gateway. - listNetworkACLLists (Lists all network ACLs.) + listNetworkACLLists + Lists all network ACLs. - addResourceDetail (Adds detail for the Resource.) + addResourceDetail + Adds detail for the Resource. - removeResourceDetail (Removes detail for the Resource.) + removeResourceDetail + Removes details of the resource. - listResourceDetails (List resource details.) + listResourceDetails + Lists resource details. - addNiciraNvpDevice (Adds a Nicira NVP device.) + addNiciraNvpDevice + Adds a Nicira NVP device. - deleteNiciraNvpDevice (Deletes a Nicira NVP device.) + deleteNiciraNvpDevice + Deletes a Nicira NVP device. - listNiciraNvpDevices (Lists Nicira NVP devices.) + listNiciraNvpDevices + Lists Nicira NVP devices. - listNiciraNvpDeviceNetworks (Lists network that are using a Nicira NVP device.) + listNiciraNvpDeviceNetworks + Lists network that are using a Nicira NVP device. - addBigSwitchVnsDevice (Adds a BigSwitch VNS device.) + addBigSwitchVnsDevice + Adds a BigSwitch VNS device. - deleteBigSwitchVnsDevice (Deletes a BigSwitch VNS device.) + deleteBigSwitchVnsDevice + Deletes a BigSwitch VNS device. - listBigSwitchVnsDevices (Lists BigSwitch VNS devices.) + listBigSwitchVnsDevices + Lists BigSwitch VNS devices. - configureSimulator (Configures a simulator.) + configureSimulator + Configures a simulator. - listApis (Lists all the available APIs on the server, provided by the API Discovery - plugin.) + listApis + Lists all the available APIs on the server, provided by the API Discovery plugin. - getApiLimit (Get the API limit count for the caller.) + getApiLimit + Gets the API limit count for the caller. - resetApiLimit (Reset the API count.) + resetApiLimit + Resets the API count. - assignToGlobalLoadBalancerRule (Assign load balancer rule or list of load balancer rules - to a global load balancer rules.) + assignToGlobalLoadBalancerRule + Assigns load balancer rule or list of load balancer rules to a global load balancer + rules. - removeFromGlobalLoadBalancerRule (Removes a load balancer rule association with global - load balancer rule) + removeFromGlobalLoadBalancerRule + Removes a load balancer rule association with global load balancer rule. - listVMSnapshot (List virtual machine snapshot by conditions) + listVMSnapshot + Lists virtual machine snapshot by conditions. - createLoadBalancer (Creates a Load Balancer) + createLoadBalancer + Creates a load balancer. - listLoadBalancers (Lists Load Balancers) + listLoadBalancers + Lists load balancers. - deleteLoadBalancer (Deletes a load balancer) + deleteLoadBalancer + Deletes a load balancer. - configureInternalLoadBalancerElement (Configures an Internal Load Balancer element.) - + configureInternalLoadBalancerElement + Configures an Internal Load Balancer element. - createInternalLoadBalancerElement (Create an Internal Load Balancer element.) + createInternalLoadBalancerElement + Creates an Internal Load Balancer element. - listInternalLoadBalancerElements (Lists all available Internal Load Balancer elements.) - + listInternalLoadBalancerElements + Lists all available Internal Load Balancer elements. - createAffinityGroup (Creates an affinity or anti-affinity group.) + createAffinityGroup + Creates an affinity or anti-affinity group. - deleteAffinityGroup (Deletes an affinity group.) + deleteAffinityGroup + Deletes an affinity group. - listAffinityGroups (Lists all the affinity groups.) + listAffinityGroups + Lists all the affinity groups. - updateVMAffinityGroup (Updates the affinity or anti-affinity group associations of a VM. - The VM has to be stopped and restarted for the new properties to take effect.) + updateVMAffinityGroup + Updates the affinity or anti-affinity group associations of a VM. The VM has to be + stopped and restarted for the new properties to take effect. - listAffinityGroupTypes (Lists affinity group types available.) + listAffinityGroupTypes + Lists affinity group types available. - stopInternalLoadBalancerVM (Stops an Internal LB VM.) + stopInternalLoadBalancerVM + Stops an Internal LB VM. - startInternalLoadBalancerVM (Starts an existing Internal LB VM.) + startInternalLoadBalancerVM + Starts an existing Internal LB VM. - listInternalLoadBalancerVMs (List internal LB VMs.) + listInternalLoadBalancerVMs + Lists internal LB VMs. - listNetworkIsolationMethods (Lists supported methods of network isolation.) + listNetworkIsolationMethods + Lists supported methods of network isolation. - dedicateZone (Dedicates a zone.) + dedicateZone + Dedicates a zone. - dedicatePod (Dedicates a pod.) + dedicatePod + Dedicates a pod. - dedicateCluster (Dedicate an existing cluster.) + dedicateCluster + Dedicates an existing cluster. - dedicateHost (Dedicates a host.) + dedicateHost + Dedicates a host. - releaseDedicatedZone (Release dedication of zone.) + releaseDedicatedZone + Releases dedication of zone. - releaseDedicatedPod (Release dedication for the pod.) + releaseDedicatedPod + Releases dedication for the pod. - releaseDedicatedCluster (Release dedication for cluster.) + releaseDedicatedCluster + Releases dedication for cluster. - releaseDedicatedHost (Release dedication for host.) + releaseDedicatedHost + Releases dedication for host. - listDedicatedZones (List dedicated zones.) + listDedicatedZones + Lists dedicated zones. - listDedicatedPods (Lists dedicated pods.) + listDedicatedPods + Lists dedicated pods. - listDedicatedClusters (Lists dedicated clusters.) + listDedicatedClusters + Lists dedicated clusters. - listDedicatedHosts (Lists dedicated hosts.) + listDedicatedHosts + Lists dedicated hosts.
diff --git a/docs/en-US/admin-alerts.xml b/docs/en-US/admin-alerts.xml index 5354c5e9b8e..e98f79de06f 100644 --- a/docs/en-US/admin-alerts.xml +++ b/docs/en-US/admin-alerts.xml @@ -31,5 +31,98 @@ The Management Server cluster runs low on CPU, memory, or storage resources The Management Server loses heartbeat from a Host for more than 3 minutes The Host cluster runs low on CPU, memory, or storage resources - + +
+ + Sending Alerts to External SNMP and Syslog Managers + In addition to showing administrator alerts on the Dashboard in the &PRODUCT; UI and + sending them in email, &PRODUCT; can also send the same alerts to external SNMP or + Syslog management software. This is useful if you prefer to use an SNMP or Syslog + manager to monitor your cloud. + The alerts which can be sent are listed in . You can also + display the most up to date list by calling the API command listAlerts. +
+ SNMP Alert Details + The supported protocol is SNMP version 2. + Each SNMP trap contains the following information: message, podId, dataCenterId, clusterId, and generationTime. +
+
+ Syslog Alert Details + &PRODUCT; generates a syslog message for every alert. Each syslog message incudes + the fields alertType, message, podId, dataCenterId, and clusterId, in the following + format. If any field does not have a valid value, it will not be included. + Date severity_level Management_Server_IP_Address/Name alertType:: value dataCenterId:: value podId:: value clusterId:: value message:: value + For example: + Mar 4 10:13:47 WARN localhost alertType:: managementNode message:: Management server node 127.0.0.1 is up +
+
+ Configuring SNMP and Syslog Managers + To configure one or more SNMP managers or Syslog managers to receive alerts from + &PRODUCT;: + + For an SNMP manager, install the &PRODUCT; MIB file on your SNMP manager system. + This maps the SNMP OIDs to trap types that can be more easily read by users. + The file must be publicly available. + For more information on how to install this file, consult the documentation provided with the SNMP manager. + + Edit the file /etc/cloudstack/management/log4j-cloud.xml. + # vi /etc/cloudstack/management/log4j-cloud.xml + + + Add an entry using the syntax shown below. Follow the appropriate example + depending on whether you are adding an SNMP manager or a Syslog manager. To specify + multiple external managers, separate the IP addresses and other configuration values + with commas (,). + + The recommended maximum number of SNMP or Syslog managers is 20 for + each. + + + The following example shows how to configure two SNMP managers at IP addresses + 10.1.1.1 and 10.1.1.2. Substitute your own IP addresses, ports, and communities. Do + not change the other values (name, threshold, class, and layout values). + <appender name="SNMP" class="org.apache.cloudstack.alert.snmp.SnmpTrapAppender"> + <param name="Threshold" value="WARN"/> <!-- Do not edit. The alert feature assumes WARN. --> + <param name="SnmpManagerIpAddresses" value="10.1.1.1,10.1.1.2"/> + <param name="SnmpManagerPorts" value="162,162"/> + <param name="SnmpManagerCommunities" value="public,public"/> + <layout class="org.apache.cloudstack.alert.snmp.SnmpEnhancedPatternLayout"> <!-- Do not edit --> + <param name="PairDelimeter" value="//"/> + <param name="KeyValueDelimeter" value="::"/> + </layout> +</appender> + The following example shows how to configure two Syslog managers at IP + addresses 10.1.1.1 and 10.1.1.2. Substitute your own IP addresses. You can + set Facility to any syslog-defined value, such as LOCAL0 - LOCAL7. Do not + change the other values. + <appender name="ALERTSYSLOG"> + <param name="Threshold" value="WARN"/> + <param name="SyslogHosts" value="10.1.1.1,10.1.1.2"/> + <param name="Facility" value="LOCAL6"/> + <layout> + <param name="ConversionPattern" value=""/> + </layout> +</appender> + + + If your cloud has multiple Management Server nodes, repeat these steps to edit + log4j-cloud.xml on every instance. + + + If you have made these changes while the Management Server is running, wait a + few minutes for the change to take effect. + + + Troubleshooting: If no alerts appear at the + configured SNMP or Syslog manager after a reasonable amount of time, it is likely that + there is an error in the syntax of the <appender> entry in log4j-cloud.xml. Check + to be sure that the format and settings are correct. +
+
+ Deleting an SNMP or Syslog Manager + To remove an external SNMP manager or Syslog manager so that it no longer receives + alerts from &PRODUCT;, remove the corresponding entry from the file + /etc/cloudstack/management/log4j-cloud.xml. +
+
diff --git a/docs/en-US/attaching-volume.xml b/docs/en-US/attaching-volume.xml index 7511ec32a4d..bb9196a93bb 100644 --- a/docs/en-US/attaching-volume.xml +++ b/docs/en-US/attaching-volume.xml @@ -37,7 +37,7 @@ In Select View, choose Volumes. - 4. Click the volume name in the Volumes list, then click the Attach Disk button + Click the volume name in the Volumes list, then click the Attach Disk button diff --git a/docs/en-US/basic-zone-configuration.xml b/docs/en-US/basic-zone-configuration.xml index eb8b5068f76..965aff3f644 100644 --- a/docs/en-US/basic-zone-configuration.xml +++ b/docs/en-US/basic-zone-configuration.xml @@ -65,7 +65,7 @@ Choose which traffic types will be carried by the physical network. The traffic types are management, public, guest, and storage traffic. For more information about the types, roll over the icons to display their tool tips, or see Basic Zone Network Traffic Types. This screen starts out with some traffic types already assigned. To add more, drag and drop traffic types onto the network. You can also change the network name if desired. - 3. (Introduced in version 3.0.1) Assign a network traffic label to each traffic type on the physical network. These labels must match the labels you have already defined on the hypervisor host. To assign each label, click the Edit button under the traffic type icon. A popup dialog appears where you can type the label, then click OK. + Assign a network traffic label to each traffic type on the physical network. These labels must match the labels you have already defined on the hypervisor host. To assign each label, click the Edit button under the traffic type icon. A popup dialog appears where you can type the label, then click OK. These traffic labels will be defined only for the hypervisor selected for the first cluster. For all other hypervisors, the labels can be configured after the zone is created. Click Next. diff --git a/docs/en-US/best-practices-for-vms.xml b/docs/en-US/best-practices-for-vms.xml index bba20c6fce3..f2656a09e5d 100644 --- a/docs/en-US/best-practices-for-vms.xml +++ b/docs/en-US/best-practices-for-vms.xml @@ -22,18 +22,46 @@ -->
- Best Practices for Virtual Machines - The &PRODUCT; administrator should monitor the total number of VM instances in each - cluster, and disable allocation to the cluster if the total is approaching the maximum that - the hypervisor can handle. Be sure to leave a safety margin to allow for the possibility of - one or more hosts failing, which would increase the VM load on the other hosts as the VMs - are automatically redeployed. Consult the documentation for your chosen hypervisor to find - the maximum permitted number of VMs per host, then use &PRODUCT; global configuration - settings to set this as the default limit. Monitor the VM activity in each cluster at all - times. Keep the total number of VMs below a safe level that allows for the occasional host - failure. For example, if there are N hosts in the cluster, and you want to allow for one - host in the cluster to be down at any given time, the total number of VM instances you can - permit in the cluster is at most (N-1) * (per-host-limit). Once a cluster reaches this - number of VMs, use the &PRODUCT; UI to disable allocation of more VMs to the - cluster. + Best Practices for Virtual Machines + For VMs to work as expected and provide excellent service, follow these guidelines. +
+ Monitor VMs for Max Capacity + The &PRODUCT; administrator should monitor the total number of VM instances in each + cluster, and disable allocation to the cluster if the total is approaching the maximum that + the hypervisor can handle. Be sure to leave a safety margin to allow for the possibility of + one or more hosts failing, which would increase the VM load on the other hosts as the VMs + are automatically redeployed. Consult the documentation for your chosen hypervisor to find + the maximum permitted number of VMs per host, then use &PRODUCT; global configuration + settings to set this as the default limit. Monitor the VM activity in each cluster at all + times. Keep the total number of VMs below a safe level that allows for the occasional host + failure. For example, if there are N hosts in the cluster, and you want to allow for one + host in the cluster to be down at any given time, the total number of VM instances you can + permit in the cluster is at most (N-1) * (per-host-limit). Once a cluster reaches this + number of VMs, use the &PRODUCT; UI to disable allocation of more VMs to the + cluster. +
+
+ Install Required Tools and Drivers + Be sure the following are installed on each VM: + + For XenServer, install PV drivers and Xen tools on each VM. + This will enable live migration and clean guest shutdown. + Xen tools are required in order for dynamic CPU and RAM scaling to work. + For vSphere, install VMware Tools on each VM. + This will enable console view to work properly. + VMware Tools are required in order for dynamic CPU and RAM scaling to work. + + To be sure that Xen tools or VMware Tools is installed, use one of the following techniques: + + Create each VM from a template that already has the tools installed; or, + When registering a new template, the administrator or user can indicate whether tools are + installed on the template. This can be done through the UI + or using the updateTemplate API; or, + If a user deploys a virtual machine with a template that does not have + Xen tools or VMware Tools, and later installs the tools on the VM, + then the user can inform &PRODUCT; using the updateVirtualMachine API. + After installing the tools and updating the virtual machine, stop + and start the VM. + +
diff --git a/docs/en-US/best-practices-primary-storage.xml b/docs/en-US/best-practices-primary-storage.xml index 0c9a22fcb18..279b95c0de1 100644 --- a/docs/en-US/best-practices-primary-storage.xml +++ b/docs/en-US/best-practices-primary-storage.xml @@ -25,7 +25,9 @@
Best Practices for Primary Storage - The speed of primary storage will impact guest performance. If possible, choose smaller, higher RPM drives for primary storage. - Ensure that nothing is stored on the server. Adding the server to &PRODUCT; will destroy any existing data + The speed of primary storage will impact guest performance. If possible, choose smaller, higher RPM drives or SSDs for primary storage. + There are two ways CloudStack can leverage primary storage: + Static: This is CloudStack's traditional way of handling storage. In this model, a preallocated amount of storage (ex. a volume from a SAN) is given to CloudStack. CloudStack then permits many of its volumes to be created on this storage (can be root and/or data disks). If using this technique, ensure that nothing is stored on the storage. Adding the storage to &PRODUCT; will destroy any existing data. + Dynamic: This is a newer way for CloudStack to manage storage. In this model, a storage system (rather than a preallocated amount of storage) is given to CloudStack. CloudStack, working in concert with a storage plug-in, dynamically creates volumes on the storage system and each volume on the storage system maps to a single CloudStack volume. This is highly useful for features such as storage Quality of Service. Currently this feature is supported for data disks (Disk Offerings).
diff --git a/docs/en-US/change-network-offering-on-guest-network.xml b/docs/en-US/change-network-offering-on-guest-network.xml index 2c7db3e9176..de3a80ecddc 100644 --- a/docs/en-US/change-network-offering-on-guest-network.xml +++ b/docs/en-US/change-network-offering-on-guest-network.xml @@ -20,34 +20,49 @@ KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ---> +-->
- Changing the Network Offering on a Guest Network - A user or administrator can change the network offering that is associated with an existing guest network. - - Log in to the &PRODUCT; UI as an administrator or end user. - If you are changing from a network offering that uses the &PRODUCT; virtual router to one - that uses external devices as network service providers, you must first stop all the - VMs on the network. - See "Stopping and Starting Virtual Machines" in the Administrator's Guide. - See . - In the left navigation, choose Network. - Click the name of the network you want to modify. - In the Details tab, click Edit. - - - - - EditButton.png: button to edit a network - - - In Network Offering, choose the new network offering, then click Apply. - A prompt is displayed asking whether you want to keep the existing CIDR. This is to let you - know that if you change the network offering, the CIDR will be affected. Choose No - to proceed with the change. - Wait for the update to complete. Don’t try to restart VMs until the network change is - complete. - If you stopped any VMs, restart them. - -
- + Changing the Network Offering on a Guest Network + A user or administrator can change the network offering that is associated with an existing + guest network. + + + Log in to the &PRODUCT; UI as an administrator or end user. + + + If you are changing from a network offering that uses the &PRODUCT; virtual router to + one that uses external devices as network service providers, you must first stop all the VMs + on the network. + + + In the left navigation, choose Network. + + + Click the name of the network you want to modify. + + + In the Details tab, click Edit. + + + + + EditButton.png: button to edit a network + + + + + In Network Offering, choose the new network offering, then click Apply. + A prompt is displayed asking whether you want to keep the existing CIDR. This is to let + you know that if you change the network offering, the CIDR will be affected. + If you upgrade between virtual router as a provider and an external network device as + provider, acknowledge the change of CIDR to continue, so choose Yes. + + + Wait for the update to complete. Don’t try to restart VMs until the network change is + complete. + + + If you stopped any VMs, restart them. + + + diff --git a/docs/en-US/changed-API-commands-4.2.xml b/docs/en-US/changed-API-commands-4.2.xml index b1008875a51..8fda9cc13bd 100644 --- a/docs/en-US/changed-API-commands-4.2.xml +++ b/docs/en-US/changed-API-commands-4.2.xml @@ -217,9 +217,9 @@ updateCluster - The following new request parameters are added: cpuovercommitratio - (optional), memoryovercommitratio (optional) - The following new response parameters are added: cpuovercommitratio, + The following new request parameters are removed: cpuovercommitratio, + memoryovercommitratio + The following new response parameters are removed: cpuovercommitratio, memoryovercommitratio @@ -388,24 +388,22 @@ listCapabilities The following new response parameters are added: apilimitinterval and - apilimitmax. - See . + apilimitmax. createServiceOffering The following new request parameters are added: deploymentplanner (optional), isvolatile (optional), serviceofferingdetails (optional). isvolatie indicates whether the service offering includes Volatile VM capability, - which will discard the VM's root disk and create a new one on reboot. See . + which will discard the VM's root disk and create a new one on reboot. The following new response parameters are added: deploymentplanner, isvolatile restoreVirtualMachine - The following request parameter is added: templateID (optional). This is used - to point to the new template ID when the base image is updated. See . + The following request parameter is added: templateID (optional). This is used to point to the + new template ID when the base image is updated. The parameter templateID can be an ISO + ID in case of restore vm deployed using ISO. The following response parameters are added: diskioread, diskiowrite, diskkbsread, diskkbswrite, displayvm, isdynamicallyscalable, affinitygroup @@ -561,11 +559,9 @@ addCluster - The following request parameters are added: cpuovercommitratio (optional), - guestvswitchtype (optional), guestvswitchtype (optional), memoryovercommitratio + The following request parameters are added: guestvswitchtype (optional), guestvswitchtype (optional), publicvswitchtype (optional), publicvswitchtype (optional) - See . - The following request parameters are added: cpuovercommitratio, + The following request parameters are removed: cpuovercommitratio, memoryovercommitratio @@ -573,7 +569,6 @@ updateCluster The following request parameters are added: cpuovercommitratio, ramovercommitratio - See . @@ -584,7 +579,6 @@ The following request parameters are added: hypervisor (optional), provider (optional), scope (optional) The following request parameters have been made mandatory: podid, clusterid - See . The following response parameter has been added: hypervisor, scope, suitableformigration @@ -592,7 +586,6 @@ listStoragePools The following request parameter is added: scope (optional) - See . The following response parameters are added: hypervisor, scope, suitableformigration diff --git a/docs/en-US/changing-service-offering-for-vm.xml b/docs/en-US/changing-service-offering-for-vm.xml index 4fc9ef4270b..5e37880cac0 100644 --- a/docs/en-US/changing-service-offering-for-vm.xml +++ b/docs/en-US/changing-service-offering-for-vm.xml @@ -22,33 +22,168 @@ under the License. -->
- Changing the Service Offering for a VM - To upgrade or downgrade the level of compute resources available to a virtual machine, you can change the VM's compute offering. - - Log in to the &PRODUCT; UI as a user or admin. - In the left navigation, click Instances. - Choose the VM that you want to work with. - Click the Stop button to stop the VM. - - - - - StopButton.png: button to stop a VM - - - - Click the Change Service button. - - - - - ChangeServiceButton.png: button to change the service of a - VM - - - The Change service dialog box is displayed. - Select the offering you want to apply to the selected VM. - Click OK. - -
- + Changing the Service Offering for a VM + To upgrade or downgrade the level of compute resources available to a virtual machine, you + can change the VM's compute offering. + + + Log in to the &PRODUCT; UI as a user or admin. + + + In the left navigation, click Instances. + + + Choose the VM that you want to work with. + + + (Skip this step if you have enabled dynamic VM scaling; see .) + Click the Stop button to stop the VM. + + + + + StopButton.png: button to stop a VM + + + + + + Click the Change Service button. + + + + + ChangeServiceButton.png: button to change the service of a VM + + + The Change service dialog box is displayed. + + + Select the offering you want to apply to the selected VM. + + + Click OK. + + +
+ + CPU and Memory Scaling for Running VMs + (Supported on VMware and XenServer) + It is not always possible to accurately predict the CPU and RAM requirements when you + first deploy a VM. You might need to increase these resources at any time during the life of a + VM. You can dynamically modify CPU and RAM levels to scale up these resources for a running VM + without incurring any downtime. + Dynamic CPU and RAM scaling can be used in the following cases: + + + User VMs on hosts running VMware and XenServer. + + + System VMs on VMware. + + + VMware Tools or XenServer Tools must be installed on the virtual machine. + + + The new requested CPU and RAM values must be within the constraints allowed by the + hypervisor and the VM operating system. + + + New VMs that are created after the installation of &PRODUCT; 4.2 can use the dynamic + scaling feature. If you are upgrading from a previous version of &PRODUCT;, your existing + VMs created with previous versions will not have the dynamic scaling capability unless you + update them using the following procedure. + + +
+
+ Updating Existing VMs + If you are upgrading from a previous version of &PRODUCT;, and you want your existing VMs + created with previous versions to have the dynamic scaling capability, update the VMs using + the following steps: + + + Make sure the zone-level setting enable.dynamic.scale.vm is set to true. In the left + navigation bar of the &PRODUCT; UI, click Infrastructure, then click Zones, click the zone + you want, and click the Settings tab. + + + Install Xen tools (for XenServer hosts) or VMware Tools (for VMware hosts) on each VM + if they are not already installed. + + + Stop the VM. + + + Click the Edit button. + + + Click the Dynamically Scalable checkbox. + + + Click Apply. + + + Restart the VM. + + +
+
+ Configuring Dynamic CPU and RAM Scaling + To configure this feature, use the following new global configuration variables: + + + enable.dynamic.scale.vm: Set to True to enable the feature. By default, the feature is + turned off. + + + scale.retry: How many times to attempt the scaling operation. Default = 2. + + +
+
+ How to Dynamically Scale CPU and RAM + To modify the CPU and/or RAM capacity of a virtual machine, you need to change the compute + offering of the VM to a new compute offering that has the desired CPU and RAM values. You can + use the same steps described above in , but + skip the step where you stop the virtual machine. Of course, you might have to create a new + compute offering first. + When you submit a dynamic scaling request, the resources will be scaled up on the current + host if possible. If the host does not have enough resources, the VM will be live migrated to + another host in the same cluster. If there is no host in the cluster that can fulfill the + requested level of CPU and RAM, the scaling operation will fail. The VM will continue to run + as it was before. +
+
+ Limitations + + + You can not do dynamic scaling for system VMs on XenServer. + + + &PRODUCT; will not check to be sure that the new CPU and RAM levels are compatible + with the OS running on the VM. + + + When scaling memory or CPU for a Linux VM on VMware, you might need to run scripts in + addition to the other steps mentioned above. For more information, see Hot adding memory in Linux (1012764) in the VMware Knowledge Base. + + + (VMware) If resources are not available on the current host, scaling up will fail on + VMware because of a known issue where &PRODUCT; and vCenter calculate the available + capacity differently. For more information, see https://issues.apache.org/jira/browse/CLOUDSTACK-1809. + + + On VMs running Linux 64-bit and Windows 7 32-bit operating systems, if the VM is + initially assigned a RAM of less than 3 GB, it can be dynamically scaled up to 3 GB, but + not more. This is due to a known issue with these operating systems, which will freeze if + an attempt is made to dynamically scale from less than 3 GB to more than 3 GB. + + +
+ diff --git a/docs/en-US/configure-acl.xml b/docs/en-US/configure-acl.xml index c89210b3c50..3ac2b7462c4 100644 --- a/docs/en-US/configure-acl.xml +++ b/docs/en-US/configure-acl.xml @@ -22,9 +22,11 @@ Configuring Network Access Control List Define Network Access Control List (ACL) on the VPC virtual router to control incoming (ingress) and outgoing (egress) traffic between the VPC tiers, and the tiers and Internet. By - default, all incoming and outgoing traffic to the guest networks is blocked. To open the ports, - you must create a new network ACL. The network ACLs can be created for the tiers only if the - NetworkACL service is supported. + default, all incoming traffic to the guest networks is blocked and all outgoing traffic from + guest networks is allowed, once you add an ACL rule for outgoing traffic, then only outgoing + traffic specified in this ACL rule is allowed, the rest is blocked. To open the ports, you must + create a new network ACL. The network ACLs can be created for the tiers only if the NetworkACL + service is supported.
About Network ACL Lists In &PRODUCT; terminology, Network ACL is a group of Network ACL items. Network ACL items @@ -35,8 +37,8 @@ VPC tiers within a VPC. A Tier is associated with a Network ACL at all the times. Each tier can be associated with only one ACL. The default Network ACL is used when no ACL is associated. Default behavior is all the - incoming and outgoing traffic is blocked to the tiers. Default network ACL cannot be removed - or modified. Contents of the default Network ACL is: + incoming traffic is blocked and outgoing traffic is allowed from the tiers. Default network + ACL cannot be removed or modified. Contents of the default Network ACL is: @@ -222,7 +224,7 @@
- Assigning a Custom ACL List to a Tier + Creating a Tier with Custom ACL List Create a VPC. diff --git a/docs/en-US/configure-vpn.xml b/docs/en-US/configure-vpn.xml index 5d25620b3eb..f389f30efc3 100644 --- a/docs/en-US/configure-vpn.xml +++ b/docs/en-US/configure-vpn.xml @@ -22,7 +22,7 @@ under the License. -->
- Configuring VPN + Configuring Remote Access VPN To set up VPN for the cloud: Log in to the &PRODUCT; UI as an administrator or end user. diff --git a/docs/en-US/creating-compute-offerings.xml b/docs/en-US/creating-compute-offerings.xml index 31f974196fb..5c5033afabb 100644 --- a/docs/en-US/creating-compute-offerings.xml +++ b/docs/en-US/creating-compute-offerings.xml @@ -54,6 +54,10 @@ hosts CPU cap: Whether to limit the level of CPU usage even if spare capacity is available. + isVolatile: If checked, VMs created from this service + offering will have their root disks reset upon reboot. This is useful for + secure environments that need a fresh start on every boot and for desktops + that should not retain state. Public: Indicate whether the service offering should be available all domains or only some domains. Choose Yes to make it available to all domains. Choose No to limit the scope to a subdomain; &PRODUCT; diff --git a/docs/en-US/creating-disk-offerings.xml b/docs/en-US/creating-disk-offerings.xml index 12bb2aca785..627311e4418 100644 --- a/docs/en-US/creating-disk-offerings.xml +++ b/docs/en-US/creating-disk-offerings.xml @@ -24,7 +24,7 @@
Creating a New Disk Offering - To create a system service offering: + To create a new disk offering: Log in with admin privileges to the &PRODUCT; UI. In the left navigation bar, click Service Offerings. @@ -32,11 +32,15 @@ Click Add Disk Offering. In the dialog, make the following choices: - Name. Any desired name for the system offering. + Name. Any desired name for the disk offering. Description. A short description of the offering that can be displayed to users Custom Disk Size. If checked, the user can set their own disk size. If not checked, the root administrator must define a value in Disk Size. Disk Size. Appears only if Custom Disk Size is not selected. Define the volume size in GB. - (Optional)Storage Tags. The tags that should be associated with the primary storage for this disk. Tags are a comma separated list of attributes of the storage. For example "ssd,blue". Tags are also added on Primary Storage. &PRODUCT; matches tags on a disk offering to tags on the storage. If a tag is present on a disk offering that tag (or tags) must also be present on Primary Storage for the volume to be provisioned. If no such primary storage exists, allocation from the disk offering will fail.. + QoS Type. Three options: Empty (no Quality of Service), hypervisor (rate limiting enforced on the hypervisor side), and storage (guaranteed minimum and maximum IOPS enforced on the storage side). If leveraging QoS, make sure that the hypervisor or storage system supports this feature. + Custom IOPS. If checked, the user can set their own IOPS. If not checked, the root administrator can define values. If the root admin does not set values when using storage QoS, default values are used (the defauls can be overridden if the proper parameters are passed into &PRODUCT; when creating the primary storage in question). + Min IOPS. Appears only if storage QoS is to be used. Set a guaranteed minimum number of IOPS to be enforced on the storage side. + Max IOPS. Appears only if storage QoS is to be used. Set a maximum number of IOPS to be enforced on the storage side (the system may go above this limit in certain circumstances for short intervals). + (Optional)Storage Tags. The tags that should be associated with the primary storage for this disk. Tags are a comma separated list of attributes of the storage. For example "ssd,blue". Tags are also added on Primary Storage. &PRODUCT; matches tags on a disk offering to tags on the storage. If a tag is present on a disk offering that tag (or tags) must also be present on Primary Storage for the volume to be provisioned. If no such primary storage exists, allocation from the disk offering will fail.. Public. Indicate whether the service offering should be available all domains or only some domains. Choose Yes to make it available to all domains. Choose No to limit the scope to a subdomain; &PRODUCT; will then prompt for the subdomain's name. Click Add. diff --git a/docs/en-US/creating-network-offerings.xml b/docs/en-US/creating-network-offerings.xml index 6e25b27e9ad..4f75781c3cb 100644 --- a/docs/en-US/creating-network-offerings.xml +++ b/docs/en-US/creating-network-offerings.xml @@ -241,7 +241,7 @@ For information on Elastic IP, see . - Redundant router capability. Available only when + Redundant router capability: Available only when Virtual Router is selected as the Source NAT provider. Select this option if you want to use two virtual routers in the network for uninterrupted connection: one operating as the master virtual router and the other as the backup. The master virtual router @@ -251,7 +251,7 @@ reliability if one host is down. - Conserve mode. Indicate whether to use conserve + Conserve mode: Indicate whether to use conserve mode. In this mode, network resources are allocated only when the first virtual machine starts in the network. When conservative mode is off, the public IP can only be used for a single service. For example, a public IP used for a port forwarding rule cannot be @@ -264,9 +264,18 @@ - Tags. Network tag to specify which physical network + Tags: Network tag to specify which physical network to use. + + Default egress policy: Configure the default policy + for firewall egress rules. Options are Allow and Deny. Default is Allow if no egress + policy is specified, which indicates that all the egress traffic is accepted when a + guest network is created from this offering. + To block the egress traffic for a guest network, select Deny. In this case, when you + configure an egress rules for an isolated guest network, rules are added to allow the + specified traffic. + diff --git a/docs/en-US/creating-shared-network.xml b/docs/en-US/creating-shared-network.xml index 6c2b50466ba..e6a018f39d5 100644 --- a/docs/en-US/creating-shared-network.xml +++ b/docs/en-US/creating-shared-network.xml @@ -23,7 +23,7 @@ Configuring a Shared Guest Network - Log in to the CloudPlatform UI as administrator. + Log in to the &PRODUCT; UI as administrator. In the left navigation, choose Infrastructure. diff --git a/docs/en-US/delete-event-alerts.xml b/docs/en-US/delete-event-alerts.xml index 5958b721940..c0d56719818 100644 --- a/docs/en-US/delete-event-alerts.xml +++ b/docs/en-US/delete-event-alerts.xml @@ -44,19 +44,44 @@ - Archived alerts or events cannot be viewed in the UI, or by using the API. They are + Archived alerts or events cannot be viewed in the UI or by using the API. They are maintained in the database for auditing or compliance purposes. - +
Permissions Consider the following: - - - - The root admin can delete or archive one or multiple alerts or events. - - - The domain admin or end user can delete or archive one or multiple events. - - + + + The root admin can delete or archive one or multiple alerts or events. + + + The domain admin or end user can delete or archive one or multiple events. + + +
+
+ Procedure + + + Log in as administrator to the &PRODUCT; UI. + + + In the left navigation, click Events. + + + Perform either of the following: + + + To archive events, click Archive Events, and specify event type and date. + + + To archive events, click Delete Events, and specify event type and date. + + + + + Click OK. + + +
diff --git a/docs/en-US/detach-move-volumes.xml b/docs/en-US/detach-move-volumes.xml index 7103c305c4f..8922db12161 100644 --- a/docs/en-US/detach-move-volumes.xml +++ b/docs/en-US/detach-move-volumes.xml @@ -24,9 +24,8 @@
Detaching and Moving Volumes - This procedure is different from moving disk volumes from one storage pool to another. See - VM Storage Migration - + This procedure is different from moving volumes from one storage pool to another as described in . + A volume can be detached from a guest VM and attached to another guest. Both &PRODUCT; administrators and users can detach volumes from VMs and move them to other VMs. If the two VMs are in different clusters, and the volume is large, it may take several diff --git a/docs/en-US/egress-firewall-rule.xml b/docs/en-US/egress-firewall-rule.xml index 9b45e2e02a2..93d5a814547 100644 --- a/docs/en-US/egress-firewall-rule.xml +++ b/docs/en-US/egress-firewall-rule.xml @@ -19,80 +19,150 @@ under the License. -->
- Creating Egress Firewall Rules in an Advanced Zone + Egress Firewall Rules in an Advanced Zone The egress traffic originates from a private network to a public network, such as the - Internet. By default, the egress traffic is blocked, so no outgoing traffic is allowed from a - guest network to the Internet. However, you can control the egress traffic in an Advanced zone - by creating egress firewall rules. When an egress firewall rule is applied, the traffic specific - to the rule is allowed and the remaining traffic is blocked. When all the firewall rules are - removed the default policy, Block, is applied. - Egress firewall rules are supported on Juniper SRX and virtual router. - - The egress firewall rules are not supported on shared networks. - - Consider the following scenarios to apply egress firewall rules: - - - Allow the egress traffic from specified source CIDR. The Source CIDR is part of guest - network CIDR. - - - Allow the egress traffic with destination protocol TCP,UDP,ICMP, or ALL. - - - Allow the egress traffic with destination protocol and port range. The port range is - specified for TCP, UDP or for ICMP type and code. - - - To configure an egress firewall rule: - - - Log in to the &PRODUCT; UI as an administrator or end user. - - - In the left navigation, choose Network. - - - In Select view, choose Guest networks, then click the Guest network you want. - - - To add an egress rule, click the Egress rules tab and fill out the following fields to - specify what type of traffic is allowed to be sent out of VM instances in this guest - network: - - - - - - egress-firewall-rule.png: adding an egress firewall rule - - - - - CIDR: (Add by CIDR only) To send traffic only to - the IP addresses within a particular address block, enter a CIDR or a comma-separated - list of CIDRs. The CIDR is the base IP address of the destination. For example, - 192.168.0.0/22. To allow all CIDRs, set to 0.0.0.0/0. - - - Protocol: The networking protocol that VMs uses to - send outgoing traffic. The TCP and UDP protocols are typically used for data exchange - and end-user communications. The ICMP protocol is typically used to send error messages - or network monitoring data. - - - Start Port, End Port: (TCP, UDP only) A range of - listening ports that are the destination for the outgoing traffic. If you are opening a - single port, use the same number in both fields. - - - ICMP Type, ICMP Code: (ICMP only) The type of - message and error code that are sent. - - - - - Click Add. - - + Internet. By default, the egress traffic is blocked in default network offerings, so no outgoing + traffic is allowed from a guest network to the Internet. However, you can control the egress + traffic in an Advanced zone by creating egress firewall rules. When an egress firewall rule is + applied, the traffic specific to the rule is allowed and the remaining traffic is blocked. When + all the firewall rules are removed the default policy, Block, is applied. +
+ Prerequisites and Guidelines + Consider the following scenarios to apply egress firewall rules: + + + Egress firewall rules are supported on Juniper SRX and virtual router. + + + The egress firewall rules are not supported on shared networks. + + + Allow the egress traffic from specified source CIDR. The Source CIDR is part of guest + network CIDR. + + + Allow the egress traffic with protocol TCP,UDP,ICMP, or ALL. + + + Allow the egress traffic with protocol and destination port range. The port range is + specified for TCP, UDP or for ICMP type and code. + + + The default policy is Allow for the new network offerings, whereas on upgrade existing + network offerings with firewall service providers will have the default egress policy + Deny. + + +
+
+ Configuring an Egress Firewall Rule + + + Log in to the &PRODUCT; UI as an administrator or end user. + + + In the left navigation, choose Network. + + + In Select view, choose Guest networks, then click the Guest network you want. + + + To add an egress rule, click the Egress rules tab and fill out the following fields to + specify what type of traffic is allowed to be sent out of VM instances in this guest + network: + + + + + + egress-firewall-rule.png: adding an egress firewall rule + + + + + CIDR: (Add by CIDR only) To send traffic only to + the IP addresses within a particular address block, enter a CIDR or a comma-separated + list of CIDRs. The CIDR is the base IP address of the destination. For example, + 192.168.0.0/22. To allow all CIDRs, set to 0.0.0.0/0. + + + Protocol: The networking protocol that VMs uses + to send outgoing traffic. The TCP and UDP protocols are typically used for data + exchange and end-user communications. The ICMP protocol is typically used to send + error messages or network monitoring data. + + + Start Port, End Port: (TCP, UDP only) A range of + listening ports that are the destination for the outgoing traffic. If you are opening + a single port, use the same number in both fields. + + + ICMP Type, ICMP Code: (ICMP only) The type of + message and error code that are sent. + + + + + Click Add. + + +
+
+ Configuring the Default Egress Policy + The default egress policy for Isolated guest network is configured by using Network + offering. Use the create network offering option to determine whether the default policy + should be block or allow all the traffic to the public network from a guest network. Use this + network offering to create the network. If no policy is specified, by default all the traffic + is allowed from the guest network that you create by using this network offering. + You have two options: Allow and Deny. + + Allow + If you select Allow for a network offering, by default egress traffic is allowed. + However, when an egress rule is configured for a guest network, rules are applied to block + the specified traffic and rest are allowed. If no egress rules are configured for the + network, egress traffic is accepted. + + + Deny + If you select Deny for a network offering, by default egress traffic for the guest + network is blocked. However, when an egress rules is configured for a guest network, rules + are applied to allow the specified traffic. While implementing a guest network, &PRODUCT; + adds the firewall egress rule specific to the default egress policy for the guest + network. + + This feature is supported only on virtual router and Juniper SRX. + + + Create a network offering with your desirable default egress policy: + + + Log in with admin privileges to the &PRODUCT; UI. + + + In the left navigation bar, click Service Offerings. + + + In Select Offering, choose Network Offering. + + + Click Add Network Offering. + + + In the dialog, make necessary choices, including firewall provider. + + + In the Default egress policy field, specify the behaviour. + + + Click OK. + + + + + Create an isolated network by using this network offering. + Based on your selection, the network will have the egress public traffic blocked or + allowed. + + +
diff --git a/docs/en-US/global-config.xml b/docs/en-US/global-config.xml index 11952c382ac..407d97d2ee4 100644 --- a/docs/en-US/global-config.xml +++ b/docs/en-US/global-config.xml @@ -1,5 +1,5 @@ - %BOOK_ENTITIES; ]> @@ -19,115 +19,112 @@ under the License. --> - Global Configuration Parameters -
- Setting Global Configuration Parameters - &PRODUCT; provides parameters that you can set to control many aspects of the cloud. When - &PRODUCT; is first installed, and periodically thereafter, you might need to modify these - settings. - - - Log in to the UI as administrator. - - - In the left navigation bar, click Global Settings. - - - In Select View, choose one of the following: - - - Global Settings. This displays a list of the parameters with brief descriptions and - current values. - - - Hypervisor Capabilities. This displays a list of hypervisor versions with the - maximum number of guests supported for each. - - - - - Use the search box to narrow down the list to those you are interested in. - - - Click the Edit icon to modify a value. If you are viewing Hypervisor Capabilities, you - must click the name of the hypervisor first to display the editing screen. - - -
+ Setting Configuration Parameters
- About Global Configuration Parameters + About Configuration Parameters &PRODUCT; provides a variety of settings you can use to set limits, configure features, and enable or disable features in the cloud. Once your Management Server is running, you might - need to set some of these global configuration parameters, depending on what optional features - you are setting up. - To modify global configuration parameters, use the steps in "Setting Global Configuration - Parameters." + need to set some of these configuration parameters, depending on what optional features + you are setting up. + You can set default values at the global level, which will be in effect throughout the cloud unless you override them at a lower level. + You can make local settings, which will override the global configuration parameter values, at the level of an account, zone, cluster, or primary storage. The documentation for each &PRODUCT; feature should direct you to the names of the applicable - parameters. Many of them are discussed in the &PRODUCT; Administration Guide. The following table + parameters. The following table shows a few of the more useful parameters. - - Field - Value - + + Field + Value + - - management.network.cidr - A CIDR that describes the network that the management CIDRs reside on. This - variable must be set for deployments that use vSphere. It is recommended to be set for - other deployments as well. Example: 192.168.3.0/24. - - - xen.setup.multipath - For XenServer nodes, this is a true/false variable that instructs CloudStack to - enable iSCSI multipath on the XenServer Hosts when they are added. This defaults to false. - Set it to true if you would like CloudStack to enable multipath. - If this is true for a NFS-based deployment multipath will still be enabled on the - XenServer host. However, this does not impact NFS operation and is harmless. - - - secstorage.allowed.internal.sites - This is used to protect your internal network from rogue attempts to download - arbitrary files using the template download feature. This is a comma-separated list of CIDRs. - If a requested URL matches any of these CIDRs the Secondary Storage VM will use the private - network interface to fetch the URL. Other URLs will go through the public interface. - We suggest you set this to 1 or 2 hardened internal machines where you keep your templates. - For example, set it to 192.168.1.66/32. - - - use.local.storage - Determines whether CloudStack will use storage that is local to the Host for data - disks, templates, and snapshots. By default CloudStack will not use this storage. You should - change this to true if you want to use local storage and you understand the reliability and - feature drawbacks to choosing local storage. - - - host - This is the IP address of the Management Server. If you are using multiple - Management Servers you should enter a load balanced IP address that is reachable via - the private network. - - - default.page.size - Maximum number of items per page that can be returned by a CloudStack API command. - The limit applies at the cloud level and can vary from cloud to cloud. You can override this - with a lower value on a particular API call by using the page and pagesize API command parameters. - For more information, see the Developer's Guide. Default: 500. - - - ha.tag - The label you want to use throughout the cloud to designate certain hosts as dedicated - HA hosts. These hosts will be used only for HA-enabled VMs that are restarting due to the failure - of another host. For example, you could set this to ha_host. Specify the ha.tag value as a host tag - when you add a new host to the cloud. - + + management.network.cidr + A CIDR that describes the network that the management CIDRs reside on. This + variable must be set for deployments that use vSphere. It is recommended to be set for + other deployments as well. Example: 192.168.3.0/24. + + + xen.setup.multipath + For XenServer nodes, this is a true/false variable that instructs CloudStack to + enable iSCSI multipath on the XenServer Hosts when they are added. This defaults to false. + Set it to true if you would like CloudStack to enable multipath. + If this is true for a NFS-based deployment multipath will still be enabled on the + XenServer host. However, this does not impact NFS operation and is harmless. + + + secstorage.allowed.internal.sites + This is used to protect your internal network from rogue attempts to download + arbitrary files using the template download feature. This is a comma-separated list of CIDRs. + If a requested URL matches any of these CIDRs the Secondary Storage VM will use the private + network interface to fetch the URL. Other URLs will go through the public interface. + We suggest you set this to 1 or 2 hardened internal machines where you keep your templates. + For example, set it to 192.168.1.66/32. + + + use.local.storage + Determines whether CloudStack will use storage that is local to the Host for data + disks, templates, and snapshots. By default CloudStack will not use this storage. You should + change this to true if you want to use local storage and you understand the reliability and + feature drawbacks to choosing local storage. + + + host + This is the IP address of the Management Server. If you are using multiple + Management Servers you should enter a load balanced IP address that is reachable via + the private network. + + + default.page.size + Maximum number of items per page that can be returned by a CloudStack API command. + The limit applies at the cloud level and can vary from cloud to cloud. You can override this + with a lower value on a particular API call by using the page and pagesize API command parameters. + For more information, see the Developer's Guide. Default: 500. + + + ha.tag + The label you want to use throughout the cloud to designate certain hosts as dedicated + HA hosts. These hosts will be used only for HA-enabled VMs that are restarting due to the failure + of another host. For example, you could set this to ha_host. Specify the ha.tag value as a host tag + when you add a new host to the cloud. +
+
+ Setting Global Configuration Parameters + Use the following steps to set global configuration parameters. These values will be the defaults in effect throughout your &PRODUCT; deployment. + + Log in to the UI as administrator. + In the left navigation bar, click Global Settings. + In Select View, choose one of the following: + + Global Settings. This displays a list of the parameters with brief descriptions and current values. + Hypervisor Capabilities. This displays a list of hypervisor versions with the maximum number of guests supported for each. + + + Use the search box to narrow down the list to those you are interested in. + In the Actions column, click the Edit icon to modify a value. If you are viewing Hypervisor Capabilities, you must click the name of the hypervisor first to display the editing screen. + +
+
+ Setting Local Configuration Parameters + Use the following steps to set local configuration parameters for an account, zone, cluster, or primary storage. + These values will override the global configuration settings. + + Log in to the UI as administrator. + In the left navigation bar, click Infrastructure or Accounts, depending on where you want to set a value. + Find the name of the particular resource that you want to work with. For example, if you are in Infrastructure, + click View All on the Zones, Clusters, or Primary Storage area. + Click the name of the resource where you want to set a limit. + Click the Settings tab. + Use the search box to narrow down the list to those you are interested in. + In the Actions column, click the Edit icon to modify a value. + +
diff --git a/docs/en-US/guest-traffic.xml b/docs/en-US/guest-traffic.xml index bca635582a8..943073ebc97 100644 --- a/docs/en-US/guest-traffic.xml +++ b/docs/en-US/guest-traffic.xml @@ -23,15 +23,21 @@ -->
Guest Traffic - A network can carry guest traffic only between VMs within one zone. Virtual machines in different zones cannot communicate with each other using their IP addresses; they must communicate with each other by routing through a public IP address. - This figure illustrates a typical guest traffic setup: - - - - - Depicts a guest traffic setup. - - The Management Server automatically creates a virtual router for each network. A virtual router is a special virtual machine that runs on the hosts. Each virtual router has three network interfaces. Its eth0 interface serves as the gateway for the guest traffic and has the IP address of 10.1.1.1. Its eth1 interface is used by the system to configure the virtual router. Its eth2 interface is assigned a public IP address for public traffic. + A network can carry guest traffic only between VMs within one zone. Virtual machines in different zones cannot communicate with each other using their IP addresses; they must communicate with each other by routing through a public IP address. + See a typical guest traffic setup given below: + + + + + guest-traffic-setup.png: Depicts a guest traffic setup + + Typically, the Management Server automatically creates a virtual router for each network. A + virtual router is a special virtual machine that runs on the hosts. Each virtual router in an + isolated network has three network interfaces. If multiple public VLAN is used, the router will + have multiple public interfaces. Its eth0 interface serves as the gateway for the guest traffic + and has the IP address of 10.1.1.1. Its eth1 interface is used by the system to configure the + virtual router. Its eth2 interface is assigned a public IP address for public traffic. If + multiple public VLAN is used, the router will have multiple public interfaces. The virtual router provides DHCP and will automatically assign an IP address for each guest VM within the IP range assigned for the network. The user can manually reconfigure guest VMs to assume different IP addresses. Source NAT is automatically configured in the virtual router to forward outbound traffic for all guest VMs
diff --git a/docs/en-US/health-checks-for-lb-rules.xml b/docs/en-US/health-checks-for-lb-rules.xml new file mode 100644 index 00000000000..be2aec0dff4 --- /dev/null +++ b/docs/en-US/health-checks-for-lb-rules.xml @@ -0,0 +1,51 @@ + + +%BOOK_ENTITIES; +]> + + +
+ + Health Checks for Load Balancer Rules + (NetScaler load balancer only; requires NetScaler version 10.0) + + Health checks are used in load-balanced applications to ensure that requests are forwarded + only to running, available services. + When creating a load balancer rule, you can specify a health check policy. + This is in addition to specifying the + stickiness policy, algorithm, and other load balancer rule options. + You can configure one health check policy per load balancer rule. + Any load balancer rule defined on a NetScaler load balancer in &PRODUCT; can have a health check policy. + The policy consists of a ping path, thresholds to define "healthy" and "unhealthy" states, + health check frequency, and timeout wait interval. + When a health check policy is in effect, + the load balancer will stop forwarding requests to any resources that are found to be unhealthy. + If the resource later becomes available again, the periodic health check + will discover it, and the resource will once again be added to the pool of resources that can + receive requests from the load balancer. + At any given time, the most recent result of the health check is displayed in the UI. + For any VM that is attached to a load balancer rule with a health check configured, + the state will be shown as UP or DOWN in the UI depending on the result of the most recent health check. + You can delete or modify existing health check policies. + To configure how often the health check is performed by default, use the global + configuration setting healthcheck.update.interval (default value is 600 seconds). + You can override this value for an individual health check policy. + For details on how to set a health check policy using the UI, see . +
\ No newline at end of file diff --git a/docs/en-US/host-allocation.xml b/docs/en-US/host-allocation.xml index f5bc53c7fbf..dddffd553ac 100644 --- a/docs/en-US/host-allocation.xml +++ b/docs/en-US/host-allocation.xml @@ -1,5 +1,5 @@ - %BOOK_ENTITIES; ]> @@ -23,10 +23,101 @@ -->
- Host Allocation - The system automatically picks the most appropriate host to run each virtual machine. End users may specify the zone in which the virtual machine will be created. End users do not have control over which host will run the virtual machine instance. - &PRODUCT; administrators can specify that certain hosts should have a preference for particular types of guest instances. For example, an administrator could state that a host should have a preference to run Windows guests. The default host allocator will attempt to place guests of that OS type on such hosts first. If no such host is available, the allocator will place the instance wherever there is sufficient physical capacity. - Both vertical and horizontal allocation is allowed. Vertical allocation consumes all the resources of a given host before allocating any guests on a second host. This reduces power consumption in the cloud. Horizontal allocation places a guest on each host in a round-robin fashion. This may yield better performance to the guests in some cases. &PRODUCT; also allows an element of CPU over-provisioning as configured by the administrator. Over-provisioning allows the administrator to commit more CPU cycles to the allocated guests than are actually available from the hardware. - &PRODUCT; also provides a pluggable interface for adding new allocators. These custom allocators can provide any policy the administrator desires. - + Assigning VMs to Hosts + At any point in time, each virtual machine instance is running on a single host. + How does &PRODUCT; determine which host to place a VM on? There are several ways: + + Automatic default host allocation. &PRODUCT; can automatically pick + the most appropriate host to run each virtual machine. + Instance type preferences. &PRODUCT; administrators can specify that certain hosts should have a preference for particular types of guest instances. + For example, an administrator could state that a host should have a preference to run Windows guests. + The default host allocator will attempt to place guests of that OS type on such hosts first. + If no such host is available, the allocator will place the instance wherever there is sufficient physical capacity. + Vertical and horizontal allocation. + Vertical allocation consumes all the resources of a given host before allocating any guests on a second host. + This reduces power consumption in the cloud. Horizontal allocation places a guest on each host in a round-robin fashion. + This may yield better performance to the guests in some cases. + End user preferences. + Users can not control exactly which host will run a given VM instance, + but they can specify a zone for the VM. + &PRODUCT; is then restricted to allocating the VM only to one of the hosts in that zone. + Host tags. The administrator can assign tags to hosts. These tags can be used to + specify which host a VM should use. + The &PRODUCT; administrator decides whether to define host tags, then create a service offering using those tags and offer it to the user. + + Affinity groups. + By defining affinity groups and assigning VMs to them, the user or administrator can + influence (but not dictate) which VMs should run on separate hosts. + This feature is to let users specify that certain VMs won't be on the same host. + &PRODUCT; also provides a pluggable interface for adding new allocators. + These custom allocators can provide any policy the administrator desires. + +
+ Affinity Groups + By defining affinity groups and assigning VMs to them, the user or administrator can + influence (but not dictate) which VMs should run on separate hosts. + This feature is to let users specify that VMs with the same “host anti-affinity†type won’t be on the same host. + This serves to increase fault tolerance. + If a host fails, another VM offering the same service (for example, hosting the user's website) is still up and running on another host. + The scope of an affinity group is per user account. + Creating a New Affinity Group + To add an affinity group: + + Log in to the &PRODUCT; UI as an administrator or user. + In the left navigation bar, click Affinity Groups. + Click Add affinity group. In the dialog box, fill in the following fields: + + Name. Give the group a name. + Description. Any desired text to tell more about the purpose of the group. + Type. The only supported type shipped with &PRODUCT; is Host Anti-Affinity. + This indicates that the VMs in this group should avoid being placed on the same VM with each other. + If you see other types in this list, it means that your installation of &PRODUCT; has been extended + with customized affinity group plugins. + + + + Assign a New VM to an Affinity Group + To assign a new VM to an affinity group: + + Create the VM as usual, as described in . + In the Add Instance wizard, there is a new Affinity tab where you can select the affinity group. + + Change Affinity Group for an Existing VM + To assign an existing VM to an affinity group: + + Log in to the &PRODUCT; UI as an administrator or user. + In the left navigation bar, click Instances. + Click the name of the VM you want to work with. + Stop the VM by clicking the Stop button. + Click the Change Affinity button. + + + + + change-affinity-button.png: button to assign an affinity group + to a virtual machine + + + + + View Members of an Affinity Group + To see which VMs are currently assigned to a particular affinity group: + + In the left navigation bar, click Affinity Groups. + Click the name of the group you are interested in. + Click View Instances. The members of the group are listed. + From here, you can click the name of any VM in the list to access all its details and controls. + + Delete an Affinity Group + To delete an affinity group: + + In the left navigation bar, click Affinity Groups. + Click the name of the group you are interested in. + Click Delete. + Any VM that is a member of the affinity group will be disassociated from the group. + The former group members will continue to run normally on the current hosts, but if the + VM is restarted, it will no longer follow the host allocation rules from its former + affinity group. + +
diff --git a/docs/en-US/hypervisor-host-install-agent.xml b/docs/en-US/hypervisor-host-install-agent.xml index 41b6719bbaf..e339165d0da 100644 --- a/docs/en-US/hypervisor-host-install-agent.xml +++ b/docs/en-US/hypervisor-host-install-agent.xml @@ -31,4 +31,49 @@ In Ubuntu: $ apt-get install cloudstack-agent The host is now ready to be added to a cluster. This is covered in a later section, see . It is recommended that you continue to read the documentation before adding the host! +
+ Configure CPU model for KVM guest (Optional) + In additional,the &PRODUCT; Agent allows host administrator to control the guest CPU model which is exposed to KVM instances. By default, the CPU model of KVM instance is likely QEMU Virtual CPU version x.x.x with least CPU features exposed. There are a couple of reasons to specify the CPU model: + + To maximise performance of instances by exposing new host CPU features to the KVM instances; + To ensure a consistent default CPU across all machines,removing reliance of variable QEMU defaults; + + For the most part it will be sufficient for the host administrator to specify the guest CPU config in the per-host configuration file (/etc/cloudstack/agent/agent.properties). This will be achieved by introducing two new configuration parameters: + guest.cpu.mode=custom|host-model|host-passthrough +guest.cpu.model=from /usr/share/libvirt/cpu_map.xml(only valid when guest.cpu.mode=custom) + + There are three choices to fulfill the cpu model changes: + + + custom: you can explicitly specify one of the supported named model in /usr/share/libvirt/cpu_map.xml + + + host-model: libvirt will identify the CPU model in /usr/share/libvirt/cpu_map.xml which most closely matches the host, and then request additional CPU flags to complete the match. This should give close to maximum functionality/performance, which maintaining good reliability/compatibility if the guest is migrated to another host with slightly different host CPUs. + + + host-passthrough: libvirt will tell KVM to passthrough the host CPU with no modifications. The difference to host-model, instead of just matching feature flags, every last detail of the host CPU is matched. This gives absolutely best performance, and can be important to some apps which check low level CPU details, but it comes at a cost with respect to migration: the guest can only be migrated to an exactly matching host CPU. + + + Here are some examples: + + + custom + guest.cpu.mode=custom +guest.cpu.model=SandyBridge + + + + host-model + guest.cpu.mode=host-model + + + host-passthrough + guest.cpu.mode=host-passthrough + + + + host-passthrough may lead to migration failure,if you have this problem,you should use host-model or custom + +
+
diff --git a/docs/en-US/hypervisor-host-install-libvirt.xml b/docs/en-US/hypervisor-host-install-libvirt.xml index d3d6b9b4e80..c4be67e643f 100644 --- a/docs/en-US/hypervisor-host-install-libvirt.xml +++ b/docs/en-US/hypervisor-host-install-libvirt.xml @@ -46,6 +46,11 @@ so it looks like: libvirtd_opts="-d -l" + + In order to have the VNC Console work we have to make sure it will bind on 0.0.0.0. We do this by editing /etc/libvirt/qemu.conf + Make sure this parameter is set: + vnc_listen = "0.0.0.0" + Restart libvirt In RHEL or CentOS: diff --git a/docs/en-US/images/add-tier.png b/docs/en-US/images/add-tier.png index 881671e2133..0994dbd0a5a 100644 Binary files a/docs/en-US/images/add-tier.png and b/docs/en-US/images/add-tier.png differ diff --git a/docs/en-US/images/addAccount-icon.png b/docs/en-US/images/addAccount-icon.png new file mode 100644 index 00000000000..4743dbef2cf Binary files /dev/null and b/docs/en-US/images/addAccount-icon.png differ diff --git a/docs/en-US/images/change-affinity-button.png b/docs/en-US/images/change-affinity-button.png new file mode 100644 index 00000000000..c21ef758dc2 Binary files /dev/null and b/docs/en-US/images/change-affinity-button.png differ diff --git a/docs/en-US/images/dedicate-resource-button.png b/docs/en-US/images/dedicate-resource-button.png new file mode 100644 index 00000000000..0ac38e00eca Binary files /dev/null and b/docs/en-US/images/dedicate-resource-button.png differ diff --git a/docs/en-US/images/gslb.png b/docs/en-US/images/gslb.png index 9f13580c560..f0a04db45e1 100644 Binary files a/docs/en-US/images/gslb.png and b/docs/en-US/images/gslb.png differ diff --git a/docs/en-US/images/plugin1.jpg b/docs/en-US/images/plugin1.jpg new file mode 100644 index 00000000000..970233d8475 Binary files /dev/null and b/docs/en-US/images/plugin1.jpg differ diff --git a/docs/en-US/images/plugin2.jpg b/docs/en-US/images/plugin2.jpg new file mode 100644 index 00000000000..9c8a6107ba9 Binary files /dev/null and b/docs/en-US/images/plugin2.jpg differ diff --git a/docs/en-US/images/plugin3.jpg b/docs/en-US/images/plugin3.jpg new file mode 100644 index 00000000000..07fae790e22 Binary files /dev/null and b/docs/en-US/images/plugin3.jpg differ diff --git a/docs/en-US/images/plugin4.jpg b/docs/en-US/images/plugin4.jpg new file mode 100644 index 00000000000..2bcec9f773a Binary files /dev/null and b/docs/en-US/images/plugin4.jpg differ diff --git a/docs/en-US/images/plugin_intro.jpg b/docs/en-US/images/plugin_intro.jpg new file mode 100644 index 00000000000..113ffb32781 Binary files /dev/null and b/docs/en-US/images/plugin_intro.jpg differ diff --git a/docs/en-US/images/replace-acl-icon.png b/docs/en-US/images/replace-acl-icon.png index 6a15d4565dd..ae953ba2032 100644 Binary files a/docs/en-US/images/replace-acl-icon.png and b/docs/en-US/images/replace-acl-icon.png differ diff --git a/docs/en-US/images/vds-name.png b/docs/en-US/images/vds-name.png new file mode 100644 index 00000000000..bf5b4fcf35c Binary files /dev/null and b/docs/en-US/images/vds-name.png differ diff --git a/docs/en-US/images/view-systemvm-details.png b/docs/en-US/images/view-systemvm-details.png new file mode 100755 index 00000000000..bce270bf258 Binary files /dev/null and b/docs/en-US/images/view-systemvm-details.png differ diff --git a/docs/en-US/ip-forwarding-firewalling.xml b/docs/en-US/ip-forwarding-firewalling.xml index d7a24571429..d1beb2eb0f2 100644 --- a/docs/en-US/ip-forwarding-firewalling.xml +++ b/docs/en-US/ip-forwarding-firewalling.xml @@ -20,15 +20,16 @@ -->
IP Forwarding and Firewalling - By default, all incoming traffic to the public IP address is rejected. - All outgoing traffic from the guests is also blocked by default. - To allow outgoing traffic, follow the procedure in . + By default, all incoming traffic to the public IP address is rejected. All outgoing traffic + from the guests is also blocked by default. + To allow outgoing traffic, follow the procedure in . To allow incoming traffic, users may set up firewall rules and/or port forwarding rules. For example, you can use a firewall rule to open a range of ports on the public IP address, such as 33 through 44. Then use port forwarding rules to direct traffic from individual ports within that range to specific ports on user VMs. For example, one port forwarding rule could route incoming traffic on the public IP's port 33 to port 100 on one user VM's private IP. - +
diff --git a/docs/en-US/ip-vlan-tenant.xml b/docs/en-US/ip-vlan-tenant.xml index 42124f0f446..d58d49be63a 100644 --- a/docs/en-US/ip-vlan-tenant.xml +++ b/docs/en-US/ip-vlan-tenant.xml @@ -19,19 +19,26 @@ under the License. -->
- Dedicated Resources: Public IP Addresses and VLANs Per Account + Reserving Public IP Addresses and VLANs for Accounts &PRODUCT; provides you the ability to reserve a set of public IP addresses and VLANs - exclusively for an account. During zone creation, you can continue to define a set of VLANs and + exclusively for an account. During zone creation, you can continue defining a set of VLANs and multiple public IP ranges. This feature extends the functionality to enable you to dedicate a fixed set of VLANs and guest IP addresses for a tenant. + Note that if an account has consumed all the VLANs and IPs dedicated to it, the account can + acquire two more resources from the system. &PRODUCT; provides the root admin with two + configuration parameter to modify this default behavior—use.system.public.ips and + use.system.guest.vlans. These global parameters enable the root admin to disallow an account + from acquiring public IPs and guest VLANs from the system, if the account has dedicated + resources and these dedicated resources have all been consumed. Both these configurations are + configurable at the account level. This feature provides you the following capabilities: Reserve a VLAN range and public IP address range from an Advanced zone and assign it to - a domain or account + an account - Disassociate a VLAN and public IP address range from an domain or account + Disassociate a VLAN and public IP address range from an account View the number of public IP addresses allocated to an account diff --git a/docs/en-US/limit-accounts-domains.xml b/docs/en-US/limit-accounts-domains.xml index a864ee27ef3..78a642b3a5a 100644 --- a/docs/en-US/limit-accounts-domains.xml +++ b/docs/en-US/limit-accounts-domains.xml @@ -164,7 +164,7 @@
- Per-Domain Limits + Limiting Resource Usage in a Domain &PRODUCT; allows the configuration of limits on a domain basis. With a domain limit in place, all users still have their account limits. They are additionally limited, as a group, to not exceed the resource limits set on their domain. Domain limits aggregate the usage of diff --git a/docs/en-US/linux-installation.xml b/docs/en-US/linux-installation.xml index b560ee0d5bd..14a2f51b3d2 100644 --- a/docs/en-US/linux-installation.xml +++ b/docs/en-US/linux-installation.xml @@ -22,32 +22,49 @@ under the License. -->
- Linux OS Installation - Use the following steps to begin the Linux OS installation: - - Download the script file cloud-set-guest-password: - - Linux: - - Windows: - - - - Copy this file to /etc/init.d.On some Linux distributions, copy the file to /etc/rc.d/init.d. - - Run the following command to make the script executable:chmod +x /etc/init.d/cloud-set-guest-password - - Depending on the Linux distribution, continue with the appropriate step.On Fedora, CentOS/RHEL, and Debian, run:chkconfig --add cloud-set-guest-password - On Ubuntu with VMware tools, link the script file to the /etc/network/if-up and - /etc/network/if-down folders, and run the script: - #ln -s /etc/init.d/cloud-set-guest-password /etc/network/if-up/cloud-set-guest-password + Linux OS Installation + Use the following steps to begin the Linux OS installation: + + + Download the script file cloud-set-guest-password: + + + Linux: + + + + Windows: + + + + + + Copy this file to /etc/init.d. + On some Linux distributions, copy the file to + /etc/rc.d/init.d. + + + Run the following command to make the script executable: + chmod +x /etc/init.d/cloud-set-guest-password + + + Depending on the Linux distribution, continue with the appropriate step. + On Fedora, CentOS/RHEL, and Debian, run: + chkconfig --add cloud-set-guest-password + On Ubuntu with VMware tools, link the script file to the + /etc/network/if-up and /etc/network/if-down + folders, and run the script: + #ln -s /etc/init.d/cloud-set-guest-password /etc/network/if-up/cloud-set-guest-password #ln -s /etc/init.d/cloud-set-guest-password /etc/network/if-down/cloud-set-guest-password - If you are using Ubuntu 11.04, start by creating a directory called /var/lib/dhcp3 on your Ubuntu machine (works around a known issue with this version of Ubuntu). On all Ubuntu versions: Run “sudo update-rc.d cloud-set-guest-password defaults 98â€. To test, run "mkpasswd" and check that it is generating a new password. If the “mkpasswd†command does not exist, run "sudo apt-get install whois" (or sudo apt-get install mkpasswd, depending on your Ubuntu version) and repeat. - - - -
+ If you are using Ubuntu 11.04, start by creating a directory + called /var/lib/dhcp3 on your Ubuntu machine (works around a known issue with this version + of Ubuntu). On all Ubuntu versions: Run “sudo update-rc.d cloud-set-guest-password defaults + 98â€. To test, run "mkpasswd" and check that it is generating a new password. If the + “mkpasswd†command does not exist, run "sudo apt-get install whois" (or sudo apt-get install + mkpasswd, depending on your Ubuntu version) and repeat. + + +
diff --git a/docs/en-US/load-balancer-rules.xml b/docs/en-US/load-balancer-rules.xml index 77739001966..884647c6f8b 100644 --- a/docs/en-US/load-balancer-rules.xml +++ b/docs/en-US/load-balancer-rules.xml @@ -37,4 +37,5 @@ +
diff --git a/docs/en-US/manual-live-migration.xml b/docs/en-US/manual-live-migration.xml index 225f0ba3317..1daa6d3d937 100644 --- a/docs/en-US/manual-live-migration.xml +++ b/docs/en-US/manual-live-migration.xml @@ -26,10 +26,12 @@ The &PRODUCT; administrator can move a running VM from one host to another without interrupting service to users or going into maintenance mode. This is called manual live migration, and can be done under the following conditions: The root administrator is logged in. Domain admins and users can not perform manual live migration of VMs. - The VM is running. Stopped VMs can not be live migrated. - The destination host must be in the same cluster as the original host. - The VM must not be using local disk storage. + The VM is running. Stopped VMs can not be live migrated. The destination host must have enough available capacity. If not, the VM will remain in the "migrating" state until memory becomes available. + (KVM) The VM must not be using local disk storage. (On XenServer and VMware, VM live migration + with local disk is enabled by &PRODUCT; support for XenMotion and vMotion.) + (KVM) The destination host must be in the same cluster as the original host. + (On XenServer and VMware, VM live migration from one cluster to another is enabled by &PRODUCT; support for XenMotion and vMotion.) To manually live migrate a virtual machine @@ -44,7 +46,10 @@ Migrateinstance.png: button to migrate an instance - From the list of hosts, choose the one to which you want to move the VM. + From the list of suitable hosts, choose the one to which you want to move the VM. + If the VM's storage has to be migrated along with the VM, this will be noted in the host + list. &PRODUCT; will take care of the storage migration for you. + Click OK.
diff --git a/docs/en-US/migrate-datadisk-volume-new-storage-pool.xml b/docs/en-US/migrate-datadisk-volume-new-storage-pool.xml index 552fb319341..1ed6bbd7cd3 100644 --- a/docs/en-US/migrate-datadisk-volume-new-storage-pool.xml +++ b/docs/en-US/migrate-datadisk-volume-new-storage-pool.xml @@ -23,13 +23,56 @@ -->
- Migrating a Data Disk Volume to a New Storage Pool - - Log in to the &PRODUCT; UI as a user or admin. - Detach the data disk from the VM. See Detaching and Moving Volumes (but skip the “reattach†step at the end. You will do that after migrating to new storage). - Call the &PRODUCT; API command migrateVolume and pass in the volume ID and the ID of any storage pool in the zone. - Watch for the volume status to change to Migrating, then back to Ready. - Attach the volume to any desired VM running in the same cluster as the new storage server. See Attaching a Volume - + Migrating a Data Volume to a New Storage Pool + There are two situations when you might want to migrate a disk: + + Move the disk to new storage, but leave it attached to the same running VM. + Detach the disk from its current VM, move it to new storage, and attach it to a new VM. + +
+ Migrating Storage For a Running VM + (Supported on XenServer and VMware) + + Log in to the &PRODUCT; UI as a user or admin. + In the left navigation bar, click Instances, click the VM name, and click View Volumes. + Click the volume you want to migrate. + Detach the disk from the VM. + See but skip the “reattach†step at the end. You + will do that after migrating to new storage. + Click the Migrate Volume button + + + + + Migrateinstance.png: button to migrate a volume + + + and choose the destination from the dropdown list. + Watch for the volume status to change to Migrating, then back to Ready. +
- +
+ Migrating Storage and Attaching to a Different VM + + Log in to the &PRODUCT; UI as a user or admin. + Detach the disk from the VM. + See but skip the “reattach†step at the end. You + will do that after migrating to new storage. + Click the Migrate Volume button + + + + + Migrateinstance.png: button to migrate a volume + + + and choose the destination from the dropdown list. + Watch for the volume status to change to Migrating, then back to Ready. You can find the + volume by clicking Storage in the left navigation bar. Make sure that Volumes is + displayed at the top of the window, in the Select View dropdown. + Attach the volume to any desired VM running in the same cluster as the new storage server. See + + + +
+
diff --git a/docs/en-US/migrate-vm-rootvolume-volume-new-storage-pool.xml b/docs/en-US/migrate-vm-rootvolume-volume-new-storage-pool.xml index d615cfe7a5b..3bcaff53c63 100644 --- a/docs/en-US/migrate-vm-rootvolume-volume-new-storage-pool.xml +++ b/docs/en-US/migrate-vm-rootvolume-volume-new-storage-pool.xml @@ -23,15 +23,25 @@ -->
Migrating a VM Root Volume to a New Storage Pool - When migrating the root disk volume, the VM must first be stopped, and users can not access the VM. After migration is complete, the VM can be restarted. - - Log in to the &PRODUCT; UI as a user or admin. - Detach the data disk from the VM. See Detaching and Moving Volumes (but skip the “reattach†step at the end. You will do that after migrating to new storage). - Stop the VM. - Use the &PRODUCT; API command, migrateVirtualMachine, with the ID of the VM to migrate and - the IDs of a destination host and destination storage pool in the same zone. - Watch for the VM status to change to Migrating, then back to Stopped. - Restart the VM. - -
- + (XenServer, VMware) You can live migrate a VM's root disk from one storage pool to another, without stopping the VM first. + (KVM) When migrating the root disk volume, the VM must first be stopped, and users can not access the VM. After migration is complete, the VM can be restarted. + + Log in to the &PRODUCT; UI as a user or admin. + In the left navigation bar, click Instances, and click the VM name. + (KVM only) Stop the VM. + Click the Migrate button + + + + + Migrateinstance.png: button to migrate a VM or volume + + + and choose the destination from the dropdown list. + If the VM's storage has to be migrated along with the VM, this will be noted in the host + list. &PRODUCT; will take care of the storage migration for you. + Watch for the volume status to change to Migrating, then back to Running (or Stopped, in the case of KVM). This + can take some time. + (KVM only) Restart the VM. + + \ No newline at end of file diff --git a/docs/en-US/multiple-ip-nic.xml b/docs/en-US/multiple-ip-nic.xml index 790befcc081..344dc8df16f 100644 --- a/docs/en-US/multiple-ip-nic.xml +++ b/docs/en-US/multiple-ip-nic.xml @@ -75,9 +75,9 @@ Click Acquire New Secondary IP, and click Yes in the confirmation dialog. - You need to specify the secondary IP address on the guest VM. &PRODUCT; will not - automatically configure the acquired IP address on the VM. Ensure that you assign IPs to - NIC each time the VM reboots. + You need to configure the IP on the guest VM NIC manually. &PRODUCT; will not + automatically configure the acquired IP address on the VM. Ensure that the IP address + configuration persist on VM reboot. Within a few moments, the new IP address should appear with the state Allocated. You can now use the IP address in Port Forwarding or StaticNAT rules. diff --git a/docs/en-US/networks.xml b/docs/en-US/networks.xml index 5d0c82ab8d6..b28f985a147 100644 --- a/docs/en-US/networks.xml +++ b/docs/en-US/networks.xml @@ -32,8 +32,13 @@ xmlns:xi="http://www.w3.org/2001/XInclude"/> + + + + + @@ -47,10 +52,6 @@ - - - - diff --git a/docs/en-US/non-contiguous-vlan.xml b/docs/en-US/non-contiguous-vlan.xml index 79fac835c7e..193b91697c3 100644 --- a/docs/en-US/non-contiguous-vlan.xml +++ b/docs/en-US/non-contiguous-vlan.xml @@ -20,49 +20,48 @@ under the License. -->
- Non Contiguous VLAN Ranges + Adding Non Contiguous VLAN Ranges &PRODUCT; provides you with the flexibility to add non contiguous VLAN ranges to your network. The administrator can either update an existing VLAN range or add multiple non contiguous VLAN ranges while creating a zone. You can also use the UpdatephysicalNetwork API to extend the VLAN range. -
- Adding a New VLAN Range - - Log in to the CloudPlatform UI as an administrator or end user. - - Ensure that the VLAN range does not already exist. - - - Check whether the new VLAN range overlaps with any existing ones. If overlaps, extend - the existing range. If does not overlap, add the new range. - - - In the left navigation, choose Infrastructure. On Zones, click View More, then click the zone to - which you want to work with. - - - Click Physical Network. - - - In the Guest node of the diagram, click Configure. - - - Click Add VLAN Ranges button - - - - - add-vlan-ico.png: button to add a VLAN range. - - - The Add VLAN Ranges dialog is displayed. - - - Specify the start and end of the VLAN range. - - - Click OK. - - -
+ + + Log in to the &PRODUCT; UI as an administrator or end user. + + + Ensure that the VLAN range does not already exist. + + + In the left navigation, choose Infrastructure. + + + On Zones, click View More, then click the zone to which you want to work with. + + + Click Physical Network. + + + In the Guest node of the diagram, click Configure. + + + Click Edit + + + + + edit-icon.png: button to edit the VLAN range. + + + The VLAN Ranges field now is editable. + + + Specify the start and end of the VLAN range in comma-separated list. + Specify all the VLANs you want to use, VLANs not specified will be removed if you are + adding new ranges to the existing list. + + + Click Apply. + +
diff --git a/docs/en-US/over-provisioning-service-offering-limits.xml b/docs/en-US/over-provisioning-service-offering-limits.xml index 64a162745e5..e2b3b3db54c 100644 --- a/docs/en-US/over-provisioning-service-offering-limits.xml +++ b/docs/en-US/over-provisioning-service-offering-limits.xml @@ -24,8 +24,138 @@
Over-Provisioning and Service Offering Limits - &PRODUCT; performs CPU over-provisioning based on an over-provisioning ratio configured by the administrator. This is defined by the cpu.overprovisioning.factor global configuration variable. - &PRODUCT; performs CPU over-provisioning based on an over-provisioning ratio configured by the administrator. This is defined by the cpu.overprovisioning.factor global configuration variable - Service offerings limits (e.g. 1 GHz, 1 core) are strictly enforced for core count. For example, a guest with a service offering of one core will have only one core available to it regardless of other activity on the Host. + (Supported for XenServer, KVM, and VMware) + CPU and memory (RAM) over-provisioning factors can be set for each cluster to change the + number of VMs that can run on each host in the cluster. This helps optimize the use of + resources. By increasing the over-provisioning ratio, more resource capacity will be used. If + the ratio is set to 1, no over-provisioning is done. + The administrator can also set global default over-provisioning ratios + in the cpu.overprovisioning.factor and mem.overprovisioning.factor global configuration variables. + The default value of these variables is 1: over-provisioning is turned off by default. + + Over-provisioning ratios are dynamically substituted in &PRODUCT;'s capacity + calculations. For example: + Capacity = 2 GB + Over-provisioning factor = 2 + Capacity after over-provisioning = 4 GB + With this configuration, suppose you deploy 3 VMs of 1 GB each: + Used = 3 GB + Free = 1 GB + The administrator can specify a memory over-provisioning ratio, and can specify both CPU and + memory over-provisioning ratios on a per-cluster basis. + In any given cloud, the optimum number of VMs for each host is affected by such things as + the hypervisor, storage, and hardware configuration. These may be different for each cluster in + the same cloud. A single global over-provisioning setting can not provide the best utilization + for all the different clusters in the cloud. It has to be set for the lowest common denominator. + The per-cluster setting provides a finer granularity for better utilization of resources, no + matter where the &PRODUCT; placement algorithm decides to place a VM. + The overprovisioning settings can be used along with dedicated resources (assigning a + specific cluster to an account) to effectively offer different levels of service to + different accounts. For example, an account paying for a more expensive level of service + could be assigned to a dedicated cluster with an over-provisioning ratio of 1, and a + lower-paying account to a cluster with a ratio of 2. + When a new host is added to a cluster, &PRODUCT; will assume the host has the + capability to perform the CPU and RAM over-provisioning which is configured for that + cluster. It is up to the administrator to be sure the host is actually suitable for the + level of over-provisioning which has been set. +
+ Limitations on Over-Provisioning in XenServer and KVM + + In XenServer, due to a constraint of this hypervisor, you can not use an + over-provisioning factor greater than 4. + The KVM hypervisor can not manage memory allocation to VMs dynamically. + &PRODUCT; sets the minimum and maximum amount of memory that a VM can use. + The hypervisor adjusts the memory within the set limits based on the memory contention. + +
+
+ Requirements for Over-Provisioning + Several prerequisites are required in order for over-provisioning to function + properly. The feature is dependent on the OS type, hypervisor capabilities, and certain + scripts. It is the administrator's responsibility to ensure that these requirements are + met. +
+ Balloon Driver + All VMs should have a balloon driver installed in them. The hypervisor + communicates with the balloon driver to free up and make the memory available to a + VM. + + XenServer + The balloon driver can be found as a part of xen pv or PVHVM drivers. The xen + pvhvm drivers are included in upstream linux kernels 2.6.36+. + + + VMware + The balloon driver can be found as a part of the VMware tools. All the VMs that + are deployed in a over-provisioned cluster should have the VMware tools + installed. + + + KVM + All VMs are required to support the virtio drivers. These drivers are installed + in all Linux kernel versions 2.6.25 and greater. The administrator must set + CONFIG_VIRTIO_BALLOON=y in the virtio configuration. + +
+
+ Hypervisor capabilities + The hypervisor must be capable of using the memory ballooning. + + XenServer + The DMC (Dynamic Memory Control) capability of the hypervisor should be enabled. + Only XenServer Advanced and above versions have this feature. + + + VMware, KVM + Memory ballooning is supported by default. + +
+
+
+ Setting Over-Provisioning Ratios + There are two ways the root admin can set CPU and RAM over-provisioning ratios. First, the + global configuration settings cpu.overprovisioning.factor and mem.overprovisioning.factor will + be applied when a new cluster is created. Later, the ratios can be modified for an existing + cluster. + Only VMs deployed after the change are affected by the new setting. + If you want VMs deployed before the change to adopt the new over-provisioning ratio, + you must stop and restart the VMs. + When this is done, &PRODUCT; recalculates or scales the used and + reserved capacities based on the new over-provisioning ratios, + to ensure that &PRODUCT; is correctly tracking the amount of free capacity. + It is safer not to deploy additional new VMs while the capacity recalculation is underway, in + case the new values for available capacity are not high enough to accommodate the new VMs. + Just wait for the new used/available values to become available, to be sure there is room + for all the new VMs you want. + To change the over-provisioning ratios for an existing cluster: + + + Log in as administrator to the &PRODUCT; UI. + + + In the left navigation bar, click Infrastructure. + + + Under Clusters, click View All. + + + Select the cluster you want to work with, and click the Edit button. + + + Fill in your desired over-provisioning multipliers in the fields CPU overcommit + ratio and RAM overcommit ratio. The value which is intially shown in these + fields is the default value inherited from the global configuration settings. + + + In XenServer, due to a constraint of this hypervisor, you can not use an + over-provisioning factor greater than 4. + + + +
+
+ Service Offering Limits and Over-Provisioning + Service offering limits (e.g. 1 GHz, 1 core) are strictly enforced for core count. For example, a guest with a service offering of one core will have only one core available to it regardless of other activity on the Host. Service offering limits for gigahertz are enforced only in the presence of contention for CPU resources. For example, suppose that a guest was created with a service offering of 1 GHz on a Host that has 2 GHz cores, and that guest is the only guest running on the Host. The guest will have the full 2 GHz available to it. When multiple guests are attempting to use the CPU a weighting factor is used to schedule CPU resources. The weight is based on the clock speed in the service offering. Guests receive a CPU allocation that is proportionate to the GHz in the service offering. For example, a guest created from a 2 GHz service offering will receive twice the CPU allocation as a guest created from a 1 GHz service offering. &PRODUCT; does not perform memory over-provisioning. -
+
+ \ No newline at end of file diff --git a/docs/en-US/password-storage-engine.xml b/docs/en-US/password-storage-engine.xml index 05661055e9b..8bbc96fcac2 100644 --- a/docs/en-US/password-storage-engine.xml +++ b/docs/en-US/password-storage-engine.xml @@ -22,11 +22,13 @@
Changing the Default Password Encryption Passwords are encoded when creating or updating users. &PRODUCT; allows you to determine the - default encoding and authentication mechanism for admin and user logins. A new configurable list - called UserPasswordEncoders to allow you to separately configure the order of - preference for encoding and authentication schemes. - Additionally, plain text user authenticator has been changed to use SHA256SALT as the - default encoding algorithm because it is more secure compared to MD5 hashing. It does a simple + default encoding and authentication mechanism for admin and user logins. Two new configurable + lists have been introduced—userPasswordEncoders and userAuthenticators. + userPasswordEncoders allows you to configure the order of preference for encoding passwords, + whereas userAuthenticators allows you to configure the order in which authentication schemes are + invoked to validate user passwords. + Additionally, the plain text user authenticator has been modified not to convert supplied + passwords to their md5 sums before checking them with the database entries. It performs a simple string comparison between retrieved and supplied login passwords instead of comparing the retrieved md5 hash of the stored password against the supplied md5 hash of the password because clients no longer hash the password. The following method determines what encoding scheme is @@ -35,11 +37,15 @@ loaded as per the sequence specified in the UserPasswordEncoders property in the ComponentContext.xml or nonossComponentContext.xml files. The order of authentication schemes is determined by the UserAuthenticators - property in the same files. When a new authenticator or encoder is added, you can add them to - this list. While doing so, ensure that the new authenticator or encoder is specified as a bean - in both these files. The administrator can change the ordering of both these properties as - preferred to change the order of schemes. Modify the following list properties available in - client/tomcatconf/nonossComponentContext.xml.in or + property in the same files. If Non-OSS components, such as VMware environments, are to be + deployed, modify the UserPasswordEncoders and UserAuthenticators lists + in the nonossComponentContext.xml file, for OSS environments, such as + XenServer or KVM, modify the ComponentContext.xml file. It is recommended + to make uniform changes across both the files. When a new authenticator or encoder is added, you + can add them to this list. While doing so, ensure that the new authenticator or encoder is + specified as a bean in both these files. The administrator can change the ordering of both these + properties as preferred to change the order of schemes. Modify the following list properties + available in client/tomcatconf/nonossComponentContext.xml.in or client/tomcatconf/componentContext.xml.in as applicable, to the desired order: <property name="UserAuthenticators"> @@ -62,7 +68,7 @@ the encoded password is stored in the user table's password column. If it fails for any reason, the MD5UserAuthenticator will be tried next, and the order continues. For UserAuthenticators, SHA256Salt authentication is tried first. If it succeeds, the - user is logged into the Management server. If it fails, MD5 is tried next, and attempts - continues until any of them succeeds and the user logs in . If none of them works, the user is + user is logged into the Management server. If it fails, md5 is tried next, and attempts + continues until any of them succeeds and the user logs in . If none of them works, the user is returned an invalid credential message.
diff --git a/docs/en-US/portable-ip.xml b/docs/en-US/portable-ip.xml index 83d5b43b206..f9ae395de20 100644 --- a/docs/en-US/portable-ip.xml +++ b/docs/en-US/portable-ip.xml @@ -24,10 +24,10 @@ About Portable IP Portable IPs in &PRODUCT; are region-level pool of IPs, which are elastic in nature, that can be transferred across geographically separated zones. As an administrator, you can - provision a pool of portable IPs at region level and are available for user consumption. The - users can acquire portable IPs if admin has provisioned portable public IPs at the region - level they are part of. These IPs can be use for any service within an advanced zone. You can - also use portable IPs for EIP services in basic zones. + provision a pool of portable public IPs at region level and are available for user + consumption. The users can acquire portable IPs if admin has provisioned portable IPs at the + region level they are part of. These IPs can be use for any service within an advanced zone. + You can also use portable IPs for EIP services in basic zones. The salient features of Portable IP are as follows: IP is statically allocated @@ -36,25 +36,26 @@ IP need not be associated with a network - Network association is transferable across networks + IP association is transferable across networks IP is transferable across both Basic and Advanced zones - IP is transferable across VPC, non-VPC Isolated and Shared networks + IP is transferable across VPC, non-VPC isolated and shared networks - - - - + Portable IP transfer is available only for static NAT. + + Guidelines + Before transferring to another network, ensure that no network rules (Firewall, Static + NAT, Port Forwarding, and so on) exist on that portable IP. +
Configuring Portable IPs - Log in to the &PRODUCT; UI as an administrator or end user. @@ -129,4 +130,16 @@
+
+ Transferring Portable IP + An IP can be transferred from one network to another only if Static NAT is enabled. + However, when a portable IP is associated with a network, you can use it for any service in + the network. + To transfer a portable IP across the networks, execute the following API: + http://localhost:8096/client/api?command=enableStaticNat&response=json&ipaddressid=a4bc37b2-4b4e-461d-9a62-b66414618e36&virtualmachineid=a242c476-ef37-441e-9c7b-b303e2a9cb4f&networkid=6e7cd8d1-d1ba-4c35-bdaf-333354cbd49810 + Replace the UUID with appropriate UUID. For example, if you want to transfer a portable IP + to network X and VM Y in a network, execute the following: + http://localhost:8096/client/api?command=enableStaticNat&response=json&ipaddressid=a4bc37b2-4b4e-461d-9a62-b66414618e36&virtualmachineid=Y&networkid=X + +
diff --git a/docs/en-US/primary-storage-add.xml b/docs/en-US/primary-storage-add.xml index 067cf7114dc..d18dece54d9 100644 --- a/docs/en-US/primary-storage-add.xml +++ b/docs/en-US/primary-storage-add.xml @@ -39,9 +39,10 @@
- Adding Primary Stroage + Adding Primary Storage When you create a new zone, the first primary storage is added as part of that procedure. You can add primary storage servers at any time, such as when adding a new cluster or adding more servers to an existing cluster. - Be sure there is nothing stored on the server. Adding the server to &PRODUCT; will destroy any existing data. + When using preallocated storage for primary storage, be sure there is nothing on the storage (ex. you have an empty SAN volume or an empty NFS share). Adding the storage to &PRODUCT; will destroy any existing data. + Primary storage can also be added at the zone level through the &PRODUCT; API (adding zone-level primary storage is not yet supported through the &PRODUCT; UI).Once primary storage has been added at the zone level, it can be managed through the &PRODUCT; UI. Log in to the &PRODUCT; UI (see ). In the left navigation, choose Infrastructure. In Zones, click View More, then click the zone in which you want to add the primary storage. @@ -51,8 +52,9 @@ Provide the following information in the dialog. The information required varies depending on your choice in Protocol. - Pod. The pod for the storage device. - Cluster. The cluster for the storage device. + Scope. Indicate whether the storage is available to all hosts in the zone or only to hosts in a single cluster. + Pod. (Visible only if you choose Cluster in the Scope field.) The pod for the storage device. + Cluster. (Visible only if you choose Cluster in the Scope field.) The cluster for the storage device. Name. The name of the storage device. Protocol. For XenServer, choose either NFS, iSCSI, or PreSetup. For KVM, choose NFS or SharedMountPoint. For vSphere choose either VMFS (iSCSI or FiberChannel) or NFS. Server (for NFS, iSCSI, or PreSetup). The IP address or DNS name of the storage device. @@ -69,6 +71,93 @@ Click OK. - +
+
+ Configuring a Storage Plug-in + + Primary storage that is based on a custom plug-in (ex. SolidFire) must be added through the &PRODUCT; API (described later in this section). There is no support at this time through the &PRODUCT; UI to add this type of primary storage (although most of its features are available through the &PRODUCT; UI). + + + At this time, a custom storage plug-in, such as the SolidFire storage plug-in, can only be leveraged for data disks (through Disk Offerings). + + + The SolidFire storage plug-in for &PRODUCT; is part of the standard &PRODUCT; install. There is no additional work required to add this component. + + Adding primary storage that is based on the SolidFire plug-in enables &PRODUCT; to provide hard quality-of-service (QoS) guarantees. + When used with Disk Offerings, an administrator is able to build an environment in which a data disk that a user creates leads to the dynamic creation of a SolidFire volume, which has guaranteed performance. Such a SolidFire volume is associated with one (and only ever one) &PRODUCT; volume, so performance of the &PRODUCT; volume does not vary depending on how heavily other tenants are using the system. + The createStoragePool API has been augmented to support plugable storage providers. The following is a list of parameters to use when adding storage to &PRODUCT; that is based on the SolidFire plug-in: + + + command=createStoragePool + + + scope=zone + + + zoneId=[your zone id] + + + name=[name for primary storage] + + + hypervisor=Any + + + provider=SolidFire + + + capacityIops=[whole number of IOPS from the SAN to give to &PRODUCT;] + + + capacityBytes=[whole number of bytes from the SAN to give to &PRODUCT;] + + + The url parameter is somewhat unique in that its value can contain additional key/value pairs. + + url=[key/value pairs detailed below (values are URL encoded; for example, '=' is represented as '%3D')] + + MVIP%3D[Management Virtual IP Address] (can be suffixed with :[port number]) + + + SVIP%3D[Storage Virtual IP Address] (can be suffixed with :[port number]) + + + clusterAdminUsername%3D[cluster admin's username] + + + clusterAdminPassword%3D[cluster admin's password] + + + clusterDefaultMinIops%3D[Min IOPS (whole number) to set for a volume; used if Min IOPS is not specified by administrator or user] + + + clusterDefaultMaxIops%3D[Max IOPS (whole number) to set for a volume; used if Max IOPS is not specified by administrator or user] + + + clusterDefaultBurstIopsPercentOfMaxIops%3D[Burst IOPS is determined by (Min IOPS * clusterDefaultBurstIopsPercentOfMaxIops parameter) (can be a decimal value)] + + + + Example URL to add primary storage to &PRODUCT; based on the SolidFire plug-in (note that URL encoding is used with the value of the url key, so '%3A' equals ':','%3B' equals '&' and '%3D' equals '='): + + http://127.0.0.1:8080/client/api?command=createStoragePool + &scope=zone + &zoneId=cf4e6ddf-8ae7-4194-8270-d46733a52b55 + &name=SolidFire_121258566 + &url=MVIP%3D192.168.138.180%3A443 + %3BSVIP%3D192.168.56.7 + %3BclusterAdminUsername%3Dadmin + %3BclusterAdminPassword%3Dpassword + %3BclusterDefaultMinIops%3D200 + %3BclusterDefaultMaxIops%3D300 + %3BclusterDefaultBurstIopsPercentOfMaxIop%3D2.5 + &provider=SolidFire + &tags=SolidFire_SAN_1 + &capacityIops=4000000 + &capacityBytes=2251799813685248 + &hypervisor=Any + &response=json + &apiKey=VrrkiZQWFFgSdA6k3DYtoKLcrgQJjZXoSWzicHXt8rYd9Bl47p8L39p0p8vfDpiljtlcMLn_jatMSqCWv5Cs-Q&signature=wqf8KzcPpY2JmT1Sxk%2F%2BWbgX3l8%3D +
diff --git a/docs/en-US/pvlan.xml b/docs/en-US/pvlan.xml index d569507f973..38b25319faf 100644 --- a/docs/en-US/pvlan.xml +++ b/docs/en-US/pvlan.xml @@ -113,19 +113,19 @@ ports. Configure the switch port connected to the router in PVLAN promiscuous trunk mode, which would translate an isolated VLAN to primary VLAN for the PVLAN-unaware router. Note that only Cisco Catalyst 4500 has the PVLAN promiscuous trunk mode to connect - both normal VLAN and PVLAN to a PVLAN-unaware switch. For other Catalyst PVLAN support - switch, connect the switch to upper switch by using cables. The number of cables should be - greater than the number of PVLANs used. + both normal VLAN and PVLAN to a PVLAN-unaware switch. For the other Catalyst PVLAN support + switch, connect the switch to upper switch by using cables, one each for a PVLAN + pair. Configure private VLAN on your physical switches out-of-band. - Before you use PVLAN on XenServer and KVM, enable Open vSwitch (OVS) . + Before you use PVLAN on XenServer and KVM, enable Open vSwitch (OVS). - OVS on XenServer and KVM does not support PVLAN. Therefore, simulate PVLAN on OVS - for XenServer and KVM by modifying the flow table and tagging every traffic leaving - guest VMs with the secondary VLAN ID. + OVS on XenServer and KVM does not support PVLAN natively. Therefore, &PRODUCT; + managed to simulate PVLAN on OVS for XenServer and KVM by modifying the flow + table. @@ -134,7 +134,7 @@ Creating a PVLAN-Enabled Guest Network - Log in to the CloudPlatform UI as administrator. + Log in to the &PRODUCT; UI as administrator. In the left navigation, choose Infrastructure. @@ -176,8 +176,8 @@ VLAN ID: The unique ID of the VLAN. - Isolated VLAN ID: The unique ID of the Secondary - Isolated VLAN. + Secondary Isolated VLAN ID: The unique ID of the + Secondary Isolated VLAN. For the description on Secondary Isolated VLAN, see . @@ -223,15 +223,15 @@ IP Range: A range of IP addresses that are accessible from the Internet and are assigned to the guest VMs. - If one NIC is used, these IPs should be in the same CIDR in the case of - IPv6. + - + Network Domain: A custom DNS suffix at the level of a network. If you want to assign a special domain name to the guest VM network, diff --git a/docs/en-US/region-add.xml b/docs/en-US/region-add.xml index 802e462ce16..212047ad89b 100644 --- a/docs/en-US/region-add.xml +++ b/docs/en-US/region-add.xml @@ -30,9 +30,8 @@ The First Region: The Default Region If you do not take action to define regions, then all the zones in your cloud will be automatically grouped into a single default region. This region is assigned the region - ID of 1. - You can change the name or URL of the default region by using the API command updateRegion. For example: - http://<IP_of_Management_Server>:8080/client/api?command=updateRegion&id=1&name=Northern&endpoint=http://<region_1_IP_address_here>:8080/client&apiKey=miVr6X7u6bN_sdahOBpjNejPgEsT35eXq-jB8CG20YI3yaxXcgpyuaIRmFI_EJTVwZ0nUkkJbPmY3y2bciKwFQ&signature=Lxx1DM40AjcXU%2FcaiK8RAP0O1hU%3D + ID of 1. You can change the name or URL of the default region by displaying the region in + the &PRODUCT; UI and clicking the Edit button.
Adding a Region @@ -43,17 +42,29 @@ geographic area where you want to set up the new region. Use the steps in the Installation guide. When you come to the step where you set up the database, use the additional command-line flag -r <region_id> to set a - region ID for the new region. The default region is automatically assigned a - region ID of 1, so your first additional region might be region 2. - cloudstack-setup-databases cloud:<dbpassword>@localhost --deploy-as=root:<password> -e <encryption_type> -m <management_server_key> -k <database_key> -r <region_id> + region ID for the new region. The default region is automatically assigned a + region ID of 1, so your first additional region might be region 2. + cloudstack-setup-databases cloud:<dbpassword>@localhost --deploy-as=root:<password> -e <encryption_type> -m <management_server_key> -k <database_key> -r <region_id> By the end of the installation procedure, the Management Server should have been started. Be sure that the Management Server installation was successful and complete. - Add region 2 to region 1. Use the API command addRegion. (For information about how to make an API call, see the Developer's Guide.) - http://<IP_of_region_1_Management_Server>:8080/client/api?command=addRegion&id=2&name=Western&endpoint=http://<region_2_IP_address_here>:8080/client&apiKey=miVr6X7u6bN_sdahOBpjNejPgEsT35eXq-jB8CG20YI3yaxXcgpyuaIRmFI_EJTVwZ0nUkkJbPmY3y2bciKwFQ&signature=Lxx1DM40AjcXU%2FcaiK8RAP0O1hU%3D - - Now perform the same command in reverse, adding region 1 to region 2. - http://<IP_of_region_2_Management_Server>:8080/client/api?command=addRegion&id=1&name=Northern&endpoint=http://<region_1_IP_address_here>:8080/client&apiKey=miVr6X7u6bN_sdahOBpjNejPgEsT35eXq-jB8CG20YI3yaxXcgpyuaIRmFI_EJTVwZ0nUkkJbPmY3y2bciKwFQ&signature=Lxx1DM40AjcXU%2FcaiK8RAP0O1hU%3D + Now add the new region to region 1 in &PRODUCT;. + + Log in to &PRODUCT; in the first region as root administrator + (that is, log in to <region.1.IP.address>:8080/client). + In the left navigation bar, click Regions. + Click Add Region. In the dialog, fill in the following fields: + + ID. A unique identifying number. Use the same number + you set in the database during Management Server installation in the new region; + for example, 2. + Name. Give the new region a descriptive name. + Endpoint. The URL where you can log in to the Management Server in the new region. + This has the format <region.2.IP.address>:8080/client. + + + + Now perform the same procedure in reverse. Log in to region 2, and add region 1. Copy the account, user, and domain tables from the region 1 database to the region 2 database. In the following commands, it is assumed that you have set the root password on the database, which is a &PRODUCT; recommended best practice. Substitute your own MySQL @@ -84,16 +95,23 @@ Install &PRODUCT; in each additional region. Set the region ID for each region during the database setup step. cloudstack-setup-databases cloud:<dbpassword>@localhost --deploy-as=root:<password> -e <encryption_type> -m <management_server_key> -k <database_key> -r <region_id> Once the Management Server is running, add your new region to all existing regions by - repeatedly calling the API command addRegion. For example, if you were adding + repeatedly using the Add Region button in the UI. For example, if you were adding region 3: - http://<IP_of_region_1_Management_Server>:8080/client/api?command=addRegion&id=3&name=Eastern&endpoint=http://<region_3_IP_address_here>:8080/client&apiKey=miVr6X7u6bN_sdahOBpjNejPgEsT35eXq-jB8CG20YI3yaxXcgpyuaIRmFI_EJTVwZ0nUkkJbPmY3y2bciKwFQ&signature=Lxx1DM40AjcXU%2FcaiK8RAP0O1hU%3D - -http://<IP_of_region_2_Management_Server>:8080/client/api?command=addRegion&id=3&name=Eastern&endpoint=http://<region_3_IP_address_here>:8080/client&apiKey=miVr6X7u6bN_sdahOBpjNejPgEsT35eXq-jB8CG20YI3yaxXcgpyuaIRmFI_EJTVwZ0nUkkJbPmY3y2bciKwFQ&signature=Lxx1DM40AjcXU%2FcaiK8RAP0O1hU%3D + + Log in to &PRODUCT; in the first region as root administrator + (that is, log in to <region.1.IP.address>:8080/client), and add a region with ID 3, the name of region 3, and the endpoint <region.3.IP.address>:8080/client. + Log in to &PRODUCT; in the second region as root administrator (that is, log in to <region.2.IP.address>:8080/client), and add a region with ID 3, the name of region 3, and the endpoint <region.3.IP.address>:8080/client. + + Repeat the procedure in reverse to add all existing regions to the new region. For example, for the third region, add the other two existing regions: - http://<IP_of_region_3_Management_Server>:8080/client/api?command=addRegion&id=1&name=Northern&endpoint=http://<region_1_IP_address_here>:8080/client&apiKey=miVr6X7u6bN_sdahOBpjNejPgEsT35eXq-jB8CG20YI3yaxXcgpyuaIRmFI_EJTVwZ0nUkkJbPmY3y2bciKwFQ&signature=Lxx1DM40AjcXU%2FcaiK8RAP0O1hU%3D - -http://<IP_of_region_3_Management_Server>:8080/client/api?command=addRegion&id=2&name=Western&endpoint=http://<region_2_IP_address_here>:8080/client&apiKey=miVr6X7u6bN_sdahOBpjNejPgEsT35eXq-jB8CG20YI3yaxXcgpyuaIRmFI_EJTVwZ0nUkkJbPmY3y2bciKwFQ&signature=Lxx1DM40AjcXU%2FcaiK8RAP0O1hU%3D + + Log in to &PRODUCT; in the third region as root administrator + (that is, log in to <region.3.IP.address>:8080/client). + Add a region with ID 1, the name of region 1, and the endpoint <region.1.IP.address>:8080/client. + Add a region with ID 2, the name of region 2, and the endpoint <region.2.IP.address>:8080/client. + + Copy the account, user, and domain tables from any existing region's database to the new region's database. In the following commands, it is assumed that you have set the root password on the @@ -109,7 +127,7 @@ http://<IP_of_region_3_Management_Server>:8080/client/api?command=addRegio - Remove project accounts. Run these commands on the region 2 database: + Remove project accounts. Run these commands on the region 3 database: mysql> delete from account where type = 5; Set the default zone as null: @@ -120,9 +138,14 @@ http://<IP_of_region_3_Management_Server>:8080/client/api?command=addRegio
Deleting a Region - To delete a region, use the API command removeRegion. Repeat the call to remove the region from all other regions. For example, to remove the 3rd region in a three-region cloud: - http://<IP_of_region_1_Management_Server>:8080/client/api?command=removeRegion&id=3&apiKey=miVr6X7u6bN_sdahOBpjNejPgEsT35eXq-jB8CG20YI3yaxXcgpyuaIRmFI_EJTVwZ0nUkkJbPmY3y2bciKwFQ&signature=Lxx1DM40AjcXU%2FcaiK8RAP0O1hU%3D - -http://<IP_of_region_2_Management_Server>:8080/client/api?command=removeRegion&id=3&apiKey=miVr6X7u6bN_sdahOBpjNejPgEsT35eXq-jB8CG20YI3yaxXcgpyuaIRmFI_EJTVwZ0nUkkJbPmY3y2bciKwFQ&signature=Lxx1DM40AjcXU%2FcaiK8RAP0O1hU%3D + Log in to each of the other regions, navigate to the one you want to delete, and click Remove Region. + For example, to remove the third region in a 3-region cloud: + + Log in to <region.1.IP.address>:8080/client. + In the left navigation bar, click Regions. + Click the name of the region you want to delete. + Click the Remove Region button. + Repeat these steps for <region.2.IP.address>:8080/client. +
diff --git a/docs/en-US/reserved-ip-addresses-non-csvms.xml b/docs/en-US/reserved-ip-addresses-non-csvms.xml index 18ba3ca0e42..0f20b634f11 100644 --- a/docs/en-US/reserved-ip-addresses-non-csvms.xml +++ b/docs/en-US/reserved-ip-addresses-non-csvms.xml @@ -28,8 +28,8 @@ of the IP address space that is primarily provided to the guest network. In an Advanced zone, an IP address range or a CIDR is assigned to a network when the network is defined. The &PRODUCT; virtual router acts as the DHCP server and uses CIDR for assigning IP - addresses to the guest VMs. If you decide to reserve IP ranges for non-&PRODUCT; purposes, you - can specify a part of the IP address range or the CIDR that should only be allocated by the DHCP + addresses to the guest VMs. If you decide to reserve CIDR for non-&PRODUCT; purposes, you can + specify a part of the IP address range or the CIDR that should only be allocated by the DHCP service of the virtual router to the guest VMs created in &PRODUCT;. The remaining IPs in that network are called Reserved IP Range. When IP reservation is configured, the administrator can add additional VMs or physical servers that are not part of &PRODUCT; to the same network and @@ -39,6 +39,9 @@ IP Reservation Considerations Consider the following before you reserve an IP range for non-&PRODUCT; machines: + + IP Reservation is supported only in Isolated networks. + IP Reservation can be applied only when the network is in Implemented state. diff --git a/docs/en-US/reset-volume-on-reboot.xml b/docs/en-US/reset-volume-on-reboot.xml new file mode 100644 index 00000000000..da423ab4c60 --- /dev/null +++ b/docs/en-US/reset-volume-on-reboot.xml @@ -0,0 +1,32 @@ + + +%BOOK_ENTITIES; +]> + + +
+ + Reset VM to New Root Disk on Reboot + You can specify that you want to discard the root disk and create a new one whenever a given + VM is rebooted. This is useful for secure environments that need a fresh start on every boot and + for desktops that should not retain state. The IP address of the VM will not change due to this + operation. + To enable root disk reset on VM reboot: + When creating a new service offering, set the parameter isVolatile to True. VMs created from + this service offering will have their disks reset upon reboot. See . +
diff --git a/docs/en-US/runtime-behavior-of-primary-storage.xml b/docs/en-US/runtime-behavior-of-primary-storage.xml index 479ebce1ce1..5e17a4f77a4 100644 --- a/docs/en-US/runtime-behavior-of-primary-storage.xml +++ b/docs/en-US/runtime-behavior-of-primary-storage.xml @@ -25,6 +25,7 @@
Runtime Behavior of Primary Storage Root volumes are created automatically when a virtual machine is created. Root volumes are deleted when the VM is destroyed. Data volumes can be created and dynamically attached to VMs. Data volumes are not deleted when VMs are destroyed. - Administrators should monitor the capacity of primary storage devices and add additional primary storage as needed. See the Advanced Installation Guide. - Administrators add primary storage to the system by creating a &PRODUCT; storage pool. Each storage pool is associated with a cluster. + Administrators should monitor the capacity of primary storage devices and add additional primary storage as needed. See the Advanced Installation Guide. + Administrators add primary storage to the system by creating a &PRODUCT; storage pool. Each storage pool is associated with a cluster or a zone. + With regards to data disks, when a user executes a Disk Offering to create a data disk, the information is initially written to the CloudStack database only. Upon the first request that the data disk be attached to a VM, CloudStack determines what storage to place the volume on and space is taken from that storage (either from preallocated storage or from a storage system (ex. a SAN), depending on how the primary storage was added to CloudStack).
diff --git a/docs/en-US/secondary-storage-add.xml b/docs/en-US/secondary-storage-add.xml index e1f45cdec66..9dd1e7d9319 100644 --- a/docs/en-US/secondary-storage-add.xml +++ b/docs/en-US/secondary-storage-add.xml @@ -39,10 +39,49 @@ When you create a new zone, the first secondary storage is added as part of that procedure. You can add secondary storage servers at any time to add more servers to an existing zone. Be sure there is nothing stored on the server. Adding the server to &PRODUCT; will destroy any existing data. - If you are going to use Swift for cloud-wide secondary storage, you must add the Swift storage to &PRODUCT; before you add the local zone secondary storage servers. See . - To prepare for local zone secondary storage, you should have created and mounted an NFS share during Management Server installation. See .See Preparing NFS Shares in the Installation Guide. + To prepare for the zone-based Secondary Staging Store, you should have created and mounted an NFS share during Management Server installation. See .See Preparing NFS Shares in the Installation Guide. Make sure you prepared the system VM template during Management Server installation. See .See Prepare the System VM Template in the Installation Guide. - Now that the secondary storage server for per-zone storage is prepared, add it to &PRODUCT;. Secondary storage is added as part of the procedure for adding a new zone. See . + Log in to the &PRODUCT; UI as root administrator. + In the left navigation bar, click Infrastructure. + In Secondary Storage, click View All. + Click Add Secondary Storage. + Fill in the following fields: + + Name. Give the storage a descriptive name. + Provider. Choose S3, Swift, or NFS, then fill in the related fields which appear. + The fields will vary depending on the storage provider; for more information, consult the + provider's documentation (such as the S3 or Swift website). + NFS can be used for zone-based storage, and the others for region-wide storage. + You can use only a single S3 or Swift account per region. + Create NFS Secondary Staging Store. This box must always be checked. + Even if the UI allows you to uncheck this box, do not do so. + This checkbox and the three fields below it must be filled in. + Even when Swift or S3 is used as the secondary storage provider, an NFS + staging storage in each zone is still required. + Zone. The zone where the NFS Secondary Staging Store is to be located. + NFS server. The name of the zone's Secondary Staging Store. + Path. The path to the zone's Secondary Staging Store. + + +
+ Adding an NFS Secondary Staging Store for Each Zone + Every zone must have at least one NFS store provisioned; multiple NFS servers are + allowed per zone. To provision an NFS Staging Store for a zone: + + Log in to the &PRODUCT; UI as root administrator. + In the left navigation bar, click Infrastructure. + In Secondary Storage, click View All. + In Select View, choose Secondary Staging Store. + Click the Add NFS Secondary Staging Store button. + Fill out the dialog box fields, then click OK: + + Zone. The zone where the NFS Secondary Staging Store is to be located. + NFS server. The name of the zone's Secondary Staging Store. + Path. The path to the zone's Secondary Staging Store. + + + +
diff --git a/docs/en-US/shared-networks.xml b/docs/en-US/shared-networks.xml index 65657cef828..b8a86f1f5f9 100644 --- a/docs/en-US/shared-networks.xml +++ b/docs/en-US/shared-networks.xml @@ -24,8 +24,8 @@
Shared Networks A shared network can be accessed by virtual machines that belong to many different accounts. - Network Isolation on shared networks is accomplished using techniques such as security groups - (supported only in basic zones). + Network Isolation on shared networks is accomplished by using techniques such as security + groups, which is supported only in Basic zones in &PRODUCT; 3.0.3 and later versions. Shared Networks are created by the administrator @@ -38,11 +38,15 @@ designated by the administrator - Shared Networks are isolated by security groups + Shared Networks can be isolated by security groups Public Network is a shared network that is not shown to the end users + + Source NAT per zone is not supported in Shared Network when the service provider is + virtual router. However, Source NAT per account is supported. + For information, see .
diff --git a/docs/en-US/site-to-site-vpn.xml b/docs/en-US/site-to-site-vpn.xml index a5899eac4f1..9a41a0adf82 100644 --- a/docs/en-US/site-to-site-vpn.xml +++ b/docs/en-US/site-to-site-vpn.xml @@ -3,6 +3,7 @@ %BOOK_ENTITIES; ]> + + +
+ VMware Volume Snapshot Performance + When you take a snapshot of a data or root volume on VMware, &PRODUCT; uses an + efficient storage technique to improve performance. + A snapshot is not immediately exported from vCenter to a mounted NFS + share and packaged into an OVA file format. This operation would consume time and resources. + Instead, the original file formats (e.g., VMDK) provided by vCenter are + retained. An OVA file will only be created as needed, on demand. To generate the OVA, + &PRODUCT; uses information in a properties file (*.ova.meta) which it stored along with + the original snapshot data. + For upgrading customers: This process applies only to newly created snapshots after upgrade to &PRODUCT; + 4.2. Snapshots that have already been taken and stored in OVA format will continue to + exist in that format, and will continue to work as expected. + +
\ No newline at end of file diff --git a/docs/en-US/stopping-and-starting-vms.xml b/docs/en-US/stopping-and-starting-vms.xml index 1c8bd808394..25c1f494b92 100644 --- a/docs/en-US/stopping-and-starting-vms.xml +++ b/docs/en-US/stopping-and-starting-vms.xml @@ -24,6 +24,6 @@
Stopping and Starting VMs - Once a VM instance is created, you can stop, restart, or delete it as needed. In the &PRODUCT; UI, click Instances, select the VM, and use the Stop, Start, Reboot, and Destroy links. + Once a VM instance is created, you can stop, restart, or delete it as needed. In the &PRODUCT; UI, click Instances, select the VM, and use the Stop, Start, Reboot, and Destroy buttons.
diff --git a/docs/en-US/storage-plugins.xml b/docs/en-US/storage-plugins.xml new file mode 100644 index 00000000000..e6612c199d8 --- /dev/null +++ b/docs/en-US/storage-plugins.xml @@ -0,0 +1,144 @@ + + +%BOOK_ENTITIES; +]> + + + + + Writing a Storage Plugin + This section gives an outline of how to implement a plugin + to integrate a third-party storage provider. + For details and an example, you will need to read the code. + + Example code is available at: + plugins/storage/volume/sample + + + Third party storage providers can integrate with &PRODUCT; to provide + either primary storage or secondary storage. + For example, &PRODUCT; provides plugins for + Amazon Simple Storage Service (S3) or OpenStack + Object Storage (Swift). Additional third party object storages can be integrated with &PRODUCT; + by writing plugin software that uses the object storage plugin framework. + Several new interfaces are available so that + storage providers can develop vendor-specific plugins based on well-defined + contracts that can be seamlessly managed by &PRODUCT;. + Artifacts such as templates, ISOs and snapshots are kept in storage which &PRODUCT; + refers to as secondary storage. To improve scalability and performance, as when a number + of hosts access secondary storage concurrently, object storage can be used for secondary + storage. Object storage can also provide built-in high availability capability. When using + object storage, access to secondary storage data can be made available across multiple + zones in a region. This is a huge benefit, as it is no longer necessary to copy templates, + snapshots etc. across zones as would be needed in an environment + using only zone-based NFS storage. + The user enables a storage plugin through the UI. + A new dialog box choice is offered to select the storage + provider. Depending on the provider you select, additional input fields may appear so that + you can provide the additional details required by that provider, such as a user name and + password for a third-party storage account. + +
+ Overview of How to Write a Storage Plugin + To add a third-party storage option to &PRODUCT;, implement the following interfaces in Java: + + DataStoreDriver + DataStoreLifecycle + DataStoreProvider + In addition to implementing the interfaces, you have to hardcode your plugin's required additional + input fields into the code for the Add Secondary Storage + or Add Primary Storage dialog box. + Place your .jar file in plugins/storage/volume/ or plugins/storage/image/. + Edit /client/tomcatconf/componentContext.xml.in. + Edit client/pom.xml. + +
+
+ Implementing DataStoreDriver + DataStoreDriver contains the code that &PRODUCT; will use to provision the object store, when needed. + You must implement the following methods: + + getTO() + getStoreTO() + createAsync() + deleteAsync() + + The following methods are optional: + + resize() + canCopy() is optional. If you set it to true, then you must implement copyAsync(). + +
+
+ Implementing DataStoreLifecycle + DataStoreLifecycle contains the code to manage the storage operations for ongoing use of the storage. + Several operations are needed, like create, maintenance mode, delete, etc. + You must implement the following methods: + + initialize() + maintain() + cancelMaintain() + deleteDataStore() + Implement one of the attach*() methods depending on what scope you want the storage to have: attachHost(), attachCluster(), or attachZone(). + +
+
+ Implementing DataStoreProvider + DataStoreProvider contains the main code of the data store. + You must implement the following methods: + + getDatastoreLifeCycle() + getDataStoreDriver() + getTypes(). Returns one or more types of storage for which this data store provider can be used. + For secondary object storage, return IMAGE, and for a Secondary Staging Store, return ImageCache. + configure(). First initialize the lifecycle implementation and the driver implementation, + then call registerDriver() to register the new object store provider instance with &PRODUCT;. + getName(). Returns the unique name of your provider; for example, + this can be used to get the name to display in the UI. + + The following methods are optional: + + getHostListener() is optional; it's for monitoring the status of the host. + +
+
+ Place the .jar File in the Right Directory + For a secondary storage plugin, place your .jar file here: + plugins/storage/image/ + For a primary storage plugin, place your .jar file here: + plugins/storage/volume/ +
+
+ Edit Configuration Files + First, edit the following file tell &PRODUCT; to include your .jar file. + Add a line to this file to tell the &PRODUCT; Management Server that it now has a dependency on your code: + client/pom.xml + Place some facts about your code in the following file so &PRODUCT; can run it: + /client/tomcatconf/componentContext.xml.in + In the section “Deployment configurations of various adapters,†add this: + <bean>id=â€some unique ID†class=â€package name of your implementation of DataStoreProviderâ€</bean> + In the section “Storage Providers,†add this: + <property name=â€providersâ€> + <ref local=â€same ID from the bean tag's id attributeâ€> +</property> + +
+ + diff --git a/docs/en-US/third-party-ui-plugin.xml b/docs/en-US/third-party-ui-plugin.xml new file mode 100644 index 00000000000..297fdaa857f --- /dev/null +++ b/docs/en-US/third-party-ui-plugin.xml @@ -0,0 +1,364 @@ + + +%BOOK_ENTITIES; +]> + + + + Third-Party UI Plugin Framework + Using the new third-party plugin framework, you can write and install extensions to + &PRODUCT;. The installed and enabled plugins will appear in the UI alongside the + other features. + The code for the plugin is simply placed in a special directory + within &PRODUCT;’s installed code at any time after &PRODUCT; installation. The new plugin + appears only when it is enabled by the cloud administrator. + + + + + + plugin_intro.jpg: New plugin button in product navbar + + + The left navigation bar of the &PRODUCT; UI has a new Plugins button to help you work with UI plugins. +
+ How to Write a Plugin: Overview + The basic procedure for writing a plugin is: + + + Write the code and create the other files needed. You will need the plugin code + itself (in Javascript), a thumbnail image, the plugin listing, and a CSS file. + + + + + + plugin1.jpg: Write the plugin code + + + All UI plugins have the following set of files: + +-- cloudstack/ + +-- ui/ + +-- plugins/ + +-- csMyFirstPlugin/ + +-- config.js --> Plugin metadata (title, author, vendor URL, etc.) + +-- icon.png --> Icon, shown on side nav bar and plugin listing + (should be square, and ~50x50px) + +-- csMyFirstPlugin.css --> CSS file, loaded automatically when plugin loads + +-- csMyFirstPlugin.js --> Main JS file, containing plugin code + + The same files must also be present at /tomcat/webapps/client/plugins. + + + The &PRODUCT; administrator adds the folder containing your plugin code under the + &PRODUCT; PLUGINS folder. + + + + + + plugin2.jpg: The plugin code is placed in the PLUGINS folder + + + + + The administrator also adds the name of your plugin to the plugin.js file in the + PLUGINS folder. + + + + + + plugin3.jpg: The plugin name is added to plugin.js in the PLUGINS + folder + + + + + The next time the user refreshes the UI in the browser, your plugin will appear in + the left navigation bar. + + + + + + plugin4.jpg: The plugin appears in the UI + + + + +
+
+ How to Write a Plugin: Implementation Details + This section requires an understanding of JavaScript and the &PRODUCT; API. You don't + need knowledge of specific frameworks for this tutorial (jQuery, etc.), since the + &PRODUCT; UI handles the front-end rendering for you. + There is much more to the &PRODUCT; UI framework than can be described here. The UI is + very flexible to handle many use cases, so there are countless options and variations. The + best reference right now is to read the existing code for the main UI, which is in the /ui + folder. Plugins are written in a very similar way to the main UI. + + + Create the directory to hold your plugin. + All plugins are composed of set of required files in the directory + /ui/plugins/pluginID, where pluginID is a short name for your plugin. It's recommended + that you prefix your folder name (for example, bfMyPlugin) to avoid naming conflicts + with other people's plugins. + In this example, the plugin is named csMyFirstPlugin. + $ cd cloudstack/ui/plugins +$ mkdir csMyFirstPlugin +$ ls -l + +total 8 +drwxr-xr-x 2 bgregory staff 68 Feb 11 14:44 csMyFirstPlugin +-rw-r--r-- 1 bgregory staff 101 Feb 11 14:26 plugins.js + + + + Change to your new plugin directory. + $ cd csMyFirstPlugin + + + + Set up the listing. + Add the file config.js, using your favorite editor. + $ vi config.js + Add the following content to config.js. This information will be displayed on the + plugin listing page in the UI: + (function (cloudStack) { + cloudStack.plugins.csMyFirstPlugin.config = { + title: 'My first plugin', + desc: 'Tutorial plugin', + externalLink: 'http://www.cloudstack.org/', + authorName: 'Test Plugin Developer', + authorEmail: 'plugin.developer@example.com' + }; +}(cloudStack)); + + + + Add a new main section. + Add the file csMyFirstPlugin.js, using your favorite editor. + $ vi csMyFirstPlugin.js + Add the following content to csMyFirstPlugin.js: + (function (cloudStack) { + cloudStack.plugins.csMyFirstPlugin = function(plugin) { + plugin.ui.addSection({ + id: 'csMyFirstPlugin', + title: 'My Plugin', + preFilter: function(args) { + return isAdmin(); + }, + show: function() { + return $('<div>').html('Content will go here'); + } + }); + }; +}(cloudStack)); + + + + Register the plugin. + You now have the minimal content needed to run the plugin, so you can activate the + plugin in the UI by adding it to plugins.js. First, edit the file: + $ cd cloudstack/ui/plugins +$ vi plugins.js + + Now add the following to plugins.js: + (function($, cloudStack) { + cloudStack.plugins = [ + 'csMyFirstPlugin' + ]; +}(jQuery, cloudStack)); + + + + Check the plugin in the UI. + First, copy all the plugin code that you have created so far to + /tomcat/webapps/client/plugins. Then refresh the browser and click Plugins in the side + navigation bar. You should see your new plugin. + + + Make the plugin do something. + Right now, you just have placeholder content in the new plugin. It's time to add + real code. In this example, you will write a basic list view, which renders data from + an API call. You will list all virtual machines owned by the logged-in user. To do + this, replace the 'show' function in the plugin code with a 'listView' block, + containing the required syntax for a list view. To get the data, use the + listVirtualMachines API call. Without any parameters, it will return VMs only for your + active user. Use the provided 'apiCall' helper method to handle the server call. Of + course, you are free to use any other method for making the AJAX call (for example, + jQuery's $.ajax method). + First, open your plugin's JavaScript source file in your favorite editor: + $ cd csMyFirstPlugin +$ vi csMyFirstPlugin.js + + Add the following code in csMyFirstPlugin.js: + (function (cloudStack) { + cloudStack.plugins.csMyFirstPlugin = function(plugin) { + plugin.ui.addSection({ + id: 'csMyFirstPlugin', + title: 'My Plugin', + preFilter: function(args) { + return isAdmin(); + }, + + // Render page as a list view + listView: { + id: 'testPluginInstances', + fields: { + name: { label: 'label.name' }, + instancename: { label: 'label.internal.name' }, + displayname: { label: 'label.display.name' }, + zonename: { label: 'label.zone.name' } + }, + dataProvider: function(args) { + // API calls go here, to retrive the data asynchronously + // + // On successful retrieval, call + // args.response.success({ data: [data array] }); + plugin.ui.apiCall('listVirtualMachines', { + success: function(json) { + var vms = json.listvirtualmachinesresponse.virtualmachine; + + args.response.success({ data: vms }); + }, + error: function(errorMessage) { + args.response.error(errorMessage) + } + }); + } + } + }); + }; +}(cloudStack)); + + + + Test the plugin. + First, copy all the plugin code that you have created so far to + /tomcat/webapps/client/plugins. Then refresh the browser. You can see that your + placeholder content was replaced with a list table, containing 4 columns of virtual + machine data. + + + Add an action button. + Let's add an action button to the list view, which will reboot the VM. To do this, + add an actions block under listView. After specifying the correct format, the actions + will appear automatically to the right of each row of data. + $ vi csMyFirstPlugin.js + + Now add the following new code in csMyFirstPlugin.js. (The dots ... show where we + have omitted some existing code for the sake of space. Don't actually cut and paste + that part): + ... + listView: { + id: 'testPluginInstances', + ... + + actions: { + // The key/ID you specify here will determine what icon is + // shown in the UI for this action, + // and will be added as a CSS class to the action's element + // (i.e., '.action.restart') + // + // -- here, 'restart' is a predefined name in &PRODUCT; that will + // automatically show a 'reboot' arrow as an icon; + // this can be changed in csMyFirstPlugin.css + restart: { + label: 'Restart VM', + messages: { + confirm: function() { return 'Are you sure you want to restart this VM?' }, + notification: function() { return 'Rebooted VM' } + }, + action: function(args) { + // Get the instance object of the selected row from context + // + // -- all currently loaded state is stored in 'context' as objects, + // such as the selected list view row, + // the selected section, and active user + // + // -- for list view actions, the object's key will be the same as + // listView.id, specified above; + // always make sure you specify an 'id' for the listView, + // or else it will be 'undefined!' + var instance = args.context.testPluginInstances[0]; + + plugin.ui.apiCall('rebootVirtualMachine', { + // These will be appended to the API request + // + // i.e., rebootVirtualMachine&id=... + data: { + id: instance.id + }, + success: function(json) { + args.response.success({ + // This is an async job, so success here only indicates + // that the job was initiated. + // + // To pass the job ID to the notification UI + // (for checking to see when action is completed), + // '_custom: { jobID: ... }' needs to always be passed on success, + // in the same format as below + _custom: { jobId: json.rebootvirtualmachineresponse.jobid } + }); + }, + + + error: function(errorMessage) { + args.response.error(errorMessage); // Cancel action, show error message returned + } + }); + }, + + // Because rebootVirtualMachine is an async job, we need to add + // a poll function, which will perodically check + // the management server to see if the job is ready + // (via pollAsyncJobResult API call) + // + // The plugin API provides a helper function, 'plugin.ui.pollAsyncJob', + / which will work for most jobs + // in &PRODUCT; + notification: { + poll: plugin.ui.pollAsyncJob + } + } + }, + + dataProvider: function(args) { + ... +... + + + + Add the thumbnail icon. + Create an icon file; it should be square, about 50x50 pixels, and named icon.png. + Copy it into the same directory with your plugin code: + cloudstack/ui/plugins/csMyFirstPlugin/icon.png. + + + Add the stylesheet. + Create a CSS file, with the same name as your .js file. Copy it into the same + directory with your plugin code: + cloudstack/ui/plugins/csMyFirstPlugin/csMyFirstPlugin.css. + + +
+ diff --git a/docs/en-US/update-iso-vm.xml b/docs/en-US/update-iso-vm.xml new file mode 100644 index 00000000000..98105f51198 --- /dev/null +++ b/docs/en-US/update-iso-vm.xml @@ -0,0 +1,47 @@ + + +%BOOK_ENTITIES; +]> + + +
+ + Changing a VM's Base Image + Every VM is created from a base image, which is a template or ISO which has been created and + stored in &PRODUCT;. Both cloud administrators and end users can create and modify templates, + ISOs, and VMs. + In &PRODUCT;, you can change an existing VM's base image from one template to another, + or from one ISO to another. (You can not change from an ISO to a template, or from a + template to an ISO). + For example, suppose there is a + template based on a particular operating system, and the OS vendor releases a software patch. + The administrator or user naturally wants to apply the patch and then make sure existing VMs + start using it. Whether a software update is involved or not, it's also possible to simply + switch a VM from its current template to any other desired template. + To change a VM's base image, call the restoreVirtualMachine API command and pass in the + virtual machine ID and a new template ID. The template ID parameter may refer to either a + template or an ISO, depending on which type of base image the VM was already using (it must + match the previous type of image). When this call occurs, the VM's root disk is first destroyed, + then a new root disk is created from the source designated in the template ID parameter. The new + root disk is attached to the VM, and now the VM is based on the new template. + You can also omit the template ID parameter from the restoreVirtualMachine call. In this + case, the VM's root disk is destroyed and recreated, but from the same template or ISO that was + already in use by the VM. +
\ No newline at end of file diff --git a/docs/en-US/user-data-and-meta-data.xml b/docs/en-US/user-data-and-meta-data.xml index 3f03449554a..34007011de1 100644 --- a/docs/en-US/user-data-and-meta-data.xml +++ b/docs/en-US/user-data-and-meta-data.xml @@ -24,7 +24,7 @@
User Data and Meta Data - &PRODUCT; provides API access to attach user data to a deployed VM. Deployed VMs also have access to instance metadata via the virtual router. + &PRODUCT; provides API access to attach up to 32KB of user data to a deployed VM. Deployed VMs also have access to instance metadata via the virtual router. User data can be accessed once the IP address of the virtual router is known. Once the IP address is known, use the following steps to access the user data: Run the following command to find the virtual router. diff --git a/docs/en-US/user-services-overview.xml b/docs/en-US/user-services-overview.xml index 12504e6ca4e..ad27375dd1d 100644 --- a/docs/en-US/user-services-overview.xml +++ b/docs/en-US/user-services-overview.xml @@ -24,7 +24,7 @@ User Services Overview - In addition to the physical and logical infrastructure of your cloud, + In addition to the physical and logical infrastructure of your cloud and the &PRODUCT; software and servers, you also need a layer of user services so that people can actually make use of the cloud. This means not just a user UI, but a set of options and resources that users can @@ -48,8 +48,8 @@ root disk, and other choices. See Creating a New Compute Offering. Disk Offerings, defined by the &PRODUCT; administrator, - provide a choice of disk size for primary data storage. See Creating a - New Disk Offering. + provide a choice of disk size and IOPS (Quality of Service) for primary + data storage. See Creating a New Disk Offering. Network Offerings, defined by the &PRODUCT; administrator, describe the feature set that is available to end users from the virtual diff --git a/docs/en-US/using-vpn-with-mac.xml b/docs/en-US/using-vpn-with-mac.xml index a41dcab5e02..769682445e4 100644 --- a/docs/en-US/using-vpn-with-mac.xml +++ b/docs/en-US/using-vpn-with-mac.xml @@ -23,7 +23,7 @@ -->
- Using VPN with Mac OS X + Using Remote Access VPN with Mac OS X First, be sure you've configured the VPN settings in your &PRODUCT; install. This section is only concerned with connecting via Mac OS X to your VPN. Note, these instructions were written on Mac OS X 10.7.5. They may differ slightly in older or newer releases of Mac OS X. diff --git a/docs/en-US/using-vpn-with-windows.xml b/docs/en-US/using-vpn-with-windows.xml index c5d95ddd3e0..82e556c58a4 100644 --- a/docs/en-US/using-vpn-with-windows.xml +++ b/docs/en-US/using-vpn-with-windows.xml @@ -23,7 +23,7 @@ -->
- Using VPN with Windows + Using Remote Access VPN with Windows The procedure to use VPN varies by Windows version. Generally, the user must edit the VPN properties and make sure that the default route is not the VPN. The following steps are for Windows L2TP clients on Windows Vista. The commands should be similar for other Windows versions. Log in to the &PRODUCT; UI and click on the source NAT IP for the account. The VPN tab should display the IPsec preshared key. Make a note of this and the source NAT IP. The UI also lists one or more users and their passwords. Choose one of these users, or, if none exists, add a user and password. diff --git a/docs/en-US/virtual-machines.xml b/docs/en-US/virtual-machines.xml index 802e8e1702f..8d8847853db 100644 --- a/docs/en-US/virtual-machines.xml +++ b/docs/en-US/virtual-machines.xml @@ -26,10 +26,16 @@ + +
+ Resetting the Virtual Machine Root Volume on Reboot + For secure environments, and to ensure that VM state is not persisted across reboots, + you can reset the root disk. For more information, see . +
diff --git a/docs/en-US/vlan-assign-isolated-nw.xml b/docs/en-US/vlan-assign-isolated-nw.xml index 2ed0129cfdf..424ecd2ac4a 100644 --- a/docs/en-US/vlan-assign-isolated-nw.xml +++ b/docs/en-US/vlan-assign-isolated-nw.xml @@ -21,14 +21,18 @@ -->
Assigning VLANs to Isolated Networks - &PRODUCT; provides you the ability to control VLAN assignment to Isolated networks. You can - assign a VLAN ID when a network is created, just the way it's done for Shared networks. + &PRODUCT; provides you the ability to control VLAN assignment to Isolated networks. As a + Root admin, you can assign a VLAN ID when a network is created, just the way it's done for + Shared networks. The former behaviour also is supported — VLAN is randomly allocated to a network from the VNET range of the physical network when the network turns to Implemented state. The VLAN is released back to the VNET pool when the network shuts down as a part of the Network Garbage Collection. The VLAN can be re-used either by the same network when it is implemented again, or by any other network. On each subsequent implementation of a network, a new VLAN can be assigned. + Only the Root admin can assign VLANs because the regular users or domain admin are not aware + of the physical network topology. They cannot even view what VLAN is assigned to a + network. To enable you to assign VLANs to Isolated networks, diff --git a/docs/en-US/vm-storage-migration.xml b/docs/en-US/vm-storage-migration.xml index e0dad57faa0..51c6f34a757 100644 --- a/docs/en-US/vm-storage-migration.xml +++ b/docs/en-US/vm-storage-migration.xml @@ -24,15 +24,23 @@
VM Storage Migration Supported in XenServer, KVM, and VMware. - - This procedure is different from moving disk volumes from one VM to another. See Detaching - and Moving Volumes . - - You can migrate a virtual machine’s root disk volume or any additional data disk volume from - one storage pool to another in the same zone. - You can use the storage migration feature to achieve some commonly desired administration - goals, such as balancing the load on storage pools and increasing the reliability of virtual - machines by moving them away from any storage pool that is experiencing issues. + This procedure is different from moving disk volumes from one VM to another as described in + . + + You can migrate a virtual machine’s root disk volume or any additional data disk volume from one storage pool to another in the same zone. + You can use the storage migration feature to achieve some commonly desired administration goals, such as balancing the load on storage pools and increasing the reliability of virtual machines by moving them away from any storage pool that is experiencing issues. + On XenServer and VMware, live migration of VM storage is enabled through &PRODUCT; + support for XenMotion and vMotion. + Live storage migration allows VMs to be moved from one host to another, where the VMs are + not located on storage shared between the two hosts. It provides the option to live + migrate a VM’s disks along with the VM itself. It is possible to migrate a VM from one + XenServer resource pool / VMware cluster to another, or to migrate a VM whose disks are on + local storage, or even to migrate a VM’s disks from one storage repository to another, all + while the VM is running. + Because of a limitation in VMware, live migration of storage for a VM is allowed only + if the source and target storage pool are accessible to the source host; that is, the host + where the VM is running when the live migration operation is requested. +
- Configuring a vSphere Cluster with VMware Distributed Virtual Switch - &PRODUCT; supports VMware vNetwork Distributed Switch (VDS) for virtual network configuration - in a VMware vSphere environment. This section helps you configure VMware VDS in a &PRODUCT; - deployment. Each vCenter server instance can support up to 128 VDS instances and each VDS - instance can manage up to 500 VMware hosts. + Configuring a VMware Datacenter with VMware Distributed Virtual Switch + &PRODUCT; supports VMware vNetwork Distributed Switch (VDS) for virtual network + configuration in a VMware vSphere environment. This section helps you configure VMware VDS in a + &PRODUCT; deployment. Each vCenter server instance can support up to 128 VDS instances and each + VDS instance can manage up to 500 VMware hosts.
About VMware Distributed Virtual Switch VMware VDS is an aggregation of host-level virtual switches on a VMware vCenter server. @@ -41,30 +41,130 @@ Prerequisites and Guidelines - Do not attempt to configure VDS by altering VMware traffic label when configuring - physical networks. This will only work for Standard Virtual Switch and should not be - distributed. + VMware VDS is supported only on Public and Guest traffic in &PRODUCT;. VMware VDS does not support multiple VDS per traffic type. If a user has many VDS switches, only one can be used for Guest traffic and another one for Public traffic. + + Additional switches of any type can be added for each cluster in the same zone. While + adding the clusters with different switch type, traffic labels is overridden at the + cluster level. + Management and Storage network does not support VDS. Therefore, use Standard Switch for these networks. + + When you remove a guest network, the corresponding dvportgroup will not be removed on + the vCenter. You must manually delete them on the vCenter. +
+
+ Preparation Checklist + For a smoother configuration of VMware VDS, note down the VDS name you have added in the + datacenter before you start: + + + + + + vds-name.png: Name of the dvSwitch as specified in the vCenter. + + + Use this VDS name when you specify the switch name in the traffic label while creating the + zone. Traffic label format is [["Name of vSwitch/dvSwitch/EthernetPortProfile"][,"VLAN + ID"[,"vSwitch Type"]]] + The possible values for traffic labels are: + + empty string + dvSwitch0 + dvSwitch0,200 + dvSwitch1,300,vmwaredvs + myEthernetPortProfile,,nexusdvs + dvSwitch0,,vmwaredvs + + + + + + + + traffic-label.png: Traffic label specified while zone creation. + + + + + + + + + + Fields + Name + Description + + + + + 1 + Represents the name of the virtual / distributed virtual switch at + vCenter. + The default value depends on the type of virtual switch: + vSwitch0: If type of virtual switch is VMware + vNetwork Standard virtual switch + dvSwitch0: If type of virtual switch is VMware + vNetwork Distributed virtual switch + epp0: If type of virtual switch is Cisco Nexus + 1000v Distributed virtual switch + + + 2 + VLAN ID to be used for this traffic wherever applicable. + This field would be used for only public traffic as of now. In case of guest traffic this + field would be ignored and could be left empty for guest traffic. By default empty + string would be assumed which translates to untagged VLAN for that specific traffic + type. + + + 3 + Type of virtual switch. Specified as string. + Possible valid values are vmwaredvs, vmwaresvs, nexusdvs. + vmwaresvs: Represents VMware vNetwork Standard + virtual switch + vmwaredvs: Represents VMware vNetwork + distributed virtual switch + nexusdvs: Represents Cisco Nexus 1000v + distributed virtual switch. + If nothing specified (left empty), zone-level default virtual switch would be + defaulted, based on the value of global parameter you specify. + Following are the global configuration parameters: + vmware.use.dvswitch: Set to true to enable any + kind (VMware DVS and Cisco Nexus 1000v) of distributed virtual switch in a &PRODUCT; + deployment. If set to false, the virtual switch that can be used in that &PRODUCT; + deployment is Standard virtual switch. + vmware.use.nexus.vswitch: This parameter is + ignored if vmware.use.dvswitch is set to false. Set to true to enable Cisco Nexus + 1000v distributed virtual switch in a &PRODUCT; deployment. + + + + +
Enabling Virtual Distributed Switch in &PRODUCT; To make a &PRODUCT; deployment VDS enabled, set the vmware.use.dvswitch parameter to true by using the Global Settings page in the &PRODUCT; UI and restart the Management Server. Unless you enable the vmware.use.dvswitch parameter, you cannot see any UI options specific to - VDS, and &PRODUCT; ignores the VDS-specific parameters given in the AddClusterCmd API call. - Additionally, &PRODUCT; uses VDS for virtual network infrastructure if the value of - vmware.use.dvswitch parameter is true and the value of vmware.use.nexus.dvswitch parameter is - false. + VDS, and &PRODUCT; ignores the VDS-specific parameters that you specify. Additionally, + &PRODUCT; uses VDS for virtual network infrastructure if the value of vmware.use.dvswitch + parameter is true and the value of vmware.use.nexus.dvswitch parameter is false. Another + global parameter that defines VDS configuration is vmware.ports.per.dvportgroup. This is the + default number of ports per VMware dvPortGroup in a VMware environment. Default value is 256. + This number directly associated with the number of guest network you can create. &PRODUCT; supports orchestration of virtual networks in a deployment with a mix of Virtual Distributed Switch, Standard Virtual Switch and Nexus 1000v Virtual Switch.
@@ -97,12 +197,12 @@ Cluster Name Enter the name of the cluster you created in vCenter. For example, - "cloud.cluster". + "cloudcluster". vCenter Host - Enter the name or the IP address of the vCenter host where you have deployed the VMware - VDS. + Enter the name or the IP address of the vCenter host where you have + deployed the VMware VDS. vCenter User name @@ -116,7 +216,7 @@ vCenter Datacenter Enter the vCenter datacenter that the cluster is in. For example, - "cloud.dc.VM". + "clouddcVM". Override Public Traffic @@ -154,40 +254,4 @@
-
- Removing VMware Virtual Switch - - - In the vCenter datacenter that is served by the VDS, ensure that you delete all the - hosts in the corresponding cluster. - - - Log in with Admin permissions to the &PRODUCT; administrator UI. - - - In the left navigation bar, select Infrastructure. - - - In the Infrastructure page, click View all under Clusters. - - - Select the cluster where you want to remove the virtual switch. - - - In the VMware dvSwitch tab, click the name of the virtual switch. - - - In the Details page, click Delete VMware dvSwitch icon. - - - - - DeleteButton.png: button to delete dvSwitch - - - - Click Yes in the confirmation dialog box. - - -
diff --git a/docs/en-US/vmware-install.xml b/docs/en-US/vmware-install.xml index fd88fc7c0cb..282cf2ec6e2 100644 --- a/docs/en-US/vmware-install.xml +++ b/docs/en-US/vmware-install.xml @@ -406,7 +406,7 @@ esxcfg-firewall -o 59000-60000,tcp,out,vncextras before you start: - vCenter Credentials + vCenter credentials Nexus 1000v VSM IP address diff --git a/docs/en-US/vnmc-cisco.xml b/docs/en-US/vnmc-cisco.xml index 809c1517b8e..b0785fc953f 100644 --- a/docs/en-US/vnmc-cisco.xml +++ b/docs/en-US/vnmc-cisco.xml @@ -21,62 +21,127 @@
External Guest Firewall Integration for Cisco VNMC (Optional) Cisco Virtual Network Management Center (VNMC) provides centralized multi-device and policy - management for Cisco Network Virtual Services. When Cisco VNMC is integrated with ASA 1000v - Cloud Firewall and Cisco Nexus 1000v dvSwitch in &PRODUCT; you will be able to: + management for Cisco Network Virtual Services. You can integrate Cisco VNMC with &PRODUCT; to + leverage the firewall and NAT service offered by ASA 1000v Cloud Firewall. Use it in a Cisco + Nexus 1000v dvSwitch-enabled cluster in &PRODUCT;. In such a deployment, you will be able to: - Configure Cisco ASA 1000v Firewalls + Configure Cisco ASA 1000v firewalls. You can configure one per guest network. - Create and apply security profiles that contain ACL policy sets for both ingress and - egress traffic, connection timeout, NAT policy sets, and TCP intercept + Use Cisco ASA 1000v firewalls to create and apply security profiles that contain ACL + policy sets for both ingress and egress traffic. + + + Use Cisco ASA 1000v firewalls to create and apply Source NAT, Port Forwarding, and + Static NAT policy sets. &PRODUCT; supports Cisco VNMC on Cisco Nexus 1000v dvSwich-enabled VMware hypervisors. -
- Use Cases - - - A Cloud administrator adds VNMC as a network element by using the admin API - addCiscoVnmcResource after specifying the credentials - - - A Cloud administrator adds ASA 1000v appliances by using the admin API - addCiscoAsa1000vResource. You can configure one per guest network. - - - A Cloud administrator creates an Isolated guest network offering by using ASA 1000v as - the service provider for Firewall, Source NAT, Port Forwarding, and Static NAT. - - -
Using Cisco ASA 1000v Firewall, Cisco Nexus 1000v dvSwitch, and Cisco VNMC in a Deployment -
- Prerequisites +
+ Guidelines - Ensure that Cisco ASA 1000v appliance is set up externally and then registered with - &PRODUCT; by using the admin API. Typically, you can create a pool of ASA 1000v - appliances and register them with &PRODUCT;. - Specify the following to set up a Cisco ASA 1000v instance: + Cisco ASA 1000v firewall is supported only in Isolated Guest Networks. + + + Cisco ASA 1000v firewall is not supported on VPC. + + + Cisco ASA 1000v firewall is not supported for load balancing. + + + When a guest network is created with Cisco VNMC firewall provider, an additional + public IP is acquired along with the Source NAT IP. The Source NAT IP is used for the + rules, whereas the additional IP is used to for the ASA outside interface. Ensure that + this additional public IP is not released. You can identify this IP as soon as the + network is in implemented state and before acquiring any further public IPs. The + additional IP is the one that is not marked as Source NAT. You can find the IP used for + the ASA outside interface by looking at the Cisco VNMC used in your guest + network. + + + Use the public IP address range from a single subnet. You cannot add IP addresses + from different subnets. + + + Only one ASA instance per VLAN is allowed because multiple VLANS cannot be trunked + to ASA ports. Therefore, you can use only one ASA instance in a guest network. + + + Only one Cisco VNMC per zone is allowed. + + + Supported only in Inline mode deployment with load balancer. + + + The ASA firewall rule is applicable to all the public IPs in the guest network. + Unlike the firewall rules created on virtual router, a rule created on the ASA device is + not tied to a specific public IP. + + + Use a version of Cisco Nexus 1000v dvSwitch that support the vservice command. For + example: nexus-1000v.4.2.1.SV1.5.2b.bin + Cisco VNMC requires the vservice command to be available on the Nexus switch to + create a guest network in &PRODUCT;. + + +
+
+ Prerequisites + + + Configure Cisco Nexus 1000v dvSwitch in a vCenter environment. + Create Port profiles for both internal and external network interfaces on Cisco + Nexus 1000v dvSwitch. Note down the inside port profile, which needs to be provided + while adding the ASA appliance to &PRODUCT;. + For information on configuration, see . + + + Deploy and configure Cisco VNMC. + For more information, see Installing Cisco Virtual Network Management Center and Configuring Cisco Virtual Network Management Center. + + + Register Cisco Nexus 1000v dvSwitch with Cisco VNMC. + For more information, see Registering a Cisco Nexus 1000V with Cisco VNMC. + + + Create Inside and Outside port profiles in Cisco Nexus 1000v dvSwitch. + For more information, see . + + + Deploy and Cisco ASA 1000v appliance. + For more information, see Setting Up the ASA 1000V Using VNMC. + Typically, you create a pool of ASA 1000v appliances and register them with + &PRODUCT;. + Specify the following while setting up a Cisco ASA 1000v instance: - ESX host IP + VNMC host IP. - Standalone or HA mode + Ensure that you add ASA appliance in VNMC mode. Port profiles for the Management and HA network interfaces. This need to be - pre-created on Nexus dvSwitch switch. + pre-created on Cisco Nexus 1000v dvSwitch. - Port profiles for both internal and external network interfaces. This need to be - pre-created on Nexus dvSwitch switch, and to be updated appropriately while - implementing guest networks. + Internal and external port profiles. The Management IP for Cisco ASA 1000v appliance. Specify the gateway such that @@ -89,29 +154,13 @@ VNMC credentials + + + Register Cisco ASA 1000v with VNMC. After Cisco ASA 1000v instance is powered on, register VNMC from the ASA console. - - Ensure that Cisco VNMC appliance is set up externally and then registered with - &PRODUCT; by using the admin API. A single VNMC instance manages multiple ASA1000v - appliances. - - - Ensure that Cisco Nexus 1000v appliance is set up and configured in &PRODUCT; when - adding VMware cluster. - - -
-
- Guidelines - When a guest network is created with Cisco VNMC firewall provider, an additional public - IP is by default acquired along with the Source NAT IP. The Source NAT IP is used for the - ASA outside interface, whereas the addition IP is used to workaround an ASA limitation. - Ensure that this additional public IP is not released. You can identify this IP as soon as - the network is in implemented state and before acquiring any further public IPs. The - additional IP is the one that is not marked as Source NAT. You can find the IP used for the - ASA outside interface by looking at the Cisco VNMC used in your guest network. +
Using Cisco ASA 1000v Services @@ -156,7 +205,7 @@ Choose the zone you want to work with. - Click the Network tab. + Click the Physical Network tab. In the Network Service Providers node of the diagram, click Configure. @@ -166,7 +215,7 @@ Click Cisco VNMC. - Click View VNMC Devices + Click View VNMC Devices. Click the Add VNMC Device and provide the following: @@ -204,7 +253,7 @@ Choose the zone you want to work with. - Click the Network tab. + Click the Physical Network tab. In the Network Service Providers node of the diagram, click Configure. @@ -220,15 +269,16 @@ Click the Add CiscoASA1000v Resource and provide the following: - Host: The management IP address of the ASA 1000v instance. The IP address is used - to connect to ASA 1000V. + Host: The management IP address of the ASA 1000v + instance. The IP address is used to connect to ASA 1000V. - Inside Port Profile: The Inside Port Profile configuration on Cisco Nexus1000v - dvSwitch. + Inside Port Profile: The Inside Port Profile + configured on Cisco Nexus1000v dvSwitch. - Cluster: The VMware cluster to which you are adding the ASA 1000v instance. + Cluster: The VMware cluster to which you are + adding the ASA 1000v instance. Ensure that the cluster is Cisco Nexus 1000v dvSwitch enabled. @@ -312,4 +362,39 @@
+
+ Reusing ASA 1000v Appliance in new Guest Networks + You can reuse an ASA 1000v appliance in a new guest network after the necessary cleanup. + Typically, ASA 1000v is cleaned up when the logical edge firewall is cleaned up in VNMC. If + this cleanup does not happen, you need to reset the appliance to its factory settings for use + in new guest networks. As part of this, enable SSH on the appliance and store the SSH + credentials by registering on VNMC. + + + Open a command line on the ASA appliance: + + + Run the following: + ASA1000V(config)# reload + You are prompted with the following message: + System config has been modified. Save? [Y]es/[N]o:" + + + Enter N. + You will get the following confirmation message: + "Proceed with reload? [confirm]" + + + Restart the appliance. + + + + + Register the ASA 1000v appliance with the VNMC: + ASA1000V(config)# vnmc policy-agent +ASA1000V(config-vnmc-policy-agent)# registration host vnmc_ip_address +ASA1000V(config-vnmc-policy-agent)# shared-secret key where key is the shared secret for authentication of the ASA 1000V connection to the Cisco VNMC + + +
diff --git a/docs/en-US/vpn.xml b/docs/en-US/vpn.xml index ccb3e861310..1f8098ca962 100644 --- a/docs/en-US/vpn.xml +++ b/docs/en-US/vpn.xml @@ -22,24 +22,41 @@ under the License. -->
- VPN - &PRODUCT; account owners can create virtual private networks (VPN) to access their virtual machines. If the guest network is instantiated from a network offering that offers the Remote Access VPN service, the virtual router (based on the System VM) is used to provide the service. &PRODUCT; provides a L2TP-over-IPsec-based remote access VPN service to guest virtual networks. Since each network gets its own virtual router, VPNs are not shared across the networks. VPN clients native to Windows, Mac OS X and iOS can be used to connect to the guest networks. The account owner can create and manage users for their VPN. &PRODUCT; does not use its account database for this purpose but uses a separate table. The VPN user database is shared across all the VPNs created by the account owner. All VPN users get access to all VPNs created by the account owner. - Make sure that not all traffic goes through the VPN. That is, the route installed by the VPN should be only for the guest network and not for all traffic. - - - Road Warrior / Remote Access. Users want to be able to - connect securely from a home or office to a private network in the cloud. Typically, - the IP address of the connecting client is dynamic and cannot be preconfigured on - the VPN server. - Site to Site. In this scenario, two private subnets are - connected over the public Internet with a secure VPN tunnel. The cloud user’s subnet - (for example, an office network) is connected through a gateway to the network in - the cloud. The address of the user’s gateway must be preconfigured on the VPN server - in the cloud. Note that although L2TP-over-IPsec can be used to set up Site-to-Site - VPNs, this is not the primary intent of this feature. For more information, see - - - - - + Remote Access VPN + &PRODUCT; account owners can create virtual private networks (VPN) to access their virtual + machines. If the guest network is instantiated from a network offering that offers the Remote + Access VPN service, the virtual router (based on the System VM) is used to provide the service. + &PRODUCT; provides a L2TP-over-IPsec-based remote access VPN service to guest virtual networks. + Since each network gets its own virtual router, VPNs are not shared across the networks. VPN + clients native to Windows, Mac OS X and iOS can be used to connect to the guest networks. The + account owner can create and manage users for their VPN. &PRODUCT; does not use its account + database for this purpose but uses a separate table. The VPN user database is shared across all + the VPNs created by the account owner. All VPN users get access to all VPNs created by the + account owner. + + Make sure that not all traffic goes through the VPN. That is, the route installed by the + VPN should be only for the guest network and not for all traffic. + + + + + Road Warrior / Remote Access. Users want to be able to + connect securely from a home or office to a private network in the cloud. Typically, the IP + address of the connecting client is dynamic and cannot be preconfigured on the VPN + server. + + + Site to Site. In this scenario, two private subnets are + connected over the public Internet with a secure VPN tunnel. The cloud user’s subnet (for + example, an office network) is connected through a gateway to the network in the cloud. The + address of the user’s gateway must be preconfigured on the VPN server in the cloud. Note + that although L2TP-over-IPsec can be used to set up Site-to-Site VPNs, this is not the + primary intent of this feature. For more information, see + + + + + +
diff --git a/docs/en-US/whats-new.xml b/docs/en-US/whats-new.xml index c129c1e9ff5..04733c71a75 100644 --- a/docs/en-US/whats-new.xml +++ b/docs/en-US/whats-new.xml @@ -26,7 +26,7 @@ What's New in the API for 4.2 - +
What's New in the API for 4.1 diff --git a/docs/en-US/working-with-hosts.xml b/docs/en-US/working-with-hosts.xml index 83cd8b2bdc6..d1fc74fd207 100644 --- a/docs/en-US/working-with-hosts.xml +++ b/docs/en-US/working-with-hosts.xml @@ -34,6 +34,6 @@ - + diff --git a/docs/en-US/working-with-iso.xml b/docs/en-US/working-with-iso.xml index 03e18ee3535..9872106ceec 100644 --- a/docs/en-US/working-with-iso.xml +++ b/docs/en-US/working-with-iso.xml @@ -29,4 +29,5 @@ ISO images may be stored in the system and made available with a privacy level similar to templates. ISO images are classified as either bootable or not bootable. A bootable ISO image is one that contains an OS image. &PRODUCT; allows a user to boot a guest VM off of an ISO image. Users can also attach ISO images to guest VMs. For example, this enables installing PV drivers into Windows. ISO images are not hypervisor-specific. +
diff --git a/docs/en-US/working-with-snapshots.xml b/docs/en-US/working-with-snapshots.xml index b984439203c..2d8cada48c1 100644 --- a/docs/en-US/working-with-snapshots.xml +++ b/docs/en-US/working-with-snapshots.xml @@ -28,9 +28,10 @@ Snapshots may be taken for volumes, including both root and data disks. The administrator places a limit on the number of stored snapshots per user. Users can create new volumes from the snapshot for recovery of particular files and they can create templates from snapshots to boot from a restored disk. Users can create snapshots manually or by setting up automatic recurring snapshot policies. Users can also create disk volumes from snapshots, which may be attached to a VM like any other disk volume. Snapshots of both root disks and data disks are supported. However, &PRODUCT; does not currently support booting a VM from a recovered root disk. A disk recovered from snapshot of a root disk is treated as a regular data disk; the data on recovered disk can be accessed by attaching the disk to a VM. A completed snapshot is copied from primary storage to secondary storage, where it is stored until deleted or purged by newer snapshot. - + +
diff --git a/docs/en-US/working-with-system-vm.xml b/docs/en-US/working-with-system-vm.xml index 70f7dd1aa4e..073d0772561 100644 --- a/docs/en-US/working-with-system-vm.xml +++ b/docs/en-US/working-with-system-vm.xml @@ -32,6 +32,7 @@ parameter on the &PRODUCT; UI or by calling the listConfigurations API. + diff --git a/docs/en-US/working-with-volumes.xml b/docs/en-US/working-with-volumes.xml index 6832cffe339..5de5e6c7bd8 100644 --- a/docs/en-US/working-with-volumes.xml +++ b/docs/en-US/working-with-volumes.xml @@ -47,6 +47,7 @@ +
diff --git a/docs/en-US/zone-add.xml b/docs/en-US/zone-add.xml index 3ca5789cd99..4137b671ee2 100644 --- a/docs/en-US/zone-add.xml +++ b/docs/en-US/zone-add.xml @@ -24,46 +24,17 @@
Adding a Zone - These steps assume you have already logged in to the &PRODUCT; UI. See . + When you add a new zone, you will be prompted to configure the zone’s physical network and add the first pod, cluster, host, primary storage, and secondary storage. - (Optional) If you are going to use Swift for cloud-wide secondary storage, you need to add it before you add zones. - - Log in to the &PRODUCT; UI as administrator. - If this is your first time visiting the UI, you will see the guided tour splash screen. Choose “Experienced user.†The Dashboard appears. - In the left navigation bar, click Global Settings. - In the search box, type swift.enable and click the search button. - Click the edit button and set swift.enable to true. - - - - - edit-icon.png: button to modify data - - - - Restart the Management Server. - # service cloudstack-management restart - - Refresh the &PRODUCT; UI browser tab and log back in. - - + Log in to the &PRODUCT; UI as the root administrator. See . In the left navigation, choose Infrastructure. On Zones, click View More. - (Optional) If you are using Swift storage, click Enable Swift. Provide the following: - - URL. The Swift URL. - Account. The Swift account. - Username. The Swift account’s username. - Key. The Swift key. - - Click Add Zone. The zone creation wizard will appear. Choose one of the following network types: Basic. For AWS-style networking. Provides a single network where each VM instance is assigned an IP directly from the network. Guest isolation can be provided through layer-3 means such as security groups (IP address source filtering). Advanced. For more sophisticated network topologies. This network model provides the most flexibility in defining guest networks and providing custom network offerings such as firewall, VPN, or load balancer support. - For more information about the network types, see . The rest of the steps differ depending on whether you chose Basic or Advanced. Continue with the steps that apply to you: diff --git a/docs/qig/en-US/Book_Info.xml b/docs/qig/en-US/Book_Info.xml index e356de4415a..98cbcb49327 100644 --- a/docs/qig/en-US/Book_Info.xml +++ b/docs/qig/en-US/Book_Info.xml @@ -27,7 +27,7 @@ Quick Install Guide Prescriptive instructions for deploying Apache CloudStack Apache CloudStack - 4.0.2 + 4.2.0 0 0 diff --git a/engine/api/pom.xml b/engine/api/pom.xml index 1b8f26c2320..cecdd50652a 100644 --- a/engine/api/pom.xml +++ b/engine/api/pom.xml @@ -16,7 +16,7 @@ org.apache.cloudstack cloud-engine - 4.2.0-SNAPSHOT + 4.2.0 ../pom.xml diff --git a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java index 085fbbdb232..4950597963d 100644 --- a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java +++ b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java @@ -61,4 +61,6 @@ public interface TemplateService { void downloadBootstrapSysTemplate(DataStore store); void addSystemVMTemplatesToSecondary(DataStore store); + + void associateCrosszoneTemplatesToZone(long dcId); } diff --git a/engine/api/src/org/apache/cloudstack/storage/command/CopyCommand.java b/engine/api/src/org/apache/cloudstack/storage/command/CopyCommand.java index f14f37ebd49..c591fafbfe3 100644 --- a/engine/api/src/org/apache/cloudstack/storage/command/CopyCommand.java +++ b/engine/api/src/org/apache/cloudstack/storage/command/CopyCommand.java @@ -16,8 +16,13 @@ // under the License. package org.apache.cloudstack.storage.command; +import org.apache.cloudstack.storage.to.TemplateObjectTO; +import org.apache.cloudstack.storage.to.VolumeObjectTO; + import com.cloud.agent.api.Command; import com.cloud.agent.api.to.DataTO; +import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.storage.DataStoreRole; public final class CopyCommand extends Command implements StorageSubSystemCommand { private DataTO srcTO; @@ -31,13 +36,33 @@ public final class CopyCommand extends Command implements StorageSubSystemComman this.srcTO = srcData; this.destTO = destData; this.setWait(timeout); - this.executeInSequence = executeInSequence; + this.executeInSequence = executeInSequence; // default is to run in parallel, so false here +/* + // special handling for vmware parallel vm deployment bug https://issues.apache.org/jira/browse/CLOUDSTACK-3568 + if (srcTO instanceof TemplateObjectTO && destTO instanceof VolumeObjectTO) { + // create a volume wrapper vm from a template on primary storage + TemplateObjectTO srcTmplt = (TemplateObjectTO) srcTO; + VolumeObjectTO destVol = (VolumeObjectTO) destTO; + if (srcTmplt.getHypervisorType() == HypervisorType.VMware && srcTmplt.getDataStore().getRole() == DataStoreRole.Primary + && destVol.getDataStore().getRole() == DataStoreRole.Primary) { + this.executeInSequence = true; + } + } +*/ } public DataTO getDestTO() { return this.destTO; } + public void setSrcTO(DataTO srcTO) { + this.srcTO = srcTO; + } + + public void setDestTO(DataTO destTO) { + this.destTO = destTO; + } + public DataTO getSrcTO() { return this.srcTO; } diff --git a/engine/api/src/org/apache/cloudstack/storage/datastore/db/ImageStoreDao.java b/engine/api/src/org/apache/cloudstack/storage/datastore/db/ImageStoreDao.java index 70e9bb386e0..f95e66cd498 100644 --- a/engine/api/src/org/apache/cloudstack/storage/datastore/db/ImageStoreDao.java +++ b/engine/api/src/org/apache/cloudstack/storage/datastore/db/ImageStoreDao.java @@ -31,6 +31,8 @@ public interface ImageStoreDao extends GenericDao { List findByScope(ZoneScope scope); + List findRegionImageStores(); + List findImageCacheByScope(ZoneScope scope); List listImageStores(); diff --git a/engine/api/src/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java b/engine/api/src/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java index 669dd25f5d3..38dd884eff1 100644 --- a/engine/api/src/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java +++ b/engine/api/src/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java @@ -28,7 +28,6 @@ import com.cloud.utils.db.GenericDao; * Data Access Object for storage_pool table */ public interface PrimaryDataStoreDao extends GenericDao { - /** * @param datacenterId -- the id of the datacenter (availability zone) */ @@ -42,19 +41,16 @@ public interface PrimaryDataStoreDao extends GenericDao { /** * Set capacity of storage pool in bytes * @param id pool id. - * @param capacity capacity in bytes + * @param capacityBytes capacity in bytes */ - void updateCapacity(long id, long capacity); + void updateCapacityBytes(long id, long capacityBytes); /** - * Set available bytes of storage pool in bytes - * - * @param id - * pool id. - * @param available - * available capacity in bytes + * Set iops capacity of storage pool + * @param id pool id. + * @param capacityIops iops capacity */ - void updateAvailable(long id, long available); + void updateCapacityIops(long id, long capacityIops); StoragePoolVO persist(StoragePoolVO pool, Map details); diff --git a/engine/api/src/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java b/engine/api/src/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java index 96e18fc6277..2c27b0e952d 100644 --- a/engine/api/src/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java +++ b/engine/api/src/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java @@ -145,18 +145,17 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase } @Override - public void updateAvailable(long id, long available) { + public void updateCapacityBytes(long id, long capacityBytes) { StoragePoolVO pool = createForUpdate(id); - pool.setUsedBytes(available); + pool.setCapacityBytes(capacityBytes); update(id, pool); } @Override - public void updateCapacity(long id, long capacity) { + public void updateCapacityIops(long id, long capacityIops) { StoragePoolVO pool = createForUpdate(id); - pool.setCapacityBytes(capacity); + pool.setCapacityIops(capacityIops); update(id, pool); - } @Override diff --git a/engine/api/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java b/engine/api/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java index f9037150c93..e350a763449 100644 --- a/engine/api/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java +++ b/engine/api/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java @@ -40,4 +40,5 @@ StateDao listDestroyed(long storeId); + List findBySnapshotId(long snapshotId); } diff --git a/engine/api/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreVO.java b/engine/api/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreVO.java index 300df1e9673..0fe5e088043 100644 --- a/engine/api/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreVO.java +++ b/engine/api/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreVO.java @@ -193,7 +193,7 @@ public class SnapshotDataStoreVO implements StateObject implements C ZoneClusterSearch = createSearchBuilder(); ZoneClusterSearch.and("dataCenterId", ZoneClusterSearch.entity().getDataCenterId(), SearchCriteria.Op.EQ); ZoneClusterSearch.done(); + + ClusterIdSearch = createSearchBuilder(Long.class); + ClusterIdSearch.selectField(ClusterIdSearch.entity().getId()); + ClusterIdSearch.and("dataCenterId", ClusterIdSearch.entity().getDataCenterId(), Op.EQ); + ClusterIdSearch.done(); } @Override @@ -168,11 +175,11 @@ public class ClusterDaoImpl extends GenericDaoBase implements C while (rs.next()) { Long podId = rs.getLong(1); Long clusterIdInPod = rs.getLong(2); - if(result.containsKey(podId)){ + if (result.containsKey(podId)) { List clusterList = result.get(podId); clusterList.add(clusterIdInPod); result.put(podId, clusterList); - }else{ + } else { List clusterList = new ArrayList(); clusterList.add(clusterIdInPod); result.put(podId, clusterList); @@ -191,13 +198,12 @@ public class ClusterDaoImpl extends GenericDaoBase implements C GenericSearchBuilder clusterIdSearch = createSearchBuilder(Long.class); clusterIdSearch.selectField(clusterIdSearch.entity().getId()); clusterIdSearch.and("dataCenterId", clusterIdSearch.entity().getDataCenterId(), Op.EQ); - if(podId != null){ + if (podId != null) { clusterIdSearch.and("podId", clusterIdSearch.entity().getPodId(), Op.EQ); } clusterIdSearch.and("allocationState", clusterIdSearch.entity().getAllocationState(), Op.EQ); clusterIdSearch.done(); - SearchCriteria sc = clusterIdSearch.create(); sc.addAnd("dataCenterId", SearchCriteria.Op.EQ, zoneId); if (podId != null) { @@ -250,4 +256,10 @@ public class ClusterDaoImpl extends GenericDaoBase implements C return result; } + @Override + public List listAllCusters(long zoneId) { + SearchCriteria sc = ClusterIdSearch.create(); + sc.setParameters("dataCenterId", zoneId); + return customSearch(sc, null); + } } diff --git a/engine/schema/src/com/cloud/dc/dao/DataCenterVnetDao.java b/engine/schema/src/com/cloud/dc/dao/DataCenterVnetDao.java index e2e6b795d35..28cd0278620 100644 --- a/engine/schema/src/com/cloud/dc/dao/DataCenterVnetDao.java +++ b/engine/schema/src/com/cloud/dc/dao/DataCenterVnetDao.java @@ -33,7 +33,7 @@ public interface DataCenterVnetDao extends GenericDao { public void delete(long physicalNetworkId); - public void deleteRange(Transaction txn, long dcId, long physicalNetworkId, int start, int end); + public void deleteVnets(Transaction txn, long dcId, long physicalNetworkId, List vnets); public void lockRange(long dcId, long physicalNetworkId, Integer start, Integer end); @@ -48,4 +48,6 @@ public interface DataCenterVnetDao extends GenericDao { public int countVnetsDedicatedToAccount(long dcId, long accountId); List listVnetsByPhysicalNetworkAndDataCenter(long dcId, long physicalNetworkId); + + int countAllocatedVnets(long physicalNetworkId); } diff --git a/engine/schema/src/com/cloud/dc/dao/DataCenterVnetDaoImpl.java b/engine/schema/src/com/cloud/dc/dao/DataCenterVnetDaoImpl.java index ced2982cf9d..d3a2409dc96 100755 --- a/engine/schema/src/com/cloud/dc/dao/DataCenterVnetDaoImpl.java +++ b/engine/schema/src/com/cloud/dc/dao/DataCenterVnetDaoImpl.java @@ -74,6 +74,12 @@ public class DataCenterVnetDaoImpl extends GenericDaoBase sc = DcSearchAllocated.create(); + sc.setParameters("physicalNetworkId", physicalNetworkId); + return listBy(sc).size(); + } public List listAllocatedVnetsInRange(long dcId, long physicalNetworkId, Integer start, Integer end) { SearchCriteria sc = DcSearchAllocatedInRange.create(); sc.setParameters("dc",dcId); @@ -110,9 +116,10 @@ public class DataCenterVnetDaoImpl extends GenericDaoBase argument each string is a vlan. not a vlanRange. public void add(long dcId, long physicalNetworkId, List vnets) { String insertVnet = "INSERT INTO `cloud`.`op_dc_vnet_alloc` (vnet, data_center_id, physical_network_id) VALUES ( ?, ?, ?)"; @@ -133,15 +140,18 @@ public class DataCenterVnetDaoImpl extends GenericDaoBase argument each string is a vlan. not a vlanRange. + public void deleteVnets(Transaction txn, long dcId, long physicalNetworkId, List vnets) { + String deleteVnet = "DELETE FROM `cloud`.`op_dc_vnet_alloc` WHERE data_center_id=? AND physical_network_id=? AND taken IS NULL AND vnet=?"; try { PreparedStatement stmt = txn.prepareAutoCloseStatement(deleteVnet); - stmt.setLong(1,dcId); - stmt.setLong(2,physicalNetworkId); - stmt.setString(3,((Integer)start).toString()); - stmt.setString(4,((Integer)end).toString()); - stmt.execute(); + for (int i =0; i <= vnets.size()-1; i++) { + stmt.setLong(1,dcId); + stmt.setLong(2,physicalNetworkId); + stmt.setString(3, vnets.get(i)); + stmt.addBatch(); + } + stmt.executeBatch(); } catch (SQLException e) { throw new CloudRuntimeException("Exception caught adding vnet ", e); } diff --git a/engine/schema/src/com/cloud/dc/dao/HostPodDao.java b/engine/schema/src/com/cloud/dc/dao/HostPodDao.java index 03f7155d0d2..1babef16d03 100644 --- a/engine/schema/src/com/cloud/dc/dao/HostPodDao.java +++ b/engine/schema/src/com/cloud/dc/dao/HostPodDao.java @@ -24,12 +24,13 @@ import com.cloud.utils.db.GenericDao; import com.cloud.vm.VirtualMachine; public interface HostPodDao extends GenericDao { - public List listByDataCenterId(long id); + public List listByDataCenterId(long id); public HostPodVO findByName(String name, long dcId); - - public HashMap> getCurrentPodCidrSubnets(long zoneId, long podIdToSkip); + + public HashMap> getCurrentPodCidrSubnets(long zoneId, long podIdToSkip); public List listDisabledPods(long zoneId); + public List listAllPods(long zoneId); } diff --git a/engine/schema/src/com/cloud/dc/dao/HostPodDaoImpl.java b/engine/schema/src/com/cloud/dc/dao/HostPodDaoImpl.java index 07b4ad13db6..14b2931dcc5 100644 --- a/engine/schema/src/com/cloud/dc/dao/HostPodDaoImpl.java +++ b/engine/schema/src/com/cloud/dc/dao/HostPodDaoImpl.java @@ -44,6 +44,7 @@ public class HostPodDaoImpl extends GenericDaoBase implements H protected SearchBuilder DataCenterAndNameSearch; protected SearchBuilder DataCenterIdSearch; + protected GenericSearchBuilder PodIdSearch; public HostPodDaoImpl() { DataCenterAndNameSearch = createSearchBuilder(); @@ -54,6 +55,12 @@ public class HostPodDaoImpl extends GenericDaoBase implements H DataCenterIdSearch = createSearchBuilder(); DataCenterIdSearch.and("dcId", DataCenterIdSearch.entity().getDataCenterId(), SearchCriteria.Op.EQ); DataCenterIdSearch.done(); + + PodIdSearch = createSearchBuilder(Long.class); + PodIdSearch.selectField(PodIdSearch.entity().getId()); + PodIdSearch.and("dataCenterId", PodIdSearch.entity().getDataCenterId(), Op.EQ); + PodIdSearch.and("allocationState", PodIdSearch.entity().getAllocationState(), Op.EQ); + PodIdSearch.done(); } @Override @@ -118,17 +125,16 @@ public class HostPodDaoImpl extends GenericDaoBase implements H @Override public List listDisabledPods(long zoneId) { - GenericSearchBuilder podIdSearch = createSearchBuilder(Long.class); - podIdSearch.selectField(podIdSearch.entity().getId()); - podIdSearch.and("dataCenterId", podIdSearch.entity().getDataCenterId(), Op.EQ); - podIdSearch.and("allocationState", podIdSearch.entity().getAllocationState(), Op.EQ); - podIdSearch.done(); - - - SearchCriteria sc = podIdSearch.create(); + SearchCriteria sc = PodIdSearch.create(); sc.addAnd("dataCenterId", SearchCriteria.Op.EQ, zoneId); sc.addAnd("allocationState", SearchCriteria.Op.EQ, Grouping.AllocationState.Disabled); return customSearch(sc, null); } + @Override + public List listAllPods(long zoneId) { + SearchCriteria sc = PodIdSearch.create(); + sc.addAnd("dataCenterId", SearchCriteria.Op.EQ, zoneId); + return customSearch(sc, null); + } } diff --git a/engine/schema/src/com/cloud/dc/dao/VlanDaoImpl.java b/engine/schema/src/com/cloud/dc/dao/VlanDaoImpl.java index eb3bde9d005..6f5a01f1476 100755 --- a/engine/schema/src/com/cloud/dc/dao/VlanDaoImpl.java +++ b/engine/schema/src/com/cloud/dc/dao/VlanDaoImpl.java @@ -347,7 +347,7 @@ public class VlanDaoImpl extends GenericDaoBase implements VlanDao @Override public List listZoneWideNonDedicatedVlans(long zoneId) { SearchCriteria sc = ZoneWideNonDedicatedVlanSearch.create(); - sc.setParameters("ZoneWideNonDedicatedVlanSearch", "zoneId", zoneId); + sc.setParameters("zoneId", zoneId); return listBy(sc); } diff --git a/engine/schema/src/com/cloud/host/dao/HostDao.java b/engine/schema/src/com/cloud/host/dao/HostDao.java index 8ceb8f23132..b007bb135a5 100755 --- a/engine/schema/src/com/cloud/host/dao/HostDao.java +++ b/engine/schema/src/com/cloud/host/dao/HostDao.java @@ -43,7 +43,7 @@ public interface HostDao extends GenericDao, StateDao findLostHosts(long timeout); + List findLostHosts(long timeout); List findAndUpdateDirectAgentToLoad(long lastPingSecondsAfter, Long limit, long managementServerId); @@ -61,15 +61,14 @@ public interface HostDao extends GenericDao, StateDao findAndUpdateApplianceToLoad(long lastPingSecondsAfter, long managementServerId); + List findAndUpdateApplianceToLoad(long lastPingSecondsAfter, long managementServerId); boolean updateResourceState(ResourceState oldState, ResourceState.Event event, ResourceState newState, Host vo); - HostVO findByGuid(String guid); - - HostVO findByTypeNameAndZoneId(long zoneId, String name, Host.Type type); - List findHypervisorHostInCluster(long clusterId); + HostVO findByGuid(String guid); + HostVO findByTypeNameAndZoneId(long zoneId, String name, Host.Type type); + List findHypervisorHostInCluster(long clusterId); /** * @param type @@ -86,4 +85,6 @@ public interface HostDao extends GenericDao, StateDao findByClusterId(Long clusterId); List listByDataCenterId(long id); + + List listAllHosts(long zoneId); } diff --git a/engine/schema/src/com/cloud/host/dao/HostDaoImpl.java b/engine/schema/src/com/cloud/host/dao/HostDaoImpl.java index dd26941aa00..b373737ee2b 100755 --- a/engine/schema/src/com/cloud/host/dao/HostDaoImpl.java +++ b/engine/schema/src/com/cloud/host/dao/HostDaoImpl.java @@ -105,7 +105,7 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao protected SearchBuilder ManagedRoutingServersSearch; protected SearchBuilder SecondaryStorageVMSearch; - + protected GenericSearchBuilder HostIdSearch; protected GenericSearchBuilder HostsInStatusSearch; protected GenericSearchBuilder CountRoutingByDc; protected SearchBuilder HostTransferSearch; @@ -319,7 +319,6 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao CountRoutingByDc.and("dc", CountRoutingByDc.entity().getDataCenterId(), SearchCriteria.Op.EQ); CountRoutingByDc.and("type", CountRoutingByDc.entity().getType(), SearchCriteria.Op.EQ); CountRoutingByDc.and("status", CountRoutingByDc.entity().getStatus(), SearchCriteria.Op.EQ); - CountRoutingByDc.done(); ManagedDirectConnectSearch = createSearchBuilder(); @@ -370,6 +369,11 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao HostsInClusterSearch.and("server", HostsInClusterSearch.entity().getManagementServerId(), SearchCriteria.Op.NNULL); HostsInClusterSearch.done(); + HostIdSearch = createSearchBuilder(Long.class); + HostIdSearch.selectField(HostIdSearch.entity().getId()); + HostIdSearch.and("dataCenterId", HostIdSearch.entity().getDataCenterId(), Op.EQ); + HostIdSearch.done(); + _statusAttr = _allAttributes.get("status"); _msIdAttr = _allAttributes.get("managementServerId"); _pingTimeAttr = _allAttributes.get("lastPinged"); @@ -392,7 +396,7 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao @Override public List listByDataCenterId(long id) { SearchCriteria sc = DcSearch.create(); - sc.setParameters("dcId", id); + sc.setParameters("dc", id); sc.setParameters("status", Status.Up); sc.setParameters("type", Host.Type.Routing); sc.setParameters("resourceState", ResourceState.Enabled); @@ -1027,4 +1031,10 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao return listBy(sc); } + @Override + public List listAllHosts(long zoneId) { + SearchCriteria sc = HostIdSearch.create(); + sc.addAnd("dataCenterId", SearchCriteria.Op.EQ, zoneId); + return customSearch(sc, null); + } } diff --git a/engine/schema/src/com/cloud/network/dao/PhysicalNetworkVO.java b/engine/schema/src/com/cloud/network/dao/PhysicalNetworkVO.java index f68eee1de5c..684a6008e9d 100644 --- a/engine/schema/src/com/cloud/network/dao/PhysicalNetworkVO.java +++ b/engine/schema/src/com/cloud/network/dao/PhysicalNetworkVO.java @@ -206,7 +206,7 @@ public class PhysicalNetworkVO implements PhysicalNetwork { public List> getVnet() { List > vnetList = new ArrayList>(); if (vnet != null) { - String [] Temp = vnet.split(";"); + String [] Temp = vnet.split(","); String [] vnetSplit = null; for (String vnetRange : Temp){ vnetSplit = vnetRange.split("-"); diff --git a/engine/schema/src/com/cloud/network/security/dao/SecurityGroupRulesDaoImpl.java b/engine/schema/src/com/cloud/network/security/dao/SecurityGroupRulesDaoImpl.java index 18ef57fbcd8..cb3baac3350 100644 --- a/engine/schema/src/com/cloud/network/security/dao/SecurityGroupRulesDaoImpl.java +++ b/engine/schema/src/com/cloud/network/security/dao/SecurityGroupRulesDaoImpl.java @@ -85,12 +85,4 @@ public class SecurityGroupRulesDaoImpl extends GenericDaoBase sc = createSearchCriteria(); - sc.addAnd("ruleUuid", SearchCriteria.Op.EQ, uuid); - SecurityGroupRulesVO rule = findOneIncludingRemovedBy(sc); - SecurityGroupRulesVO newRule = new SecurityGroupRulesVO(rule.getRuleId()); - return newRule; - } } diff --git a/engine/schema/src/com/cloud/network/vpc/VpcGatewayVO.java b/engine/schema/src/com/cloud/network/vpc/VpcGatewayVO.java index 7df2dfd236e..2c592cd4fbf 100644 --- a/engine/schema/src/com/cloud/network/vpc/VpcGatewayVO.java +++ b/engine/schema/src/com/cloud/network/vpc/VpcGatewayVO.java @@ -63,7 +63,7 @@ public class VpcGatewayVO implements VpcGateway { long zoneId; @Column(name="network_id") - Long networkId; + long networkId; @Column(name=GenericDao.CREATED_COLUMN) Date created; @@ -110,7 +110,7 @@ public class VpcGatewayVO implements VpcGateway { * @param account_id * @param sourceNat */ - public VpcGatewayVO(String ip4Address, Type type, Long vpcId, long zoneId, Long networkId, String vlanTag, + public VpcGatewayVO(String ip4Address, Type type, long vpcId, long zoneId, long networkId, String vlanTag, String gateway, String netmask, long accountId, long domainId, boolean sourceNat, long networkACLId) { this.ip4Address = ip4Address; this.type = type; @@ -160,7 +160,7 @@ public class VpcGatewayVO implements VpcGateway { } @Override - public Long getNetworkId() { + public long getNetworkId() { return networkId; } diff --git a/engine/schema/src/com/cloud/network/vpc/dao/VpcGatewayDao.java b/engine/schema/src/com/cloud/network/vpc/dao/VpcGatewayDao.java index 42144b6bbcd..55fc2af6644 100644 --- a/engine/schema/src/com/cloud/network/vpc/dao/VpcGatewayDao.java +++ b/engine/schema/src/com/cloud/network/vpc/dao/VpcGatewayDao.java @@ -16,16 +16,15 @@ // under the License. package com.cloud.network.vpc.dao; +import java.util.List; + import com.cloud.network.vpc.VpcGateway; import com.cloud.network.vpc.VpcGatewayVO; import com.cloud.utils.db.GenericDao; -import java.util.List; - public interface VpcGatewayDao extends GenericDao{ VpcGatewayVO getPrivateGatewayForVpc(long vpcId); - VpcGatewayVO getVpnGatewayForVpc(long vpcId); Long getNetworkAclIdForPrivateIp(long vpcId, long networkId, String ipaddr); diff --git a/engine/schema/src/com/cloud/network/vpc/dao/VpcGatewayDaoImpl.java b/engine/schema/src/com/cloud/network/vpc/dao/VpcGatewayDaoImpl.java index a8cb2b38c43..13c37c4e0e6 100644 --- a/engine/schema/src/com/cloud/network/vpc/dao/VpcGatewayDaoImpl.java +++ b/engine/schema/src/com/cloud/network/vpc/dao/VpcGatewayDaoImpl.java @@ -55,14 +55,6 @@ public class VpcGatewayDaoImpl extends GenericDaoBase implem return findOneBy(sc); } - @Override - public VpcGatewayVO getVpnGatewayForVpc(long vpcId) { - SearchCriteria sc = AllFieldsSearch.create(); - sc.setParameters("vpcId", vpcId); - sc.setParameters("type", VpcGateway.Type.Vpn); - - return findOneBy(sc); - } @Override public Long getNetworkAclIdForPrivateIp (long vpcId, long networkId, String ipaddr) { diff --git a/engine/schema/src/com/cloud/offerings/NetworkOfferingVO.java b/engine/schema/src/com/cloud/offerings/NetworkOfferingVO.java index 406d98a5286..1ab463bd64a 100755 --- a/engine/schema/src/com/cloud/offerings/NetworkOfferingVO.java +++ b/engine/schema/src/com/cloud/offerings/NetworkOfferingVO.java @@ -134,7 +134,8 @@ public class NetworkOfferingVO implements NetworkOffering { boolean egressdefaultpolicy; @Column(name = "concurrent_connections") - Integer concurrent_connections; + Integer concurrentConnections; + @Override public String getDisplayText() { @@ -433,10 +434,10 @@ public class NetworkOfferingVO implements NetworkOffering { this.publicLb = publicLb; } public Integer getConcurrentConnections() { - return this.concurrent_connections; + return this.concurrentConnections; } public void setConcurrentConnections(Integer concurrent_connections) { - this.concurrent_connections = concurrent_connections; + this.concurrentConnections = concurrent_connections; } } diff --git a/engine/schema/src/com/cloud/storage/S3VO.java b/engine/schema/src/com/cloud/storage/S3VO.java deleted file mode 100644 index e30da0cbc2d..00000000000 --- a/engine/schema/src/com/cloud/storage/S3VO.java +++ /dev/null @@ -1,205 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package com.cloud.storage; - -import java.util.Date; - -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.GeneratedValue; -import javax.persistence.GenerationType; -import javax.persistence.Id; -import javax.persistence.Table; - -import com.cloud.agent.api.to.S3TO; -import com.cloud.utils.db.GenericDao; - -//TODO: this will be removed after object_store merge. -@Entity -@Table(name = "s3") -public class S3VO implements S3 { - - public static final String ID_COLUMN_NAME = "id"; - - @Id - @GeneratedValue(strategy = GenerationType.IDENTITY) - @Column(name = ID_COLUMN_NAME) - private long id; - - @Column(name = "uuid") - private String uuid; - - @Column(name = "access_key") - private String accessKey; - - @Column(name = "secret_key") - private String secretKey; - - @Column(name = "end_point") - private String endPoint; - - @Column(name = "bucket") - private String bucketName; - - @Column(name = "https") - private Integer httpsFlag; - - @Column(name = "connection_timeout") - private Integer connectionTimeout; - - @Column(name = "max_error_retry") - private Integer maxErrorRetry; - - @Column(name = "socket_timeout") - private Integer socketTimeout; - - @Column(name = GenericDao.CREATED_COLUMN) - private Date created; - - public S3VO() { - super(); - } - - public S3VO(final String uuid, final String accessKey, final String secretKey, final String endPoint, - final String bucketName, final Boolean httpsFlag, final Integer connectionTimeout, - final Integer maxErrorRetry, final Integer socketTimeout, final Date created) { - - super(); - - this.uuid = uuid; - this.accessKey = accessKey; - this.secretKey = secretKey; - this.endPoint = endPoint; - this.bucketName = bucketName; - - Integer value = null; - if (httpsFlag != null) { - value = httpsFlag == false ? 0 : 1; - } - this.httpsFlag = value; - - this.connectionTimeout = connectionTimeout; - this.maxErrorRetry = maxErrorRetry; - this.socketTimeout = socketTimeout; - this.created = created; - - } - - @Override - public S3TO toS3TO() { - - Boolean httpsFlag = null; - if (this.httpsFlag != null) { - httpsFlag = this.httpsFlag == 0 ? false : true; - } - - return new S3TO(this.id, this.uuid, this.accessKey, this.secretKey, this.endPoint, this.bucketName, httpsFlag, - this.connectionTimeout, this.maxErrorRetry, this.socketTimeout, this.created, false); - - } - - public long getId() { - return this.id; - } - - public void setId(final long id) { - this.id = id; - } - - public String getUuid() { - return this.uuid; - } - - public void setUuid(final String uuid) { - this.uuid = uuid; - } - - public String getAccessKey() { - return this.accessKey; - } - - public void setAccessKey(final String accessKey) { - this.accessKey = accessKey; - } - - public String getSecretKey() { - return this.secretKey; - } - - public void setSecretKey(final String secretKey) { - this.secretKey = secretKey; - } - - public String getEndPoint() { - return this.endPoint; - } - - public void setEndPoint(final String endPoint) { - this.endPoint = endPoint; - } - - public String getBucketName() { - return this.bucketName; - } - - public void setBucketName(final String bucketName) { - this.bucketName = bucketName; - } - - public Integer getHttpsFlag() { - return this.httpsFlag; - } - - public void setHttpsFlag(final Integer httpsFlag) { - this.httpsFlag = httpsFlag; - } - - public Integer getConnectionTimeout() { - return this.connectionTimeout; - } - - public void setConnectionTimeout(final int connectionTimeout) { - this.connectionTimeout = connectionTimeout; - } - - public Integer getMaxErrorRetry() { - return this.maxErrorRetry; - } - - public void setMaxErrorRetry(final int maxErrorRetry) { - this.maxErrorRetry = maxErrorRetry; - } - - public Integer getSocketTimeout() { - return this.socketTimeout; - } - - public void setSocketTimeout(final int socketTimeout) { - this.socketTimeout = socketTimeout; - } - - public Date getCreated() { - return this.created; - } - - public void setCreated(final Date created) { - this.created = created; - } - -} diff --git a/engine/schema/src/com/cloud/storage/SwiftVO.java b/engine/schema/src/com/cloud/storage/SwiftVO.java deleted file mode 100644 index 1389242d21c..00000000000 --- a/engine/schema/src/com/cloud/storage/SwiftVO.java +++ /dev/null @@ -1,113 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package com.cloud.storage; - -import java.util.Date; -import java.util.UUID; - -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.GeneratedValue; -import javax.persistence.GenerationType; -import javax.persistence.Id; -import javax.persistence.Table; - -import org.apache.cloudstack.api.InternalIdentity; - -import com.cloud.agent.api.to.SwiftTO; -import com.cloud.utils.db.GenericDao; - -@Entity -@Table(name = "swift") -public class SwiftVO implements Swift, InternalIdentity { - - @Id - @GeneratedValue(strategy = GenerationType.IDENTITY) - @Column(name = "id") - private long id; - - @Column(name = "url") - String url; - - @Column(name = "account") - String account; - - @Column(name = "username") - String userName; - - @Column(name = "key") - String key; - - @Column(name = "uuid") - String uuid = UUID.randomUUID().toString(); - - @Column(name = GenericDao.CREATED_COLUMN) - private Date created; - - public SwiftVO() { - } - - public SwiftVO(String url, String account, String userName, String key) { - this.url = url; - this.account = account; - this.userName = userName; - this.key = key; - } - - @Override - public long getId() { - return id; - } - - @Override - public String getUrl() { - return url; - } - - @Override - public String getAccount() { - return account; - } - - @Override - public String getUserName() { - return userName; - } - - @Override - public String getKey() { - return key; - } - - public Date getCreated() { - return created; - } - - @Override - public SwiftTO toSwiftTO() { - return null; - } - - @Override - public String getUuid() { - return this.uuid; - } - - public void setUuid(String uuid) { - this.uuid = uuid; - } -} diff --git a/engine/schema/src/com/cloud/storage/VMTemplateS3VO.java b/engine/schema/src/com/cloud/storage/VMTemplateS3VO.java deleted file mode 100644 index e106bf7d1a5..00000000000 --- a/engine/schema/src/com/cloud/storage/VMTemplateS3VO.java +++ /dev/null @@ -1,194 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package com.cloud.storage; - -import com.cloud.utils.db.GenericDaoBase; -import org.apache.cloudstack.api.InternalIdentity; - -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.GeneratedValue; -import javax.persistence.GenerationType; -import javax.persistence.Id; -import javax.persistence.Table; - -import java.text.DateFormat; -import java.util.Date; - -@Entity -@Table(name = "template_s3_ref") -public class VMTemplateS3VO implements InternalIdentity { - - public static final String S3_ID_COLUMN_NAME = "s3_id"; - - public static final String TEMPLATE_ID_COLUMN_NAME = "template_id"; - - @Id - @GeneratedValue(strategy = GenerationType.IDENTITY) - private long id; - - @Column(name = S3_ID_COLUMN_NAME) - private long s3Id; - - @Column(name = TEMPLATE_ID_COLUMN_NAME) - private long templateId; - - @Column(name = GenericDaoBase.CREATED_COLUMN) - private Date created; - - @Column(name = "size") - private Long size; - - @Column(name = "physical_size") - private Long physicalSize; - - public VMTemplateS3VO() { - super(); - } - - public VMTemplateS3VO(final long s3Id, final long templateId, final Date created, final Long size, - final Long physicalSize) { - - super(); - - this.s3Id = s3Id; - this.templateId = templateId; - this.created = created; - this.size = size; - this.physicalSize = physicalSize; - - } - - @Override - public boolean equals(final Object thatObject) { - - if (this == thatObject) { - return true; - } - - if (thatObject == null || getClass() != thatObject.getClass()) { - return false; - } - - final VMTemplateS3VO thatVMTemplateS3VO = (VMTemplateS3VO) thatObject; - - if (this.id != thatVMTemplateS3VO.id) { - return false; - } - - if (this.s3Id != thatVMTemplateS3VO.s3Id) { - return false; - } - - if (this.templateId != thatVMTemplateS3VO.templateId) { - return false; - } - - if (this.created != null ? !created.equals(thatVMTemplateS3VO.created) : thatVMTemplateS3VO.created != null) { - return false; - } - - if (this.physicalSize != null ? !physicalSize.equals(thatVMTemplateS3VO.physicalSize) - : thatVMTemplateS3VO.physicalSize != null) { - return false; - } - - if (this.size != null ? !size.equals(thatVMTemplateS3VO.size) : thatVMTemplateS3VO.size != null) { - return false; - } - - return true; - } - - @Override - public int hashCode() { - - int result = (int) (this.id ^ (this.id >>> 32)); - - result = 31 * result + (int) (this.s3Id ^ (this.s3Id >>> 32)); - result = 31 * result + (int) (this.templateId ^ (this.templateId >>> 32)); - result = 31 * result + (this.created != null ? this.created.hashCode() : 0); - result = 31 * result + (this.size != null ? this.size.hashCode() : 0); - result = 31 * result + (this.physicalSize != null ? this.physicalSize.hashCode() : 0); - - return result; - - } - - public long getId() { - return this.id; - } - - public void setId(final long id) { - this.id = id; - } - - public long getS3Id() { - return this.s3Id; - } - - public void setS3Id(final long s3Id) { - this.s3Id = s3Id; - } - - public long getTemplateId() { - return this.templateId; - } - - public void setTemplateId(final long templateId) { - this.templateId = templateId; - } - - public Date getCreated() { - return this.created; - } - - public void setCreated(final Date created) { - this.created = created; - } - - public Long getSize() { - return this.size; - } - - public void setSize(final Long size) { - this.size = size; - } - - public Long getPhysicalSize() { - return this.physicalSize; - } - - public void setPhysicalSize(final Long physicalSize) { - this.physicalSize = physicalSize; - } - - @Override - public String toString() { - - final StringBuilder stringBuilder = new StringBuilder("VMTemplateS3VO [ id: ").append(id).append(", created: ") - .append(DateFormat.getDateTimeInstance().format(created)).append(", physicalSize: ") - .append(physicalSize).append(", size: ").append(size).append(", templateId: ").append(templateId) - .append(", s3Id: ").append(s3Id).append(" ]"); - - return stringBuilder.toString(); - - } - -} diff --git a/engine/schema/src/com/cloud/storage/VMTemplateStoragePoolVO.java b/engine/schema/src/com/cloud/storage/VMTemplateStoragePoolVO.java index 9331b038e49..b9886e08237 100644 --- a/engine/schema/src/com/cloud/storage/VMTemplateStoragePoolVO.java +++ b/engine/schema/src/com/cloud/storage/VMTemplateStoragePoolVO.java @@ -266,6 +266,11 @@ public class VMTemplateStoragePoolVO implements VMTemplateStorageResourceAssoc, return this.state; } + //TODO: this should be revisited post-4.2 to completely use state transition machine + public void setState(ObjectInDataStoreStateMachine.State state) { + this.state = state; + } + public long getUpdatedCount() { return this.updatedCount; } diff --git a/engine/schema/src/com/cloud/storage/VolumeVO.java b/engine/schema/src/com/cloud/storage/VolumeVO.java index 1445e99d727..ea3d6bffa67 100755 --- a/engine/schema/src/com/cloud/storage/VolumeVO.java +++ b/engine/schema/src/com/cloud/storage/VolumeVO.java @@ -152,6 +152,9 @@ public class VolumeVO implements Volume { @Column(name = "vm_snapshot_chain_size") private Long vmSnapshotChainSize; + + @Column(name = "iso_id") + private long isoId; @Transient // @Column(name="reservation") @@ -235,6 +238,7 @@ public class VolumeVO implements Volume { this.chainInfo = that.getChainInfo(); this.templateId = that.getTemplateId(); this.deviceId = that.getDeviceId(); + this.format = that.getFormat(); this.uuid = UUID.randomUUID().toString(); } @@ -561,4 +565,12 @@ public class VolumeVO implements Volume { public Long getVmSnapshotChainSize(){ return this.vmSnapshotChainSize; } + + public Long getIsoId() { + return this.isoId; + } + + public void setIsoId(long isoId) { + this.isoId =isoId; + } } diff --git a/engine/schema/src/com/cloud/storage/dao/S3DaoImpl.java b/engine/schema/src/com/cloud/storage/dao/S3DaoImpl.java deleted file mode 100644 index 7316f018037..00000000000 --- a/engine/schema/src/com/cloud/storage/dao/S3DaoImpl.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package com.cloud.storage.dao; - -import com.cloud.agent.api.to.S3TO; -import com.cloud.storage.S3VO; -import com.cloud.utils.db.GenericDaoBase; - -import javax.ejb.Local; - -import org.springframework.stereotype.Component; - -@Component -@Local(S3Dao.class) -public class S3DaoImpl extends GenericDaoBase implements S3Dao { - - @Override - public S3TO getS3TO(final Long id) { - - if (id != null) { - - final S3VO s3VO = findById(id); - if (s3VO != null) { - return s3VO.toS3TO(); - } - - } - - // NOTE: Excluded listAll / shuffle operation implemented in - // SwiftDaoImpl ... - - return null; - - } -} diff --git a/engine/schema/src/com/cloud/storage/dao/VMTemplateDao.java b/engine/schema/src/com/cloud/storage/dao/VMTemplateDao.java index c3d44bdb6aa..700ccf574b6 100755 --- a/engine/schema/src/com/cloud/storage/dao/VMTemplateDao.java +++ b/engine/schema/src/com/cloud/storage/dao/VMTemplateDao.java @@ -66,7 +66,7 @@ public interface VMTemplateDao extends GenericDao { VMTemplateVO findSystemVMTemplate(long zoneId); - VMTemplateVO findSystemVMTemplate(long zoneId, HypervisorType hType); + VMTemplateVO findSystemVMReadyTemplate(long zoneId, HypervisorType hypervisorType); VMTemplateVO findRoutingTemplate(HypervisorType type, String templateName); diff --git a/engine/schema/src/com/cloud/storage/dao/VMTemplateDaoImpl.java b/engine/schema/src/com/cloud/storage/dao/VMTemplateDaoImpl.java index 9e7599052a7..2aca203ca6d 100755 --- a/engine/schema/src/com/cloud/storage/dao/VMTemplateDaoImpl.java +++ b/engine/schema/src/com/cloud/storage/dao/VMTemplateDaoImpl.java @@ -23,11 +23,15 @@ import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Map; +import java.util.Collections; import javax.ejb.Local; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.storage.VMTemplateStorageResourceAssoc; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -75,6 +79,8 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem DomainDao _domainDao; @Inject DataCenterDao _dcDao; + @Inject + TemplateDataStoreDao _templateDataStoreDao; private static final String SELECT_S3_CANDIDATE_TEMPLATES = "SELECT t.id, t.unique_name, t.name, t.public, t.featured, " + "t.type, t.hvm, t.bits, t.url, t.format, t.created, t.account_id, t.checksum, t.display_text, " @@ -87,6 +93,7 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem protected SearchBuilder UniqueNameSearch; protected SearchBuilder tmpltTypeSearch; protected SearchBuilder tmpltTypeHyperSearch; + protected SearchBuilder readySystemTemplateSearch; protected SearchBuilder tmpltTypeHyperSearch2; protected SearchBuilder AccountIdSearch; @@ -326,6 +333,24 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem hostHyperSearch.done(); tmpltTypeHyperSearch.done(); + readySystemTemplateSearch = createSearchBuilder(); + readySystemTemplateSearch.and("removed", readySystemTemplateSearch.entity().getRemoved(), SearchCriteria.Op.NULL); + readySystemTemplateSearch.and("templateType", readySystemTemplateSearch.entity().getTemplateType(), SearchCriteria.Op.EQ); + SearchBuilder templateDownloadSearch = _templateDataStoreDao.createSearchBuilder(); + templateDownloadSearch.and("downloadState", templateDownloadSearch.entity().getDownloadState(), SearchCriteria.Op.EQ); + readySystemTemplateSearch.join("vmTemplateJoinTemplateStoreRef", templateDownloadSearch, templateDownloadSearch.entity().getTemplateId(), + readySystemTemplateSearch.entity().getId(), JoinBuilder.JoinType.INNER); + SearchBuilder hostHyperSearch2 = _hostDao.createSearchBuilder(); + hostHyperSearch2.and("type", hostHyperSearch2.entity().getType(), SearchCriteria.Op.EQ); + hostHyperSearch2.and("zoneId", hostHyperSearch2.entity().getDataCenterId(), SearchCriteria.Op.EQ); + hostHyperSearch2.and("removed", hostHyperSearch2.entity().getRemoved(), SearchCriteria.Op.NULL); + hostHyperSearch2.groupBy(hostHyperSearch2.entity().getHypervisorType()); + + readySystemTemplateSearch.join("tmplHyper", hostHyperSearch2, hostHyperSearch2.entity().getHypervisorType(), + readySystemTemplateSearch.entity().getHypervisorType(), JoinBuilder.JoinType.INNER); + hostHyperSearch2.done(); + readySystemTemplateSearch.done(); + tmpltTypeHyperSearch2 = createSearchBuilder(); tmpltTypeHyperSearch2.and("templateType", tmpltTypeHyperSearch2.entity().getTemplateType(), SearchCriteria.Op.EQ); @@ -764,22 +789,26 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem } @Override - public VMTemplateVO findSystemVMTemplate(long zoneId, HypervisorType hType) { - SearchCriteria sc = tmpltTypeHyperSearch.create(); + public VMTemplateVO findSystemVMReadyTemplate(long zoneId, HypervisorType hypervisorType) { + SearchCriteria sc = readySystemTemplateSearch.create(); sc.setParameters("templateType", Storage.TemplateType.SYSTEM); sc.setJoinParameters("tmplHyper", "type", Host.Type.Routing); sc.setJoinParameters("tmplHyper", "zoneId", zoneId); + sc.setJoinParameters("vmTemplateJoinTemplateStoreRef", "downloadState", VMTemplateStorageResourceAssoc.Status.DOWNLOADED); // order by descending order of id List tmplts = listBy(sc, new Filter(VMTemplateVO.class, "id", false, null, null)); - for (VMTemplateVO tmplt : tmplts) { - if (tmplt.getHypervisorType() == hType) { - return tmplt; + if (tmplts.size() > 0) { + if (hypervisorType == HypervisorType.Any) { + return tmplts.get(0); } - } - if (tmplts.size() > 0 && hType == HypervisorType.Any) { - return tmplts.get(0); + for (VMTemplateVO tmplt : tmplts) { + if (tmplt.getHypervisorType() == hypervisorType) { + return tmplt; + } + } + } return null; } diff --git a/engine/schema/src/com/cloud/storage/dao/VMTemplateS3DaoImpl.java b/engine/schema/src/com/cloud/storage/dao/VMTemplateS3DaoImpl.java deleted file mode 100644 index d49645d944a..00000000000 --- a/engine/schema/src/com/cloud/storage/dao/VMTemplateS3DaoImpl.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package com.cloud.storage.dao; - -import static com.cloud.utils.db.SearchCriteria.Op.*; -import static com.cloud.storage.VMTemplateS3VO.*; - -import com.cloud.storage.VMTemplateS3VO; -import com.cloud.utils.db.GenericDaoBase; -import com.cloud.utils.db.SearchBuilder; -import com.cloud.utils.db.SearchCriteria; - -import javax.ejb.Local; - -import org.springframework.stereotype.Component; - -import java.util.List; - -@Component -@Local(VMTemplateS3Dao.class) -public class VMTemplateS3DaoImpl extends GenericDaoBase implements VMTemplateS3Dao { - - private final SearchBuilder searchBuilder; - - public VMTemplateS3DaoImpl() { - - super(); - - this.searchBuilder = createSearchBuilder(); - this.searchBuilder.and(S3_ID_COLUMN_NAME, this.searchBuilder.entity().getS3Id(), EQ) - .and(TEMPLATE_ID_COLUMN_NAME, this.searchBuilder.entity().getTemplateId(), EQ).done(); - - } - - @Override - public List listByS3Id(final long s3id) { - - final SearchCriteria criteria = this.searchBuilder.create(); - - criteria.setParameters(S3_ID_COLUMN_NAME, s3id); - - return this.listBy(criteria); - - } - - @Override - public VMTemplateS3VO findOneByTemplateId(final long templateId) { - - final SearchCriteria criteria = this.searchBuilder.create(); - - criteria.setParameters(TEMPLATE_ID_COLUMN_NAME, templateId); - - return this.findOneBy(criteria); - - } - - @Override - public VMTemplateS3VO findOneByS3Template(final long s3Id, final long templateId) { - - final SearchCriteria criteria = this.searchBuilder.create(); - - criteria.setParameters(S3_ID_COLUMN_NAME, s3Id); - criteria.setParameters(TEMPLATE_ID_COLUMN_NAME, templateId); - - return this.findOneBy(criteria); - - } - - @Override - public void expungeAllByTemplateId(long templateId) { - - final SearchCriteria criteria = this.searchBuilder.create(); - - criteria.setParameters(TEMPLATE_ID_COLUMN_NAME, templateId); - - this.expunge(criteria); - - } - -} diff --git a/engine/schema/src/com/cloud/storage/dao/VolumeDao.java b/engine/schema/src/com/cloud/storage/dao/VolumeDao.java index 7b58e7d3400..1f5083a8e14 100755 --- a/engine/schema/src/com/cloud/storage/dao/VolumeDao.java +++ b/engine/schema/src/com/cloud/storage/dao/VolumeDao.java @@ -19,6 +19,7 @@ package com.cloud.storage.dao; import java.util.List; import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.storage.ScopeType; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.Volume; import com.cloud.storage.VolumeVO; @@ -75,7 +76,7 @@ public interface VolumeDao extends GenericDao, StateDao findReadyRootVolumesByInstance(long instanceId); List listPoolIdsByVolumeCount(long dcId, Long podId, Long clusterId, long accountId); - + List listZoneWidePoolIdsByVolumeCount(long dcId, long accountId); /** * Gets the Total Primary Storage space allocated for an account * @@ -93,4 +94,11 @@ public interface VolumeDao extends GenericDao, StateDao implements Vol // need to account for zone-wide primary storage where storage_pool has // null-value pod and cluster, where hypervisor information is stored in // storage_pool - protected static final String SELECT_HYPERTYPE_FROM_VOLUME = "SELECT s.hypervisor, c.hypervisor_type from volumes v, storage_pool s, cluster c where v.pool_id = s.id and s.cluster_id = c.id and v.id = ?"; + protected static final String SELECT_HYPERTYPE_FROM_CLUSTER_VOLUME = "SELECT c.hypervisor_type from volumes v, storage_pool s, cluster c where v.pool_id = s.id and s.cluster_id = c.id and v.id = ?"; + protected static final String SELECT_HYPERTYPE_FROM_ZONE_VOLUME = "SELECT s.hypervisor from volumes v, storage_pool s where v.pool_id = s.id and v.id = ?"; + protected static final String SELECT_POOLSCOPE = "SELECT s.scope from storage_pool s, volumes v where s.id = v.pool_id and v.id = ?"; private static final String ORDER_POOLS_NUMBER_OF_VOLUMES_FOR_ACCOUNT = "SELECT pool.id, SUM(IF(vol.state='Ready' AND vol.account_id = ?, 1, 0)) FROM `cloud`.`storage_pool` pool LEFT JOIN `cloud`.`volumes` vol ON pool.id = vol.pool_id WHERE pool.data_center_id = ? " + " AND pool.pod_id = ? AND pool.cluster_id = ? " + " GROUP BY pool.id ORDER BY 2 ASC "; - + private static final String ORDER_ZONE_WIDE_POOLS_NUMBER_OF_VOLUMES_FOR_ACCOUNT = "SELECT pool.id, SUM(IF(vol.state='Ready' AND vol.account_id = ?, 1, 0)) FROM `cloud`.`storage_pool` pool LEFT JOIN `cloud`.`volumes` vol ON pool.id = vol.pool_id WHERE pool.data_center_id = ? " + + " AND pool.scope = 'ZONE' AND pool.status='Up' " + " GROUP BY pool.id ORDER BY 2 ASC "; @Override public List findDetachedByAccount(long accountId) { SearchCriteria sc = DetachedAccountIdSearch.create(); @@ -234,25 +239,31 @@ public class VolumeDaoImpl extends GenericDaoBase implements Vol /* lookup from cluster of pool */ Transaction txn = Transaction.currentTxn(); PreparedStatement pstmt = null; - + String sql = null; try { - String sql = SELECT_HYPERTYPE_FROM_VOLUME; - pstmt = txn.prepareAutoCloseStatement(sql); - pstmt.setLong(1, volumeId); - ResultSet rs = pstmt.executeQuery(); - if (rs.next()) { - String hypervisor; - if (rs.getString(1) != null) - hypervisor = rs.getString(1); + ScopeType scope = getVolumeStoragePoolScope(volumeId); + if (scope != null ) { + if (scope == ScopeType.CLUSTER || scope == ScopeType.HOST) + sql = SELECT_HYPERTYPE_FROM_CLUSTER_VOLUME; + else if (scope == ScopeType.ZONE) + sql = SELECT_HYPERTYPE_FROM_ZONE_VOLUME; else - hypervisor = rs.getString(2); - return HypervisorType.getType(hypervisor); + s_logger.error("Unhandled scope type '" + scope + "' when running getHypervisorType on volume id " + volumeId); + + pstmt = txn.prepareAutoCloseStatement(sql); + pstmt.setLong(1, volumeId); + ResultSet rs = pstmt.executeQuery(); + if (rs.next()) { + if (rs.getString(1) != null) { + return HypervisorType.getType(rs.getString(1)); + } + } } return HypervisorType.None; } catch (SQLException e) { - throw new CloudRuntimeException("DB Exception on: " + SELECT_HYPERTYPE_FROM_VOLUME, e); + throw new CloudRuntimeException("DB Exception on: " + sql, e); } catch (Throwable e) { - throw new CloudRuntimeException("Caught: " + SELECT_HYPERTYPE_FROM_VOLUME, e); + throw new CloudRuntimeException("Caught: " + sql, e); } } @@ -470,6 +481,30 @@ public class VolumeDaoImpl extends GenericDaoBase implements Vol } } + @Override + public List listZoneWidePoolIdsByVolumeCount(long dcId, long accountId) { + + Transaction txn = Transaction.currentTxn(); + PreparedStatement pstmt = null; + List result = new ArrayList(); + try { + String sql = ORDER_ZONE_WIDE_POOLS_NUMBER_OF_VOLUMES_FOR_ACCOUNT; + pstmt = txn.prepareAutoCloseStatement(sql); + pstmt.setLong(1, accountId); + pstmt.setLong(2, dcId); + + ResultSet rs = pstmt.executeQuery(); + while (rs.next()) { + result.add(rs.getLong(1)); + } + return result; + } catch (SQLException e) { + throw new CloudRuntimeException("DB Exception on: " + ORDER_ZONE_WIDE_POOLS_NUMBER_OF_VOLUMES_FOR_ACCOUNT, e); + } catch (Throwable e) { + throw new CloudRuntimeException("Caught: " + ORDER_ZONE_WIDE_POOLS_NUMBER_OF_VOLUMES_FOR_ACCOUNT, e); + } + } + @Override @DB(txn = false) public Pair getNonDestroyedCountAndTotalByPool(long poolId) { @@ -494,4 +529,33 @@ public class VolumeDaoImpl extends GenericDaoBase implements Vol txn.commit(); return result; } + + @Override + public ScopeType getVolumeStoragePoolScope(long volumeId) { + // finding the storage scope where the volume is present + Transaction txn = Transaction.currentTxn(); + PreparedStatement pstmt = null; + + try { + String sql = SELECT_POOLSCOPE; + pstmt = txn.prepareAutoCloseStatement(sql); + pstmt.setLong(1, volumeId); + ResultSet rs = pstmt.executeQuery(); + if (rs.next()) { + String scope = rs.getString(1); + if (scope != null) { + try { + return Enum.valueOf(ScopeType.class, scope.toUpperCase()); + } catch (Exception e) { + throw new InvalidParameterValueException("invalid scope for pool " + scope); + } + } + } + } catch (SQLException e) { + throw new CloudRuntimeException("DB Exception on: " + SELECT_POOLSCOPE, e); + } catch (Throwable e) { + throw new CloudRuntimeException("Caught: " + SELECT_POOLSCOPE, e); + } + return null; + } } diff --git a/engine/schema/src/com/cloud/upgrade/dao/Upgrade2214to30.java b/engine/schema/src/com/cloud/upgrade/dao/Upgrade2214to30.java index e1b56df8da8..78ee674e069 100644 --- a/engine/schema/src/com/cloud/upgrade/dao/Upgrade2214to30.java +++ b/engine/schema/src/com/cloud/upgrade/dao/Upgrade2214to30.java @@ -79,7 +79,7 @@ public class Upgrade2214to30 extends Upgrade30xBase implements DbUpgrade { // drop keys dropKeysIfExist(conn); //update templete ID for system Vms - updateSystemVms(conn); + //updateSystemVms(conn); This is not required as system template update is handled during 4.2 upgrade // update domain network ref updateDomainNetworkRef(conn); // update networks that use redundant routers to the new network offering @@ -601,139 +601,6 @@ public class Upgrade2214to30 extends Upgrade30xBase implements DbUpgrade { } } - private void updateSystemVms(Connection conn){ - PreparedStatement pstmt = null; - ResultSet rs = null; - boolean xenserver = false; - boolean kvm = false; - boolean VMware = false; - s_logger.debug("Updating System Vm template IDs"); - try{ - //Get all hypervisors in use - try { - pstmt = conn.prepareStatement("select distinct(hypervisor_type) from `cloud`.`cluster` where removed is null"); - rs = pstmt.executeQuery(); - while(rs.next()){ - if("XenServer".equals(rs.getString(1))){ - xenserver = true; - } else if("KVM".equals(rs.getString(1))){ - kvm = true; - } else if("VMware".equals(rs.getString(1))){ - VMware = true; - } - } - } catch (SQLException e) { - throw new CloudRuntimeException("Error while listing hypervisors in use", e); - } - - s_logger.debug("Updating XenSever System Vms"); - //XenServer - try { - //Get 3.0.0 or later xenserer system Vm template Id - pstmt = conn.prepareStatement("select max(id) from `cloud`.`vm_template` where name like 'systemvm-xenserver-%' and removed is null"); - rs = pstmt.executeQuery(); - if(rs.next()){ - long templateId = rs.getLong(1); - rs.close(); - pstmt.close(); - // change template type to SYSTEM - pstmt = conn.prepareStatement("update `cloud`.`vm_template` set type='SYSTEM' where id = ?"); - pstmt.setLong(1, templateId); - pstmt.executeUpdate(); - pstmt.close(); - // update templete ID of system Vms - pstmt = conn.prepareStatement("update `cloud`.`vm_instance` set vm_template_id = ? where type <> 'User' and hypervisor_type = 'XenServer'"); - pstmt.setLong(1, templateId); - pstmt.executeUpdate(); - pstmt.close(); - } else { - if (xenserver){ - throw new CloudRuntimeException("3.0.0 or later XenServer SystemVm template not found. Cannot upgrade system Vms"); - } else { - s_logger.warn("3.0.0 or later XenServer SystemVm template not found. XenServer hypervisor is not used, so not failing upgrade"); - } - } - } catch (SQLException e) { - throw new CloudRuntimeException("Error while updating XenServer systemVm template", e); - } - - //KVM - s_logger.debug("Updating KVM System Vms"); - try { - //Get 3.0.0 or later KVM system Vm template Id - pstmt = conn.prepareStatement("select max(id) from `cloud`.`vm_template` where name like 'systemvm-kvm-%' and removed is null"); - rs = pstmt.executeQuery(); - if(rs.next()){ - long templateId = rs.getLong(1); - rs.close(); - pstmt.close(); - // change template type to SYSTEM - pstmt = conn.prepareStatement("update `cloud`.`vm_template` set type='SYSTEM' where id = ?"); - pstmt.setLong(1, templateId); - pstmt.executeUpdate(); - pstmt.close(); - // update templete ID of system Vms - pstmt = conn.prepareStatement("update `cloud`.`vm_instance` set vm_template_id = ? where type <> 'User' and hypervisor_type = 'KVM'"); - pstmt.setLong(1, templateId); - pstmt.executeUpdate(); - pstmt.close(); - } else { - if (kvm){ - throw new CloudRuntimeException("3.0.0 or later KVM SystemVm template not found. Cannot upgrade system Vms"); - } else { - s_logger.warn("3.0.0 or later KVM SystemVm template not found. KVM hypervisor is not used, so not failing upgrade"); - } - } - } catch (SQLException e) { - throw new CloudRuntimeException("Error while updating KVM systemVm template", e); - } - - //VMware - s_logger.debug("Updating VMware System Vms"); - try { - //Get 3.0.0 or later VMware system Vm template Id - pstmt = conn.prepareStatement("select max(id) from `cloud`.`vm_template` where name like 'systemvm-vmware-%' and removed is null"); - rs = pstmt.executeQuery(); - if(rs.next()){ - long templateId = rs.getLong(1); - rs.close(); - pstmt.close(); - // change template type to SYSTEM - pstmt = conn.prepareStatement("update `cloud`.`vm_template` set type='SYSTEM' where id = ?"); - pstmt.setLong(1, templateId); - pstmt.executeUpdate(); - pstmt.close(); - // update templete ID of system Vms - pstmt = conn.prepareStatement("update `cloud`.`vm_instance` set vm_template_id = ? where type <> 'User' and hypervisor_type = 'VMware'"); - pstmt.setLong(1, templateId); - pstmt.executeUpdate(); - pstmt.close(); - } else { - if (VMware){ - throw new CloudRuntimeException("3.0.0 or later VMware SystemVm template not found. Cannot upgrade system Vms"); - } else { - s_logger.warn("3.0.0 or later VMware SystemVm template not found. VMware hypervisor is not used, so not failing upgrade"); - } - } - } catch (SQLException e) { - throw new CloudRuntimeException("Error while updating VMware systemVm template", e); - } - s_logger.debug("Updating System Vm Template IDs Complete"); - } - finally { - try { - if (rs != null) { - rs.close(); - } - - if (pstmt != null) { - pstmt.close(); - } - } catch (SQLException e) { - } - } - } - private void createNetworkOfferingServices(Connection conn, String externalOfferingName) { PreparedStatement pstmt = null; ResultSet rs = null; diff --git a/engine/schema/src/com/cloud/upgrade/dao/Upgrade302to40.java b/engine/schema/src/com/cloud/upgrade/dao/Upgrade302to40.java index 6f31fdd2b8e..45f5f1bc2b8 100644 --- a/engine/schema/src/com/cloud/upgrade/dao/Upgrade302to40.java +++ b/engine/schema/src/com/cloud/upgrade/dao/Upgrade302to40.java @@ -64,7 +64,7 @@ public class Upgrade302to40 extends Upgrade30xBase implements DbUpgrade { @Override public void performDataMigration(Connection conn) { - updateVmWareSystemVms(conn); + //updateVmWareSystemVms(conn); This is not required as system template update is handled during 4.2 upgrade correctVRProviders(conn); correctMultiplePhysicaNetworkSetups(conn); addHostDetailsUniqueKey(conn); @@ -74,6 +74,7 @@ public class Upgrade302to40 extends Upgrade30xBase implements DbUpgrade { setupExternalNetworkDevices(conn); fixZoneUsingExternalDevices(conn); encryptConfig(conn); + encryptClusterDetails(conn); } @Override @@ -86,54 +87,6 @@ public class Upgrade302to40 extends Upgrade30xBase implements DbUpgrade { return new File[] { new File(script) }; } - private void updateVmWareSystemVms(Connection conn){ - PreparedStatement pstmt = null; - ResultSet rs = null; - boolean VMware = false; - try { - pstmt = conn.prepareStatement("select distinct(hypervisor_type) from `cloud`.`cluster` where removed is null"); - rs = pstmt.executeQuery(); - while(rs.next()){ - if("VMware".equals(rs.getString(1))){ - VMware = true; - } - } - } catch (SQLException e) { - throw new CloudRuntimeException("Error while iterating through list of hypervisors in use", e); - } - // Just update the VMware system template. Other hypervisor templates are unchanged from previous 3.0.x versions. - s_logger.debug("Updating VMware System Vms"); - try { - //Get 4.0 VMware system Vm template Id - pstmt = conn.prepareStatement("select id from `cloud`.`vm_template` where name = 'systemvm-vmware-4.0' and removed is null"); - rs = pstmt.executeQuery(); - if(rs.next()){ - long templateId = rs.getLong(1); - rs.close(); - pstmt.close(); - // change template type to SYSTEM - pstmt = conn.prepareStatement("update `cloud`.`vm_template` set type='SYSTEM' where id = ?"); - pstmt.setLong(1, templateId); - pstmt.executeUpdate(); - pstmt.close(); - // update templete ID of system Vms - pstmt = conn.prepareStatement("update `cloud`.`vm_instance` set vm_template_id = ? where type <> 'User' and hypervisor_type = 'VMware'"); - pstmt.setLong(1, templateId); - pstmt.executeUpdate(); - pstmt.close(); - } else { - if (VMware){ - throw new CloudRuntimeException("4.0 VMware SystemVm template not found. Cannot upgrade system Vms"); - } else { - s_logger.warn("4.0 VMware SystemVm template not found. VMware hypervisor is not used, so not failing upgrade"); - } - } - } catch (SQLException e) { - throw new CloudRuntimeException("Error while updating VMware systemVm template", e); - } - s_logger.debug("Updating System Vm Template IDs Complete"); - } - private void correctVRProviders(Connection conn) { PreparedStatement pstmtVR = null; ResultSet rsVR = null; @@ -1120,4 +1073,42 @@ public class Upgrade302to40 extends Upgrade30xBase implements DbUpgrade { } s_logger.debug("Done encrypting Config values"); } + + private void encryptClusterDetails(Connection conn) { + s_logger.debug("Encrypting cluster details"); + PreparedStatement pstmt = null; + ResultSet rs = null; + try { + pstmt = conn.prepareStatement("select id, value from `cloud`.`cluster_details` where name = 'password'"); + rs = pstmt.executeQuery(); + while (rs.next()) { + long id = rs.getLong(1); + String value = rs.getString(2); + if (value == null) { + continue; + } + String encryptedValue = DBEncryptionUtil.encrypt(value); + pstmt = conn.prepareStatement("update `cloud`.`cluster_details` set value=? where id=?"); + pstmt.setBytes(1, encryptedValue.getBytes("UTF-8")); + pstmt.setLong(2, id); + pstmt.executeUpdate(); + } + } catch (SQLException e) { + throw new CloudRuntimeException("Unable encrypt cluster_details values ", e); + } catch (UnsupportedEncodingException e) { + throw new CloudRuntimeException("Unable encrypt cluster_details values ", e); + } finally { + try { + if (rs != null) { + rs.close(); + } + + if (pstmt != null) { + pstmt.close(); + } + } catch (SQLException e) { + } + } + s_logger.debug("Done encrypting cluster_details"); + } } diff --git a/engine/schema/src/com/cloud/upgrade/dao/Upgrade304to305.java b/engine/schema/src/com/cloud/upgrade/dao/Upgrade304to305.java index f9b7e5ea94a..bfbce898540 100644 --- a/engine/schema/src/com/cloud/upgrade/dao/Upgrade304to305.java +++ b/engine/schema/src/com/cloud/upgrade/dao/Upgrade304to305.java @@ -19,6 +19,7 @@ package com.cloud.upgrade.dao; import java.io.File; +import java.io.UnsupportedEncodingException; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -27,6 +28,7 @@ import java.util.ArrayList; import java.util.List; import java.util.UUID; +import com.cloud.utils.crypt.DBEncryptionUtil; import org.apache.log4j.Logger; import com.cloud.utils.exception.CloudRuntimeException; @@ -56,13 +58,8 @@ public class Upgrade304to305 extends Upgrade30xBase implements DbUpgrade { if (script == null) { throw new CloudRuntimeException("Unable to find db/schema-304to305.sql"); } - - String vmWareTemplateInsertScript = Script.findScript("", "db/vmwaretmplinsert-304to305.sql"); - if (vmWareTemplateInsertScript == null) { - throw new CloudRuntimeException("Unable to find db/vmwaretmplinsert-304to305.sql"); - } - return new File[] { new File(vmWareTemplateInsertScript), new File(script) }; + return new File[] { new File(script) }; } @Override @@ -71,8 +68,9 @@ public class Upgrade304to305 extends Upgrade30xBase implements DbUpgrade { addVpcProvider(conn); updateRouterNetworkRef(conn); fixZoneUsingExternalDevices(conn); - updateSystemVms(conn); +// updateSystemVms(conn); fixForeignKeys(conn); + encryptClusterDetails(conn); } @Override @@ -460,4 +458,42 @@ public class Upgrade304to305 extends Upgrade30xBase implements DbUpgrade { throw new CloudRuntimeException("Unable to execute ssh_keypairs table update for adding domain_id foreign key", e); } } + + private void encryptClusterDetails(Connection conn) { + s_logger.debug("Encrypting cluster details"); + PreparedStatement pstmt = null; + ResultSet rs = null; + try { + pstmt = conn.prepareStatement("select id, value from `cloud`.`cluster_details` where name = 'password'"); + rs = pstmt.executeQuery(); + while (rs.next()) { + long id = rs.getLong(1); + String value = rs.getString(2); + if (value == null) { + continue; + } + String encryptedValue = DBEncryptionUtil.encrypt(value); + pstmt = conn.prepareStatement("update `cloud`.`cluster_details` set value=? where id=?"); + pstmt.setBytes(1, encryptedValue.getBytes("UTF-8")); + pstmt.setLong(2, id); + pstmt.executeUpdate(); + } + } catch (SQLException e) { + throw new CloudRuntimeException("Unable encrypt cluster_details values ", e); + } catch (UnsupportedEncodingException e) { + throw new CloudRuntimeException("Unable encrypt cluster_details values ", e); + } finally { + try { + if (rs != null) { + rs.close(); + } + + if (pstmt != null) { + pstmt.close(); + } + } catch (SQLException e) { + } + } + s_logger.debug("Done encrypting cluster_details"); + } } diff --git a/engine/schema/src/com/cloud/upgrade/dao/Upgrade410to420.java b/engine/schema/src/com/cloud/upgrade/dao/Upgrade410to420.java old mode 100644 new mode 100755 index 5052573d91a..8ff07dfa9dd --- a/engine/schema/src/com/cloud/upgrade/dao/Upgrade410to420.java +++ b/engine/schema/src/com/cloud/upgrade/dao/Upgrade410to420.java @@ -17,16 +17,6 @@ package com.cloud.upgrade.dao; -import com.cloud.deploy.DeploymentPlanner; -import com.cloud.storage.Upload; -import com.cloud.storage.UploadVO; -import com.cloud.utils.DateUtil; -import com.cloud.utils.exception.CloudRuntimeException; -import com.cloud.utils.script.Script; -import com.cloud.hypervisor.Hypervisor.HypervisorType; - -import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProvider; -import org.apache.log4j.Logger; import java.io.File; import java.sql.Connection; import java.sql.Date; @@ -35,47 +25,61 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Types; import java.util.ArrayList; -import java.util.List; -import java.util.UUID; -import java.util.Set; -import java.util.HashSet; -import java.util.Map; import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProvider; +import org.apache.log4j.Logger; + +import com.cloud.deploy.DeploymentPlanner; +import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.network.Networks.TrafficType; import com.cloud.network.vpc.NetworkACL; +import com.cloud.utils.Pair; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.Script; public class Upgrade410to420 implements DbUpgrade { - final static Logger s_logger = Logger.getLogger(Upgrade410to420.class); + final static Logger s_logger = Logger.getLogger(Upgrade410to420.class); - @Override - public String[] getUpgradableVersionRange() { - return new String[] { "4.1.0", "4.2.0" }; - } + @Override + public String[] getUpgradableVersionRange() { + return new String[] { "4.1.0", "4.2.0" }; + } - @Override - public String getUpgradedVersion() { - return "4.2.0"; - } + @Override + public String getUpgradedVersion() { + return "4.2.0"; + } - @Override - public boolean supportsRollingUpgrade() { - return false; - } + @Override + public boolean supportsRollingUpgrade() { + return false; + } - @Override - public File[] getPrepareScripts() { - String script = Script.findScript("", "db/schema-410to420.sql"); + @Override + public File[] getPrepareScripts() { + String script = Script.findScript("", "db/schema-410to420.sql"); if (script == null) { throw new CloudRuntimeException("Unable to find db/schema-410to420.sql"); } return new File[] { new File(script) }; - } + } - @Override - public void performDataMigration(Connection conn) { + @Override + public void performDataMigration(Connection conn) { + movePrivateZoneToDedicatedResource(conn); upgradeVmwareLabels(conn); persistLegacyZones(conn); + persistVswitchConfiguration(conn); createPlaceHolderNics(conn); updateRemoteAccessVpn(conn); updateSystemVmTemplates(conn); @@ -92,7 +96,7 @@ public class Upgrade410to420 implements DbUpgrade { correctExternalNetworkDevicesSetup(conn); removeFirewallServiceFromSharedNetworkOfferingWithSGService(conn); fix22xKVMSnapshots(conn); - setKVMSnapshotFlag(conn); + setKVMSnapshotFlag(conn); addIndexForAlert(conn); fixBaremetalForeignKeys(conn); // storage refactor related migration @@ -101,9 +105,424 @@ public class Upgrade410to420 implements DbUpgrade { migrateVolumeHostRef(conn); migrateTemplateHostRef(conn); migrateSnapshotStoreRef(conn); + migrateS3ToImageStore(conn); + migrateSwiftToImageStore(conn); fixNiciraKeys(conn); fixRouterKeys(conn); updateConcurrentConnectionsInNetworkOfferings(conn); + migrateDatafromIsoIdInVolumesTable(conn); + setRAWformatForRBDVolumes(conn); + migrateVolumeOnSecondaryStorage(conn); + createFullCloneFlag(conn); + } + + private void createFullCloneFlag(Connection conn) { + ResultSet rs = null; + PreparedStatement delete = null; + PreparedStatement query = null; + PreparedStatement update = null; + int numRows = 0; + try { + delete = conn.prepareStatement("delete from `cloud`.`configuration` where name='vmware.create.full.clone';"); + delete.executeUpdate(); + query = conn.prepareStatement("select count(*) from `cloud`.`data_center`"); + rs = query.executeQuery(); + if (rs.next()) { + numRows = rs.getInt(1); + } + if (numRows > 0) { + update = conn.prepareStatement("insert into `cloud`.`configuration` (`category`, `instance`, `component`, `name`, `value`, `description`) VALUES ('Advanced', 'DEFAULT', 'UserVmManager', 'vmware.create.full.clone' , 'false', 'If set to true, creates VMs as full clones on ESX hypervisor');"); + } else { + update = conn.prepareStatement("insert into `cloud`.`configuration` (`category`, `instance`, `component`, `name`, `value`, `description`) VALUES ('Advanced', 'DEFAULT', 'UserVmManager', 'vmware.create.full.clone' , 'true', 'If set to true, creates VMs as full clones on ESX hypervisor');"); + } + update.executeUpdate(); + } catch (SQLException e) { + throw new CloudRuntimeException("Failed to set global flag vmware.create.full.clone: ", e); + } finally { + if (update != null) { + try { + update.close(); + } catch (SQLException e) { + + } + } + if (query != null) { + try { + query.close(); + } catch (SQLException e) { + + } + } + if (delete != null) { + try { + delete.close(); + } catch (SQLException e) { + + } + } + } + } + + private void migrateVolumeOnSecondaryStorage(Connection conn) { + PreparedStatement sql = null; + try { + sql = conn.prepareStatement("update `cloud`.`volumes` set state='Uploaded' where state='UploadOp'"); + sql.executeUpdate(); + } catch (SQLException e) { + throw new CloudRuntimeException("Failed to upgrade volume state: ", e); + } finally { + if (sql != null) { + try { + sql.close(); + } catch (SQLException e) { + + } + } + } + } + + private void persistVswitchConfiguration(Connection conn) { + PreparedStatement clustersQuery = null; + ResultSet clusters = null; + Long clusterId; + String clusterHypervisorType; + final String NEXUS_GLOBAL_CONFIG_PARAM_NAME = "vmware.use.nexus.vswitch"; + final String DVS_GLOBAL_CONFIG_PARAM_NAME = "vmware.use.dvswitch"; + final String VSWITCH_GLOBAL_CONFIG_PARAM_CATEGORY = "Network"; + final String VMWARE_STANDARD_VSWITCH = "vmwaresvs"; + final String NEXUS_1000V_DVSWITCH = "nexusdvs"; + String paramValStr; + boolean readGlobalConfigParam = false; + boolean nexusEnabled = false; + String publicVswitchType = VMWARE_STANDARD_VSWITCH; + String guestVswitchType = VMWARE_STANDARD_VSWITCH; + String defaultPublicVswitchName = "vSwitch0"; + String defaultGuestVswitchName = "vSwitch0"; + String publicVswitchName = null; + String guestVswitchName = null; + Map>> detailsMap = new HashMap>>(); + List> detailsList; + + try { + clustersQuery = conn.prepareStatement("select id, hypervisor_type from `cloud`.`cluster` where removed is NULL"); + clusters = clustersQuery.executeQuery(); + while(clusters.next()) { + clusterHypervisorType = clusters.getString("hypervisor_type"); + clusterId = clusters.getLong("id"); + if (clusterHypervisorType.equalsIgnoreCase("VMware")) { + if (!readGlobalConfigParam) { + paramValStr = getConfigurationParameter(conn, VSWITCH_GLOBAL_CONFIG_PARAM_CATEGORY, NEXUS_GLOBAL_CONFIG_PARAM_NAME); + if(paramValStr.equalsIgnoreCase("true")) { + nexusEnabled = true; + } + } + // Set default values if cloud level setting is turned on for nexus 1000v. + if (nexusEnabled) { + publicVswitchType = NEXUS_1000V_DVSWITCH; + guestVswitchType = NEXUS_1000V_DVSWITCH; + defaultPublicVswitchName = "publicEthernetPortProfile"; + defaultGuestVswitchName = "guestEthernetPortProfile"; + } + // Read zone level settings from zone wide traffic labels for guest traffic and public traffic + guestVswitchName = getDefaultTrafficLabel(conn, TrafficType.Guest.toString()); + publicVswitchName = getDefaultTrafficLabel(conn, TrafficType.Public.toString()); + if (guestVswitchName == null) { + guestVswitchName = defaultGuestVswitchName; + } + if (publicVswitchName == null) { + publicVswitchName = defaultPublicVswitchName; + } + detailsList = new ArrayList>(); + detailsList.add(new Pair(ApiConstants.VSWITCH_TYPE_GUEST_TRAFFIC, guestVswitchType)); + detailsList.add(new Pair(ApiConstants.VSWITCH_NAME_GUEST_TRAFFIC, guestVswitchName)); + detailsList.add(new Pair(ApiConstants.VSWITCH_TYPE_PUBLIC_TRAFFIC, publicVswitchType)); + detailsList.add(new Pair(ApiConstants.VSWITCH_NAME_PUBLIC_TRAFFIC, publicVswitchName)); + detailsMap.put(clusterId, detailsList); + + updateClusterDetails(conn, detailsMap); + s_logger.debug("Persist vSwitch Configuration: Successfully persisted vswitch configuration for cluster " + clusterId); + } else { + s_logger.debug("Persist vSwitch Configuration: Ignoring cluster " + clusterId + " with hypervisor type " + clusterHypervisorType); + continue; + } + } // End cluster iteration + + if (nexusEnabled) { + // If Nexus global parameter is true, then set DVS configuration parameter to true. TODOS: Document that this mandates that MS need to be restarted. + setConfigurationParameter(conn, VSWITCH_GLOBAL_CONFIG_PARAM_CATEGORY, DVS_GLOBAL_CONFIG_PARAM_NAME, "true"); + } + } catch (SQLException e) { + String msg = "Unable to persist vswitch configuration of VMware clusters." + e.getMessage(); + s_logger.error(msg); + throw new CloudRuntimeException(msg, e); + } finally { + try { + if (clusters != null) { + clusters.close(); + } + if (clustersQuery != null) { + clustersQuery.close(); + } + } catch (SQLException e) { + } + } + } + + private void updateClusterDetails(Connection conn, Map>> detailsMap) { + PreparedStatement clusterDetailsInsert = null; + // Insert cluster details into cloud.cluster_details table for existing VMware clusters + // Input parameter detailMap is a map of clusterId and list of key value pairs for that cluster + Long clusterId; + String key; + String val; + List> keyValues; + try { + Iterator clusterIt = detailsMap.keySet().iterator(); + while (clusterIt.hasNext()) { + clusterId = clusterIt.next(); + keyValues = detailsMap.get(clusterId); + for (Pair keyValuePair : keyValues) { + clusterDetailsInsert = conn.prepareStatement("INSERT INTO `cloud`.`cluster_details` (cluster_id, name, value) VALUES (?, ?, ?)"); + key = keyValuePair.first(); + val = keyValuePair.second(); + clusterDetailsInsert.setLong(1, clusterId); + clusterDetailsInsert.setString(2, key); + clusterDetailsInsert.setString(3, val); + clusterDetailsInsert.executeUpdate(); + } + s_logger.debug("Inserted vswitch configuration details into cloud.cluster_details for cluster with id " + clusterId + "."); + } + } catch (SQLException e) { + throw new CloudRuntimeException("Unable insert cluster details into cloud.cluster_details table.", e); + } finally { + try { + if (clusterDetailsInsert != null) { + clusterDetailsInsert.close(); + } + } catch (SQLException e) { + } + } + } + + private String getDefaultTrafficLabel(Connection conn, String trafficType) { + ResultSet rs = null; + PreparedStatement pstmt = null; + try { + pstmt = conn.prepareStatement("select vmware_network_label from physical_network_traffic_types where vmware_network_label is not NULL and traffic_type='" + trafficType + "';"); + rs = pstmt.executeQuery(); + + while (rs.next()) { + String label = rs.getString("vmware_network_label"); + // Handle case of label specified as [vswitch_name,vlan_id] + return label.split(",")[0]; + } + } catch (SQLException e) { + throw new CloudRuntimeException("Unable read default traffic label for " + trafficType + ". ", e); + } finally { + try { + if (rs != null) { + rs.close(); + } + if (pstmt != null) { + pstmt.close(); + } + } catch (SQLException e) { + } + } + return null; + } + + private String getConfigurationParameter(Connection conn, String category, String paramName) { + ResultSet rs = null; + PreparedStatement pstmt = null; + try { + pstmt = conn.prepareStatement("select value from `cloud`.`configuration` where category='" + category + "' and value is not NULL and name = '" + paramName + "';"); + rs = pstmt.executeQuery(); + while (rs.next()) { + return rs.getString("value"); + } + } catch (SQLException e) { + throw new CloudRuntimeException("Unable read global configuration parameter " + paramName + ". ", e); + } finally { + try { + if (rs != null) { + rs.close(); + } + if (pstmt != null) { + pstmt.close(); + } + } catch (SQLException e) { + } + } + return "false"; + } + + private void setConfigurationParameter(Connection conn, String category, String paramName, String paramVal) { + PreparedStatement pstmt = null; + try { + pstmt = conn.prepareStatement("UPDATE `cloud`.`configuration` SET value = '" + paramVal + "' WHERE name = '" + paramName + "';"); + s_logger.debug("Updating global configuration parameter " + paramName + " with value " + paramVal + ". Update SQL statement is " + pstmt); + pstmt.executeUpdate(); + } catch (SQLException e) { + throw new CloudRuntimeException("Unable to set global configuration parameter " + paramName + " to " + paramVal + ". ", e); + } finally { + try { + if (pstmt != null) { + pstmt.close(); + } + } catch (SQLException e) { + } + } + } + + private void movePrivateZoneToDedicatedResource(Connection conn) { + + PreparedStatement pstmt = null; + ResultSet rs = null; + PreparedStatement pstmtUpdate = null; + PreparedStatement pstmt3 = null; + ResultSet rs3 = null; + + + try { + + pstmt3 = conn.prepareStatement("SELECT distinct(`domain_id`) FROM `cloud`.`data_center` WHERE `domain_id` IS NOT NULL AND removed IS NULL"); + rs3 = pstmt3.executeQuery(); + + while (rs3.next()) { + long domainId = rs3.getLong(1); + long affinityGroupId = 0; + + // create or find an affinity group for this domain of type + // 'ExplicitDedication' + PreparedStatement pstmt2 = null; + ResultSet rs2 = null; + pstmt2 = conn + .prepareStatement("SELECT affinity_group.id FROM `cloud`.`affinity_group` INNER JOIN `cloud`.`affinity_group_domain_map` ON affinity_group.id=affinity_group_domain_map.affinity_group_id WHERE affinity_group.type = 'ExplicitDedication' AND affinity_group.acl_type = 'Domain' AND (affinity_group_domain_map.domain_id = ?)"); + pstmt2.setLong(1, domainId); + rs2 = pstmt2.executeQuery(); + if (rs2.next()) { + // group exists, use it + affinityGroupId = rs2.getLong(1); + } else { + // create new group + rs2.close(); + pstmt2.close(); + + pstmt2 = conn.prepareStatement("SELECT name FROM `cloud`.`domain` where id = ?"); + pstmt2.setLong(1, domainId); + rs2 = pstmt2.executeQuery(); + String domainName = ""; + if (rs2.next()) { + domainName = rs2.getString(1); + } + rs2.close(); + pstmt2.close(); + // create new domain level group for this domain + String type = "ExplicitDedication"; + String uuid = UUID.randomUUID().toString(); + String groupName = "DedicatedGrp-domain-" + domainName; + s_logger.debug("Adding AffinityGroup of type " + type + " for domain id " + domainId); + + String sql = "INSERT INTO `cloud`.`affinity_group` (`name`, `type`, `uuid`, `description`, `domain_id`, `account_id`, `acl_type`) VALUES (?, ?, ?, ?, 1, 1, 'Domain')"; + pstmtUpdate = conn.prepareStatement(sql); + pstmtUpdate.setString(1, groupName); + pstmtUpdate.setString(2, type); + pstmtUpdate.setString(3, uuid); + pstmtUpdate.setString(4, "dedicated resources group"); + pstmtUpdate.executeUpdate(); + pstmtUpdate.close(); + + pstmt2 = conn + .prepareStatement("SELECT affinity_group.id FROM `cloud`.`affinity_group` where uuid = ?"); + pstmt2.setString(1, uuid); + rs2 = pstmt2.executeQuery(); + if (rs2.next()) { + affinityGroupId = rs2.getLong(1); + } + + // add the domain map + String sqlMap = "INSERT INTO `cloud`.`affinity_group_domain_map` (`domain_id`, `affinity_group_id`) VALUES (?, ?)"; + pstmtUpdate = conn.prepareStatement(sqlMap); + pstmtUpdate.setLong(1, domainId); + pstmtUpdate.setLong(2, affinityGroupId); + pstmtUpdate.executeUpdate(); + pstmtUpdate.close(); + + } + + rs2.close(); + pstmt2.close(); + + pstmt = conn.prepareStatement("SELECT `id` FROM `cloud`.`data_center` WHERE `domain_id` = ? AND removed IS NULL"); + pstmt.setLong(1, domainId); + rs = pstmt.executeQuery(); + + while (rs.next()) { + long zoneId = rs.getLong(1); + dedicateZone(conn, zoneId, domainId, affinityGroupId); + } + } + + } catch (SQLException e) { + throw new CloudRuntimeException("Exception while Moving private zone information to dedicated resources", e); + } finally { + if (pstmtUpdate != null) { + try { + pstmtUpdate.close(); + } catch (SQLException e) { + } + } + if (rs != null) { + try { + rs.close(); + } catch (SQLException e) { + } + } + if (pstmt != null) { + try { + pstmt.close(); + } catch (SQLException e) { + } + } + if (rs3 != null) { + try { + rs3.close(); + } catch (SQLException e) { + } + } + if (pstmt3 != null) { + try { + pstmt3.close(); + } catch (SQLException e) { + } + } + + } + } + + private void dedicateZone(Connection conn, long zoneId, long domainId, long affinityGroupId) { + PreparedStatement pstmtUpdate2 = null; + try { + // create the dedicated resources entry + String sql = "INSERT INTO `cloud`.`dedicated_resources` (`uuid`,`data_center_id`, `domain_id`, `affinity_group_id`) VALUES (?, ?, ?, ?)"; + pstmtUpdate2 = conn.prepareStatement(sql); + pstmtUpdate2.setString(1, UUID.randomUUID().toString()); + pstmtUpdate2.setLong(2, zoneId); + pstmtUpdate2.setLong(3, domainId); + pstmtUpdate2.setLong(4, affinityGroupId); + pstmtUpdate2.executeUpdate(); + pstmtUpdate2.close(); + } catch (SQLException e) { + throw new CloudRuntimeException("Exception while saving zone to dedicated resources", e); + } finally { + if (pstmtUpdate2 != null) { + try { + pstmtUpdate2.close(); + } catch (SQLException e) { + } + } + } } private void fixBaremetalForeignKeys(Connection conn) { @@ -113,12 +532,12 @@ public class Upgrade410to420 implements DbUpgrade { keys.add("fk_external_dhcp_devices_pod_id"); keys.add("fk_external_dhcp_devices_physical_network_id"); DbUpgradeUtils.dropKeysIfExist(conn, "baremetal_dhcp_devices", keys, true); - + keys.add("fk_external_pxe_devices_nsp_id"); keys.add("fk_external_pxe_devices_host_id"); keys.add("fk_external_pxe_devices_physical_network_id"); DbUpgradeUtils.dropKeysIfExist(conn, "baremetal_pxe_devices", keys, true); - + PreparedStatement pstmt = null; try { pstmt = conn.prepareStatement("ALTER TABLE `cloud`.`baremetal_dhcp_devices` ADD CONSTRAINT `fk_external_dhcp_devices_nsp_id` FOREIGN KEY (`nsp_id`) REFERENCES `physical_network_service_providers` (`id`) ON DELETE CASCADE"); @@ -127,8 +546,6 @@ public class Upgrade410to420 implements DbUpgrade { pstmt = conn.prepareStatement("ALTER TABLE `cloud`.`baremetal_dhcp_devices` ADD CONSTRAINT `fk_external_dhcp_devices_host_id` FOREIGN KEY (`host_id`) REFERENCES `host`(`id`) ON DELETE CASCADE"); pstmt.executeUpdate(); pstmt.close(); - pstmt = conn.prepareStatement("ALTER TABLE `cloud`.`baremetal_dhcp_devices` ADD CONSTRAINT `fk_external_dhcp_devices_pod_id` FOREIGN KEY (`pod_id`) REFERENCES `host_pod_ref`(`id`) ON DELETE CASCADE"); - pstmt.executeUpdate(); pstmt.close(); pstmt = conn.prepareStatement("ALTER TABLE `cloud`.`baremetal_dhcp_devices` ADD CONSTRAINT `fk_external_dhcp_devices_physical_network_id` FOREIGN KEY (`physical_network_id`) REFERENCES `physical_network`(`id`) ON DELETE CASCADE"); pstmt.executeUpdate(); @@ -255,8 +672,8 @@ public class Upgrade410to420 implements DbUpgrade { } - private void updateSystemVmTemplates(Connection conn) { - // TODO: system vm template migration after storage refactoring + private void updateSystemVmTemplates(Connection conn) { + // TODO: system vm template migration after storage refactoring PreparedStatement pstmt = null; ResultSet rs = null; s_logger.debug("Updating System Vm template IDs"); @@ -268,16 +685,16 @@ public class Upgrade410to420 implements DbUpgrade { rs = pstmt.executeQuery(); while(rs.next()){ switch (HypervisorType.getType(rs.getString(1))) { - case XenServer: hypervisorsListInUse.add(HypervisorType.XenServer); - break; - case KVM: hypervisorsListInUse.add(HypervisorType.KVM); - break; - case VMware: hypervisorsListInUse.add(HypervisorType.VMware); - break; - case Hyperv: hypervisorsListInUse.add(HypervisorType.Hyperv); - break; - case LXC: hypervisorsListInUse.add(HypervisorType.LXC); - break; + case XenServer: hypervisorsListInUse.add(HypervisorType.XenServer); + break; + case KVM: hypervisorsListInUse.add(HypervisorType.KVM); + break; + case VMware: hypervisorsListInUse.add(HypervisorType.VMware); + break; + case Hyperv: hypervisorsListInUse.add(HypervisorType.Hyperv); + break; + case LXC: hypervisorsListInUse.add(HypervisorType.LXC); + break; } } } catch (SQLException e) { @@ -286,19 +703,37 @@ public class Upgrade410to420 implements DbUpgrade { Map NewTemplateNameList = new HashMap(){ { put(HypervisorType.XenServer, "systemvm-xenserver-4.2"); - put(HypervisorType.VMware, "systemvm-vmware-4.2"); - put(HypervisorType.KVM, "systemvm-kvm-4.2"); - put(HypervisorType.LXC, "systemvm-lxc-4.2"); - put(HypervisorType.Hyperv, "systemvm-hyperv-4.2"); + put(HypervisorType.VMware, "systemvm-vmware-4.2"); + put(HypervisorType.KVM, "systemvm-kvm-4.2"); + put(HypervisorType.LXC, "systemvm-lxc-4.2"); + put(HypervisorType.Hyperv, "systemvm-hyperv-4.2"); } }; Map routerTemplateConfigurationNames = new HashMap(){ { put(HypervisorType.XenServer, "router.template.xen"); - put(HypervisorType.VMware, "router.template.vmware"); - put(HypervisorType.KVM, "router.template.kvm"); - put(HypervisorType.LXC, "router.template.lxc"); - put(HypervisorType.Hyperv, "router.template.hyperv"); + put(HypervisorType.VMware, "router.template.vmware"); + put(HypervisorType.KVM, "router.template.kvm"); + put(HypervisorType.LXC, "router.template.lxc"); + put(HypervisorType.Hyperv, "router.template.hyperv"); + } + }; + + Map newTemplateUrl = new HashMap(){ + { put(HypervisorType.XenServer, "http://download.cloud.com/templates/4.2/systemvmtemplate-2013-07-12-master-xen.vhd.bz2"); + put(HypervisorType.VMware, "http://download.cloud.com/templates/4.2/systemvmtemplate-4.2-vh7.ova"); + put(HypervisorType.KVM, "http://download.cloud.com/templates/4.2/systemvmtemplate-2013-06-12-master-kvm.qcow2.bz2"); + put(HypervisorType.LXC, "http://download.cloud.com/templates/acton/acton-systemvm-02062012.qcow2.bz2"); + put(HypervisorType.Hyperv, "http://download.cloud.com/templates/4.2/systemvmtemplate-2013-06-12-master-xen.vhd.bz2"); + } + }; + + Map newTemplateChecksum = new HashMap(){ + { put(HypervisorType.XenServer, "fb1b6e032a160d86f2c28feb5add6d83"); + put(HypervisorType.VMware, "8fde62b1089e5844a9cd3b9b953f9596"); + put(HypervisorType.KVM, "6cea42b2633841648040becb588bd8f0"); + put(HypervisorType.LXC, "2755de1f9ef2ce4d6f2bee2efbb4da92"); + put(HypervisorType.Hyperv, "fb1b6e032a160d86f2c28feb5add6d83"); } }; @@ -306,13 +741,18 @@ public class Upgrade410to420 implements DbUpgrade { s_logger.debug("Updating " + hypervisorAndTemplateName.getKey() + " System Vms"); try { //Get 4.2.0 system Vm template Id for corresponding hypervisor - pstmt = conn.prepareStatement("select id from `cloud`.`vm_template` where name like ? and removed is null order by id desc limit 1"); + pstmt = conn.prepareStatement("select id from `cloud`.`vm_template` where name = ? and removed is null order by id desc limit 1"); pstmt.setString(1, hypervisorAndTemplateName.getValue()); rs = pstmt.executeQuery(); if(rs.next()){ long templateId = rs.getLong(1); rs.close(); pstmt.close(); + // Mark the old system templates as removed + pstmt = conn.prepareStatement("UPDATE `cloud`.`vm_template` SET removed = now() WHERE hypervisor_type = ? AND type = 'SYSTEM' AND removed is null"); + pstmt.setString(1, hypervisorAndTemplateName.getKey().toString()); + pstmt.executeUpdate(); + pstmt.close(); // change template type to SYSTEM pstmt = conn.prepareStatement("update `cloud`.`vm_template` set type='SYSTEM' where id = ?"); pstmt.setLong(1, templateId); @@ -332,9 +772,16 @@ public class Upgrade410to420 implements DbUpgrade { pstmt.close(); } else { if (hypervisorsListInUse.contains(hypervisorAndTemplateName.getKey())){ - throw new CloudRuntimeException("4.2.0 " + hypervisorAndTemplateName.getKey() + " SystemVm template not found. Cannot upgrade system Vms"); + // throw new CloudRuntimeException("4.2.0 " + hypervisorAndTemplateName.getKey() + " SystemVm template not found. Cannot upgrade system Vms"); } else { s_logger.warn("4.2.0 " + hypervisorAndTemplateName.getKey() + " SystemVm template not found. " + hypervisorAndTemplateName.getKey() + " hypervisor is not used, so not failing upgrade"); + // Update the latest template URLs for corresponding hypervisor + pstmt = conn.prepareStatement("UPDATE `cloud`.`vm_template` SET url = ? , checksum = ? WHERE hypervisor_type = ? AND type = 'SYSTEM' AND removed is null order by id desc limit 1"); + pstmt.setString(1, newTemplateUrl.get(hypervisorAndTemplateName.getKey())); + pstmt.setString(2, newTemplateChecksum.get(hypervisorAndTemplateName.getKey())); + pstmt.setString(3, hypervisorAndTemplateName.getKey().toString()); + pstmt.executeUpdate(); + pstmt.close(); } } } catch (SQLException e) { @@ -378,17 +825,17 @@ public class Upgrade410to420 implements DbUpgrade { } } } - */ - } + */ + } - //KVM snapshot flag: only turn on if Customers is using snapshot; + //KVM snapshot flag: only turn on if Customers is using snapshot; private void setKVMSnapshotFlag(Connection conn) { s_logger.debug("Verify and set the KVM snapshot flag if snapshot was used. "); PreparedStatement pstmt = null; ResultSet rs = null; try { - int numRows = 0; - pstmt = conn.prepareStatement("select count(*) from `cloud`.`snapshots` where hypervisor_type = 'KVM'"); + int numRows = 0; + pstmt = conn.prepareStatement("select count(*) from `cloud`.`snapshots` where hypervisor_type = 'KVM'"); rs = pstmt.executeQuery(); if(rs.next()){ numRows = rs.getInt(1); @@ -396,8 +843,8 @@ public class Upgrade410to420 implements DbUpgrade { rs.close(); pstmt.close(); if (numRows > 0){ - //Add the configuration flag - pstmt = conn.prepareStatement("UPDATE `cloud`.`configuration` SET value = ? WHERE name = 'KVM.snapshot.enabled'"); + //Add the configuration flag + pstmt = conn.prepareStatement("UPDATE `cloud`.`configuration` SET value = ? WHERE name = 'kvm.snapshot.enabled'"); pstmt.setString(1, "true"); pstmt.executeUpdate(); } @@ -418,9 +865,9 @@ public class Upgrade410to420 implements DbUpgrade { s_logger.debug("Done set KVM snapshot flag. "); } - private void updatePrimaryStore(Connection conn) { - PreparedStatement sql = null; - PreparedStatement sql2 = null; + private void updatePrimaryStore(Connection conn) { + PreparedStatement sql = null; + PreparedStatement sql2 = null; try { sql = conn.prepareStatement("update storage_pool set storage_provider_name = ? , scope = ? where pool_type = 'Filesystem' or pool_type = 'LVM'"); sql.setString(1, DataStoreProvider.DEFAULT_PRIMARY); @@ -448,46 +895,84 @@ public class Upgrade410to420 implements DbUpgrade { } } } - } + } //update the cluster_details table with default overcommit ratios. private void updateCluster_details(Connection conn) { PreparedStatement pstmt = null; PreparedStatement pstmt1 = null; PreparedStatement pstmt2 =null; - ResultSet rs = null; - + PreparedStatement pstmt3 = null; + ResultSet rs1 = null; + ResultSet rscpu_global = null; + ResultSet rsmem_global = null; try { - pstmt = conn.prepareStatement("select id from `cloud`.`cluster`"); - pstmt1=conn.prepareStatement("INSERT INTO `cloud`.`cluster_details` (cluster_id, name, value) VALUES(?, 'cpuOvercommitRatio', '1')"); - pstmt2=conn.prepareStatement("INSERT INTO `cloud`.`cluster_details` (cluster_id, name, value) VALUES(?, 'memoryOvercommitRatio', '1')"); - rs = pstmt.executeQuery(); - while (rs.next()) { - long id = rs.getLong(1); - //update cluster_details table with the default overcommit ratios. - pstmt1.setLong(1,id); - pstmt1.execute(); - pstmt2.setLong(1,id); - pstmt2.execute(); + pstmt = conn.prepareStatement("select id, hypervisor_type from `cloud`.`cluster`"); + pstmt1=conn.prepareStatement("INSERT INTO `cloud`.`cluster_details` (cluster_id, name, value) VALUES(?, 'cpuOvercommitRatio', ?)"); + pstmt2=conn.prepareStatement("INSERT INTO `cloud`.`cluster_details` (cluster_id, name, value) VALUES(?, 'memoryOvercommitRatio', ?)"); + pstmt3=conn.prepareStatement("select value from `cloud`.`configuration` where name=?"); + pstmt3.setString(1,"cpu.overprovisioning.factor"); + rscpu_global = pstmt3.executeQuery(); + String global_cpu_overprovisioning_factor = "1"; + if (rscpu_global.next()) + global_cpu_overprovisioning_factor = rscpu_global.getString(1); + pstmt3.setString(1,"mem.overprovisioning.factor"); + rsmem_global = pstmt3.executeQuery(); + String global_mem_overprovisioning_factor = "1"; + if (rsmem_global.next()) + global_mem_overprovisioning_factor = rsmem_global.getString(1); + rs1 = pstmt.executeQuery(); + + while (rs1.next()) { + long id = rs1.getLong(1); + String hypervisor_type = rs1.getString(2); + if (hypervisor_type.equalsIgnoreCase(HypervisorType.VMware.toString())) { + pstmt1.setLong(1,id); + pstmt1.setString(2,global_cpu_overprovisioning_factor); + pstmt1.execute(); + pstmt2.setLong(1,id); + pstmt2.setString(2,global_mem_overprovisioning_factor); + pstmt2.execute(); + }else { + //update cluster_details table with the default overcommit ratios. + pstmt1.setLong(1,id); + pstmt1.setString(2,"1"); + pstmt1.execute(); + pstmt2.setLong(1,id); + pstmt2.setString(2,"1"); + pstmt2.execute(); + } } } catch (SQLException e) { throw new CloudRuntimeException("Unable to update cluster_details with default overcommit ratios.", e); } finally { try { - if (rs != null) { - rs.close(); + if (rs1 != null) { + rs1.close(); + } + if (rsmem_global != null) { + rsmem_global.close(); + } + if (rscpu_global != null) { + rscpu_global.close(); } if (pstmt != null) { pstmt.close(); } + if (pstmt2 != null) { + pstmt2.close(); + } + if (pstmt3 != null) { + pstmt3.close(); + } } catch (SQLException e) { } } } - @Override - public File[] getCleanupScripts() { + @Override + public File[] getCleanupScripts() { String script = Script.findScript("", "db/schema-410to420-cleanup.sql"); if (script == null) { throw new CloudRuntimeException("Unable to find db/schema-410to420-cleanup.sql"); @@ -534,7 +1019,7 @@ public class Upgrade410to420 implements DbUpgrade { try { // update the existing vmware traffic labels - pstmt = conn.prepareStatement("select name,value from `cloud`.`configuration` where category='Hidden' and value is not NULL and name REGEXP 'vmware\\.*\\.vswitch';"); + pstmt = conn.prepareStatement("select name,value from `cloud`.`configuration` where category='Hidden' and value is not NULL and name REGEXP 'vmware*.vswitch';"); rsParams = pstmt.executeQuery(); while (rsParams.next()) { trafficTypeVswitchParam = rsParams.getString("name"); @@ -547,11 +1032,11 @@ public class Upgrade410to420 implements DbUpgrade { } else if (trafficTypeVswitchParam.equals("vmware.guest.vswitch")) { trafficType = "Guest"; } - s_logger.debug("Updating vmware label for " + trafficType + " traffic. Update SQL statement is " + pstmt); pstmt = conn.prepareStatement("select physical_network_id, traffic_type, vmware_network_label from physical_network_traffic_types where vmware_network_label is not NULL and traffic_type='" + trafficType + "';"); rsLabel = pstmt.executeQuery(); newLabel = getNewLabel(rsLabel, trafficTypeVswitchParamValue); pstmt = conn.prepareStatement("update physical_network_traffic_types set vmware_network_label = " + newLabel + " where traffic_type = '" + trafficType + "' and vmware_network_label is not NULL;"); + s_logger.debug("Updating vmware label for " + trafficType + " traffic. Update SQL statement is " + pstmt); pstmt.executeUpdate(); } } catch (SQLException e) { @@ -581,7 +1066,6 @@ public class Upgrade410to420 implements DbUpgrade { Long vmwareDcId = 1L; Long zoneId; Long clusterId; - String clusterHypervisorType; boolean legacyZone; boolean ignoreZone; Long count; @@ -598,12 +1082,13 @@ public class Upgrade410to420 implements DbUpgrade { String value; try { - clustersQuery = conn.prepareStatement("select id, hypervisor_type from `cloud`.`cluster` where removed is NULL"); pstmt = conn.prepareStatement("select id from `cloud`.`data_center` where removed is NULL"); rs = pstmt.executeQuery(); while (rs.next()) { zoneId = rs.getLong("id"); + clustersQuery = conn.prepareStatement("select id from `cloud`.`cluster` where removed is NULL AND data_center_id=? AND hypervisor_type='VMware'"); + clustersQuery.setLong(1, zoneId); legacyZone = false; ignoreZone = true; count = 0L; @@ -616,39 +1101,36 @@ public class Upgrade410to420 implements DbUpgrade { dcOfPreviousCluster = null; dcOfCurrentCluster = null; do { - clusterHypervisorType = clusters.getString("hypervisor_type"); clusterId = clusters.getLong("id"); - if (clusterHypervisorType.equalsIgnoreCase("VMware")) { - ignoreZone = false; - clusterDetailsQuery = conn.prepareStatement("select value from `cloud`.`cluster_details` where name='url' and cluster_id=?"); - clusterDetailsQuery.setLong(1, clusterId); - clusterDetails = clusterDetailsQuery.executeQuery(); - clusterDetails.next(); - url = clusterDetails.getString("value"); - tokens = url.split("/"); // url format - http://vcenter/dc/cluster - vc = tokens[2]; - dcName = tokens[3]; - if (count > 0) { - dcOfPreviousCluster = dcOfCurrentCluster; - dcOfCurrentCluster = dcName + "@" + vc; - if (!dcOfPreviousCluster.equals(dcOfCurrentCluster)) { - legacyZone = true; - s_logger.debug("Marking the zone " + zoneId + " as legacy zone."); - } + ignoreZone = false; + clusterDetailsQuery = conn.prepareStatement("select value from `cloud`.`cluster_details` where name='url' and cluster_id=?"); + clusterDetailsQuery.setLong(1, clusterId); + clusterDetails = clusterDetailsQuery.executeQuery(); + clusterDetails.next(); + url = clusterDetails.getString("value"); + tokens = url.split("/"); // url format - http://vcenter/dc/cluster + vc = tokens[2]; + dcName = tokens[3]; + dcOfPreviousCluster = dcOfCurrentCluster; + dcOfCurrentCluster = dcName + "@" + vc; + if (count > 0) { + if (!dcOfPreviousCluster.equalsIgnoreCase(dcOfCurrentCluster)) { + legacyZone = true; + s_logger.debug("Marking the zone " + zoneId + " as legacy zone."); } - } else { - s_logger.debug("Ignoring zone " + zoneId + " with hypervisor type " + clusterHypervisorType); - break; } count++; } while (clusters.next()); + + // Ignore the zone without even one VMware cluster. if (ignoreZone) { - continue; // Ignore the zone with hypervisors other than VMware + continue; } } if (legacyZone) { listOfLegacyZones.add(zoneId); } else { + // Associate DC with the zone. assert(clusterDetails != null) : "Couldn't retrieve details of cluster!"; s_logger.debug("Discovered non-legacy zone " + zoneId + ". Processing the zone to associate with VMware datacenter."); @@ -751,18 +1233,18 @@ public class Upgrade410to420 implements DbUpgrade { pstmt = conn.prepareStatement("SELECT network_id, gateway, ip4_address FROM `cloud`.`nics` WHERE reserver_name IN ('DirectNetworkGuru','DirectPodBasedNetworkGuru') and vm_type='DomainRouter' AND removed IS null"); rs = pstmt.executeQuery(); while (rs.next()) { - Long networkId = rs.getLong(1); - String gateway = rs.getString(2); - String ip = rs.getString(3); - String uuid = UUID.randomUUID().toString(); - //Insert placeholder nic for each Domain router nic in Shared network - pstmt = conn.prepareStatement("INSERT INTO `cloud`.`nics` (uuid, ip4_address, gateway, network_id, state, strategy, vm_type) VALUES (?, ?, ?, ?, 'Reserved', 'PlaceHolder', 'DomainRouter')"); - pstmt.setString(1, uuid); - pstmt.setString(2, ip); - pstmt.setString(3, gateway); - pstmt.setLong(4, networkId); - pstmt.executeUpdate(); - s_logger.debug("Created placeholder nic for the ipAddress " + ip); + Long networkId = rs.getLong(1); + String gateway = rs.getString(2); + String ip = rs.getString(3); + String uuid = UUID.randomUUID().toString(); + //Insert placeholder nic for each Domain router nic in Shared network + pstmt = conn.prepareStatement("INSERT INTO `cloud`.`nics` (uuid, ip4_address, gateway, network_id, state, strategy, vm_type, default_nic, created) VALUES (?, ?, ?, ?, 'Reserved', 'PlaceHolder', 'DomainRouter', 0, now())"); + pstmt.setString(1, uuid); + pstmt.setString(2, ip); + pstmt.setString(3, gateway); + pstmt.setLong(4, networkId); + pstmt.executeUpdate(); + s_logger.debug("Created placeholder nic for the ipAddress " + ip + " and network " + networkId); } } catch (SQLException e) { @@ -790,14 +1272,14 @@ public class Upgrade410to420 implements DbUpgrade { rs = pstmt.executeQuery(); long id=1; while (rs.next()) { - String uuid = UUID.randomUUID().toString(); - Long ipId = rs.getLong(1); - pstmt = conn.prepareStatement("UPDATE `cloud`.`remote_access_vpn` set uuid=?, id=? where vpn_server_addr_id=?"); - pstmt.setString(1, uuid); - pstmt.setLong(2, id); - pstmt.setLong(3, ipId); - pstmt.executeUpdate(); - id++; + String uuid = UUID.randomUUID().toString(); + Long ipId = rs.getLong(1); + pstmt = conn.prepareStatement("UPDATE `cloud`.`remote_access_vpn` set uuid=?, id=? where vpn_server_addr_id=?"); + pstmt.setString(1, uuid); + pstmt.setLong(2, id); + pstmt.setLong(3, ipId); + pstmt.executeUpdate(); + id++; } } catch (SQLException e) { throw new CloudRuntimeException("Unable to update id/uuid of remote_access_vpn table", e); @@ -1154,15 +1636,15 @@ public class Upgrade410to420 implements DbUpgrade { String uuid = UUID.randomUUID().toString(); //Add internal LB VM to the list of physical network service providers pstmt = conn.prepareStatement("INSERT INTO `cloud`.`physical_network_service_providers` " + - "(uuid, physical_network_id, provider_name, state, load_balance_service_provided, destination_physical_network_id)" + - " VALUES (?, ?, 'InternalLbVm', 'Enabled', 1, 0)"); + "(uuid, physical_network_id, provider_name, state, load_balance_service_provided, destination_physical_network_id)" + + " VALUES (?, ?, 'InternalLbVm', 'Enabled', 1, 0)"); pstmt.setString(1, uuid); pstmt.setLong(2, pNtwkId); pstmt.executeUpdate(); //Add internal lb vm to the list of physical network elements PreparedStatement pstmt1 = conn.prepareStatement("SELECT id FROM `cloud`.`physical_network_service_providers`" + - " WHERE physical_network_id=? AND provider_name='InternalLbVm'"); + " WHERE physical_network_id=? AND provider_name='InternalLbVm'"); pstmt1.setLong(1, pNtwkId); ResultSet rs1 = pstmt1.executeQuery(); while (rs1.next()) { @@ -1292,7 +1774,7 @@ public class Upgrade410to420 implements DbUpgrade { // Above path should change to /snapshots/1/2/6/i-2-6-VM_ROOT-6_20121219072022 int index = backUpPath.indexOf("snapshots"+File.separator); if (index > 1){ - String correctedPath = File.separator + backUpPath.substring(index); + String correctedPath = backUpPath.substring(index); s_logger.debug("Updating Snapshot with id: "+id+" original backup path: "+backUpPath+ " updated backup path: "+correctedPath); pstmt = conn.prepareStatement("UPDATE `cloud`.`snapshots` set backup_snap_id=? where id = ?"); pstmt.setString(1, correctedPath); @@ -1413,7 +1895,7 @@ public class Upgrade410to420 implements DbUpgrade { try{ s_logger.debug("Adding F5 Big IP load balancer with host id " + hostId + " in to physical network" + physicalNetworkId); String insertF5 = "INSERT INTO `cloud`.`external_load_balancer_devices` (physical_network_id, host_id, provider_name, " + - "device_name, capacity, is_dedicated, device_state, allocation_state, is_inline, is_managed, uuid) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; + "device_name, capacity, is_dedicated, device_state, allocation_state, is_managed, uuid) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; pstmtUpdate = conn.prepareStatement(insertF5); pstmtUpdate.setLong(1, physicalNetworkId); pstmtUpdate.setLong(2, hostId); @@ -1424,8 +1906,7 @@ public class Upgrade410to420 implements DbUpgrade { pstmtUpdate.setString(7, "Enabled"); pstmtUpdate.setString(8, "Shared"); pstmtUpdate.setBoolean(9, false); - pstmtUpdate.setBoolean(10, false); - pstmtUpdate.setString(11, UUID.randomUUID().toString()); + pstmtUpdate.setString(10, UUID.randomUUID().toString()); pstmtUpdate.executeUpdate(); }catch (SQLException e) { throw new CloudRuntimeException("Exception while adding F5 load balancer device" , e); @@ -1676,26 +2157,238 @@ public class Upgrade410to420 implements DbUpgrade { } } - // migrate secondary storages (NFS, S3, Swift) from host, s3, swift tables to image_store table + // migrate secondary storages NFS from host tables to image_store table private void migrateSecondaryStorageToImageStore(Connection conn) { + PreparedStatement storeInsert = null; + PreparedStatement storeDetailInsert = null; + PreparedStatement nfsQuery = null; + PreparedStatement pstmt = null; + ResultSet rs = null; + ResultSet storeInfo = null; + + s_logger.debug("Migrating secondary storage to image store"); + boolean hasS3orSwift = false; + try { + s_logger.debug("Checking if we need to migrate NFS secondary storage to image store or staging store"); + int numRows = 0; + pstmt = conn.prepareStatement("select count(*) from `cloud`.`s3`"); + rs = pstmt.executeQuery(); + if(rs.next()){ + numRows = rs.getInt(1); + } + rs.close(); + pstmt.close(); + if (numRows > 0){ + hasS3orSwift = true; + } else{ + // check if there is swift storage + pstmt = conn.prepareStatement("select count(*) from `cloud`.`swift`"); + rs = pstmt.executeQuery(); + if(rs.next()){ + numRows = rs.getInt(1); + } + rs.close(); + pstmt.close(); + if ( numRows > 0){ + hasS3orSwift = true; + } + } + + String store_role = "Image"; + if ( hasS3orSwift){ + store_role = "ImageCache"; + } + + s_logger.debug("Migrating NFS secondary storage to " + store_role + " store"); + + storeDetailInsert = conn + .prepareStatement("INSERT INTO `cloud`.`image_store_details` (store_id, name, value) values(?, ?, ?)"); + + // migrate NFS secondary storage, for nfs, keep previous host_id as the store_id + storeInsert = conn + .prepareStatement("INSERT INTO `cloud`.`image_store` (id, uuid, name, image_provider_name, protocol, url, data_center_id, scope, role, parent, total_size, created) values(?, ?, ?, 'NFS', 'nfs', ?, ?, 'ZONE', ?, ?, ?, ?)"); + nfsQuery = conn + .prepareStatement("select id, uuid, url, data_center_id, parent, total_size, created from `cloud`.`host` where type = 'SecondaryStorage' and removed is null"); + rs = nfsQuery.executeQuery(); + + while (rs.next()) { + Long nfs_id = rs.getLong("id"); + String nfs_uuid = rs.getString("uuid"); + String nfs_url = rs.getString("url"); + String nfs_parent = rs.getString("parent"); + int nfs_dcid = rs.getInt("data_center_id"); + Long nfs_totalsize = rs.getObject("total_size") != null ? rs.getLong("total_size") : null; + Date nfs_created = rs.getDate("created"); + + // insert entry in image_store table and image_store_details + // table and store host_id and store_id mapping + storeInsert.setLong(1, nfs_id); + storeInsert.setString(2, nfs_uuid); + storeInsert.setString(3, nfs_uuid); + storeInsert.setString(4, nfs_url); + storeInsert.setInt(5, nfs_dcid); + storeInsert.setString(6, store_role); + storeInsert.setString(7, nfs_parent); + if (nfs_totalsize != null){ + storeInsert.setLong(8, nfs_totalsize); + } + else{ + storeInsert.setNull(8, Types.BIGINT); + } + storeInsert.setDate(9, nfs_created); + storeInsert.executeUpdate(); + } + } + catch (SQLException e) { + String msg = "Unable to migrate secondary storages." + e.getMessage(); + s_logger.error(msg); + throw new CloudRuntimeException(msg, e); + } finally { + try { + if (rs != null) { + rs.close(); + } + if (storeInfo != null) { + storeInfo.close(); + } + + if (storeInsert != null) { + storeInsert.close(); + } + if (storeDetailInsert != null) { + storeDetailInsert.close(); + } + if (nfsQuery != null) { + nfsQuery.close(); + } + if (pstmt != null) { + pstmt.close(); + } + } catch (SQLException e) { + } + } + s_logger.debug("Completed migrating secondary storage to image store"); + } + + // migrate volume_host_ref to volume_store_ref + private void migrateVolumeHostRef(Connection conn) { + PreparedStatement volStoreInsert = null; + PreparedStatement volStoreUpdate = null; + + s_logger.debug("Updating volume_store_ref table from volume_host_ref table"); + try { + volStoreInsert = conn + .prepareStatement("INSERT INTO `cloud`.`volume_store_ref` (store_id, volume_id, zone_id, created, last_updated, job_id, download_pct, size, physical_size, download_state, checksum, error_str, local_path, install_path, url, destroyed, update_count, ref_cnt, state) select host_id, volume_id, zone_id, created, last_updated, job_id, download_pct, size, physical_size, download_state, checksum, error_str, local_path, install_path, url, destroyed, 0, 0, 'Allocated' from `cloud`.`volume_host_ref`"); + int rowCount = volStoreInsert.executeUpdate(); + s_logger.debug("Insert modified " + rowCount + " rows"); + + volStoreUpdate = conn.prepareStatement("update `cloud`.`volume_store_ref` set state = 'Ready' where download_state = 'DOWNLOADED'"); + rowCount = volStoreUpdate.executeUpdate(); + s_logger.debug("Update modified " + rowCount + " rows"); + } + catch (SQLException e) { + String msg = "Unable to migrate volume_host_ref." + e.getMessage(); + s_logger.error(msg); + throw new CloudRuntimeException(msg, e); + } finally { + try{ + if (volStoreInsert != null) { + volStoreInsert.close(); + } + if (volStoreUpdate != null) { + volStoreUpdate.close(); + } + } catch (SQLException e) { + } + } + s_logger.debug("Completed updating volume_store_ref table from volume_host_ref table"); + } + + // migrate template_host_ref to template_store_ref + private void migrateTemplateHostRef(Connection conn) { + PreparedStatement tmplStoreInsert = null; + PreparedStatement tmplStoreUpdate = null; + + s_logger.debug("Updating template_store_ref table from template_host_ref table"); + try { + tmplStoreInsert = conn + .prepareStatement("INSERT INTO `cloud`.`template_store_ref` (store_id, template_id, created, last_updated, job_id, download_pct, size, physical_size, download_state, error_str, local_path, install_path, url, destroyed, is_copy, update_count, ref_cnt, store_role, state) select host_id, template_id, created, last_updated, job_id, download_pct, size, physical_size, download_state, error_str, local_path, install_path, url, destroyed, is_copy, 0, 0, 'Image', 'Allocated' from `cloud`.`template_host_ref`"); + int rowCount = tmplStoreInsert.executeUpdate(); + s_logger.debug("Insert modified " + rowCount + " rows"); + + tmplStoreUpdate = conn.prepareStatement("update `cloud`.`template_store_ref` set state = 'Ready' where download_state = 'DOWNLOADED'"); + rowCount = tmplStoreUpdate.executeUpdate(); + s_logger.debug("Update modified " + rowCount + " rows"); + } + catch (SQLException e) { + String msg = "Unable to migrate template_host_ref." + e.getMessage(); + s_logger.error(msg); + throw new CloudRuntimeException(msg, e); + } finally { + try{ + if (tmplStoreInsert != null) { + tmplStoreInsert.close(); + } + if (tmplStoreUpdate != null) { + tmplStoreUpdate.close(); + } + } catch (SQLException e) { + } + } + s_logger.debug("Completed updating template_store_ref table from template_host_ref table"); + } + + // migrate some entry contents of snapshots to snapshot_store_ref + private void migrateSnapshotStoreRef(Connection conn) { + PreparedStatement snapshotStoreInsert = null; + + s_logger.debug("Updating snapshot_store_ref table from snapshots table"); + try { + //Update all snapshots except KVM snapshots + snapshotStoreInsert = conn + .prepareStatement("INSERT INTO `cloud`.`snapshot_store_ref` (store_id, snapshot_id, created, size, parent_snapshot_id, install_path, volume_id, update_count, ref_cnt, store_role, state) select sechost_id, id, created, size, prev_snap_id, CONCAT('snapshots', '/', account_id, '/', volume_id, '/', backup_snap_id), volume_id, 0, 0, 'Image', 'Ready' from `cloud`.`snapshots` where status = 'BackedUp' and hypervisor_type <> 'KVM' and sechost_id is not null and removed is null"); + int rowCount = snapshotStoreInsert.executeUpdate(); + s_logger.debug("Inserted " + rowCount + " snapshots into snapshot_store_ref"); + + //backsnap_id for KVM snapshots is complate path. CONCAT is not required + snapshotStoreInsert = conn + .prepareStatement("INSERT INTO `cloud`.`snapshot_store_ref` (store_id, snapshot_id, created, size, parent_snapshot_id, install_path, volume_id, update_count, ref_cnt, store_role, state) select sechost_id, id, created, size, prev_snap_id, backup_snap_id, volume_id, 0, 0, 'Image', 'Ready' from `cloud`.`snapshots` where status = 'BackedUp' and hypervisor_type = 'KVM' and sechost_id is not null and removed is null"); + rowCount = snapshotStoreInsert.executeUpdate(); + s_logger.debug("Inserted " + rowCount + " KVM snapshots into snapshot_store_ref"); + } + catch (SQLException e) { + String msg = "Unable to migrate snapshot_store_ref." + e.getMessage(); + s_logger.error(msg); + throw new CloudRuntimeException(msg, e); + } finally { + try{ + if (snapshotStoreInsert != null) { + snapshotStoreInsert.close(); + } + } catch (SQLException e) { + } + } + s_logger.debug("Completed updating snapshot_store_ref table from snapshots table"); + } + + // migrate secondary storages S3 from s3 tables to image_store table + private void migrateS3ToImageStore(Connection conn) { PreparedStatement storeInsert = null; PreparedStatement storeDetailInsert = null; PreparedStatement storeQuery = null; PreparedStatement s3Query = null; - PreparedStatement swiftQuery = null; - PreparedStatement nfsQuery = null; ResultSet rs = null; ResultSet storeInfo = null; Long storeId = null; + Map s3_store_id_map = new HashMap(); - + s_logger.debug("Migrating S3 to image store"); try { storeQuery = conn.prepareStatement("select id from `cloud`.`image_store` where uuid = ?"); storeDetailInsert = conn .prepareStatement("INSERT INTO `cloud`.`image_store_details` (store_id, name, value) values(?, ?, ?)"); - /* - // migrate S3 secondary storage + // migrate S3 to image_store storeInsert = conn .prepareStatement("INSERT INTO `cloud`.`image_store` (uuid, name, image_provider_name, protocol, scope, role, created) values(?, ?, 'S3', ?, 'REGION', 'Image', ?)"); s3Query = conn @@ -1710,8 +2403,7 @@ public class Upgrade410to420 implements DbUpgrade { String s3_endpoint = rs.getString("end_point"); String s3_bucket = rs.getString("bucket"); boolean s3_https = rs.getObject("https") != null ? (rs.getInt("https") == 0 ? false : true) : false; - Integer s3_connectiontimeout = rs.getObject("connection_timeout") != null ? rs - .getInt("connection_timeout") : null; + Integer s3_connectiontimeout = rs.getObject("connection_timeout") != null ? rs.getInt("connection_timeout") : null; Integer s3_retry = rs.getObject("max_error_retry") != null ? rs.getInt("max_error_retry") : null; Integer s3_sockettimeout = rs.getObject("socket_timeout") != null ? rs.getInt("socket_timeout") : null; Date s3_created = rs.getDate("created"); @@ -1758,12 +2450,184 @@ public class Upgrade410to420 implements DbUpgrade { } s3_store_id_map.put(s3_id, storeId); } + } catch (SQLException e) { + String msg = "Unable to migrate S3 secondary storages." + e.getMessage(); + s_logger.error(msg); + throw new CloudRuntimeException(msg, e); + } finally { + try { + if (rs != null) { + rs.close(); + } + if (storeInfo != null) { + storeInfo.close(); + } + + if (storeInsert != null) { + storeInsert.close(); + } + if (storeDetailInsert != null) { + storeDetailInsert.close(); + } + if (storeQuery != null) { + storeQuery.close(); + } + if (s3Query != null) { + s3Query.close(); + } + } catch (SQLException e) { + } + } + + s_logger.debug("Migrating template_s3_ref to template_store_ref"); + migrateTemplateS3Ref(conn, s3_store_id_map); + + s_logger.debug("Migrating s3 backedup snapshots to snapshot_store_ref"); + migrateSnapshotS3Ref(conn, s3_store_id_map); + + s_logger.debug("Completed migrating S3 secondary storage to image store"); + } + + // migrate template_s3_ref to template_store_ref + private void migrateTemplateS3Ref(Connection conn, Map s3StoreMap) { + PreparedStatement tmplStoreInsert = null; + PreparedStatement s3Query = null; + ResultSet rs = null; + + s_logger.debug("Updating template_store_ref table from template_s3_ref table"); + try{ + tmplStoreInsert = conn + .prepareStatement("INSERT INTO `cloud`.`template_store_ref` (store_id, template_id, created, download_pct, size, physical_size, download_state, local_path, install_path, update_count, ref_cnt, store_role, state) values(?, ?, ?, 100, ?, ?, 'DOWNLOADED', '?', '?', 0, 0, 'Image', 'Ready')"); + s3Query = conn + .prepareStatement("select template_s3_ref.s3_id, template_s3_ref.template_id, template_s3_ref.created, template_s3_ref.size, template_s3_ref.physical_size, vm_template.account_id from `cloud`.`template_s3_ref`, `cloud`.`vm_template` where vm_template.id = template_s3_ref.template_id"); + rs = s3Query.executeQuery(); + + while (rs.next()) { + Long s3_id = rs.getLong("s3_id"); + Long s3_tmpl_id = rs.getLong("template_id"); + Date s3_created = rs.getDate("created"); + Long s3_size = rs.getObject("size") != null ? rs.getLong("size") : null; + Long s3_psize = rs.getObject("physical_size") != null ? rs.getLong("physical_size") : null; + Long account_id = rs.getLong("account_id"); + + tmplStoreInsert.setLong(1, s3StoreMap.get(s3_id)); + tmplStoreInsert.setLong(2, s3_tmpl_id); + tmplStoreInsert.setDate(3, s3_created); + if (s3_size != null) { + tmplStoreInsert.setLong(4, s3_size); + } else { + tmplStoreInsert.setNull(4, Types.BIGINT); + } + if (s3_psize != null) { + tmplStoreInsert.setLong(5, s3_psize); + } else { + tmplStoreInsert.setNull(5, Types.BIGINT); + } + String path = "template/tmpl/" + account_id + "/" + s3_tmpl_id; + tmplStoreInsert.setString(6, path); + tmplStoreInsert.setString(7, path); + tmplStoreInsert.executeUpdate(); + } + } + catch (SQLException e) { + String msg = "Unable to migrate template_s3_ref." + e.getMessage(); + s_logger.error(msg); + throw new CloudRuntimeException(msg, e); + } finally { + try { + if (rs != null) { + rs.close(); + } + if (tmplStoreInsert != null) { + tmplStoreInsert.close(); + } + if (s3Query != null) { + s3Query.close(); + } + } catch (SQLException e) { + } + } + s_logger.debug("Completed migrating template_s3_ref table."); + } + + // migrate some entry contents of snapshots to snapshot_store_ref + private void migrateSnapshotS3Ref(Connection conn, Map s3StoreMap) { + PreparedStatement snapshotStoreInsert = null; + PreparedStatement s3Query = null; + ResultSet rs = null; + + + s_logger.debug("Updating snapshot_store_ref table from snapshots table for s3"); + try { + snapshotStoreInsert = conn + .prepareStatement("INSERT INTO `cloud`.`snapshot_store_ref` (store_id, snapshot_id, created, size, parent_snapshot_id, install_path, volume_id, update_count, ref_cnt, store_role, state) values(?, ?, ?, ?, ?, ?, ?, 0, 0, 'Image', 'Ready')"); + s3Query = conn + .prepareStatement("select s3_id, id, created, size, prev_snap_id, CONCAT('snapshots', '/', account_id, '/', volume_id, '/', backup_snap_id), volume_id, 0, 0, 'Image', 'Ready' from `cloud`.`snapshots` where status = 'BackedUp' and hypervisor_type <> 'KVM' and s3_id is not null and removed is null"); + rs = s3Query.executeQuery(); + + while (rs.next()) { + Long s3_id = rs.getLong("s3_id"); + Long snapshot_id = rs.getLong("id"); + Date s3_created = rs.getDate("created"); + Long s3_size = rs.getObject("size") != null ? rs.getLong("size") : null; + Long s3_prev_id = rs.getObject("prev_snap_id") != null ? rs.getLong("prev_snap_id") : null; + String install_path = rs.getString(6); + Long s3_vol_id = rs.getLong("volume_id"); + + snapshotStoreInsert.setLong(1, s3StoreMap.get(s3_id)); + snapshotStoreInsert.setLong(2, snapshot_id); + snapshotStoreInsert.setDate(3, s3_created); + if (s3_size != null) { + snapshotStoreInsert.setLong(4, s3_size); + } else { + snapshotStoreInsert.setNull(4, Types.BIGINT); + } + if (s3_prev_id != null) { + snapshotStoreInsert.setLong(5, s3_prev_id); + } else { + snapshotStoreInsert.setNull(5, Types.BIGINT); + } + snapshotStoreInsert.setString(6, install_path); + snapshotStoreInsert.setLong(7, s3_vol_id); + snapshotStoreInsert.executeUpdate(); + } + } + catch (SQLException e) { + String msg = "Unable to migrate s3 backedup snapshots to snapshot_store_ref." + e.getMessage(); + s_logger.error(msg); + throw new CloudRuntimeException(msg, e); + } finally { + try{ + if (snapshotStoreInsert != null) { + snapshotStoreInsert.close(); + } + } catch (SQLException e) { + } + } + s_logger.debug("Completed updating snapshot_store_ref table from s3 snapshots entries"); + } + + // migrate secondary storages Swift from swift tables to image_store table + private void migrateSwiftToImageStore(Connection conn) { + PreparedStatement storeInsert = null; + PreparedStatement storeDetailInsert = null; + PreparedStatement storeQuery = null; + PreparedStatement swiftQuery = null; + ResultSet rs = null; + ResultSet storeInfo = null; + Long storeId = null; + Map swift_store_id_map = new HashMap(); + + s_logger.debug("Migrating Swift to image store"); + try { + storeQuery = conn.prepareStatement("select id from `cloud`.`image_store` where uuid = ?"); + storeDetailInsert = conn + .prepareStatement("INSERT INTO `cloud`.`image_store_details` (store_id, name, value) values(?, ?, ?)"); // migrate SWIFT secondary storage storeInsert = conn .prepareStatement("INSERT INTO `cloud`.`image_store` (uuid, name, image_provider_name, protocol, url, scope, role, created) values(?, ?, 'Swift', 'http', ?, 'REGION', 'Image', ?)"); - swiftQuery = conn - .prepareStatement("select id, uuid, url, account, username, key, created from `cloud`.`swift`"); + swiftQuery = conn.prepareStatement("select id, uuid, url, account, username, swift.key, created from `cloud`.`swift`"); rs = swiftQuery.executeQuery(); while (rs.next()) { @@ -1805,52 +2669,9 @@ public class Upgrade410to420 implements DbUpgrade { } swift_store_id_map.put(swift_id, storeId); } - */ - - // migrate NFS secondary storage, for nfs, keep previous host_id as the store_id - storeInsert = conn - .prepareStatement("INSERT INTO `cloud`.`image_store` (id, uuid, name, image_provider_name, protocol, url, data_center_id, scope, role, parent, total_size, created) values(?, ?, ?, 'NFS', 'nfs', ?, ?, 'ZONE', 'Image', ?, ?, ?)"); - nfsQuery = conn - .prepareStatement("select id, uuid, url, data_center_id, parent, total_size, created from `cloud`.`host` where type = 'SecondaryStorage' and removed is null"); - rs = nfsQuery.executeQuery(); - - while (rs.next()) { - Long nfs_id = rs.getLong("id"); - String nfs_uuid = rs.getString("uuid"); - String nfs_url = rs.getString("url"); - String nfs_parent = rs.getString("parent"); - int nfs_dcid = rs.getInt("data_center_id"); - Long nfs_totalsize = rs.getObject("total_size") != null ? rs.getLong("total_size") : null; - Date nfs_created = rs.getDate("created"); - - // insert entry in image_store table and image_store_details - // table and store host_id and store_id mapping - storeInsert.setLong(1, nfs_id); - storeInsert.setString(2, nfs_uuid); - storeInsert.setString(3, nfs_uuid); - storeInsert.setString(4, nfs_url); - storeInsert.setInt(5, nfs_dcid); - storeInsert.setString(6, nfs_parent); - if (nfs_totalsize != null){ - storeInsert.setLong(7, nfs_totalsize); - } - else{ - storeInsert.setNull(7, Types.BIGINT); - } - storeInsert.setDate(8, nfs_created); - storeInsert.executeUpdate(); - - storeQuery.setString(1, nfs_uuid); - storeInfo = storeQuery.executeQuery(); - if (storeInfo.next()) { - storeId = storeInfo.getLong("id"); - } - - //host_store_id_map.put(nfs_id, storeId); - } } catch (SQLException e) { - String msg = "Unable to migrate secondary storages." + e.getMessage(); + String msg = "Unable to migrate swift secondary storages." + e.getMessage(); s_logger.error(msg); throw new CloudRuntimeException(msg, e); } finally { @@ -1874,102 +2695,125 @@ public class Upgrade410to420 implements DbUpgrade { if (swiftQuery != null) { swiftQuery.close(); } - if (s3Query != null) { - s3Query.close(); - } - if (nfsQuery != null) { - nfsQuery.close(); - } } catch (SQLException e) { } } + + s_logger.debug("Migrating template_swift_ref to template_store_ref"); + migrateTemplateSwiftRef(conn, swift_store_id_map); + + s_logger.debug("Migrating swift backedup snapshots to snapshot_store_ref"); + migrateSnapshotSwiftRef(conn, swift_store_id_map); + + s_logger.debug("Completed migrating Swift secondary storage to image store"); } - // migrate volume_host_ref to volume_store_ref - private void migrateVolumeHostRef(Connection conn) { - PreparedStatement volStoreInsert = null; - PreparedStatement volStoreUpdate = null; - - try { - - volStoreInsert = conn - .prepareStatement("INSERT INTO `cloud`.`volume_store_ref` (store_id, volume_id, zone_id, created, last_updated, job_id, download_pct, size, physical_size, download_state, checksum, error_str, local_path, install_path, url, destroyed, state) select host_id, volume_id, zone_id, created, last_updated, job_id, download_pct, size, physical_size, download_state, checksum, error_str, local_path, install_path, url, destroyed, 'Allocated' from `cloud`.`volume_host_ref`"); - volStoreInsert.executeUpdate(); - - volStoreUpdate = conn.prepareStatement("update `cloud`.`volume_store_ref` set state = 'Ready' where download_state = 'DOWNLOADED'"); - volStoreUpdate.executeUpdate(); - } - catch (SQLException e) { - String msg = "Unable to migrate volume_host_ref." + e.getMessage(); - s_logger.error(msg); - throw new CloudRuntimeException(msg, e); - } finally { - try{ - if (volStoreInsert != null) { - volStoreInsert.close(); - } - if (volStoreUpdate != null) { - volStoreUpdate.close(); - } - } catch (SQLException e) { - } - } - } - - // migrate template_host_ref to template_store_ref - private void migrateTemplateHostRef(Connection conn) { + // migrate template_s3_ref to template_store_ref + private void migrateTemplateSwiftRef(Connection conn, Map swiftStoreMap) { PreparedStatement tmplStoreInsert = null; - PreparedStatement tmplStoreUpdate = null; + PreparedStatement s3Query = null; + ResultSet rs = null; + s_logger.debug("Updating template_store_ref table from template_swift_ref table"); try { - tmplStoreInsert = conn - .prepareStatement("INSERT INTO `cloud`.`template_store_ref` (store_id, template_id, created, last_updated, job_id, download_pct, size, physical_size, download_state, error_str, local_path, install_path, url, destroyed, is_copy, store_role, state) select host_id, template_id, created, last_updated, job_id, download_pct, size, physical_size, download_state, error_str, local_path, install_path, url, destroyed, is_copy, 'Image', 'Allocated' from `cloud`.`template_host_ref`"); - tmplStoreInsert.executeUpdate(); + .prepareStatement("INSERT INTO `cloud`.`template_store_ref` (store_id, template_id, created, download_pct, size, physical_size, download_state, local_path, install_path, update_count, ref_cnt, store_role, state) values(?, ?, ?, 100, ?, ?, 'DOWNLOADED', '?', '?', 0, 0, 'Image', 'Ready')"); + s3Query = conn.prepareStatement("select swift_id, template_id, created, path, size, physical_size from `cloud`.`template_swift_ref`"); + rs = s3Query.executeQuery(); - tmplStoreUpdate = conn.prepareStatement("update `cloud`.`template_store_ref` set state = 'Ready' where download_state = 'DOWNLOADED'"); - tmplStoreUpdate.executeUpdate(); - } - catch (SQLException e) { - String msg = "Unable to migrate template_host_ref." + e.getMessage(); + while (rs.next()) { + Long swift_id = rs.getLong("swift_id"); + Long tmpl_id = rs.getLong("template_id"); + Date created = rs.getDate("created"); + String path = rs.getString("path"); + Long size = rs.getObject("size") != null ? rs.getLong("size") : null; + Long psize = rs.getObject("physical_size") != null ? rs.getLong("physical_size") : null; + + tmplStoreInsert.setLong(1, swiftStoreMap.get(swift_id)); + tmplStoreInsert.setLong(2, tmpl_id); + tmplStoreInsert.setDate(3, created); + if (size != null) { + tmplStoreInsert.setLong(4, size); + } else { + tmplStoreInsert.setNull(4, Types.BIGINT); + } + if (psize != null) { + tmplStoreInsert.setLong(5, psize); + } else { + tmplStoreInsert.setNull(5, Types.BIGINT); + } + tmplStoreInsert.setString(6, path); + tmplStoreInsert.setString(7, path); + tmplStoreInsert.executeUpdate(); + } + } catch (SQLException e) { + String msg = "Unable to migrate template_swift_ref." + e.getMessage(); s_logger.error(msg); throw new CloudRuntimeException(msg, e); } finally { - try{ + try { + if (rs != null) { + rs.close(); + } if (tmplStoreInsert != null) { tmplStoreInsert.close(); } - if (tmplStoreUpdate != null) { - tmplStoreUpdate.close(); + if (s3Query != null) { + s3Query.close(); } } catch (SQLException e) { } } + s_logger.debug("Completed migrating template_swift_ref table."); } // migrate some entry contents of snapshots to snapshot_store_ref - private void migrateSnapshotStoreRef(Connection conn) { + private void migrateSnapshotSwiftRef(Connection conn, Map swiftStoreMap) { PreparedStatement snapshotStoreInsert = null; + PreparedStatement s3Query = null; + ResultSet rs = null; + s_logger.debug("Updating snapshot_store_ref table from snapshots table for swift"); try { snapshotStoreInsert = conn - .prepareStatement("INSERT INTO `cloud`.`snapshot_store_ref` (store_id, snapshot_id, created, size, parent_snapshot_id, install_path, state) select sechost_id, id, created, size, prev_snap_id, path, 'Ready' from `cloud`.`snapshots` where status = 'BackedUp' and sechost_id is not null and removed is null"); - snapshotStoreInsert.executeUpdate(); - } - catch (SQLException e) { - String msg = "Unable to migrate snapshot_store_ref." + e.getMessage(); + .prepareStatement("INSERT INTO `cloud`.`snapshot_store_ref` (store_id, snapshot_id, created, size, parent_snapshot_id, install_path, volume_id, update_count, ref_cnt, store_role, state) values(?, ?, ?, ?, ?, ?, ?, 0, 0, 'Image', 'Ready')"); + s3Query = conn + .prepareStatement("select swift_id, id, created, size, prev_snap_id, CONCAT('snapshots', '/', account_id, '/', volume_id, '/', backup_snap_id), volume_id, 0, 0, 'Image', 'Ready' from `cloud`.`snapshots` where status = 'BackedUp' and hypervisor_type <> 'KVM' and swift_id is not null and removed is null"); + rs = s3Query.executeQuery(); + + while (rs.next()) { + Long swift_id = rs.getLong("swift_id"); + Long snapshot_id = rs.getLong("id"); + Date created = rs.getDate("created"); + Long size = rs.getLong("size"); + Long prev_id = rs.getLong("prev_snap_id"); + String install_path = rs.getString(6); + Long vol_id = rs.getLong("volume_id"); + + snapshotStoreInsert.setLong(1, swiftStoreMap.get(swift_id)); + snapshotStoreInsert.setLong(2, snapshot_id); + snapshotStoreInsert.setDate(3, created); + snapshotStoreInsert.setLong(4, size); + snapshotStoreInsert.setLong(5, prev_id); + snapshotStoreInsert.setString(6, install_path); + snapshotStoreInsert.setLong(7, vol_id); + snapshotStoreInsert.executeUpdate(); + } + } catch (SQLException e) { + String msg = "Unable to migrate swift backedup snapshots to snapshot_store_ref." + e.getMessage(); s_logger.error(msg); throw new CloudRuntimeException(msg, e); } finally { - try{ + try { if (snapshotStoreInsert != null) { snapshotStoreInsert.close(); } } catch (SQLException e) { } } + s_logger.debug("Completed updating snapshot_store_ref table from swift snapshots entries"); } - + private void fixNiciraKeys(Connection conn) { //First drop the key if it exists. List keys = new ArrayList(); @@ -1994,7 +2838,7 @@ public class Upgrade410to420 implements DbUpgrade { } } } - + private void fixRouterKeys(Connection conn) { //First drop the key if it exists. List keys = new ArrayList(); @@ -2030,8 +2874,8 @@ public class Upgrade410to420 implements DbUpgrade { pstmt = conn.prepareStatement("SELECT * FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = 'cloud' AND TABLE_NAME = 'network_offerings' AND COLUMN_NAME = 'concurrent_connections'"); rs = pstmt.executeQuery(); if (!rs.next()) { - pstmt = conn.prepareStatement("ALTER TABLE `cloud`.`network_offerings` ADD COLUMN `concurrent_connections` int(10) unsigned COMMENT 'Load Balancer(haproxy) maximum number of concurrent connections(global max)'"); - pstmt.executeUpdate(); + pstmt = conn.prepareStatement("ALTER TABLE `cloud`.`network_offerings` ADD COLUMN `concurrent_connections` int(10) unsigned COMMENT 'Load Balancer(haproxy) maximum number of concurrent connections(global max)'"); + pstmt.executeUpdate(); } }catch (SQLException e) { throw new CloudRuntimeException("migration of concurrent connections from network_detais failed"); @@ -2078,4 +2922,33 @@ public class Upgrade410to420 implements DbUpgrade { } } } + + private void migrateDatafromIsoIdInVolumesTable(Connection conn) { + PreparedStatement pstmt = null; + ResultSet rs = null; + + try { + pstmt = conn.prepareStatement("SELECT iso_id1 From `cloud`.`volumes`"); + rs = pstmt.executeQuery(); + if (rs.next()) { + pstmt = conn.prepareStatement("ALTER TABLE `cloud`.`volumes` DROP COLUMN `iso_id`"); + pstmt.executeUpdate(); + pstmt = conn.prepareStatement("ALTER TABLE `cloud`.`volumes` CHANGE COLUMN `iso_id1` `iso_id` bigint(20) unsigned COMMENT 'The id of the iso from which the volume was created'"); + pstmt.executeUpdate(); + } + }catch (SQLException e) { + //implies iso_id1 is not present, so do nothing. + } + } + + protected void setRAWformatForRBDVolumes(Connection conn) { + PreparedStatement pstmt = null; + try { + s_logger.debug("Setting format to RAW for all volumes on RBD primary storage pools"); + pstmt = conn.prepareStatement("UPDATE volumes SET format = 'RAW' WHERE pool_id IN(SELECT id FROM storage_pool WHERE pool_type = 'RBD')"); + pstmt.executeUpdate(); + } catch (SQLException e) { + throw new CloudRuntimeException("Failed to update volume format to RAW for volumes on RBD pools due to exception ", e); + } + } } diff --git a/engine/schema/src/com/cloud/usage/dao/UsageDaoImpl.java b/engine/schema/src/com/cloud/usage/dao/UsageDaoImpl.java index 9dd1c59994f..8b79257d93a 100644 --- a/engine/schema/src/com/cloud/usage/dao/UsageDaoImpl.java +++ b/engine/schema/src/com/cloud/usage/dao/UsageDaoImpl.java @@ -66,7 +66,7 @@ public class UsageDaoImpl extends GenericDaoBase implements Usage private static final String UPDATE_VM_DISK_STATS = "UPDATE cloud_usage.vm_disk_statistics SET net_io_read=?, net_io_write=?, current_io_read=?, current_io_write=?, agg_io_read=?, agg_io_write=?, " + "net_bytes_read=?, net_bytes_write=?, current_bytes_read=?, current_bytes_write=?, agg_bytes_read=?, agg_bytes_write=? WHERE id=?"; private static final String INSERT_USGAE_RECORDS = "INSERT INTO cloud_usage.cloud_usage (zone_id, account_id, domain_id, description, usage_display, usage_type, raw_usage, vm_instance_id, vm_name, offering_id, template_id, " + - "usage_id, type, size, network_id, start_date, end_date, virtual_size) VALUES (?,?,?,?,?,?,?,?,?, ?, ?, ?,?,?,?,?,?)"; + "usage_id, type, size, network_id, start_date, end_date, virtual_size) VALUES (?,?,?,?,?,?,?,?,?, ?, ?, ?,?,?,?,?,?,?)"; protected final static TimeZone s_gmtTimeZone = TimeZone.getTimeZone("GMT"); diff --git a/engine/schema/src/com/cloud/user/AccountVO.java b/engine/schema/src/com/cloud/user/AccountVO.java index 77110aedb95..4a7e73bb304 100644 --- a/engine/schema/src/com/cloud/user/AccountVO.java +++ b/engine/schema/src/com/cloud/user/AccountVO.java @@ -65,7 +65,7 @@ public class AccountVO implements Account { @Column(name="default_zone_id") private Long defaultZoneId = null; - + @Column(name = "default") boolean isDefault; @@ -77,7 +77,7 @@ public class AccountVO implements Account { this.id = id; this.uuid = UUID.randomUUID().toString(); } - + public AccountVO(String accountName, long domainId, String networkDomain, short type, String uuid) { this.accountName = accountName; this.domainId = domainId; @@ -161,7 +161,7 @@ public class AccountVO implements Account { @Override public String toString() { - return new StringBuilder("Acct[").append(id).append("-").append(accountName).append("]").toString(); + return new StringBuilder("Acct[").append(uuid).append("-").append(accountName).append("]").toString(); } @Override diff --git a/engine/schema/src/com/cloud/vm/dao/NicDao.java b/engine/schema/src/com/cloud/vm/dao/NicDao.java index 37249dd909e..79bd4d2b3d6 100644 --- a/engine/schema/src/com/cloud/vm/dao/NicDao.java +++ b/engine/schema/src/com/cloud/vm/dao/NicDao.java @@ -71,4 +71,6 @@ public interface NicDao extends GenericDao { NicVO findByInstanceIdAndIpAddressAndVmtype(long instanceId, String ipaddress, VirtualMachine.Type type); List listByNetworkIdTypeAndGatewayAndBroadcastUri(long networkId, VirtualMachine.Type vmType, String gateway, URI broadcastUri); + + int countNicsForStartingVms(long networkId); } diff --git a/engine/schema/src/com/cloud/vm/dao/NicDaoImpl.java b/engine/schema/src/com/cloud/vm/dao/NicDaoImpl.java index d6433a45ffb..a7b7625e39c 100644 --- a/engine/schema/src/com/cloud/vm/dao/NicDaoImpl.java +++ b/engine/schema/src/com/cloud/vm/dao/NicDaoImpl.java @@ -16,8 +16,18 @@ // under the License. package com.cloud.vm.dao; +import java.net.URI; +import java.util.List; + +import javax.annotation.PostConstruct; +import javax.ejb.Local; +import javax.inject.Inject; + +import org.springframework.stereotype.Component; + import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.GenericSearchBuilder; +import com.cloud.utils.db.JoinBuilder; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.SearchCriteria.Func; @@ -25,25 +35,27 @@ import com.cloud.utils.db.SearchCriteria.Op; import com.cloud.vm.Nic; import com.cloud.vm.Nic.State; import com.cloud.vm.NicVO; +import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; -import org.springframework.stereotype.Component; - -import javax.ejb.Local; -import java.net.URI; -import java.util.List; @Component @Local(value=NicDao.class) public class NicDaoImpl extends GenericDaoBase implements NicDao { - private final SearchBuilder AllFieldsSearch; - private final GenericSearchBuilder IpSearch; - private final SearchBuilder NonReleasedSearch; - final GenericSearchBuilder CountBy; + private SearchBuilder AllFieldsSearch; + private GenericSearchBuilder IpSearch; + private SearchBuilder NonReleasedSearch; + private GenericSearchBuilder CountBy; + private GenericSearchBuilder CountByForStartingVms; + @Inject + VMInstanceDao _vmDao; - public NicDaoImpl() { - super(); + public NicDaoImpl() { + } + + @PostConstruct + protected void init() { AllFieldsSearch = createSearchBuilder(); AllFieldsSearch.and("instance", AllFieldsSearch.entity().getInstanceId(), Op.EQ); AllFieldsSearch.and("network", AllFieldsSearch.entity().getNetworkId(), Op.EQ); @@ -54,6 +66,7 @@ public class NicDaoImpl extends GenericDaoBase implements NicDao { AllFieldsSearch.and("broadcastUri", AllFieldsSearch.entity().getBroadcastUri(), Op.EQ); AllFieldsSearch.and("secondaryip", AllFieldsSearch.entity().getSecondaryIp(), Op.EQ); AllFieldsSearch.and("nicid", AllFieldsSearch.entity().getId(), Op.EQ); + AllFieldsSearch.and("strategy", AllFieldsSearch.entity().getReservationStrategy(), Op.EQ); AllFieldsSearch.done(); IpSearch = createSearchBuilder(String.class); @@ -73,6 +86,15 @@ public class NicDaoImpl extends GenericDaoBase implements NicDao { CountBy.and("vmId", CountBy.entity().getInstanceId(), Op.EQ); CountBy.and("removed", CountBy.entity().getRemoved(), Op.NULL); CountBy.done(); + + CountByForStartingVms = createSearchBuilder(Integer.class); + CountByForStartingVms.select(null, Func.COUNT, CountByForStartingVms.entity().getId()); + CountByForStartingVms.and("networkId", CountByForStartingVms.entity().getNetworkId(), Op.EQ); + CountByForStartingVms.and("removed", CountByForStartingVms.entity().getRemoved(), Op.NULL); + SearchBuilder join1 = _vmDao.createSearchBuilder(); + join1.and("state", join1.entity().getState(), Op.EQ); + CountByForStartingVms.join("vm", join1, CountByForStartingVms.entity().getInstanceId(), join1.entity().getId(), JoinBuilder.JoinType.INNER); + CountByForStartingVms.done(); } @Override @@ -256,4 +278,13 @@ public class NicDaoImpl extends GenericDaoBase implements NicDao { return listBy(sc); } + @Override + public int countNicsForStartingVms(long networkId) { + SearchCriteria sc = CountByForStartingVms.create(); + sc.setParameters("networkId", networkId); + sc.setJoinParameters("vm", "state", VirtualMachine.State.Starting); + List results = customSearch(sc, null); + return results.get(0); + } + } diff --git a/engine/schema/src/com/cloud/vm/dao/VMInstanceDaoImpl.java b/engine/schema/src/com/cloud/vm/dao/VMInstanceDaoImpl.java index bc3efbbf854..3a7dde78a6d 100644 --- a/engine/schema/src/com/cloud/vm/dao/VMInstanceDaoImpl.java +++ b/engine/schema/src/com/cloud/vm/dao/VMInstanceDaoImpl.java @@ -228,6 +228,19 @@ public class VMInstanceDaoImpl extends GenericDaoBase implem _updateTimeAttr = _allAttributes.get("updateTime"); assert _updateTimeAttr != null : "Couldn't get this updateTime attribute"; + + SearchBuilder nicSearch = _nicDao.createSearchBuilder(); + nicSearch.and("networkId", nicSearch.entity().getNetworkId(), SearchCriteria.Op.EQ); + + DistinctHostNameSearch = createSearchBuilder(String.class); + DistinctHostNameSearch.selectField(DistinctHostNameSearch.entity().getHostName()); + + DistinctHostNameSearch.and("types", DistinctHostNameSearch.entity().getType(), SearchCriteria.Op.IN); + DistinctHostNameSearch.and("removed", DistinctHostNameSearch.entity().getRemoved(), SearchCriteria.Op.NULL); + DistinctHostNameSearch.join("nicSearch", nicSearch, DistinctHostNameSearch.entity().getId(), + nicSearch.entity().getInstanceId(), JoinBuilder.JoinType.INNER); + DistinctHostNameSearch.done(); + } @Override @@ -629,21 +642,6 @@ public class VMInstanceDaoImpl extends GenericDaoBase implem @Override public List listDistinctHostNames(long networkId, VirtualMachine.Type... types) { - if (DistinctHostNameSearch == null) { - - SearchBuilder nicSearch = _nicDao.createSearchBuilder(); - nicSearch.and("networkId", nicSearch.entity().getNetworkId(), SearchCriteria.Op.EQ); - - DistinctHostNameSearch = createSearchBuilder(String.class); - DistinctHostNameSearch.selectField(DistinctHostNameSearch.entity().getHostName()); - - DistinctHostNameSearch.and("types", DistinctHostNameSearch.entity().getType(), SearchCriteria.Op.IN); - DistinctHostNameSearch.and("removed", DistinctHostNameSearch.entity().getRemoved(), SearchCriteria.Op.NULL); - DistinctHostNameSearch.join("nicSearch", nicSearch, DistinctHostNameSearch.entity().getId(), - nicSearch.entity().getInstanceId(), JoinBuilder.JoinType.INNER); - DistinctHostNameSearch.done(); - } - SearchCriteria sc = DistinctHostNameSearch.create(); if (types != null && types.length != 0) { sc.setParameters("types", (Object[]) types); diff --git a/engine/schema/src/org/apache/cloudstack/affinity/AffinityGroupDomainMapVO.java b/engine/schema/src/org/apache/cloudstack/affinity/AffinityGroupDomainMapVO.java new file mode 100644 index 00000000000..ea37194a117 --- /dev/null +++ b/engine/schema/src/org/apache/cloudstack/affinity/AffinityGroupDomainMapVO.java @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.affinity; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; + +import com.cloud.domain.PartOf; +import org.apache.cloudstack.api.InternalIdentity; + +@Entity +@Table(name = "affinity_group_domain_map") +public class AffinityGroupDomainMapVO implements PartOf, InternalIdentity { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + long id; + + @Column(name = "domain_id") + long domainId; + + @Column(name = "affinity_group_id") + private long affinityGroupId; + + @Column(name = "subdomain_access") + public Boolean subdomainAccess; + + protected AffinityGroupDomainMapVO() { + } + + public AffinityGroupDomainMapVO(long affinityGroupId, long domainId, Boolean subdomainAccess) { + this.affinityGroupId = affinityGroupId; + this.domainId = domainId; + this.subdomainAccess = subdomainAccess; + } + + @Override + public long getId() { + return id; + } + + @Override + public long getDomainId() { + return domainId; + } + + public long getAffinityGroupId() { + return affinityGroupId; + } + + public Boolean isSubdomainAccess() { + return subdomainAccess; + } + +} diff --git a/engine/schema/src/org/apache/cloudstack/affinity/AffinityGroupVO.java b/engine/schema/src/org/apache/cloudstack/affinity/AffinityGroupVO.java index f418cefd781..44f8dd85c5b 100644 --- a/engine/schema/src/org/apache/cloudstack/affinity/AffinityGroupVO.java +++ b/engine/schema/src/org/apache/cloudstack/affinity/AffinityGroupVO.java @@ -20,11 +20,14 @@ import java.util.UUID; import javax.persistence.Column; import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; import javax.persistence.GeneratedValue; import javax.persistence.GenerationType; import javax.persistence.Id; import javax.persistence.Table; +import org.apache.cloudstack.acl.ControlledEntity; @Entity @Table(name = ("affinity_group")) @@ -52,17 +55,22 @@ public class AffinityGroupVO implements AffinityGroup { @Column(name = "uuid") private String uuid; + @Column(name = "acl_type") + @Enumerated(value = EnumType.STRING) + ControlledEntity.ACLType aclType; + public AffinityGroupVO() { this.uuid = UUID.randomUUID().toString(); } - public AffinityGroupVO(String name, String type, String description, long domainId, long accountId) { + public AffinityGroupVO(String name, String type, String description, long domainId, long accountId, ACLType aclType) { this.name = name; this.description = description; this.domainId = domainId; this.accountId = accountId; this.uuid = UUID.randomUUID().toString(); this.type = type; + this.aclType = aclType; } @Override @@ -104,6 +112,11 @@ public class AffinityGroupVO implements AffinityGroup { return type; } + @Override + public ControlledEntity.ACLType getAclType() { + return aclType; + } + @Override public String toString() { StringBuilder buf = new StringBuilder("AffinityGroup["); diff --git a/engine/schema/src/org/apache/cloudstack/affinity/dao/AffinityGroupDao.java b/engine/schema/src/org/apache/cloudstack/affinity/dao/AffinityGroupDao.java index 296e7b1d043..6310a2f66b4 100644 --- a/engine/schema/src/org/apache/cloudstack/affinity/dao/AffinityGroupDao.java +++ b/engine/schema/src/org/apache/cloudstack/affinity/dao/AffinityGroupDao.java @@ -18,6 +18,7 @@ package org.apache.cloudstack.affinity.dao; import java.util.List; +import org.apache.cloudstack.affinity.AffinityGroup; import org.apache.cloudstack.affinity.AffinityGroupVO; import com.cloud.utils.db.GenericDao; @@ -26,5 +27,12 @@ public interface AffinityGroupDao extends GenericDao { boolean isNameInUse(Long accountId, Long domainId, String name); AffinityGroupVO findByAccountAndName(Long accountId, String name); List findByAccountAndNames(Long accountId, String... names); - int removeByAccountId(long accountId); + + int removeByAccountId(long accountId); + + AffinityGroup findDomainLevelGroupByName(Long domainId, String affinityGroupName); + + AffinityGroup findByAccountAndType(Long accountId, String string); + + AffinityGroup findDomainLevelGroupByType(Long domainId, String string); } diff --git a/engine/schema/src/org/apache/cloudstack/affinity/dao/AffinityGroupDaoImpl.java b/engine/schema/src/org/apache/cloudstack/affinity/dao/AffinityGroupDaoImpl.java index d189d609ff2..0be00887caa 100644 --- a/engine/schema/src/org/apache/cloudstack/affinity/dao/AffinityGroupDaoImpl.java +++ b/engine/schema/src/org/apache/cloudstack/affinity/dao/AffinityGroupDaoImpl.java @@ -20,18 +20,29 @@ import java.util.List; import javax.annotation.PostConstruct; import javax.ejb.Local; +import javax.inject.Inject; + +import org.apache.cloudstack.acl.ControlledEntity; +import org.apache.cloudstack.affinity.AffinityGroup; +import org.apache.cloudstack.affinity.AffinityGroupDomainMapVO; import org.apache.cloudstack.affinity.AffinityGroupVO; -import org.springframework.stereotype.Component; + import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.db.JoinBuilder.JoinType; @Local(value = { AffinityGroupDao.class }) public class AffinityGroupDaoImpl extends GenericDaoBase implements AffinityGroupDao { private SearchBuilder AccountIdSearch; private SearchBuilder AccountIdNameSearch; private SearchBuilder AccountIdNamesSearch; - + private SearchBuilder DomainLevelNameSearch; + private SearchBuilder AccountIdTypeSearch; + @Inject + AffinityGroupDomainMapDao _groupDomainDao; + + private SearchBuilder DomainLevelTypeSearch; public AffinityGroupDaoImpl() { @@ -51,6 +62,30 @@ public class AffinityGroupDaoImpl extends GenericDaoBase AccountIdNamesSearch.and("accountId", AccountIdNamesSearch.entity().getAccountId(), SearchCriteria.Op.EQ); AccountIdNamesSearch.and("groupNames", AccountIdNamesSearch.entity().getName(), SearchCriteria.Op.IN); AccountIdNameSearch.done(); + + SearchBuilder domainMapSearch = _groupDomainDao.createSearchBuilder(); + domainMapSearch.and("domainId", domainMapSearch.entity().getDomainId(), SearchCriteria.Op.EQ); + + DomainLevelNameSearch = createSearchBuilder(); + DomainLevelNameSearch.and("name", DomainLevelNameSearch.entity().getName(), SearchCriteria.Op.EQ); + DomainLevelNameSearch.and("aclType", DomainLevelNameSearch.entity().getAclType(), SearchCriteria.Op.EQ); + DomainLevelNameSearch.join("domainMapSearch", domainMapSearch, domainMapSearch.entity().getAffinityGroupId(), + DomainLevelNameSearch.entity().getId(), JoinType.INNER); + DomainLevelNameSearch.done(); + + AccountIdTypeSearch = createSearchBuilder(); + AccountIdTypeSearch.and("accountId", AccountIdTypeSearch.entity().getAccountId(), SearchCriteria.Op.EQ); + AccountIdTypeSearch.and("type", AccountIdTypeSearch.entity().getType(), SearchCriteria.Op.EQ); + + SearchBuilder domainTypeSearch = _groupDomainDao.createSearchBuilder(); + domainTypeSearch.and("domainId", domainTypeSearch.entity().getDomainId(), SearchCriteria.Op.EQ); + DomainLevelTypeSearch = createSearchBuilder(); + DomainLevelTypeSearch.and("type", DomainLevelTypeSearch.entity().getType(), SearchCriteria.Op.EQ); + DomainLevelTypeSearch.and("aclType", DomainLevelTypeSearch.entity().getAclType(), SearchCriteria.Op.EQ); + DomainLevelTypeSearch.join("domainTypeSearch", domainTypeSearch, + domainTypeSearch.entity().getAffinityGroupId(), + DomainLevelTypeSearch.entity().getId(), JoinType.INNER); + DomainLevelTypeSearch.done(); } @Override @@ -99,4 +134,31 @@ public class AffinityGroupDaoImpl extends GenericDaoBase sc.setParameters("accountId", accountId); return expunge(sc); } + + @Override + public AffinityGroup findDomainLevelGroupByName(Long domainId, String affinityGroupName) { + SearchCriteria sc = DomainLevelNameSearch.create(); + sc.setParameters("aclType", ControlledEntity.ACLType.Domain); + sc.setParameters("name", affinityGroupName); + sc.setJoinParameters("domainMapSearch", "domainId", domainId); + return findOneBy(sc); + } + + @Override + public AffinityGroup findByAccountAndType(Long accountId, String type) { + SearchCriteria sc = AccountIdTypeSearch.create(); + sc.setParameters("accountId", accountId); + sc.setParameters("type", type); + + return findOneBy(sc); + } + + @Override + public AffinityGroup findDomainLevelGroupByType(Long domainId, String type) { + SearchCriteria sc = DomainLevelTypeSearch.create(); + sc.setParameters("aclType", ControlledEntity.ACLType.Domain); + sc.setParameters("type", type); + sc.setJoinParameters("domainTypeSearch", "domainId", domainId); + return findOneBy(sc); + } } diff --git a/plugins/hypervisors/simulator/src/com/cloud/server/ManagementServerSimulatorImpl.java b/engine/schema/src/org/apache/cloudstack/affinity/dao/AffinityGroupDomainMapDao.java similarity index 68% rename from plugins/hypervisors/simulator/src/com/cloud/server/ManagementServerSimulatorImpl.java rename to engine/schema/src/org/apache/cloudstack/affinity/dao/AffinityGroupDomainMapDao.java index 728271c41f0..07be976f202 100644 --- a/plugins/hypervisors/simulator/src/com/cloud/server/ManagementServerSimulatorImpl.java +++ b/engine/schema/src/org/apache/cloudstack/affinity/dao/AffinityGroupDomainMapDao.java @@ -14,18 +14,18 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -package com.cloud.server; - +package org.apache.cloudstack.affinity.dao; import java.util.List; -import com.cloud.api.commands.ConfigureSimulator; +import org.apache.cloudstack.affinity.AffinityGroupDomainMapVO; + +import com.cloud.utils.db.GenericDao; + +public interface AffinityGroupDomainMapDao extends GenericDao { + + AffinityGroupDomainMapVO findByAffinityGroup(long affinityGroupId); + + List listByDomain(Object... domainId); -public class ManagementServerSimulatorImpl extends ManagementServerImpl { - @Override - public List> getCommands() { - List> cmdList = super.getCommands(); - cmdList.add(ConfigureSimulator.class); - return cmdList; - } } diff --git a/engine/schema/src/org/apache/cloudstack/affinity/dao/AffinityGroupDomainMapDaoImpl.java b/engine/schema/src/org/apache/cloudstack/affinity/dao/AffinityGroupDomainMapDaoImpl.java new file mode 100644 index 00000000000..3777ef5399a --- /dev/null +++ b/engine/schema/src/org/apache/cloudstack/affinity/dao/AffinityGroupDomainMapDaoImpl.java @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.affinity.dao; + +import java.util.List; + +import javax.annotation.PostConstruct; +import org.apache.cloudstack.affinity.AffinityGroupDomainMapVO; + +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.db.SearchCriteria.Op; + +public class AffinityGroupDomainMapDaoImpl extends GenericDaoBase implements + AffinityGroupDomainMapDao { + + private SearchBuilder ListByAffinityGroup; + + private SearchBuilder DomainsSearch; + + + public AffinityGroupDomainMapDaoImpl() { + } + + @PostConstruct + protected void init() { + ListByAffinityGroup = createSearchBuilder(); + ListByAffinityGroup.and("affinityGroupId", ListByAffinityGroup.entity().getAffinityGroupId(), + SearchCriteria.Op.EQ); + ListByAffinityGroup.done(); + + DomainsSearch = createSearchBuilder(); + DomainsSearch.and("domainId", DomainsSearch.entity().getDomainId(), Op.IN); + DomainsSearch.done(); + } + + @Override + public AffinityGroupDomainMapVO findByAffinityGroup(long affinityGroupId) { + SearchCriteria sc = ListByAffinityGroup.create(); + sc.setParameters("affinityGroupId", affinityGroupId); + return findOneBy(sc); + } + + @Override + public List listByDomain(Object... domainId) { + SearchCriteria sc = DomainsSearch.create(); + sc.setParameters("domainId", (Object[]) domainId); + + return listBy(sc); + } + +} diff --git a/engine/schema/src/org/apache/cloudstack/region/RegionVO.java b/engine/schema/src/org/apache/cloudstack/region/RegionVO.java index 6890bc850a8..36db8dd13b0 100644 --- a/engine/schema/src/org/apache/cloudstack/region/RegionVO.java +++ b/engine/schema/src/org/apache/cloudstack/region/RegionVO.java @@ -35,7 +35,12 @@ public class RegionVO implements Region{ @Column(name="end_point") private String endPoint; - + + @Column(name="gslb_service_enabled") + private boolean gslbEnabled; + + @Column(name="portableip_service_enabled") + private boolean portableipEnabled; public boolean getGslbEnabled() { return gslbEnabled; @@ -45,9 +50,6 @@ public class RegionVO implements Region{ this.gslbEnabled = gslbEnabled; } - @Column(name="gslb_service_enabled") - private boolean gslbEnabled; - public RegionVO() { } @@ -78,10 +80,35 @@ public class RegionVO implements Region{ this.endPoint = endPoint; } - @Override public boolean checkIfServiceEnabled(Service service) { - return gslbEnabled; + if (Service.Gslb.equals(service)) { + return gslbEnabled; + } else if (Service.PortableIp.equals(service)) { + return portableipEnabled; + } else { + assert false: "Unknown Region level Service"; + return false; + } } + @Override + public void enableService(org.apache.cloudstack.region.Region.Service service) { + if (Service.Gslb.equals(service)) { + this.gslbEnabled = true; + } else if (Service.PortableIp.equals(service)) { + this.portableipEnabled = true; + } else { + assert false: "Unknown Region level Service"; + return; + } + } + + public boolean getPortableipEnabled() { + return portableipEnabled; + } + + public void setPortableipEnabled(boolean portableipEnabled) { + this.portableipEnabled = portableipEnabled; + } } diff --git a/engine/service/pom.xml b/engine/service/pom.xml index 47c0edcefc0..970e90117c6 100644 --- a/engine/service/pom.xml +++ b/engine/service/pom.xml @@ -23,7 +23,7 @@ org.apache.cloudstack cloud-engine - 4.2.0-SNAPSHOT + 4.2.0 cloud-engine-service war diff --git a/engine/storage/cache/pom.xml b/engine/storage/cache/pom.xml index f00f6cd1498..0edee9b6e2d 100644 --- a/engine/storage/cache/pom.xml +++ b/engine/storage/cache/pom.xml @@ -16,7 +16,7 @@ org.apache.cloudstack cloud-engine - 4.2.0-SNAPSHOT + 4.2.0 ../../pom.xml diff --git a/engine/storage/datamotion/pom.xml b/engine/storage/datamotion/pom.xml index 8a3698c94d3..3d107f99d40 100644 --- a/engine/storage/datamotion/pom.xml +++ b/engine/storage/datamotion/pom.xml @@ -16,7 +16,7 @@ org.apache.cloudstack cloud-engine - 4.2.0-SNAPSHOT + 4.2.0 ../../pom.xml diff --git a/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java b/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java index e17306a2b9c..4aa01476527 100644 --- a/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java +++ b/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java @@ -79,7 +79,8 @@ import com.cloud.utils.db.DB; import com.cloud.utils.exception.CloudRuntimeException; @Component -public class AncientDataMotionStrategy implements DataMotionStrategy { +public class + AncientDataMotionStrategy implements DataMotionStrategy { private static final Logger s_logger = Logger.getLogger(AncientDataMotionStrategy.class); @Inject EndPointSelector selector; @@ -176,17 +177,31 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { return zoneScope; } + private Scope pickCacheScopeForCopy(DataObject srcData, DataObject destData) { + Scope srcScope = srcData.getDataStore().getScope(); + Scope destScope = destData.getDataStore().getScope(); + + Scope selectedScope = null; + if (srcScope.getScopeId() != null) { + selectedScope = getZoneScope(srcScope); + } else if (destScope.getScopeId() != null) { + selectedScope = getZoneScope(destScope); + } else { + s_logger.warn("Cannot find a zone-wide scope for movement that needs a cache storage"); + } + return selectedScope; + } + protected Answer copyObject(DataObject srcData, DataObject destData) { String value = configDao.getValue(Config.PrimaryStorageDownloadWait.toString()); int _primaryStorageDownloadWait = NumbersUtil.parseInt(value, Integer.parseInt(Config.PrimaryStorageDownloadWait.getDefaultValue())); Answer answer = null; - boolean usingCache = false; DataObject cacheData = null; DataObject srcForCopy = srcData; try { if (needCacheStorage(srcData, destData)) { - Scope destScope = getZoneScope(destData.getDataStore().getScope()); + Scope destScope = pickCacheScopeForCopy(srcData, destData); srcForCopy = cacheData = cacheMgr.createCacheObject(srcData, destScope); } @@ -195,15 +210,21 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { answer = ep.sendMessage(cmd); if (cacheData != null) { - if (answer == null || !answer.getResult()) { + if (srcData.getType() == DataObjectType.VOLUME && destData.getType() == DataObjectType.VOLUME) { + // volume transfer from primary to secondary or vice versa. Volume transfer between primary pools are already handled by copyVolumeBetweenPools cacheMgr.deleteCacheObject(srcForCopy); } else { - cacheMgr.releaseCacheObject(srcForCopy); + // for template, we want to leave it on cache for performance reason + if (answer == null || !answer.getResult()) { + cacheMgr.deleteCacheObject(srcForCopy); + } else { + cacheMgr.releaseCacheObject(srcForCopy); + } } } return answer; } catch (Exception e) { - s_logger.debug("copy object failed: " + e.toString()); + s_logger.debug("copy object failed: ", e); if (cacheData != null) { cacheMgr.deleteCacheObject(cacheData); } @@ -328,6 +349,10 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { CopyCommand cmd = new CopyCommand(cacheData.getTO(), destData.getTO(), _copyvolumewait, _mgmtServer.getExecuteInSequence()); EndPoint ep = selector.select(cacheData, destData); Answer answer = ep.sendMessage(cmd); + // delete volume on cache store + if (cacheData != null) { + cacheMgr.deleteCacheObject(cacheData); + } return answer; } @@ -426,7 +451,8 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { Answer answer = null; try { if (needCacheStorage(srcData, destData)) { - cacheData = cacheMgr.getCacheObject(srcData, destData.getDataStore().getScope()); + Scope selectedScope = pickCacheScopeForCopy(srcData, destData); + cacheData = cacheMgr.getCacheObject(srcData, selectedScope); CopyCommand cmd = new CopyCommand(srcData.getTO(), destData.getTO(), _backupsnapshotwait, _mgmtServer.getExecuteInSequence()); cmd.setCacheTO(cacheData.getTO()); @@ -437,26 +463,25 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { EndPoint ep = selector.select(srcData, destData); answer = ep.sendMessage(cmd); } - // clean up cache entry in case of failure - if (answer == null || !answer.getResult()) { - if (cacheData != null) { - cacheMgr.deleteCacheObject(cacheData); - } - } - return answer; - } catch (Exception e) { - s_logger.debug("copy snasphot failed: " + e.toString()); + if (cacheData != null) { cacheMgr.deleteCacheObject(cacheData); } - throw new CloudRuntimeException(e.toString()); + + return answer; + } catch (Exception e) { + s_logger.debug("copy snasphot failed: ", e); + if (cacheData != null) { + cacheMgr.deleteCacheObject(cacheData); + } + throw new CloudRuntimeException(e); } } @Override public Void copyAsync(Map volumeMap, VirtualMachineTO vmTo, Host srcHost, Host destHost, - AsyncCompletionCallback callback) { + AsyncCompletionCallback callback) { CopyCommandResult result = new CopyCommandResult(null, null); result.setResult("Unsupported operation requested for copying data."); callback.complete(result); diff --git a/engine/storage/image/pom.xml b/engine/storage/image/pom.xml index c4d2d1b2542..d7ba5279937 100644 --- a/engine/storage/image/pom.xml +++ b/engine/storage/image/pom.xml @@ -16,7 +16,7 @@ org.apache.cloudstack cloud-engine - 4.2.0-SNAPSHOT + 4.2.0 ../../pom.xml diff --git a/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java b/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java index 9eef3992e3e..0113bc0cfcd 100644 --- a/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java +++ b/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java @@ -73,6 +73,7 @@ import com.cloud.exception.ResourceAllocationException; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.storage.DataStoreRole; import com.cloud.storage.StoragePool; +import com.cloud.storage.Storage.TemplateType; import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.VMTemplateZoneVO; @@ -86,6 +87,7 @@ import com.cloud.user.Account; import com.cloud.user.AccountManager; import com.cloud.user.ResourceLimitService; import com.cloud.utils.UriUtils; +import com.cloud.utils.db.GlobalLock; import com.cloud.utils.exception.CloudRuntimeException; @Component @@ -238,6 +240,8 @@ public class TemplateServiceImpl implements TemplateService { TemplateDataStoreVO tmpltHost = _vmTemplateStoreDao .findByStoreTemplate(store.getId(), template.getId()); if (tmpltHost == null || tmpltHost.getState() != ObjectInDataStoreStateMachine.State.Ready) { + associateTemplateToZone(template.getId(), dcId); + s_logger.info("Downloading builtin template " + template.getUniqueName() + " to data center: " + dcId); TemplateInfo tmplt = _templateFactory.getTemplate(template.getId(), DataStoreRole.Image); createTemplateAsync(tmplt, store, null); } @@ -252,184 +256,208 @@ public class TemplateServiceImpl implements TemplateService { return; } long storeId = store.getId(); - Long zoneId = store.getScope().getScopeId(); - Map templateInfos = listTemplate(store); - if (templateInfos == null) { - return; - } + // add lock to make template sync for a data store only be done once + String lockString = "templatesync.storeId:" + storeId; + GlobalLock syncLock = GlobalLock.getInternLock(lockString); + try { + if (syncLock.lock(3)) { + try{ + Long zoneId = store.getScope().getScopeId(); - Set toBeDownloaded = new HashSet(); - List allTemplates = null; - if (zoneId == null) { - // region wide store - allTemplates = _templateDao.listAllActive(); - } else { - // zone wide store - allTemplates = _templateDao.listAllInZone(zoneId); - } - List rtngTmplts = _templateDao.listAllSystemVMTemplates(); - List defaultBuiltin = _templateDao.listDefaultBuiltinTemplates(); - - if (rtngTmplts != null) { - for (VMTemplateVO rtngTmplt : rtngTmplts) { - if (!allTemplates.contains(rtngTmplt)) { - allTemplates.add(rtngTmplt); - } - } - } - - if (defaultBuiltin != null) { - for (VMTemplateVO builtinTmplt : defaultBuiltin) { - if (!allTemplates.contains(builtinTmplt)) { - allTemplates.add(builtinTmplt); - } - } - } - - toBeDownloaded.addAll(allTemplates); - - for (VMTemplateVO tmplt : allTemplates) { - String uniqueName = tmplt.getUniqueName(); - TemplateDataStoreVO tmpltStore = _vmTemplateStoreDao.findByStoreTemplate(storeId, tmplt.getId()); - if (templateInfos.containsKey(uniqueName)) { - TemplateProp tmpltInfo = templateInfos.remove(uniqueName); - toBeDownloaded.remove(tmplt); - if (tmpltStore != null) { - s_logger.info("Template Sync found " + uniqueName + " already in the image store"); - if (tmpltStore.getDownloadState() != Status.DOWNLOADED) { - tmpltStore.setErrorString(""); + Map templateInfos = listTemplate(store); + if (templateInfos == null) { + return; } - if (tmpltInfo.isCorrupted()) { - tmpltStore.setDownloadState(Status.DOWNLOAD_ERROR); - String msg = "Template " + tmplt.getName() + ":" + tmplt.getId() - + " is corrupted on secondary storage " + tmpltStore.getId(); - tmpltStore.setErrorString(msg); - s_logger.info("msg"); - if (tmplt.getUrl() == null) { - msg = "Private Template (" + tmplt + ") with install path " + tmpltInfo.getInstallPath() - + "is corrupted, please check in image store: " + tmpltStore.getDataStoreId(); - s_logger.warn(msg); - } else { - toBeDownloaded.add(tmplt); - } + Set toBeDownloaded = new HashSet(); + List allTemplates = null; + if (zoneId == null) { + // region wide store + allTemplates = _templateDao.listAllActive(); } else { - tmpltStore.setDownloadPercent(100); - tmpltStore.setDownloadState(Status.DOWNLOADED); - tmpltStore.setInstallPath(tmpltInfo.getInstallPath()); - tmpltStore.setSize(tmpltInfo.getSize()); - tmpltStore.setPhysicalSize(tmpltInfo.getPhysicalSize()); - tmpltStore.setLastUpdated(new Date()); - // update size in vm_template table - VMTemplateVO tmlpt = _templateDao.findById(tmplt.getId()); - tmlpt.setSize(tmpltInfo.getSize()); - _templateDao.update(tmplt.getId(), tmlpt); + // zone wide store + allTemplates = _templateDao.listAllInZone(zoneId); + } + List rtngTmplts = _templateDao.listAllSystemVMTemplates(); + List defaultBuiltin = _templateDao.listDefaultBuiltinTemplates(); - // Skipping limit checks for SYSTEM Account and for the templates created from volumes or snapshots - // which already got checked and incremented during createTemplate API call. - if (tmpltInfo.getSize() > 0 && tmplt.getAccountId() != Account.ACCOUNT_ID_SYSTEM && tmplt.getUrl() != null) { - long accountId = tmplt.getAccountId(); - try { - _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(accountId), - com.cloud.configuration.Resource.ResourceType.secondary_storage, - tmpltInfo.getSize() - UriUtils.getRemoteSize(tmplt.getUrl())); - } catch (ResourceAllocationException e) { - s_logger.warn(e.getMessage()); - _alertMgr.sendAlert(AlertManager.ALERT_TYPE_RESOURCE_LIMIT_EXCEEDED, zoneId, null, - e.getMessage(), e.getMessage()); - } finally { - _resourceLimitMgr.recalculateResourceCount(accountId, _accountMgr.getAccount(accountId) - .getDomainId(), com.cloud.configuration.Resource.ResourceType.secondary_storage - .getOrdinal()); + if (rtngTmplts != null) { + for (VMTemplateVO rtngTmplt : rtngTmplts) { + if (!allTemplates.contains(rtngTmplt)) { + allTemplates.add(rtngTmplt); } } } - _vmTemplateStoreDao.update(tmpltStore.getId(), tmpltStore); - } else { - tmpltStore = new TemplateDataStoreVO(storeId, tmplt.getId(), new Date(), 100, Status.DOWNLOADED, - null, null, null, tmpltInfo.getInstallPath(), tmplt.getUrl()); - tmpltStore.setSize(tmpltInfo.getSize()); - tmpltStore.setPhysicalSize(tmpltInfo.getPhysicalSize()); - tmpltStore.setDataStoreRole(store.getRole()); - _vmTemplateStoreDao.persist(tmpltStore); - // update size in vm_template table - VMTemplateVO tmlpt = _templateDao.findById(tmplt.getId()); - tmlpt.setSize(tmpltInfo.getSize()); - _templateDao.update(tmplt.getId(), tmlpt); - associateTemplateToZone(tmplt.getId(), zoneId); + if (defaultBuiltin != null) { + for (VMTemplateVO builtinTmplt : defaultBuiltin) { + if (!allTemplates.contains(builtinTmplt)) { + allTemplates.add(builtinTmplt); + } + } + } + + toBeDownloaded.addAll(allTemplates); + + for (VMTemplateVO tmplt : allTemplates) { + String uniqueName = tmplt.getUniqueName(); + TemplateDataStoreVO tmpltStore = _vmTemplateStoreDao.findByStoreTemplate(storeId, tmplt.getId()); + if (templateInfos.containsKey(uniqueName)) { + TemplateProp tmpltInfo = templateInfos.remove(uniqueName); + toBeDownloaded.remove(tmplt); + if (tmpltStore != null) { + s_logger.info("Template Sync found " + uniqueName + " already in the image store"); + if (tmpltStore.getDownloadState() != Status.DOWNLOADED) { + tmpltStore.setErrorString(""); + } + if (tmpltInfo.isCorrupted()) { + tmpltStore.setDownloadState(Status.DOWNLOAD_ERROR); + String msg = "Template " + tmplt.getName() + ":" + tmplt.getId() + + " is corrupted on secondary storage " + tmpltStore.getId(); + tmpltStore.setErrorString(msg); + s_logger.info("msg"); + if (tmplt.getUrl() == null) { + msg = "Private Template (" + tmplt + ") with install path " + tmpltInfo.getInstallPath() + + "is corrupted, please check in image store: " + tmpltStore.getDataStoreId(); + s_logger.warn(msg); + } else { + s_logger.info("Removing template_store_ref entry for corrupted template " + tmplt.getName()); + _vmTemplateStoreDao.remove(tmpltStore.getId()); + toBeDownloaded.add(tmplt); + } + + } else { + tmpltStore.setDownloadPercent(100); + tmpltStore.setDownloadState(Status.DOWNLOADED); + tmpltStore.setState(ObjectInDataStoreStateMachine.State.Ready); + tmpltStore.setInstallPath(tmpltInfo.getInstallPath()); + tmpltStore.setSize(tmpltInfo.getSize()); + tmpltStore.setPhysicalSize(tmpltInfo.getPhysicalSize()); + tmpltStore.setLastUpdated(new Date()); + // update size in vm_template table + VMTemplateVO tmlpt = _templateDao.findById(tmplt.getId()); + tmlpt.setSize(tmpltInfo.getSize()); + _templateDao.update(tmplt.getId(), tmlpt); + + // Skipping limit checks for SYSTEM Account and for the templates created from volumes or snapshots + // which already got checked and incremented during createTemplate API call. + if (tmpltInfo.getSize() > 0 && tmplt.getAccountId() != Account.ACCOUNT_ID_SYSTEM && tmplt.getUrl() != null) { + long accountId = tmplt.getAccountId(); + try { + _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(accountId), + com.cloud.configuration.Resource.ResourceType.secondary_storage, + tmpltInfo.getSize() - UriUtils.getRemoteSize(tmplt.getUrl())); + } catch (ResourceAllocationException e) { + s_logger.warn(e.getMessage()); + _alertMgr.sendAlert(AlertManager.ALERT_TYPE_RESOURCE_LIMIT_EXCEEDED, zoneId, null, + e.getMessage(), e.getMessage()); + } finally { + _resourceLimitMgr.recalculateResourceCount(accountId, _accountMgr.getAccount(accountId) + .getDomainId(), com.cloud.configuration.Resource.ResourceType.secondary_storage + .getOrdinal()); + } + } + } + _vmTemplateStoreDao.update(tmpltStore.getId(), tmpltStore); + } else { + tmpltStore = new TemplateDataStoreVO(storeId, tmplt.getId(), new Date(), 100, Status.DOWNLOADED, + null, null, null, tmpltInfo.getInstallPath(), tmplt.getUrl()); + tmpltStore.setSize(tmpltInfo.getSize()); + tmpltStore.setPhysicalSize(tmpltInfo.getPhysicalSize()); + tmpltStore.setDataStoreRole(store.getRole()); + _vmTemplateStoreDao.persist(tmpltStore); + + // update size in vm_template table + VMTemplateVO tmlpt = _templateDao.findById(tmplt.getId()); + tmlpt.setSize(tmpltInfo.getSize()); + _templateDao.update(tmplt.getId(), tmlpt); + associateTemplateToZone(tmplt.getId(), zoneId); + } + } else { + s_logger.info("Template Sync did not find " + uniqueName + " on image store " + storeId + ", may request download based on available hypervisor types"); + if (tmpltStore != null) { + s_logger.info("Removing leftover template " + uniqueName + " entry from template store table"); + // remove those leftover entries + _vmTemplateStoreDao.remove(tmpltStore.getId()); + } + } + } + + if (toBeDownloaded.size() > 0) { + /* Only download templates whose hypervirsor type is in the zone */ + List availHypers = _clusterDao.getAvailableHypervisorInZone(zoneId); + if (availHypers.isEmpty()) { + /* + * This is for cloudzone, local secondary storage resource + * started before cluster created + */ + availHypers.add(HypervisorType.KVM); + } + /* Baremetal need not to download any template */ + availHypers.remove(HypervisorType.BareMetal); + availHypers.add(HypervisorType.None); // bug 9809: resume ISO + // download. + for (VMTemplateVO tmplt : toBeDownloaded) { + if (tmplt.getUrl() == null) { // If url is null we can't + s_logger.info("Skip downloading template " + tmplt.getUniqueName() + " since no url is specified."); + continue; + } + // if this is private template, skip sync to a new image store + if (!tmplt.isPublicTemplate() && !tmplt.isFeatured() && tmplt.getTemplateType() != TemplateType.SYSTEM) { + s_logger.info("Skip sync downloading private template " + tmplt.getUniqueName() + " to a new image store"); + continue; + } + + if (availHypers.contains(tmplt.getHypervisorType())) { + s_logger.info("Downloading template " + tmplt.getUniqueName() + " to image store " + + store.getName()); + associateTemplateToZone(tmplt.getId(), zoneId); + TemplateInfo tmpl = _templateFactory.getTemplate(tmplt.getId(), DataStoreRole.Image); + createTemplateAsync(tmpl, store, null); + } else { + s_logger.info("Skip downloading template " + tmplt.getUniqueName() + " since current data center does not have hypervisor " + + tmplt.getHypervisorType().toString()); + } + } + } + + for (String uniqueName : templateInfos.keySet()) { + TemplateProp tInfo = templateInfos.get(uniqueName); + if (_tmpltMgr.templateIsDeleteable(tInfo.getId())) { + // we cannot directly call deleteTemplateSync here to + // reuse delete logic since in this case, our db does not have + // this template at all. + TemplateObjectTO tmplTO = new TemplateObjectTO(); + tmplTO.setDataStore(store.getTO()); + tmplTO.setPath(tInfo.getInstallPath()); + tmplTO.setId(tInfo.getId()); + DeleteCommand dtCommand = new DeleteCommand(tmplTO); + EndPoint ep = _epSelector.select(store); + Answer answer = ep.sendMessage(dtCommand); + if (answer == null || !answer.getResult()) { + s_logger.info("Failed to deleted template at store: " + store.getName()); + + } else { + String description = "Deleted template " + tInfo.getTemplateName() + " on secondary storage " + + storeId; + s_logger.info(description); + } + + } + } } - } else { - if (tmpltStore != null) { - s_logger.info("Template Sync did not find " + uniqueName + " on image store " + storeId - + ", may request download based on available hypervisor types"); - s_logger.info("Removing leftover template " + uniqueName + " entry from template store table"); - // remove those leftover entries - _vmTemplateStoreDao.remove(tmpltStore.getId()); + finally{ + syncLock.unlock(); } } - } - - if (toBeDownloaded.size() > 0) { - /* Only download templates whose hypervirsor type is in the zone */ - List availHypers = _clusterDao.getAvailableHypervisorInZone(zoneId); - if (availHypers.isEmpty()) { - /* - * This is for cloudzone, local secondary storage resource - * started before cluster created - */ - availHypers.add(HypervisorType.KVM); - } - /* Baremetal need not to download any template */ - availHypers.remove(HypervisorType.BareMetal); - availHypers.add(HypervisorType.None); // bug 9809: resume ISO - // download. - for (VMTemplateVO tmplt : toBeDownloaded) { - if (tmplt.getUrl() == null) { // If url is null we can't - // initiate the download - continue; - } - - // if this is private template, skip - if (!tmplt.isPublicTemplate() && !tmplt.isFeatured()) { - continue; - } - if (availHypers.contains(tmplt.getHypervisorType())) { - s_logger.info("Downloading template " + tmplt.getUniqueName() + " to image store " - + store.getName()); - associateTemplateToZone(tmplt.getId(), zoneId); - TemplateInfo tmpl = _templateFactory.getTemplate(tmplt.getId(), DataStoreRole.Image); - createTemplateAsync(tmpl, store, null); - } - } - } - - for (String uniqueName : templateInfos.keySet()) { - TemplateProp tInfo = templateInfos.get(uniqueName); - if (_tmpltMgr.templateIsDeleteable(tInfo.getId())) { - // we cannot directly call deleteTemplateSync here to - // reuse delete logic since in this case, our db does not have - // this template at all. - TemplateObjectTO tmplTO = new TemplateObjectTO(); - tmplTO.setDataStore(store.getTO()); - tmplTO.setPath(tInfo.getInstallPath()); - tmplTO.setId(tInfo.getId()); - DeleteCommand dtCommand = new DeleteCommand(tmplTO); - EndPoint ep = _epSelector.select(store); - Answer answer = ep.sendMessage(dtCommand); - if (answer == null || !answer.getResult()) { - s_logger.info("Failed to deleted template at store: " + store.getName()); - - } else { - String description = "Deleted template " + tInfo.getTemplateName() + " on secondary storage " - + storeId; - s_logger.info(description); - } - + else { + s_logger.info("Couldn't get global lock on " + lockString + ", another thread may be doing template sync on data store " + storeId + " now."); } + } finally { + syncLock.releaseRef(); } } @@ -459,6 +487,23 @@ public class TemplateServiceImpl implements TemplateService { } } + // update template_zone_ref for cross-zone template for newly added zone + @Override + public void associateCrosszoneTemplatesToZone(long dcId){ + VMTemplateZoneVO tmpltZone; + + List allTemplates = _templateDao.listAll(); + for (VMTemplateVO vt: allTemplates){ + if (vt.isCrossZones()) { + tmpltZone = _vmTemplateZoneDao.findByZoneTemplate(dcId, vt.getId()); + if (tmpltZone == null) { + VMTemplateZoneVO vmTemplateZone = new VMTemplateZoneVO(dcId, vt.getId(), new Date()); + _vmTemplateZoneDao.persist(vmTemplateZone); + } + } + } + } + private Map listTemplate(DataStore ssStore) { ListTemplateCommand cmd = new ListTemplateCommand(ssStore.getTO()); EndPoint ep = _epSelector.select(ssStore); diff --git a/engine/storage/image/src/org/apache/cloudstack/storage/image/store/TemplateObject.java b/engine/storage/image/src/org/apache/cloudstack/storage/image/store/TemplateObject.java index ba5a7d19605..0a1836e1af9 100644 --- a/engine/storage/image/src/org/apache/cloudstack/storage/image/store/TemplateObject.java +++ b/engine/storage/image/src/org/apache/cloudstack/storage/image/store/TemplateObject.java @@ -167,7 +167,7 @@ public class TemplateObject implements TemplateInfo { } finally { // in case of OperationFailed, expunge the entry if (event == ObjectInDataStoreStateMachine.Event.OperationFailed) { - objectInStoreMgr.delete(this); + objectInStoreMgr.deleteIfNotReady(this); } } } @@ -198,10 +198,19 @@ public class TemplateObject implements TemplateInfo { templateStoreRef.setDownloadPercent(100); templateStoreRef.setDownloadState(Status.DOWNLOADED); templateStoreRef.setSize(newTemplate.getSize()); + if (newTemplate.getPhysicalSize() != null) { + templateStoreRef.setPhysicalSize(newTemplate.getPhysicalSize()); + } templateStoreDao.update(templateStoreRef.getId(), templateStoreRef); if (this.getDataStore().getRole() == DataStoreRole.Image) { VMTemplateVO templateVO = this.imageDao.findById(this.getId()); - templateVO.setFormat(newTemplate.getFormat()); + if (newTemplate.getFormat() != null) { + templateVO.setFormat(newTemplate.getFormat()); + } + if (newTemplate.getName() != null ){ + // For template created from snapshot, template name is determine by resource code. + templateVO.setUniqueName(newTemplate.getName()); + } templateVO.setSize(newTemplate.getSize()); this.imageDao.update(templateVO.getId(), templateVO); } @@ -218,7 +227,7 @@ public class TemplateObject implements TemplateInfo { } finally { // in case of OperationFailed, expunge the entry if (event == ObjectInDataStoreStateMachine.Event.OperationFailed) { - objectInStoreMgr.delete(this); + objectInStoreMgr.deleteIfNotReady(this); } } } diff --git a/engine/storage/integration-test/pom.xml b/engine/storage/integration-test/pom.xml index 8a9dc15579c..1f0888594c2 100644 --- a/engine/storage/integration-test/pom.xml +++ b/engine/storage/integration-test/pom.xml @@ -16,7 +16,7 @@ org.apache.cloudstack cloud-engine - 4.2.0-SNAPSHOT + 4.2.0 ../../pom.xml diff --git a/engine/storage/integration-test/test/org/apache/cloudstack/storage/allocator/StorageAllocatorTest.java b/engine/storage/integration-test/test/org/apache/cloudstack/storage/allocator/StorageAllocatorTest.java index 90696cae806..2fc701a1214 100644 --- a/engine/storage/integration-test/test/org/apache/cloudstack/storage/allocator/StorageAllocatorTest.java +++ b/engine/storage/integration-test/test/org/apache/cloudstack/storage/allocator/StorageAllocatorTest.java @@ -22,6 +22,12 @@ import java.util.UUID; import javax.inject.Inject; +import com.cloud.configuration.Config; +import com.cloud.configuration.ConfigurationVO; +import com.cloud.configuration.dao.ConfigurationDao; +import com.cloud.user.Account; +import com.cloud.utils.db.DB; +import com.cloud.vm.VMInstanceVO; import junit.framework.Assert; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProvider; @@ -87,6 +93,8 @@ public class StorageAllocatorTest { StoragePoolDetailsDao poolDetailsDao; @Inject DataStoreProviderManager providerMgr; + @Inject + ConfigurationDao configDao; Long dcId = 1l; Long podId = 1l; Long clusterId = 1l; @@ -98,7 +106,13 @@ public class StorageAllocatorTest { StoragePoolVO storage = null; @Before + @DB public void setup() throws Exception { + ConfigurationVO cfg = configDao.findByName(Config.VmAllocationAlgorithm.key()); + if (cfg == null) { + ConfigurationVO configVO = new ConfigurationVO("test", "DEFAULT", "test", Config.VmAllocationAlgorithm.key(), "userdispersing", null); + configDao.persist(configVO); + } ComponentContext.initComponentsLifeCycle(); } @@ -120,7 +134,7 @@ public class StorageAllocatorTest { cluster = clusterDao.persist(cluster); clusterId = cluster.getId(); - DataStoreProvider provider = providerMgr.getDataStoreProvider("cloudstack primary data store provider"); + DataStoreProvider provider = providerMgr.getDataStoreProvider(DataStoreProvider.DEFAULT_PRIMARY); storage = new StoragePoolVO(); storage.setDataCenterId(dcId); storage.setPodId(podId); @@ -163,7 +177,7 @@ public class StorageAllocatorTest { try { createDb(); - DataStoreProvider provider = providerMgr.getDataStoreProvider("cloudstack primary data store provider"); + DataStoreProvider provider = providerMgr.getDataStoreProvider(DataStoreProvider.DEFAULT_PRIMARY); storage = new StoragePoolVO(); storage.setDataCenterId(dcId); storage.setPodId(podId); @@ -312,7 +326,10 @@ public class StorageAllocatorTest { createDb(); StoragePoolVO pool = storagePoolDao.findById(storagePoolId); + pool.setHypervisor(HypervisorType.KVM); pool.setScope(ScopeType.ZONE); + pool.setClusterId(null); + pool.setPodId(null); storagePoolDao.update(pool.getId(), pool); DiskProfile profile = new DiskProfile(volume, diskOffering, HypervisorType.KVM); @@ -321,6 +338,8 @@ public class StorageAllocatorTest { Mockito.when( storageMgr.storagePoolHasEnoughSpace(Matchers.anyListOf(Volume.class), Matchers.any(StoragePool.class))).thenReturn(true); + Mockito.when(storageMgr.storagePoolHasEnoughIops(Matchers.anyListOf(Volume.class), + Matchers.any(StoragePool.class))).thenReturn(true); DeploymentPlan plan = new DataCenterDeployment(dcId, podId, clusterId, null, null, null); int foundAcct = 0; for (StoragePoolAllocator allocator : allocators) { @@ -340,6 +359,49 @@ public class StorageAllocatorTest { } } + @Test + public void testCLOUDSTACK3481() { + try { + createDb(); + + StoragePoolVO pool = storagePoolDao.findById(storagePoolId); + pool.setHypervisor(HypervisorType.KVM); + pool.setScope(ScopeType.ZONE); + pool.setClusterId(null); + pool.setPodId(null); + storagePoolDao.update(pool.getId(), pool); + + + DiskProfile profile = new DiskProfile(volume, diskOffering, HypervisorType.KVM); + VirtualMachineProfile vmProfile = Mockito.mock(VirtualMachineProfile.class); + Account account = Mockito.mock(Account.class); + Mockito.when(account.getAccountId()).thenReturn(1L); + Mockito.when(vmProfile.getHypervisorType()).thenReturn(HypervisorType.KVM); + Mockito.when(vmProfile.getOwner()).thenReturn(account); + Mockito.when( + storageMgr.storagePoolHasEnoughSpace(Matchers.anyListOf(Volume.class), + Matchers.any(StoragePool.class))).thenReturn(true); + Mockito.when(storageMgr.storagePoolHasEnoughIops(Matchers.anyListOf(Volume.class), + Matchers.any(StoragePool.class))).thenReturn(true); + DeploymentPlan plan = new DataCenterDeployment(dcId, podId, clusterId, null, null, null); + int foundAcct = 0; + for (StoragePoolAllocator allocator : allocators) { + List pools = allocator.allocateToPool(profile, vmProfile, plan, new ExcludeList(), 1); + if (!pools.isEmpty()) { + Assert.assertEquals(pools.get(0).getId(), storage.getId()); + foundAcct++; + } + } + + if (foundAcct > 1 || foundAcct == 0) { + Assert.fail(); + } + } catch (Exception e) { + cleanDb(); + Assert.fail(); + } + } + @Test public void testPoolStateIsNotUp() { try { diff --git a/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/ChildTestConfiguration.java b/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/ChildTestConfiguration.java index 817262c28d7..40c5a72d7a7 100644 --- a/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/ChildTestConfiguration.java +++ b/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/ChildTestConfiguration.java @@ -18,6 +18,8 @@ package org.apache.cloudstack.storage.test; import java.io.IOException; +import com.cloud.event.ActionEventUtils; +import com.cloud.event.dao.EventDaoImpl; import org.apache.cloudstack.acl.APIChecker; import org.apache.cloudstack.engine.service.api.OrchestrationService; import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; @@ -106,7 +108,7 @@ import com.cloud.vm.snapshot.dao.VMSnapshotDaoImpl; VMSnapshotDaoImpl.class, OCFS2ManagerImpl.class, ClusterDetailsDaoImpl.class, SecondaryStorageVmDaoImpl.class, ConsoleProxyDaoImpl.class, StoragePoolWorkDaoImpl.class, StorageCacheManagerImpl.class, UserDaoImpl.class, DataCenterDaoImpl.class, StoragePoolDetailsDaoImpl.class, DomainDaoImpl.class, DownloadMonitorImpl.class, - AccountDaoImpl.class }, includeFilters = { @Filter(value = Library.class, type = FilterType.CUSTOM) }, + AccountDaoImpl.class, ActionEventUtils.class, EventDaoImpl.class}, includeFilters = { @Filter(value = Library.class, type = FilterType.CUSTOM) }, useDefaultFilters = false) public class ChildTestConfiguration extends TestConfiguration { @@ -185,12 +187,6 @@ public class ChildTestConfiguration extends TestConfiguration { return Mockito.mock(VirtualMachineManager.class); } - - @Bean - public SnapshotManager snapshotMgr() { - return Mockito.mock(SnapshotManager.class); - } - @Bean public ResourceManager resourceMgr() { return Mockito.mock(ResourceManager.class); diff --git a/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/DirectAgentManagerSimpleImpl.java b/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/DirectAgentManagerSimpleImpl.java index ac50e9bde95..763a49e97a6 100644 --- a/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/DirectAgentManagerSimpleImpl.java +++ b/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/DirectAgentManagerSimpleImpl.java @@ -26,6 +26,10 @@ import java.util.Map; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.host.Host; +import com.cloud.host.Status; +import com.cloud.utils.fsm.NoTransitionException; +import com.cloud.utils.fsm.StateMachine2; import org.apache.log4j.Logger; import com.cloud.agent.AgentManager; @@ -65,6 +69,9 @@ public class DirectAgentManagerSimpleImpl extends ManagerBase implements AgentMa ClusterDao clusterDao; @Inject ClusterDetailsDao clusterDetailsDao; + @Inject + HostDao _hostDao; + protected StateMachine2 _statusStateMachine = Status.getStateMachine(); @Override public boolean configure(String name, Map params) throws ConfigurationException { @@ -249,8 +256,12 @@ public class DirectAgentManagerSimpleImpl extends ManagerBase implements AgentMa @Override public boolean agentStatusTransitTo(HostVO host, Event e, long msId) { - // TODO Auto-generated method stub - return false; + try { + return _statusStateMachine.transitTo(host, e, host.getId(), _hostDao); + } catch (NoTransitionException e1) { + e1.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. + } + return true; } @Override diff --git a/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/EndpointSelectorTest.java b/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/EndpointSelectorTest.java new file mode 100644 index 00000000000..15233bfe8db --- /dev/null +++ b/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/EndpointSelectorTest.java @@ -0,0 +1,327 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.test; + +import com.cloud.agent.AgentManager; +import com.cloud.cluster.LockMasterListener; +import com.cloud.dc.ClusterVO; +import com.cloud.dc.DataCenter; +import com.cloud.dc.DataCenterVO; +import com.cloud.dc.HostPodVO; +import com.cloud.dc.dao.ClusterDao; +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.dc.dao.HostPodDao; +import com.cloud.host.Host; +import com.cloud.host.HostVO; +import com.cloud.host.Status; +import com.cloud.host.dao.HostDao; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.org.Cluster; +import com.cloud.org.Managed; +import com.cloud.resource.ResourceState; +import com.cloud.storage.DataStoreRole; +import com.cloud.storage.ScopeType; +import com.cloud.storage.Storage; +import com.cloud.storage.StoragePoolHostVO; +import com.cloud.storage.StoragePoolStatus; +import com.cloud.storage.dao.SnapshotDao; +import com.cloud.storage.dao.SnapshotPolicyDao; +import com.cloud.storage.dao.StoragePoolHostDao; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; +import com.cloud.user.User; +import com.cloud.utils.component.ComponentContext; +import com.cloud.utils.db.Merovingian2; +import junit.framework.Assert; +import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope; +import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProvider; +import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; +import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; +import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreProvider; +import org.apache.cloudstack.engine.subsystem.api.storage.Scope; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService; +import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; +import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; +import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +import javax.inject.Inject; +import java.net.URI; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.UUID; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(locations = { "classpath:/fakeDriverTestContext.xml" }) +public class EndpointSelectorTest { + @Inject + SnapshotService snapshotService; + @Inject + SnapshotDao snapshotDao; + @Inject + SnapshotDataFactory snapshotDataFactory; + @Inject + PrimaryDataStoreProvider primaryDataStoreProvider; + @Inject + SnapshotDataStoreDao snapshotDataStoreDao; + @Inject + VolumeDao volumeDao; + @Inject + VolumeService volumeService; + @Inject + VolumeDataFactory volumeDataFactory; + @Inject + DataCenterDao dcDao; + Long dcId; + @Inject + HostPodDao podDao; + Long podId; + @Inject + ClusterDao clusterDao; + Long clusterId; + @Inject + ImageStoreDao imageStoreDao; + ImageStoreVO imageStore; + @Inject + AccountManager accountManager; + LockMasterListener lockMasterListener; + VolumeInfo vol = null; + FakePrimaryDataStoreDriver driver = new FakePrimaryDataStoreDriver(); + @Inject + MockStorageMotionStrategy mockStorageMotionStrategy; + Merovingian2 _lockMaster; + @Inject + DataStoreManager dataStoreManager; + @Inject + PrimaryDataStoreDao primaryDataStoreDao; + @Inject + SnapshotPolicyDao snapshotPolicyDao; + @Inject + HostDao hostDao; + @Inject + StoragePoolHostDao storagePoolHostDao; + @Inject + EndPointSelector endPointSelector; + @Inject + AgentManager agentMgr; + @Before + public void setUp() { + // create data center + + DataCenterVO dc = new DataCenterVO(UUID.randomUUID().toString(), "test", "8.8.8.8", null, "10.0.0.1", null, + "10.0.0.1/24", null, null, DataCenter.NetworkType.Basic, null, null, true, true, null, null); + dc = dcDao.persist(dc); + dcId = dc.getId(); + // create pod + + HostPodVO pod = new HostPodVO(UUID.randomUUID().toString(), dc.getId(), "10.223.0.1", + "10.233.2.2/25", 8, "test"); + pod = podDao.persist(pod); + podId = pod.getId(); + // create xen cluster + ClusterVO cluster = new ClusterVO(dc.getId(), pod.getId(), "devcloud cluster"); + cluster.setHypervisorType(Hypervisor.HypervisorType.XenServer.toString()); + cluster.setClusterType(Cluster.ClusterType.CloudManaged); + cluster.setManagedState(Managed.ManagedState.Managed); + cluster = clusterDao.persist(cluster); + clusterId = cluster.getId(); + + imageStore = new ImageStoreVO(); + imageStore.setName(UUID.randomUUID().toString()); + imageStore.setDataCenterId(dcId); + imageStore.setProviderName(DataStoreProvider.NFS_IMAGE); + imageStore.setRole(DataStoreRole.Image); + imageStore.setUrl(UUID.randomUUID().toString()); + imageStore.setUuid(UUID.randomUUID().toString()); + imageStore.setProtocol("nfs"); + imageStore = imageStoreDao.persist(imageStore); + + when(primaryDataStoreProvider.configure(Mockito.anyMap())).thenReturn(true); + Set types = new HashSet(); + types.add(DataStoreProvider.DataStoreProviderType.PRIMARY); + + when(primaryDataStoreProvider.getTypes()).thenReturn(types); + when(primaryDataStoreProvider.getName()).thenReturn(DataStoreProvider.DEFAULT_PRIMARY); + when(primaryDataStoreProvider.getDataStoreDriver()).thenReturn(driver); + User user = mock(User.class); + when(user.getId()).thenReturn(1L); + Account account = mock(Account.class); + when(account.getId()).thenReturn(1L); + when(accountManager.getSystemAccount()).thenReturn(account); + when(accountManager.getSystemUser()).thenReturn(user); + + if(Merovingian2.getLockMaster() == null) { + _lockMaster = Merovingian2.createLockMaster(1234); + } else { + _lockMaster = Merovingian2.getLockMaster(); + } + _lockMaster.cleanupThisServer(); + ComponentContext.initComponentsLifeCycle(); + } + + public DataStore createPrimaryDataStore(ScopeType scope) { + String uuid = UUID.randomUUID().toString(); + List pools = primaryDataStoreDao.findPoolByName(uuid); + if (pools.size() > 0) { + return dataStoreManager.getPrimaryDataStore(pools.get(0).getId()); + } + + StoragePoolVO pool = new StoragePoolVO(); + if (scope != ScopeType.ZONE) { + pool.setClusterId(clusterId); + } + pool.setDataCenterId(dcId); + + pool.setHostAddress(uuid); + pool.setPath(uuid); + pool.setPort(0); + pool.setName(uuid); + pool.setUuid(uuid); + pool.setStatus(StoragePoolStatus.Up); + pool.setPoolType(Storage.StoragePoolType.NetworkFilesystem); + pool.setPodId(podId); + pool.setScope(scope); + pool.setStorageProviderName(DataStoreProvider.DEFAULT_PRIMARY); + pool = primaryDataStoreDao.persist(pool); + DataStore store = dataStoreManager.getPrimaryDataStore(pool.getId()); + return store; + } + + public HostVO createHost(Hypervisor.HypervisorType hypervisorType) { + String uuid = UUID.randomUUID().toString(); + HostVO host = new HostVO(uuid); + host.setName("devcloud xen host"); + host.setType(Host.Type.Routing); + host.setPrivateIpAddress(uuid); + host.setDataCenterId(dcId); + host.setVersion("6.0.1"); + host.setAvailable(true); + host.setSetup(true); + host.setPodId(podId); + host.setLastPinged(0); + host.setResourceState(ResourceState.Enabled); + host.setHypervisorType(hypervisorType); + host.setClusterId(clusterId); + + + host = hostDao.persist(host); + agentMgr.agentStatusTransitTo(host, Status.Event.AgentConnected, 1L); + host = hostDao.findById(host.getId()); + agentMgr.agentStatusTransitTo(host, Status.Event.Ready, 1L); + return hostDao.findById(host.getId()); + } + + public void addStorageToHost(DataStore store, HostVO host) { + StoragePoolHostVO storagePoolHostVO = new StoragePoolHostVO(store.getId(), host.getId(), UUID.randomUUID().toString()); + storagePoolHostDao.persist(storagePoolHostVO); + } + + @Test + public void testMixZonePrimaryStorages() { + Long srcStoreId = null; + Long destStoreId = imageStore.getId(); + DataStore store = createPrimaryDataStore(ScopeType.ZONE); + srcStoreId = store.getId(); + HostVO host = createHost(Hypervisor.HypervisorType.VMware); + addStorageToHost(store, host); + + store = createPrimaryDataStore(ScopeType.ZONE); + host = createHost(Hypervisor.HypervisorType.VMware); + addStorageToHost(store, host); + + Long xenStoreId = null; + store = createPrimaryDataStore(ScopeType.CLUSTER); + xenStoreId = store.getId(); + host = createHost(Hypervisor.HypervisorType.XenServer); + addStorageToHost(store, host); + + store = createPrimaryDataStore(ScopeType.CLUSTER); + host = createHost(Hypervisor.HypervisorType.XenServer); + addStorageToHost(store, host); + + ZoneScope srcScope = new ZoneScope(dcId); + + DataStore srcStore = mock(DataStore.class); + DataStore destStore = mock(DataStore.class); + + when(srcStore.getScope()).thenReturn(srcScope); + when(srcStore.getRole()).thenReturn(DataStoreRole.Primary); + when(srcStore.getId()).thenReturn(srcStoreId); + when(destStore.getScope()).thenReturn(srcScope); + when(destStore.getRole()).thenReturn(DataStoreRole.Image); + when(destStore.getId()).thenReturn(destStoreId); + + + + DataObject srcObj = mock(DataObject.class); + DataObject destObj = mock(DataObject.class); + when(srcObj.getDataStore()).thenReturn(srcStore); + when(destObj.getDataStore()).thenReturn(destStore); + EndPoint ep = endPointSelector.select(srcObj, destObj); + + Assert.assertTrue(ep != null); + Long hostId = ep.getId(); + HostVO newHost = hostDao.findById(hostId); + Assert.assertTrue(newHost.getHypervisorType() == Hypervisor.HypervisorType.VMware); + + when(srcStore.getRole()).thenReturn(DataStoreRole.Image); + when(srcStore.getId()).thenReturn(destStoreId); + when(destStore.getId()).thenReturn(srcStoreId); + when(destStore.getRole()).thenReturn(DataStoreRole.Primary); + ep = endPointSelector.select(srcObj, destObj); + + Assert.assertTrue(ep != null); + hostId = ep.getId(); + newHost = hostDao.findById(hostId); + Assert.assertTrue(newHost.getHypervisorType() == Hypervisor.HypervisorType.VMware); + + ClusterScope clusterScope = new ClusterScope(clusterId, podId, dcId); + when(srcStore.getRole()).thenReturn(DataStoreRole.Primary); + when(srcStore.getScope()).thenReturn(clusterScope); + when(srcStore.getId()).thenReturn(xenStoreId); + ep = endPointSelector.select(srcStore); + Assert.assertTrue(ep != null); + newHost = hostDao.findById(ep.getId()); + Assert.assertTrue(newHost.getHypervisorType() == Hypervisor.HypervisorType.XenServer); + + + + } + +} diff --git a/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/FakeDriverTestConfiguration.java b/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/FakeDriverTestConfiguration.java new file mode 100644 index 00000000000..75eda90c864 --- /dev/null +++ b/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/FakeDriverTestConfiguration.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.test; + +import com.cloud.storage.snapshot.SnapshotScheduler; +import com.cloud.storage.snapshot.SnapshotSchedulerImpl; +import com.cloud.user.DomainManager; +import com.cloud.utils.component.ComponentContext; +import org.apache.cloudstack.engine.subsystem.api.storage.DataMotionStrategy; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProvider; +import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; +import org.apache.cloudstack.storage.datastore.provider.CloudStackPrimaryDataStoreProviderImpl; +import org.apache.cloudstack.storage.datastore.type.DataStoreType; +import org.apache.cloudstack.storage.endpoint.DefaultEndPointSelector; +import org.mockito.Mockito; +import org.springframework.context.annotation.Bean; + +import java.util.HashSet; +import java.util.Set; + +public class FakeDriverTestConfiguration extends ChildTestConfiguration{ + @Bean + public CloudStackPrimaryDataStoreProviderImpl dataStoreProvider() { + CloudStackPrimaryDataStoreProviderImpl provider = Mockito.mock(CloudStackPrimaryDataStoreProviderImpl.class); + + return provider; + } + + @Bean + public DataMotionStrategy dataMotionStrategy() { + DataMotionStrategy strategy = new MockStorageMotionStrategy(); + return strategy; + } + + @Bean + public SnapshotScheduler SnapshotScheduler() { + return Mockito.mock(SnapshotScheduler.class); + } + + @Bean + public DomainManager DomainManager() { + return Mockito.mock(DomainManager.class); + } + + @Override + @Bean + public EndPointSelector selector() { + return ComponentContext.inject(DefaultEndPointSelector.class); + } + +} diff --git a/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/FakePrimaryDataStoreDriver.java b/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/FakePrimaryDataStoreDriver.java new file mode 100644 index 00000000000..810afd11577 --- /dev/null +++ b/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/FakePrimaryDataStoreDriver.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.test; + +import com.cloud.agent.api.to.DataStoreTO; +import com.cloud.agent.api.to.DataTO; +import org.apache.cloudstack.engine.subsystem.api.storage.ChapInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; +import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult; +import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; +import org.apache.cloudstack.framework.async.AsyncCompletionCallback; +import org.apache.cloudstack.storage.command.CommandResult; +import org.apache.cloudstack.storage.command.CreateObjectAnswer; +import org.apache.cloudstack.storage.to.SnapshotObjectTO; + +import java.util.UUID; + +public class FakePrimaryDataStoreDriver implements PrimaryDataStoreDriver { + boolean snapshotResult = true; + @Override + public ChapInfo getChapInfo(VolumeInfo volumeInfo) { + return null; //To change body of implemented methods use File | Settings | File Templates. + } + + @Override + public void takeSnapshot(SnapshotInfo snapshot, AsyncCompletionCallback callback) { + CreateCmdResult result = new CreateCmdResult(null, null); + if (snapshotResult) { + SnapshotObjectTO newSnap = new SnapshotObjectTO(); + newSnap.setPath(UUID.randomUUID().toString()); + + CreateObjectAnswer answer = new CreateObjectAnswer(newSnap); + result.setAnswer(answer); + } else { + result.setResult("Failed to create snapshot"); + } + callback.complete(result); + return; + } + + public void makeTakeSnapshotSucceed(boolean success) { + snapshotResult = success; + } + + @Override + public void revertSnapshot(SnapshotInfo snapshot, AsyncCompletionCallback callback) { + //To change body of implemented methods use File | Settings | File Templates. + } + + @Override + public DataTO getTO(DataObject data) { + return null; //To change body of implemented methods use File | Settings | File Templates. + } + + @Override + public DataStoreTO getStoreTO(DataStore store) { + return null; //To change body of implemented methods use File | Settings | File Templates. + } + + @Override + public void createAsync(DataStore store, DataObject data, AsyncCompletionCallback callback) { + //To change body of implemented methods use File | Settings | File Templates. + } + + @Override + public void deleteAsync(DataStore store, DataObject data, AsyncCompletionCallback callback) { + CommandResult result = new CommandResult(); + result.setSuccess(true); + callback.complete(result); + return; + } + + @Override + public void copyAsync(DataObject srcdata, DataObject destData, AsyncCompletionCallback callback) { + //To change body of implemented methods use File | Settings | File Templates. + } + + @Override + public boolean canCopy(DataObject srcData, DataObject destData) { + return false; //To change body of implemented methods use File | Settings | File Templates. + } + + @Override + public void resize(DataObject data, AsyncCompletionCallback callback) { + //To change body of implemented methods use File | Settings | File Templates. + } +} diff --git a/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/MockStorageMotionStrategy.java b/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/MockStorageMotionStrategy.java index 52ccf410c8d..6c0bd556480 100644 --- a/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/MockStorageMotionStrategy.java +++ b/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/MockStorageMotionStrategy.java @@ -19,25 +19,39 @@ package org.apache.cloudstack.storage.test; import java.util.Map; +import java.util.UUID; +import com.cloud.agent.api.to.DataObjectType; +import com.cloud.agent.api.to.DataTO; +import com.cloud.storage.Storage; import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; import org.apache.cloudstack.engine.subsystem.api.storage.DataMotionStrategy; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; import org.apache.cloudstack.framework.async.AsyncCompletionCallback; import com.cloud.agent.api.to.VirtualMachineTO; import com.cloud.host.Host; +import org.apache.cloudstack.storage.command.CopyCmdAnswer; +import org.apache.cloudstack.storage.snapshot.SnapshotObject; +import org.apache.cloudstack.storage.to.SnapshotObjectTO; +import org.apache.cloudstack.storage.to.TemplateObjectTO; public class MockStorageMotionStrategy implements DataMotionStrategy { + boolean success = true; @Override public boolean canHandle(DataObject srcData, DataObject destData) { // TODO Auto-generated method stub return true; } + public void makeBackupSnapshotSucceed(boolean success) { + this.success = success; + } + @Override public boolean canHandle(Map volumeMap, Host srcHost, Host destHost) { return true; @@ -45,7 +59,32 @@ public class MockStorageMotionStrategy implements DataMotionStrategy { @Override public Void copyAsync(DataObject srcData, DataObject destData, AsyncCompletionCallback callback) { - CopyCommandResult result = new CopyCommandResult("something", null); + CopyCmdAnswer answer = null; + DataTO data = null; + if (!success) { + CopyCommandResult result = new CopyCommandResult(null, null); + result.setResult("Failed"); + callback.complete(result); + } + if (destData.getType() == DataObjectType.SNAPSHOT) { + SnapshotInfo srcSnapshot = (SnapshotInfo)srcData; + + SnapshotObjectTO newSnapshot = new SnapshotObjectTO(); + newSnapshot.setPath(UUID.randomUUID().toString()); + if (srcSnapshot.getParent() != null) { + newSnapshot.setParentSnapshotPath(srcSnapshot.getParent().getPath()); + } + data = newSnapshot; + } else if (destData.getType() == DataObjectType.TEMPLATE) { + TemplateObjectTO newTemplate = new TemplateObjectTO(); + newTemplate.setPath(UUID.randomUUID().toString()); + newTemplate.setFormat(Storage.ImageFormat.QCOW2); + newTemplate.setSize(10L); + newTemplate.setPhysicalSize(10L); + data = newTemplate; + } + answer = new CopyCmdAnswer(data); + CopyCommandResult result = new CopyCommandResult("something", answer); callback.complete(result); return null; } diff --git a/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/SnapshotTestWithFakeData.java b/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/SnapshotTestWithFakeData.java new file mode 100644 index 00000000000..2aaabdaf430 --- /dev/null +++ b/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/SnapshotTestWithFakeData.java @@ -0,0 +1,359 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.test; + +import com.cloud.cluster.LockMasterListener; +import com.cloud.dc.ClusterVO; +import com.cloud.dc.DataCenter; +import com.cloud.dc.DataCenterVO; +import com.cloud.dc.HostPodVO; +import com.cloud.dc.dao.ClusterDao; +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.dc.dao.HostPodDao; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.org.Cluster; +import com.cloud.org.Managed; +import com.cloud.storage.CreateSnapshotPayload; +import com.cloud.storage.DataStoreRole; +import com.cloud.storage.ScopeType; +import com.cloud.storage.Snapshot; +import com.cloud.storage.SnapshotPolicyVO; +import com.cloud.storage.SnapshotVO; +import com.cloud.storage.Storage; +import com.cloud.storage.StoragePoolStatus; +import com.cloud.storage.Volume; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.SnapshotDao; +import com.cloud.storage.dao.SnapshotPolicyDao; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; +import com.cloud.user.User; +import com.cloud.utils.DateUtil; +import com.cloud.utils.component.ComponentContext; +import com.cloud.utils.db.DB; +import com.cloud.utils.db.Merovingian2; +import junit.framework.Assert; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProvider; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; +import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver; +import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreProvider; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotResult; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService; +import org.apache.cloudstack.storage.datastore.PrimaryDataStore; +import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; +import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.storage.volume.VolumeObject; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +import javax.inject.Inject; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(locations = { "classpath:/fakeDriverTestContext.xml" }) +public class SnapshotTestWithFakeData { + @Inject + SnapshotService snapshotService; + @Inject + SnapshotDao snapshotDao; + @Inject + PrimaryDataStoreDao primaryDataStoreDao; + @Inject + DataStoreManager dataStoreManager; + @Inject + SnapshotDataFactory snapshotDataFactory; + @Inject + PrimaryDataStoreProvider primaryDataStoreProvider; + @Inject + SnapshotDataStoreDao snapshotDataStoreDao; + @Inject + VolumeDao volumeDao; + @Inject + VolumeService volumeService; + @Inject + VolumeDataFactory volumeDataFactory; + @Inject + DataCenterDao dcDao; + Long dcId; + @Inject + HostPodDao podDao; + Long podId; + @Inject + ClusterDao clusterDao; + Long clusterId; + @Inject + ImageStoreDao imageStoreDao; + ImageStoreVO imageStore; + @Inject + AccountManager accountManager; + LockMasterListener lockMasterListener; + VolumeInfo vol = null; + FakePrimaryDataStoreDriver driver = new FakePrimaryDataStoreDriver(); + @Inject + MockStorageMotionStrategy mockStorageMotionStrategy; + Merovingian2 _lockMaster; + @Inject + SnapshotPolicyDao snapshotPolicyDao; + + @Before + public void setUp() { + // create data center + + DataCenterVO dc = new DataCenterVO(UUID.randomUUID().toString(), "test", "8.8.8.8", null, "10.0.0.1", null, + "10.0.0.1/24", null, null, DataCenter.NetworkType.Basic, null, null, true, true, null, null); + dc = dcDao.persist(dc); + dcId = dc.getId(); + // create pod + + HostPodVO pod = new HostPodVO(UUID.randomUUID().toString(), dc.getId(), "10.223.0.1", + "10.233.2.2/25", 8, "test"); + pod = podDao.persist(pod); + podId = pod.getId(); + // create xen cluster + ClusterVO cluster = new ClusterVO(dc.getId(), pod.getId(), "devcloud cluster"); + cluster.setHypervisorType(Hypervisor.HypervisorType.XenServer.toString()); + cluster.setClusterType(Cluster.ClusterType.CloudManaged); + cluster.setManagedState(Managed.ManagedState.Managed); + cluster = clusterDao.persist(cluster); + clusterId = cluster.getId(); + + imageStore = new ImageStoreVO(); + imageStore.setName(UUID.randomUUID().toString()); + imageStore.setDataCenterId(dcId); + imageStore.setProviderName(DataStoreProvider.NFS_IMAGE); + imageStore.setRole(DataStoreRole.Image); + imageStore.setUrl(UUID.randomUUID().toString()); + imageStore.setUuid(UUID.randomUUID().toString()); + imageStore.setProtocol("nfs"); + imageStore = imageStoreDao.persist(imageStore); + + when(primaryDataStoreProvider.configure(Mockito.anyMap())).thenReturn(true); + Set types = new HashSet(); + types.add(DataStoreProvider.DataStoreProviderType.PRIMARY); + + when(primaryDataStoreProvider.getTypes()).thenReturn(types); + when(primaryDataStoreProvider.getName()).thenReturn(DataStoreProvider.DEFAULT_PRIMARY); + when(primaryDataStoreProvider.getDataStoreDriver()).thenReturn(driver); + User user = mock(User.class); + when(user.getId()).thenReturn(1L); + Account account = mock(Account.class); + when(account.getId()).thenReturn(1L); + when(accountManager.getSystemAccount()).thenReturn(account); + when(accountManager.getSystemUser()).thenReturn(user); + + if(Merovingian2.getLockMaster() == null) { + _lockMaster = Merovingian2.createLockMaster(1234); + } else { + _lockMaster = Merovingian2.getLockMaster(); + } + _lockMaster.cleanupThisServer(); + ComponentContext.initComponentsLifeCycle(); + } + + @After + public void tearDown() throws Exception { + _lockMaster.cleanupThisServer(); + } + private SnapshotVO createSnapshotInDb() { + Snapshot.Type snapshotType = Snapshot.Type.RECURRING; + SnapshotVO snapshotVO = new SnapshotVO(dcId, 2, 1, 1L, 1L, UUID.randomUUID() + .toString(), (short) snapshotType.ordinal(), snapshotType.name(), 100, + Hypervisor.HypervisorType.XenServer); + return this.snapshotDao.persist(snapshotVO); + } + + private SnapshotVO createSnapshotInDb(Long volumeId) { + Snapshot.Type snapshotType = Snapshot.Type.DAILY; + SnapshotVO snapshotVO = new SnapshotVO(dcId, 2, 1, volumeId, 1L, UUID.randomUUID() + .toString(), (short) snapshotType.ordinal(), snapshotType.name(), 100, + Hypervisor.HypervisorType.XenServer); + return this.snapshotDao.persist(snapshotVO); + } + + private VolumeInfo createVolume(Long templateId, DataStore store) { + VolumeVO volume = new VolumeVO(Volume.Type.DATADISK, UUID.randomUUID().toString(), dcId, 1L, 1L, 1L, 1000, 0L, 0L, ""); + ; + volume.setPoolId(store.getId()); + + volume = volumeDao.persist(volume); + VolumeInfo volumeInfo = volumeDataFactory.getVolume(volume.getId(), store); + volumeInfo.stateTransit(Volume.Event.CreateRequested); + volumeInfo.stateTransit(Volume.Event.OperationSucceeded); + return volumeInfo; + } + private DataStore createDataStore() throws URISyntaxException { + StoragePoolVO pool = new StoragePoolVO(); + pool.setClusterId(clusterId); + pool.setDataCenterId(dcId); + URI uri = new URI("nfs://jfkdkf/fjdkfj"); + pool.setHostAddress(uri.getHost()); + pool.setPath(uri.getPath()); + pool.setPort(0); + pool.setName(UUID.randomUUID().toString()); + pool.setUuid(UUID.randomUUID().toString()); + pool.setStatus(StoragePoolStatus.Up); + pool.setPoolType(Storage.StoragePoolType.NetworkFilesystem); + pool.setPodId(podId); + pool.setScope(ScopeType.CLUSTER); + pool.setStorageProviderName(DataStoreProvider.DEFAULT_PRIMARY); + pool = this.primaryDataStoreDao.persist(pool); + DataStore store = this.dataStoreManager.getPrimaryDataStore(pool.getId()); + return store; + } + //@Test + public void testTakeSnapshot() throws URISyntaxException { + SnapshotVO snapshotVO = createSnapshotInDb(); + DataStore store = createDataStore(); + try { + SnapshotInfo snapshotInfo = snapshotDataFactory.getSnapshot(snapshotVO.getId(), store); + SnapshotResult result = snapshotService.takeSnapshot(snapshotInfo); + Assert.assertTrue(result.isSuccess()); + SnapshotDataStoreVO storeRef = snapshotDataStoreDao.findByStoreSnapshot(store.getRole(), store.getId(), snapshotVO.getId()); + Assert.assertTrue(storeRef != null); + Assert.assertTrue(storeRef.getState() == ObjectInDataStoreStateMachine.State.Ready); + snapshotInfo = result.getSnashot(); + boolean deletResult = snapshotService.deleteSnapshot(snapshotInfo); + Assert.assertTrue(deletResult); + snapshotDataStoreDao.expunge(storeRef.getId()); + } finally { + snapshotDao.expunge(snapshotVO.getId()); + primaryDataStoreDao.remove(store.getId()); + } + } + + //@Test + public void testTakeSnapshotWithFailed() throws URISyntaxException { + SnapshotVO snapshotVO = createSnapshotInDb(); + DataStore store = null; + try { + store = createDataStore(); + FakePrimaryDataStoreDriver dataStoreDriver = (FakePrimaryDataStoreDriver)store.getDriver(); + dataStoreDriver.makeTakeSnapshotSucceed(false); + SnapshotInfo snapshotInfo = snapshotDataFactory.getSnapshot(snapshotVO.getId(), store); + SnapshotResult result = snapshotService.takeSnapshot(snapshotInfo); + Assert.assertFalse(result.isSuccess()); + SnapshotDataStoreVO storeRef = snapshotDataStoreDao.findByStoreSnapshot(store.getRole(), store.getId(), snapshotVO.getId()); + Assert.assertTrue(storeRef == null); + } finally { + snapshotDao.expunge(snapshotVO.getId()); + if (store != null) { + primaryDataStoreDao.remove(store.getId()); + } + } + } + + //@Test + public void testTakeSnapshotFromVolume() throws URISyntaxException { + DataStore store = createDataStore(); + FakePrimaryDataStoreDriver dataStoreDriver = (FakePrimaryDataStoreDriver)store.getDriver(); + dataStoreDriver.makeTakeSnapshotSucceed(false); + VolumeInfo volumeInfo = createVolume(1L, store); + Assert.assertTrue(volumeInfo.getState() == Volume.State.Ready); + SnapshotInfo result = volumeService.takeSnapshot(volumeInfo); + Assert.assertTrue(volumeInfo.getState() == Volume.State.Ready); + Assert.assertTrue(result == null); + } + + protected SnapshotPolicyVO createSnapshotPolicy(Long volId) { + SnapshotPolicyVO policyVO = new SnapshotPolicyVO(volId, "jfkd", "fdfd", DateUtil.IntervalType.DAILY, 8); + policyVO = snapshotPolicyDao.persist(policyVO); + return policyVO; + } + + @Test + public void testConcurrentSnapshot() throws URISyntaxException, InterruptedException, ExecutionException { + DataStore store = createDataStore(); + final FakePrimaryDataStoreDriver dataStoreDriver = (FakePrimaryDataStoreDriver)store.getDriver(); + dataStoreDriver.makeTakeSnapshotSucceed(true); + final VolumeInfo volumeInfo = createVolume(1L, store); + Assert.assertTrue(volumeInfo.getState() == Volume.State.Ready); + vol = volumeInfo; + // final SnapshotPolicyVO policyVO = createSnapshotPolicy(vol.getId()); + + + ExecutorService pool = Executors.newFixedThreadPool(2); + boolean result = false; + List> future = new ArrayList>(); + for(int i = 0; i < 12; i++) { + final int cnt = i; + Future task = pool.submit(new Callable() { + @Override + public Boolean call() throws Exception { + boolean r = true; + try { + SnapshotVO snapshotVO = createSnapshotInDb(vol.getId()); + VolumeObject volumeObject = (VolumeObject)vol; + Account account = mock(Account.class); + when(account.getId()).thenReturn(1L); + CreateSnapshotPayload createSnapshotPayload = mock(CreateSnapshotPayload.class); + when(createSnapshotPayload.getAccount()).thenReturn(account); + when(createSnapshotPayload.getSnapshotId()).thenReturn(snapshotVO.getId()); + when(createSnapshotPayload.getSnapshotPolicyId()).thenReturn(0L); + volumeObject.addPayload(createSnapshotPayload); + if (cnt > 8) { + mockStorageMotionStrategy.makeBackupSnapshotSucceed(false); + } + SnapshotInfo newSnapshot = volumeService.takeSnapshot(vol); + if (newSnapshot == null) { + r = false; + } + } catch (Exception e) { + r = false; + } + return r; + } + }); + Assert.assertTrue(task.get()); + } + + } +} diff --git a/engine/storage/integration-test/test/resource/fakeDriverTestContext.xml b/engine/storage/integration-test/test/resource/fakeDriverTestContext.xml new file mode 100644 index 00000000000..b7ef363ff04 --- /dev/null +++ b/engine/storage/integration-test/test/resource/fakeDriverTestContext.xml @@ -0,0 +1,91 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/engine/storage/integration-test/test/resource/storageContext.xml b/engine/storage/integration-test/test/resource/storageContext.xml index f9c891a036f..664f1e3a290 100644 --- a/engine/storage/integration-test/test/resource/storageContext.xml +++ b/engine/storage/integration-test/test/resource/storageContext.xml @@ -47,7 +47,6 @@ - @@ -62,7 +61,6 @@ - @@ -82,7 +80,6 @@ - diff --git a/engine/storage/pom.xml b/engine/storage/pom.xml index cc561161602..9c638f1ea3d 100644 --- a/engine/storage/pom.xml +++ b/engine/storage/pom.xml @@ -16,7 +16,7 @@ org.apache.cloudstack cloud-engine - 4.2.0-SNAPSHOT + 4.2.0 ../pom.xml diff --git a/engine/storage/snapshot/pom.xml b/engine/storage/snapshot/pom.xml index 350a9a9eed6..fa43b607b8d 100644 --- a/engine/storage/snapshot/pom.xml +++ b/engine/storage/snapshot/pom.xml @@ -16,7 +16,7 @@ org.apache.cloudstack cloud-engine - 4.2.0-SNAPSHOT + 4.2.0 ../../pom.xml diff --git a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotObject.java b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotObject.java index 2fc576b8b8f..7d823a3b407 100644 --- a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotObject.java +++ b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotObject.java @@ -38,6 +38,7 @@ import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.storage.DataStoreRole; import com.cloud.storage.Snapshot; import com.cloud.storage.SnapshotVO; +import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.SnapshotDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.utils.component.ComponentContext; @@ -107,6 +108,7 @@ public class SnapshotObject implements SnapshotInfo { .create(SnapshotDataStoreVO.class); sc.addAnd(sc.getEntity().getDataStoreId(), Op.EQ, this.store.getId()); sc.addAnd(sc.getEntity().getRole(), Op.EQ, this.store.getRole()); + sc.addAnd(sc.getEntity().getState(), Op.NIN, State.Destroying, State.Destroyed, State.Error); sc.addAnd(sc.getEntity().getParentSnapshotId(), Op.EQ, this.getId()); SnapshotDataStoreVO vo = sc.find(); if (vo == null) { @@ -159,7 +161,7 @@ public class SnapshotObject implements SnapshotInfo { throw new CloudRuntimeException("Failed to update state: " + e.toString()); } finally { if (event == ObjectInDataStoreStateMachine.Event.OperationFailed) { - objectInStoreMgr.delete(this); + objectInStoreMgr.deleteIfNotReady(this); } } } @@ -261,18 +263,32 @@ public class SnapshotObject implements SnapshotInfo { snapshotStore.setParentSnapshotId(0L); } this.snapshotStoreDao.update(snapshotStore.getId(), snapshotStore); + + // update side-effect of snapshot operation + if(snapshotTO.getVolume() != null && snapshotTO.getVolume().getPath() != null) { + VolumeVO vol = this.volumeDao.findByUuid(snapshotTO.getVolume().getUuid()); + if(vol != null) { + s_logger.info("Update volume path change due to snapshot operation, volume " + vol.getId() + " path: " + + vol.getPath() + "->" + snapshotTO.getVolume().getPath()); + vol.setPath(snapshotTO.getVolume().getPath()); + this.volumeDao.update(vol.getId(), vol); + } else { + s_logger.error("Cound't find the original volume with uuid: " + snapshotTO.getVolume().getUuid()); + } + } } else { throw new CloudRuntimeException("Unknown answer: " + answer.getClass()); } } catch (RuntimeException ex) { if (event == ObjectInDataStoreStateMachine.Event.OperationFailed) { - objectInStoreMgr.delete(this); + objectInStoreMgr.deleteIfNotReady(this); } throw ex; } this.processEvent(event); } + @Override public void incRefCount() { if (this.store == null) { return; diff --git a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java index f8d9cbc3af9..00d96316e24 100644 --- a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java +++ b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java @@ -210,10 +210,6 @@ public class SnapshotServiceImpl implements SnapshotService { try { result = future.get(); - if (result.isFailed()) { - s_logger.debug("Failed to create snapshot:" + result.getResult()); - throw new CloudRuntimeException(result.getResult()); - } return result; } catch (InterruptedException e) { s_logger.debug("Failed to create snapshot", e); @@ -222,7 +218,6 @@ public class SnapshotServiceImpl implements SnapshotService { s_logger.debug("Failed to create snapshot", e); throw new CloudRuntimeException("Failed to create snapshot", e); } - } // if a snapshot has parent snapshot, the new snapshot should be stored in @@ -252,7 +247,6 @@ public class SnapshotServiceImpl implements SnapshotService { AsyncCallFuture future = new AsyncCallFuture(); SnapshotResult result = new SnapshotResult(snapshot, null); try { - snapObj.processEvent(Snapshot.Event.BackupToSecondary); DataStore imageStore = this.findSnapshotImageStore(snapshot); diff --git a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/XenserverSnapshotStrategy.java b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/XenserverSnapshotStrategy.java index ebbce5b16a4..93d6a3ca5b3 100644 --- a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/XenserverSnapshotStrategy.java +++ b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/XenserverSnapshotStrategy.java @@ -18,6 +18,7 @@ package org.apache.cloudstack.storage.snapshot; import javax.inject.Inject; +import com.cloud.utils.db.DB; import org.apache.cloudstack.engine.subsystem.api.storage.*; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.Event; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.State; @@ -58,11 +59,43 @@ public class XenserverSnapshotStrategy extends SnapshotStrategyBase { @Inject SnapshotDataFactory snapshotDataFactory; + protected boolean needFullBackup(SnapshotInfo snapshot) { + if (snapshot == null) { + return true; + } + boolean fullBackup = false; + int _deltaSnapshotMax = NumbersUtil.parseInt(configDao.getValue("snapshot.delta.max"), + SnapshotManager.DELTAMAX); + int deltaSnap = _deltaSnapshotMax; + SnapshotDataStoreVO parentSnapshotOnBackupStore = this.snapshotStoreDao.findBySnapshot(snapshot.getId(), DataStoreRole.Image); + int i; + for (i = 1; (i < deltaSnap && (parentSnapshotOnBackupStore != null)); i++) { + Long prevBackupId = parentSnapshotOnBackupStore.getParentSnapshotId(); + + if (prevBackupId == 0 || prevBackupId == null) { + break; + } + + parentSnapshotOnBackupStore = this.snapshotStoreDao.findBySnapshot(prevBackupId, DataStoreRole.Image); + } + if (i >= deltaSnap) { + fullBackup = true; + } + return fullBackup; + } + + protected boolean isEmptySnapshot(SnapshotInfo snapshotInfo, SnapshotInfo parentSnapshot) { + if (parentSnapshot != null && snapshotInfo.getPath().equalsIgnoreCase(parentSnapshot.getPath())) { + return true; + } else { + return false; + } + } @Override public SnapshotInfo backupSnapshot(SnapshotInfo snapshot) { SnapshotInfo parentSnapshot = snapshot.getParent(); - - if (parentSnapshot != null && snapshot.getPath().equalsIgnoreCase(parentSnapshot.getPath())) { + boolean fullBackup = needFullBackup(parentSnapshot); + if (!fullBackup && isEmptySnapshot(snapshot, parentSnapshot)) { s_logger.debug("backup an empty snapshot"); // don't need to backup this snapshot SnapshotDataStoreVO parentSnapshotOnBackupStore = this.snapshotStoreDao.findBySnapshot( @@ -93,55 +126,59 @@ public class XenserverSnapshotStrategy extends SnapshotStrategyBase { } } - // determine full snapshot backup or not - - boolean fullBackup = false; - - if (parentSnapshot != null) { - int _deltaSnapshotMax = NumbersUtil.parseInt(configDao.getValue("snapshot.delta.max"), - SnapshotManager.DELTAMAX); - int deltaSnap = _deltaSnapshotMax; - - int i; - SnapshotDataStoreVO parentSnapshotOnBackupStore = null; - for (i = 1; i < deltaSnap; i++) { - parentSnapshotOnBackupStore = this.snapshotStoreDao.findBySnapshot(parentSnapshot.getId(), - DataStoreRole.Image); - if (parentSnapshotOnBackupStore == null) { - break; - } - Long prevBackupId = parentSnapshotOnBackupStore.getParentSnapshotId(); - - if (prevBackupId == 0) { - break; - } - - parentSnapshotOnBackupStore = this.snapshotStoreDao.findBySnapshot(prevBackupId, DataStoreRole.Image); - } - if (i >= deltaSnap) { - fullBackup = true; + //if fullbackup, delete the snapshot on primary storage in db, so that, snapshot.getparent will return empty, thus full snapshot on the backend + if (fullBackup && parentSnapshot != null) { + s_logger.debug("reach delta snapshots max, delete the snapshot on primary storage in db"); + SnapshotDataStoreVO parentSnapshotOnPrimary = snapshotStoreDao.findByStoreSnapshot(DataStoreRole.Primary, parentSnapshot.getDataStore().getId(), + parentSnapshot.getId()); + if (parentSnapshotOnPrimary != null) { + snapshotStoreDao.remove(parentSnapshotOnPrimary.getId()); } } - - snapshot.addPayload(fullBackup); return this.snapshotSvr.backupSnapshot(snapshot); } protected boolean deleteSnapshotChain(SnapshotInfo snapshot) { s_logger.debug("delete snapshot chain for snapshot: " + snapshot.getId()); boolean result = false; - while (snapshot != null && (snapshot.getState() == Snapshot.State.Destroying || snapshot.getState() - == Snapshot.State.Destroyed || snapshot.getState() == Snapshot.State.Error)) { - SnapshotInfo child = snapshot.getChild(); + boolean resultIsSet = false; //need to track, the snapshot itself is deleted or not. + try { + while (snapshot != null && (snapshot.getState() == Snapshot.State.Destroying || snapshot.getState() + == Snapshot.State.Destroyed || snapshot.getState() == Snapshot.State.Error)) { + SnapshotInfo child = snapshot.getChild(); - if (child != null) { - s_logger.debug("the snapshot has child, can't delete it on the storage"); - break; + if (child != null) { + s_logger.debug("the snapshot has child, can't delete it on the storage"); + break; + } + s_logger.debug("Snapshot: " + snapshot.getId() + " doesn't have children, so it's ok to delete it and its parents"); + SnapshotInfo parent = snapshot.getParent(); + boolean deleted = false; + if (parent != null) { + if (parent.getPath() != null && parent.getPath().equalsIgnoreCase(snapshot.getPath())) { + //NOTE: if both snapshots share the same path, it's for xenserver's empty delta snapshot. We can't delete the snapshot on the backend, as parent snapshot still reference to it + //Instead, mark it as destroyed in the db. + s_logger.debug("for empty delta snapshot, only mark it as destroyed in db"); + snapshot.processEvent(Event.DestroyRequested); + snapshot.processEvent(Event.OperationSuccessed); + deleted = true; + if (!resultIsSet) { + result = true; + resultIsSet = true; + } + } + } + if (!deleted) { + boolean r = this.snapshotSvr.deleteSnapshot(snapshot); + if (!resultIsSet) { + result = r; + resultIsSet = true; + } + } + snapshot = parent; } - s_logger.debug("Snapshot: " + snapshot.getId() + " doesn't have children, so it's ok to delete it and its parents"); - SnapshotInfo parent = snapshot.getParent(); - result = this.snapshotSvr.deleteSnapshot(snapshot); - snapshot = parent; + } catch (Exception e) { + s_logger.debug("delete snapshot failed: ", e); } return result; } @@ -153,7 +190,6 @@ public class XenserverSnapshotStrategy extends SnapshotStrategyBase { return true; } - if (snapshotVO.getState() == Snapshot.State.CreatedOnPrimary) { s_logger.debug("delete snapshot on primary storage:"); snapshotVO.setState(Snapshot.State.Destroyed); @@ -163,16 +199,17 @@ public class XenserverSnapshotStrategy extends SnapshotStrategyBase { if (!Snapshot.State.BackedUp.equals(snapshotVO.getState())) { throw new InvalidParameterValueException("Can't delete snapshotshot " + snapshotId - + " due to it is not in BackedUp Status"); + + " due to it is in " + snapshotVO.getState() + " Status"); } - // firt mark the snapshot as destroyed, so that ui can't see it, but we - // may not destroy the snapshot on the storage, as other snaphosts may + // first mark the snapshot as destroyed, so that ui can't see it, but we + // may not destroy the snapshot on the storage, as other snapshots may // depend on it. SnapshotInfo snapshotOnImage = this.snapshotDataFactory.getSnapshot(snapshotId, DataStoreRole.Image); if (snapshotOnImage == null) { s_logger.debug("Can't find snapshot on backup storage, delete it in db"); snapshotDao.remove(snapshotId); + return true; } SnapshotObject obj = (SnapshotObject) snapshotOnImage; @@ -201,46 +238,60 @@ public class XenserverSnapshotStrategy extends SnapshotStrategyBase { } catch (NoTransitionException e1) { s_logger.debug("Failed to change snapshot state: " + e.toString()); } + return false; } return true; } @Override + @DB public SnapshotInfo takeSnapshot(SnapshotInfo snapshot) { - SnapshotResult result = snapshotSvr.takeSnapshot(snapshot); - if (result.isFailed()) { - s_logger.debug("Failed to take snapshot: " + result.getResult()); - throw new CloudRuntimeException(result.getResult()); + SnapshotVO snapshotVO = snapshotDao.acquireInLockTable(snapshot.getId()); + if (snapshotVO == null) { + throw new CloudRuntimeException("Failed to get lock on snapshot:" + snapshot.getId()); } - snapshot = result.getSnashot(); - DataStore primaryStore = snapshot.getDataStore(); - SnapshotInfo backupedSnapshot = this.backupSnapshot(snapshot); try { - SnapshotInfo parent = snapshot.getParent(); - if (backupedSnapshot != null && parent != null) { - Long parentSnapshotId = parent.getId(); - while (parentSnapshotId != null && parentSnapshotId != 0L) { - SnapshotDataStoreVO snapshotDataStoreVO = snapshotStoreDao.findByStoreSnapshot(primaryStore.getRole(),primaryStore.getId(), parentSnapshotId); + SnapshotResult result = snapshotSvr.takeSnapshot(snapshot); + if (result.isFailed()) { + s_logger.debug("Failed to take snapshot: " + result.getResult()); + throw new CloudRuntimeException(result.getResult()); + } + snapshot = result.getSnashot(); + DataStore primaryStore = snapshot.getDataStore(); + + SnapshotInfo backupedSnapshot = this.backupSnapshot(snapshot); + + try { + SnapshotInfo parent = snapshot.getParent(); + if (backupedSnapshot != null && parent != null) { + Long parentSnapshotId = parent.getId(); + while (parentSnapshotId != null && parentSnapshotId != 0L) { + SnapshotDataStoreVO snapshotDataStoreVO = snapshotStoreDao.findByStoreSnapshot(primaryStore.getRole(),primaryStore.getId(), parentSnapshotId); + if (snapshotDataStoreVO != null) { + parentSnapshotId = snapshotDataStoreVO.getParentSnapshotId(); + snapshotStoreDao.remove(snapshotDataStoreVO.getId()); + } else { + parentSnapshotId = null; + } + } + SnapshotDataStoreVO snapshotDataStoreVO = snapshotStoreDao.findByStoreSnapshot(primaryStore.getRole(), primaryStore.getId(), + snapshot.getId()); if (snapshotDataStoreVO != null) { - parentSnapshotId = snapshotDataStoreVO.getParentSnapshotId(); - snapshotStoreDao.remove(snapshotDataStoreVO.getId()); - } else { - parentSnapshotId = null; + snapshotDataStoreVO.setParentSnapshotId(0L); + snapshotStoreDao.update(snapshotDataStoreVO.getId(), snapshotDataStoreVO); } } - SnapshotDataStoreVO snapshotDataStoreVO = snapshotStoreDao.findByStoreSnapshot(primaryStore.getRole(), primaryStore.getId(), - snapshot.getId()); - if (snapshotDataStoreVO != null) { - snapshotDataStoreVO.setParentSnapshotId(0L); - snapshotStoreDao.update(snapshotDataStoreVO.getId(), snapshotDataStoreVO); - } + } catch (Exception e) { + s_logger.debug("Failed to clean up snapshots on primary storage", e); + } + return backupedSnapshot; + } finally { + if (snapshotVO != null) { + snapshotDao.releaseFromLockTable(snapshot.getId()); } - } catch (Exception e) { - s_logger.debug("Failed to clean up snapshots on primary storage", e); } - return backupedSnapshot; } @Override diff --git a/engine/storage/src/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java b/engine/storage/src/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java index 89e09748ea3..e5a1d8bf864 100755 --- a/engine/storage/src/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java +++ b/engine/storage/src/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java @@ -183,14 +183,6 @@ public abstract class AbstractStoragePoolAllocator extends AdapterBase implement return false; } - - DiskOfferingVO diskOffering = _diskOfferingDao.findById(dskCh.getDiskOfferingId()); - if (diskOffering.getSystemUse() && pool.getPoolType() == StoragePoolType.RBD) { - s_logger.debug("Skipping RBD pool " + pool.getName() - + " as a suitable pool. RBD is not supported for System VM's"); - return false; - } - Long clusterId = pool.getClusterId(); ClusterVO cluster = _clusterDao.findById(clusterId); if (!(cluster.getHypervisorType() == dskCh.getHypervisorType())) { diff --git a/engine/storage/src/org/apache/cloudstack/storage/allocator/ZoneWideStoragePoolAllocator.java b/engine/storage/src/org/apache/cloudstack/storage/allocator/ZoneWideStoragePoolAllocator.java index 300d932a31c..f1723c0fc09 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/allocator/ZoneWideStoragePoolAllocator.java +++ b/engine/storage/src/org/apache/cloudstack/storage/allocator/ZoneWideStoragePoolAllocator.java @@ -17,10 +17,13 @@ package org.apache.cloudstack.storage.allocator; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import javax.inject.Inject; +import com.cloud.user.Account; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; @@ -101,4 +104,35 @@ public class ZoneWideStoragePoolAllocator extends AbstractStoragePoolAllocator { } return suitablePools; } + @Override + protected List reorderPoolsByNumberOfVolumes(DeploymentPlan plan, List pools, + Account account) { + if (account == null) { + return pools; + } + long dcId = plan.getDataCenterId(); + + List poolIdsByVolCount = _volumeDao.listZoneWidePoolIdsByVolumeCount(dcId, + account.getAccountId()); + if (s_logger.isDebugEnabled()) { + s_logger.debug("List of pools in ascending order of number of volumes for account id: " + + account.getAccountId() + " is: " + poolIdsByVolCount); + } + + // now filter the given list of Pools by this ordered list + Map poolMap = new HashMap(); + for (StoragePool pool : pools) { + poolMap.put(pool.getId(), pool); + } + List matchingPoolIds = new ArrayList(poolMap.keySet()); + + poolIdsByVolCount.retainAll(matchingPoolIds); + + List reorderedPools = new ArrayList(); + for (Long id : poolIdsByVolCount) { + reorderedPools.add(poolMap.get(id)); + } + + return reorderedPools; + } } diff --git a/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManager.java b/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManager.java index fbd315e3826..3dc6ac497a9 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManager.java +++ b/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManager.java @@ -31,6 +31,8 @@ public interface ObjectInDataStoreManager { public boolean delete(DataObject dataObj); + public boolean deleteIfNotReady(DataObject dataObj); + public DataObject get(DataObject dataObj, DataStore store); public boolean update(DataObject vo, Event event) throws NoTransitionException, ConcurrentOperationException; diff --git a/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java b/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java index c673776357e..652df43c785 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java +++ b/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java @@ -232,6 +232,60 @@ public class ObjectInDataStoreManagerImpl implements ObjectInDataStoreManager { return false; } + @Override + public boolean deleteIfNotReady(DataObject dataObj) { + long objId = dataObj.getId(); + DataStore dataStore = dataObj.getDataStore(); + if (dataStore.getRole() == DataStoreRole.Primary) { + if (dataObj.getType() == DataObjectType.TEMPLATE) { + VMTemplateStoragePoolVO destTmpltPool = templatePoolDao.findByPoolTemplate(dataStore.getId(), objId); + if (destTmpltPool != null && destTmpltPool.getState() != ObjectInDataStoreStateMachine.State.Ready) { + return templatePoolDao.remove(destTmpltPool.getId()); + } else { + s_logger.warn("Template " + objId + " is not found on storage pool " + dataStore.getId() + ", so no need to delete"); + return true; + } + } else if (dataObj.getType() == DataObjectType.SNAPSHOT) { + SnapshotDataStoreVO destSnapshotStore = snapshotDataStoreDao.findByStoreSnapshot(dataStore.getRole(), dataStore.getId(), objId); + if (destSnapshotStore != null && destSnapshotStore.getState() != ObjectInDataStoreStateMachine.State.Ready) { + snapshotDataStoreDao.remove(destSnapshotStore.getId()); + } + return true; + } + } else { + // Image store + switch (dataObj.getType()) { + case TEMPLATE: + TemplateDataStoreVO destTmpltStore = templateDataStoreDao.findByStoreTemplate(dataStore.getId(), objId); + if (destTmpltStore != null && destTmpltStore.getState() != ObjectInDataStoreStateMachine.State.Ready) { + return templateDataStoreDao.remove(destTmpltStore.getId()); + } else { + s_logger.warn("Template " + objId + " is not found on image store " + dataStore.getId() + ", so no need to delete"); + return true; + } + case SNAPSHOT: + SnapshotDataStoreVO destSnapshotStore = snapshotDataStoreDao.findByStoreSnapshot(dataStore.getRole(), dataStore.getId(), objId); + if (destSnapshotStore != null && destSnapshotStore.getState() != ObjectInDataStoreStateMachine.State.Ready) { + return snapshotDataStoreDao.remove(destSnapshotStore.getId()); + } else { + s_logger.warn("Snapshot " + objId + " is not found on image store " + dataStore.getId() + ", so no need to delete"); + return true; + } + case VOLUME: + VolumeDataStoreVO destVolumeStore = volumeDataStoreDao.findByStoreVolume(dataStore.getId(), objId); + if (destVolumeStore != null && destVolumeStore.getState() != ObjectInDataStoreStateMachine.State.Ready) { + return volumeDataStoreDao.remove(destVolumeStore.getId()); + } else { + s_logger.warn("Volume " + objId + " is not found on image store " + dataStore.getId() + ", so no need to delete"); + return true; + } + } + } + + s_logger.warn("Unsupported data object (" + dataObj.getType() + ", " + dataObj.getDataStore() + "), no need to delete from object in store ref table"); + return false; + } + @Override public boolean update(DataObject data, Event event) throws NoTransitionException, ConcurrentOperationException { DataObjectInStore obj = this.findObject(data, data.getDataStore()); diff --git a/engine/storage/src/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java b/engine/storage/src/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java index c3f52ffcf29..98c6a3fc001 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java +++ b/engine/storage/src/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java @@ -55,7 +55,8 @@ public class DefaultEndPointSelector implements EndPointSelector { private static final Logger s_logger = Logger.getLogger(DefaultEndPointSelector.class); @Inject HostDao hostDao; - private String findOneHostOnPrimaryStorage = "select id from host where " + "status = 'Up' and type = 'Routing' "; + private String findOneHostOnPrimaryStorage = "select h.id from host h, storage_pool_host_ref s where h.status = 'Up' and h.type = 'Routing' and h.resource_state = 'Enabled' and" + + " h.id = s.host_id and s.pool_id = ? "; protected boolean moveBetweenPrimaryImage(DataStore srcStore, DataStore destStore) { DataStoreRole srcRole = srcStore.getRole(); @@ -90,18 +91,18 @@ public class DefaultEndPointSelector implements EndPointSelector { } @DB - protected EndPoint findEndPointInScope(Scope scope, String sqlBase) { + protected EndPoint findEndPointInScope(Scope scope, String sqlBase, Long poolId) { StringBuilder sbuilder = new StringBuilder(); sbuilder.append(sqlBase); if (scope.getScopeType() == ScopeType.HOST) { - sbuilder.append(" and id = "); + sbuilder.append(" and h.id = "); sbuilder.append(scope.getScopeId()); } else if (scope.getScopeType() == ScopeType.CLUSTER) { - sbuilder.append(" and cluster_id = "); + sbuilder.append(" and h.cluster_id = "); sbuilder.append(scope.getScopeId()); } else if (scope.getScopeType() == ScopeType.ZONE) { - sbuilder.append(" and data_center_id = "); + sbuilder.append(" and h.data_center_id = "); sbuilder.append(scope.getScopeId()); } // TODO: order by rand() is slow if there are lot of hosts @@ -114,6 +115,7 @@ public class DefaultEndPointSelector implements EndPointSelector { try { pstmt = txn.prepareStatement(sql); + pstmt.setLong(1, poolId); rs = pstmt.executeQuery(); while (rs.next()) { long id = rs.getLong(1); @@ -146,17 +148,26 @@ public class DefaultEndPointSelector implements EndPointSelector { Scope srcScope = srcStore.getScope(); Scope destScope = destStore.getScope(); Scope selectedScope = null; + Long poolId = null; + // assumption, at least one of scope should be zone, find the least // scope if (srcScope.getScopeType() != ScopeType.ZONE) { selectedScope = srcScope; + poolId = srcStore.getId(); } else if (destScope.getScopeType() != ScopeType.ZONE) { selectedScope = destScope; + poolId = destStore.getId(); } else { // if both are zone scope selectedScope = srcScope; + if (srcStore.getRole() == DataStoreRole.Primary) { + poolId = srcStore.getId(); + } else if (destStore.getRole() == DataStoreRole.Primary) { + poolId = destStore.getId(); + } } - return findEndPointInScope(selectedScope, findOneHostOnPrimaryStorage); + return findEndPointInScope(selectedScope, findOneHostOnPrimaryStorage, poolId); } @Override @@ -166,7 +177,14 @@ public class DefaultEndPointSelector implements EndPointSelector { if (moveBetweenPrimaryImage(srcStore, destStore)) { return findEndPointForImageMove(srcStore, destStore); } else if (moveBetweenCacheAndImage(srcStore, destStore)) { - EndPoint ep = findEndpointForImageStorage(destStore); + // pick ssvm based on image cache dc + DataStore selectedStore = null; + if (srcStore.getRole() == DataStoreRole.ImageCache) { + selectedStore = srcStore; + } else { + selectedStore = destStore; + } + EndPoint ep = findEndpointForImageStorage(selectedStore); return ep; } else if (moveBetweenImages(srcStore, destStore)) { EndPoint ep = findEndpointForImageStorage(destStore); @@ -177,7 +195,7 @@ public class DefaultEndPointSelector implements EndPointSelector { } protected EndPoint findEndpointForPrimaryStorage(DataStore store) { - return findEndPointInScope(store.getScope(), findOneHostOnPrimaryStorage); + return findEndPointInScope(store.getScope(), findOneHostOnPrimaryStorage, store.getId()); } protected EndPoint findEndpointForImageStorage(DataStore store) { @@ -221,7 +239,7 @@ public class DefaultEndPointSelector implements EndPointSelector { public EndPoint select(DataStore store) { if (store.getRole() == DataStoreRole.Primary) { return findEndpointForPrimaryStorage(store); - } else if (store.getRole() == DataStoreRole.Image) { + } else if (store.getRole() == DataStoreRole.Image || store.getRole() == DataStoreRole.ImageCache) { // in case there is no ssvm, directly send down command hypervisor // host // otherwise, send to localhost for bootstrap system vm template diff --git a/engine/storage/src/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java b/engine/storage/src/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java index c8d4e4533fa..2905f081061 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java +++ b/engine/storage/src/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java @@ -106,7 +106,8 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { } @Override - public void createAsync(DataStore dataStore, DataObject data, AsyncCompletionCallback callback) { + public void + createAsync(DataStore dataStore, DataObject data, AsyncCompletionCallback callback) { CreateContext context = new CreateContext(callback, data); AsyncCallbackDispatcher caller = AsyncCallbackDispatcher .create(this); diff --git a/engine/storage/src/org/apache/cloudstack/storage/image/db/ImageStoreDaoImpl.java b/engine/storage/src/org/apache/cloudstack/storage/image/db/ImageStoreDaoImpl.java index acbbc7d74a8..b9ef9c307af 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/image/db/ImageStoreDaoImpl.java +++ b/engine/storage/src/org/apache/cloudstack/storage/image/db/ImageStoreDaoImpl.java @@ -38,6 +38,7 @@ import com.cloud.utils.db.SearchCriteria; public class ImageStoreDaoImpl extends GenericDaoBase implements ImageStoreDao { private SearchBuilder nameSearch; private SearchBuilder providerSearch; + private SearchBuilder regionSearch; @Override public boolean configure(String name, Map params) throws ConfigurationException { @@ -50,9 +51,14 @@ public class ImageStoreDaoImpl extends GenericDaoBase implem providerSearch = createSearchBuilder(); providerSearch.and("providerName", providerSearch.entity().getProviderName(), SearchCriteria.Op.EQ); - providerSearch.and("role", providerSearch.entity().getProviderName(), SearchCriteria.Op.EQ); + providerSearch.and("role", providerSearch.entity().getRole(), SearchCriteria.Op.EQ); providerSearch.done(); + regionSearch = createSearchBuilder(); + regionSearch.and("scope", regionSearch.entity().getScope(), SearchCriteria.Op.EQ); + regionSearch.and("role", regionSearch.entity().getRole(), SearchCriteria.Op.EQ); + regionSearch.done(); + return true; } @@ -86,6 +92,14 @@ public class ImageStoreDaoImpl extends GenericDaoBase implem return listBy(sc); } + @Override + public List findRegionImageStores() { + SearchCriteria sc = regionSearch.create(); + sc.setParameters("scope", ScopeType.REGION); + sc.setParameters("role", DataStoreRole.Image); + return listBy(sc); + } + @Override public List findImageCacheByScope(ZoneScope scope) { SearchCriteria sc = createSearchCriteria(); diff --git a/engine/storage/src/org/apache/cloudstack/storage/image/db/SnapshotDataStoreDaoImpl.java b/engine/storage/src/org/apache/cloudstack/storage/image/db/SnapshotDataStoreDaoImpl.java index f33d51aebe7..23deed3cb0a 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/image/db/SnapshotDataStoreDaoImpl.java +++ b/engine/storage/src/org/apache/cloudstack/storage/image/db/SnapshotDataStoreDaoImpl.java @@ -25,6 +25,7 @@ import java.util.Map; import javax.naming.ConfigurationException; +import com.cloud.utils.db.DB; import org.apache.cloudstack.engine.subsystem.api.storage.DataObjectInStore; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.Event; @@ -51,6 +52,7 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase cacheSearch; private SearchBuilder snapshotSearch; private SearchBuilder storeSnapshotSearch; + private SearchBuilder snapshotIdSearch; private String parentSearch = "select store_id, store_role, snapshot_id from cloud.snapshot_store_ref where store_id = ? " + " and store_role = ? and volume_id = ? and state = 'Ready'" + " order by created DESC " + @@ -68,6 +70,7 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase sc = storeSearch.create(); sc.setParameters("store_id", id); sc.setParameters("store_role", role); + sc.setParameters("state", ObjectInDataStoreStateMachine.State.Destroyed); return listBy(sc); } @@ -171,6 +179,7 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase findBySnapshotId(long snapshotId) { + SearchCriteria sc = snapshotIdSearch.create(); + sc.setParameters("snapshot_id", snapshotId); + return listBy(sc); + } + @Override public List listDestroyed(long id) { SearchCriteria sc = destroyedSearch.create(); diff --git a/engine/schema/src/com/cloud/storage/dao/S3Dao.java b/engine/storage/src/org/apache/cloudstack/storage/image/format/RAW.java similarity index 65% rename from engine/schema/src/com/cloud/storage/dao/S3Dao.java rename to engine/storage/src/org/apache/cloudstack/storage/image/format/RAW.java index ebea3531339..84adff2ef88 100644 --- a/engine/schema/src/com/cloud/storage/dao/S3Dao.java +++ b/engine/storage/src/org/apache/cloudstack/storage/image/format/RAW.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an @@ -16,14 +16,17 @@ * specific language governing permissions and limitations * under the License. */ -package com.cloud.storage.dao; +package org.apache.cloudstack.storage.image.format; -import com.cloud.agent.api.to.S3TO; -import com.cloud.storage.S3VO; -import com.cloud.utils.db.GenericDao; +import org.apache.cloudstack.storage.BaseType; +import org.springframework.stereotype.Component; -public interface S3Dao extends GenericDao { +@Component("imageformat_raw") +public class RAW extends BaseType implements ImageFormat { + private final String type = "RAW"; - S3TO getS3TO(final Long id); - -} + @Override + public String toString() { + return type; + } +} \ No newline at end of file diff --git a/engine/storage/volume/pom.xml b/engine/storage/volume/pom.xml index 19357ab11e4..da62ec954d4 100644 --- a/engine/storage/volume/pom.xml +++ b/engine/storage/volume/pom.xml @@ -16,7 +16,7 @@ org.apache.cloudstack cloud-engine - 4.2.0-SNAPSHOT + 4.2.0 ../../pom.xml diff --git a/engine/storage/volume/src/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java b/engine/storage/volume/src/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java index 0432381ddb6..bbccfcdafb3 100644 --- a/engine/storage/volume/src/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java +++ b/engine/storage/volume/src/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java @@ -23,6 +23,7 @@ import java.util.List; import javax.inject.Inject; +import com.cloud.utils.db.GlobalLock; import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreDriver; @@ -225,34 +226,52 @@ public class PrimaryDataStoreImpl implements PrimaryDataStore { public DataObject create(DataObject obj) { // create template on primary storage if (obj.getType() == DataObjectType.TEMPLATE) { - VMTemplateStoragePoolVO templateStoragePoolRef = templatePoolDao.findByPoolTemplate(this.getId(), - obj.getId()); - if (templateStoragePoolRef == null) { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Not found (templateId: " + obj.getId() + ", poolId: " + this.getId() + ") in template_spool_ref"); + try{ + String templateIdPoolIdString = "templateId:" + obj.getId() + "poolId:" + this.getId(); + VMTemplateStoragePoolVO templateStoragePoolRef; + GlobalLock lock = GlobalLock.getInternLock(templateIdPoolIdString); + if (!lock.lock(5)) { + s_logger.debug("Couldn't lock the db on the string " + templateIdPoolIdString); + return null; } try { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Persisting (templateId: " + obj.getId() + ", poolId: " + this.getId() + ") to template_spool_ref"); - } - templateStoragePoolRef = new VMTemplateStoragePoolVO(this.getId(), obj.getId()); - templateStoragePoolRef = templatePoolDao.persist(templateStoragePoolRef); - } catch (Throwable t) { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Failed to insert (templateId: " + obj.getId() + ", poolId: " + this.getId() + ") to template_spool_ref", t); - } - templateStoragePoolRef = templatePoolDao.findByPoolTemplate(this.getId(), obj.getId()); + templateStoragePoolRef = templatePoolDao.findByPoolTemplate(this.getId(), + obj.getId()); if (templateStoragePoolRef == null) { - throw new CloudRuntimeException("Failed to create template storage pool entry"); - } else { + if (s_logger.isDebugEnabled()) { - s_logger.debug("Another thread already inserts " + templateStoragePoolRef.getId() + " to template_spool_ref", t); + s_logger.debug("Not found (" + templateIdPoolIdString + ") in template_spool_ref, persisting it"); } + templateStoragePoolRef = new VMTemplateStoragePoolVO(this.getId(), obj.getId()); + templateStoragePoolRef = templatePoolDao.persist(templateStoragePoolRef); } + } catch (Throwable t) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Failed to insert (" + templateIdPoolIdString + ") to template_spool_ref", t); + } + templateStoragePoolRef = templatePoolDao.findByPoolTemplate(this.getId(), obj.getId()); + if (templateStoragePoolRef == null) { + throw new CloudRuntimeException("Failed to create template storage pool entry"); + } else { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Another thread already inserts " + templateStoragePoolRef.getId() + " to template_spool_ref", t); + } + } + }finally { + lock.unlock(); + lock.releaseRef(); } + } catch (Exception e){ + s_logger.debug("Caught exception ", e); } } else if (obj.getType() == DataObjectType.SNAPSHOT) { return objectInStoreMgr.create(obj, this); + } else if (obj.getType() == DataObjectType.VOLUME) { + VolumeVO vol = volumeDao.findById(obj.getId()); + if (vol != null) { + vol.setPoolId(this.getId()); + volumeDao.update(vol.getId(), vol); + } } return objectInStoreMgr.get(obj, this); diff --git a/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeDataFactoryImpl.java b/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeDataFactoryImpl.java index 2512c49348c..1d75ba1529b 100644 --- a/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeDataFactoryImpl.java +++ b/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeDataFactoryImpl.java @@ -74,6 +74,9 @@ public class VolumeDataFactoryImpl implements VolumeDataFactory { @Override public VolumeInfo getVolume(long volumeId) { VolumeVO volumeVO = volumeDao.findById(volumeId); + if (volumeVO == null) { + return null; + } VolumeObject vol = null; if (volumeVO.getPoolId() == null) { DataStore store = null; diff --git a/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeObject.java b/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeObject.java index b5968b676d7..f5a1276cf2d 100644 --- a/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeObject.java +++ b/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeObject.java @@ -293,7 +293,7 @@ public class VolumeObject implements VolumeInfo { // in case of OperationFailed, expunge the entry if (event == ObjectInDataStoreStateMachine.Event.OperationFailed && (this.volumeVO.getState() != Volume.State.Copying && this.volumeVO.getState() != Volume.State.Uploaded)) { - objectInStoreMgr.delete(this); + objectInStoreMgr.deleteIfNotReady(this); } } @@ -309,7 +309,7 @@ public class VolumeObject implements VolumeInfo { } finally { // in case of OperationFailed, expunge the entry if (event == ObjectInDataStoreStateMachine.Event.OperationFailed) { - objectInStoreMgr.delete(this); + objectInStoreMgr.deleteIfNotReady(this); } } } @@ -335,7 +335,11 @@ public class VolumeObject implements VolumeInfo { return this.volumeVO.getPath(); } else { DataObjectInStore objInStore = this.objectInStoreMgr.findObject(this, dataStore); - return objInStore.getInstallPath(); + if (objInStore != null) { + return objInStore.getInstallPath(); + } else { + return null; + } } } @@ -471,7 +475,12 @@ public class VolumeObject implements VolumeInfo { VolumeVO vol = this.volumeDao.findById(this.getId()); VolumeObjectTO newVol = (VolumeObjectTO) cpyAnswer.getNewData(); vol.setPath(newVol.getPath()); - vol.setSize(newVol.getSize()); + if (newVol.getSize() != null) { + vol.setSize(newVol.getSize()); + } + if (newVol.getFormat() != null) { + vol.setFormat(newVol.getFormat()); + } vol.setPoolId(this.getDataStore().getId()); volumeDao.update(vol.getId(), vol); } else if (answer instanceof CreateObjectAnswer) { @@ -479,8 +488,13 @@ public class VolumeObject implements VolumeInfo { VolumeObjectTO newVol = (VolumeObjectTO) createAnswer.getData(); VolumeVO vol = this.volumeDao.findById(this.getId()); vol.setPath(newVol.getPath()); - vol.setSize(newVol.getSize()); + if (newVol.getSize() != null) { + vol.setSize(newVol.getSize()); + } vol.setPoolId(this.getDataStore().getId()); + if (newVol.getFormat() != null) { + vol.setFormat(newVol.getFormat()); + } volumeDao.update(vol.getId(), vol); } } else { @@ -498,13 +512,15 @@ public class VolumeObject implements VolumeInfo { this.getId()); VolumeObjectTO newVol = (VolumeObjectTO) cpyAnswer.getNewData(); volStore.setInstallPath(newVol.getPath()); - volStore.setSize(newVol.getSize()); + if (newVol.getSize() != null) { + volStore.setSize(newVol.getSize()); + } this.volumeStoreDao.update(volStore.getId(), volStore); } } } catch (RuntimeException ex) { if (event == ObjectInDataStoreStateMachine.Event.OperationFailed) { - objectInStoreMgr.delete(this); + objectInStoreMgr.deleteIfNotReady(this); } throw ex; } @@ -560,7 +576,9 @@ public class VolumeObject implements VolumeInfo { VolumeVO vol = this.volumeDao.findById(this.getId()); VolumeObjectTO newVol = (VolumeObjectTO) cpyAnswer.getNewData(); vol.setPath(newVol.getPath()); - vol.setSize(newVol.getSize()); + if (newVol.getSize() != null) { + vol.setSize(newVol.getSize()); + } vol.setPoolId(this.getDataStore().getId()); volumeDao.update(vol.getId(), vol); } else if (answer instanceof CreateObjectAnswer) { @@ -568,7 +586,9 @@ public class VolumeObject implements VolumeInfo { VolumeObjectTO newVol = (VolumeObjectTO) createAnswer.getData(); VolumeVO vol = this.volumeDao.findById(this.getId()); vol.setPath(newVol.getPath()); - vol.setSize(newVol.getSize()); + if (newVol.getSize() != null) { + vol.setSize(newVol.getSize()); + } vol.setPoolId(this.getDataStore().getId()); volumeDao.update(vol.getId(), vol); } @@ -587,13 +607,15 @@ public class VolumeObject implements VolumeInfo { this.getId()); VolumeObjectTO newVol = (VolumeObjectTO) cpyAnswer.getNewData(); volStore.setInstallPath(newVol.getPath()); - volStore.setSize(newVol.getSize()); + if (newVol.getSize() != null) { + volStore.setSize(newVol.getSize()); + } this.volumeStoreDao.update(volStore.getId(), volStore); } } } catch (RuntimeException ex) { if (event == ObjectInDataStoreStateMachine.Event.OperationFailed) { - objectInStoreMgr.delete(this); + objectInStoreMgr.deleteIfNotReady(this); } throw ex; } diff --git a/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java b/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java index 7b1d030767c..1e6858955fb 100644 --- a/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java +++ b/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java @@ -49,6 +49,7 @@ import org.apache.cloudstack.framework.async.AsyncCallbackDispatcher; import org.apache.cloudstack.framework.async.AsyncCompletionCallback; import org.apache.cloudstack.framework.async.AsyncRpcContext; import org.apache.cloudstack.storage.command.CommandResult; +import org.apache.cloudstack.storage.command.CopyCmdAnswer; import org.apache.cloudstack.storage.command.DeleteCommand; import org.apache.cloudstack.storage.datastore.DataObjectManager; import org.apache.cloudstack.storage.datastore.ObjectInDataStoreManager; @@ -90,6 +91,7 @@ import com.cloud.user.AccountManager; import com.cloud.user.ResourceLimitService; import com.cloud.utils.NumbersUtil; import com.cloud.utils.db.DB; +import com.cloud.utils.db.GlobalLock; import com.cloud.utils.exception.CloudRuntimeException; @Component @@ -292,15 +294,19 @@ public class VolumeServiceImpl implements VolumeService { CommandResult result = callback.getResult(); VolumeObject vo = context.getVolume(); VolumeApiResult apiResult = new VolumeApiResult(vo); - if (result.isSuccess()) { - vo.processEvent(Event.OperationSuccessed); - if (canVolumeBeRemoved(vo.getId())) { - s_logger.info("Volume " + vo.getId() + " is not referred anywhere, remove it from volumes table"); - volDao.remove(vo.getId()); + try { + if (result.isSuccess()) { + vo.processEvent(Event.OperationSuccessed); + if (canVolumeBeRemoved(vo.getId())) { + s_logger.info("Volume " + vo.getId() + " is not referred anywhere, remove it from volumes table"); + volDao.remove(vo.getId()); + } + } else { + vo.processEvent(Event.OperationFailed); + apiResult.setResult(result.getResult()); } - } else { - vo.processEvent(Event.OperationFailed); - apiResult.setResult(result.getResult()); + } catch (Exception e) { + s_logger.debug("ignore delete volume status update failure, it will be picked up by storage clean up thread later", e); } context.getFuture().complete(apiResult); return null; @@ -511,16 +517,41 @@ public class VolumeServiceImpl implements VolumeService { AsyncCallbackDispatcher callback, CreateVolumeFromBaseImageContext context) { DataObject vo = context.vo; + DataObject tmplOnPrimary = context.templateOnStore; CopyCommandResult result = callback.getResult(); VolumeApiResult volResult = new VolumeApiResult((VolumeObject) vo); if (result.isSuccess()) { vo.processEvent(Event.OperationSuccessed, result.getAnswer()); - } else { + } else { vo.processEvent(Event.OperationFailed); volResult.setResult(result.getResult()); + // hack for Vmware: host is down, previously download template to the host needs to be re-downloaded, so we need to reset + // template_spool_ref entry here to NOT_DOWNLOADED and Allocated state + Answer ans = result.getAnswer(); + if ( ans != null && ans instanceof CopyCmdAnswer && ans.getDetails().contains("request template reload")){ + if (tmplOnPrimary != null){ + s_logger.info("Reset template_spool_ref entry so that vmware template can be reloaded in next try"); + VMTemplateStoragePoolVO templatePoolRef = _tmpltPoolDao.findByPoolTemplate(tmplOnPrimary.getDataStore().getId(), tmplOnPrimary.getId()); + if (templatePoolRef != null) { + long templatePoolRefId = templatePoolRef.getId(); + templatePoolRef = _tmpltPoolDao.acquireInLockTable(templatePoolRefId, 1200); + if (templatePoolRef == null) { + s_logger.warn("Reset Template State On Pool failed - unable to lock TemplatePoolRef " + templatePoolRefId); + } + + try { + templatePoolRef.setDownloadState(VMTemplateStorageResourceAssoc.Status.NOT_DOWNLOADED); + templatePoolRef.setState(ObjectInDataStoreStateMachine.State.Allocated); + _tmpltPoolDao.update(templatePoolRefId, templatePoolRef); + } finally { + _tmpltPoolDao.releaseFromLockTable(templatePoolRefId); + } + } + } + } } - + AsyncCallFuture future = context.getFuture(); future.complete(volResult); return null; @@ -734,7 +765,7 @@ public class VolumeServiceImpl implements VolumeService { AsyncCallFuture future = context.future; VolumeApiResult res = new VolumeApiResult(destVolume); try { - if (res.isFailed()) { + if (result.isFailed()) { srcVolume.processEvent(Event.OperationFailed); // back to Ready state in Volume table destVolume.processEventOnly(Event.OperationFailed); res.setResult(result.getResult()); @@ -888,7 +919,7 @@ public class VolumeServiceImpl implements VolumeService { future.complete(res); } } catch (Exception e) { - s_logger.error("Failed to process copy volume callback", e); + s_logger.error("Failed to process migrate volume callback", e); res.setResult(e.toString()); future.complete(res); } @@ -1108,124 +1139,143 @@ public class VolumeServiceImpl implements VolumeService { @Override public void handleVolumeSync(DataStore store) { if (store == null) { - s_logger.warn("Huh? ssHost is null"); + s_logger.warn("Huh? image store is null"); return; } long storeId = store.getId(); - Map volumeInfos = listVolume(store); - if (volumeInfos == null) { - return; - } - - List dbVolumes = _volumeStoreDao.listByStoreId(storeId); - List toBeDownloaded = new ArrayList(dbVolumes); - for (VolumeDataStoreVO volumeStore : dbVolumes) { - VolumeVO volume = _volumeDao.findById(volumeStore.getVolumeId()); - if (volume == null ){ - s_logger.warn("Volume_store_ref shows that volume " + volumeStore.getVolumeId() + " is on image store " + storeId - + ", but the volume is not found in volumes table, potentially some bugs in deleteVolume, so we just treat this volume to be deleted and mark it as destroyed"); - volumeStore.setDestroyed(true); - _volumeStoreDao.update(volumeStore.getId(), volumeStore); - continue; - } - // Exists then don't download - if (volumeInfos.containsKey(volume.getId())) { - TemplateProp volInfo = volumeInfos.remove(volume.getId()); - toBeDownloaded.remove(volumeStore); - s_logger.info("Volume Sync found " + volume.getUuid() + " already in the volume image store table"); - if (volumeStore.getDownloadState() != Status.DOWNLOADED) { - volumeStore.setErrorString(""); - } - if (volInfo.isCorrupted()) { - volumeStore.setDownloadState(Status.DOWNLOAD_ERROR); - String msg = "Volume " + volume.getUuid() + " is corrupted on image store "; - volumeStore.setErrorString(msg); - s_logger.info("msg"); - if (volumeStore.getDownloadUrl() == null) { - msg = "Volume (" + volume.getUuid() + ") with install path " + volInfo.getInstallPath() - + "is corrupted, please check in image store: " + volumeStore.getDataStoreId(); - s_logger.warn(msg); - } else { - toBeDownloaded.add(volumeStore); + // add lock to make template sync for a data store only be done once + String lockString = "volumesync.storeId:" + storeId; + GlobalLock syncLock = GlobalLock.getInternLock(lockString); + try { + if ( syncLock.lock(3)){ + try { + Map volumeInfos = listVolume(store); + if (volumeInfos == null) { + return; } - } else { // Put them in right status - volumeStore.setDownloadPercent(100); - volumeStore.setDownloadState(Status.DOWNLOADED); - volumeStore.setInstallPath(volInfo.getInstallPath()); - volumeStore.setSize(volInfo.getSize()); - volumeStore.setPhysicalSize(volInfo.getPhysicalSize()); - volumeStore.setLastUpdated(new Date()); - _volumeStoreDao.update(volumeStore.getId(), volumeStore); + List dbVolumes = _volumeStoreDao.listByStoreId(storeId); + List toBeDownloaded = new ArrayList(dbVolumes); + for (VolumeDataStoreVO volumeStore : dbVolumes) { + VolumeVO volume = _volumeDao.findById(volumeStore.getVolumeId()); + if (volume == null ){ + s_logger.warn("Volume_store_ref shows that volume " + volumeStore.getVolumeId() + " is on image store " + storeId + + ", but the volume is not found in volumes table, potentially some bugs in deleteVolume, so we just treat this volume to be deleted and mark it as destroyed"); + volumeStore.setDestroyed(true); + _volumeStoreDao.update(volumeStore.getId(), volumeStore); + continue; + } + // Exists then don't download + if (volumeInfos.containsKey(volume.getId())) { + TemplateProp volInfo = volumeInfos.remove(volume.getId()); + toBeDownloaded.remove(volumeStore); + s_logger.info("Volume Sync found " + volume.getUuid() + " already in the volume image store table"); + if (volumeStore.getDownloadState() != Status.DOWNLOADED) { + volumeStore.setErrorString(""); + } + if (volInfo.isCorrupted()) { + volumeStore.setDownloadState(Status.DOWNLOAD_ERROR); + String msg = "Volume " + volume.getUuid() + " is corrupted on image store "; + volumeStore.setErrorString(msg); + s_logger.info("msg"); + if (volumeStore.getDownloadUrl() == null) { + msg = "Volume (" + volume.getUuid() + ") with install path " + volInfo.getInstallPath() + + "is corrupted, please check in image store: " + volumeStore.getDataStoreId(); + s_logger.warn(msg); + } else { + s_logger.info("Removing volume_store_ref entry for corrupted volume " + volume.getName()); + _volumeStoreDao.remove(volumeStore.getId()); + toBeDownloaded.add(volumeStore); + } - if (volume.getSize() == 0) { - // Set volume size in volumes table - volume.setSize(volInfo.getSize()); - _volumeDao.update(volumeStore.getVolumeId(), volume); + } else { // Put them in right status + volumeStore.setDownloadPercent(100); + volumeStore.setDownloadState(Status.DOWNLOADED); + volumeStore.setState(ObjectInDataStoreStateMachine.State.Ready); + volumeStore.setInstallPath(volInfo.getInstallPath()); + volumeStore.setSize(volInfo.getSize()); + volumeStore.setPhysicalSize(volInfo.getPhysicalSize()); + volumeStore.setLastUpdated(new Date()); + _volumeStoreDao.update(volumeStore.getId(), volumeStore); + + if (volume.getSize() == 0) { + // Set volume size in volumes table + volume.setSize(volInfo.getSize()); + _volumeDao.update(volumeStore.getVolumeId(), volume); + } + + if (volInfo.getSize() > 0) { + try { + _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(volume.getAccountId()), + com.cloud.configuration.Resource.ResourceType.secondary_storage, volInfo.getSize() + - volInfo.getPhysicalSize()); + } catch (ResourceAllocationException e) { + s_logger.warn(e.getMessage()); + _alertMgr.sendAlert(AlertManager.ALERT_TYPE_RESOURCE_LIMIT_EXCEEDED, + volume.getDataCenterId(), volume.getPodId(), e.getMessage(), e.getMessage()); + } finally { + _resourceLimitMgr.recalculateResourceCount(volume.getAccountId(), volume.getDomainId(), + com.cloud.configuration.Resource.ResourceType.secondary_storage.getOrdinal()); + } + } + } + continue; + } + // Volume is not on secondary but we should download. + if (volumeStore.getDownloadState() != Status.DOWNLOADED) { + s_logger.info("Volume Sync did not find " + volume.getName() + " ready on image store " + storeId + + ", will request download to start/resume shortly"); + toBeDownloaded.add(volumeStore); + } } - if (volInfo.getSize() > 0) { - try { - _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(volume.getAccountId()), - com.cloud.configuration.Resource.ResourceType.secondary_storage, volInfo.getSize() - - volInfo.getPhysicalSize()); - } catch (ResourceAllocationException e) { - s_logger.warn(e.getMessage()); - _alertMgr.sendAlert(AlertManager.ALERT_TYPE_RESOURCE_LIMIT_EXCEEDED, - volume.getDataCenterId(), volume.getPodId(), e.getMessage(), e.getMessage()); - } finally { - _resourceLimitMgr.recalculateResourceCount(volume.getAccountId(), volume.getDomainId(), - com.cloud.configuration.Resource.ResourceType.secondary_storage.getOrdinal()); + // Download volumes which haven't been downloaded yet. + if (toBeDownloaded.size() > 0) { + for (VolumeDataStoreVO volumeHost : toBeDownloaded) { + if (volumeHost.getDownloadUrl() == null) { // If url is null we + s_logger.info("Skip downloading volume " + volumeHost.getVolumeId() + " since no download url is specified."); + continue; + } + s_logger.debug("Volume " + volumeHost.getVolumeId() + " needs to be downloaded to " + store.getName()); + // TODO: pass a callback later + VolumeInfo vol = volFactory.getVolume(volumeHost.getVolumeId()); + createVolumeAsync(vol, store); + } + } + + // Delete volumes which are not present on DB. + for (Long uniqueName : volumeInfos.keySet()) { + TemplateProp tInfo = volumeInfos.get(uniqueName); + + //we cannot directly call expungeVolumeAsync here to + // reuse delete logic since in this case, our db does not have + // this template at all. + VolumeObjectTO tmplTO = new VolumeObjectTO(); + tmplTO.setDataStore(store.getTO()); + tmplTO.setPath(tInfo.getInstallPath()); + tmplTO.setId(tInfo.getId()); + DeleteCommand dtCommand = new DeleteCommand(tmplTO); + EndPoint ep = _epSelector.select(store); + Answer answer = ep.sendMessage(dtCommand); + if (answer == null || !answer.getResult()) { + s_logger.info("Failed to deleted volume at store: " + store.getName()); + + } else { + String description = "Deleted volume " + tInfo.getTemplateName() + " on secondary storage " + storeId; + s_logger.info(description); } } } - continue; - } - // Volume is not on secondary but we should download. - if (volumeStore.getDownloadState() != Status.DOWNLOADED) { - s_logger.info("Volume Sync did not find " + volume.getName() + " ready on image store " + storeId - + ", will request download to start/resume shortly"); - toBeDownloaded.add(volumeStore); - } - } - - // Download volumes which haven't been downloaded yet. - if (toBeDownloaded.size() > 0) { - for (VolumeDataStoreVO volumeHost : toBeDownloaded) { - if (volumeHost.getDownloadUrl() == null) { // If url is null we - // can't initiate the - // download - continue; + finally{ + syncLock.unlock(); } - s_logger.debug("Volume " + volumeHost.getVolumeId() + " needs to be downloaded to " + store.getName()); - // TODO: pass a callback later - VolumeInfo vol = volFactory.getVolume(volumeHost.getVolumeId()); - createVolumeAsync(vol, store); } - } - - // Delete volumes which are not present on DB. - for (Long uniqueName : volumeInfos.keySet()) { - TemplateProp tInfo = volumeInfos.get(uniqueName); - - //we cannot directly call expungeVolumeAsync here to - // reuse delete logic since in this case, our db does not have - // this template at all. - VolumeObjectTO tmplTO = new VolumeObjectTO(); - tmplTO.setDataStore(store.getTO()); - tmplTO.setPath(tInfo.getInstallPath()); - tmplTO.setId(tInfo.getId()); - DeleteCommand dtCommand = new DeleteCommand(tmplTO); - EndPoint ep = _epSelector.select(store); - Answer answer = ep.sendMessage(dtCommand); - if (answer == null || !answer.getResult()) { - s_logger.info("Failed to deleted volume at store: " + store.getName()); - - } else { - String description = "Deleted volume " + tInfo.getTemplateName() + " on secondary storage " + storeId; - s_logger.info(description); + else { + s_logger.info("Couldn't get global lock on " + lockString + ", another thread may be doing volume sync on data store " + storeId + " now."); } + } finally { + syncLock.releaseRef(); } } @@ -1248,13 +1298,15 @@ public class VolumeServiceImpl implements VolumeService { @Override public SnapshotInfo takeSnapshot(VolumeInfo volume) { VolumeObject vol = (VolumeObject) volume; - vol.stateTransit(Volume.Event.SnapshotRequested); - + boolean result = vol.stateTransit(Volume.Event.SnapshotRequested); + if (!result) { + s_logger.debug("Failed to transit state"); + } SnapshotInfo snapshot = null; try { snapshot = snapshotMgr.takeSnapshot(volume); } catch (Exception e) { - s_logger.debug("Take snapshot: " + volume.getId() + " failed: " + e.toString()); + s_logger.debug("Take snapshot: " + volume.getId() + " failed", e); } finally { if (snapshot != null) { vol.stateTransit(Volume.Event.OperationSucceeded); diff --git a/framework/events/pom.xml b/framework/events/pom.xml index 747c5a1a667..dbe8d9cb4a0 100644 --- a/framework/events/pom.xml +++ b/framework/events/pom.xml @@ -15,7 +15,7 @@ org.apache.cloudstack cloudstack-framework - 4.2.0-SNAPSHOT + 4.2.0 ../pom.xml diff --git a/framework/ipc/pom.xml b/framework/ipc/pom.xml index 2c2131f01c1..a3f9fa70d87 100644 --- a/framework/ipc/pom.xml +++ b/framework/ipc/pom.xml @@ -16,7 +16,7 @@ org.apache.cloudstack cloudstack-framework - 4.2.0-SNAPSHOT + 4.2.0 ../pom.xml diff --git a/framework/jobs/pom.xml b/framework/jobs/pom.xml index cf1fdd585a6..c2497b44fa4 100644 --- a/framework/jobs/pom.xml +++ b/framework/jobs/pom.xml @@ -23,7 +23,7 @@ org.apache.cloudstack cloudstack-framework - 4.2.0-SNAPSHOT + 4.2.0 ../pom.xml diff --git a/framework/pom.xml b/framework/pom.xml index ddcdcb0439a..31b6fbddb25 100644 --- a/framework/pom.xml +++ b/framework/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack - 4.2.0-SNAPSHOT + 4.2.0 install diff --git a/framework/rest/pom.xml b/framework/rest/pom.xml index ab884059685..27e2b894869 100644 --- a/framework/rest/pom.xml +++ b/framework/rest/pom.xml @@ -22,7 +22,7 @@ org.apache.cloudstack cloudstack-framework - 4.2.0-SNAPSHOT + 4.2.0 ../pom.xml cloud-framework-rest diff --git a/packaging/centos63/cloud.spec b/packaging/centos63/cloud.spec index 0a6327f9460..2b814f871df 100644 --- a/packaging/centos63/cloud.spec +++ b/packaging/centos63/cloud.spec @@ -85,7 +85,6 @@ Requires: %{name}-common = %{_ver} Requires: %{name}-awsapi = %{_ver} Obsoletes: cloud-client < 4.1.0 Obsoletes: cloud-client-ui < 4.1.0 -Obsoletes: cloud-daemonize < 4.1.0 Obsoletes: cloud-server < 4.1.0 Obsoletes: cloud-test < 4.1.0 Provides: cloud-client @@ -105,6 +104,7 @@ Obsoletes: cloud-deps < 4.1.0 Obsoletes: cloud-python < 4.1.0 Obsoletes: cloud-setup < 4.1.0 Obsoletes: cloud-cli < 4.1.0 +Obsoletes: cloud-daemonize < 4.1.0 Group: System Environment/Libraries %description common The Apache CloudStack files shared between agent and management server @@ -286,6 +286,8 @@ install -D agent/target/transformed/agent.properties ${RPM_BUILD_ROOT}%{_sysconf install -D agent/target/transformed/environment.properties ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/agent/environment.properties install -D agent/target/transformed/log4j-cloud.xml ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/agent/log4j-cloud.xml install -D agent/target/transformed/cloud-setup-agent ${RPM_BUILD_ROOT}%{_bindir}/%{name}-setup-agent +install -D agent/target/transformed/cloudstack-agent-upgrade ${RPM_BUILD_ROOT}%{_bindir}/%{name}-agent-upgrade +install -D agent/target/transformed/libvirtqemuhook ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib/libvirtqemuhook install -D agent/target/transformed/cloud-ssh ${RPM_BUILD_ROOT}%{_bindir}/%{name}-ssh install -D plugins/hypervisors/kvm/target/cloud-plugin-hypervisor-kvm-%{_maventag}.jar ${RPM_BUILD_ROOT}%{_datadir}/%name-agent/lib/cloud-plugin-hypervisor-kvm-%{_maventag}.jar cp plugins/hypervisors/kvm/target/dependencies/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib @@ -367,6 +369,7 @@ sed -i /"cloud soft nofile"/d /etc/security/limits.conf echo "cloud hard nofile 4096" >> /etc/security/limits.conf echo "cloud soft nofile 4096" >> /etc/security/limits.conf rm -rf %{_localstatedir}/cache/cloud +rm -rf %{_localstatedir}/cache/cloudstack # user harcoded here, also hardcoded on wscript # save old configs if they exist (for upgrade). Otherwise we may lose them @@ -400,7 +403,9 @@ fi if [ -f "%{_sysconfdir}/cloud.rpmsave/management/db.properties" ]; then mv %{_sysconfdir}/%{name}/management/db.properties %{_sysconfdir}/%{name}/management/db.properties.rpmnew cp -p %{_sysconfdir}/cloud.rpmsave/management/db.properties %{_sysconfdir}/%{name}/management - cp -p %{_sysconfdir}/cloud.rpmsave/management/key %{_sysconfdir}/%{name}/management + if [ -f "%{_sysconfdir}/cloud.rpmsave/management/key" ]; then + cp -p %{_sysconfdir}/cloud.rpmsave/management/key %{_sysconfdir}/%{name}/management + fi # make sure we only do this on the first install of this RPM, don't want to overwrite on a reinstall mv %{_sysconfdir}/cloud.rpmsave/management/db.properties %{_sysconfdir}/cloud.rpmsave/management/db.properties.rpmsave fi @@ -408,27 +413,33 @@ fi # Choose server.xml and tomcat.conf links based on old config, if exists serverxml=%{_sysconfdir}/%{name}/management/server.xml oldserverxml=%{_sysconfdir}/cloud.rpmsave/management/server.xml -if [ -L $oldserverxml ] ; then - if stat -c %N $oldserverxml | grep -q server-nonssl ; then - if [ -L $serverxml ]; then rm -f $serverxml; fi - ln -s %{_sysconfdir}/%{name}/management/server-nonssl.xml $serverxml - elif stat -c %N $oldserverxml| grep -q server-ssl ; then - if [ -L $serverxml ]; then rm -f $serverxml; fi +if [ -f $oldserverxml ] || [ -L $oldserverxml ]; then + if stat -c %N $oldserverxml| grep -q server-ssl ; then + if [ -f $serverxml ] || [ -L $serverxml ]; then rm -f $serverxml; fi ln -s %{_sysconfdir}/%{name}/management/server-ssl.xml $serverxml + echo Please verify the server.xml in saved folder, and make the required changes manually , saved folder available at $oldserverxml + else + if [ -f $serverxml ] || [ -L $serverxml ]; then rm -f $serverxml; fi + ln -s %{_sysconfdir}/%{name}/management/server-nonssl.xml $serverxml + echo Please verify the server.xml in saved folder, and make the required changes manually , saved folder available at $oldserverxml + fi else echo "Unable to determine ssl settings for server.xml, please run cloudstack-setup-management manually" fi + tomcatconf=%{_sysconfdir}/%{name}/management/tomcat6.conf oldtomcatconf=%{_sysconfdir}/cloud.rpmsave/management/tomcat6.conf -if [ -L $oldtomcatconf ] ; then - if stat -c %N $oldtomcatconf | grep -q tomcat6-nonssl ; then - if [ -L $tomcatconf ]; then rm -f $tomcatconf; fi - ln -s %{_sysconfdir}/%{name}/management/tomcat6-nonssl.conf $tomcatconf - elif stat -c %N $oldtomcatconf| grep -q tomcat6-ssl ; then - if [ -L $tomcatconf ]; then rm -f $tomcatconf; fi +if [ -f $oldtomcatconf ] || [ -L $oldtomcatconf ] ; then + if stat -c %N $oldtomcatconf| grep -q tomcat6-ssl ; then + if [ -f $tomcatconf ] || [ -L $tomcatconf ]; then rm -f $tomcatconf; fi ln -s %{_sysconfdir}/%{name}/management/tomcat6-ssl.conf $tomcatconf + echo Please verify the tomcat6.conf in saved folder, and make the required changes manually , saved folder available at $oldtomcatconf + else + if [ -f $tomcatconf ] || [ -L $tomcatconf ]; then rm -f $tomcatconf; fi + ln -s %{_sysconfdir}/%{name}/management/tomcat6-nonssl.conf $tomcatconf + echo Please verify the tomcat6.conf in saved folder, and make the required changes manually , saved folder available at $oldtomcatconf fi else echo "Unable to determine ssl settings for tomcat.conf, please run cloudstack-setup-management manually" @@ -538,12 +549,14 @@ fi %files agent %attr(0755,root,root) %{_bindir}/%{name}-setup-agent +%attr(0755,root,root) %{_bindir}/%{name}-agent-upgrade %attr(0755,root,root) %{_bindir}/%{name}-ssh %attr(0755,root,root) %{_sysconfdir}/init.d/%{name}-agent %attr(0755,root,root) %{_datadir}/%{name}-common/scripts/network/cisco %config(noreplace) %{_sysconfdir}/%{name}/agent %dir %{_localstatedir}/log/%{name}/agent %attr(0644,root,root) %{_datadir}/%{name}-agent/lib/*.jar +%attr(0755,root,root) %{_datadir}/%{name}-agent/lib/libvirtqemuhook %dir %{_datadir}/%{name}-agent/plugins %{_defaultdocdir}/%{name}-agent-%{version}/LICENSE %{_defaultdocdir}/%{name}-agent-%{version}/NOTICE diff --git a/packaging/centos63/package.sh b/packaging/centos63/package.sh index ec56c0f2f6b..f30a0e7120a 100755 --- a/packaging/centos63/package.sh +++ b/packaging/centos63/package.sh @@ -42,8 +42,8 @@ if echo $VERSION | grep SNAPSHOT ; then DEFPRE="-D_prerelease 1" DEFREL="-D_rel SNAPSHOT" else + REALVER=$VERSION DEFVER="-D_ver $REALVER" - DEFPRE= DEFREL="-D_rel 1" fi @@ -58,7 +58,7 @@ mkdir -p $RPMDIR/SOURCES/$PACK_PROJECT-$VERSION cp cloud.spec $RPMDIR/SPECS -(cd $RPMDIR; rpmbuild --define "_topdir $RPMDIR" "$DEFVER" "$DEFREL" "$DEFPRE" -ba SPECS/cloud.spec) +(cd $RPMDIR; rpmbuild --define "_topdir $RPMDIR" "${DEFVER}" "${DEFREL}" ${DEFPRE+"${DEFPRE}"} -ba SPECS/cloud.spec) exit } @@ -80,7 +80,6 @@ if echo $VERSION | grep SNAPSHOT ; then else REALVER=`echo $VERSION` DEFVER="-D_ver $REALVER" - DEFPRE= DEFREL="-D_rel 1" fi @@ -96,7 +95,7 @@ mkdir -p $RPMDIR/SOURCES/$PACK_PROJECT-$VERSION cp cloud.spec $RPMDIR/SPECS -(cd $RPMDIR; rpmbuild --define "_topdir $RPMDIR" "$DEFVER" "$DEFREL" "$DEFPRE" "$DEFOSSNOSS" -bb SPECS/cloud.spec) +(cd $RPMDIR; rpmbuild --define "_topdir $RPMDIR" "${DEFVER}" "${DEFREL}" ${DEFPRE+\"${DEFPRE}\"} "${DEFOSSNOSS}" -bb SPECS/cloud.spec) exit } diff --git a/patches/pom.xml b/patches/pom.xml index 438ec6e03aa..9b1d0865db6 100644 --- a/patches/pom.xml +++ b/patches/pom.xml @@ -17,7 +17,7 @@ org.apache.cloudstack cloudstack - 4.2.0-SNAPSHOT + 4.2.0 install diff --git a/patches/systemvm/debian/config/etc/dnsmasq.conf.tmpl b/patches/systemvm/debian/config/etc/dnsmasq.conf.tmpl index 38e5a8bbc96..a3e0bc84856 100644 --- a/patches/systemvm/debian/config/etc/dnsmasq.conf.tmpl +++ b/patches/systemvm/debian/config/etc/dnsmasq.conf.tmpl @@ -632,3 +632,5 @@ log-facility=/var/log/dnsmasq.log # Include a another lot of configuration options. #conf-file=/etc/dnsmasq.more.conf conf-dir=/etc/dnsmasq.d + +dhcp-optsfile=/etc/dhcpopts.txt diff --git a/patches/systemvm/debian/config/etc/init.d/cloud-early-config b/patches/systemvm/debian/config/etc/init.d/cloud-early-config index a552d44ccaf..d847c2470a6 100755 --- a/patches/systemvm/debian/config/etc/init.d/cloud-early-config +++ b/patches/systemvm/debian/config/etc/init.d/cloud-early-config @@ -196,6 +196,7 @@ patch() { patch_log4j() { log_it "Updating log4j-cloud.xml" +mkdir -p /usr/local/cloud/systemvm/conf cat << "EOF" > /usr/local/cloud/systemvm/conf/temp.xml @@ -484,7 +485,7 @@ disable_hvc() { enable_vpc_rpsrfs() { local enable=$1 - if [ $eanble -eq 0] + if [ $enable -eq 0 ] then echo 0 > /etc/rpsrfsenable else @@ -497,7 +498,7 @@ enable_vpc_rpsrfs() { enable_rpsrfs() { local enable=$1 - if [ $eanble -eq 0] + if [ $enable -eq 0 ] then echo 0 > /etc/rpsrfsenable return 0 @@ -634,6 +635,9 @@ setup_common() { setup_dnsmasq() { log_it "Setting up dnsmasq" + + touch /etc/dhcpopts.txt + [ -z $DHCP_RANGE ] && [ $ETH0_IP ] && DHCP_RANGE=$ETH0_IP [ $ETH0_IP6 ] && DHCP_RANGE_IP6=$ETH0_IP6 [ -z $DOMAIN ] && DOMAIN="cloudnine.internal" @@ -724,6 +728,15 @@ setup_dnsmasq() { then echo "$ETH0_IP6 data-server" >> /etc/hosts fi +#add the dhcp-client-update only if dnsmasq version is 2.6 and above + dnsmasqVersion=$(dnsmasq -v | grep version -m 1 | grep -o "[[:digit:]]\.[[:digit:]]") + major=$(echo "$dnsmasqVersion" | cut -d '.' -f 1) + minor=$(echo "$dnsmasqVersion" | cut -d '.' -f 2) + if [ "$major" -eq '2' -a "$minor" -ge '6' ] || [ "$major" -gt '2' ] + then + sed -i -e "/^dhcp-client-update/d" /etc/dnsmasq.conf + echo 'dhcp-client-update' >> /etc/dnsmasq.conf + fi } setup_sshd(){ @@ -754,7 +767,15 @@ setup_vpc_apache2() { } +clean_ipalias_config() { +rm -f /etc/apache2/conf.d/ports.*.meta-data.conf +rm -f /etc/apache2/sites-available/ipAlias* +rm -f /etc/apache2/sites-enabled/ipAlias* +rm -rf /etc/failure_config +} + setup_apache2() { + clean_ipalias_config log_it "Setting up apache web server" local ip=$1 [ -f /etc/apache2/sites-available/default ] && sed -i -e "s///" /etc/apache2/sites-available/default @@ -916,10 +937,6 @@ setup_router() { setup_vpcrouter() { log_it "Setting up VPC virtual router system vm" - if [ "$hyp" == "vmware" ]; then - setup_vmware_extra_nics - fi - if [ -f /etc/hosts ]; then grep -q $NAME /etc/hosts || echo "127.0.0.1 $NAME" >> /etc/hosts; fi @@ -964,6 +981,11 @@ EOF if [ "$hyp" == "vmware" ] then ip route add $MGMTNET via $LOCAL_GW dev eth0 + + # a hacking way to activate vSwitch under VMware + ping -n -c 3 $LOCAL_GW & + sleep 3 + pkill ping fi fi diff --git a/patches/systemvm/debian/config/opt/cloud/bin/ipassoc.sh b/patches/systemvm/debian/config/opt/cloud/bin/ipassoc.sh index ae2d7e4296c..0b1bfbad0f6 100755 --- a/patches/systemvm/debian/config/opt/cloud/bin/ipassoc.sh +++ b/patches/systemvm/debian/config/opt/cloud/bin/ipassoc.sh @@ -332,6 +332,14 @@ enable_rpsrfs() { fi } + +check_for_iface_numof_ips () { + ip addr show $ethDev | grep 'inet ' + return $? +} + + + #set -x sflag=0 lflag= @@ -358,7 +366,7 @@ then if_keep_state=1 fi -while getopts 'sfADna:l:c:g:' OPTION +while getopts 'sfADdna:l:c:g:' OPTION do case $OPTION in A) Aflag=1 @@ -382,6 +390,8 @@ do ;; n) nflag=1 ;; + d) dflag=1 + ;; ?) usage unlock_exit 2 $lock $locked ;; @@ -401,6 +411,12 @@ then unlock_exit 2 $lock $locked fi + if [ "$Dflag" == "1" ] && [ "$dflag" == "1" ] + then + check_for_iface_numof_ips + unlock_exit $? $lock $locked + + fi if [ "$Aflag" == "1" ] && [ "$nflag" == "1" ] then diff --git a/patches/systemvm/debian/config/opt/cloud/bin/passwd_server_ip b/patches/systemvm/debian/config/opt/cloud/bin/passwd_server_ip index 8d62dffa231..46228606e53 100755 --- a/patches/systemvm/debian/config/opt/cloud/bin/passwd_server_ip +++ b/patches/systemvm/debian/config/opt/cloud/bin/passwd_server_ip @@ -20,7 +20,7 @@ addr=$1; while [ "$ENABLED" == "1" ] do - socat -lf /var/log/cloud.log TCP4-LISTEN:8080,reuseaddr,crnl,bind=$addr SYSTEM:"/opt/cloud/bin/serve_password.sh \"\$SOCAT_PEERADDR\"" + socat -lf /var/log/cloud.log TCP4-LISTEN:8080,reuseaddr,fork,crnl,bind=$addr SYSTEM:"/opt/cloud/bin/serve_password.sh \"\$SOCAT_PEERADDR\"" rc=$? if [ $rc -ne 0 ] diff --git a/patches/systemvm/debian/config/opt/cloud/bin/serve_password.sh b/patches/systemvm/debian/config/opt/cloud/bin/serve_password.sh index b829b540666..a3a2732cd2b 100755 --- a/patches/systemvm/debian/config/opt/cloud/bin/serve_password.sh +++ b/patches/systemvm/debian/config/opt/cloud/bin/serve_password.sh @@ -62,7 +62,7 @@ do break fi - request=$(echo $input | grep "DomU_Request:" | cut -d: -f2 | sed 's/^[ \t]*//') + request=$(echo "$input" | grep "DomU_Request:" | cut -d: -f2 | sed 's/^[ \t]*//') if [ "$request" != "" ] then diff --git a/patches/systemvm/debian/config/root/createIpAlias.sh b/patches/systemvm/debian/config/root/createIpAlias.sh index 54981954214..cd273f69ad9 100755 --- a/patches/systemvm/debian/config/root/createIpAlias.sh +++ b/patches/systemvm/debian/config/root/createIpAlias.sh @@ -28,9 +28,38 @@ then exit 1 fi +PORTS_CONF=/etc/apache2/ports.conf +PORTS_CONF_BAK=/etc/ports.conf.bak +FAIL_DIR=/etc/failure_config +CMDLINE=$(cat /var/cache/cloud/cmdline | tr '\n' ' ') + +if [ ! -d "$FAIL_DIR" ] + then + mkdir "$FAIL_DIR" +fi +#bakup ports.conf +cp "$PORTS_CONF" "$PORTS_CONF_BAK" + +domain=$(echo "$CMDLINE" | grep -o " domain=.* " | sed -e 's/domain=//' | awk '{print $1}') + +setup_apache2() { + local ip=$1 + logger -t cloud "Setting up apache web server for $ip" + cp /etc/apache2/sites-available/default /etc/apache2/sites-available/ipAlias.${ip}.meta-data + cp /etc/apache2/sites-available/default-ssl /etc/apache2/sites-available/ipAlias.${ip}-ssl.meta-data + cp /etc/apache2/ports.conf /etc/apache2/conf.d/ports.${ip}.meta-data.conf + sed -i -e "s//\nServerName $domain/" /etc/apache2/sites-available/ipAlias.${ip}.meta-data + sed -i -e "s//\nServerName $domain/" /etc/apache2/sites-available/ipAlias.${ip}-ssl.meta-data + sed -i -e "/NameVirtualHost .*:80/d" /etc/apache2/conf.d/ports.${ip}.meta-data.conf + sed -i -e "s/Listen .*:80/Listen $ip:80/g" /etc/apache2/conf.d/ports.${ip}.meta-data.conf + sed -i -e "s/Listen .*:443/Listen $ip:443/g" /etc/apache2/conf.d/ports.${ip}.meta-data.conf + ln -s /etc/apache2/sites-available/ipAlias.${ip}.meta-data /etc/apache2/sites-enabled/ipAlias.${ip}.meta-data + ln -s /etc/apache2/sites-available/ipAlias.${ip}-ssl.meta-data /etc/apache2/sites-enabled/ipAlias.${ip}-ssl.meta-data +} var="$1" cert="/root/.ssh/id_rsa.cloud" +config_ips="" while [ -n "$var" ] do @@ -39,8 +68,33 @@ do routerip=$(echo $var1 | cut -f2 -d ":") netmask=$(echo $var1 | cut -f3 -d ":") ifconfig eth0:$alias_count $routerip netmask $netmask up + setup_apache2 "$routerip" + config_ips="${config_ips}"$routerip":" var=$( echo $var | sed "s/${var1}-//" ) done + +#restarting the apache server for the config to take effect. +service apache2 restart +result=$? +if [ "$result" -ne "0" ] +then + logger -t cloud "createIpAlias.sh: could not configure apache2 server" + logger -t cloud "createIpAlias.sh: reverting to the old config" + logger -t cloud "createIpAlias.sh: moving out the failure config to $FAIL_DIR" + while [ -n "$config_ips" ] + do + ip=$( echo $config_ips | cut -f1 -d ":" ) + mv "/etc/apache2/sites-available/ipAlias.${ip}.meta-data" "$FAIL_DIR/ipAlias.${ip}.meta-data" + mv "/etc/apache2/sites-available/ipAlias.${ip}-ssl.meta-data" "$FAIL_DIR/ipAlias.${ip}-ssl.meta-data" + mv "/etc/apache2/conf.d/ports.${ip}.meta-data.conf" "$FAIL_DIR/ports.${ip}.meta-data.conf" + rm -f "/etc/apache2/sites-enabled/ipAlias.${ip}.meta-data" + rm -f "/etc/apache2/sites-enabled/ipAlias.${ip}-ssl.meta-data" + config_ips=$( echo $config_ips | sed "s/${ip}://" ) + done + service apache2 restart + unlock_exit $result $lock $locked +fi + #restaring the password service to enable it on the ip aliases /etc/init.d/cloud-passwd-srvr restart unlock_exit $? $lock $locked \ No newline at end of file diff --git a/patches/systemvm/debian/config/root/deleteIpAlias.sh b/patches/systemvm/debian/config/root/deleteIpAlias.sh index fa228fb694f..47edb925450 100755 --- a/patches/systemvm/debian/config/root/deleteIpAlias.sh +++ b/patches/systemvm/debian/config/root/deleteIpAlias.sh @@ -21,7 +21,6 @@ usage() { } source /root/func.sh - lock="biglock" locked=$(getLockFile $lock) if [ "$locked" != "1" ] @@ -29,6 +28,16 @@ then exit 1 fi +remove_apache_config() { +local ip=$1 + logger -t cloud "removing apache web server config for $ip" + rm -f "/etc/apache2/sites-available/ipAlias.${ip}.meta-data" + rm -f "/etc/apache2/sites-available/ipAlias.${ip}-ssl.meta-data" + rm -f "/etc/apache2/conf.d/ports.${ip}.meta-data.conf" + rm -f "/etc/apache2/sites-enabled/ipAlias.${ip}-ssl.meta-data" + rm -f "/etc/apache2/sites-enabled/ipAlias.${ip}.meta-data" +} + var="$1" cert="/root/.ssh/id_rsa.cloud" @@ -36,12 +45,16 @@ while [[ !( "$var" == "-" ) ]] do var1=$(echo $var | cut -f1 -d "-") alias_count=$( echo $var1 | cut -f1 -d ":" ) + routerip=$( echo $var1 | cut -f2 -d ":" ) ifconfig eth0:$alias_count down + remove_apache_config "$routerip" var=$( echo $var | sed "s/${var1}-//" ) done +#restarting the apache server for the config to take effect. +service apache2 restart releaseLockFile $lock $locked #recreating the active ip aliases /root/createIpAlias.sh $2 -unlock_exit $? $lock $locked +unlock_exit $? $lock $locked \ No newline at end of file diff --git a/patches/systemvm/debian/config/root/dnsmasq.sh b/patches/systemvm/debian/config/root/dnsmasq.sh index c6ab07a764a..8fae25c5b5e 100755 --- a/patches/systemvm/debian/config/root/dnsmasq.sh +++ b/patches/systemvm/debian/config/root/dnsmasq.sh @@ -55,8 +55,8 @@ count=0 # fetching the dns Ips from the command line. -dns1=$(echo "$CMDLINE" | grep -o " dns1=.* " | sed -e 's/dns1=//' | awk '{print $1}') -dns2=$(echo "$CMDLINE" | grep -o " dns2=.* " | sed -e 's/dns2=//' | awk '{print $1}') +dns1=$(echo "$CMDLINE" | grep -o " dns1=[[:digit:]].* " | sed -e 's/dns1=//' | awk '{print $1}') +dns2=$(echo "$CMDLINE" | grep -o " dns2=[[:digit:]].* " | sed -e 's/dns2=//' | awk '{print $1}') dns_servers="${dns1}" if [ -n "$dns2" ] @@ -89,19 +89,18 @@ done #logging the configuration being removed. log="" -log="${log}"`grep "^dhcp-option=6.*" "$DHCP_CONFIG_MAIN"`"\n" -log="${log}"`grep "^dhcp-option=option:router.*" "$DHCP_CONFIG_MAIN"`"\n" -log="${log}"`grep "^dhcp-range=.*" "$DHCP_CONFIG_MAIN"`"\n" -echo -e "$log" > log.dnsmasq.txt +log="${log}"`grep "^dhcp-option=6" "$DHCP_CONFIG_MAIN"`"\n" +log="${log}"`grep "^dhcp-option=option:router" "$DHCP_CONFIG_MAIN"`"\n" +log="${log}"`grep "^dhcp-range=" "$DHCP_CONFIG_MAIN"`"\n" if [ "$log" != '\n\n\n' ] then #Cleaning the existing dhcp confgiuration logger -t cloud "dnsmasq.sh: remvoing the primaryip confg from dnsmasq.conf and adding it to /etc/dnsmaq.d/multiple_ranges.conf" logger -t cloud "dnsmasq.sh: config removed from dnsmasq.conf is $log" - sed -i -e '/dhcp-option=6.*/d' "$DHCP_CONFIG_MAIN" - sed -i -e '/dhcp-option=option:router.*/d' "$DHCP_CONFIG_MAIN" - sed -i -e '/dhcp-range=.*/d' "$DHCP_CONFIG_MAIN" + sed -i -e '/dhcp-option=6/d' "$DHCP_CONFIG_MAIN" + sed -i -e '/dhcp-option=option:router/d' "$DHCP_CONFIG_MAIN" + sed -i -e '/^dhcp-range=/d' "$DHCP_CONFIG_MAIN" fi #wrting the new config into the config file. diff --git a/patches/systemvm/debian/config/root/redundant_router/enable_pubip.sh.templ b/patches/systemvm/debian/config/root/redundant_router/enable_pubip.sh.templ index 0e42ec4968a..0e2d03a9041 100644 --- a/patches/systemvm/debian/config/root/redundant_router/enable_pubip.sh.templ +++ b/patches/systemvm/debian/config/root/redundant_router/enable_pubip.sh.templ @@ -16,12 +16,16 @@ # specific language governing permissions and limitations # under the License. +ip link|grep BROADCAST|grep -v eth0|grep -v eth1|cut -d ":" -f 2 > /tmp/iflist +ip addr show eth2 | grep "inet" 2>&1 > /dev/null +is_init=$? + set -e -ip link|grep BROADCAST|grep -v eth0|grep -v eth1|cut -d ":" -f 2 > /tmp/iflist while read i do - if [ "$i" == "eth2" ] + # if eth2'ip has already been configured, we would use ifconfig rather than ifdown/ifup + if [ "$i" == "eth2" -a "$is_init" != "0" ] then ifdown $i ifup $i diff --git a/plugins/acl/static-role-based/pom.xml b/plugins/acl/static-role-based/pom.xml index e40cecb9d65..23093422f7b 100644 --- a/plugins/acl/static-role-based/pom.xml +++ b/plugins/acl/static-role-based/pom.xml @@ -26,7 +26,7 @@ org.apache.cloudstack cloudstack-plugins - 4.2.0-SNAPSHOT + 4.2.0 ../../pom.xml diff --git a/plugins/affinity-group-processors/explicit-dedication/pom.xml b/plugins/affinity-group-processors/explicit-dedication/pom.xml index bb3c595841a..a6068490aff 100644 --- a/plugins/affinity-group-processors/explicit-dedication/pom.xml +++ b/plugins/affinity-group-processors/explicit-dedication/pom.xml @@ -23,7 +23,7 @@ org.apache.cloudstack cloudstack-plugins - 4.2.0-SNAPSHOT + 4.2.0 ../../pom.xml diff --git a/plugins/affinity-group-processors/explicit-dedication/src/org/apache/cloudstack/affinity/ExplicitDedicationProcessor.java b/plugins/affinity-group-processors/explicit-dedication/src/org/apache/cloudstack/affinity/ExplicitDedicationProcessor.java index a0eb56cbb8a..364c057df85 100644 --- a/plugins/affinity-group-processors/explicit-dedication/src/org/apache/cloudstack/affinity/ExplicitDedicationProcessor.java +++ b/plugins/affinity-group-processors/explicit-dedication/src/org/apache/cloudstack/affinity/ExplicitDedicationProcessor.java @@ -43,6 +43,10 @@ import com.cloud.domain.dao.DomainDao; import com.cloud.exception.AffinityConflictException; import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; +import com.cloud.utils.db.DB; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.db.Transaction; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineProfile; @@ -87,161 +91,176 @@ public class ExplicitDedicationProcessor extends AffinityProcessorBase implement VirtualMachine vm = vmProfile.getVirtualMachine(); List vmGroupMappings = _affinityGroupVMMapDao.findByVmIdType(vm.getId(), getType()); DataCenter dc = _dcDao.findById(vm.getDataCenterId()); - long domainId = vm.getDomainId(); - long accountId = vm.getAccountId(); + List resourceList = new ArrayList(); - for (AffinityGroupVMMapVO vmGroupMapping : vmGroupMappings) { - if (vmGroupMapping != null) { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Processing affinity group of type 'ExplicitDedication' for VM Id: " + vm.getId()); + if (vmGroupMappings != null && !vmGroupMappings.isEmpty()) { + + for (AffinityGroupVMMapVO vmGroupMapping : vmGroupMappings) { + if (vmGroupMapping != null) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Processing affinity group " + vmGroupMapping.getAffinityGroupId() + + "of type 'ExplicitDedication' for VM Id: " + vm.getId()); + } + + long affinityGroupId = vmGroupMapping.getAffinityGroupId(); + + // List dr = + // _dedicatedDao.listByAccountId(accountId); + // List drOfDomain = + // searchInDomainResources(domainId); + // List drOfParentDomain = + // searchInParentDomainResources(domainId); + + List dr = _dedicatedDao.listByAffinityGroupId(affinityGroupId); + resourceList.addAll(dr); + + } + } + + boolean canUse = false; + + if (plan.getHostId() != null) { + HostVO host = _hostDao.findById(plan.getHostId()); + ClusterVO clusterofHost = _clusterDao.findById(host.getClusterId()); + HostPodVO podOfHost = _podDao.findById(host.getPodId()); + DataCenterVO zoneOfHost = _dcDao.findById(host.getDataCenterId()); + if (resourceList != null && resourceList.size() != 0) { + for (DedicatedResourceVO resource : resourceList) { + if ((resource.getHostId() != null && resource.getHostId() == plan.getHostId()) + || (resource.getClusterId() != null && resource.getClusterId() == clusterofHost.getId()) + || (resource.getPodId() != null && resource.getPodId() == podOfHost.getId()) + || (resource.getDataCenterId() != null && resource.getDataCenterId() == zoneOfHost + .getId())) { + canUse = true; + } + } + } + if (!canUse) { + throw new CloudRuntimeException("Cannot use this host " + host.getName() + + " for explicit dedication"); + } + } else if (plan.getClusterId() != null) { + ClusterVO cluster = _clusterDao.findById(plan.getClusterId()); + HostPodVO podOfCluster = _podDao.findById(cluster.getPodId()); + DataCenterVO zoneOfCluster = _dcDao.findById(cluster.getDataCenterId()); + List hostToUse = new ArrayList(); + // check whether this cluster or its pod is dedicated + if (resourceList != null && resourceList.size() != 0) { + for (DedicatedResourceVO resource : resourceList) { + if ((resource.getClusterId() != null && resource.getClusterId() == cluster.getId()) + || (resource.getPodId() != null && resource.getPodId() == podOfCluster.getId()) + || (resource.getDataCenterId() != null && resource.getDataCenterId() == zoneOfCluster + .getId())) { + canUse = true; + } + + // check for all dedicated host; if it belongs to this + // cluster + if (!canUse) { + if (resource.getHostId() != null) { + HostVO dHost = _hostDao.findById(resource.getHostId()); + if (dHost.getClusterId() == cluster.getId()) { + hostToUse.add(dHost); + } + } + } + + } } - List dr = _dedicatedDao.listByAccountId(accountId); - List drOfDomain = searchInDomainResources(domainId); - List drOfParentDomain = searchInParentDomainResources(domainId); - List resourceList = new ArrayList(); - resourceList.addAll(dr); - resourceList.addAll(drOfDomain); - resourceList.addAll(drOfParentDomain); - boolean canUse = false; + if (hostToUse.isEmpty() && !canUse) { + throw new CloudRuntimeException("Cannot use this cluster " + cluster.getName() + + " for explicit dedication"); + } - if (plan.getHostId() != null) { - HostVO host = _hostDao.findById(plan.getHostId()); - ClusterVO clusterofHost = _clusterDao.findById(host.getClusterId()); - HostPodVO podOfHost = _podDao.findById(host.getPodId()); - DataCenterVO zoneOfHost = _dcDao.findById(host.getDataCenterId()); - if (resourceList != null && resourceList.size() != 0) { - for(DedicatedResourceVO resource : resourceList){ - if ((resource.getHostId() != null && resource.getHostId() == plan.getHostId()) || - (resource.getClusterId() != null && resource.getClusterId() == clusterofHost.getId()) || - (resource.getPodId() != null && resource.getPodId() == podOfHost.getId()) || - (resource.getDataCenterId() != null && resource.getDataCenterId() == zoneOfHost.getId())){ - canUse = true; - } + if (hostToUse != null && hostToUse.size() != 0) { + // add other non-dedicated hosts to avoid list + List hostList = _hostDao.findByClusterId(cluster.getId()); + for (HostVO host : hostList) { + if (!hostToUse.contains(host)) { + avoid.addHost(host.getId()); } } - if (!canUse) { - throw new CloudRuntimeException("Cannot use this host " + host.getName() + " for explicit dedication"); - } - } else if (plan.getClusterId() != null) { - ClusterVO cluster = _clusterDao.findById(plan.getClusterId()); - HostPodVO podOfCluster = _podDao.findById(cluster.getPodId()); - DataCenterVO zoneOfCluster = _dcDao.findById(cluster.getDataCenterId()); - List hostToUse = new ArrayList(); - // check whether this cluster or its pod is dedicated - if (resourceList != null && resourceList.size() != 0) { - for(DedicatedResourceVO resource : resourceList){ - if ((resource.getClusterId() != null && resource.getClusterId() == cluster.getId()) || - (resource.getPodId() != null && resource.getPodId() == podOfCluster.getId()) || - (resource.getDataCenterId() != null && resource.getDataCenterId() == zoneOfCluster.getId())){ - canUse = true; - } + } - // check for all dedicated host; if it belongs to this cluster - if (!canUse){ - if (resource.getHostId() != null) { - HostVO dHost = _hostDao.findById(resource.getHostId()); - if (dHost.getClusterId() == cluster.getId()) { - hostToUse.add(dHost); - } + } else if (plan.getPodId() != null) { + HostPodVO pod = _podDao.findById(plan.getPodId()); + DataCenterVO zoneOfPod = _dcDao.findById(pod.getDataCenterId()); + List clustersToUse = new ArrayList(); + List hostsToUse = new ArrayList(); + // check whether this cluster or its pod is dedicated + if (resourceList != null && resourceList.size() != 0) { + for (DedicatedResourceVO resource : resourceList) { + if ((resource.getPodId() != null && resource.getPodId() == pod.getId()) + || (resource.getDataCenterId() != null && resource.getDataCenterId() == zoneOfPod + .getId())) { + canUse = true; + } + + // check for all dedicated cluster/host; if it belongs + // to + // this pod + if (!canUse) { + if (resource.getClusterId() != null) { + ClusterVO dCluster = _clusterDao.findById(resource.getClusterId()); + if (dCluster.getPodId() == pod.getId()) { + clustersToUse.add(dCluster); } } - - } - } - - if (hostToUse.isEmpty() && !canUse) { - throw new CloudRuntimeException("Cannot use this cluster " + cluster.getName() + " for explicit dedication"); - } - - if (hostToUse != null && hostToUse.size() != 0) { - // add other non-dedicated hosts to avoid list - List hostList = _hostDao.findByClusterId(cluster.getId()); - for (HostVO host : hostList){ - if (!hostToUse.contains(host)) { - avoid.addHost(host.getId()); - } - } - } - - } else if (plan.getPodId() != null) { - HostPodVO pod = _podDao.findById(plan.getPodId()); - DataCenterVO zoneOfPod = _dcDao.findById(pod.getDataCenterId()); - List clustersToUse = new ArrayList(); - List hostsToUse = new ArrayList(); - // check whether this cluster or its pod is dedicated - if (resourceList != null && resourceList.size() != 0) { - for(DedicatedResourceVO resource : resourceList){ - if ((resource.getPodId() != null && resource.getPodId() == pod.getId()) || - (resource.getDataCenterId() != null && resource.getDataCenterId() == zoneOfPod.getId())){ - canUse = true; - } - - // check for all dedicated cluster/host; if it belongs to this pod - if (!canUse){ - if (resource.getClusterId() != null) { - ClusterVO dCluster = _clusterDao.findById(resource.getClusterId()); - if (dCluster.getPodId() == pod.getId()) { - clustersToUse.add(dCluster); - } - } - if (resource.getHostId() != null) { - HostVO dHost = _hostDao.findById(resource.getHostId()); - if (dHost.getPodId() == pod.getId()) { - hostsToUse.add(dHost); - } + if (resource.getHostId() != null) { + HostVO dHost = _hostDao.findById(resource.getHostId()); + if (dHost.getPodId() == pod.getId()) { + hostsToUse.add(dHost); } } + } + } + } + + if (hostsToUse.isEmpty() && clustersToUse.isEmpty() && !canUse) { + throw new CloudRuntimeException("Cannot use this pod " + pod.getName() + " for explicit dedication"); + } + + if (clustersToUse != null && clustersToUse.size() != 0) { + // add other non-dedicated clusters to avoid list + List clusterList = _clusterDao.listByPodId(pod.getId()); + for (ClusterVO cluster : clusterList) { + if (!clustersToUse.contains(cluster)) { + avoid.addCluster(cluster.getId()); } } + } - if (hostsToUse.isEmpty() && clustersToUse.isEmpty() && !canUse) { - throw new CloudRuntimeException("Cannot use this pod " + pod.getName() + " for explicit dedication"); - } - - if (clustersToUse != null && clustersToUse.size() != 0) { - // add other non-dedicated clusters to avoid list - List clusterList = _clusterDao.listByPodId(pod.getId()); - for (ClusterVO cluster : clusterList){ - if (!clustersToUse.contains(cluster)) { - avoid.addCluster(cluster.getId()); - } - } - } - - if (hostsToUse != null && hostsToUse.size() != 0) { - // add other non-dedicated hosts to avoid list - List hostList = _hostDao.findByPodId(pod.getId()); - for (HostVO host : hostList){ - if (!hostsToUse.contains(host)) { - avoid.addHost(host.getId()); - } + if (hostsToUse != null && hostsToUse.size() != 0) { + // add other non-dedicated hosts to avoid list + List hostList = _hostDao.findByPodId(pod.getId()); + for (HostVO host : hostList) { + if (!hostsToUse.contains(host)) { + avoid.addHost(host.getId()); } } + } + } else { + // check all resources under this zone + if (resourceList != null && resourceList.size() != 0) { + avoid = updateAvoidList(resourceList, avoid, dc); } else { - //check all resources under this zone - if (dr != null && dr.size() != 0) { - avoid = updateAvoidList(dr, avoid, dc); - } else if(drOfDomain != null && drOfDomain.size() != 0){ - avoid = updateAvoidList(drOfDomain, avoid, dc); - } else if(drOfParentDomain != null && drOfParentDomain.size() != 0){ - avoid = updateAvoidList(drOfParentDomain, avoid, dc); - } else { - avoid.addDataCenter(dc.getId()); - if (s_logger.isDebugEnabled()) { - s_logger.debug("No dedicated resources available for this domain or account"); - } - } - + avoid.addDataCenter(dc.getId()); if (s_logger.isDebugEnabled()) { - s_logger.debug("ExplicitDedicationProcessor returns Avoid List as: Deploy avoids pods: " + avoid.getPodsToAvoid() + ", clusters: " - + avoid.getClustersToAvoid() + ", hosts: " + avoid.getHostsToAvoid()); + s_logger.debug("No dedicated resources available for this domain or account under this group"); } } + + if (s_logger.isDebugEnabled()) { + s_logger.debug("ExplicitDedicationProcessor returns Avoid List as: Deploy avoids pods: " + + avoid.getPodsToAvoid() + ", clusters: " + avoid.getClustersToAvoid() + ", hosts: " + + avoid.getHostsToAvoid()); + } } } + } private ExcludeList updateAvoidList(List dedicatedResources, ExcludeList avoidList, DataCenter dc) { @@ -260,7 +279,7 @@ public class ExplicitDedicationProcessor extends AffinityProcessorBase implement List hostList = _hostDao.findByClusterId(dr.getClusterId()); for (HostVO host : hostList) { DedicatedResourceVO dHost = _dedicatedDao.findByHostId(host.getId()); - if (dHost != null) { + if (dHost != null && !dedicatedResources.contains(dHost)) { avoidList.addHost(host.getId()); } else { includeList.addHost(host.getId()); @@ -275,7 +294,8 @@ public class ExplicitDedicationProcessor extends AffinityProcessorBase implement //add all cluster under this pod in includeList List clusterList = _clusterDao.listByPodId(dr.getPodId()); for (ClusterVO cluster : clusterList) { - if (_dedicatedDao.findByClusterId(cluster.getId()) != null) { + DedicatedResourceVO dCluster = _dedicatedDao.findByClusterId(cluster.getId()); + if (dCluster != null && !dedicatedResources.contains(dCluster)) { avoidList.addCluster(cluster.getId()); } else { includeList.addCluster(cluster.getId()); @@ -284,7 +304,8 @@ public class ExplicitDedicationProcessor extends AffinityProcessorBase implement //add all hosts inside this pod in includeList List hostList = _hostDao.findByPodId(dr.getPodId()); for (HostVO host : hostList) { - if (_dedicatedDao.findByHostId(host.getId()) != null) { + DedicatedResourceVO dHost = _dedicatedDao.findByHostId(host.getId()); + if (dHost != null && !dedicatedResources.contains(dHost)) { avoidList.addHost(host.getId()); } else { includeList.addHost(host.getId()); @@ -297,7 +318,8 @@ public class ExplicitDedicationProcessor extends AffinityProcessorBase implement //add all Pod under this data center in includeList List podList = _podDao.listByDataCenterId(dr.getDataCenterId()); for (HostPodVO pod : podList) { - if (_dedicatedDao.findByPodId(pod.getId()) != null) { + DedicatedResourceVO dPod = _dedicatedDao.findByPodId(pod.getId()); + if (dPod != null && !dedicatedResources.contains(dPod)) { avoidList.addPod(pod.getId()); } else { includeList.addPod(pod.getId()); @@ -305,7 +327,8 @@ public class ExplicitDedicationProcessor extends AffinityProcessorBase implement } List clusterList = _clusterDao.listClustersByDcId(dr.getDataCenterId()); for (ClusterVO cluster : clusterList) { - if (_dedicatedDao.findByClusterId(cluster.getId()) != null) { + DedicatedResourceVO dCluster = _dedicatedDao.findByClusterId(cluster.getId()); + if (dCluster != null && !dedicatedResources.contains(dCluster)) { avoidList.addCluster(cluster.getId()); } else { includeList.addCluster(cluster.getId()); @@ -314,7 +337,8 @@ public class ExplicitDedicationProcessor extends AffinityProcessorBase implement //add all hosts inside this in includeList List hostList = _hostDao.listByDataCenterId(dr.getDataCenterId()); for (HostVO host : hostList) { - if (_dedicatedDao.findByHostId(host.getId()) != null) { + DedicatedResourceVO dHost = _dedicatedDao.findByHostId(host.getId()); + if (dHost != null && !dedicatedResources.contains(dHost)) { avoidList.addHost(host.getId()); } else { includeList.addHost(host.getId()); @@ -380,4 +404,53 @@ public class ExplicitDedicationProcessor extends AffinityProcessorBase implement return domainIds; } + @Override + public boolean isAdminControlledGroup() { + return true; + } + + @Override + public boolean canBeSharedDomainWide() { + return true; + } + + @DB + @Override + public void handleDeleteGroup(AffinityGroup group) { + // When a group of the 'ExplicitDedication' type gets deleted, make sure + // to remove the dedicated resources in the group as well. + if (group != null) { + List dedicatedResources = _dedicatedDao.listByAffinityGroupId(group.getId()); + if (!dedicatedResources.isEmpty()) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Releasing the dedicated resources under group: " + group); + } + Transaction txn = Transaction.currentTxn(); + txn.start(); + + SearchBuilder listByAffinityGroup = _dedicatedDao.createSearchBuilder(); + listByAffinityGroup.and("affinityGroupId", listByAffinityGroup.entity().getAffinityGroupId(), + SearchCriteria.Op.EQ); + listByAffinityGroup.done(); + SearchCriteria sc = listByAffinityGroup.create(); + sc.setParameters("affinityGroupId", group.getId()); + + _dedicatedDao.lockRows(sc, null, true); + _dedicatedDao.remove(sc); + + txn.commit(); + } else { + if (s_logger.isDebugEnabled()) { + s_logger.debug("No dedicated resources to releease under group: " + group); + } + } + } + + return; + } + + @Override + public boolean subDomainAccess() { + return true; + } } diff --git a/plugins/affinity-group-processors/host-anti-affinity/pom.xml b/plugins/affinity-group-processors/host-anti-affinity/pom.xml index 669febd7db8..5f68be26a44 100644 --- a/plugins/affinity-group-processors/host-anti-affinity/pom.xml +++ b/plugins/affinity-group-processors/host-anti-affinity/pom.xml @@ -23,7 +23,7 @@ org.apache.cloudstack cloudstack-plugins - 4.2.0-SNAPSHOT + 4.2.0 ../../pom.xml diff --git a/plugins/alert-handlers/snmp-alerts/pom.xml b/plugins/alert-handlers/snmp-alerts/pom.xml index b5cebf31b7a..0b1db5d33ce 100644 --- a/plugins/alert-handlers/snmp-alerts/pom.xml +++ b/plugins/alert-handlers/snmp-alerts/pom.xml @@ -22,7 +22,7 @@ cloudstack-plugins org.apache.cloudstack - 4.2.0-SNAPSHOT + 4.2.0 ../../pom.xml 4.0.0 diff --git a/plugins/alert-handlers/syslog-alerts/pom.xml b/plugins/alert-handlers/syslog-alerts/pom.xml index 21aa54a7be2..00a7fea0558 100644 --- a/plugins/alert-handlers/syslog-alerts/pom.xml +++ b/plugins/alert-handlers/syslog-alerts/pom.xml @@ -22,7 +22,7 @@ cloudstack-plugins org.apache.cloudstack - 4.2.0-SNAPSHOT + 4.2.0 ../../pom.xml 4.0.0 diff --git a/plugins/api/discovery/pom.xml b/plugins/api/discovery/pom.xml index 5d9ad75ea3a..abcc4466c21 100644 --- a/plugins/api/discovery/pom.xml +++ b/plugins/api/discovery/pom.xml @@ -26,7 +26,7 @@ org.apache.cloudstack cloudstack-plugins - 4.2.0-SNAPSHOT + 4.2.0 ../../pom.xml diff --git a/plugins/api/rate-limit/pom.xml b/plugins/api/rate-limit/pom.xml index 5645f0b3a32..8b808a1372c 100644 --- a/plugins/api/rate-limit/pom.xml +++ b/plugins/api/rate-limit/pom.xml @@ -23,7 +23,7 @@ org.apache.cloudstack cloudstack-plugins - 4.2.0-SNAPSHOT + 4.2.0 ../../pom.xml diff --git a/plugins/dedicated-resources/pom.xml b/plugins/dedicated-resources/pom.xml index 4c908f4ff96..054f8cd03a3 100644 --- a/plugins/dedicated-resources/pom.xml +++ b/plugins/dedicated-resources/pom.xml @@ -23,7 +23,7 @@ org.apache.cloudstack cloudstack-plugins - 4.2.0-SNAPSHOT + 4.2.0 ../pom.xml diff --git a/plugins/dedicated-resources/src/org/apache/cloudstack/api/commands/ListDedicatedClustersCmd.java b/plugins/dedicated-resources/src/org/apache/cloudstack/api/commands/ListDedicatedClustersCmd.java index f3947876581..613f2e750e0 100644 --- a/plugins/dedicated-resources/src/org/apache/cloudstack/api/commands/ListDedicatedClustersCmd.java +++ b/plugins/dedicated-resources/src/org/apache/cloudstack/api/commands/ListDedicatedClustersCmd.java @@ -21,12 +21,14 @@ import java.util.List; import javax.inject.Inject; +import org.apache.cloudstack.affinity.AffinityGroupResponse; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseListCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.BaseCmd.CommandType; import org.apache.cloudstack.api.response.ClusterResponse; import org.apache.cloudstack.api.response.DomainResponse; import org.apache.cloudstack.api.response.ListResponse; @@ -60,6 +62,9 @@ public class ListDedicatedClustersCmd extends BaseListCmd { description = "the name of the account associated with the cluster. Must be used with domainId.") private String accountName; + @Parameter(name = ApiConstants.AFFINITY_GROUP_ID, type = CommandType.UUID, entityType = AffinityGroupResponse.class, description = "list dedicated clusters by affinity group") + private Long affinityGroupId; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -76,6 +81,10 @@ public class ListDedicatedClustersCmd extends BaseListCmd { return accountName; } + public Long getAffinityGroupId() { + return affinityGroupId; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/plugins/dedicated-resources/src/org/apache/cloudstack/api/commands/ListDedicatedHostsCmd.java b/plugins/dedicated-resources/src/org/apache/cloudstack/api/commands/ListDedicatedHostsCmd.java index 736251b36d6..5b1a41094f0 100644 --- a/plugins/dedicated-resources/src/org/apache/cloudstack/api/commands/ListDedicatedHostsCmd.java +++ b/plugins/dedicated-resources/src/org/apache/cloudstack/api/commands/ListDedicatedHostsCmd.java @@ -21,12 +21,14 @@ import java.util.List; import javax.inject.Inject; +import org.apache.cloudstack.affinity.AffinityGroupResponse; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseListCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.BaseCmd.CommandType; import org.apache.cloudstack.api.response.DomainResponse; import org.apache.cloudstack.api.response.HostResponse; import org.apache.cloudstack.api.response.ListResponse; @@ -60,6 +62,9 @@ public class ListDedicatedHostsCmd extends BaseListCmd { description = "the name of the account associated with the host. Must be used with domainId.") private String accountName; + @Parameter(name = ApiConstants.AFFINITY_GROUP_ID, type = CommandType.UUID, entityType = AffinityGroupResponse.class, description = "list dedicated hosts by affinity group") + private Long affinityGroupId; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -76,6 +81,10 @@ public class ListDedicatedHostsCmd extends BaseListCmd { return accountName; } + public Long getAffinityGroupId() { + return affinityGroupId; + } + ///////////////////////////////////////////////////// /////////////// API Implementation///////////////////l ///////////////////////////////////////////////////// diff --git a/plugins/dedicated-resources/src/org/apache/cloudstack/api/commands/ListDedicatedPodsCmd.java b/plugins/dedicated-resources/src/org/apache/cloudstack/api/commands/ListDedicatedPodsCmd.java index da59edae8d3..6f16d65d19e 100644 --- a/plugins/dedicated-resources/src/org/apache/cloudstack/api/commands/ListDedicatedPodsCmd.java +++ b/plugins/dedicated-resources/src/org/apache/cloudstack/api/commands/ListDedicatedPodsCmd.java @@ -21,12 +21,14 @@ import java.util.List; import javax.inject.Inject; +import org.apache.cloudstack.affinity.AffinityGroupResponse; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseListCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.BaseCmd.CommandType; import org.apache.cloudstack.api.response.DomainResponse; import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.PodResponse; @@ -60,6 +62,9 @@ public class ListDedicatedPodsCmd extends BaseListCmd { description = "the name of the account associated with the pod. Must be used with domainId.") private String accountName; + @Parameter(name = ApiConstants.AFFINITY_GROUP_ID, type = CommandType.UUID, entityType = AffinityGroupResponse.class, description = "list dedicated pods by affinity group") + private Long affinityGroupId; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -76,6 +81,10 @@ public class ListDedicatedPodsCmd extends BaseListCmd { return accountName; } + public Long getAffinityGroupId() { + return affinityGroupId; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/plugins/dedicated-resources/src/org/apache/cloudstack/api/commands/ListDedicatedZonesCmd.java b/plugins/dedicated-resources/src/org/apache/cloudstack/api/commands/ListDedicatedZonesCmd.java index a21f129f5be..1fc1a1961f0 100644 --- a/plugins/dedicated-resources/src/org/apache/cloudstack/api/commands/ListDedicatedZonesCmd.java +++ b/plugins/dedicated-resources/src/org/apache/cloudstack/api/commands/ListDedicatedZonesCmd.java @@ -21,12 +21,14 @@ import java.util.List; import javax.inject.Inject; +import org.apache.cloudstack.affinity.AffinityGroupResponse; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseListCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.BaseCmd.CommandType; import org.apache.cloudstack.api.response.DomainResponse; import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.ZoneResponse; @@ -60,6 +62,9 @@ public class ListDedicatedZonesCmd extends BaseListCmd { description = "the name of the account associated with the zone. Must be used with domainId.") private String accountName; + @Parameter(name = ApiConstants.AFFINITY_GROUP_ID, type = CommandType.UUID, entityType = AffinityGroupResponse.class, description = "list dedicated zones by affinity group") + private Long affinityGroupId; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -76,6 +81,10 @@ public class ListDedicatedZonesCmd extends BaseListCmd { return accountName; } + public Long getAffinityGroupId() { + return affinityGroupId; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/plugins/dedicated-resources/src/org/apache/cloudstack/api/response/DedicateClusterResponse.java b/plugins/dedicated-resources/src/org/apache/cloudstack/api/response/DedicateClusterResponse.java index 3c8dde3fd08..e45938c6dea 100644 --- a/plugins/dedicated-resources/src/org/apache/cloudstack/api/response/DedicateClusterResponse.java +++ b/plugins/dedicated-resources/src/org/apache/cloudstack/api/response/DedicateClusterResponse.java @@ -16,6 +16,8 @@ // under the License. package org.apache.cloudstack.api.response; +import javax.persistence.Column; + import org.apache.cloudstack.api.BaseResponse; import com.cloud.serializer.Param; @@ -37,6 +39,10 @@ public class DedicateClusterResponse extends BaseResponse { @SerializedName("accountid") @Param(description="the Account ID of the cluster") private String accountId; + @SerializedName("affinitygroupid") + @Param(description = "the Dedication Affinity Group ID of the cluster") + private String affinityGroupId; + public String getId() { return id; } @@ -76,4 +82,12 @@ public class DedicateClusterResponse extends BaseResponse { public void setAccountId(String accountId) { this.accountId = accountId; } + + public String getAffinityGroupId() { + return affinityGroupId; + } + + public void setAffinityGroupId(String affinityGroupId) { + this.affinityGroupId = affinityGroupId; + } } diff --git a/plugins/dedicated-resources/src/org/apache/cloudstack/api/response/DedicateHostResponse.java b/plugins/dedicated-resources/src/org/apache/cloudstack/api/response/DedicateHostResponse.java index cea31fe392b..83126a2490e 100644 --- a/plugins/dedicated-resources/src/org/apache/cloudstack/api/response/DedicateHostResponse.java +++ b/plugins/dedicated-resources/src/org/apache/cloudstack/api/response/DedicateHostResponse.java @@ -37,6 +37,10 @@ public class DedicateHostResponse extends BaseResponse { @SerializedName("accountid") @Param(description="the Account ID of the host") private String accountId; + @SerializedName("affinitygroupid") + @Param(description = "the Dedication Affinity Group ID of the host") + private String affinityGroupId; + public String getId() { return id; } @@ -76,4 +80,12 @@ public class DedicateHostResponse extends BaseResponse { public void setAccountId(String accountId) { this.accountId = accountId; } + + public String getAffinityGroupId() { + return affinityGroupId; + } + + public void setAffinityGroupId(String affinityGroupId) { + this.affinityGroupId = affinityGroupId; + } } diff --git a/plugins/dedicated-resources/src/org/apache/cloudstack/api/response/DedicatePodResponse.java b/plugins/dedicated-resources/src/org/apache/cloudstack/api/response/DedicatePodResponse.java index 4bcaa61c269..3705666433d 100644 --- a/plugins/dedicated-resources/src/org/apache/cloudstack/api/response/DedicatePodResponse.java +++ b/plugins/dedicated-resources/src/org/apache/cloudstack/api/response/DedicatePodResponse.java @@ -40,6 +40,10 @@ public class DedicatePodResponse extends BaseResponse { @SerializedName("accountid") @Param(description="the Account Id to which the Pod is dedicated") private String accountId; + @SerializedName("affinitygroupid") + @Param(description = "the Dedication Affinity Group ID of the pod") + private String affinityGroupId; + public String getId() { return id; } @@ -79,4 +83,12 @@ public class DedicatePodResponse extends BaseResponse { public void setAccountId(String accountId) { this.accountId = accountId; } + + public String getAffinityGroupId() { + return affinityGroupId; + } + + public void setAffinityGroupId(String affinityGroupId) { + this.affinityGroupId = affinityGroupId; + } } diff --git a/plugins/dedicated-resources/src/org/apache/cloudstack/api/response/DedicateZoneResponse.java b/plugins/dedicated-resources/src/org/apache/cloudstack/api/response/DedicateZoneResponse.java index 57497cd3484..ee067ed1a1a 100644 --- a/plugins/dedicated-resources/src/org/apache/cloudstack/api/response/DedicateZoneResponse.java +++ b/plugins/dedicated-resources/src/org/apache/cloudstack/api/response/DedicateZoneResponse.java @@ -40,6 +40,10 @@ public class DedicateZoneResponse extends BaseResponse { @SerializedName("accountid") @Param(description="the Account Id to which the Zone is dedicated") private String accountId; + @SerializedName("affinitygroupid") + @Param(description = "the Dedication Affinity Group ID of the zone") + private String affinityGroupId; + public String getId() { return id; } @@ -80,4 +84,11 @@ public class DedicateZoneResponse extends BaseResponse { this.accountId = accountId; } + public String getAffinityGroupId() { + return affinityGroupId; + } + + public void setAffinityGroupId(String affinityGroupId) { + this.affinityGroupId = affinityGroupId; + } } diff --git a/plugins/dedicated-resources/src/org/apache/cloudstack/dedicated/DedicatedResourceManagerImpl.java b/plugins/dedicated-resources/src/org/apache/cloudstack/dedicated/DedicatedResourceManagerImpl.java index c321b22176e..cedb33bf8c9 100755 --- a/plugins/dedicated-resources/src/org/apache/cloudstack/dedicated/DedicatedResourceManagerImpl.java +++ b/plugins/dedicated-resources/src/org/apache/cloudstack/dedicated/DedicatedResourceManagerImpl.java @@ -25,6 +25,11 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; import com.cloud.utils.component.AdapterBase; + +import org.apache.cloudstack.acl.ControlledEntity; +import org.apache.cloudstack.affinity.AffinityGroup; +import org.apache.cloudstack.affinity.AffinityGroupService; +import org.apache.cloudstack.affinity.dao.AffinityGroupDao; import org.apache.cloudstack.api.commands.DedicateClusterCmd; import org.apache.cloudstack.api.commands.DedicateHostCmd; import org.apache.cloudstack.api.commands.DedicatePodCmd; @@ -92,6 +97,10 @@ public class DedicatedResourceManagerImpl implements DedicatedService { @Inject AccountManager _accountMgr; @Inject UserVmDao _userVmDao; @Inject ConfigurationDao _configDao; + @Inject AffinityGroupDao _affinityGroupDao; + + @Inject + AffinityGroupService _affinityGroupService; private int capacityReleaseInterval; @@ -212,13 +221,29 @@ public class DedicatedResourceManagerImpl implements DedicatedService { Transaction txn = Transaction.currentTxn(); txn.start(); - DedicatedResourceVO dedicatedResource = new DedicatedResourceVO(zoneId, null, null, null, null, null); + // find or create the affinity group by name under this account/domain + AffinityGroup group = findOrCreateDedicatedAffinityGroup(domainId, accountId); + if (group == null) { + s_logger.error("Unable to dedicate zone due to, failed to create dedication affinity group"); + throw new CloudRuntimeException("Failed to dedicate zone. Please contact Cloud Support."); + } + + DedicatedResourceVO dedicatedResource = new DedicatedResourceVO(zoneId, null, null, null, null, null, + group.getId()); try { dedicatedResource.setDomainId(domainId); if (accountId != null) { dedicatedResource.setAccountId(accountId); } dedicatedResource = _dedicatedDao.persist(dedicatedResource); + + // save the domainId in the zone + dc.setDomainId(domainId); + if (!_zoneDao.update(zoneId, dc)) { + throw new CloudRuntimeException( + "Failed to dedicate zone, could not set domainId. Please contact Cloud Support."); + } + } catch (Exception e) { s_logger.error("Unable to dedicate zone due to " + e.getMessage(), e); throw new CloudRuntimeException("Failed to dedicate zone. Please contact Cloud Support."); @@ -329,7 +354,14 @@ public class DedicatedResourceManagerImpl implements DedicatedService { Transaction txn = Transaction.currentTxn(); txn.start(); - DedicatedResourceVO dedicatedResource = new DedicatedResourceVO(null, podId, null, null, null, null); + // find or create the affinity group by name under this account/domain + AffinityGroup group = findOrCreateDedicatedAffinityGroup(domainId, accountId); + if (group == null) { + s_logger.error("Unable to dedicate zone due to, failed to create dedication affinity group"); + throw new CloudRuntimeException("Failed to dedicate zone. Please contact Cloud Support."); + } + DedicatedResourceVO dedicatedResource = new DedicatedResourceVO(null, podId, null, null, null, null, + group.getId()); try { dedicatedResource.setDomainId(domainId); if (accountId != null) { @@ -432,7 +464,14 @@ public class DedicatedResourceManagerImpl implements DedicatedService { Transaction txn = Transaction.currentTxn(); txn.start(); - DedicatedResourceVO dedicatedResource = new DedicatedResourceVO(null, null, clusterId, null, null, null); + // find or create the affinity group by name under this account/domain + AffinityGroup group = findOrCreateDedicatedAffinityGroup(domainId, accountId); + if (group == null) { + s_logger.error("Unable to dedicate zone due to, failed to create dedication affinity group"); + throw new CloudRuntimeException("Failed to dedicate zone. Please contact Cloud Support."); + } + DedicatedResourceVO dedicatedResource = new DedicatedResourceVO(null, null, clusterId, null, null, null, + group.getId()); try { dedicatedResource.setDomainId(domainId); if (accountId != null) { @@ -520,7 +559,14 @@ public class DedicatedResourceManagerImpl implements DedicatedService { Transaction txn = Transaction.currentTxn(); txn.start(); - DedicatedResourceVO dedicatedResource = new DedicatedResourceVO(null, null, null, hostId, null, null); + // find or create the affinity group by name under this account/domain + AffinityGroup group = findOrCreateDedicatedAffinityGroup(domainId, accountId); + if (group == null) { + s_logger.error("Unable to dedicate zone due to, failed to create dedication affinity group"); + throw new CloudRuntimeException("Failed to dedicate zone. Please contact Cloud Support."); + } + DedicatedResourceVO dedicatedResource = new DedicatedResourceVO(null, null, null, hostId, null, null, + group.getId()); try { dedicatedResource.setDomainId(domainId); if (accountId != null) { @@ -538,6 +584,45 @@ public class DedicatedResourceManagerImpl implements DedicatedService { return result; } + private AffinityGroup findOrCreateDedicatedAffinityGroup(Long domainId, Long accountId) { + if(domainId == null){ + return null; + } + + AffinityGroup group = null; + String accountName = null; + String affinityGroupName = null; + + if (accountId != null) { + AccountVO account = _accountDao.findById(accountId); + accountName = account.getAccountName(); + + group = _affinityGroupDao.findByAccountAndType(accountId, "ExplicitDedication"); + if (group != null) { + return group; + } + // default to a groupname with account/domain information + affinityGroupName = "DedicatedGrp-" + accountName; + + } else { + // domain level group + group = _affinityGroupDao.findDomainLevelGroupByType(domainId, "ExplicitDedication"); + if (group != null) { + return group; + } + // default to a groupname with account/domain information + String domainName = _domainDao.findById(domainId).getName(); + affinityGroupName = "DedicatedGrp-domain-" + domainName; + } + + + group = _affinityGroupService.createAffinityGroupInternal(accountName, domainId, affinityGroupName, + "ExplicitDedication", "dedicated resources group"); + + return group; + + } + private List getVmsOnHost(long hostId) { List vms = _userVmDao.listUpByHostId(hostId); List vmsByLastHostId = _userVmDao.listByLastHostId(hostId); @@ -620,10 +705,12 @@ public class DedicatedResourceManagerImpl implements DedicatedService { DataCenterVO dc = _zoneDao.findById(resource.getDataCenterId()); DomainVO domain = _domainDao.findById(resource.getDomainId()); AccountVO account = _accountDao.findById(resource.getAccountId()); + AffinityGroup group = _affinityGroupDao.findById(resource.getAffinityGroupId()); dedicateZoneResponse.setId(resource.getUuid()); dedicateZoneResponse.setZoneId(dc.getUuid()); dedicateZoneResponse.setZoneName(dc.getName()); dedicateZoneResponse.setDomainId(domain.getUuid()); + dedicateZoneResponse.setAffinityGroupId(group.getUuid()); if (account != null) { dedicateZoneResponse.setAccountId(account.getUuid()); } @@ -637,10 +724,12 @@ public class DedicatedResourceManagerImpl implements DedicatedService { HostPodVO pod = _podDao.findById(resource.getPodId()); DomainVO domain = _domainDao.findById(resource.getDomainId()); AccountVO account = _accountDao.findById(resource.getAccountId()); + AffinityGroup group = _affinityGroupDao.findById(resource.getAffinityGroupId()); dedicatePodResponse.setId(resource.getUuid()); dedicatePodResponse.setPodId(pod.getUuid()); dedicatePodResponse.setPodName(pod.getName()); dedicatePodResponse.setDomainId(domain.getUuid()); + dedicatePodResponse.setAffinityGroupId(group.getUuid()); if (account != null) { dedicatePodResponse.setAccountId(account.getUuid()); } @@ -654,10 +743,12 @@ public class DedicatedResourceManagerImpl implements DedicatedService { ClusterVO cluster = _clusterDao.findById(resource.getClusterId()); DomainVO domain = _domainDao.findById(resource.getDomainId()); AccountVO account = _accountDao.findById(resource.getAccountId()); + AffinityGroup group = _affinityGroupDao.findById(resource.getAffinityGroupId()); dedicateClusterResponse.setId(resource.getUuid()); dedicateClusterResponse.setClusterId(cluster.getUuid()); dedicateClusterResponse.setClusterName(cluster.getName()); dedicateClusterResponse.setDomainId(domain.getUuid()); + dedicateClusterResponse.setAffinityGroupId(group.getUuid()); if (account != null) { dedicateClusterResponse.setAccountId(account.getUuid()); } @@ -671,10 +762,12 @@ public class DedicatedResourceManagerImpl implements DedicatedService { HostVO host = _hostDao.findById(resource.getHostId()); DomainVO domain = _domainDao.findById(resource.getDomainId()); AccountVO account = _accountDao.findById(resource.getAccountId()); + AffinityGroup group = _affinityGroupDao.findById(resource.getAffinityGroupId()); dedicateHostResponse.setId(resource.getUuid()); dedicateHostResponse.setHostId(host.getUuid()); dedicateHostResponse.setHostName(host.getName()); dedicateHostResponse.setDomainId(domain.getUuid()); + dedicateHostResponse.setAffinityGroupId(group.getUuid()); if (account != null) { dedicateHostResponse.setAccountId(account.getUuid()); } @@ -706,6 +799,8 @@ public class DedicatedResourceManagerImpl implements DedicatedService { Long domainId = cmd.getDomainId(); String accountName = cmd.getAccountName(); Long accountId = null; + Long affinityGroupId = cmd.getAffinityGroupId(); + if (accountName != null) { if (domainId != null) { Account account = _accountDao.findActiveAccount(accountName, domainId); @@ -716,7 +811,8 @@ public class DedicatedResourceManagerImpl implements DedicatedService { throw new InvalidParameterValueException("Please specify the domain id of the account: " + accountName); } } - Pair, Integer> result = _dedicatedDao.searchDedicatedZones(zoneId, domainId, accountId); + Pair, Integer> result = _dedicatedDao.searchDedicatedZones(zoneId, domainId, + accountId, affinityGroupId); return new Pair, Integer>(result.first(), result.second()); } @@ -726,6 +822,8 @@ public class DedicatedResourceManagerImpl implements DedicatedService { Long domainId = cmd.getDomainId(); String accountName = cmd.getAccountName(); Long accountId = null; + Long affinityGroupId = cmd.getAffinityGroupId(); + if (accountName != null) { if (domainId != null) { Account account = _accountDao.findActiveAccount(accountName, domainId); @@ -736,7 +834,8 @@ public class DedicatedResourceManagerImpl implements DedicatedService { throw new InvalidParameterValueException("Please specify the domain id of the account: " + accountName); } } - Pair, Integer> result = _dedicatedDao.searchDedicatedPods(podId, domainId, accountId); + Pair, Integer> result = _dedicatedDao.searchDedicatedPods(podId, domainId, accountId, + affinityGroupId); return new Pair, Integer>(result.first(), result.second()); } @@ -746,6 +845,8 @@ public class DedicatedResourceManagerImpl implements DedicatedService { Long domainId = cmd.getDomainId(); String accountName = cmd.getAccountName(); Long accountId = null; + Long affinityGroupId = cmd.getAffinityGroupId(); + if (accountName != null) { if (domainId != null) { Account account = _accountDao.findActiveAccount(accountName, domainId); @@ -756,7 +857,8 @@ public class DedicatedResourceManagerImpl implements DedicatedService { throw new InvalidParameterValueException("Please specify the domain id of the account: " + accountName); } } - Pair, Integer> result = _dedicatedDao.searchDedicatedClusters(clusterId, domainId, accountId); + Pair, Integer> result = _dedicatedDao.searchDedicatedClusters(clusterId, domainId, + accountId, affinityGroupId); return new Pair, Integer>(result.first(), result.second()); } @@ -765,6 +867,8 @@ public class DedicatedResourceManagerImpl implements DedicatedService { Long hostId = cmd.getHostId(); Long domainId = cmd.getDomainId(); String accountName = cmd.getAccountName(); + Long affinityGroupId = cmd.getAffinityGroupId(); + Long accountId = null; if (accountName != null) { if (domainId != null) { @@ -777,7 +881,7 @@ public class DedicatedResourceManagerImpl implements DedicatedService { } } - Pair, Integer> result = _dedicatedDao.searchDedicatedHosts(hostId, domainId, accountId); + Pair, Integer> result = _dedicatedDao.searchDedicatedHosts(hostId, domainId, accountId, affinityGroupId); return new Pair, Integer>(result.first(), result.second()); } @@ -808,7 +912,30 @@ public class DedicatedResourceManagerImpl implements DedicatedService { if (!_dedicatedDao.remove(resourceId)) { throw new CloudRuntimeException("Failed to delete Resource " + resourceId); } + if (zoneId != null) { + // remove the domainId set in zone + DataCenterVO dc = _zoneDao.findById(zoneId); + if (dc != null) { + dc.setDomainId(null); + dc.setDomain(null); + if (!_zoneDao.update(zoneId, dc)) { + throw new CloudRuntimeException( + "Failed to release dedicated zone, could not clear domainId. Please contact Cloud Support."); + } + } + } + txn.commit(); + + // find the group associated and check if there are any more + // resources under that group + List resourcesInGroup = _dedicatedDao.listByAffinityGroupId(resource + .getAffinityGroupId()); + if (resourcesInGroup.isEmpty()) { + // delete the group + _affinityGroupService.deleteAffinityGroup(resource.getAffinityGroupId(), null, null, null); + } + } return true; } diff --git a/plugins/dedicated-resources/test/org/apache/cloudstack/dedicated/manager/DedicatedApiUnitTest.java b/plugins/dedicated-resources/test/org/apache/cloudstack/dedicated/manager/DedicatedApiUnitTest.java index 58aae238d88..66f9d5f14d7 100644 --- a/plugins/dedicated-resources/test/org/apache/cloudstack/dedicated/manager/DedicatedApiUnitTest.java +++ b/plugins/dedicated-resources/test/org/apache/cloudstack/dedicated/manager/DedicatedApiUnitTest.java @@ -28,6 +28,8 @@ import javax.naming.ConfigurationException; import junit.framework.Assert; +import org.apache.cloudstack.affinity.AffinityGroupService; +import org.apache.cloudstack.affinity.dao.AffinityGroupDao; import org.apache.cloudstack.dedicated.DedicatedResourceManagerImpl; import org.apache.cloudstack.test.utils.SpringUtils; import org.apache.log4j.Logger; @@ -208,28 +210,28 @@ public class DedicatedApiUnitTest { @Test(expected = CloudRuntimeException.class) public void dedicateZoneExistTest() { - DedicatedResourceVO dr = new DedicatedResourceVO(10L, null, null, null, domainId, accountId); + DedicatedResourceVO dr = new DedicatedResourceVO(10L, null, null, null, domainId, accountId, 12L); when(_dedicatedDao.findByZoneId(10L)).thenReturn(dr); _dedicatedService.dedicateZone(10L, domainId, accountName); } @Test(expected = CloudRuntimeException.class) public void dedicatePodExistTest() { - DedicatedResourceVO dr = new DedicatedResourceVO(null, 10L, null, null, domainId, accountId); + DedicatedResourceVO dr = new DedicatedResourceVO(null, 10L, null, null, domainId, accountId, 12L); when(_dedicatedDao.findByPodId(10L)).thenReturn(dr); _dedicatedService.dedicatePod(10L, domainId, accountName); } @Test(expected = CloudRuntimeException.class) public void dedicateClusterExistTest() { - DedicatedResourceVO dr = new DedicatedResourceVO(null, null, 10L, null, domainId, accountId); + DedicatedResourceVO dr = new DedicatedResourceVO(null, null, 10L, null, domainId, accountId, 12L); when(_dedicatedDao.findByClusterId(10L)).thenReturn(dr); _dedicatedService.dedicateCluster(10L, domainId, accountName); } @Test(expected = CloudRuntimeException.class) public void dedicateHostExistTest() { - DedicatedResourceVO dr = new DedicatedResourceVO(null, null, null, 10L, domainId, accountId); + DedicatedResourceVO dr = new DedicatedResourceVO(null, null, null, 10L, domainId, accountId, 12L); when(_dedicatedDao.findByHostId(10L)).thenReturn(dr); _dedicatedService.dedicateHost(10L, domainId, accountName); } @@ -306,6 +308,16 @@ public class DedicatedApiUnitTest { return Mockito.mock(ConfigurationDao.class); } + @Bean + public AffinityGroupService affinityGroupService() { + return Mockito.mock(AffinityGroupService.class); + } + + @Bean + public AffinityGroupDao affinityGroupDao() { + return Mockito.mock(AffinityGroupDao.class); + } + public static class Library implements TypeFilter { @Override public boolean match(MetadataReader mdr, MetadataReaderFactory arg1) throws IOException { diff --git a/plugins/deployment-planners/implicit-dedication/pom.xml b/plugins/deployment-planners/implicit-dedication/pom.xml index 18555923668..796cdb98b4e 100644 --- a/plugins/deployment-planners/implicit-dedication/pom.xml +++ b/plugins/deployment-planners/implicit-dedication/pom.xml @@ -23,7 +23,7 @@ org.apache.cloudstack cloudstack-plugins - 4.2.0-SNAPSHOT + 4.2.0 ../../pom.xml diff --git a/plugins/deployment-planners/user-concentrated-pod/pom.xml b/plugins/deployment-planners/user-concentrated-pod/pom.xml index df7c660630e..02a45866688 100644 --- a/plugins/deployment-planners/user-concentrated-pod/pom.xml +++ b/plugins/deployment-planners/user-concentrated-pod/pom.xml @@ -23,7 +23,7 @@ org.apache.cloudstack cloudstack-plugins - 4.2.0-SNAPSHOT + 4.2.0 ../../pom.xml diff --git a/plugins/deployment-planners/user-dispersing/pom.xml b/plugins/deployment-planners/user-dispersing/pom.xml index 0e5dbd58eb6..744b6acf6e5 100644 --- a/plugins/deployment-planners/user-dispersing/pom.xml +++ b/plugins/deployment-planners/user-dispersing/pom.xml @@ -23,7 +23,7 @@ org.apache.cloudstack cloudstack-plugins - 4.2.0-SNAPSHOT + 4.2.0 ../../pom.xml diff --git a/plugins/event-bus/rabbitmq/pom.xml b/plugins/event-bus/rabbitmq/pom.xml index bd4d0977c04..0556329c383 100644 --- a/plugins/event-bus/rabbitmq/pom.xml +++ b/plugins/event-bus/rabbitmq/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.2.0-SNAPSHOT + 4.2.0 ../../pom.xml diff --git a/plugins/file-systems/netapp/pom.xml b/plugins/file-systems/netapp/pom.xml index 0e6f427da36..600a9586462 100644 --- a/plugins/file-systems/netapp/pom.xml +++ b/plugins/file-systems/netapp/pom.xml @@ -23,7 +23,7 @@ org.apache.cloudstack cloudstack-plugins - 4.2.0-SNAPSHOT + 4.2.0 ../../pom.xml diff --git a/plugins/host-allocators/random/pom.xml b/plugins/host-allocators/random/pom.xml index 6fc76fe8dad..596afe6be9e 100644 --- a/plugins/host-allocators/random/pom.xml +++ b/plugins/host-allocators/random/pom.xml @@ -23,7 +23,7 @@ org.apache.cloudstack cloudstack-plugins - 4.2.0-SNAPSHOT + 4.2.0 ../../pom.xml diff --git a/plugins/hypervisors/baremetal/pom.xml b/plugins/hypervisors/baremetal/pom.xml index 328bd963c91..aa82ff1ddf7 100755 --- a/plugins/hypervisors/baremetal/pom.xml +++ b/plugins/hypervisors/baremetal/pom.xml @@ -21,7 +21,7 @@ org.apache.cloudstack cloudstack-plugins - 4.2.0-SNAPSHOT + 4.2.0 ../../pom.xml cloud-plugin-hypervisor-baremetal diff --git a/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BareMetalPingServiceImpl.java b/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BareMetalPingServiceImpl.java index 77958ae6a1c..55a63bb1baf 100755 --- a/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BareMetalPingServiceImpl.java +++ b/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BareMetalPingServiceImpl.java @@ -33,7 +33,7 @@ import javax.inject.Inject; import org.apache.cloudstack.api.AddBaremetalPxeCmd; import org.apache.cloudstack.api.AddBaremetalPxePingServerCmd; -import org.apache.cloudstack.api.ListBaremetalPxePingServersCmd; +import org.apache.cloudstack.api.ListBaremetalPxeServersCmd; import org.apache.log4j.Logger; import com.cloud.agent.api.Answer; @@ -271,34 +271,13 @@ public class BareMetalPingServiceImpl extends BareMetalPxeServiceBase implements @Override public BaremetalPxeResponse getApiResponse(BaremetalPxeVO vo) { - BaremetalPxePingResponse response = new BaremetalPxePingResponse(); - response.setId(String.valueOf(vo.getId())); - response.setPhysicalNetworkId(String.valueOf(vo.getPhysicalNetworkId())); - response.setPodId(String.valueOf(vo.getPodId())); - Map details = _hostDetailsDao.findDetails(vo.getHostId()); - response.setPingStorageServerIp(details.get(BaremetalPxeService.PXE_PARAM_PING_STORAGE_SERVER_IP)); - response.setPingDir(details.get(BaremetalPxeService.PXE_PARAM_PING_ROOT_DIR)); - response.setTftpDir(details.get(BaremetalPxeService.PXE_PARAM_TFTP_DIR)); - return response; + return null; } @Override - public List listPxeServers(ListBaremetalPxePingServersCmd cmd) { - SearchCriteriaService sc = SearchCriteria2.create(BaremetalPxeVO.class); - sc.addAnd(sc.getEntity().getDeviceType(), Op.EQ, BaremetalPxeType.PING.toString()); - if (cmd.getPodId() != null) { - sc.addAnd(sc.getEntity().getPodId(), Op.EQ, cmd.getPodId()); - if (cmd.getId() != null) { - sc.addAnd(sc.getEntity().getId(), Op.EQ, cmd.getId()); - } - } - List vos = sc.list(); - List responses = new ArrayList(vos.size()); - for (BaremetalPxeVO vo : vos) { - responses.add(getApiResponse(vo)); - } - return responses; + public List listPxeServers(ListBaremetalPxeServersCmd cmd) { + return null; } diff --git a/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalDhcpElement.java b/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalDhcpElement.java index 96d702d7501..34e83027da7 100755 --- a/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalDhcpElement.java +++ b/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalDhcpElement.java @@ -65,7 +65,8 @@ public class BaremetalDhcpElement extends AdapterBase implements DhcpServiceProv static { Capability cap = new Capability(BaremetalDhcpManager.BAREMETAL_DHCP_SERVICE_CAPABITLITY); Map baremetalCaps = new HashMap(); - baremetalCaps.put(cap, null); + baremetalCaps.put(cap, null); + baremetalCaps.put(Capability.DhcpAccrossMultipleSubnets, Boolean.TRUE.toString()); capabilities = new HashMap>(); capabilities.put(Service.Dhcp, baremetalCaps); } diff --git a/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalDhcpManagerImpl.java b/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalDhcpManagerImpl.java index 17e4481c535..9d18478626b 100755 --- a/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalDhcpManagerImpl.java +++ b/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalDhcpManagerImpl.java @@ -223,14 +223,9 @@ public class BaremetalDhcpManagerImpl extends ManagerBase implements BaremetalDh + " is in shutdown state in the physical network: " + cmd.getPhysicalNetworkId() + "to add this device"); } - HostPodVO pod = _podDao.findById(cmd.getPodId()); - if (pod == null) { - throw new IllegalArgumentException("Could not find pod with ID: " + cmd.getPodId()); - } - - List dhcps = _resourceMgr.listAllUpAndEnabledHosts(Host.Type.BaremetalDhcp, null, cmd.getPodId(), zoneId); + List dhcps = _resourceMgr.listAllUpAndEnabledHosts(Host.Type.BaremetalDhcp, null, null, zoneId); if (dhcps.size() != 0) { - throw new IllegalArgumentException("Already had a DHCP server in Pod: " + cmd.getPodId() + " zone: " + zoneId); + throw new IllegalArgumentException("Already had a DHCP server in zone: " + zoneId); } URI uri; @@ -242,16 +237,17 @@ public class BaremetalDhcpManagerImpl extends ManagerBase implements BaremetalDh } String ipAddress = uri.getHost(); - String guid = getDhcpServerGuid(Long.toString(zoneId) + "-" + Long.toString(cmd.getPodId()), "ExternalDhcp", ipAddress); + if (ipAddress == null) { + ipAddress = cmd.getUrl(); // the url is raw ip. For backforward compatibility, we have to support http://ip format as well + } + String guid = getDhcpServerGuid(Long.toString(zoneId), "ExternalDhcp", ipAddress); Map params = new HashMap(); params.put("type", cmd.getDhcpType()); params.put("zone", Long.toString(zoneId)); - params.put("pod", cmd.getPodId().toString()); params.put("ip", ipAddress); params.put("username", cmd.getUsername()); params.put("password", cmd.getPassword()); params.put("guid", guid); - params.put("gateway", pod.getGateway()); String dns = zone.getDns1(); if (dns == null) { dns = zone.getDns2(); @@ -284,7 +280,6 @@ public class BaremetalDhcpManagerImpl extends ManagerBase implements BaremetalDh vo.setHostId(dhcpServer.getId()); vo.setNetworkServiceProviderId(ntwkSvcProvider.getId()); vo.setPhysicalNetworkId(cmd.getPhysicalNetworkId()); - vo.setPodId(cmd.getPodId()); Transaction txn = Transaction.currentTxn(); txn.start(); _extDhcpDao.persist(vo); @@ -296,26 +291,32 @@ public class BaremetalDhcpManagerImpl extends ManagerBase implements BaremetalDh public BaremetalDhcpResponse generateApiResponse(BaremetalDhcpVO vo) { BaremetalDhcpResponse response = new BaremetalDhcpResponse(); response.setDeviceType(vo.getDeviceType()); - response.setId(String.valueOf(vo.getId())); - response.setPhysicalNetworkId(String.valueOf(vo.getPhysicalNetworkId())); - response.setProviderId(String.valueOf(vo.getNetworkServiceProviderId())); + response.setId(vo.getUuid()); + HostVO host = _hostDao.findById(vo.getHostId()); + response.setUrl(host.getPrivateIpAddress()); + PhysicalNetworkVO nwVO = _physicalNetworkDao.findById(vo.getPhysicalNetworkId()); + response.setPhysicalNetworkId(nwVO.getUuid()); + PhysicalNetworkServiceProviderVO providerVO = _physicalNetworkServiceProviderDao.findById(vo.getNetworkServiceProviderId()); + response.setProviderId(providerVO.getUuid()); + response.setObjectName("baremetaldhcp"); return response; } @Override public List listBaremetalDhcps(ListBaremetalDhcpCmd cmd) { + List responses = new ArrayList(); + if (cmd.getId() != null) { + BaremetalDhcpVO vo = _extDhcpDao.findById(cmd.getId()); + responses.add(generateApiResponse(vo)); + return responses; + } + SearchCriteriaService sc = SearchCriteria2.create(BaremetalDhcpVO.class); if (cmd.getDeviceType() != null) { sc.addAnd(sc.getEntity().getDeviceType(), Op.EQ, cmd.getDeviceType()); } - if (cmd.getPodId() != null) { - sc.addAnd(sc.getEntity().getPodId(), Op.EQ, cmd.getPodId()); - if (cmd.getId() != null) { - sc.addAnd(sc.getEntity().getId(), Op.EQ, cmd.getId()); - } - } + List vos = sc.list(); - List responses = new ArrayList(vos.size()); for (BaremetalDhcpVO vo : vos) { responses.add(generateApiResponse(vo)); } diff --git a/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalDhcpResourceBase.java b/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalDhcpResourceBase.java old mode 100644 new mode 100755 index 4496d5d0e70..2a17a436842 --- a/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalDhcpResourceBase.java +++ b/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalDhcpResourceBase.java @@ -55,8 +55,6 @@ public class BaremetalDhcpResourceBase extends ManagerBase implements ServerReso String _password; String _ip; String _zoneId; - String _podId; - String _gateway; String _dns; @Override @@ -67,8 +65,6 @@ public class BaremetalDhcpResourceBase extends ManagerBase implements ServerReso _username = (String)params.get("username"); _password = (String)params.get("password"); _zoneId = (String)params.get("zone"); - _podId = (String)params.get("pod"); - _gateway = (String)params.get("gateway"); _dns = (String)params.get("dns"); if (_guid == null) { @@ -79,10 +75,6 @@ public class BaremetalDhcpResourceBase extends ManagerBase implements ServerReso throw new ConfigurationException("No Zone specified"); } - if (_podId == null) { - throw new ConfigurationException("No Pod specified"); - } - if (_ip == null) { throw new ConfigurationException("No IP specified"); } @@ -95,10 +87,6 @@ public class BaremetalDhcpResourceBase extends ManagerBase implements ServerReso throw new ConfigurationException("No password specified"); } - if (_gateway == null) { - throw new ConfigurationException("No gateway specified"); - } - if (_dns == null) { throw new ConfigurationException("No dns specified"); } @@ -131,7 +119,6 @@ public class BaremetalDhcpResourceBase extends ManagerBase implements ServerReso StartupExternalDhcpCommand cmd = new StartupExternalDhcpCommand(); cmd.setName(_name); cmd.setDataCenter(_zoneId); - cmd.setPod(_podId); cmd.setPrivateIpAddress(_ip); cmd.setStorageIpAddress(""); cmd.setVersion(""); diff --git a/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalDhcpResponse.java b/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalDhcpResponse.java index 1875d3947a0..82506476be2 100755 --- a/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalDhcpResponse.java +++ b/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalDhcpResponse.java @@ -41,6 +41,9 @@ public class BaremetalDhcpResponse extends BaseResponse { @SerializedName(ApiConstants.DHCP_SERVER_TYPE) @Param(description="name of the provider") private String deviceType; + @SerializedName(ApiConstants.URL) @Param(description="url") + private String url; + public String getId() { return id; } @@ -72,4 +75,12 @@ public class BaremetalDhcpResponse extends BaseResponse { public void setDeviceType(String deviceType) { this.deviceType = deviceType; } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } } diff --git a/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalDnsmasqResource.java b/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalDnsmasqResource.java old mode 100644 new mode 100755 index 6841c525107..d0fb2b4c098 --- a/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalDnsmasqResource.java +++ b/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalDnsmasqResource.java @@ -73,11 +73,13 @@ public class BaremetalDnsmasqResource extends BaremetalDhcpResourceBase { throw new ConfigurationException("Can not find script prepare_dnsmasq.sh at " + prepareDnsmasq); } scp.put(prepareDnsmasqPath, "/usr/bin/", "0755"); - + + /* String prepareCmd = String.format("sh /usr/bin/prepare_dnsmasq.sh %1$s %2$s %3$s", _gateway, _dns, _ip); if (!SSHCmdHelper.sshExecuteCmd(sshConnection, prepareCmd)) { throw new ConfigurationException("prepare dnsmasq at " + _ip + " failed"); } + */ s_logger.debug("Dnsmasq resource configure successfully"); return true; diff --git a/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalKickStartServiceImpl.java b/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalKickStartServiceImpl.java index 5858e7ebd61..582841cbaf6 100755 --- a/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalKickStartServiceImpl.java +++ b/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalKickStartServiceImpl.java @@ -29,7 +29,7 @@ import javax.inject.Inject; import org.apache.cloudstack.api.AddBaremetalKickStartPxeCmd; import org.apache.cloudstack.api.AddBaremetalPxeCmd; -import org.apache.cloudstack.api.ListBaremetalPxePingServersCmd; +import org.apache.cloudstack.api.ListBaremetalPxeServersCmd; import org.apache.log4j.Logger; import com.cloud.agent.api.Answer; @@ -199,6 +199,9 @@ public class BaremetalKickStartServiceImpl extends BareMetalPxeServiceBase imple throw new IllegalArgumentException(e.getMessage()); } String ipAddress = uri.getHost(); + if (ipAddress == null) { + ipAddress = cmd.getUrl(); + } String guid = getPxeServerGuid(Long.toString(zoneId), BaremetalPxeType.KICK_START.toString(), ipAddress); @@ -236,27 +239,28 @@ public class BaremetalKickStartServiceImpl extends BareMetalPxeServiceBase imple @Override public BaremetalPxeResponse getApiResponse(BaremetalPxeVO vo) { - BaremetalPxeKickStartResponse response = new BaremetalPxeKickStartResponse(); - response.setId(String.valueOf(vo.getId())); - response.setPhysicalNetworkId(String.valueOf(vo.getPhysicalNetworkId())); - response.setPodId(String.valueOf(vo.getPodId())); - Map details = _hostDetailsDao.findDetails(vo.getHostId()); - response.setTftpDir(details.get(BaremetalPxeService.PXE_PARAM_TFTP_DIR)); + BaremetalPxeResponse response = new BaremetalPxeResponse(); + response.setId(vo.getUuid()); + HostVO host = _hostDao.findById(vo.getHostId()); + response.setUrl(host.getPrivateIpAddress()); + PhysicalNetworkServiceProviderVO providerVO = _physicalNetworkServiceProviderDao.findById(vo.getNetworkServiceProviderId()); + response.setPhysicalNetworkId(providerVO.getUuid()); + PhysicalNetworkVO nwVO = _physicalNetworkDao.findById(vo.getPhysicalNetworkId()); + response.setPhysicalNetworkId(nwVO.getUuid()); + response.setObjectName("baremetalpxeserver"); return response; } @Override - public List listPxeServers(ListBaremetalPxePingServersCmd cmd) { - SearchCriteriaService sc = SearchCriteria2.create(BaremetalPxeVO.class); - sc.addAnd(sc.getEntity().getDeviceType(), Op.EQ, BaremetalPxeType.KICK_START.toString()); - if (cmd.getPodId() != null) { - sc.addAnd(sc.getEntity().getPodId(), Op.EQ, cmd.getPodId()); - if (cmd.getId() != null) { - sc.addAnd(sc.getEntity().getId(), Op.EQ, cmd.getId()); - } + public List listPxeServers(ListBaremetalPxeServersCmd cmd) { + List responses = new ArrayList(); + if (cmd.getId() != null) { + BaremetalPxeVO vo = _pxeDao.findById(cmd.getId()); + responses.add(getApiResponse(vo)); + return responses; } - List vos = sc.list(); - List responses = new ArrayList(vos.size()); + + List vos = _pxeDao.listAll(); for (BaremetalPxeVO vo : vos) { responses.add(getApiResponse(vo)); } diff --git a/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalPxeManager.java b/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalPxeManager.java index 7984ddd9750..7289eb243ee 100755 --- a/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalPxeManager.java +++ b/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalPxeManager.java @@ -25,7 +25,7 @@ package com.cloud.baremetal.networkservice; import java.util.List; import org.apache.cloudstack.api.AddBaremetalPxeCmd; -import org.apache.cloudstack.api.ListBaremetalPxePingServersCmd; +import org.apache.cloudstack.api.ListBaremetalPxeServersCmd; import com.cloud.baremetal.database.BaremetalPxeVO; import com.cloud.deploy.DeployDestination; @@ -37,7 +37,6 @@ import com.cloud.utils.component.Manager; import com.cloud.utils.component.PluggableService; import com.cloud.vm.NicProfile; import com.cloud.vm.ReservationContext; -import com.cloud.vm.UserVmVO; import com.cloud.vm.VirtualMachineProfile; public interface BaremetalPxeManager extends Manager, PluggableService { @@ -56,7 +55,7 @@ public interface BaremetalPxeManager extends Manager, PluggableService { BaremetalPxeResponse getApiResponse(BaremetalPxeVO vo); - List listPxeServers(ListBaremetalPxePingServersCmd cmd); + List listPxeServers(ListBaremetalPxeServersCmd cmd); boolean addUserData(NicProfile nic, VirtualMachineProfile vm); diff --git a/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalPxeManagerImpl.java b/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalPxeManagerImpl.java index 555adf792d0..6ed1a8ba8e0 100755 --- a/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalPxeManagerImpl.java +++ b/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalPxeManagerImpl.java @@ -34,7 +34,7 @@ import javax.naming.ConfigurationException; import org.apache.cloudstack.api.AddBaremetalKickStartPxeCmd; import org.apache.cloudstack.api.AddBaremetalPxeCmd; import org.apache.cloudstack.api.AddBaremetalPxePingServerCmd; -import org.apache.cloudstack.api.ListBaremetalPxePingServersCmd; +import org.apache.cloudstack.api.ListBaremetalPxeServersCmd; import org.apache.log4j.Logger; import com.cloud.agent.AgentManager; @@ -178,8 +178,8 @@ public class BaremetalPxeManagerImpl extends ManagerBase implements BaremetalPxe } @Override - public List listPxeServers(ListBaremetalPxePingServersCmd cmd) { - return getServiceByType(BaremetalPxeManager.BaremetalPxeType.PING.toString()).listPxeServers(cmd); + public List listPxeServers(ListBaremetalPxeServersCmd cmd) { + return getServiceByType(BaremetalPxeType.KICK_START.toString()).listPxeServers(cmd); } @Override @@ -246,7 +246,7 @@ public class BaremetalPxeManagerImpl extends ManagerBase implements BaremetalPxe List> cmds = new ArrayList>(); cmds.add(AddBaremetalKickStartPxeCmd.class); cmds.add(AddBaremetalPxePingServerCmd.class); - cmds.add(ListBaremetalPxePingServersCmd.class); + cmds.add(ListBaremetalPxeServersCmd.class); return cmds; } } diff --git a/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalPxeResponse.java b/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalPxeResponse.java index 2103020cfef..ef4fd57563a 100755 --- a/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalPxeResponse.java +++ b/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalPxeResponse.java @@ -33,9 +33,9 @@ public class BaremetalPxeResponse extends BaseResponse { @SerializedName(ApiConstants.PROVIDER) @Param(description="name of the provider") private String providerId; - - @SerializedName(ApiConstants.POD_ID) @Param(description="pod id where the device is in") - private String podId; + + @SerializedName(ApiConstants.URL) @Param(description="url") + private String url; public String getId() { return id; @@ -61,11 +61,11 @@ public class BaremetalPxeResponse extends BaseResponse { this.providerId = providerId; } - public String getPodId() { - return podId; - } + public String getUrl() { + return url; + } - public void setPodId(String podId) { - this.podId = podId; - } + public void setUrl(String url) { + this.url = url; + } } diff --git a/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalPxeService.java b/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalPxeService.java index af6a6c8043c..0a0a91b4e84 100755 --- a/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalPxeService.java +++ b/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalPxeService.java @@ -25,11 +25,10 @@ package com.cloud.baremetal.networkservice; import java.util.List; import org.apache.cloudstack.api.AddBaremetalPxeCmd; -import org.apache.cloudstack.api.ListBaremetalPxePingServersCmd; +import org.apache.cloudstack.api.ListBaremetalPxeServersCmd; import com.cloud.baremetal.database.BaremetalPxeVO; import com.cloud.deploy.DeployDestination; -import com.cloud.host.Host; import com.cloud.uservm.UserVm; import com.cloud.utils.component.Adapter; import com.cloud.vm.NicProfile; @@ -47,7 +46,7 @@ public interface BaremetalPxeService extends Adapter { BaremetalPxeResponse getApiResponse(BaremetalPxeVO vo); - List listPxeServers(ListBaremetalPxePingServersCmd cmd); + List listPxeServers(ListBaremetalPxeServersCmd cmd); String getPxeServiceType(); diff --git a/plugins/hypervisors/baremetal/src/org/apache/cloudstack/api/AddBaremetalDhcpCmd.java b/plugins/hypervisors/baremetal/src/org/apache/cloudstack/api/AddBaremetalDhcpCmd.java index 7c56a8aeb02..a8c15e26b34 100755 --- a/plugins/hypervisors/baremetal/src/org/apache/cloudstack/api/AddBaremetalDhcpCmd.java +++ b/plugins/hypervisors/baremetal/src/org/apache/cloudstack/api/AddBaremetalDhcpCmd.java @@ -44,7 +44,7 @@ import com.cloud.exception.ResourceUnavailableException; import com.cloud.user.UserContext; @APICommand(name="addBaremetalDhcp", description="adds a baremetal dhcp server", responseObject = BaremetalDhcpResponse.class) public class AddBaremetalDhcpCmd extends BaseAsyncCmd { - private static final String s_name = "addexternaldhcpresponse"; + private static final String s_name = "addbaremetaldhcpresponse"; public static final Logger s_logger = Logger.getLogger(AddBaremetalDhcpCmd.class); @Inject BaremetalDhcpManager mgr; @@ -55,9 +55,6 @@ public class AddBaremetalDhcpCmd extends BaseAsyncCmd { @Parameter(name=ApiConstants.PHYSICAL_NETWORK_ID, type=CommandType.UUID, entityType=PhysicalNetworkResponse.class, required=true, description="the Physical Network ID") private Long physicalNetworkId; - @Parameter(name=ApiConstants.POD_ID, type=CommandType.UUID, entityType=PodResponse.class, required = true, description="Pod Id") - private Long podId; - @Parameter(name=ApiConstants.DHCP_SERVER_TYPE, type=CommandType.STRING, required = true, description="Type of dhcp device") private String dhcpType; @@ -86,7 +83,6 @@ public class AddBaremetalDhcpCmd extends BaseAsyncCmd { try { BaremetalDhcpVO vo = mgr.addDchpServer(this); BaremetalDhcpResponse response = mgr.generateApiResponse(vo); - response.setObjectName(s_name); response.setResponseName(getCommandName()); this.setResponseObject(response); } catch (Exception e) { @@ -105,14 +101,6 @@ public class AddBaremetalDhcpCmd extends BaseAsyncCmd { return UserContext.current().getCaller().getId(); } - public Long getPodId() { - return podId; - } - - public void setPodId(Long podId) { - this.podId = podId; - } - public String getDhcpType() { return dhcpType; } diff --git a/plugins/hypervisors/baremetal/src/org/apache/cloudstack/api/AddBaremetalPxeCmd.java b/plugins/hypervisors/baremetal/src/org/apache/cloudstack/api/AddBaremetalPxeCmd.java index e1b39e945a3..fe311e32af5 100755 --- a/plugins/hypervisors/baremetal/src/org/apache/cloudstack/api/AddBaremetalPxeCmd.java +++ b/plugins/hypervisors/baremetal/src/org/apache/cloudstack/api/AddBaremetalPxeCmd.java @@ -20,6 +20,7 @@ package org.apache.cloudstack.api; import javax.inject.Inject; +import com.cloud.baremetal.networkservice.BaremetalPxeResponse; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; @@ -42,7 +43,7 @@ import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; import com.cloud.user.UserContext; public class AddBaremetalPxeCmd extends BaseAsyncCmd { - private static final String s_name = "addexternalpxeresponse"; + private static final String s_name = "addbaremetalpxeresponse"; public static final Logger s_logger = Logger.getLogger(AddBaremetalPxeCmd.class); @Inject BaremetalPxeManager pxeMgr; @@ -82,6 +83,9 @@ public class AddBaremetalPxeCmd extends BaseAsyncCmd { ResourceAllocationException, NetworkRuleConflictException { try { BaremetalPxeVO vo = pxeMgr.addPxeServer(this); + BaremetalPxeResponse rsp = pxeMgr.getApiResponse(vo); + rsp.setResponseName(getCommandName()); + this.setResponseObject(rsp); } catch (Exception e) { s_logger.warn("Unable to add external pxe server with url: " + getUrl(), e); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage()); diff --git a/plugins/hypervisors/baremetal/src/org/apache/cloudstack/api/ListBaremetalDhcpCmd.java b/plugins/hypervisors/baremetal/src/org/apache/cloudstack/api/ListBaremetalDhcpCmd.java index a147ca2a29c..412a3d4615b 100755 --- a/plugins/hypervisors/baremetal/src/org/apache/cloudstack/api/ListBaremetalDhcpCmd.java +++ b/plugins/hypervisors/baremetal/src/org/apache/cloudstack/api/ListBaremetalDhcpCmd.java @@ -43,7 +43,7 @@ import com.cloud.exception.ResourceUnavailableException; @APICommand(name="listBaremetalDhcp", description="list baremetal dhcp servers", responseObject = BaremetalDhcpResponse.class) public class ListBaremetalDhcpCmd extends BaseListCmd { private static final Logger s_logger = Logger.getLogger(ListBaremetalDhcpCmd.class); - private static final String s_name = "listexternaldhcpresponse"; + private static final String s_name = "listbaremetaldhcpresponse"; @Inject BaremetalDhcpManager _dhcpMgr; // /////////////////////////////////////////////////// @@ -52,9 +52,6 @@ public class ListBaremetalDhcpCmd extends BaseListCmd { @Parameter(name = ApiConstants.ID, type = CommandType.LONG, description = "DHCP server device ID") private Long id; - @Parameter(name = ApiConstants.POD_ID, type = CommandType.LONG, description = "Pod ID where pxe server is in") - private Long podId; - @Parameter(name = ApiConstants.DHCP_SERVER_TYPE, type = CommandType.STRING, description = "Type of DHCP device") private String deviceType; @@ -66,14 +63,6 @@ public class ListBaremetalDhcpCmd extends BaseListCmd { this.id = id; } - public Long getPodId() { - return podId; - } - - public void setPodId(Long podId) { - this.podId = podId; - } - public String getDeviceType() { return deviceType; } @@ -90,6 +79,7 @@ public class ListBaremetalDhcpCmd extends BaseListCmd { List dhcpResponses = _dhcpMgr.listBaremetalDhcps(this); response.setResponses(dhcpResponses); response.setResponseName(getCommandName()); + response.setObjectName("baremetaldhcps"); this.setResponseObject(response); } catch (Exception e) { s_logger.debug("Exception happend while executing ListBaremetalDhcpCmd"); diff --git a/plugins/hypervisors/baremetal/src/org/apache/cloudstack/api/ListBaremetalPxePingServersCmd.java b/plugins/hypervisors/baremetal/src/org/apache/cloudstack/api/ListBaremetalPxeServersCmd.java similarity index 71% rename from plugins/hypervisors/baremetal/src/org/apache/cloudstack/api/ListBaremetalPxePingServersCmd.java rename to plugins/hypervisors/baremetal/src/org/apache/cloudstack/api/ListBaremetalPxeServersCmd.java index 926ad1d9c7e..3463024e45e 100755 --- a/plugins/hypervisors/baremetal/src/org/apache/cloudstack/api/ListBaremetalPxePingServersCmd.java +++ b/plugins/hypervisors/baremetal/src/org/apache/cloudstack/api/ListBaremetalPxeServersCmd.java @@ -22,15 +22,8 @@ import java.util.List; import javax.inject.Inject; -import org.apache.cloudstack.api.APICommand; -import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.ApiErrorCode; -import org.apache.cloudstack.api.BaseCmd; -import org.apache.cloudstack.api.BaseCmd.CommandType; -import org.apache.cloudstack.api.BaseListCmd; -import org.apache.cloudstack.api.Parameter; -import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.PhysicalNetworkResponse; import org.apache.log4j.Logger; import com.cloud.baremetal.networkservice.BaremetalPxeManager; @@ -41,10 +34,10 @@ import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.NetworkRuleConflictException; import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; -@APICommand(name="listBaremetalPxePingServer", description="list baremetal ping pxe server", responseObject = BaremetalPxePingResponse.class) -public class ListBaremetalPxePingServersCmd extends BaseListCmd { - private static final Logger s_logger = Logger.getLogger(ListBaremetalPxePingServersCmd.class); - private static final String s_name = "listpingpxeserverresponse"; +@APICommand(name="listBaremetalPxeServers", description="list baremetal pxe server", responseObject = BaremetalPxeResponse.class) +public class ListBaremetalPxeServersCmd extends BaseListCmd { + private static final Logger s_logger = Logger.getLogger(ListBaremetalPxeServersCmd.class); + private static final String s_name = "listbaremetalpxeserversresponse"; @Inject BaremetalPxeManager _pxeMgr; @@ -52,11 +45,8 @@ public class ListBaremetalPxePingServersCmd extends BaseListCmd { // ////////////// API parameters ///////////////////// // /////////////////////////////////////////////////// - @Parameter(name = ApiConstants.ID, type = CommandType.LONG, description = "Ping pxe server device ID") + @Parameter(name = ApiConstants.ID, type = CommandType.LONG, description = "Pxe server device ID") private Long id; - - @Parameter(name = ApiConstants.POD_ID, type = CommandType.LONG, description = "Pod ID where pxe server is in") - private Long podId; public Long getId() { return id; @@ -66,14 +56,6 @@ public class ListBaremetalPxePingServersCmd extends BaseListCmd { this.id = id; } - public Long getPodId() { - return podId; - } - - public void setPodId(Long podId) { - this.podId = podId; - } - @Override public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { @@ -82,9 +64,10 @@ public class ListBaremetalPxePingServersCmd extends BaseListCmd { List pxeResponses = _pxeMgr.listPxeServers(this); response.setResponses(pxeResponses); response.setResponseName(getCommandName()); + response.setObjectName("baremetalpxeservers"); this.setResponseObject(response); } catch (Exception e) { - s_logger.debug("Exception happend while executing ListPingPxeServersCmd" ,e); + s_logger.debug("Exception happened while executing ListPingPxeServersCmd" ,e); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage()); } } @@ -93,5 +76,4 @@ public class ListBaremetalPxePingServersCmd extends BaseListCmd { public String getCommandName() { return s_name; } - } diff --git a/plugins/hypervisors/kvm/pom.xml b/plugins/hypervisors/kvm/pom.xml index 1babe7cbf56..b058b25d946 100644 --- a/plugins/hypervisors/kvm/pom.xml +++ b/plugins/hypervisors/kvm/pom.xml @@ -16,7 +16,7 @@ org.apache.cloudstack cloudstack-plugins - 4.2.0-SNAPSHOT + 4.2.0 ../../pom.xml @@ -45,6 +45,12 @@ rados ${cs.rados-java.version} + + net.java.dev.jna + jna + provided + ${cs.jna.version} + install diff --git a/plugins/hypervisors/kvm/src/com/cloud/ha/KVMInvestigator.java b/plugins/hypervisors/kvm/src/com/cloud/ha/KVMInvestigator.java new file mode 100644 index 00000000000..4d8322d5c76 --- /dev/null +++ b/plugins/hypervisors/kvm/src/com/cloud/ha/KVMInvestigator.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.cloud.ha; + +import com.cloud.agent.AgentManager; +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.CheckOnHostAnswer; +import com.cloud.agent.api.CheckOnHostCommand; +import com.cloud.host.HostVO; +import com.cloud.host.Status; +import com.cloud.host.dao.HostDao; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.resource.ResourceManager; +import com.cloud.utils.component.AdapterBase; +import com.cloud.vm.VMInstanceVO; +import org.apache.log4j.Logger; + +import javax.ejb.Local; +import javax.inject.Inject; +import java.util.List; + +@Local(value=Investigator.class) +public class KVMInvestigator extends AdapterBase implements Investigator { + private final static Logger s_logger = Logger.getLogger(KVMInvestigator.class); + @Inject + HostDao _hostDao; + @Inject + AgentManager _agentMgr; + @Inject + ResourceManager _resourceMgr; + @Override + public Boolean isVmAlive(VMInstanceVO vm, HostVO host) { + Status status = isAgentAlive(host); + if (status == null) { + return null; + } + return status == Status.Up ? true : null; + } + + @Override + public Status isAgentAlive(HostVO agent) { + if (agent.getHypervisorType() != Hypervisor.HypervisorType.KVM) { + return null; + } + CheckOnHostCommand cmd = new CheckOnHostCommand(agent); + List neighbors = _resourceMgr.listHostsInClusterByStatus(agent.getClusterId(), Status.Up); + for (HostVO neighbor : neighbors) { + if (neighbor.getId() == agent.getId() || neighbor.getHypervisorType() != Hypervisor.HypervisorType.KVM) { + continue; + } + try { + Answer answer = _agentMgr.easySend(neighbor.getId(), cmd); + if (answer != null) { + return answer.getResult() ? Status.Down : Status.Up; + } + } catch (Exception e) { + s_logger.debug("Failed to send command to host: " + neighbor.getId()); + } + } + + return null; + } +} diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java index 195cf409e5d..e3779a77cd7 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java @@ -45,6 +45,7 @@ public class BridgeVifDriver extends VifDriverBase { private static final Object _vnetBridgeMonitor = new Object(); private String _modifyVlanPath; + private String bridgeNameSchema; @Override public void configure(Map params) throws ConfigurationException { @@ -60,6 +61,8 @@ public class BridgeVifDriver extends VifDriverBase { networkScriptsDir = "scripts/vm/network/vnet"; } + bridgeNameSchema = (String) params.get("network.bridge.name.schema"); + String value = (String) params.get("scripts.timeout"); _timeout = NumbersUtil.parseInt(value, 30 * 60) * 1000; @@ -144,13 +147,15 @@ public class BridgeVifDriver extends VifDriverBase { } private String setVnetBrName(String pifName, String vnetId) { - String brName = "br" + pifName + "-"+ vnetId; - String oldStyleBrName = "cloudVirBr" + vnetId; - - String cmdout = Script.runSimpleBashScript("brctl show | grep " + oldStyleBrName); - if (cmdout != null && cmdout.contains(oldStyleBrName)) { - s_logger.info("Using old style bridge name for vlan " + vnetId + " because existing bridge " + oldStyleBrName + " was found"); - brName = oldStyleBrName; + String brName = null; + if (bridgeNameSchema != null) { + if (bridgeNameSchema.equalsIgnoreCase("3.0")) { + brName = "cloudVirBr" + vnetId; + } else if (bridgeNameSchema.equalsIgnoreCase("4.0")) { + brName = "br" + pifName + "-"+ vnetId; + } + } else { + brName = "br" + pifName + "-"+ vnetId; } return brName; diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 56699afec40..3ee811ff835 100755 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -21,6 +21,7 @@ import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; +import java.io.BufferedOutputStream; import java.io.FileReader; import java.io.IOException; import java.io.InputStream; @@ -56,6 +57,7 @@ import java.util.regex.Pattern; import javax.ejb.Local; import javax.naming.ConfigurationException; +import com.cloud.agent.api.CheckOnHostCommand; import org.apache.cloudstack.storage.command.StorageSubSystemCommand; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; import org.apache.cloudstack.storage.to.VolumeObjectTO; @@ -191,6 +193,7 @@ import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.hypervisor.kvm.resource.KVMHABase.NfsStoragePool; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.ClockDef; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.ConsoleDef; +import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.CpuModeDef; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.CpuTuneDef; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.DevicesDef; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.DiskDef; @@ -242,6 +245,13 @@ import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachine.State; import com.cloud.vm.VirtualMachineName; +import com.ceph.rados.Rados; +import com.ceph.rados.RadosException; +import com.ceph.rados.IoCTX; +import com.ceph.rbd.Rbd; +import com.ceph.rbd.RbdImage; +import com.ceph.rbd.RbdException; + /** * LibvirtComputingResource execute requests on the computing/routing host using * the libvirt API @@ -361,6 +371,8 @@ ServerResource { private boolean _can_bridge_firewall; protected String _localStoragePath; protected String _localStorageUUID; + protected String _guestCpuMode; + protected String _guestCpuModel; private final Map _pifs = new HashMap(); private final Map _vmStats = new ConcurrentHashMap(); @@ -752,6 +764,20 @@ ServerResource { s_logger.trace("Ignoring libvirt error.", e); } + _guestCpuMode = (String) params.get("guest.cpu.mode"); + if (_guestCpuMode != null) { + _guestCpuModel = (String) params.get("guest.cpu.model"); + + if(_hypervisorLibvirtVersion < (9 * 1000 + 10)) { + s_logger.warn("LibVirt version 0.9.10 required for guest cpu mode, but version " + + prettyVersion(_hypervisorLibvirtVersion) + " detected, so it will be disabled"); + _guestCpuMode = ""; + _guestCpuModel = ""; + } + params.put("guest.cpu.mode", _guestCpuMode); + params.put("guest.cpu.model", _guestCpuModel); + } + String[] info = NetUtils.getNetworkParams(_privateNic); _monitor = new KVMHAMonitor(null, info[0], _heartBeatPath); @@ -1071,7 +1097,7 @@ ServerResource { } } - private void passCmdLine(String vmName, String cmdLine) + private boolean passCmdLine(String vmName, String cmdLine) throws InternalErrorException { final Script command = new Script(_patchViaSocketPath, 5*1000, s_logger); String result; @@ -1080,7 +1106,9 @@ ServerResource { result = command.execute(); if (result != null) { s_logger.debug("passcmd failed:" + result); + return false; } + return true; } boolean isDirectAttachedNetwork(String type) { @@ -1258,6 +1286,8 @@ ServerResource { return this.storageHandler.handleStorageCommands((StorageSubSystemCommand)cmd); } else if (cmd instanceof PvlanSetupCommand) { return execute((PvlanSetupCommand) cmd); + } else if (cmd instanceof CheckOnHostCommand) { + return execute((CheckOnHostCommand)cmd); } else { s_logger.warn("Unsupported command "); return Answer.createUnsupportedCommandAnswer(cmd); @@ -1391,6 +1421,26 @@ ServerResource { } + protected Answer execute(CheckOnHostCommand cmd) { + ExecutorService executors = Executors.newSingleThreadExecutor(); + List pools = _monitor.getStoragePools(); + KVMHAChecker ha = new KVMHAChecker(pools, cmd.getHost().getPrivateNetwork().getIp()); + Future future = executors.submit(ha); + try { + Boolean result = future.get(); + if (result) { + return new Answer(cmd, false, "Heart is still beating..."); + } else { + return new Answer(cmd); + } + } catch (InterruptedException e) { + return new Answer(cmd, false, "can't get status of host:"); + } catch (ExecutionException e) { + return new Answer(cmd, false, "can't get status of host:"); + } + + } + protected Storage.StorageResourceType getStorageResourceType() { return Storage.StorageResourceType.STORAGE_POOL; } @@ -1517,35 +1567,66 @@ ServerResource { String path = vol.getPath(); String type = getResizeScriptType(pool, vol); - if (type == null) { - return new ResizeVolumeAnswer(cmd, false, "Unsupported volume format: pool type '" - + pool.getType() + "' and volume format '" + vol.getFormat() + "'"); - } else if (type.equals("QCOW2") && shrinkOk) { - return new ResizeVolumeAnswer(cmd, false, "Unable to shrink volumes of type " + type); + /** + * RBD volumes can't be resized via a Bash script or via libvirt + * + * libvirt-java doesn't implemented resizing volumes, so we have to do this manually + * + * Future fix would be to hand this over to libvirt + */ + if (pool.getType() == StoragePoolType.RBD) { + try { + Rados r = new Rados(pool.getAuthUserName()); + r.confSet("mon_host", pool.getSourceHost() + ":" + pool.getSourcePort()); + r.confSet("key", pool.getAuthSecret()); + r.connect(); + s_logger.debug("Succesfully connected to Ceph cluster at " + r.confGet("mon_host")); + + IoCTX io = r.ioCtxCreate(pool.getSourceDir()); + Rbd rbd = new Rbd(io); + RbdImage image = rbd.open(vol.getName()); + + s_logger.debug("Resizing RBD volume " + vol.getName() + " to " + newSize + " bytes"); + image.resize(newSize); + rbd.close(image); + + r.ioCtxDestroy(io); + s_logger.debug("Succesfully resized RBD volume " + vol.getName() + " to " + newSize + " bytes"); + } catch (RadosException e) { + return new ResizeVolumeAnswer(cmd, false, e.toString()); + } catch (RbdException e) { + return new ResizeVolumeAnswer(cmd, false, e.toString()); + } + } else { + if (type == null) { + return new ResizeVolumeAnswer(cmd, false, "Unsupported volume format: pool type '" + + pool.getType() + "' and volume format '" + vol.getFormat() + "'"); + } else if (type.equals("QCOW2") && shrinkOk) { + return new ResizeVolumeAnswer(cmd, false, "Unable to shrink volumes of type " + type); + } + + s_logger.debug("got to the stage where we execute the volume resize, params:" + + path + "," + currentSize + "," + newSize + "," + type + "," + vmInstanceName + "," + shrinkOk); + final Script resizecmd = new Script(_resizeVolumePath, + _cmdsTimeout, s_logger); + resizecmd.add("-s",String.valueOf(newSize)); + resizecmd.add("-c",String.valueOf(currentSize)); + resizecmd.add("-p",path); + resizecmd.add("-t",type); + resizecmd.add("-r",String.valueOf(shrinkOk)); + resizecmd.add("-v",vmInstanceName); + String result = resizecmd.execute(); + + if (result != null) { + return new ResizeVolumeAnswer(cmd, false, result); + } } - s_logger.debug("got to the stage where we execute the volume resize, params:" - + path + "," + currentSize + "," + newSize + "," + type + "," + vmInstanceName + "," + shrinkOk); - final Script resizecmd = new Script(_resizeVolumePath, - _cmdsTimeout, s_logger); - resizecmd.add("-s",String.valueOf(newSize)); - resizecmd.add("-c",String.valueOf(currentSize)); - resizecmd.add("-p",path); - resizecmd.add("-t",type); - resizecmd.add("-r",String.valueOf(shrinkOk)); - resizecmd.add("-v",vmInstanceName); - String result = resizecmd.execute(); - - if (result == null) { - - /* fetch new size as seen from libvirt, don't want to assume anything */ - pool = _storagePoolMgr.getStoragePool(spool.getType(), spool.getUuid()); - long finalSize = pool.getPhysicalDisk(volid).getVirtualSize(); - s_logger.debug("after resize, size reports as " + finalSize + ", requested " + newSize); - return new ResizeVolumeAnswer(cmd, true, "success", finalSize); - } - - return new ResizeVolumeAnswer(cmd, false, result); + /* fetch new size as seen from libvirt, don't want to assume anything */ + pool = _storagePoolMgr.getStoragePool(spool.getType(), spool.getUuid()); + long finalSize = pool.getPhysicalDisk(volid).getVirtualSize(); + s_logger.debug("after resize, size reports as " + finalSize + ", requested " + newSize); + return new ResizeVolumeAnswer(cmd, true, "success", finalSize); } catch (CloudRuntimeException e) { String error = "failed to resize volume: " + e; s_logger.debug(error); @@ -1641,6 +1722,25 @@ ServerResource { return new Answer(cmd, true, result); } + private void vifHotUnPlug (Connect conn, String vmName, String vlanId, + String macAddr) throws InternalErrorException, LibvirtException { + + Domain vm = null; + conn = LibvirtConnection.getConnectionByVmName(vmName); + vm = getDomain(conn, vmName); + List pluggedNics = getInterfaces(conn, vmName); + for (InterfaceDef pluggedNic : pluggedNics) { + if (pluggedNic.getMacAddress().equalsIgnoreCase(macAddr)) { + vm.detachDevice(pluggedNic.toString()); + // We don't know which "traffic type" is associated with + // each interface at this point, so inform all vif drivers + for (VifDriver vifDriver : getAllVifDrivers()) { + vifDriver.unplug(pluggedNic); + } + } + } + } + private void VifHotPlug(Connect conn, String vmName, String vlanId, String macAddr) throws InternalErrorException, LibvirtException { NicTO nicTO = new NicTO(); @@ -1660,9 +1760,10 @@ ServerResource { private PlugNicAnswer execute(PlugNicCommand cmd) { NicTO nic = cmd.getNic(); String vmName = cmd.getVmName(); + Domain vm = null; try { Connect conn = LibvirtConnection.getConnectionByVmName(vmName); - Domain vm = getDomain(conn, vmName); + vm = getDomain(conn, vmName); List pluggedNics = getInterfaces(conn, vmName); Integer nicnum = 0; for (InterfaceDef pluggedNic : pluggedNics) { @@ -1682,6 +1783,14 @@ ServerResource { String msg = " Plug Nic failed due to " + e.toString(); s_logger.warn(msg, e); return new PlugNicAnswer(cmd, false, msg); + } finally { + if (vm != null) { + try { + vm.free(); + } catch (LibvirtException l) { + s_logger.trace("Ignoring libvirt error.", l); + } + } } } @@ -1689,9 +1798,10 @@ ServerResource { Connect conn; NicTO nic = cmd.getNic(); String vmName = cmd.getVmName(); + Domain vm = null; try { conn = LibvirtConnection.getConnectionByVmName(vmName); - Domain vm = getDomain(conn, vmName); + vm = getDomain(conn, vmName); List pluggedNics = getInterfaces(conn, vmName); for (InterfaceDef pluggedNic : pluggedNics) { if (pluggedNic.getMacAddress().equalsIgnoreCase(nic.getMac())) { @@ -1709,6 +1819,14 @@ ServerResource { String msg = " Unplug Nic failed due to " + e.toString(); s_logger.warn(msg, e); return new UnPlugNicAnswer(cmd, false, msg); + } finally { + if (vm != null) { + try { + vm.free(); + } catch (LibvirtException l) { + s_logger.trace("Ignoring libvirt error.", l); + } + } } } @@ -1944,6 +2062,20 @@ ServerResource { } else { results[i++] = ip.getPublicIp() + " - success"; ; + + if (!ip.isAdd()) { + result = _virtRouterResource.checkPublicIpsCount(routerName, + routerIp, ip.getPublicIp(), ip.isFirstIP(), + ip.isSourceNat(), ip.getVlanId(), ip.getVlanGateway(), + ip.getVlanNetmask(), ip.getVifMacAddress(), nicNum, newNic); + + if (result != null) { + // There are no ips on the vm so delete the vif + networkUsage(routerIp, "deleteVif", "eth" + nicNum); + vifHotUnPlug(conn, routerName, ip.getVlanId(), + ip.getVifMacAddress()); + } + } } } return new IpAssocAnswer(cmd, results); @@ -1975,12 +2107,6 @@ ServerResource { cmd.getPool().getType(), cmd.getPool().getUuid()); - if (primaryPool.getType() == StoragePoolType.RBD) { - s_logger.debug("Snapshots are not supported on RBD volumes"); - return new ManageSnapshotAnswer(cmd, false, - "Snapshots are not supported on RBD volumes"); - } - KVMPhysicalDisk disk = primaryPool.getPhysicalDisk(cmd .getVolumePath()); if (state == DomainInfo.DomainState.VIR_DOMAIN_RUNNING @@ -2007,23 +2133,63 @@ ServerResource { vm.resume(); } } else { + /** + * For RBD we can't use libvirt to do our snapshotting or any Bash scripts. + * libvirt also wants to store the memory contents of the Virtual Machine, + * but that's not possible with RBD since there is no way to store the memory + * contents in RBD. + * + * So we rely on the Java bindings for RBD to create our snapshot + * + * This snapshot might not be 100% consistent due to writes still being in the + * memory of the Virtual Machine, but if the VM runs a kernel which supports + * barriers properly (>2.6.32) this won't be any different then pulling the power + * cord out of a running machine. + */ + if (primaryPool.getType() == StoragePoolType.RBD) { + try { + Rados r = new Rados(primaryPool.getAuthUserName()); + r.confSet("mon_host", primaryPool.getSourceHost() + ":" + primaryPool.getSourcePort()); + r.confSet("key", primaryPool.getAuthSecret()); + r.connect(); + s_logger.debug("Succesfully connected to Ceph cluster at " + r.confGet("mon_host")); - /* VM is not running, create a snapshot by ourself */ - final Script command = new Script(_manageSnapshotPath, - _cmdsTimeout, s_logger); - if (cmd.getCommandSwitch().equalsIgnoreCase( - ManageSnapshotCommand.CREATE_SNAPSHOT)) { - command.add("-c", disk.getPath()); + IoCTX io = r.ioCtxCreate(primaryPool.getSourceDir()); + Rbd rbd = new Rbd(io); + RbdImage image = rbd.open(disk.getName()); + + if (cmd.getCommandSwitch().equalsIgnoreCase( + ManageSnapshotCommand.CREATE_SNAPSHOT)) { + s_logger.debug("Attempting to create RBD snapshot " + disk.getName() + "@" + snapshotName); + image.snapCreate(snapshotName); + } else { + s_logger.debug("Attempting to remove RBD snapshot " + disk.getName() + "@" + snapshotName); + image.snapRemove(snapshotName); + } + + rbd.close(image); + r.ioCtxDestroy(io); + } catch (Exception e) { + s_logger.error("A RBD snapshot operation on " + disk.getName() + " failed. The error was: " + e.getMessage()); + } } else { - command.add("-d", snapshotPath); - } + /* VM is not running, create a snapshot by ourself */ + final Script command = new Script(_manageSnapshotPath, + _cmdsTimeout, s_logger); + if (cmd.getCommandSwitch().equalsIgnoreCase( + ManageSnapshotCommand.CREATE_SNAPSHOT)) { + command.add("-c", disk.getPath()); + } else { + command.add("-d", snapshotPath); + } - command.add("-n", snapshotName); - String result = command.execute(); - if (result != null) { - s_logger.debug("Failed to manage snapshot: " + result); - return new ManageSnapshotAnswer(cmd, false, - "Failed to manage snapshot: " + result); + command.add("-n", snapshotName); + String result = command.execute(); + if (result != null) { + s_logger.debug("Failed to manage snapshot: " + result); + return new ManageSnapshotAnswer(cmd, false, + "Failed to manage snapshot: " + result); + } } } return new ManageSnapshotAnswer(cmd, cmd.getSnapshotId(), @@ -2065,16 +2231,74 @@ ServerResource { cmd.getPrimaryStoragePoolNameLabel()); KVMPhysicalDisk snapshotDisk = primaryPool.getPhysicalDisk(cmd .getVolumePath()); - Script command = new Script(_manageSnapshotPath, _cmdsTimeout, - s_logger); - command.add("-b", snapshotDisk.getPath()); - command.add("-n", snapshotName); - command.add("-p", snapshotDestPath); - command.add("-t", snapshotName); - String result = command.execute(); - if (result != null) { - s_logger.debug("Failed to backup snaptshot: " + result); - return new BackupSnapshotAnswer(cmd, false, result, null, true); + + /** + * RBD snapshots can't be copied using qemu-img, so we have to use + * the Java bindings for librbd here. + * + * These bindings will read the snapshot and write the contents to + * the secondary storage directly + * + * It will stop doing so if the amount of time spend is longer then + * cmds.timeout + */ + if (primaryPool.getType() == StoragePoolType.RBD) { + try { + Rados r = new Rados(primaryPool.getAuthUserName()); + r.confSet("mon_host", primaryPool.getSourceHost() + ":" + primaryPool.getSourcePort()); + r.confSet("key", primaryPool.getAuthSecret()); + r.connect(); + s_logger.debug("Succesfully connected to Ceph cluster at " + r.confGet("mon_host")); + + IoCTX io = r.ioCtxCreate(primaryPool.getSourceDir()); + Rbd rbd = new Rbd(io); + RbdImage image = rbd.open(snapshotDisk.getName(), snapshotName); + + long startTime = System.currentTimeMillis() / 1000; + + File fh = new File(snapshotDestPath); + BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(fh)); + int chunkSize = 4194304; + long offset = 0; + s_logger.debug("Backuping up RBD snapshot " + snapshotName + " to " + snapshotDestPath); + while(true) { + byte[] buf = new byte[chunkSize]; + + int bytes = image.read(offset, buf, chunkSize); + if (bytes <= 0) { + break; + } + bos.write(buf, 0, bytes); + offset += bytes; + } + s_logger.debug("Completed backing up RBD snapshot " + snapshotName + " to " + snapshotDestPath + ". Bytes written: " + offset); + bos.close(); + r.ioCtxDestroy(io); + } catch (RadosException e) { + s_logger.error("A RADOS operation failed. The error was: " + e.getMessage()); + return new BackupSnapshotAnswer(cmd, false, e.toString(), null, true); + } catch (RbdException e) { + s_logger.error("A RBD operation on " + snapshotDisk.getName() + " failed. The error was: " + e.getMessage()); + return new BackupSnapshotAnswer(cmd, false, e.toString(), null, true); + } catch (FileNotFoundException e) { + s_logger.error("Failed to open " + snapshotDestPath + ". The error was: " + e.getMessage()); + return new BackupSnapshotAnswer(cmd, false, e.toString(), null, true); + } catch (IOException e) { + s_logger.debug("An I/O error occured during a snapshot operation on " + snapshotDestPath); + return new BackupSnapshotAnswer(cmd, false, e.toString(), null, true); + } + } else { + Script command = new Script(_manageSnapshotPath, _cmdsTimeout, + s_logger); + command.add("-b", snapshotDisk.getPath()); + command.add("-n", snapshotName); + command.add("-p", snapshotDestPath); + command.add("-t", snapshotName); + String result = command.execute(); + if (result != null) { + s_logger.debug("Failed to backup snaptshot: " + result); + return new BackupSnapshotAnswer(cmd, false, result, null, true); + } } /* Delete the snapshot on primary */ @@ -2111,11 +2335,11 @@ ServerResource { vm.resume(); } } else { - command = new Script(_manageSnapshotPath, _cmdsTimeout, + Script command = new Script(_manageSnapshotPath, _cmdsTimeout, s_logger); command.add("-d", snapshotDisk.getPath()); command.add("-n", snapshotName); - result = command.execute(); + String result = command.execute(); if (result != null) { s_logger.debug("Failed to backup snapshot: " + result); return new BackupSnapshotAnswer(cmd, false, @@ -3150,10 +3374,28 @@ ServerResource { } } + protected String getUuid(String uuid) { + if (uuid == null) { + uuid = UUID.randomUUID().toString(); + } else { + try { + UUID uuid2 = UUID.fromString(uuid); + String uuid3 = uuid2.toString(); + if (!uuid3.equals(uuid)) { + uuid = UUID.randomUUID().toString(); + } + } catch (IllegalArgumentException e) { + uuid = UUID.randomUUID().toString(); + } + } + return uuid; + } protected LibvirtVMDef createVMFromSpec(VirtualMachineTO vmTO) { LibvirtVMDef vm = new LibvirtVMDef(); vm.setDomainName(vmTO.getName()); - vm.setDomUUID(vmTO.getUuid()); + String uuid = vmTO.getUuid(); + uuid = getUuid(uuid); + vm.setDomUUID(uuid); vm.setDomDescription(vmTO.getOs()); GuestDef guest = new GuestDef(); @@ -3189,23 +3431,30 @@ ServerResource { grd.setVcpuNum(vmTO.getCpus()); vm.addComp(grd); - CpuTuneDef ctd = new CpuTuneDef(); - /** - A 4.0.X/4.1.X management server doesn't send the correct JSON - command for getMinSpeed, it only sends a 'speed' field. + CpuModeDef cmd = new CpuModeDef(); + cmd.setMode(_guestCpuMode); + cmd.setModel(_guestCpuModel); + vm.addComp(cmd); - So if getMinSpeed() returns null we fall back to getSpeed(). + if (_hypervisorLibvirtVersion >= 9000) { + CpuTuneDef ctd = new CpuTuneDef(); + /** + A 4.0.X/4.1.X management server doesn't send the correct JSON + command for getMinSpeed, it only sends a 'speed' field. - This way a >4.1 agent can work communicate a <=4.1 management server + So if getMinSpeed() returns null we fall back to getSpeed(). - This change is due to the overcommit feature in 4.2 - */ - if (vmTO.getMinSpeed() != null) { - ctd.setShares(vmTO.getCpus() * vmTO.getMinSpeed()); - } else { - ctd.setShares(vmTO.getCpus() * vmTO.getSpeed()); + This way a >4.1 agent can work communicate a <=4.1 management server + + This change is due to the overcommit feature in 4.2 + */ + if (vmTO.getMinSpeed() != null) { + ctd.setShares(vmTO.getCpus() * vmTO.getMinSpeed()); + } else { + ctd.setShares(vmTO.getCpus() * vmTO.getSpeed()); + } + vm.addComp(ctd); } - vm.addComp(ctd); FeaturesDef features = new FeaturesDef(); features.addFeatures("pae"); @@ -3223,8 +3472,6 @@ ServerResource { if (vmTO.getOs().startsWith("Windows")) { clock.setClockOffset(ClockDef.ClockOffset.LOCALTIME); clock.setTimer("rtc", "catchup", null); - } else if (vmTO.getType() != VirtualMachine.Type.User) { - clock.setTimer("kvmclock", "catchup", null); } vm.addComp(clock); @@ -3321,12 +3568,19 @@ ServerResource { // pass cmdline info to system vms if (vmSpec.getType() != VirtualMachine.Type.User) { if ((_kernelVersion < 2006034) && (conn.getVersion() < 1001000)) { // CLOUDSTACK-2823: try passCmdLine some times if kernel < 2.6.34 and qemu < 1.1.0 on hypervisor (for instance, CentOS 6.4) - for (int count = 0; count < 10; count ++) { + //wait for 5 minutes at most + String controlIp = null; + for (NicTO nic : nics) { + if (nic.getType() == TrafficType.Control) { + controlIp = nic.getIp(); + } + } + for (int count = 0; count < 30; count ++) { passCmdLine(vmName, vmSpec.getBootArgs()); - try { - Thread.sleep(5000); - } catch (InterruptedException e) { - s_logger.trace("Ignoring InterruptedException.", e); + //check router is up? + boolean result = _virtRouterResource.connect(controlIp, 1, 5000); + if (result) { + break; } } } else { @@ -3411,10 +3665,10 @@ ServerResource { physicalDisk = secondaryStorage.getPhysicalDisk(volName); } else if (volume.getType() != Volume.Type.ISO) { PrimaryDataStoreTO store = (PrimaryDataStoreTO)data.getDataStore(); - pool = _storagePoolMgr.getStoragePool( - store.getPoolType(), - store.getUuid()); - physicalDisk = pool.getPhysicalDisk(data.getPath()); + physicalDisk = _storagePoolMgr.getPhysicalDisk( store.getPoolType(), + store.getUuid(), + data.getPath()); + pool = physicalDisk.getPool(); } String volPath = null; @@ -3487,10 +3741,9 @@ ServerResource { if (volume.getType() == Volume.Type.ROOT) { DataTO data = volume.getData(); PrimaryDataStoreTO store = (PrimaryDataStoreTO)data.getDataStore(); - KVMStoragePool pool = _storagePoolMgr.getStoragePool( - store.getPoolType(), - store.getUuid()); - KVMPhysicalDisk physicalDisk = pool.getPhysicalDisk(data.getPath()); + KVMPhysicalDisk physicalDisk = _storagePoolMgr.getPhysicalDisk( store.getPoolType(), + store.getUuid(), + data.getPath()); FilesystemDef rootFs = new FilesystemDef(physicalDisk.getPath(), "/"); vm.getDevices().addDevice(rootFs); break; @@ -3532,6 +3785,10 @@ ServerResource { // need to umount secondary storage String path = disk.getDiskPath(); String poolUuid = null; + if (path.endsWith("systemvm.iso")) { + //Don't need to clean up system vm iso, as it's stored in local + return true; + } if (path != null) { String[] token = path.split("/"); if (token.length > 3) { @@ -4245,6 +4502,10 @@ ServerResource { } } } catch (LibvirtException e) { + if (e.getMessage().contains("Domain not found")) { + s_logger.debug("VM " + vmName + " doesn't exist, no need to stop it"); + return null; + } s_logger.debug("Failed to stop VM :" + vmName + " :", e); return e.getMessage(); } catch (InterruptedException ie) { @@ -4853,6 +5114,13 @@ ServerResource { return new Answer(cmd, success, ""); } + private String prettyVersion(long version) { + long major = version / 1000000; + long minor = version % 1000000 / 1000; + long release = version % 1000000 % 1000; + return major + "." + minor + "." + release; + } + @Override public void setName(String name) { // TODO Auto-generated method stub diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java index 51208702169..6aaabc5be13 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java @@ -384,7 +384,7 @@ public class LibvirtVMDef { } } - enum diskProtocol { + public enum diskProtocol { RBD("rbd"), SHEEPDOG("sheepdog"); String _diskProtocol; @@ -938,6 +938,33 @@ public class LibvirtVMDef { } } + public static class CpuModeDef { + private String _mode; + private String _model; + + public void setMode(String mode) { + _mode = mode; + } + + public void setModel(String model) { + _model = model; + } + + @Override + public String toString() { + StringBuilder modeBuidler = new StringBuilder(); + if ("custom".equalsIgnoreCase(_mode) && _model != null) { + modeBuidler.append("" + + _model + ""); + } else if ("host-model".equals(_mode)) { + modeBuidler.append(""); + } else if ("host-passthrough".equals(_mode)) { + modeBuidler.append(""); + } + return modeBuidler.toString(); + } + } + public static class SerialDef { private final String _type; private final String _source; diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMPhysicalDisk.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMPhysicalDisk.java index 7f088fad28a..6b4d61e71a3 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMPhysicalDisk.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMPhysicalDisk.java @@ -27,7 +27,11 @@ public class KVMPhysicalDisk { String rbdOpts; rbdOpts = "rbd:" + image; - rbdOpts += ":mon_host=" + monHost + "\\\\:" + monPort; + rbdOpts += ":mon_host=" + monHost; + if (monPort != 6789) { + rbdOpts += "\\\\:" + monPort; + } + if (authUserName == null) { rbdOpts += ":auth_supported=none"; } else { diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java index 31d491c9494..e09c9ba44da 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java @@ -31,11 +31,40 @@ import com.cloud.hypervisor.kvm.resource.KVMHAMonitor; import com.cloud.storage.Storage.StoragePoolType; import com.cloud.storage.StorageLayer; import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.log4j.Logger; public class KVMStoragePoolManager { + private static final Logger s_logger = Logger + .getLogger(KVMStoragePoolManager.class); + private class StoragePoolInformation { + String name; + String host; + int port; + String path; + String userInfo; + boolean type; + StoragePoolType poolType; + + + public StoragePoolInformation(String name, + String host, + int port, + String path, + String userInfo, + StoragePoolType poolType, + boolean type) { + this.name = name; + this.host = host; + this.port = port; + this.path = path; + this.userInfo = userInfo; + this.type = type; + this.poolType = poolType; + } + } private StorageAdaptor _storageAdaptor; private KVMHAMonitor _haMonitor; - private final Map _storagePools = new ConcurrentHashMap(); + private final Map _storagePools = new ConcurrentHashMap(); private final Map _storageMapper = new HashMap(); private StorageAdaptor getStorageAdaptor(StoragePoolType type) { @@ -51,10 +80,10 @@ public class KVMStoragePoolManager { return adaptor; } - private void addStoragePool(String uuid) { + private void addStoragePool(String uuid, StoragePoolInformation pool) { synchronized (_storagePools) { if (!_storagePools.containsKey(uuid)) { - _storagePools.put(uuid, new Object()); + _storagePools.put(uuid, pool); } } } @@ -68,8 +97,18 @@ public class KVMStoragePoolManager { } public KVMStoragePool getStoragePool(StoragePoolType type, String uuid) { + StorageAdaptor adaptor = getStorageAdaptor(type); - return adaptor.getStoragePool(uuid); + KVMStoragePool pool = null; + try { + pool = adaptor.getStoragePool(uuid); + } catch(Exception e) { + StoragePoolInformation info = _storagePools.get(uuid); + if (info != null) { + pool = createStoragePool(info.name, info.host, info.port, info.path, info.userInfo, info.poolType, info.type); + } + } + return pool; } public KVMStoragePool getStoragePoolByURI(String uri) { @@ -98,6 +137,39 @@ public class KVMStoragePoolManager { return createStoragePool(uuid, sourceHost, 0, sourcePath, "", protocol, false); } + public KVMPhysicalDisk getPhysicalDisk(StoragePoolType type, String poolUuid, String volName) { + int cnt = 0; + int retries = 10; + KVMPhysicalDisk vol = null; + //harden get volume, try cnt times to get volume, in case volume is created on other host + String errMsg = ""; + while (cnt < retries) { + try { + KVMStoragePool pool = getStoragePool(type, poolUuid); + vol = pool.getPhysicalDisk(volName); + if (vol != null) { + break; + } + } catch (Exception e) { + s_logger.debug("Failed to find volume:" + volName + " due to" + e.toString() + ", retry:" + cnt); + errMsg = e.toString(); + } + + try { + Thread.sleep(30000); + } catch (InterruptedException e) { + } + cnt++; + } + + if (vol == null) { + throw new CloudRuntimeException(errMsg); + } else { + return vol; + } + + } + public KVMStoragePool createStoragePool( String name, String host, int port, String path, String userInfo, StoragePoolType type) { @@ -105,7 +177,8 @@ public class KVMStoragePoolManager { return createStoragePool(name, host, port, path, userInfo, type, true); } - private KVMStoragePool createStoragePool( String name, String host, int port, + //Note: due to bug CLOUDSTACK-4459, createStoragepool can be called in parallel, so need to be synced. + private synchronized KVMStoragePool createStoragePool( String name, String host, int port, String path, String userInfo, StoragePoolType type, boolean primaryStorage) { StorageAdaptor adaptor = getStorageAdaptor(type); @@ -119,7 +192,8 @@ public class KVMStoragePoolManager { PoolType.PrimaryStorage); _haMonitor.addStoragePool(nfspool); } - addStoragePool(pool.getUuid()); + StoragePoolInformation info = new StoragePoolInformation(name, host, port, path, userInfo, type, primaryStorage); + addStoragePool(pool.getUuid(), info); return pool; } @@ -131,11 +205,6 @@ public class KVMStoragePoolManager { return true; } - public boolean deleteVbdByPath(StoragePoolType type, String diskPath) { - StorageAdaptor adaptor = getStorageAdaptor(type); - return adaptor.deleteVbdByPath(diskPath); - } - public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template, String name, KVMStoragePool destPool) { StorageAdaptor adaptor = getStorageAdaptor(destPool.getType()); diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index 2f87ad49212..99ea04fc1d7 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -20,6 +20,9 @@ package com.cloud.hypervisor.kvm.storage; import java.io.File; import java.io.FileOutputStream; +import java.io.FileNotFoundException; +import java.io.BufferedOutputStream; +import java.io.IOException; import java.net.URISyntaxException; import java.text.DateFormat; import java.text.MessageFormat; @@ -32,6 +35,11 @@ import java.util.UUID; import javax.naming.ConfigurationException; +import com.cloud.agent.api.storage.CopyVolumeAnswer; +import com.cloud.agent.api.to.DataObjectType; +import com.cloud.agent.api.to.S3TO; +import com.cloud.agent.api.to.StorageFilerTO; +import com.cloud.utils.S3Utils; import org.apache.cloudstack.storage.command.AttachAnswer; import org.apache.cloudstack.storage.command.AttachCommand; import org.apache.cloudstack.storage.command.CopyCmdAnswer; @@ -50,6 +58,7 @@ import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; import org.apache.cloudstack.utils.qemu.QemuImgException; import org.apache.cloudstack.utils.qemu.QemuImgFile; import org.apache.log4j.Logger; +import org.apache.commons.io.FileUtils; import org.libvirt.Connect; import org.libvirt.Domain; import org.libvirt.DomainInfo; @@ -67,6 +76,7 @@ import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; import com.cloud.hypervisor.kvm.resource.LibvirtConnection; import com.cloud.hypervisor.kvm.resource.LibvirtDomainXMLParser; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.DiskDef; +import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.DiskDef.diskProtocol; import com.cloud.storage.JavaStorageLayer; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.Storage.StoragePoolType; @@ -80,6 +90,15 @@ import com.cloud.utils.NumbersUtil; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.Script; +import com.ceph.rados.Rados; +import com.ceph.rados.RadosException; +import com.ceph.rados.IoCTX; +import com.ceph.rbd.Rbd; +import com.ceph.rbd.RbdImage; +import com.ceph.rbd.RbdException; + +import static com.cloud.utils.S3Utils.putFile; + public class KVMStorageProcessor implements StorageProcessor { private static final Logger s_logger = Logger.getLogger(KVMStorageProcessor.class); private KVMStoragePoolManager storagePoolMgr; @@ -180,7 +199,16 @@ public class KVMStorageProcessor implements StorageProcessor { TemplateObjectTO newTemplate = new TemplateObjectTO(); newTemplate.setPath(primaryVol.getName()); - newTemplate.setFormat(ImageFormat.QCOW2); + + /** + * Force the ImageFormat for RBD templates to RAW + * + */ + if (primaryPool.getType() == StoragePoolType.RBD) { + newTemplate.setFormat(ImageFormat.RAW); + } else { + newTemplate.setFormat(ImageFormat.QCOW2); + } return new CopyCmdAnswer(newTemplate); } catch (CloudRuntimeException e) { return new CopyCmdAnswer(e.toString()); @@ -261,8 +289,12 @@ public class KVMStorageProcessor implements StorageProcessor { if (primaryPool.getType() == StoragePoolType.CLVM) { vol = templateToPrimaryDownload(templatePath, primaryPool); } else { - BaseVol = primaryPool.getPhysicalDisk(templatePath); - vol = storagePoolMgr.createDiskFromTemplate(BaseVol, UUID.randomUUID().toString(), primaryPool); + if (templatePath.contains("/mnt")) { + //upgrade issue, if the path contains path, need to extract the volume uuid from path + templatePath = templatePath.substring(templatePath.lastIndexOf(File.separator) + 1); + } + BaseVol = storagePoolMgr.getPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), templatePath); + vol = storagePoolMgr.createDiskFromTemplate(BaseVol, UUID.randomUUID().toString(), BaseVol.getPool()); } if (vol == null) { return new CopyCmdAnswer(" Can't create storage volume on storage pool"); @@ -270,7 +302,13 @@ public class KVMStorageProcessor implements StorageProcessor { VolumeObjectTO newVol = new VolumeObjectTO(); newVol.setPath(vol.getName()); - newVol.setSize(vol.getSize()); + newVol.setSize(volume.getSize()); + + if (vol.getFormat() == PhysicalDiskFormat.RAW) { + newVol.setFormat(ImageFormat.RAW); + } else if (vol.getFormat() == PhysicalDiskFormat.QCOW2) { + newVol.setFormat(ImageFormat.QCOW2); + } return new CopyCmdAnswer(newVol); } catch (CloudRuntimeException e) { @@ -281,13 +319,111 @@ public class KVMStorageProcessor implements StorageProcessor { @Override public Answer copyVolumeFromImageCacheToPrimary(CopyCommand cmd) { - // TODO Auto-generated method stub - return null; + DataTO srcData = cmd.getSrcTO(); + DataTO destData = cmd.getDestTO(); + DataStoreTO srcStore = srcData.getDataStore(); + DataStoreTO destStore = destData.getDataStore(); + VolumeObjectTO srcVol = (VolumeObjectTO) srcData; + ImageFormat srcFormat = srcVol.getFormat(); + PrimaryDataStoreTO primaryStore = (PrimaryDataStoreTO) destStore; + if (!(srcStore instanceof NfsTO)) { + return new CopyCmdAnswer("can only handle nfs storage"); + } + NfsTO nfsStore = (NfsTO)srcStore; + String srcVolumePath = srcData.getPath(); + String secondaryStorageUrl = nfsStore.getUrl(); + KVMStoragePool secondaryStoragePool = null; + KVMStoragePool primaryPool = null; + try { + try { + primaryPool = storagePoolMgr.getStoragePool( + primaryStore.getPoolType(), + primaryStore.getUuid()); + } catch (CloudRuntimeException e) { + if (e.getMessage().contains("not found")) { + primaryPool = storagePoolMgr.createStoragePool(primaryStore.getUuid(), + primaryStore.getHost(), primaryStore.getPort(), + primaryStore.getPath(), null, + primaryStore.getPoolType()); + } else { + return new CopyCmdAnswer(e.getMessage()); + } + } + + String volumeName = UUID.randomUUID().toString(); + + int index = srcVolumePath.lastIndexOf(File.separator); + String volumeDir = srcVolumePath.substring(0, index); + String srcVolumeName = srcVolumePath.substring(index + 1); + secondaryStoragePool = storagePoolMgr.getStoragePoolByURI( + secondaryStorageUrl + File.separator + volumeDir + ); + if (!srcVolumeName.endsWith(".qcow2") && srcFormat == ImageFormat.QCOW2) { + srcVolumeName = srcVolumeName + ".qcow2"; + } + KVMPhysicalDisk volume = secondaryStoragePool + .getPhysicalDisk(srcVolumeName); + volume.setFormat(PhysicalDiskFormat.valueOf(srcFormat.toString())); + KVMPhysicalDisk newDisk = storagePoolMgr.copyPhysicalDisk(volume, volumeName, + primaryPool); + VolumeObjectTO newVol = new VolumeObjectTO(); + newVol.setFormat(ImageFormat.valueOf(newDisk.getFormat().toString().toUpperCase())); + newVol.setPath(volumeName); + return new CopyCmdAnswer(newVol); + } catch (CloudRuntimeException e) { + return new CopyCmdAnswer(e.toString()); + } finally { + if (secondaryStoragePool != null) { + storagePoolMgr.deleteStoragePool(secondaryStoragePool.getType(),secondaryStoragePool.getUuid()); + } + } } @Override public Answer copyVolumeFromPrimaryToSecondary(CopyCommand cmd) { - return null; + DataTO srcData = cmd.getSrcTO(); + DataTO destData = cmd.getDestTO(); + VolumeObjectTO srcVol = (VolumeObjectTO) srcData; + VolumeObjectTO destVol = (VolumeObjectTO) destData; + ImageFormat srcFormat = srcVol.getFormat(); + ImageFormat destFormat = destVol.getFormat(); + DataStoreTO srcStore = srcData.getDataStore(); + DataStoreTO destStore = destData.getDataStore(); + PrimaryDataStoreTO primaryStore = (PrimaryDataStoreTO) srcStore; + if (!(destStore instanceof NfsTO)) { + return new CopyCmdAnswer("can only handle nfs storage"); + } + NfsTO nfsStore = (NfsTO)destStore; + String srcVolumePath = srcData.getPath(); + String destVolumePath = destData.getPath(); + String secondaryStorageUrl = nfsStore.getUrl(); + KVMStoragePool secondaryStoragePool = null; + + try { + String volumeName = UUID.randomUUID().toString(); + + KVMPhysicalDisk volume = storagePoolMgr.getPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), srcVolumePath); + String destVolumeName = volumeName + "." + destFormat.getFileExtension(); + volume.setFormat(PhysicalDiskFormat.valueOf(srcFormat.toString())); + secondaryStoragePool = storagePoolMgr.getStoragePoolByURI( + secondaryStorageUrl); + secondaryStoragePool.createFolder(destVolumePath); + storagePoolMgr.deleteStoragePool(secondaryStoragePool.getType(),secondaryStoragePool.getUuid()); + secondaryStoragePool = storagePoolMgr.getStoragePoolByURI( + secondaryStorageUrl + File.separator + destVolumePath); + storagePoolMgr.copyPhysicalDisk(volume, + destVolumeName,secondaryStoragePool); + VolumeObjectTO newVol = new VolumeObjectTO(); + newVol.setPath(destVolumePath + File.separator + destVolumeName); + newVol.setFormat(destFormat); + return new CopyCmdAnswer(newVol); + } catch (CloudRuntimeException e) { + return new CopyCmdAnswer(e.toString()); + } finally { + if (secondaryStoragePool != null) { + storagePoolMgr.deleteStoragePool(secondaryStoragePool.getType(),secondaryStoragePool.getUuid()); + } + } } @Override @@ -312,18 +448,9 @@ public class KVMStorageProcessor implements StorageProcessor { secondaryStorage = storagePoolMgr.getStoragePoolByURI(nfsImageStore.getUrl()); - try { - primary = storagePoolMgr.getStoragePool(primaryStore.getPoolType(), primaryStore.getUuid()); - } catch (CloudRuntimeException e) { - if (e.getMessage().contains("not found")) { - primary = storagePoolMgr.createStoragePool(primaryStore.getUuid(), primaryStore.getHost(), - primaryStore.getPort(), primaryStore.getPath(), null, primaryStore.getPoolType()); - } else { - return new CopyCmdAnswer(e.getMessage()); - } - } + primary = storagePoolMgr.getStoragePool(primaryStore.getPoolType(), primaryStore.getUuid()); - KVMPhysicalDisk disk = primary.getPhysicalDisk(volume.getPath()); + KVMPhysicalDisk disk = storagePoolMgr.getPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), volume.getPath()); String tmpltPath = secondaryStorage.getLocalPath() + File.separator + templateFolder; this.storageLayer.mkdirs(tmpltPath); String templateName = UUID.randomUUID().toString(); @@ -391,7 +518,9 @@ public class KVMStorageProcessor implements StorageProcessor { TemplateObjectTO newTemplate = new TemplateObjectTO(); newTemplate.setPath(templateFolder + File.separator + templateName + ".qcow2"); newTemplate.setSize(info.virtualSize); + newTemplate.setPhysicalSize(info.size); newTemplate.setFormat(ImageFormat.QCOW2); + newTemplate.setName(templateName); return new CopyCmdAnswer(newTemplate); } catch (Exception e) { s_logger.debug("Failed to create template from volume: " + e.toString()); @@ -407,7 +536,78 @@ public class KVMStorageProcessor implements StorageProcessor { public Answer createTemplateFromSnapshot(CopyCommand cmd) { return null; //To change body of implemented methods use File | Settings | File Templates. } + protected String copyToS3(File srcFile, S3TO destStore, String destPath) { + final String bucket = destStore.getBucketName(); + String key = destPath + S3Utils.SEPARATOR + srcFile.getName(); + putFile(destStore, srcFile, bucket, key); + return key; + } + protected Answer copyToObjectStore(CopyCommand cmd) { + DataTO srcData = cmd.getSrcTO(); + DataTO destData = cmd.getDestTO(); + SnapshotObjectTO snapshot = (SnapshotObjectTO) srcData; + DataStoreTO imageStore = destData.getDataStore(); + NfsTO srcStore = (NfsTO)srcData.getDataStore(); + String srcPath = srcData.getPath(); + int index = srcPath.lastIndexOf(File.separator); + String srcSnapshotDir = srcPath.substring(0, index); + String srcFileName = srcPath.substring(index + 1); + KVMStoragePool srcStorePool = null; + File srcFile = null; + try { + srcStorePool = storagePoolMgr.getStoragePoolByURI(srcStore.getUrl() + File.separator + srcSnapshotDir); + if (srcStorePool == null) { + return new CopyCmdAnswer("Can't get store:" + srcStore.getUrl()); + } + srcFile = new File(srcStorePool.getLocalPath() + File.separator + srcFileName); + if (!srcFile.exists()) { + return new CopyCmdAnswer("Can't find src file: " + srcPath); + } + String destPath = null; + if (imageStore instanceof S3TO) { + destPath = copyToS3(srcFile, (S3TO)imageStore, destData.getPath()); + } else { + return new CopyCmdAnswer("Unsupported protocol"); + } + SnapshotObjectTO newSnapshot = new SnapshotObjectTO(); + newSnapshot.setPath(destPath); + return new CopyCmdAnswer(newSnapshot); + } finally { + try { + if (srcFile != null) { + srcFile.delete(); + } + if (srcStorePool != null) { + srcStorePool.delete(); + } + } catch(Exception e) { + s_logger.debug("Failed to clean up:", e); + } + } + } + + protected Answer backupSnapshotForObjectStore(CopyCommand cmd) { + DataTO srcData = cmd.getSrcTO(); + DataTO destData = cmd.getDestTO(); + SnapshotObjectTO snapshot = (SnapshotObjectTO) srcData; + DataStoreTO imageStore = destData.getDataStore(); + DataTO cacheData = cmd.getCacheTO(); + if (cacheData == null) { + return new CopyCmdAnswer("Failed to copy to object store without cache store"); + } + DataStoreTO cacheStore = cacheData.getDataStore(); + ((SnapshotObjectTO) destData).setDataStore(cacheStore); + CopyCmdAnswer answer = (CopyCmdAnswer)backupSnapshot(cmd); + if (!answer.getResult()) { + return answer; + } + SnapshotObjectTO snapshotOnCacheStore = (SnapshotObjectTO)answer.getNewData(); + snapshotOnCacheStore.setDataStore(cacheStore); + ((SnapshotObjectTO) destData).setDataStore(imageStore); + CopyCommand newCpyCmd = new CopyCommand(snapshotOnCacheStore, destData, cmd.getWait(), cmd.executeInSequence()); + return copyToObjectStore(newCpyCmd); + } @Override public Answer backupSnapshot(CopyCommand cmd) { DataTO srcData = cmd.getSrcTO(); @@ -418,7 +618,7 @@ public class KVMStorageProcessor implements StorageProcessor { DataStoreTO imageStore = destData.getDataStore(); if (!(imageStore instanceof NfsTO)) { - return new CopyCmdAnswer("unsupported protocol"); + return backupSnapshotForObjectStore(cmd); } NfsTO nfsImageStore = (NfsTO) imageStore; @@ -441,19 +641,86 @@ public class KVMStorageProcessor implements StorageProcessor { snapshotRelPath = destSnapshot.getPath(); snapshotDestPath = ssPmountPath + File.separator + snapshotRelPath; - KVMStoragePool primaryPool = storagePoolMgr.getStoragePool(primaryStore.getPoolType(), - primaryStore.getUuid()); - KVMPhysicalDisk snapshotDisk = primaryPool.getPhysicalDisk(volumePath); - Script command = new Script(_manageSnapshotPath, cmd.getWait() * 1000, s_logger); - command.add("-b", snapshotDisk.getPath()); - command.add("-n", snapshotName); - command.add("-p", snapshotDestPath); - command.add("-t", snapshotName); - String result = command.execute(); - if (result != null) { - s_logger.debug("Failed to backup snaptshot: " + result); - return new CopyCmdAnswer(result); + KVMPhysicalDisk snapshotDisk = storagePoolMgr.getPhysicalDisk(primaryStore.getPoolType(), + primaryStore.getUuid(), volumePath); + KVMStoragePool primaryPool = snapshotDisk.getPool(); + + /** + * RBD snapshots can't be copied using qemu-img, so we have to use + * the Java bindings for librbd here. + * + * These bindings will read the snapshot and write the contents to + * the secondary storage directly + * + * It will stop doing so if the amount of time spend is longer then + * cmds.timeout + */ + if (primaryPool.getType() == StoragePoolType.RBD) { + try { + Rados r = new Rados(primaryPool.getAuthUserName()); + r.confSet("mon_host", primaryPool.getSourceHost() + ":" + primaryPool.getSourcePort()); + r.confSet("key", primaryPool.getAuthSecret()); + r.connect(); + s_logger.debug("Succesfully connected to Ceph cluster at " + r.confGet("mon_host")); + + IoCTX io = r.ioCtxCreate(primaryPool.getSourceDir()); + Rbd rbd = new Rbd(io); + RbdImage image = rbd.open(snapshotDisk.getName(), snapshotName); + + long startTime = System.currentTimeMillis() / 1000; + + File snapDir = new File(snapshotDestPath); + s_logger.debug("Attempting to create " + snapDir.getAbsolutePath() + " recursively"); + FileUtils.forceMkdir(snapDir); + + File snapFile = new File(snapshotDestPath + "/" + snapshotName); + s_logger.debug("Backing up RBD snapshot to " + snapFile.getAbsolutePath()); + BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(snapFile)); + int chunkSize = 4194304; + long offset = 0; + while(true) { + byte[] buf = new byte[chunkSize]; + + int bytes = image.read(offset, buf, chunkSize); + if (bytes <= 0) { + break; + } + bos.write(buf, 0, bytes); + offset += bytes; + } + s_logger.debug("Completed backing up RBD snapshot " + snapshotName + " to " + snapFile.getAbsolutePath() + ". Bytes written: " + offset); + bos.close(); + + s_logger.debug("Attempting to remove snapshot RBD " + snapshotName + " from image " + snapshotDisk.getName()); + image.snapRemove(snapshotName); + + r.ioCtxDestroy(io); + } catch (RadosException e) { + s_logger.error("A RADOS operation failed. The error was: " + e.getMessage()); + return new CopyCmdAnswer(e.toString()); + } catch (RbdException e) { + s_logger.error("A RBD operation on " + snapshotDisk.getName() + " failed. The error was: " + e.getMessage()); + return new CopyCmdAnswer(e.toString()); + } catch (FileNotFoundException e) { + s_logger.error("Failed to open " + snapshotDestPath + ". The error was: " + e.getMessage()); + return new CopyCmdAnswer(e.toString()); + } catch (IOException e) { + s_logger.debug("An I/O error occured during a snapshot operation on " + snapshotDestPath); + return new CopyCmdAnswer(e.toString()); + } + } else { + Script command = new Script(_manageSnapshotPath, cmd.getWait() * 1000, s_logger); + command.add("-b", snapshotDisk.getPath()); + command.add("-n", snapshotName); + command.add("-p", snapshotDestPath); + command.add("-t", snapshotName); + String result = command.execute(); + if (result != null) { + s_logger.debug("Failed to backup snaptshot: " + result); + return new CopyCmdAnswer(result); + } } + /* Delete the snapshot on primary */ DomainInfo.DomainState state = null; @@ -483,13 +750,15 @@ public class KVMStorageProcessor implements StorageProcessor { vm.resume(); } } else { - command = new Script(_manageSnapshotPath, _cmdsTimeout, s_logger); - command.add("-d", snapshotDisk.getPath()); - command.add("-n", snapshotName); - result = command.execute(); - if (result != null) { - s_logger.debug("Failed to backup snapshot: " + result); - return new CopyCmdAnswer("Failed to backup snapshot: " + result); + if (primaryPool.getType() != StoragePoolType.RBD) { + Script command = new Script(_manageSnapshotPath, _cmdsTimeout, s_logger); + command.add("-d", snapshotDisk.getPath()); + command.add("-n", snapshotName); + String result = command.execute(); + if (result != null) { + s_logger.debug("Failed to backup snapshot: " + result); + return new CopyCmdAnswer("Failed to backup snapshot: " + result); + } } } @@ -509,6 +778,7 @@ public class KVMStorageProcessor implements StorageProcessor { } } + protected synchronized String attachOrDetachISO(Connect conn, String vmName, String isoPath, boolean isAttach) throws LibvirtException, URISyntaxException, InternalErrorException { String isoXml = null; @@ -626,6 +896,7 @@ public class KVMStorageProcessor implements StorageProcessor { List disks = null; Domain dm = null; DiskDef diskdef = null; + KVMStoragePool attachingPool = attachingDisk.getPool(); try { if (!attach) { dm = conn.domainLookupByName(vmName); @@ -646,11 +917,17 @@ public class KVMStorageProcessor implements StorageProcessor { } } else { diskdef = new DiskDef(); - if (attachingDisk.getFormat() == PhysicalDiskFormat.QCOW2) { - diskdef.defFileBasedDisk(attachingDisk.getPath(), devId, DiskDef.diskBus.VIRTIO, - DiskDef.diskFmtType.QCOW2); + if (attachingPool.getType() == StoragePoolType.RBD) { + diskdef.defNetworkBasedDisk(attachingDisk.getPath(), + attachingPool.getSourceHost(), attachingPool.getSourcePort(), + attachingPool.getAuthUserName(), attachingPool.getUuid(), devId, + DiskDef.diskBus.VIRTIO, diskProtocol.RBD); + } else if (attachingDisk.getFormat() == PhysicalDiskFormat.QCOW2) { + diskdef.defFileBasedDisk(attachingDisk.getPath(), devId, + DiskDef.diskBus.VIRTIO, DiskDef.diskFmtType.QCOW2); } else if (attachingDisk.getFormat() == PhysicalDiskFormat.RAW) { - diskdef.defBlockBasedDisk(attachingDisk.getPath(), devId, DiskDef.diskBus.VIRTIO); + diskdef.defBlockBasedDisk(attachingDisk.getPath(), devId, + DiskDef.diskBus.VIRTIO); } } @@ -671,9 +948,8 @@ public class KVMStorageProcessor implements StorageProcessor { String vmName = cmd.getVmName(); try { Connect conn = LibvirtConnection.getConnectionByVmName(vmName); - KVMStoragePool primary = storagePoolMgr.getStoragePool(primaryStore.getPoolType(), primaryStore.getUuid()); + KVMPhysicalDisk phyDisk = storagePoolMgr.getPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), vol.getPath()); - KVMPhysicalDisk phyDisk = primary.getPhysicalDisk(vol.getPath()); attachOrDetachDisk(conn, true, vmName, phyDisk, disk.getDiskSeq().intValue()); return new AttachAnswer(disk); @@ -694,9 +970,8 @@ public class KVMStorageProcessor implements StorageProcessor { String vmName = cmd.getVmName(); try { Connect conn = LibvirtConnection.getConnectionByVmName(vmName); - KVMStoragePool primary = storagePoolMgr.getStoragePool(primaryStore.getPoolType(), primaryStore.getUuid()); + KVMPhysicalDisk phyDisk = storagePoolMgr.getPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), vol.getPath()); - KVMPhysicalDisk phyDisk = primary.getPhysicalDisk(vol.getPath()); attachOrDetachDisk(conn, false, vmName, phyDisk, disk.getDiskSeq().intValue()); return new DettachAnswer(disk); @@ -725,6 +1000,15 @@ public class KVMStorageProcessor implements StorageProcessor { VolumeObjectTO newVol = new VolumeObjectTO(); newVol.setPath(vol.getName()); + newVol.setSize(volume.getSize()); + + /** + * Volumes on RBD are always in RAW format + * Hardcode this to RAW since there is no other way right now + */ + if (primaryPool.getType() == StoragePoolType.RBD) { + newVol.setFormat(ImageFormat.RAW); + } return new CreateObjectAnswer(newVol); } catch (Exception e) { @@ -759,12 +1043,8 @@ public class KVMStorageProcessor implements StorageProcessor { KVMStoragePool primaryPool = storagePoolMgr.getStoragePool(primaryStore.getPoolType(), primaryStore.getUuid()); - if (primaryPool.getType() == StoragePoolType.RBD) { - s_logger.debug("Snapshots are not supported on RBD volumes"); - return new CreateObjectAnswer("Snapshots are not supported on RBD volumes"); - } - - KVMPhysicalDisk disk = primaryPool.getPhysicalDisk(volume.getPath()); + KVMPhysicalDisk disk = storagePoolMgr.getPhysicalDisk(primaryStore.getPoolType(), + primaryStore.getUuid(), volume.getPath()); if (state == DomainInfo.DomainState.VIR_DOMAIN_RUNNING && !primaryPool.isExternalSnapshot()) { String vmUuid = vm.getUUIDString(); Object[] args = new Object[] { snapshotName, vmUuid }; @@ -782,15 +1062,49 @@ public class KVMStorageProcessor implements StorageProcessor { vm.resume(); } } else { + /** + * For RBD we can't use libvirt to do our snapshotting or any Bash scripts. + * libvirt also wants to store the memory contents of the Virtual Machine, + * but that's not possible with RBD since there is no way to store the memory + * contents in RBD. + * + * So we rely on the Java bindings for RBD to create our snapshot + * + * This snapshot might not be 100% consistent due to writes still being in the + * memory of the Virtual Machine, but if the VM runs a kernel which supports + * barriers properly (>2.6.32) this won't be any different then pulling the power + * cord out of a running machine. + */ + if (primaryPool.getType() == StoragePoolType.RBD) { + try { + Rados r = new Rados(primaryPool.getAuthUserName()); + r.confSet("mon_host", primaryPool.getSourceHost() + ":" + primaryPool.getSourcePort()); + r.confSet("key", primaryPool.getAuthSecret()); + r.connect(); + s_logger.debug("Succesfully connected to Ceph cluster at " + r.confGet("mon_host")); - /* VM is not running, create a snapshot by ourself */ - final Script command = new Script(_manageSnapshotPath, this._cmdsTimeout, s_logger); - command.add("-c", disk.getPath()); - command.add("-n", snapshotName); - String result = command.execute(); - if (result != null) { - s_logger.debug("Failed to manage snapshot: " + result); - return new CreateObjectAnswer("Failed to manage snapshot: " + result); + IoCTX io = r.ioCtxCreate(primaryPool.getSourceDir()); + Rbd rbd = new Rbd(io); + RbdImage image = rbd.open(disk.getName()); + + s_logger.debug("Attempting to create RBD snapshot " + disk.getName() + "@" + snapshotName); + image.snapCreate(snapshotName); + + rbd.close(image); + r.ioCtxDestroy(io); + } catch (Exception e) { + s_logger.error("A RBD snapshot operation on " + disk.getName() + " failed. The error was: " + e.getMessage()); + } + } else { + /* VM is not running, create a snapshot by ourself */ + final Script command = new Script(_manageSnapshotPath, this._cmdsTimeout, s_logger); + command.add("-c", disk.getPath()); + command.add("-n", snapshotName); + String result = command.execute(); + if (result != null) { + s_logger.debug("Failed to manage snapshot: " + result); + return new CreateObjectAnswer("Failed to manage snapshot: " + result); + } } } @@ -832,6 +1146,7 @@ public class KVMStorageProcessor implements StorageProcessor { DataTO destData = cmd.getDestTO(); PrimaryDataStoreTO pool = (PrimaryDataStoreTO) destData.getDataStore(); DataStoreTO imageStore = srcData.getDataStore(); + VolumeObjectTO volume = snapshot.getVolume(); if (!(imageStore instanceof NfsTO)) { return new CopyCmdAnswer("unsupported protocol"); @@ -847,6 +1162,12 @@ public class KVMStorageProcessor implements StorageProcessor { + snapshotPath); KVMPhysicalDisk snapshotDisk = secondaryPool.getPhysicalDisk(snapshotName); + if (volume.getFormat() == ImageFormat.RAW) { + snapshotDisk.setFormat(PhysicalDiskFormat.RAW); + } else if (volume.getFormat() == ImageFormat.QCOW2) { + snapshotDisk.setFormat(PhysicalDiskFormat.QCOW2); + } + String primaryUuid = pool.getUuid(); KVMStoragePool primaryPool = storagePoolMgr.getStoragePool(pool.getPoolType(), primaryUuid); String volUuid = UUID.randomUUID().toString(); @@ -854,6 +1175,14 @@ public class KVMStorageProcessor implements StorageProcessor { VolumeObjectTO newVol = new VolumeObjectTO(); newVol.setPath(disk.getName()); newVol.setSize(disk.getVirtualSize()); + + /** + * We have to force the format of RBD volumes to RAW + */ + if (primaryPool.getType() == StoragePoolType.RBD) { + newVol.setFormat(ImageFormat.RAW); + } + return new CopyCmdAnswer(newVol); } catch (CloudRuntimeException e) { return new CopyCmdAnswer(e.toString()); diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java index d743cd0754d..123a9f10ffb 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java @@ -46,6 +46,7 @@ import com.ceph.rados.IoCTX; import com.ceph.rbd.Rbd; import com.ceph.rbd.RbdImage; import com.ceph.rbd.RbdException; +import com.ceph.rbd.jna.RbdSnapInfo; import com.cloud.agent.api.ManageSnapshotCommand; import com.cloud.hypervisor.kvm.resource.LibvirtConnection; @@ -71,8 +72,12 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { private StorageLayer _storageLayer; private String _mountPoint = "/mnt"; private String _manageSnapshotPath; + private String _lockfile = "KVMFILELOCK" + File.separator + ".lock"; + private static final int ACQUIRE_GLOBAL_FILELOCK_TIMEOUT_FOR_KVM = 300; // 300 seconds private String rbdTemplateSnapName = "cloudstack-base-snap"; + private int rbdFeatures = (1<<0); /* Feature 1<<0 means layering in RBD format 2 */ + private int rbdOrder = 0; /* Order 0 means 4MB blocks (the default) */ public LibvirtStorageAdaptor(StorageLayer storage) { _storageLayer = storage; @@ -83,7 +88,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { @Override public boolean createFolder(String uuid, String path) { String mountPoint = _mountPoint + File.separator + uuid; - File f = new File(mountPoint + path); + File f = new File(mountPoint + File.separator + path); if (!f.exists()) { f.mkdirs(); } @@ -96,10 +101,15 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { try { vol = pool.storageVolLookupByName(volName); } catch (LibvirtException e) { - + s_logger.debug("Can't find volume: " + e.toString()); } if (vol == null) { - storagePoolRefresh(pool); + try { + refreshPool(pool); + } catch (LibvirtException e) { + s_logger.debug("failed to refresh pool: " + e.toString()); + } + try { vol = pool.storageVolLookupByName(volName); } catch (LibvirtException e) { @@ -114,16 +124,17 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { LibvirtStorageVolumeDef volDef = new LibvirtStorageVolumeDef(UUID .randomUUID().toString(), size, format, null, null); s_logger.debug(volDef.toString()); + return pool.storageVolCreateXML(volDef.toString(), 0); } public void storagePoolRefresh(StoragePool pool) { try { synchronized (getStoragePool(pool.getUUIDString())) { - pool.refresh(0); + refreshPool(pool); } } catch (LibvirtException e) { - + s_logger.debug("refresh storage pool failed: " + e.toString()); } } @@ -356,8 +367,9 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { } LibvirtStoragePoolDef spd = getStoragePoolDef(conn, storage); StoragePoolType type = null; - if (spd.getPoolType() == LibvirtStoragePoolDef.poolType.NETFS - || spd.getPoolType() == LibvirtStoragePoolDef.poolType.DIR) { + if (spd.getPoolType() == LibvirtStoragePoolDef.poolType.NETFS) { + type = StoragePoolType.NetworkFilesystem; + } else if (spd.getPoolType() == LibvirtStoragePoolDef.poolType.DIR) { type = StoragePoolType.Filesystem; } else if (spd.getPoolType() == LibvirtStoragePoolDef.poolType.RBD) { type = StoragePoolType.RBD; @@ -384,14 +396,15 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { } } - //pool.refresh(); + pool.refresh(); pool.setCapacity(storage.getInfo().capacity); pool.setUsed(storage.getInfo().allocation); pool.setAvailable(storage.getInfo().available); return pool; } catch (LibvirtException e) { - throw new CloudRuntimeException(e.toString()); + s_logger.debug("can't get storage pool",e); + throw new CloudRuntimeException(e.toString(), e); } } @@ -408,7 +421,14 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { disk = new KVMPhysicalDisk(vol.getPath(), vol.getName(), pool); disk.setSize(vol.getInfo().allocation); disk.setVirtualSize(vol.getInfo().capacity); - if (voldef.getFormat() == null) { + + /** + * libvirt returns format = 'unknow', so we have to force + * the format to RAW for RBD storage volumes + */ + if (pool.getType() == StoragePoolType.RBD) { + disk.setFormat(PhysicalDiskFormat.RAW); + } else if (voldef.getFormat() == null) { File diskDir = new File(disk.getPath()); if (diskDir.exists() && diskDir.isDirectory()) { disk.setFormat(PhysicalDiskFormat.DIR); @@ -417,8 +437,6 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { } else { disk.setFormat(pool.getDefaultFormat()); } - } else if (pool.getType() == StoragePoolType.RBD) { - disk.setFormat(PhysicalDiskFormat.RAW); } else if (voldef.getFormat() == LibvirtStorageVolumeDef.volFormat.QCOW2) { disk.setFormat(PhysicalDiskFormat.QCOW2); } else if (voldef.getFormat() == LibvirtStorageVolumeDef.volFormat.RAW) { @@ -426,6 +444,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { } return disk; } catch (LibvirtException e) { + s_logger.debug("Failed to get physical disk:", e); throw new CloudRuntimeException(e.toString()); } @@ -468,13 +487,13 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { // if anyone is, undefine the pool so we can define it as requested. // This should be safe since a pool in use can't be removed, and no // volumes are affected by unregistering the pool with libvirt. - s_logger.debug("Didn't find an existing storage pool " + name + s_logger.debug("Didn't find an existing storage pool " + name + " by UUID, checking for pools with duplicate paths"); try { String[] poolnames = conn.listStoragePools(); for (String poolname : poolnames) { - s_logger.debug("Checking path of existing pool " + poolname + s_logger.debug("Checking path of existing pool " + poolname + " against pool we want to create"); StoragePool p = conn.storagePoolLookupByName(poolname); LibvirtStoragePoolDef pdef = getStoragePoolDef(conn, p); @@ -492,7 +511,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { } } } catch (LibvirtException e) { - s_logger.error("Failure in attempting to see if an existing storage pool might " + s_logger.error("Failure in attempting to see if an existing storage pool might " + "be using the path of the pool to be created:" + e); } @@ -535,14 +554,14 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { pool.setCapacity(sp.getInfo().capacity); pool.setUsed(sp.getInfo().allocation); pool.setAvailable(sp.getInfo().available); - + return pool; } catch (LibvirtException e) { String error = e.toString(); if (error.contains("Storage source conflict")) { throw new CloudRuntimeException("A pool matching this location already exists in libvirt, " - + " but has a different UUID/Name. Cannot create new pool without first " - + " removing it. Check for inactive pools via 'virsh pool-list --all'. " + + " but has a different UUID/Name. Cannot create new pool without first " + + " removing it. Check for inactive pools via 'virsh pool-list --all'. " + error); } else { throw new CloudRuntimeException(error); @@ -615,42 +634,120 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { StoragePool virtPool = libvirtPool.getPool(); LibvirtStorageVolumeDef.volFormat libvirtformat = null; + String volPath = null; + String volName = null; + long volAllocation = 0; + long volCapacity = 0; + + /** + * To have RBD function properly we want RBD images of format 2 + * libvirt currently defaults to format 1 + * + * For that reason we use the native RBD bindings to create the + * RBD image until libvirt creates RBD format 2 by default + */ if (pool.getType() == StoragePoolType.RBD) { format = PhysicalDiskFormat.RAW; + + try { + s_logger.info("Creating RBD image " + pool.getSourceDir() + "/" + name + " with size " + size); + + Rados r = new Rados(pool.getAuthUserName()); + r.confSet("mon_host", pool.getSourceHost() + ":" + pool.getSourcePort()); + r.confSet("key", pool.getAuthSecret()); + r.connect(); + s_logger.debug("Succesfully connected to Ceph cluster at " + r.confGet("mon_host")); + + IoCTX io = r.ioCtxCreate(pool.getSourceDir()); + Rbd rbd = new Rbd(io); + rbd.create(name, size, this.rbdFeatures, this.rbdOrder); + + r.ioCtxDestroy(io); + } catch (RadosException e) { + throw new CloudRuntimeException(e.toString()); + } catch (RbdException e) { + throw new CloudRuntimeException(e.toString()); + } + + volPath = pool.getSourceDir() + "/" + name; + volName = name; + volCapacity = size; + volAllocation = size; + } else { + + if (format == PhysicalDiskFormat.QCOW2) { + libvirtformat = LibvirtStorageVolumeDef.volFormat.QCOW2; + } else if (format == PhysicalDiskFormat.RAW) { + libvirtformat = LibvirtStorageVolumeDef.volFormat.RAW; + } else if (format == PhysicalDiskFormat.DIR) { + libvirtformat = LibvirtStorageVolumeDef.volFormat.DIR; + } else if (format == PhysicalDiskFormat.TAR) { + libvirtformat = LibvirtStorageVolumeDef.volFormat.TAR; + } + + LibvirtStorageVolumeDef volDef = new LibvirtStorageVolumeDef(name, + size, libvirtformat, null, null); + s_logger.debug(volDef.toString()); + try { + StorageVol vol = virtPool.storageVolCreateXML(volDef.toString(), 0); + volPath = vol.getPath(); + volName = vol.getName(); + volAllocation = vol.getInfo().allocation; + volCapacity = vol.getInfo().capacity; + } catch (LibvirtException e) { + throw new CloudRuntimeException(e.toString()); + } } - if (format == PhysicalDiskFormat.QCOW2) { - libvirtformat = LibvirtStorageVolumeDef.volFormat.QCOW2; - } else if (format == PhysicalDiskFormat.RAW) { - libvirtformat = LibvirtStorageVolumeDef.volFormat.RAW; - } else if (format == PhysicalDiskFormat.DIR) { - libvirtformat = LibvirtStorageVolumeDef.volFormat.DIR; - } else if (format == PhysicalDiskFormat.TAR) { - libvirtformat = LibvirtStorageVolumeDef.volFormat.TAR; - } - - LibvirtStorageVolumeDef volDef = new LibvirtStorageVolumeDef(name, - size, libvirtformat, null, null); - s_logger.debug(volDef.toString()); - try { - StorageVol vol = virtPool.storageVolCreateXML(volDef.toString(), 0); - KVMPhysicalDisk disk = new KVMPhysicalDisk(vol.getPath(), - vol.getName(), pool); - disk.setFormat(format); - disk.setSize(vol.getInfo().allocation); - disk.setVirtualSize(vol.getInfo().capacity); - return disk; - } catch (LibvirtException e) { - throw new CloudRuntimeException(e.toString()); - } + KVMPhysicalDisk disk = new KVMPhysicalDisk(volPath, volName, pool); + disk.setFormat(format); + disk.setSize(volAllocation); + disk.setVirtualSize(volCapacity); + return disk; } @Override public boolean deletePhysicalDisk(String uuid, KVMStoragePool pool) { + + /** + * RBD volume can have snapshots and while they exist libvirt + * can't remove the RBD volume + * + * We have to remove those snapshots first + */ + if (pool.getType() == StoragePoolType.RBD) { + try { + s_logger.info("Unprotecting and Removing RBD snapshots of image " + + pool.getSourcePort() + "/" + uuid + " prior to removing the image"); + + Rados r = new Rados(pool.getAuthUserName()); + r.confSet("mon_host", pool.getSourceHost() + ":" + pool.getSourcePort()); + r.confSet("key", pool.getAuthSecret()); + r.connect(); + s_logger.debug("Succesfully connected to Ceph cluster at " + r.confGet("mon_host")); + + IoCTX io = r.ioCtxCreate(pool.getSourceDir()); + Rbd rbd = new Rbd(io); + RbdImage image = rbd.open(uuid); + List snaps = image.snapList(); + for (RbdSnapInfo snap : snaps) { + image.snapUnprotect(snap.name); + image.snapRemove(snap.name); + } + + rbd.close(image); + r.ioCtxDestroy(io); + } catch (RadosException e) { + throw new CloudRuntimeException(e.toString()); + } catch (RbdException e) { + throw new CloudRuntimeException(e.toString()); + } + } + LibvirtStoragePool libvirtPool = (LibvirtStoragePool) pool; try { StorageVol vol = this.getVolume(libvirtPool.getPool(), uuid); - vol.delete(0); + deleteVol(libvirtPool, vol); vol.free(); return true; } catch (LibvirtException e) { @@ -703,6 +800,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { qemu.convert(sourceFile, destFile); } } else { + format = PhysicalDiskFormat.RAW; disk = new KVMPhysicalDisk(destPool.getSourceDir() + "/" + newUuid, newUuid, destPool); disk.setFormat(format); disk.setSize(template.getVirtualSize()); @@ -730,16 +828,11 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { * we want to copy it */ - /* Feature 1<<0 means layering in RBD format 2 */ - int rbdFeatures = (1<<0); - /* Order 0 means 4MB blocks (the default) */ - int rbdOrder = 0; - try { if ((srcPool.getSourceHost().equals(destPool.getSourceHost())) && (srcPool.getSourceDir().equals(destPool.getSourceDir()))) { /* We are on the same Ceph cluster, but we require RBD format 2 on the source image */ s_logger.debug("Trying to perform a RBD clone (layering) since we are operating in the same storage pool"); - + Rados r = new Rados(srcPool.getAuthUserName()); r.confSet("mon_host", srcPool.getSourceHost() + ":" + srcPool.getSourcePort()); r.confSet("key", srcPool.getAuthSecret()); @@ -755,7 +848,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { s_logger.debug("The source image " + srcPool.getSourceDir() + "/" + template.getName() + " is RBD format 1. We have to perform a regular copy (" + template.getVirtualSize() + " bytes)"); - rbd.create(disk.getName(), template.getVirtualSize(), rbdFeatures, rbdOrder); + rbd.create(disk.getName(), template.getVirtualSize(), this.rbdFeatures, this.rbdOrder); RbdImage destImage = rbd.open(disk.getName()); s_logger.debug("Starting to copy " + srcImage.getName() + " to " + destImage.getName() + " in Ceph pool " + srcPool.getSourceDir()); @@ -768,7 +861,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { + " is RBD format 2. We will perform a RBD clone using snapshot " + this.rbdTemplateSnapName); /* The source image is format 2, we can do a RBD snapshot+clone (layering) */ - rbd.clone(template.getName(), this.rbdTemplateSnapName, io, disk.getName(), rbdFeatures, rbdOrder); + rbd.clone(template.getName(), this.rbdTemplateSnapName, io, disk.getName(), this.rbdFeatures, this.rbdOrder); s_logger.debug("Succesfully cloned " + template.getName() + "@" + this.rbdTemplateSnapName + " to " + disk.getName()); } @@ -798,7 +891,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { s_logger.debug("Creating " + disk.getName() + " on the destination cluster " + rDest.confGet("mon_host") + " in pool " + destPool.getSourceDir()); - dRbd.create(disk.getName(), template.getVirtualSize(), rbdFeatures, rbdOrder); + dRbd.create(disk.getName(), template.getVirtualSize(), this.rbdFeatures, this.rbdOrder); RbdImage srcImage = sRbd.open(template.getName()); RbdImage destImage = dRbd.open(disk.getName()); @@ -809,7 +902,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { sRbd.close(srcImage); dRbd.close(destImage); - + rSrc.ioCtxDestroy(sIO); rDest.ioCtxDestroy(dIO); } @@ -880,12 +973,21 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { for Secondary Storage */ + KVMStoragePool srcPool = disk.getPool(); + PhysicalDiskFormat sourceFormat = disk.getFormat(); + String sourcePath = disk.getPath(); + KVMPhysicalDisk newDisk; if (destPool.getType() != StoragePoolType.RBD) { if (disk.getFormat() == PhysicalDiskFormat.TAR) { newDisk = destPool.createPhysicalDisk(name, PhysicalDiskFormat.DIR, disk.getVirtualSize()); } else { - newDisk = destPool.createPhysicalDisk(name, disk.getVirtualSize()); + /* If the source device is on a RBD storage pool force the new disk to the same format (RAW) */ + if (srcPool.getType() != StoragePoolType.RBD) { + newDisk = destPool.createPhysicalDisk(name, disk.getVirtualSize()); + } else { + newDisk = destPool.createPhysicalDisk(name, sourceFormat, disk.getVirtualSize()); + } } } else { newDisk = new KVMPhysicalDisk(destPool.getSourceDir() + "/" + name, name, destPool); @@ -894,10 +996,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { newDisk.setVirtualSize(disk.getSize()); } - KVMStoragePool srcPool = disk.getPool(); String destPath = newDisk.getPath(); - String sourcePath = disk.getPath(); - PhysicalDiskFormat sourceFormat = disk.getFormat(); PhysicalDiskFormat destFormat = newDisk.getFormat(); QemuImg qemu = new QemuImg(); @@ -942,15 +1041,21 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { * A HUGE performance gain can be achieved here if QCOW2 -> RBD format 2 can be done in one step */ s_logger.debug("The source image is not RBD, but the destination is. We will convert into RBD format 2"); - String tmpFile = "/tmp/" + name; - int rbdFeatures = (1<<0); - int rbdOrder = 0; + String sourceFile; + boolean useTmpFile = false; try { - srcFile = new QemuImgFile(sourcePath, sourceFormat); - destFile = new QemuImgFile(tmpFile); - s_logger.debug("Converting " + srcFile.getFileName() + " to " + tmpFile + " as a temporary file for RBD conversion"); - qemu.convert(srcFile, destFile); + if (sourceFormat != destFormat) { + srcFile = new QemuImgFile(sourcePath, sourceFormat); + destFile = new QemuImgFile("/tmp/" + name); + s_logger.debug("Converting " + srcFile.getFileName() + " to " + destFile.getFileName() + " as a temporary file for RBD conversion"); + qemu.convert(srcFile, destFile); + sourceFile = destFile.getFileName(); + useTmpFile = true; + } else { + // Source file is RAW, we can write directly to RBD + sourceFile = sourcePath; + } // We now convert the temporary file to a RBD image with format 2 Rados r = new Rados(destPool.getAuthUserName()); @@ -963,17 +1068,16 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { Rbd rbd = new Rbd(io); s_logger.debug("Creating RBD image " + name + " in Ceph pool " + destPool.getSourceDir() + " with RBD format 2"); - rbd.create(name, disk.getVirtualSize(), rbdFeatures, rbdOrder); + rbd.create(name, disk.getVirtualSize(), this.rbdFeatures, this.rbdOrder); RbdImage image = rbd.open(name); - // We now read the temporary file and write it to the RBD image - File fh = new File(tmpFile); + File fh = new File(sourceFile); BufferedInputStream bis = new BufferedInputStream(new FileInputStream(fh)); int chunkSize = 4194304; long offset = 0; - s_logger.debug("Reading temporary file " + tmpFile + " (" + fh.length() + " bytes) into RBD image " + name + " in chunks of " + chunkSize + " bytes"); + s_logger.debug("Reading file " + sourceFile + " (" + fh.length() + " bytes) into RBD image " + name + " in chunks of " + chunkSize + " bytes"); while(true) { byte[] buf = new byte[chunkSize]; @@ -984,10 +1088,13 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { image.write(buf, offset, bytes); offset += bytes; } - s_logger.debug("Completed writing " + tmpFile + " to RBD image " + name + ". Bytes written: " + offset); + s_logger.debug("Completed writing " + sourceFile + " to RBD image " + name + ". Bytes written: " + offset); bis.close(); - s_logger.debug("Removing temporary file " + tmpFile); - fh.delete(); + + if (useTmpFile) { + s_logger.debug("Removing temporary file " + sourceFile); + fh.delete(); + } /* Snapshot the image and protect that snapshot so we can clone (layer) from it */ s_logger.debug("Creating RBD snapshot " + this.rbdTemplateSnapName + " on image " + name); @@ -1023,11 +1130,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { srcPool.getAuthSecret(), sourcePath)); srcFile.setFormat(sourceFormat); - destFile = new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(destPool.getSourceHost(), - destPool.getSourcePort(), - destPool.getAuthUserName(), - destPool.getAuthSecret(), - destPath)); + destFile = new QemuImgFile(destPath); destFile.setFormat(destFormat); try { @@ -1057,7 +1160,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { LibvirtStoragePool libvirtPool = (LibvirtStoragePool) pool; StoragePool virtPool = libvirtPool.getPool(); try { - virtPool.refresh(0); + refreshPool(virtPool); } catch (LibvirtException e) { return false; } @@ -1069,20 +1172,41 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { return deleteStoragePool(pool.getUuid()); } - public boolean deleteVbdByPath(String diskPath) { - Connect conn; - try { - conn = LibvirtConnection.getConnection(); - StorageVol vol = conn.storageVolLookupByPath(diskPath); - if(vol != null) { - s_logger.debug("requested delete disk " + diskPath); - vol.delete(0); - } - } catch (LibvirtException e) { - s_logger.debug("Libvirt error in attempting to find and delete patch disk:" + e.toString()); - return false; - } - return true; + // refreshPool and deleteVol are used to fix CLOUDSTACK-2729/CLOUDSTACK-2780 + // They are caused by a libvirt bug (https://bugzilla.redhat.com/show_bug.cgi?id=977706) + // However, we also need to fix the issues in CloudStack source code. + // A file lock is used to prevent deleting a volume from a KVM storage pool when refresh it. + private void refreshPool(StoragePool pool) throws LibvirtException { + pool.refresh(0); + return; } + private void deleteVol(LibvirtStoragePool pool, StorageVol vol) throws LibvirtException { + vol.delete(0); + } + + private boolean lock(String path, int wait) { + File lockFile = new File(path); + lockFile.getParentFile().mkdir(); + boolean havelock = false; + try { + while (wait > 0) { + if (lockFile.createNewFile()) { + havelock = true; + break; + } + s_logger.debug("lockFile " + _lockfile + " already exists, waiting 1000 ms"); + Thread.sleep(1000); + wait--; + } + } catch (IOException e) { + } catch (InterruptedException e) { + } + return havelock; + } + + private void unlock(String path) { + File lockFile = new File(path); + lockFile.delete(); + } } diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java index 2ce517504d6..c0e10002930 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java @@ -16,14 +16,19 @@ // under the License. package com.cloud.hypervisor.kvm.storage; +import java.io.File; import java.util.List; +import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; +import org.apache.log4j.Logger; import org.libvirt.StoragePool; import com.cloud.storage.Storage.StoragePoolType; public class LibvirtStoragePool implements KVMStoragePool { + private static final Logger s_logger = Logger + .getLogger(LibvirtStoragePool.class); protected String uuid; protected String uri; protected long capacity; @@ -120,7 +125,32 @@ public class LibvirtStoragePool implements KVMStoragePool { @Override public KVMPhysicalDisk getPhysicalDisk(String volumeUuid) { - return this._storageAdaptor.getPhysicalDisk(volumeUuid, this); + KVMPhysicalDisk disk = null; + try { + disk = this._storageAdaptor.getPhysicalDisk(volumeUuid, this); + } catch (CloudRuntimeException e) { + if ((this.getStoragePoolType() != StoragePoolType.NetworkFilesystem) && + (this.getStoragePoolType() != StoragePoolType.Filesystem)) { + throw e; + } + } + + if (disk != null) { + return disk; + } + s_logger.debug("find volume bypass libvirt"); + //For network file system or file system, try to use java file to find the volume, instead of through libvirt. BUG:CLOUDSTACK-4459 + String localPoolPath = this.getLocalPath(); + File f = new File(localPoolPath + File.separator + volumeUuid); + if (!f.exists()) { + s_logger.debug("volume: " + volumeUuid + " not exist on storage pool"); + throw new CloudRuntimeException("Can't find volume:" + volumeUuid); + } + disk = new KVMPhysicalDisk(f.getPath(), volumeUuid, this); + disk.setFormat(PhysicalDiskFormat.QCOW2); + disk.setSize(f.length()); + disk.setVirtualSize(f.length()); + return disk; } @Override diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/StorageAdaptor.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/StorageAdaptor.java index dd75677db20..4956d8d4717 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/StorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/StorageAdaptor.java @@ -61,6 +61,4 @@ public interface StorageAdaptor { public boolean createFolder(String uuid, String path); - public boolean deleteVbdByPath(String path); - } diff --git a/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java b/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java index 56d6536efd5..d6e8dc2fcc2 100644 --- a/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java +++ b/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java @@ -24,11 +24,13 @@ import com.cloud.template.VirtualMachineTemplate.BootloaderType; import com.cloud.utils.Pair; import com.cloud.vm.VirtualMachine; +import junit.framework.Assert; import org.apache.commons.lang.SystemUtils; import org.junit.Assume; import org.junit.Test; import java.util.Random; +import java.util.UUID; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -37,7 +39,6 @@ public class LibvirtComputingResourceTest { String _hyperVisorType = "kvm"; Random _random = new Random(); - /** This test tests if the Agent can handle a vmSpec coming from a <=4.1 management server. @@ -102,14 +103,13 @@ public class LibvirtComputingResourceTest { vmStr += "\n"; vmStr += "\n"; vmStr += "\n"; - vmStr += "\n"; - vmStr += "" + (cpus * speed) + "\n"; - vmStr += "\n"; + //vmStr += "\n"; + //vmStr += "" + (cpus * speed) + "\n"; + //vmStr += "\n"; vmStr += "restart\n"; vmStr += "destroy\n"; vmStr += "destroy\n"; vmStr += "\n"; - assertEquals(vmStr, vm.toString()); } @@ -178,9 +178,9 @@ public class LibvirtComputingResourceTest { vmStr += "\n"; vmStr += "\n"; vmStr += "\n"; - vmStr += "\n"; - vmStr += "" + (cpus * minSpeed) + "\n"; - vmStr += "\n"; + //vmStr += "\n"; + //vmStr += "" + (cpus * minSpeed) + "\n"; + //vmStr += "\n"; vmStr += "restart\n"; vmStr += "destroy\n"; vmStr += "destroy\n"; @@ -197,4 +197,17 @@ public class LibvirtComputingResourceTest { Pair stats = LibvirtComputingResource.getNicStats("lo"); assertNotNull(stats); } + + @Test + public void testUUID() { + String uuid = "1"; + LibvirtComputingResource lcr = new LibvirtComputingResource(); + uuid =lcr.getUuid(uuid); + Assert.assertTrue(!uuid.equals("1")); + + String oldUuid = UUID.randomUUID().toString(); + uuid = oldUuid; + uuid = lcr.getUuid(uuid); + Assert.assertTrue(uuid.equals(oldUuid)); + } } diff --git a/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/LibvirtVMDefTest.java b/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/LibvirtVMDefTest.java index 2c0ff8d8b77..9db2902fe25 100644 --- a/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/LibvirtVMDefTest.java +++ b/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/LibvirtVMDefTest.java @@ -49,4 +49,24 @@ public class LibvirtVMDefTest extends TestCase { assertEquals(expected, ifDef.toString()); } + public void testCpuModeDef(){ + LibvirtVMDef.CpuModeDef cpuModeDef = new LibvirtVMDef.CpuModeDef(); + cpuModeDef.setMode("custom"); + cpuModeDef.setModel("Nehalem"); + + String expected1 = "Nehalem"; + + assertEquals(expected1, cpuModeDef.toString()); + + cpuModeDef.setMode("host-model"); + String expected2 = ""; + + assertEquals(expected2, cpuModeDef.toString()); + + cpuModeDef.setMode("host-passthrough"); + String expected3 = ""; + assertEquals(expected3, cpuModeDef.toString()); + + } + } diff --git a/engine/schema/src/com/cloud/storage/dao/VMTemplateS3Dao.java b/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/storage/KVMStorageProcessorTest.java similarity index 57% rename from engine/schema/src/com/cloud/storage/dao/VMTemplateS3Dao.java rename to plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/storage/KVMStorageProcessorTest.java index d36fb3a2257..65c9d7c344a 100644 --- a/engine/schema/src/com/cloud/storage/dao/VMTemplateS3Dao.java +++ b/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/storage/KVMStorageProcessorTest.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an @@ -16,21 +16,27 @@ * specific language governing permissions and limitations * under the License. */ -package com.cloud.storage.dao; +package com.cloud.hypervisor.kvm.storage; -import com.cloud.storage.VMTemplateS3VO; -import com.cloud.utils.db.GenericDao; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import org.junit.Before; +import org.junit.Test; -import java.util.List; +import javax.naming.ConfigurationException; +import java.util.HashMap; -public interface VMTemplateS3Dao extends GenericDao { +public class KVMStorageProcessorTest { + @Before + public void setUp() throws ConfigurationException { + } + @Test + public void testCloneVolumeFromBaseTemplate() throws Exception { - List listByS3Id(long id); - VMTemplateS3VO findOneByTemplateId(long id); + } - VMTemplateS3VO findOneByS3Template(long s3Id, long templateId); - - void expungeAllByTemplateId(long templateId); + @Test + public void testCopyVolumeFromImageCacheToPrimary() throws Exception { + } } diff --git a/plugins/hypervisors/ovm/pom.xml b/plugins/hypervisors/ovm/pom.xml index 84beff0d4eb..2339f23fcb8 100644 --- a/plugins/hypervisors/ovm/pom.xml +++ b/plugins/hypervisors/ovm/pom.xml @@ -23,7 +23,7 @@ org.apache.cloudstack cloudstack-plugins - 4.2.0-SNAPSHOT + 4.2.0 ../../pom.xml diff --git a/plugins/hypervisors/simulator/pom.xml b/plugins/hypervisors/simulator/pom.xml index e99d3559139..2273e5c6415 100644 --- a/plugins/hypervisors/simulator/pom.xml +++ b/plugins/hypervisors/simulator/pom.xml @@ -22,7 +22,7 @@ org.apache.cloudstack cloudstack-plugins - 4.2.0-SNAPSHOT + 4.2.0 ../../pom.xml org.apache.cloudstack diff --git a/plugins/hypervisors/simulator/src/com/cloud/agent/manager/SimulatorManagerImpl.java b/plugins/hypervisors/simulator/src/com/cloud/agent/manager/SimulatorManagerImpl.java index 391efeed3b8..f6bd2b651ba 100644 --- a/plugins/hypervisors/simulator/src/com/cloud/agent/manager/SimulatorManagerImpl.java +++ b/plugins/hypervisors/simulator/src/com/cloud/agent/manager/SimulatorManagerImpl.java @@ -86,6 +86,7 @@ import com.cloud.agent.api.storage.DestroyCommand; import com.cloud.agent.api.storage.ListTemplateCommand; import com.cloud.agent.api.storage.ListVolumeCommand; import com.cloud.agent.api.storage.PrimaryStorageDownloadCommand; +import com.cloud.api.commands.ConfigureSimulatorCmd; import com.cloud.resource.SimulatorStorageProcessor; import com.cloud.simulator.MockConfigurationVO; import com.cloud.simulator.MockHost; @@ -96,6 +97,7 @@ import com.cloud.storage.resource.StorageSubsystemCommandHandler; import com.cloud.storage.resource.StorageSubsystemCommandHandlerBase; import com.cloud.utils.Pair; import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.component.PluggableService; import com.cloud.utils.db.DB; import com.cloud.utils.db.Transaction; import com.cloud.utils.exception.CloudRuntimeException; @@ -110,12 +112,14 @@ import org.springframework.stereotype.Component; import javax.ejb.Local; import javax.inject.Inject; import javax.naming.ConfigurationException; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; @Component @Local(value = { SimulatorManager.class }) -public class SimulatorManagerImpl extends ManagerBase implements SimulatorManager { +public class SimulatorManagerImpl extends ManagerBase implements SimulatorManager, PluggableService { private static final Logger s_logger = Logger.getLogger(SimulatorManagerImpl.class); @Inject MockVmManager _mockVmMgr; @@ -168,6 +172,13 @@ public class SimulatorManagerImpl extends ManagerBase implements SimulatorManage return _mockAgentMgr; } + @Override + public List> getCommands() { + List> cmdList = new ArrayList>(); + cmdList.add(ConfigureSimulatorCmd.class); + return cmdList; + } + @DB @Override public Answer simulate(Command cmd, String hostGuid) { @@ -416,6 +427,8 @@ public class SimulatorManagerImpl extends ManagerBase implements SimulatorManage throw new CloudRuntimeException("Unable to configure simulator because of " + ex.getMessage(), ex); } finally { txn.close(); + txn = Transaction.open(Transaction.CLOUD_DB); + txn.close(); } return true; } diff --git a/plugins/hypervisors/simulator/src/com/cloud/api/commands/ConfigureSimulator.java b/plugins/hypervisors/simulator/src/com/cloud/api/commands/ConfigureSimulatorCmd.java similarity index 97% rename from plugins/hypervisors/simulator/src/com/cloud/api/commands/ConfigureSimulator.java rename to plugins/hypervisors/simulator/src/com/cloud/api/commands/ConfigureSimulatorCmd.java index e982665965c..b5685e42491 100755 --- a/plugins/hypervisors/simulator/src/com/cloud/api/commands/ConfigureSimulator.java +++ b/plugins/hypervisors/simulator/src/com/cloud/api/commands/ConfigureSimulatorCmd.java @@ -16,8 +16,12 @@ // under the License. package com.cloud.api.commands; -import javax.inject.Inject; - +import com.cloud.agent.manager.SimulatorManager; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.user.Account; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; @@ -27,17 +31,12 @@ import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.response.SuccessResponse; import org.apache.log4j.Logger; -import com.cloud.agent.manager.SimulatorManager; -import com.cloud.exception.ConcurrentOperationException; -import com.cloud.exception.InsufficientCapacityException; -import com.cloud.exception.ResourceAllocationException; -import com.cloud.exception.ResourceUnavailableException; -import com.cloud.user.Account; +import javax.inject.Inject; @APICommand(name = "configureSimulator", description="configure simulator", responseObject=SuccessResponse.class) -public class ConfigureSimulator extends BaseCmd { - public static final Logger s_logger = Logger.getLogger(ConfigureSimulator.class.getName()); +public class ConfigureSimulatorCmd extends BaseCmd { + public static final Logger s_logger = Logger.getLogger(ConfigureSimulatorCmd.class.getName()); private static final String s_name = "configuresimulatorresponse"; @Inject SimulatorManager _simMgr; @@ -81,4 +80,5 @@ public class ConfigureSimulator extends BaseCmd { return Account.ACCOUNT_ID_SYSTEM; } + } diff --git a/plugins/hypervisors/simulator/src/com/cloud/resource/SimulatorStorageProcessor.java b/plugins/hypervisors/simulator/src/com/cloud/resource/SimulatorStorageProcessor.java index 0131c1da2c4..c7768aa5b69 100644 --- a/plugins/hypervisors/simulator/src/com/cloud/resource/SimulatorStorageProcessor.java +++ b/plugins/hypervisors/simulator/src/com/cloud/resource/SimulatorStorageProcessor.java @@ -65,18 +65,27 @@ public class SimulatorStorageProcessor implements StorageProcessor { public Answer cloneVolumeFromBaseTemplate(CopyCommand cmd) { VolumeObjectTO volume = new VolumeObjectTO(); volume.setPath(UUID.randomUUID().toString()); + volume.setSize(100); volume.setFormat(Storage.ImageFormat.RAW); return new CopyCmdAnswer(volume); } @Override public Answer copyVolumeFromImageCacheToPrimary(CopyCommand cmd) { - return null; + VolumeObjectTO volume = new VolumeObjectTO(); + volume.setPath(UUID.randomUUID().toString()); + volume.setSize(100); + volume.setFormat(Storage.ImageFormat.RAW); + return new CopyCmdAnswer(volume); } @Override public Answer copyVolumeFromPrimaryToSecondary(CopyCommand cmd) { - return null; + VolumeObjectTO volume = new VolumeObjectTO(); + volume.setPath(UUID.randomUUID().toString()); + volume.setSize(100); + volume.setFormat(Storage.ImageFormat.RAW); + return new CopyCmdAnswer(volume); } @Override diff --git a/plugins/hypervisors/ucs/pom.xml b/plugins/hypervisors/ucs/pom.xml index 24bdc948e73..3d172870d42 100755 --- a/plugins/hypervisors/ucs/pom.xml +++ b/plugins/hypervisors/ucs/pom.xml @@ -24,12 +24,12 @@ org.apache.cloudstack cloudstack-plugins - 4.2.0-SNAPSHOT + 4.2.0 ../../pom.xml org.apache.cloudstack cloud-plugin-hypervisor-ucs - 4.2.0-SNAPSHOT + 4.2.0 Apache CloudStack Plugin - Hypervisor UCS http://maven.apache.org diff --git a/plugins/hypervisors/ucs/src/com/cloud/ucs/manager/UcsManager.java b/plugins/hypervisors/ucs/src/com/cloud/ucs/manager/UcsManager.java index 35a44596cb5..0833e31f0f3 100755 --- a/plugins/hypervisors/ucs/src/com/cloud/ucs/manager/UcsManager.java +++ b/plugins/hypervisors/ucs/src/com/cloud/ucs/manager/UcsManager.java @@ -40,4 +40,6 @@ public interface UcsManager extends Manager, PluggableService { UcsBladeResponse associateProfileToBlade(AssociateUcsProfileToBladeCmd cmd); ListResponse listUcsBlades(ListUcsBladeCmd cmd); + + void deleteUcsManager(Long id); } diff --git a/plugins/hypervisors/ucs/src/com/cloud/ucs/manager/UcsManagerImpl.java b/plugins/hypervisors/ucs/src/com/cloud/ucs/manager/UcsManagerImpl.java index fdc09d357fc..4aec48c1c85 100755 --- a/plugins/hypervisors/ucs/src/com/cloud/ucs/manager/UcsManagerImpl.java +++ b/plugins/hypervisors/ucs/src/com/cloud/ucs/manager/UcsManagerImpl.java @@ -39,7 +39,8 @@ import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.UcsBladeResponse; import org.apache.cloudstack.api.response.UcsManagerResponse; import org.apache.cloudstack.api.response.UcsProfileResponse; -import org.apache.log4j.Logger; +import org.apache.log4j.Logger; +import org.apache.cloudstack.api.DeleteUcsManagerCmd; import com.cloud.configuration.Config; import com.cloud.configuration.dao.ConfigurationDao; @@ -208,36 +209,41 @@ public class UcsManagerImpl implements UcsManager { @Override @DB - public UcsManagerResponse addUcsManager(AddUcsManagerCmd cmd) { - SearchCriteriaService q = SearchCriteria2.create(UcsManagerVO.class); - q.addAnd(q.getEntity().getUrl(), Op.EQ, cmd.getUrl()); - UcsManagerVO mgrvo = q.find(); - if (mgrvo != null) { - throw new IllegalArgumentException(String.format("duplicate UCS manager. url[%s] is used by another UCS manager already", cmd.getUrl())); - } - - UcsManagerVO vo = new UcsManagerVO(); - vo.setUuid(UUID.randomUUID().toString()); - vo.setPassword(cmd.getPassword()); - vo.setUrl(cmd.getUrl()); - vo.setUsername(cmd.getUsername()); - vo.setZoneId(cmd.getZoneId()); - vo.setName(cmd.getName()); + public UcsManagerResponse addUcsManager(AddUcsManagerCmd cmd) { + SearchCriteriaService q = SearchCriteria2.create(UcsManagerVO.class); + q.addAnd(q.getEntity().getUrl(), Op.EQ, cmd.getUrl()); + UcsManagerVO mgrvo = q.find(); + if (mgrvo != null) { + throw new IllegalArgumentException(String.format("duplicate UCS manager. url[%s] is used by another UCS manager already", cmd.getUrl())); + } + try { + UcsManagerVO vo = new UcsManagerVO(); + vo.setUuid(UUID.randomUUID().toString()); + vo.setPassword(cmd.getPassword()); + vo.setUrl(cmd.getUrl()); + vo.setUsername(cmd.getUsername()); + vo.setZoneId(cmd.getZoneId()); + vo.setName(cmd.getName()); - Transaction txn = Transaction.currentTxn(); - txn.start(); - ucsDao.persist(vo); - txn.commit(); - UcsManagerResponse rsp = new UcsManagerResponse(); - rsp.setId(String.valueOf(vo.getId())); - rsp.setName(vo.getName()); - rsp.setUrl(vo.getUrl()); - rsp.setZoneId(String.valueOf(vo.getZoneId())); + Transaction txn = Transaction.currentTxn(); + txn.start(); + mgrvo = ucsDao.persist(vo); + txn.commit(); + UcsManagerResponse rsp = new UcsManagerResponse(); + rsp.setId(String.valueOf(vo.getId())); + rsp.setName(vo.getName()); + rsp.setUrl(vo.getUrl()); + rsp.setZoneId(String.valueOf(vo.getZoneId())); - discoverBlades(vo); - - return rsp; + discoverBlades(vo); + return rsp; + } catch (CloudRuntimeException e) { + if (mgrvo != null) { + ucsDao.remove(mgrvo.getId()); + } + throw e; + } } private String getCookie(Long ucsMgrId) { @@ -403,11 +409,25 @@ public class UcsManagerImpl implements UcsManager { @Override public ListResponse listUcsManager(ListUcsManagerCmd cmd) { + List rsps = new ArrayList(); + ListResponse response = new ListResponse(); + if (cmd.getId() != null) { + UcsManagerVO vo = ucsDao.findById(cmd.getId()); + UcsManagerResponse rsp = new UcsManagerResponse(); + rsp.setObjectName("ucsmanager"); + rsp.setId(vo.getUuid()); + rsp.setName(vo.getName()); + rsp.setUrl(vo.getUrl()); + rsp.setZoneId(zoneIdToUuid(vo.getZoneId())); + rsps.add(rsp); + response.setResponses(rsps); + return response; + } + SearchCriteriaService serv = SearchCriteria2.create(UcsManagerVO.class); serv.addAnd(serv.getEntity().getZoneId(), Op.EQ, cmd.getZoneId()); List vos = serv.list(); - List rsps = new ArrayList(vos.size()); for (UcsManagerVO vo : vos) { UcsManagerResponse rsp = new UcsManagerResponse(); rsp.setObjectName("ucsmanager"); @@ -417,7 +437,6 @@ public class UcsManagerImpl implements UcsManager { rsp.setZoneId(zoneIdToUuid(vo.getZoneId())); rsps.add(rsp); } - ListResponse response = new ListResponse(); response.setResponses(rsps); return response; } @@ -484,6 +503,18 @@ public class UcsManagerImpl implements UcsManager { cmds.add(ListUcsProfileCmd.class); cmds.add(AddUcsManagerCmd.class); cmds.add(AssociateUcsProfileToBladeCmd.class); + cmds.add(DeleteUcsManagerCmd.class); return cmds; } + + @Override + public void deleteUcsManager(Long id) { + SearchCriteriaService serv = SearchCriteria2.create(UcsBladeVO.class); + serv.addAnd(serv.getEntity().getUcsManagerId(), Op.EQ, id); + List vos = serv.list(); + for (UcsBladeVO vo : vos) { + bladeDao.remove(vo.getId()); + } + ucsDao.remove(id); + } } diff --git a/plugins/hypervisors/ucs/src/org/apache/cloudstack/api/DeleteUcsManagerCmd.java b/plugins/hypervisors/ucs/src/org/apache/cloudstack/api/DeleteUcsManagerCmd.java new file mode 100644 index 00000000000..ebd9a541658 --- /dev/null +++ b/plugins/hypervisors/ucs/src/org/apache/cloudstack/api/DeleteUcsManagerCmd.java @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api; + +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.ucs.manager.UcsManager; +import com.cloud.user.Account; +import com.cloud.utils.exception.CloudRuntimeException; + +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.api.response.UcsManagerResponse; +import org.apache.log4j.Logger; + +import javax.inject.Inject; + +@APICommand(name="deleteUcsManager", description="Delete a Ucs manager", responseObject= SuccessResponse.class) +public class DeleteUcsManagerCmd extends BaseCmd { + private static final Logger logger = Logger.getLogger(DeleteUcsManagerCmd.class); + + @Inject + private UcsManager mgr; + + @Parameter(name=ApiConstants.UCS_MANAGER_ID, type= BaseCmd.CommandType.UUID, description="ucs manager id", entityType=UcsManagerResponse.class, required=true) + private Long ucsManagerId; + + public Long getUcsManagerId() { + return ucsManagerId; + } + + @Override + public void execute() throws ResourceUnavailableException, + InsufficientCapacityException, ServerApiException, + ConcurrentOperationException, ResourceAllocationException, + NetworkRuleConflictException { + try { + mgr.deleteUcsManager(ucsManagerId); + SuccessResponse rsp = new SuccessResponse(); + rsp.setResponseName(getCommandName()); + rsp.setObjectName("success"); + this.setResponseObject(rsp); + } catch (Exception e) { + logger.debug(e.getMessage(), e); + throw new CloudRuntimeException(e); + } + } + + @Override + public String getCommandName() { + return "deleteUcsManagerResponse"; + } + + @Override + public long getEntityOwnerId() { + return Account.ACCOUNT_ID_SYSTEM; + } +} diff --git a/plugins/hypervisors/ucs/src/org/apache/cloudstack/api/ListUcsManagerCmd.java b/plugins/hypervisors/ucs/src/org/apache/cloudstack/api/ListUcsManagerCmd.java index da13b173f75..7238e1ee40f 100755 --- a/plugins/hypervisors/ucs/src/org/apache/cloudstack/api/ListUcsManagerCmd.java +++ b/plugins/hypervisors/ucs/src/org/apache/cloudstack/api/ListUcsManagerCmd.java @@ -34,8 +34,10 @@ import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.BaseListCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.BaseCmd.CommandType; import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.UcsManagerResponse; +import org.apache.cloudstack.api.response.VolumeResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.log4j.Logger; @@ -51,8 +53,12 @@ import com.cloud.user.Account; public class ListUcsManagerCmd extends BaseListCmd { public static final Logger s_logger = Logger.getLogger(ListUcsManagerCmd.class); - @Parameter(name=ApiConstants.ZONE_ID, type=CommandType.UUID, description="the zone id", entityType=ZoneResponse.class, required=true) + @Parameter(name=ApiConstants.ZONE_ID, type=CommandType.UUID, description="the zone id", entityType=ZoneResponse.class) private Long zoneId; + + @Parameter(name=ApiConstants.ID, type=CommandType.UUID, entityType=UcsManagerResponse.class, + description="the ID of the ucs manager") + private Long id; @Inject private UcsManager mgr; @@ -88,4 +94,12 @@ public class ListUcsManagerCmd extends BaseListCmd { public void setZoneId(Long zoneId) { this.zoneId = zoneId; } + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } } diff --git a/plugins/hypervisors/vmware/pom.xml b/plugins/hypervisors/vmware/pom.xml index 755244f5f61..def4365f6b5 100644 --- a/plugins/hypervisors/vmware/pom.xml +++ b/plugins/hypervisors/vmware/pom.xml @@ -23,7 +23,7 @@ org.apache.cloudstack cloudstack-plugins - 4.2.0-SNAPSHOT + 4.2.0 ../../pom.xml @@ -40,7 +40,7 @@ org.apache.cloudstack cloud-engine-storage - 4.2.0-SNAPSHOT + 4.2.0 compile diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/guru/VMwareGuru.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/guru/VMwareGuru.java index 292f7e992f8..73cc8e3e5a2 100644 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/guru/VMwareGuru.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/guru/VMwareGuru.java @@ -29,8 +29,11 @@ import javax.ejb.Local; import javax.inject.Inject; import com.cloud.agent.api.storage.CreateEntityDownloadURLCommand; +import com.cloud.dc.dao.ClusterDao; import com.cloud.host.Host; +import com.cloud.server.ConfigurationServer; import com.cloud.storage.Storage; +import com.cloud.vm.dao.VMInstanceDao; import org.apache.cloudstack.storage.to.VolumeObjectTO; import org.apache.log4j.Logger; @@ -108,12 +111,17 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru { @Inject SecondaryStorageVmManager _secStorageMgr; @Inject NetworkModel _networkMgr; @Inject ConfigurationDao _configDao; + @Inject ConfigurationServer _configServer; @Inject NicDao _nicDao; @Inject PhysicalNetworkDao _physicalNetworkDao; @Inject PhysicalNetworkTrafficTypeDao _physicalNetworkTrafficTypeDao; + @Inject + VMInstanceDao _vmDao; + @Inject + ClusterDao _clusterDao; protected VMwareGuru() { super(); @@ -180,7 +188,9 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru { break; } } - + long clusterId = _hostDao.findById( _vmDao.findById(vm.getId()).getHostId()).getClusterId(); + details.put(Config.VmwareReserveCpu.key(), _configServer.getConfigValue(Config.VmwareReserveCpu.key(), Config.ConfigurationParameterScope.cluster.toString(), clusterId)); + details.put(Config.VmwareReserveMem.key(), _configServer.getConfigValue(Config.VmwareReserveMem.key(), Config.ConfigurationParameterScope.cluster.toString(), clusterId)); to.setDetails(details); if(vm.getVirtualMachine() instanceof DomainRouterVO) { @@ -270,6 +280,7 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru { // Determine the VM's OS description GuestOSVO guestOS = _guestOsDao.findById(vm.getVirtualMachine().getGuestOSId()); to.setOs(guestOS.getDisplayName()); + to.setHostName(vm.getHostName()); return to; } @@ -340,7 +351,7 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru { } } } - + if(!needDelegation) { return new Pair(Boolean.FALSE, new Long(hostId)); } @@ -414,14 +425,6 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru { return tokens[0] + "@" + vCenterIp; } - @Override - public List finalizeExpunge(VirtualMachine vm) { - UnregisterVMCommand unregisterVMCommand = new UnregisterVMCommand(vm.getInstanceName()); - List commands = new ArrayList(); - commands.add(unregisterVMCommand); - return commands; - } - @Override public List finalizeExpungeNics(VirtualMachine vm, List nics) { List commands = new ArrayList(); diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/VmwareServerDiscoverer.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/VmwareServerDiscoverer.java index 33bc3e834b7..fd7b3b48795 100755 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/VmwareServerDiscoverer.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/VmwareServerDiscoverer.java @@ -27,12 +27,8 @@ import javax.ejb.Local; import javax.inject.Inject; import javax.naming.ConfigurationException; -import org.apache.log4j.Logger; - -import com.vmware.vim25.ClusterDasConfigInfo; -import com.vmware.vim25.ManagedObjectReference; - import org.apache.cloudstack.api.ApiConstants; +import org.apache.log4j.Logger; import com.cloud.agent.api.StartupCommand; import com.cloud.agent.api.StartupRoutingCommand; @@ -79,7 +75,10 @@ import com.cloud.storage.Storage.TemplateType; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.user.Account; +import com.cloud.utils.Pair; import com.cloud.utils.UriUtils; +import com.vmware.vim25.ClusterDasConfigInfo; +import com.vmware.vim25.ManagedObjectReference; @Local(value = Discoverer.class) @@ -137,7 +136,8 @@ public class VmwareServerDiscoverer extends DiscovererBase implements s_logger.info("No pod is assigned, assuming that it is not for vmware and skip it to next discoverer"); return null; } - + boolean failureInClusterDiscovery = true; + String vsmIp = ""; ClusterVO cluster = _clusterDao.findById(clusterId); if(cluster == null || cluster.getHypervisorType() != HypervisorType.VMware) { if(s_logger.isInfoEnabled()) @@ -291,8 +291,13 @@ public class VmwareServerDiscoverer extends DiscovererBase implements if (privateTrafficLabel != null) { s_logger.info("Detected private network label : " + privateTrafficLabel); } - - if (nexusDVS) { + Pair vsmInfo = new Pair(false, 0L); + if (nexusDVS && + (guestTrafficLabelObj.getVirtualSwitchType() == VirtualSwitchType.NexusDistributedVirtualSwitch) || + ((zoneType == NetworkType.Advanced) && (publicTrafficLabelObj.getVirtualSwitchType() == VirtualSwitchType.NexusDistributedVirtualSwitch))) { + // Expect Cisco Nexus VSM details only if following 2 condition met + // 1) The global config parameter vmware.use.nexus.vswitch + // 2) Atleast 1 traffic type uses Nexus distributed virtual switch as backend. if (zoneType != NetworkType.Basic) { publicTrafficLabel = _netmgr.getDefaultPublicTrafficLabel(dcId, HypervisorType.VMware); if (publicTrafficLabel != null) { @@ -304,12 +309,12 @@ public class VmwareServerDiscoverer extends DiscovererBase implements if (guestTrafficLabel != null) { s_logger.info("Detected guest network label : " + guestTrafficLabel); } - String vsmIp = _urlParams.get("vsmipaddress"); + vsmIp = _urlParams.get("vsmipaddress"); String vsmUser = _urlParams.get("vsmusername"); String vsmPassword = _urlParams.get("vsmpassword"); String clusterName = cluster.getName(); try { - _nexusElement.validateVsmCluster(vsmIp, vsmUser, vsmPassword, clusterId, clusterName); + vsmInfo = _nexusElement.validateAndAddVsm(vsmIp, vsmUser, vsmPassword, clusterId, clusterName); } catch(ResourceInUseException ex) { DiscoveryException discEx = new DiscoveryException(ex.getLocalizedMessage() + ". The resource is " + ex.getResourceName()); throw discEx; @@ -424,7 +429,8 @@ public class VmwareServerDiscoverer extends DiscovererBase implements cluster.setGuid(UUID.nameUUIDFromBytes( String.valueOf(clusterId).getBytes()).toString()); _clusterDao.update(clusterId, cluster); - + // Flag cluster discovery success + failureInClusterDiscovery = false; return resources; } catch (DiscoveredWithErrorException e) { throw e; @@ -435,6 +441,13 @@ public class VmwareServerDiscoverer extends DiscovererBase implements } finally { if (context != null) context.close(); + if (failureInClusterDiscovery && vsmInfo.first()) { + try { + s_logger.debug("Deleting Nexus 1000v VSM " + vsmIp + " because cluster discovery and addition to zone has failed."); + _nexusElement.deleteCiscoNexusVSM(vsmInfo.second().longValue()); + } catch(Exception e) { + } + } } } @@ -645,33 +658,24 @@ public class VmwareServerDiscoverer extends DiscovererBase implements throw e; } - if (defaultVirtualSwitchType.equals(VirtualSwitchType.StandardVirtualSwitch)|| (vSwitchType == null && vSwitchName == null)) { - // Case of no cluster level override configuration defined. - // Depend only on zone wide traffic label - // If global param for dvSwitch is false return default traffic info object with vmware standard vswitch - return trafficLabelObj; - } else { - // Need to persist cluster level override configuration to db - clusterDetails = _clusterDetailsDao.findDetails(clusterId); - } - + clusterDetails = _clusterDetailsDao.findDetails(clusterId); if (vSwitchName != null) { trafficLabelObj.setVirtualSwitchName(vSwitchName); - if (trafficType == TrafficType.Guest) { - clusterDetails.put(ApiConstants.VSWITCH_NAME_GUEST_TRAFFIC, vSwitchName); - } else { - clusterDetails.put(ApiConstants.VSWITCH_NAME_PUBLIC_TRAFFIC, vSwitchName); - } + } + if (trafficType == TrafficType.Guest) { + clusterDetails.put(ApiConstants.VSWITCH_NAME_GUEST_TRAFFIC, trafficLabelObj.getVirtualSwitchName()); + } else { + clusterDetails.put(ApiConstants.VSWITCH_NAME_PUBLIC_TRAFFIC, trafficLabelObj.getVirtualSwitchName()); } if (vSwitchType != null) { validateVswitchType(vSwitchType); trafficLabelObj.setVirtualSwitchType(VirtualSwitchType.getType(vSwitchType)); - if (trafficType == TrafficType.Guest) { - clusterDetails.put(ApiConstants.VSWITCH_TYPE_GUEST_TRAFFIC, vSwitchType); - } else { - clusterDetails.put(ApiConstants.VSWITCH_TYPE_PUBLIC_TRAFFIC, vSwitchType); - } + } + if (trafficType == TrafficType.Guest) { + clusterDetails.put(ApiConstants.VSWITCH_TYPE_GUEST_TRAFFIC, trafficLabelObj.getVirtualSwitchType().toString()); + } else { + clusterDetails.put(ApiConstants.VSWITCH_TYPE_PUBLIC_TRAFFIC, trafficLabelObj.getVirtualSwitchType().toString()); } // Save cluster level override configuration to cluster details diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java index 8d6bdb8420d..a1671c9fa90 100755 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java @@ -162,6 +162,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw int _portsPerDvPortGroup = 256; boolean _nexusVSwitchActive; boolean _fullCloneFlag; + boolean _instanceNameFlag; String _serviceConsoleName; String _managemetPortGroupName; String _defaultSystemVmNicAdapterType = VirtualEthernetCardType.E1000.toString(); @@ -232,6 +233,14 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw } else { _fullCloneFlag = Boolean.parseBoolean(value); } + + value = _configDao.getValue(Config.SetVmInternalNameUsingDisplayName.key()); + if (value == null) { + _instanceNameFlag = false; + } else { + _instanceNameFlag = Boolean.parseBoolean(value); + } + _portsPerDvPortGroup = NumbersUtil.parseInt(_configDao.getValue(Config.VmwarePortsPerDVPortGroup.key()), _portsPerDvPortGroup); _serviceConsoleName = _configDao.getValue(Config.VmwareServiceConsole.key()); @@ -506,10 +515,9 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw @Override public void setupResourceStartupParams(Map params) { params.put("vmware.create.full.clone", _fullCloneFlag); + params.put("vm.instancename.flag", _instanceNameFlag); params.put("service.console.name", _serviceConsoleName); params.put("management.portgroup.name", _managemetPortGroupName); - params.put("vmware.reserve.cpu", _reserveCpu); - params.put("vmware.reserve.mem", _reserveMem); params.put("vmware.root.disk.controller", _rootDiskController); params.put("vmware.recycle.hung.wokervm", _recycleHungWorker); params.put("ports.per.dvportgroup", _portsPerDvPortGroup); @@ -755,11 +763,10 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw // Change permissions for the mountpoint script = new Script(true, "chmod", _timeout, s_logger); - script.add("777", mountPoint); + script.add("-R", "777", mountPoint); result = script.execute(); if (result != null) { s_logger.warn("Unable to set permissions for " + mountPoint + " due to " + result); - return null; } return mountPoint; } @@ -1163,9 +1170,15 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw // Check if zone with specified id exists DataCenterVO zone = _dcDao.findById(zoneId); if (zone == null) { - InvalidParameterValueException ex = new InvalidParameterValueException( - "Can't find zone by the id specified."); - throw ex; + throw new InvalidParameterValueException("Can't find zone by the id specified."); + } + // Check if zone is legacy zone + if (isLegacyZone(zoneId)) { + throw new InvalidParameterValueException("The specified zone is legacy zone. Adding VMware datacenter to legacy zone is not supported."); + } else { + if (s_logger.isTraceEnabled()) { + s_logger.trace("The specified zone is not legacy zone."); + } } } diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareStorageManager.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareStorageManager.java index 1e45eb89a91..14f293a32a1 100644 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareStorageManager.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareStorageManager.java @@ -37,4 +37,5 @@ public interface VmwareStorageManager { Answer execute(VmwareHostService hostService, DeleteVMSnapshotCommand cmd); Answer execute(VmwareHostService hostService, RevertToVMSnapshotCommand cmd); boolean execute(VmwareHostService hostService, CreateEntityDownloadURLCommand cmd); + public void createOva(String path, String name); } diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareStorageManagerImpl.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareStorageManagerImpl.java index 10b6de9858b..955b111339a 100644 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareStorageManagerImpl.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareStorageManagerImpl.java @@ -54,9 +54,6 @@ import com.cloud.hypervisor.vmware.mo.DatastoreMO; import com.cloud.hypervisor.vmware.mo.HostDatastoreBrowserMO; import com.cloud.hypervisor.vmware.mo.HostMO; import com.cloud.hypervisor.vmware.mo.HypervisorHostHelper; -import com.cloud.hypervisor.vmware.mo.SnapshotDescriptor; -import com.cloud.hypervisor.vmware.mo.SnapshotDescriptor.SnapshotInfo; -import com.cloud.hypervisor.vmware.mo.TaskMO; import com.cloud.hypervisor.vmware.mo.VirtualMachineMO; import com.cloud.hypervisor.vmware.mo.VmwareHypervisorHost; import com.cloud.hypervisor.vmware.util.VmwareContext; @@ -78,7 +75,6 @@ import com.vmware.vim25.FileQueryFlags; import com.vmware.vim25.HostDatastoreBrowserSearchResults; import com.vmware.vim25.HostDatastoreBrowserSearchSpec; import com.vmware.vim25.ManagedObjectReference; -import com.vmware.vim25.TaskEvent; import com.vmware.vim25.TaskInfo; import com.vmware.vim25.VirtualDeviceConfigSpec; import com.vmware.vim25.VirtualDeviceConfigSpecOperation; @@ -109,6 +105,21 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { return true; } + @Override + public void createOva(String path, String name) { + Script commandSync = new Script(true, "sync", 0, s_logger); + commandSync.execute(); + + Script command = new Script(false, "tar", 0, s_logger); + command.setWorkDir(path); + command.add("-cf", name + ".ova"); + command.add(name + ".ovf"); // OVF file should be the first file in OVA archive + command.add(name + "-disk0.vmdk"); + + s_logger.info("Package OVA with commmand: " + command.toString()); + command.execute(); + } + private static final Logger s_logger = Logger.getLogger(VmwareStorageManagerImpl.class); private final VmwareStorageMount _mountService; @@ -214,354 +225,358 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { @Override public Answer execute(VmwareHostService hostService, PrimaryStorageDownloadCommand cmd) { - String secondaryStorageUrl = cmd.getSecondaryStorageUrl(); - assert (secondaryStorageUrl != null); + String secondaryStorageUrl = cmd.getSecondaryStorageUrl(); + assert (secondaryStorageUrl != null); - String templateUrl = cmd.getUrl(); + String templateUrl = cmd.getUrl(); - String templateName = null; - String mountPoint = null; - if (templateUrl.endsWith(".ova")) { - int index = templateUrl.lastIndexOf("/"); - mountPoint = templateUrl.substring(0, index); - mountPoint = mountPoint.substring(secondaryStorageUrl.length() + 1); - if (!mountPoint.endsWith("/")) { - mountPoint = mountPoint + "/"; - } + String templateName = null; + String mountPoint = null; + if (templateUrl.endsWith(".ova")) { + int index = templateUrl.lastIndexOf("/"); + mountPoint = templateUrl.substring(0, index); + mountPoint = mountPoint.substring(secondaryStorageUrl.length() + 1); + if (!mountPoint.endsWith("/")) { + mountPoint = mountPoint + "/"; + } - templateName = templateUrl.substring(index + 1).replace("." + ImageFormat.OVA.getFileExtension(), ""); + templateName = templateUrl.substring(index + 1).replace("." + ImageFormat.OVA.getFileExtension(), ""); - if (templateName == null || templateName.isEmpty()) { - templateName = cmd.getName(); - } - } else { - mountPoint = templateUrl.substring(secondaryStorageUrl.length() + 1); - if (!mountPoint.endsWith("/")) { - mountPoint = mountPoint + "/"; - } - templateName = cmd.getName(); - } + if (templateName == null || templateName.isEmpty()) { + templateName = cmd.getName(); + } + } else { + mountPoint = templateUrl.substring(secondaryStorageUrl.length() + 1); + if (!mountPoint.endsWith("/")) { + mountPoint = mountPoint + "/"; + } + templateName = cmd.getName(); + } - VmwareContext context = hostService.getServiceContext(cmd); - try { - VmwareHypervisorHost hyperHost = hostService.getHyperHost(context, cmd); + VmwareContext context = hostService.getServiceContext(cmd); + try { + VmwareHypervisorHost hyperHost = hostService.getHyperHost(context, cmd); - String templateUuidName = UUID.nameUUIDFromBytes((templateName + "@" + cmd.getPoolUuid() + "-" + hyperHost.getMor().getValue()).getBytes()).toString(); - // truncate template name to 32 chars to ensure they work well with vSphere API's. - templateUuidName = templateUuidName.replace("-", ""); + String templateUuidName = UUID.nameUUIDFromBytes((templateName + "@" + cmd.getPoolUuid() + "-" + hyperHost.getMor().getValue()).getBytes()).toString(); + // truncate template name to 32 chars to ensure they work well with vSphere API's. + templateUuidName = templateUuidName.replace("-", ""); - DatacenterMO dcMo = new DatacenterMO(context, hyperHost.getHyperHostDatacenter()); - VirtualMachineMO templateMo = VmwareHelper.pickOneVmOnRunningHost(dcMo.findVmByNameAndLabel(templateUuidName), true); + DatacenterMO dcMo = new DatacenterMO(context, hyperHost.getHyperHostDatacenter()); + VirtualMachineMO templateMo = VmwareHelper.pickOneVmOnRunningHost(dcMo.findVmByNameAndLabel(templateUuidName), true); - if (templateMo == null) { - if(s_logger.isInfoEnabled()) - s_logger.info("Template " + templateName + " is not setup yet, setup template from secondary storage with uuid name: " + templateUuidName); - ManagedObjectReference morDs = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, cmd.getPoolUuid()); - assert (morDs != null); - DatastoreMO primaryStorageDatastoreMo = new DatastoreMO(context, morDs); + if (templateMo == null) { + if(s_logger.isInfoEnabled()) { + s_logger.info("Template " + templateName + " is not setup yet, setup template from secondary storage with uuid name: " + templateUuidName); + } + ManagedObjectReference morDs = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, cmd.getPoolUuid()); + assert (morDs != null); + DatastoreMO primaryStorageDatastoreMo = new DatastoreMO(context, morDs); - copyTemplateFromSecondaryToPrimary(hyperHost, - primaryStorageDatastoreMo, secondaryStorageUrl, - mountPoint, templateName, templateUuidName); - } else { - s_logger.info("Template " + templateName + " has already been setup, skip the template setup process in primary storage"); - } + copyTemplateFromSecondaryToPrimary(hyperHost, + primaryStorageDatastoreMo, secondaryStorageUrl, + mountPoint, templateName, templateUuidName); + } else { + s_logger.info("Template " + templateName + " has already been setup, skip the template setup process in primary storage"); + } - return new PrimaryStorageDownloadAnswer(templateUuidName, 0); - } catch (Throwable e) { - if (e instanceof RemoteException) { - hostService.invalidateServiceContext(context); - } + return new PrimaryStorageDownloadAnswer(templateUuidName, 0); + } catch (Throwable e) { + if (e instanceof RemoteException) { + hostService.invalidateServiceContext(context); + } - String msg = "Unable to execute PrimaryStorageDownloadCommand due to exception"; - s_logger.error(msg, e); - return new PrimaryStorageDownloadAnswer(msg); - } + String msg = "Unable to execute PrimaryStorageDownloadCommand due to exception"; + s_logger.error(msg, e); + return new PrimaryStorageDownloadAnswer(msg); + } } @Override - public Answer execute(VmwareHostService hostService, BackupSnapshotCommand cmd) { - Long accountId = cmd.getAccountId(); - Long volumeId = cmd.getVolumeId(); + @Deprecated + public Answer execute(VmwareHostService hostService, BackupSnapshotCommand cmd) { + Long accountId = cmd.getAccountId(); + Long volumeId = cmd.getVolumeId(); String secondaryStorageUrl = cmd.getSecondaryStorageUrl(); String snapshotUuid = cmd.getSnapshotUuid(); // not null: Precondition. - String prevSnapshotUuid = cmd.getPrevSnapshotUuid(); - String prevBackupUuid = cmd.getPrevBackupUuid(); + String prevSnapshotUuid = cmd.getPrevSnapshotUuid(); + String prevBackupUuid = cmd.getPrevBackupUuid(); VirtualMachineMO workerVm=null; String workerVMName = null; - String volumePath = cmd.getVolumePath(); - ManagedObjectReference morDs = null; - DatastoreMO dsMo=null; + String volumePath = cmd.getVolumePath(); + ManagedObjectReference morDs = null; + DatastoreMO dsMo=null; - // By default assume failure - String details = null; - boolean success = false; - String snapshotBackupUuid = null; + // By default assume failure + String details = null; + boolean success = false; + String snapshotBackupUuid = null; - VmwareContext context = hostService.getServiceContext(cmd); - VirtualMachineMO vmMo = null; - try { - VmwareHypervisorHost hyperHost = hostService.getHyperHost(context, cmd); - morDs = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, cmd.getPool().getUuid()); + VmwareContext context = hostService.getServiceContext(cmd); + VirtualMachineMO vmMo = null; + try { + VmwareHypervisorHost hyperHost = hostService.getHyperHost(context, cmd); + morDs = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, cmd.getPool().getUuid()); - try { - vmMo = hyperHost.findVmOnHyperHost(cmd.getVmName()); - if (vmMo == null) { - if(s_logger.isDebugEnabled()) - s_logger.debug("Unable to find owner VM for BackupSnapshotCommand on host " + hyperHost.getHyperHostName() + ", will try within datacenter"); + try { + vmMo = hyperHost.findVmOnHyperHost(cmd.getVmName()); + if (vmMo == null) { + if(s_logger.isDebugEnabled()) { + s_logger.debug("Unable to find owner VM for BackupSnapshotCommand on host " + hyperHost.getHyperHostName() + ", will try within datacenter"); + } - vmMo = hyperHost.findVmOnPeerHyperHost(cmd.getVmName()); - if(vmMo == null) { - dsMo = new DatastoreMO(hyperHost.getContext(), morDs); + vmMo = hyperHost.findVmOnPeerHyperHost(cmd.getVmName()); + if(vmMo == null) { + dsMo = new DatastoreMO(hyperHost.getContext(), morDs); - workerVMName = hostService.getWorkerName(context, cmd, 0); + workerVMName = hostService.getWorkerName(context, cmd, 0); - // attach a volume to dummay wrapper VM for taking snapshot and exporting the VM for backup - if (!hyperHost.createBlankVm(workerVMName, 1, 512, 0, false, 4, 0, VirtualMachineGuestOsIdentifier.OTHER_GUEST.value(), morDs, false)) { - String msg = "Unable to create worker VM to execute BackupSnapshotCommand"; - s_logger.error(msg); - throw new Exception(msg); - } - vmMo = hyperHost.findVmOnHyperHost(workerVMName); - if (vmMo == null) { - throw new Exception("Failed to find the newly create or relocated VM. vmName: " + workerVMName); - } - workerVm = vmMo; + // attach a volume to dummay wrapper VM for taking snapshot and exporting the VM for backup + if (!hyperHost.createBlankVm(workerVMName, null, 1, 512, 0, false, 4, 0, VirtualMachineGuestOsIdentifier.OTHER_GUEST.value(), morDs, false)) { + String msg = "Unable to create worker VM to execute BackupSnapshotCommand"; + s_logger.error(msg); + throw new Exception(msg); + } + vmMo = hyperHost.findVmOnHyperHost(workerVMName); + if (vmMo == null) { + throw new Exception("Failed to find the newly create or relocated VM. vmName: " + workerVMName); + } + workerVm = vmMo; - // attach volume to worker VM + // attach volume to worker VM String datastoreVolumePath = getVolumePathInDatastore(dsMo, volumePath + ".vmdk"); - vmMo.attachDisk(new String[] { datastoreVolumePath }, morDs); - } - } + vmMo.attachDisk(new String[] { datastoreVolumePath }, morDs); + } + } if (!vmMo.createSnapshot(snapshotUuid, "Snapshot taken for " + cmd.getSnapshotName(), false, false)) { throw new Exception("Failed to take snapshot " + cmd.getSnapshotName() + " on vm: " + cmd.getVmName()); } - snapshotBackupUuid = backupSnapshotToSecondaryStorage(vmMo, accountId, volumeId, cmd.getVolumePath(), snapshotUuid, secondaryStorageUrl, prevSnapshotUuid, prevBackupUuid, - hostService.getWorkerName(context, cmd, 1)); + snapshotBackupUuid = backupSnapshotToSecondaryStorage(vmMo, accountId, volumeId, cmd.getVolumePath(), snapshotUuid, secondaryStorageUrl, prevSnapshotUuid, prevBackupUuid, + hostService.getWorkerName(context, cmd, 1)); success = (snapshotBackupUuid != null); if (success) { details = "Successfully backedUp the snapshotUuid: " + snapshotUuid + " to secondary storage."; } - } finally { + } finally { if(vmMo != null){ ManagedObjectReference snapshotMor = vmMo.getSnapshotMor(snapshotUuid); - if (snapshotMor != null){ + if (snapshotMor != null) { vmMo.removeSnapshot(snapshotUuid, false); } } - try { - if (workerVm != null) { - // detach volume and destroy worker vm - workerVm.detachAllDisks(); - workerVm.destroy(); - } - } catch (Throwable e) { - s_logger.warn("Failed to destroy worker VM: " + workerVMName); - } - } - } catch (Throwable e) { - if (e instanceof RemoteException) { - hostService.invalidateServiceContext(context); - } + try { + if (workerVm != null) { + // detach volume and destroy worker vm + workerVm.detachAllDisks(); + workerVm.destroy(); + } + } catch (Throwable e) { + s_logger.warn("Failed to destroy worker VM: " + workerVMName); + } + } + } catch (Throwable e) { + if (e instanceof RemoteException) { + hostService.invalidateServiceContext(context); + } - s_logger.error("Unexpecpted exception ", e); + s_logger.error("Unexpecpted exception ", e); - details = "BackupSnapshotCommand exception: " + StringUtils.getExceptionStackInfo(e); - return new BackupSnapshotAnswer(cmd, false, details, snapshotBackupUuid, true); - } + details = "BackupSnapshotCommand exception: " + StringUtils.getExceptionStackInfo(e); + return new BackupSnapshotAnswer(cmd, false, details, snapshotBackupUuid, true); + } - return new BackupSnapshotAnswer(cmd, success, details, snapshotBackupUuid, true); - } + return new BackupSnapshotAnswer(cmd, success, details, snapshotBackupUuid, true); + } @Override - public Answer execute(VmwareHostService hostService, CreatePrivateTemplateFromVolumeCommand cmd) { + public Answer execute(VmwareHostService hostService, CreatePrivateTemplateFromVolumeCommand cmd) { String secondaryStoragePoolURL = cmd.getSecondaryStorageUrl(); - String volumePath = cmd.getVolumePath(); - Long accountId = cmd.getAccountId(); - Long templateId = cmd.getTemplateId(); - String details = null; + String volumePath = cmd.getVolumePath(); + Long accountId = cmd.getAccountId(); + Long templateId = cmd.getTemplateId(); + String details = null; - VmwareContext context = hostService.getServiceContext(cmd); - try { - VmwareHypervisorHost hyperHost = hostService.getHyperHost(context, cmd); + VmwareContext context = hostService.getServiceContext(cmd); + try { + VmwareHypervisorHost hyperHost = hostService.getHyperHost(context, cmd); - VirtualMachineMO vmMo = hyperHost.findVmOnHyperHost(cmd.getVmName()); - if (vmMo == null) { - if(s_logger.isDebugEnabled()) - s_logger.debug("Unable to find the owner VM for CreatePrivateTemplateFromVolumeCommand on host " + hyperHost.getHyperHostName() + ", try within datacenter"); - vmMo = hyperHost.findVmOnPeerHyperHost(cmd.getVmName()); + VirtualMachineMO vmMo = hyperHost.findVmOnHyperHost(cmd.getVmName()); + if (vmMo == null) { + if(s_logger.isDebugEnabled()) { + s_logger.debug("Unable to find the owner VM for CreatePrivateTemplateFromVolumeCommand on host " + hyperHost.getHyperHostName() + ", try within datacenter"); + } + vmMo = hyperHost.findVmOnPeerHyperHost(cmd.getVmName()); - if(vmMo == null) { - String msg = "Unable to find the owner VM for volume operation. vm: " + cmd.getVmName(); - s_logger.error(msg); - throw new Exception(msg); - } - } + if(vmMo == null) { + String msg = "Unable to find the owner VM for volume operation. vm: " + cmd.getVmName(); + s_logger.error(msg); + throw new Exception(msg); + } + } - Ternary result = createTemplateFromVolume(vmMo, - accountId, templateId, cmd.getUniqueName(), - secondaryStoragePoolURL, volumePath, - hostService.getWorkerName(context, cmd, 0)); + Ternary result = createTemplateFromVolume(vmMo, + accountId, templateId, cmd.getUniqueName(), + secondaryStoragePoolURL, volumePath, + hostService.getWorkerName(context, cmd, 0)); - return new CreatePrivateTemplateAnswer(cmd, true, null, - result.first(), result.third(), result.second(), - cmd.getUniqueName(), ImageFormat.OVA); + return new CreatePrivateTemplateAnswer(cmd, true, null, + result.first(), result.third(), result.second(), + cmd.getUniqueName(), ImageFormat.OVA); - } catch (Throwable e) { - if (e instanceof RemoteException) { - hostService.invalidateServiceContext(context); - } + } catch (Throwable e) { + if (e instanceof RemoteException) { + hostService.invalidateServiceContext(context); + } - s_logger.error("Unexpecpted exception ", e); + s_logger.error("Unexpecpted exception ", e); - details = "CreatePrivateTemplateFromVolumeCommand exception: " + StringUtils.getExceptionStackInfo(e); - return new CreatePrivateTemplateAnswer(cmd, false, details); - } - } + details = "CreatePrivateTemplateFromVolumeCommand exception: " + StringUtils.getExceptionStackInfo(e); + return new CreatePrivateTemplateAnswer(cmd, false, details); + } + } @Override - public Answer execute(VmwareHostService hostService, CreatePrivateTemplateFromSnapshotCommand cmd) { - Long accountId = cmd.getAccountId(); - Long volumeId = cmd.getVolumeId(); + public Answer execute(VmwareHostService hostService, CreatePrivateTemplateFromSnapshotCommand cmd) { + Long accountId = cmd.getAccountId(); + Long volumeId = cmd.getVolumeId(); String secondaryStorageUrl = cmd.getSecondaryStorageUrl(); - String backedUpSnapshotUuid = cmd.getSnapshotUuid(); - Long newTemplateId = cmd.getNewTemplateId(); - String details; - String uniqeName = UUID.randomUUID().toString(); + String backedUpSnapshotUuid = cmd.getSnapshotUuid(); + Long newTemplateId = cmd.getNewTemplateId(); + String details; + String uniqeName = UUID.randomUUID().toString(); - VmwareContext context = hostService.getServiceContext(cmd); - try { - Ternary result = createTemplateFromSnapshot(accountId, - newTemplateId, uniqeName, - secondaryStorageUrl, volumeId, - backedUpSnapshotUuid); + VmwareContext context = hostService.getServiceContext(cmd); + try { + Ternary result = createTemplateFromSnapshot(accountId, + newTemplateId, uniqeName, + secondaryStorageUrl, volumeId, + backedUpSnapshotUuid); - return new CreatePrivateTemplateAnswer(cmd, true, null, - result.first(), result.third(), result.second(), - uniqeName, ImageFormat.OVA); - } catch (Throwable e) { - if (e instanceof RemoteException) { - hostService.invalidateServiceContext(context); - } + return new CreatePrivateTemplateAnswer(cmd, true, null, + result.first(), result.third(), result.second(), + uniqeName, ImageFormat.OVA); + } catch (Throwable e) { + if (e instanceof RemoteException) { + hostService.invalidateServiceContext(context); + } - s_logger.error("Unexpecpted exception ", e); + s_logger.error("Unexpecpted exception ", e); - details = "CreatePrivateTemplateFromSnapshotCommand exception: " + StringUtils.getExceptionStackInfo(e); - return new CreatePrivateTemplateAnswer(cmd, false, details); - } - } + details = "CreatePrivateTemplateFromSnapshotCommand exception: " + StringUtils.getExceptionStackInfo(e); + return new CreatePrivateTemplateAnswer(cmd, false, details); + } + } @Override - public Answer execute(VmwareHostService hostService, CopyVolumeCommand cmd) { - Long volumeId = cmd.getVolumeId(); - String volumePath = cmd.getVolumePath(); - String secondaryStorageURL = cmd.getSecondaryStorageURL(); - String vmName = cmd.getVmName(); + public Answer execute(VmwareHostService hostService, CopyVolumeCommand cmd) { + Long volumeId = cmd.getVolumeId(); + String volumePath = cmd.getVolumePath(); + String secondaryStorageURL = cmd.getSecondaryStorageURL(); + String vmName = cmd.getVmName(); - VmwareContext context = hostService.getServiceContext(cmd); - try { - VmwareHypervisorHost hyperHost = hostService.getHyperHost(context, cmd); + VmwareContext context = hostService.getServiceContext(cmd); + try { + VmwareHypervisorHost hyperHost = hostService.getHyperHost(context, cmd); - Pair result; - if (cmd.toSecondaryStorage()) { - result = copyVolumeToSecStorage(hostService, - hyperHost, cmd, vmName, volumeId, cmd.getPool().getUuid(), volumePath, - secondaryStorageURL, - hostService.getWorkerName(context, cmd, 0)); - } else { - StorageFilerTO poolTO = cmd.getPool(); + Pair result; + if (cmd.toSecondaryStorage()) { + result = copyVolumeToSecStorage(hostService, + hyperHost, cmd, vmName, volumeId, cmd.getPool().getUuid(), volumePath, + secondaryStorageURL, + hostService.getWorkerName(context, cmd, 0)); + } else { + StorageFilerTO poolTO = cmd.getPool(); - ManagedObjectReference morDatastore = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, poolTO.getUuid()); - if (morDatastore == null) { - morDatastore = hyperHost.mountDatastore( - false, - poolTO.getHost(), 0, poolTO.getPath(), - poolTO.getUuid().replace("-", "")); + ManagedObjectReference morDatastore = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, poolTO.getUuid()); + if (morDatastore == null) { + morDatastore = hyperHost.mountDatastore( + false, + poolTO.getHost(), 0, poolTO.getPath(), + poolTO.getUuid().replace("-", "")); - if (morDatastore == null) { - throw new Exception("Unable to mount storage pool on host. storeUrl: " + poolTO.getHost() + ":/" + poolTO.getPath()); - } - } + if (morDatastore == null) { + throw new Exception("Unable to mount storage pool on host. storeUrl: " + poolTO.getHost() + ":/" + poolTO.getPath()); + } + } - result = copyVolumeFromSecStorage( - hyperHost, volumeId, - new DatastoreMO(context, morDatastore), - secondaryStorageURL, volumePath); - deleteVolumeDirOnSecondaryStorage(volumeId, secondaryStorageURL); - } - return new CopyVolumeAnswer(cmd, true, null, result.first(), result.second()); - } catch (Throwable e) { - if (e instanceof RemoteException) { - hostService.invalidateServiceContext(context); - } + result = copyVolumeFromSecStorage( + hyperHost, volumeId, + new DatastoreMO(context, morDatastore), + secondaryStorageURL, volumePath); + deleteVolumeDirOnSecondaryStorage(volumeId, secondaryStorageURL); + } + return new CopyVolumeAnswer(cmd, true, null, result.first(), result.second()); + } catch (Throwable e) { + if (e instanceof RemoteException) { + hostService.invalidateServiceContext(context); + } - String msg = "Unable to execute CopyVolumeCommand due to exception"; - s_logger.error(msg, e); - return new CopyVolumeAnswer(cmd, false, "CopyVolumeCommand failed due to exception: " + StringUtils.getExceptionStackInfo(e), null, null); - } - } + String msg = "Unable to execute CopyVolumeCommand due to exception"; + s_logger.error(msg, e); + return new CopyVolumeAnswer(cmd, false, "CopyVolumeCommand failed due to exception: " + StringUtils.getExceptionStackInfo(e), null, null); + } + } @Override - public Answer execute(VmwareHostService hostService, CreateVolumeFromSnapshotCommand cmd) { + public Answer execute(VmwareHostService hostService, CreateVolumeFromSnapshotCommand cmd) { - String primaryStorageNameLabel = cmd.getPrimaryStoragePoolNameLabel(); - Long accountId = cmd.getAccountId(); - Long volumeId = cmd.getVolumeId(); + String primaryStorageNameLabel = cmd.getPrimaryStoragePoolNameLabel(); + Long accountId = cmd.getAccountId(); + Long volumeId = cmd.getVolumeId(); String secondaryStorageUrl = cmd.getSecondaryStorageUrl(); - String backedUpSnapshotUuid = cmd.getSnapshotUuid(); + String backedUpSnapshotUuid = cmd.getSnapshotUuid(); - String details = null; - boolean success = false; - String newVolumeName = UUID.randomUUID().toString().replaceAll("-", ""); + String details = null; + boolean success = false; + String newVolumeName = UUID.randomUUID().toString().replaceAll("-", ""); - VmwareContext context = hostService.getServiceContext(cmd); - try { - VmwareHypervisorHost hyperHost = hostService.getHyperHost(context, cmd); - ManagedObjectReference morPrimaryDs = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, - primaryStorageNameLabel); - if (morPrimaryDs == null) { - String msg = "Unable to find datastore: " + primaryStorageNameLabel; - s_logger.error(msg); - throw new Exception(msg); - } + VmwareContext context = hostService.getServiceContext(cmd); + try { + VmwareHypervisorHost hyperHost = hostService.getHyperHost(context, cmd); + ManagedObjectReference morPrimaryDs = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, + primaryStorageNameLabel); + if (morPrimaryDs == null) { + String msg = "Unable to find datastore: " + primaryStorageNameLabel; + s_logger.error(msg); + throw new Exception(msg); + } - DatastoreMO primaryDsMo = new DatastoreMO(hyperHost.getContext(), morPrimaryDs); - details = createVolumeFromSnapshot(hyperHost, primaryDsMo, - newVolumeName, accountId, volumeId, secondaryStorageUrl, backedUpSnapshotUuid); - if (details == null) { - success = true; - } - } catch (Throwable e) { - if (e instanceof RemoteException) { - hostService.invalidateServiceContext(context); - } + DatastoreMO primaryDsMo = new DatastoreMO(hyperHost.getContext(), morPrimaryDs); + details = createVolumeFromSnapshot(hyperHost, primaryDsMo, + newVolumeName, accountId, volumeId, secondaryStorageUrl, backedUpSnapshotUuid); + if (details == null) { + success = true; + } + } catch (Throwable e) { + if (e instanceof RemoteException) { + hostService.invalidateServiceContext(context); + } - s_logger.error("Unexpecpted exception ", e); - details = "CreateVolumeFromSnapshotCommand exception: " + StringUtils.getExceptionStackInfo(e); - } + s_logger.error("Unexpecpted exception ", e); + details = "CreateVolumeFromSnapshotCommand exception: " + StringUtils.getExceptionStackInfo(e); + } - return new CreateVolumeFromSnapshotAnswer(cmd, success, details, newVolumeName); - } + return new CreateVolumeFromSnapshotAnswer(cmd, success, details, newVolumeName); + } // templateName: name in secondary storage // templateUuid: will be used at hypervisor layer private void copyTemplateFromSecondaryToPrimary(VmwareHypervisorHost hyperHost, DatastoreMO datastoreMo, String secondaryStorageUrl, - String templatePathAtSecondaryStorage, String templateName, String templateUuid) throws Exception { + String templatePathAtSecondaryStorage, String templateName, String templateUuid) throws Exception { s_logger.info("Executing copyTemplateFromSecondaryToPrimary. secondaryStorage: " - + secondaryStorageUrl + ", templatePathAtSecondaryStorage: " + templatePathAtSecondaryStorage - + ", templateName: " + templateName); + + secondaryStorageUrl + ", templatePathAtSecondaryStorage: " + templatePathAtSecondaryStorage + + ", templateName: " + templateName); String secondaryMountPoint = _mountService.getMountPoint(secondaryStorageUrl); s_logger.info("Secondary storage mount point: " + secondaryMountPoint); String srcOVAFileName = secondaryMountPoint + "/" + templatePathAtSecondaryStorage + - templateName + "." + ImageFormat.OVA.getFileExtension(); + templateName + "." + ImageFormat.OVA.getFileExtension(); String srcFileName = getOVFFilePath(srcOVAFileName); if(srcFileName == null) { @@ -591,8 +606,8 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { VirtualMachineMO vmMo = hyperHost.findVmOnHyperHost(vmName); if(vmMo == null) { String msg = "Failed to import OVA template. secondaryStorage: " - + secondaryStorageUrl + ", templatePathAtSecondaryStorage: " + templatePathAtSecondaryStorage - + ", templateName: " + templateName + ", templateUuid: " + templateUuid; + + secondaryStorageUrl + ", templatePathAtSecondaryStorage: " + templatePathAtSecondaryStorage + + ", templateName: " + templateName + ", templateUuid: " + templateUuid; s_logger.error(msg); throw new Exception(msg); } @@ -609,7 +624,7 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { } private Ternary createTemplateFromVolume(VirtualMachineMO vmMo, long accountId, long templateId, String templateUniqueName, - String secStorageUrl, String volumePath, String workerVmName) throws Exception { + String secStorageUrl, String volumePath, String workerVmName) throws Exception { String secondaryMountPoint = _mountService.getMountPoint(secStorageUrl); String installPath = getTemplateRelativeDirInSecStorage(accountId, templateId); @@ -622,7 +637,7 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { String result = command.execute(); if(result != null) { String msg = "unable to prepare template directory: " - + installPath + ", storage: " + secStorageUrl + ", error msg: " + result; + + installPath + ", storage: " + secStorageUrl + ", error msg: " + result; s_logger.error(msg); throw new Exception(msg); } @@ -645,7 +660,7 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { // 4 MB is the minimum requirement for VM memory in VMware vmMo.cloneFromCurrentSnapshot(workerVmName, 0, 4, volumeDeviceInfo.second(), - VmwareHelper.getDiskDeviceDatastore(volumeDeviceInfo.first())); + VmwareHelper.getDiskDeviceDatastore(volumeDeviceInfo.first())); clonedVm = vmMo.getRunningHost().findVmOnHyperHost(workerVmName); if(clonedVm == null) { String msg = "Unable to create dummy VM to export volume. volume path: " + volumePath; @@ -676,7 +691,7 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { } private Ternary createTemplateFromSnapshot(long accountId, long templateId, String templateUniqueName, - String secStorageUrl, long volumeId, String backedUpSnapshotUuid) throws Exception { + String secStorageUrl, long volumeId, String backedUpSnapshotUuid) throws Exception { String secondaryMountPoint = _mountService.getMountPoint(secStorageUrl); String installPath = getTemplateRelativeDirInSecStorage(accountId, templateId); @@ -701,7 +716,7 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { result = command.execute(); if(result != null) { String msg = "unable to prepare template directory: " - + installPath + ", storage: " + secStorageUrl + ", error msg: " + result; + + installPath + ", storage: " + secStorageUrl + ", error msg: " + result; s_logger.error(msg); throw new Exception(msg); } @@ -714,9 +729,9 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { command.add(installFullOVAName); result = command.execute(); if(result != null) { - String msg = "unable to copy snapshot " + snapshotFullOVAName + " to " + installFullPath; - s_logger.error(msg); - throw new Exception(msg); + String msg = "unable to copy snapshot " + snapshotFullOVAName + " to " + installFullPath; + s_logger.error(msg); + throw new Exception(msg); } // untar OVA file at template directory @@ -727,40 +742,40 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { s_logger.info("Executing command: " + command.toString()); result = command.execute(); if(result != null) { - String msg = "unable to untar snapshot " + snapshotFullOVAName + " to " - + installFullPath; - s_logger.error(msg); - throw new Exception(msg); + String msg = "unable to untar snapshot " + snapshotFullOVAName + " to " + + installFullPath; + s_logger.error(msg); + throw new Exception(msg); } } else { // there is no ova file, only ovf originally; if(new File(snapshotFullOvfName).exists()) { - command = new Script(false, "cp", _timeout, s_logger); - command.add(snapshotFullOvfName); - //command.add(installFullOvfName); - command.add(installFullPath); - result = command.execute(); - if(result != null) { + command = new Script(false, "cp", _timeout, s_logger); + command.add(snapshotFullOvfName); + //command.add(installFullOvfName); + command.add(installFullPath); + result = command.execute(); + if(result != null) { String msg = "unable to copy snapshot " + snapshotFullOvfName + " to " + installFullPath; s_logger.error(msg); throw new Exception(msg); - } + } - s_logger.info("vmdkfile parent dir: " + snapshotFullVMDKName); - File snapshotdir = new File(snapshotFullVMDKName); - // File snapshotdir = new File(snapshotRoot); - File[] ssfiles = snapshotdir.listFiles(); - // List filenames = new ArrayList(); - for (int i = 0; i < ssfiles.length; i++) { - String vmdkfile = ssfiles[i].getName(); - s_logger.info("vmdk file name: " + vmdkfile); - if(vmdkfile.toLowerCase().startsWith(backupSSUuid) && vmdkfile.toLowerCase().endsWith(".vmdk")) { - snapshotFullVMDKName += vmdkfile; - templateVMDKName += vmdkfile; - break; - } - } - if (snapshotFullVMDKName != null) { + s_logger.info("vmdkfile parent dir: " + snapshotFullVMDKName); + File snapshotdir = new File(snapshotFullVMDKName); + // File snapshotdir = new File(snapshotRoot); + File[] ssfiles = snapshotdir.listFiles(); + // List filenames = new ArrayList(); + for (int i = 0; i < ssfiles.length; i++) { + String vmdkfile = ssfiles[i].getName(); + s_logger.info("vmdk file name: " + vmdkfile); + if(vmdkfile.toLowerCase().startsWith(backupSSUuid) && vmdkfile.toLowerCase().endsWith(".vmdk")) { + snapshotFullVMDKName += vmdkfile; + templateVMDKName += vmdkfile; + break; + } + } + if (snapshotFullVMDKName != null) { command = new Script(false, "cp", _timeout, s_logger); command.add(snapshotFullVMDKName); command.add(installFullPath); @@ -771,17 +786,17 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { s_logger.error(msg); throw new Exception(msg); } - } - } else { - String msg = "unable to find any snapshot ova/ovf files" + snapshotFullOVAName + " to " + installFullPath; - s_logger.error(msg); - throw new Exception(msg); - } + } + } else { + String msg = "unable to find any snapshot ova/ovf files" + snapshotFullOVAName + " to " + installFullPath; + s_logger.error(msg); + throw new Exception(msg); + } } long physicalSize = new File(installFullPath + "/" + templateVMDKName).length(); VmdkProcessor processor = new VmdkProcessor(); - // long physicalSize = new File(installFullPath + "/" + templateUniqueName + ".ova").length(); + // long physicalSize = new File(installFullPath + "/" + templateUniqueName + ".ova").length(); Map params = new HashMap(); params.put(StorageLayer.InstanceConfigKey, _storage); processor.configure("VMDK Processor", params); @@ -797,7 +812,7 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { } private void postCreatePrivateTemplate(String installFullPath, long templateId, - String templateName, long size, long virtualSize) throws Exception { + String templateName, long size, long virtualSize) throws Exception { // TODO a bit ugly here BufferedWriter out = null; @@ -831,50 +846,52 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { out.write("ova.size=" + size); out.newLine(); } finally { - if(out != null) + if(out != null) { out.close(); + } } } private void writeMetaOvaForTemplate(String installFullPath, String ovfFilename, String vmdkFilename, String templateName, long diskSize) throws Exception { - // TODO a bit ugly here - BufferedWriter out = null; - try { - out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(installFullPath + "/" + templateName +".ova.meta"))); - out.write("ova.filename=" + templateName + ".ova"); - out.newLine(); - out.write("version=1.0"); - out.newLine(); - out.write("ovf=" + ovfFilename); - out.newLine(); - out.write("numDisks=1"); - out.newLine(); - out.write("disk1.name=" + vmdkFilename); - out.newLine(); - out.write("disk1.size=" + diskSize); - out.newLine(); - } finally { - if(out != null) - out.close(); + // TODO a bit ugly here + BufferedWriter out = null; + try { + out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(installFullPath + "/" + templateName +".ova.meta"))); + out.write("ova.filename=" + templateName + ".ova"); + out.newLine(); + out.write("version=1.0"); + out.newLine(); + out.write("ovf=" + ovfFilename); + out.newLine(); + out.write("numDisks=1"); + out.newLine(); + out.write("disk1.name=" + vmdkFilename); + out.newLine(); + out.write("disk1.size=" + diskSize); + out.newLine(); + } finally { + if(out != null) { + out.close(); } - } + } + } private String createVolumeFromSnapshot(VmwareHypervisorHost hyperHost, DatastoreMO primaryDsMo, String newVolumeName, - long accountId, long volumeId, String secStorageUrl, String snapshotBackupUuid) throws Exception { + long accountId, long volumeId, String secStorageUrl, String snapshotBackupUuid) throws Exception { restoreVolumeFromSecStorage(hyperHost, primaryDsMo, newVolumeName, - secStorageUrl, getSnapshotRelativeDirInSecStorage(accountId, volumeId), snapshotBackupUuid); + secStorageUrl, getSnapshotRelativeDirInSecStorage(accountId, volumeId), snapshotBackupUuid); return null; } private void restoreVolumeFromSecStorage(VmwareHypervisorHost hyperHost, DatastoreMO primaryDsMo, String newVolumeName, - String secStorageUrl, String secStorageDir, String backupName) throws Exception { + String secStorageUrl, String secStorageDir, String backupName) throws Exception { String secondaryMountPoint = _mountService.getMountPoint(secStorageUrl); String srcOVAFileName = secondaryMountPoint + "/" + secStorageDir + "/" - + backupName + "." + ImageFormat.OVA.getFileExtension(); + + backupName + "." + ImageFormat.OVA.getFileExtension(); String snapshotDir = ""; if (backupName.contains("/")){ snapshotDir = backupName.split("/")[0]; @@ -895,17 +912,17 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { s_logger.info("Executing command: " + command.toString()); String result = command.execute(); if(result != null) { - String msg = "Unable to unpack snapshot OVA file at: " + srcOVAFileName; - s_logger.error(msg); - throw new Exception(msg); + String msg = "Unable to unpack snapshot OVA file at: " + srcOVAFileName; + s_logger.error(msg); + throw new Exception(msg); } } else { - String msg = "Unable to find snapshot OVA file at: " + srcOVAFileName; - s_logger.error(msg); - throw new Exception(msg); - } + String msg = "Unable to find snapshot OVA file at: " + srcOVAFileName; + s_logger.error(msg); + throw new Exception(msg); + } - srcOVFFileName = getOVFFilePath(srcOVAFileName); + srcOVFFileName = getOVFFilePath(srcOVAFileName); } if(srcOVFFileName == null) { String msg = "Unable to locate OVF file in template package directory: " + srcOVAFileName; @@ -917,8 +934,9 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { try { hyperHost.importVmFromOVF(srcOVFFileName, newVolumeName, primaryDsMo, "thin"); clonedVm = hyperHost.findVmOnHyperHost(newVolumeName); - if(clonedVm == null) + if(clonedVm == null) { throw new Exception("Unable to create container VM for volume creation"); + } clonedVm.moveAllVmDiskFiles(primaryDsMo, "", false); clonedVm.detachAllDisks(); @@ -931,29 +949,30 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { } private String backupSnapshotToSecondaryStorage(VirtualMachineMO vmMo, long accountId, long volumeId, - String volumePath, String snapshotUuid, String secStorageUrl, - String prevSnapshotUuid, String prevBackupUuid, String workerVmName) throws Exception { + String volumePath, String snapshotUuid, String secStorageUrl, + String prevSnapshotUuid, String prevBackupUuid, String workerVmName) throws Exception { String backupUuid = UUID.randomUUID().toString(); exportVolumeToSecondaryStroage(vmMo, volumePath, secStorageUrl, - getSnapshotRelativeDirInSecStorage(accountId, volumeId), backupUuid, workerVmName); + getSnapshotRelativeDirInSecStorage(accountId, volumeId), backupUuid, workerVmName); return backupUuid + "/" + backupUuid; } private void exportVolumeToSecondaryStroage(VirtualMachineMO vmMo, String volumePath, - String secStorageUrl, String secStorageDir, String exportName, - String workerVmName) throws Exception { + String secStorageUrl, String secStorageDir, String exportName, + String workerVmName) throws Exception { String secondaryMountPoint = _mountService.getMountPoint(secStorageUrl); String exportPath = secondaryMountPoint + "/" + secStorageDir + "/" + exportName; - + synchronized(exportPath.intern()) { if(!new File(exportPath).exists()) { Script command = new Script(false, "mkdir", _timeout, s_logger); command.add("-p"); command.add(exportPath); - if(command.execute() != null) + if(command.execute() != null) { throw new Exception("unable to prepare snapshot backup directory"); + } } } @@ -969,7 +988,7 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { // 4 MB is the minimum requirement for VM memory in VMware vmMo.cloneFromCurrentSnapshot(workerVmName, 0, 4, volumeDeviceInfo.second(), - VmwareHelper.getDiskDeviceDatastore(volumeDeviceInfo.first())); + VmwareHelper.getDiskDeviceDatastore(volumeDeviceInfo.first())); clonedVm = vmMo.getRunningHost().findVmOnHyperHost(workerVmName); if(clonedVm == null) { String msg = "Unable to create dummy VM to export volume. volume path: " + volumePath; @@ -992,33 +1011,35 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { String snapshotMountRoot = secondaryMountPoint + "/" + getSnapshotRelativeDirInSecStorage(accountId, volumeId); File file = new File(snapshotMountRoot + "/" + backupUuid + ".ovf"); if(file.exists()) { - File snapshotdir = new File(snapshotMountRoot); - File[] ssfiles = snapshotdir.listFiles(); + File snapshotdir = new File(snapshotMountRoot); + File[] ssfiles = snapshotdir.listFiles(); // List filenames = new ArrayList(); - for (int i = 0; i < ssfiles.length; i++) { + for (int i = 0; i < ssfiles.length; i++) { String vmdkfile = ssfiles[i].getName(); if(vmdkfile.toLowerCase().startsWith(backupUuid) && vmdkfile.toLowerCase().endsWith(".vmdk")) { - // filenames.add(vmdkfile); - new File(vmdkfile).delete(); + // filenames.add(vmdkfile); + new File(vmdkfile).delete(); } - } - if(file.delete()) - return null; - } else { - File file1 = new File(snapshotMountRoot + "/" + backupUuid + ".ova"); - if(file1.exists()) { - if(file1.delete()) - return null; - } else { - return "Backup file does not exist. backupUuid: " + backupUuid; - } - } - return "Failed to delete snapshot backup file, backupUuid: " + backupUuid; - } + } + if(file.delete()) { + return null; + } + } else { + File file1 = new File(snapshotMountRoot + "/" + backupUuid + ".ova"); + if(file1.exists()) { + if(file1.delete()) { + return null; + } + } else { + return "Backup file does not exist. backupUuid: " + backupUuid; + } + } + return "Failed to delete snapshot backup file, backupUuid: " + backupUuid; + } private Pair copyVolumeToSecStorage(VmwareHostService hostService, VmwareHypervisorHost hyperHost, CopyVolumeCommand cmd, - String vmName, long volumeId, String poolId, String volumePath, - String secStorageUrl, String workerVmName) throws Exception { + String vmName, long volumeId, String poolId, String volumePath, + String secStorageUrl, String workerVmName) throws Exception { String volumeFolder = String.valueOf(volumeId) + "/"; VirtualMachineMO workerVm=null; @@ -1075,7 +1096,7 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { vmMo.createSnapshot(exportName, "Temporary snapshot for copy-volume command", false, false); exportVolumeToSecondaryStroage(vmMo, volumePath, secStorageUrl, "volumes/" + volumeFolder, exportName, - hostService.getWorkerName(hyperHost.getContext(), cmd, 1)); + hostService.getWorkerName(hyperHost.getContext(), cmd, 1)); return new Pair(volumeFolder, exportName); } finally { @@ -1095,7 +1116,7 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { } private Pair copyVolumeFromSecStorage(VmwareHypervisorHost hyperHost, long volumeId, - DatastoreMO dsMo, String secStorageUrl, String exportName) throws Exception { + DatastoreMO dsMo, String secStorageUrl, String exportName) throws Exception { String volumeFolder = String.valueOf(volumeId) + "/"; String newVolume = UUID.randomUUID().toString().replaceAll("-", ""); @@ -1104,91 +1125,91 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { return new Pair(volumeFolder, newVolume); } - //Fang: here I use a method to return the ovf and vmdk file names; Another way to do it: + // here we use a method to return the ovf and vmdk file names; Another way to do it: // create a new class, and like TemplateLocation.java and create templateOvfInfo.java to handle it; - private String getOVAFromMetafile(String metafileName) throws Exception { + private String getOVAFromMetafile(String metafileName) throws Exception { File ova_metafile = new File(metafileName); Properties props = null; FileInputStream strm = null; String ovaFileName = ""; - s_logger.info("Fang: getOVAfromMetaFile: metafileName " + metafileName); - try { - strm = new FileInputStream(ova_metafile); - if (null == strm) { - String msg = "Cannot read ova meat file. Error"; - s_logger.error(msg); - throw new Exception(msg); + s_logger.info("getOVAfromMetaFile: " + metafileName); + try { + strm = new FileInputStream(ova_metafile); + if (null == strm) { + String msg = "Cannot read ova meta file."; + s_logger.error(msg); + throw new Exception(msg); + } + + s_logger.info("loading properties from ova meta file: " + metafileName); + if (null != ova_metafile) { + props = new Properties(); + props.load(strm); + if (props == null) { + s_logger.info("getOVAfromMetaFile: props is null. "); + } + } + if (null != props) { + ovaFileName = props.getProperty("ova.filename"); + s_logger.info("ovafilename: " + ovaFileName); + String ovfFileName = props.getProperty("ovf"); + s_logger.info("ovffilename: " + ovfFileName); + int diskNum = Integer.parseInt(props.getProperty("numDisks")); + if (diskNum <= 0) { + String msg = "VMDK disk file number is 0. Error"; + s_logger.error(msg); + throw new Exception(msg); + } + String[] disks = new String[diskNum]; + for (int i = 0; i < diskNum; i++) { + // String diskNameKey = "disk" + Integer.toString(i+1) + ".name"; // Fang use this + String diskNameKey = "disk1.name"; + disks[i] = props.getProperty(diskNameKey); + s_logger.info("diskname " + disks[i]); + } + String exportDir = ova_metafile.getParent(); + s_logger.info("exportDir: " + exportDir); + // Important! we need to sync file system before we can safely use tar to work around a linux kernal bug(or feature) + s_logger.info("Sync file system before we package OVA..., before tar "); + s_logger.info("ova: " + ovaFileName + ", ovf:" + ovfFileName + ", vmdk:" + disks[0] + "."); + Script commandSync = new Script(true, "sync", 0, s_logger); + commandSync.execute(); + Script command = new Script(false, "tar", 0, s_logger); + command.setWorkDir(exportDir); // Fang: pass this in to the method? + command.add("-cf", ovaFileName); + command.add(ovfFileName); // OVF file should be the first file in OVA archive + for (String diskName : disks) { + command.add(diskName); + } + command.execute(); + s_logger.info("Package OVA for template in dir: " + exportDir + "cmd: " + command.toString()); + // to be safe, physically test existence of the target OVA file + if ((new File(exportDir + ovaFileName)).exists()) { + s_logger.info("ova file is created and ready to extract "); + return (ovaFileName); + } else { + String msg = exportDir + File.separator + ovaFileName + ".ova is not created as expected"; + s_logger.error(msg); + throw new Exception(msg); + } + } else { + String msg = "Error reading the ova meta file: " + metafileName; + s_logger.error(msg); + throw new Exception(msg); + } + } catch (Exception e) { + return null; + // Do something, re-throw the exception + } finally { + if (strm != null) { + try { + strm.close(); + } catch (Exception e) { + } + } } - s_logger.info("Fang: getOVAfromMetaFile: load strm " ); - if (null != ova_metafile) { - props = new Properties(); - props.load(strm); - if (props == null) { - s_logger.info("Fang: getOVAfromMetaFile: props is null. " ); - } - } - if (null != props) { - ovaFileName = props.getProperty("ova.filename"); - s_logger.info("Fang: ovafilename" + ovaFileName); - String ovfFileName = props.getProperty("ovf"); - s_logger.info("Fang: ovffilename" + ovfFileName); - int diskNum = Integer.parseInt(props.getProperty("numDisks")); - if (diskNum <= 0) { - String msg = "VMDK disk file number is 0. Error"; - s_logger.error(msg); - throw new Exception(msg); - } - String[] disks = new String[diskNum]; - for (int i = 0; i < diskNum; i++) { - //String diskNameKey = "disk" + Integer.toString(i+1) + ".name"; // Fang use this - String diskNameKey = "disk1.name"; - disks[i] = props.getProperty(diskNameKey); - s_logger.info("Fang: diskname " + disks[i]); - } - String exportDir = ova_metafile.getParent(); - s_logger.info("Fang: exportDir: " + exportDir); - // Important! we need to sync file system before we can safely use tar to work around a linux kernal bug(or feature) - s_logger.info("Fang: Sync file system before we package OVA..., before tar "); - s_logger.info("Fang: ova: " + ovaFileName+ ", ovf:" + ovfFileName + ", vmdk:" + disks[0] + "."); - Script commandSync = new Script(true, "sync", 0, s_logger); - commandSync.execute(); - Script command = new Script(false, "tar", 0, s_logger); - command.setWorkDir(exportDir); //Fang: pass this in to the method? - command.add("-cf", ovaFileName); - command.add(ovfFileName); // OVF file should be the first file in OVA archive - for(String diskName: disks) { - command.add(diskName); - } - command.execute(); - s_logger.info("Fang: Package OVA for template in dir: " + exportDir + "cmd: " + command.toString()); - // to be safe, physically test existence of the target OVA file - if((new File(exportDir + ovaFileName)).exists()) { - s_logger.info("Fang: ova file is created and ready to extract "); - return (ovaFileName); - } else { - String msg = exportDir + File.separator + ovaFileName + ".ova is not created as expected"; - s_logger.error(msg); - throw new Exception(msg); - } - } else { - String msg = "Error reading the ova meta file: " + metafileName; - s_logger.error(msg); - throw new Exception(msg); - } - } catch (Exception e) { - return null; - //Do something, re-throw the exception - } finally { - if (strm != null) { - try { - strm.close(); - } catch (Exception e) { - } - } - } - - } + } private String getOVFFilePath(String srcOVAFileName) { File file = new File(srcOVAFileName); @@ -1213,9 +1234,9 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { return "snapshots/" + accountId + "/" + volumeId; } - private long getVMSnapshotChainSize(VmwareContext context, VmwareHypervisorHost hyperHost, - String fileName, String poolUuid, String exceptFileName) - throws Exception{ + private long getVMSnapshotChainSize(VmwareContext context, VmwareHypervisorHost hyperHost, + String fileName, String poolUuid, String exceptFileName) + throws Exception{ long size = 0; ManagedObjectReference morDs = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, poolUuid); DatastoreMO dsMo = new DatastoreMO(context, morDs); @@ -1235,10 +1256,11 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { if (result != null) { List info = result.getFile(); for (FileInfo fi : info) { - if(exceptFileName != null && fi.getPath().contains(exceptFileName)) + if(exceptFileName != null && fi.getPath().contains(exceptFileName)) { continue; - else + } else { size = size + fi.getFileSize(); + } } } } @@ -1246,22 +1268,30 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { } private String extractSnapshotBaseFileName(String input) { - if(input == null) + if(input == null) { return null; + } String result = input; if (result.endsWith(".vmdk")){ // get rid of vmdk file extension result = result.substring(0, result.length() - (".vmdk").length()); } - if(result.split("-").length == 1) // e.g 4da6dcbd412c47b59f96c7ff6dbd7216.vmdk + if(result.contains("/")) + { + result = result.substring(result.indexOf ("/") + 1,result.length()); // remove VM name path + } + if(result.contains("ROOT") && result.split("-").length > 1) { + return "ROOT"; + } + if(result.split("-").length == 1) { return result; - if(result.split("-").length > 2) // e.g ROOT-5-4.vmdk, ROOT-5-4-000001.vmdk - return result.split("-")[0] + "-" + result.split("-")[1]; - if(result.split("-").length == 2) // e.g 4da6dcbd412c47b59f96c7ff6dbd7216-000001.vmdk + } + if(result.split("-").length == 2) { return result.split("-")[0]; - else + } else { return result; + } } - + @Override public CreateVMSnapshotAnswer execute(VmwareHostService hostService, CreateVMSnapshotCommand cmd) { List volumeTOs = cmd.getVolumeTOs(); @@ -1287,8 +1317,9 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { } vmMo = hyperHost.findVmOnHyperHost(vmName); - if(vmMo == null) + if(vmMo == null) { vmMo = hyperHost.findVmOnPeerHyperHost(vmName); + } if (vmMo == null) { String msg = "Unable to find VM for CreateVMSnapshotCommand"; s_logger.debug(msg); @@ -1309,6 +1340,9 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { if (vmdkName.endsWith(".vmdk")){ vmdkName = vmdkName.substring(0, vmdkName.length() - (".vmdk").length()); } + if(vmdkName.contains("/")) { + vmdkName = vmdkName.substring(vmdkName.indexOf ("/") + 1,vmdkName.length()); + } String baseName = extractSnapshotBaseFileName(vmdkName); mapNewDisk.put(baseName, vmdkName); } @@ -1319,12 +1353,12 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { // get volume's chain size for this VM snapshot, exclude current volume vdisk long size = getVMSnapshotChainSize(context,hyperHost,baseName + "*.vmdk", volumeTO.getPoolUuid(), newPath); - + if(volumeTO.getType()== Volume.Type.ROOT){ // add memory snapshot size size = size + getVMSnapshotChainSize(context,hyperHost,cmd.getVmName()+"*.vmsn",volumeTO.getPoolUuid(),null); } - + volumeTO.setChainSize(size); volumeTO.setPath(newPath); } @@ -1343,7 +1377,7 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { } } - @Override + @Override public DeleteVMSnapshotAnswer execute(VmwareHostService hostService, DeleteVMSnapshotCommand cmd) { List listVolumeTo = cmd.getVolumeTOs(); VirtualMachineMO vmMo = null; @@ -1354,10 +1388,11 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { try { VmwareHypervisorHost hyperHost = hostService.getHyperHost(context, cmd); vmMo = hyperHost.findVmOnHyperHost(vmName); - if(vmMo == null) + if(vmMo == null) { vmMo = hyperHost.findVmOnPeerHyperHost(vmName); + } if (vmMo == null) { - String msg = "Unable to find VM for RevertToVMSnapshotCommand"; + String msg = "Unable to find VM for DeleteVMSnapshotCommand"; s_logger.debug(msg); return new DeleteVMSnapshotAnswer(cmd, false, msg); } else { @@ -1381,6 +1416,9 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { if (vmdkName.endsWith(".vmdk")) { vmdkName = vmdkName.substring(0, vmdkName.length() - (".vmdk").length()); } + if(vmdkName.contains("/")) { + vmdkName = vmdkName.substring(vmdkName.indexOf ("/") + 1,vmdkName.length()); + } String baseName = extractSnapshotBaseFileName(vmdkName); mapNewDisk.put(baseName, vmdkName); } @@ -1393,7 +1431,7 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { if(volumeTo.getType()== Volume.Type.ROOT){ // add memory snapshot size size = size + getVMSnapshotChainSize(context,hyperHost,cmd.getVmName()+"*.vmsn",volumeTo.getPoolUuid(),null); - } + } volumeTo.setChainSize(size); volumeTo.setPath(newPath); } @@ -1406,7 +1444,7 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { } } - @Override + @Override public RevertToVMSnapshotAnswer execute(VmwareHostService hostService, RevertToVMSnapshotCommand cmd) { String snapshotName = cmd.getTarget().getSnapshotName(); String vmName = cmd.getVmName(); @@ -1432,8 +1470,9 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { HostMO hostMo = (HostMO) hyperHost; vmMo = hyperHost.findVmOnHyperHost(vmName); - if(vmMo == null) + if(vmMo == null) { vmMo = hyperHost.findVmOnPeerHyperHost(vmName); + } if (vmMo == null) { String msg = "Unable to find VM for RevertToVMSnapshotCommand"; s_logger.debug(msg); @@ -1459,16 +1498,17 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { if (vmdkName.endsWith(".vmdk")) { vmdkName = vmdkName.substring(0, vmdkName.length() - (".vmdk").length()); } - String[] s = vmdkName.split("-"); - mapNewDisk.put(s[0], vmdkName); + if(vmdkName.contains("/")) { + vmdkName = vmdkName.substring(vmdkName.indexOf ("/") + 1,vmdkName.length()); + } + String baseName = extractSnapshotBaseFileName(vmdkName); + mapNewDisk.put(baseName, vmdkName); } } - String key = null; for (VolumeTO volumeTo : listVolumeTo) { - String parentUUID = volumeTo.getPath(); - String[] s = parentUUID.split("-"); - key = s[0]; - volumeTo.setPath(mapNewDisk.get(key)); + String baseName = extractSnapshotBaseFileName(volumeTo.getPath()); + String newPath = mapNewDisk.get(baseName); + volumeTo.setPath(newPath); } if (!snapshotMemory) { vmState = VirtualMachine.State.Stopped; @@ -1486,35 +1526,6 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { } } - - private VirtualMachineMO createWorkingVM(DatastoreMO dsMo, VmwareHypervisorHost hyperHost) throws Exception { - String uniqueName = UUID.randomUUID().toString(); - VirtualMachineMO workingVM = null; - VirtualMachineConfigSpec vmConfig = new VirtualMachineConfigSpec(); - vmConfig.setName(uniqueName); - vmConfig.setMemoryMB((long) 4); - vmConfig.setNumCPUs(1); - vmConfig.setGuestId(VirtualMachineGuestOsIdentifier.OTHER_GUEST.toString()); - VirtualMachineFileInfo fileInfo = new VirtualMachineFileInfo(); - fileInfo.setVmPathName(String.format("[%s]", dsMo.getName())); - vmConfig.setFiles(fileInfo); - - VirtualLsiLogicController scsiController = new VirtualLsiLogicController(); - scsiController.setSharedBus(VirtualSCSISharing.NO_SHARING); - scsiController.setBusNumber(0); - scsiController.setKey(1); - VirtualDeviceConfigSpec scsiControllerSpec = new VirtualDeviceConfigSpec(); - scsiControllerSpec.setDevice(scsiController); - scsiControllerSpec.setOperation(VirtualDeviceConfigSpecOperation.ADD); - - vmConfig.getDeviceChange().add(scsiControllerSpec); - hyperHost.createVm(vmConfig); - workingVM = hyperHost.findVmOnHyperHost(uniqueName); - return workingVM; - } - - - private String deleteVolumeDirOnSecondaryStorage(long volumeId, String secStorageUrl) throws Exception { String secondaryMountPoint = _mountService.getMountPoint(secStorageUrl); String volumeMountRoot = secondaryMountPoint + "/" + getVolumeRelativeDirInSecStroage(volumeId); diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareContextFactory.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareContextFactory.java index 5db9da3c02d..c0b716cc394 100755 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareContextFactory.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareContextFactory.java @@ -25,22 +25,23 @@ import org.springframework.stereotype.Component; import com.cloud.hypervisor.vmware.manager.VmwareManager; import com.cloud.hypervisor.vmware.util.VmwareClient; import com.cloud.hypervisor.vmware.util.VmwareContext; +import com.cloud.hypervisor.vmware.util.VmwareContextPool; import com.cloud.utils.StringUtils; @Component public class VmwareContextFactory { - private static final Logger s_logger = Logger.getLogger(VmwareContextFactory.class); private static volatile int s_seq = 1; private static VmwareManager s_vmwareMgr; + private static VmwareContextPool s_pool; @Inject VmwareManager _vmwareMgr; static { // skip certificate check System.setProperty("axis.socketSecureFactory", "org.apache.axis.components.net.SunFakeTrustSocketFactory"); - //s_vmwareMgr = ComponentContext.inject(VmwareManagerImpl.class); + s_pool = new VmwareContextPool(); } @PostConstruct @@ -54,8 +55,6 @@ public class VmwareContextFactory { assert(vCenterPassword != null); String serviceUrl = "https://" + vCenterAddress + "/sdk/vimService"; - //String[] params = new String[] {"--url", serviceUrl, "--username", vCenterUserName, "--password", vCenterPassword }; - if(s_logger.isDebugEnabled()) s_logger.debug("initialize VmwareContext. url: " + serviceUrl + ", username: " + vCenterUserName + ", password: " + StringUtils.getMaskedPasswordForDisplay(vCenterPassword)); @@ -68,6 +67,24 @@ public class VmwareContextFactory { context.registerStockObject("serviceconsole", s_vmwareMgr.getServiceConsolePortGroupName()); context.registerStockObject("manageportgroup", s_vmwareMgr.getManagementPortGroupName()); + context.setPoolInfo(s_pool, VmwareContextPool.composePoolKey(vCenterAddress, vCenterUserName)); + s_pool.registerOutstandingContext(context); + + return context; + } + + public static VmwareContext getContext(String vCenterAddress, String vCenterUserName, String vCenterPassword) throws Exception { + VmwareContext context = s_pool.getContext(vCenterAddress, vCenterUserName); + if(context == null) + context = create(vCenterAddress, vCenterUserName, vCenterPassword); + + if(context != null) { + context.registerStockObject(VmwareManager.CONTEXT_STOCK_NAME, s_vmwareMgr); + + context.registerStockObject("serviceconsole", s_vmwareMgr.getServiceConsolePortGroupName()); + context.registerStockObject("manageportgroup", s_vmwareMgr.getManagementPortGroupName()); + } + return context; } } diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java index a33b94e9ff0..0990b3afb9f 100755 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -28,7 +28,6 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Comparator; -import java.util.concurrent.*; import java.util.Date; import java.util.GregorianCalendar; import java.util.HashMap; @@ -40,74 +39,21 @@ import java.util.Random; import java.util.Set; import java.util.TimeZone; import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import javax.inject.Inject; import javax.naming.ConfigurationException; -import com.cloud.agent.api.to.DhcpTO; +import org.apache.cloudstack.storage.command.DeleteCommand; +import org.apache.cloudstack.storage.command.StorageSubSystemCommand; +import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; +import org.apache.cloudstack.storage.to.TemplateObjectTO; +import org.apache.cloudstack.storage.to.VolumeObjectTO; import org.apache.log4j.Logger; import org.apache.log4j.NDC; -import com.google.gson.Gson; -import com.vmware.vim25.AboutInfo; -import com.vmware.vim25.BoolPolicy; -import com.vmware.vim25.ClusterDasConfigInfo; -import com.vmware.vim25.ComputeResourceSummary; -import com.vmware.vim25.DVPortConfigInfo; -import com.vmware.vim25.DVPortConfigSpec; -import com.vmware.vim25.DatastoreSummary; -import com.vmware.vim25.DistributedVirtualPort; -import com.vmware.vim25.DistributedVirtualSwitchPortConnection; -import com.vmware.vim25.DistributedVirtualSwitchPortCriteria; -import com.vmware.vim25.DynamicProperty; -import com.vmware.vim25.GuestInfo; -import com.vmware.vim25.HostCapability; -import com.vmware.vim25.HostFirewallInfo; -import com.vmware.vim25.HostFirewallRuleset; -import com.vmware.vim25.HostHostBusAdapter; -import com.vmware.vim25.HostInternetScsiHba; -import com.vmware.vim25.HostInternetScsiHbaAuthenticationProperties; -import com.vmware.vim25.HostInternetScsiHbaStaticTarget; -import com.vmware.vim25.HostInternetScsiTargetTransport; -import com.vmware.vim25.HostScsiDisk; -import com.vmware.vim25.HostScsiTopology; -import com.vmware.vim25.HostScsiTopologyInterface; -import com.vmware.vim25.HostScsiTopologyLun; -import com.vmware.vim25.HostScsiTopologyTarget; -import com.vmware.vim25.ManagedObjectReference; -import com.vmware.vim25.ObjectContent; -import com.vmware.vim25.OptionValue; -import com.vmware.vim25.PerfCounterInfo; -import com.vmware.vim25.PerfEntityMetric; -import com.vmware.vim25.PerfEntityMetricBase; -import com.vmware.vim25.PerfMetricId; -import com.vmware.vim25.PerfMetricIntSeries; -import com.vmware.vim25.PerfMetricSeries; -import com.vmware.vim25.PerfQuerySpec; -import com.vmware.vim25.PerfSampleInfo; -import com.vmware.vim25.RuntimeFaultFaultMsg; -import com.vmware.vim25.ToolsUnavailableFaultMsg; -import com.vmware.vim25.VMwareDVSPortSetting; -import com.vmware.vim25.VimPortType; -import com.vmware.vim25.VirtualDevice; -import com.vmware.vim25.VirtualDeviceBackingInfo; -import com.vmware.vim25.VirtualDeviceConfigSpec; -import com.vmware.vim25.VirtualDeviceConfigSpecOperation; -import com.vmware.vim25.VirtualDisk; -import com.vmware.vim25.VirtualEthernetCard; -import com.vmware.vim25.VirtualEthernetCardDistributedVirtualPortBackingInfo; -import com.vmware.vim25.VirtualEthernetCardNetworkBackingInfo; -import com.vmware.vim25.VirtualLsiLogicController; -import com.vmware.vim25.VirtualMachineConfigSpec; -import com.vmware.vim25.VirtualMachineFileInfo; -import com.vmware.vim25.VirtualMachineGuestOsIdentifier; -import com.vmware.vim25.VirtualMachinePowerState; -import com.vmware.vim25.VirtualMachineRelocateSpec; -import com.vmware.vim25.VirtualMachineRelocateSpecDiskLocator; -import com.vmware.vim25.VirtualMachineRuntimeInfo; -import com.vmware.vim25.VirtualSCSISharing; -import com.vmware.vim25.VmwareDistributedVirtualSwitchVlanIdSpec; - import com.cloud.agent.IAgentControl; import com.cloud.agent.api.Answer; import com.cloud.agent.api.AttachIsoCommand; @@ -145,6 +91,8 @@ import com.cloud.agent.api.GetHostStatsAnswer; import com.cloud.agent.api.GetHostStatsCommand; import com.cloud.agent.api.GetStorageStatsAnswer; import com.cloud.agent.api.GetStorageStatsCommand; +import com.cloud.agent.api.GetVmDiskStatsAnswer; +import com.cloud.agent.api.GetVmDiskStatsCommand; import com.cloud.agent.api.GetVmStatsAnswer; import com.cloud.agent.api.GetVmStatsCommand; import com.cloud.agent.api.GetVncPortAnswer; @@ -236,18 +184,15 @@ import com.cloud.agent.api.storage.CopyVolumeCommand; import com.cloud.agent.api.storage.CreateAnswer; import com.cloud.agent.api.storage.CreateCommand; import com.cloud.agent.api.storage.CreatePrivateTemplateAnswer; -import com.cloud.agent.api.storage.CreateVolumeOVAAnswer; -import com.cloud.agent.api.storage.CreateVolumeOVACommand; import com.cloud.agent.api.storage.DestroyCommand; import com.cloud.agent.api.storage.MigrateVolumeAnswer; import com.cloud.agent.api.storage.MigrateVolumeCommand; -import com.cloud.agent.api.storage.PrepareOVAPackingAnswer; -import com.cloud.agent.api.storage.PrepareOVAPackingCommand; import com.cloud.agent.api.storage.PrimaryStorageDownloadAnswer; import com.cloud.agent.api.storage.PrimaryStorageDownloadCommand; import com.cloud.agent.api.storage.ResizeVolumeAnswer; import com.cloud.agent.api.storage.ResizeVolumeCommand; import com.cloud.agent.api.to.DataStoreTO; +import com.cloud.agent.api.to.DhcpTO; import com.cloud.agent.api.to.DiskTO; import com.cloud.agent.api.to.FirewallRuleTO; import com.cloud.agent.api.to.IpAddressTO; @@ -258,6 +203,7 @@ import com.cloud.agent.api.to.StaticNatRuleTO; import com.cloud.agent.api.to.StorageFilerTO; import com.cloud.agent.api.to.VirtualMachineTO; import com.cloud.agent.api.to.VolumeTO; +import com.cloud.configuration.Config; import com.cloud.dc.DataCenter.NetworkType; import com.cloud.dc.Vlan; import com.cloud.exception.InternalErrorException; @@ -270,16 +216,18 @@ import com.cloud.hypervisor.vmware.mo.ClusterMO; import com.cloud.hypervisor.vmware.mo.CustomFieldConstants; import com.cloud.hypervisor.vmware.mo.CustomFieldsManagerMO; import com.cloud.hypervisor.vmware.mo.DatacenterMO; +import com.cloud.hypervisor.vmware.mo.DatastoreFile; import com.cloud.hypervisor.vmware.mo.DatastoreMO; import com.cloud.hypervisor.vmware.mo.DiskControllerType; import com.cloud.hypervisor.vmware.mo.FeatureKeyConstants; import com.cloud.hypervisor.vmware.mo.HostDatastoreSystemMO; -import com.cloud.hypervisor.vmware.mo.HostFirewallSystemMO; import com.cloud.hypervisor.vmware.mo.HostMO; import com.cloud.hypervisor.vmware.mo.HostStorageSystemMO; import com.cloud.hypervisor.vmware.mo.HypervisorHostHelper; import com.cloud.hypervisor.vmware.mo.NetworkDetails; import com.cloud.hypervisor.vmware.mo.VirtualEthernetCardType; +import com.cloud.hypervisor.vmware.mo.VirtualMachineDiskInfo; +import com.cloud.hypervisor.vmware.mo.VirtualMachineDiskInfoBuilder; import com.cloud.hypervisor.vmware.mo.VirtualMachineMO; import com.cloud.hypervisor.vmware.mo.VirtualSwitchType; import com.cloud.hypervisor.vmware.mo.VmwareHypervisorHost; @@ -304,8 +252,9 @@ import com.cloud.storage.VolumeManager; import com.cloud.storage.VolumeManagerImpl; import com.cloud.storage.resource.StoragePoolResource; import com.cloud.storage.resource.StorageSubsystemCommandHandler; -import com.cloud.storage.resource.StorageSubsystemCommandHandlerBase; +import com.cloud.storage.resource.VmwareStorageLayoutHelper; import com.cloud.storage.resource.VmwareStorageProcessor; +import com.cloud.storage.resource.VmwareStorageSubsystemCommandHandler; import com.cloud.storage.template.TemplateProp; import com.cloud.utils.DateUtil; import com.cloud.utils.NumbersUtil; @@ -324,12 +273,64 @@ import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachine.State; import com.cloud.vm.VirtualMachineName; import com.cloud.vm.VmDetailConstants; - -import org.apache.cloudstack.storage.command.DeleteCommand; -import org.apache.cloudstack.storage.command.StorageSubSystemCommand; -import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; -import org.apache.cloudstack.storage.to.TemplateObjectTO; -import org.apache.cloudstack.storage.to.VolumeObjectTO; +import com.google.gson.Gson; +import com.vmware.vim25.AboutInfo; +import com.vmware.vim25.BoolPolicy; +import com.vmware.vim25.ClusterDasConfigInfo; +import com.vmware.vim25.ComputeResourceSummary; +import com.vmware.vim25.CustomFieldStringValue; +import com.vmware.vim25.DVPortConfigInfo; +import com.vmware.vim25.DVPortConfigSpec; +import com.vmware.vim25.DatastoreSummary; +import com.vmware.vim25.DistributedVirtualPort; +import com.vmware.vim25.DistributedVirtualSwitchPortConnection; +import com.vmware.vim25.DistributedVirtualSwitchPortCriteria; +import com.vmware.vim25.DynamicProperty; +import com.vmware.vim25.GuestInfo; +import com.vmware.vim25.HostCapability; +import com.vmware.vim25.HostHostBusAdapter; +import com.vmware.vim25.HostInternetScsiHba; +import com.vmware.vim25.HostInternetScsiHbaAuthenticationProperties; +import com.vmware.vim25.HostInternetScsiHbaStaticTarget; +import com.vmware.vim25.HostInternetScsiTargetTransport; +import com.vmware.vim25.HostScsiDisk; +import com.vmware.vim25.HostScsiTopology; +import com.vmware.vim25.HostScsiTopologyInterface; +import com.vmware.vim25.HostScsiTopologyLun; +import com.vmware.vim25.HostScsiTopologyTarget; +import com.vmware.vim25.ManagedObjectReference; +import com.vmware.vim25.ObjectContent; +import com.vmware.vim25.OptionValue; +import com.vmware.vim25.PerfCounterInfo; +import com.vmware.vim25.PerfEntityMetric; +import com.vmware.vim25.PerfEntityMetricBase; +import com.vmware.vim25.PerfMetricId; +import com.vmware.vim25.PerfMetricIntSeries; +import com.vmware.vim25.PerfMetricSeries; +import com.vmware.vim25.PerfQuerySpec; +import com.vmware.vim25.PerfSampleInfo; +import com.vmware.vim25.RuntimeFaultFaultMsg; +import com.vmware.vim25.ToolsUnavailableFaultMsg; +import com.vmware.vim25.VMwareDVSPortSetting; +import com.vmware.vim25.VimPortType; +import com.vmware.vim25.VirtualDevice; +import com.vmware.vim25.VirtualDeviceBackingInfo; +import com.vmware.vim25.VirtualDeviceConfigSpec; +import com.vmware.vim25.VirtualDeviceConfigSpecOperation; +import com.vmware.vim25.VirtualDisk; +import com.vmware.vim25.VirtualEthernetCard; +import com.vmware.vim25.VirtualEthernetCardDistributedVirtualPortBackingInfo; +import com.vmware.vim25.VirtualEthernetCardNetworkBackingInfo; +import com.vmware.vim25.VirtualLsiLogicController; +import com.vmware.vim25.VirtualMachineConfigSpec; +import com.vmware.vim25.VirtualMachineFileInfo; +import com.vmware.vim25.VirtualMachineGuestOsIdentifier; +import com.vmware.vim25.VirtualMachinePowerState; +import com.vmware.vim25.VirtualMachineRelocateSpec; +import com.vmware.vim25.VirtualMachineRelocateSpecDiskLocator; +import com.vmware.vim25.VirtualMachineRuntimeInfo; +import com.vmware.vim25.VirtualSCSISharing; +import com.vmware.vim25.VmwareDistributedVirtualSwitchVlanIdSpec; public class VmwareResource implements StoragePoolResource, ServerResource, VmwareHostService { @@ -362,17 +363,17 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa protected String _privateNetworkVSwitchName; protected VmwareTrafficLabel _guestTrafficInfo = new VmwareTrafficLabel(TrafficType.Guest); protected VmwareTrafficLabel _publicTrafficInfo = new VmwareTrafficLabel(TrafficType.Public); + protected Map _vsmCredentials = null; protected int _portsPerDvPortGroup; protected boolean _fullCloneFlag = false; + protected boolean _instanceNameFlag = false; - protected boolean _reserveCpu = false; - protected boolean _reserveMem = false; protected boolean _recycleHungWorker = false; protected DiskControllerType _rootDiskController = DiskControllerType.ide; protected ManagedObjectReference _morHyperHost; - protected VmwareContext _serviceContext; + protected static ThreadLocal s_serviceContext = new ThreadLocal(); protected String _hostName; protected HashMap _vms = new HashMap(71); @@ -446,6 +447,8 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa answer = execute((GetHostStatsCommand) cmd); } else if (clz == GetVmStatsCommand.class) { answer = execute((GetVmStatsCommand) cmd); + } else if (clz == GetVmDiskStatsCommand.class) { + answer = execute((GetVmDiskStatsCommand) cmd); } else if (clz == CheckHealthCommand.class) { answer = execute((CheckHealthCommand) cmd); } else if (clz == StopCommand.class) { @@ -598,10 +601,10 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } } finally { + recycleServiceContext(); NDC.pop(); } - if(s_logger.isTraceEnabled()) s_logger.trace("End executeRequest(), cmd: " + cmd.getClass().getSimpleName()); @@ -612,11 +615,39 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa String path = cmd.getPath(); String vmName = cmd.getInstanceName(); long newSize = cmd.getNewSize()/1024; + boolean useWorkerVm = false; + + VmwareHypervisorHost hyperHost = getHyperHost(getServiceContext()); + String poolId = cmd.getPoolUuid(); + VirtualMachineMO vmMo = null; + DatastoreMO dsMo = null; + ManagedObjectReference morDS = null; + String vmdkDataStorePath = null; + try { - VmwareHypervisorHost hyperHost = getHyperHost(getServiceContext()); + if (vmName.equalsIgnoreCase("none")) { + // we need to spawn a worker VM to attach the volume to and + // resize the volume. + useWorkerVm = true; + vmName = this.getWorkerName(getServiceContext(), cmd, 0); + + morDS = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, poolId); + dsMo = new DatastoreMO(hyperHost.getContext(), morDS); + s_logger.info("Create worker VM " + vmName); + vmMo = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, vmName); + if (vmMo == null) { + throw new Exception("Unable to create a worker VM for volume resize"); + } + + synchronized (this) { + vmdkDataStorePath = VmwareStorageLayoutHelper.getLegacyDatastorePathFromVmdkFileName(dsMo, path + ".vmdk"); + vmMo.attachDisk(new String[] { vmdkDataStorePath }, morDS); + } + } + // find VM through datacenter (VM is not at the target host yet) - VirtualMachineMO vmMo = hyperHost.findVmOnPeerHyperHost(vmName); + vmMo = hyperHost.findVmOnPeerHyperHost(vmName); if (vmMo == null) { String msg = "VM " + vmName + " does not exist in VMware datacenter"; s_logger.error(msg); @@ -631,6 +662,9 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } VirtualDisk disk = vdisk.first(); long oldSize = disk.getCapacityInKB(); + if (newSize <= oldSize && useWorkerVm == true) { + deleteWorkerVm(vmMo, vmdkDataStorePath); + } if (newSize < oldSize){ throw new Exception("VMware doesn't support shrinking volume from larger size: " + oldSize+ " MB to a smaller size: " + newSize + " MB"); } else if(newSize == oldSize){ @@ -644,9 +678,14 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa deviceConfigSpec.setOperation(VirtualDeviceConfigSpecOperation.EDIT); vmConfigSpec.getDeviceChange().add(deviceConfigSpec); if (!vmMo.configureVm(vmConfigSpec)) { + if (useWorkerVm == true) { + deleteWorkerVm(vmMo, vmdkDataStorePath); + } throw new Exception("Failed to configure VM to resize disk. vmName: " + vmName); } - + if (useWorkerVm == true) { + deleteWorkerVm(vmMo, vmdkDataStorePath); + } return new ResizeVolumeAnswer(cmd, true, "success", newSize*1024); } catch (Exception e) { s_logger.error("Unable to resize volume",e); @@ -655,6 +694,18 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } } + private void deleteWorkerVm(VirtualMachineMO vmMo, String vmdkDataStorePath) { + try { + if (vmMo != null) { + s_logger.info("Delete worker VM " + vmMo.getVmName()); + vmMo.detachDisk(vmdkDataStorePath, false); + vmMo.destroy(); + } + } catch (Exception e) { + s_logger.error("Unable to destroy worker vm", e); + } + } + protected Answer execute(CheckNetworkCommand cmd) { if (s_logger.isInfoEnabled()) { s_logger.info("Executing resource CheckNetworkCommand " + _gson.toJson(cmd)); @@ -1085,11 +1136,8 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa s_logger.error("Unexpected exception: " + e.toString(), e); return new Answer(cmd, false, "VPCLoadBalancerConfigCommand failed due to " + VmwareHelper.getExceptionMessage(e)); } - - } - protected Answer execute(final LoadBalancerConfigCommand cmd) { if ( cmd.getVpcId() != null ) { @@ -1788,7 +1836,6 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa if (!result_gateway.first()) { throw new InternalErrorException("Unable to configure source NAT for public IP address."); } - } } @@ -1907,11 +1954,13 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa * so we assume that it's VLAN for now */ if (VirtualSwitchType.StandardVirtualSwitch == vSwitchType) { - networkInfo = HypervisorHostHelper.prepareNetwork(_publicTrafficInfo.getVirtualSwitchName(), "cloud.public", - vmMo.getRunningHost(), vlanId, null, null, _ops_timeout, true, BroadcastDomainType.Vlan, null); + synchronized(vmMo.getRunningHost().getMor().getValue().intern()) { + networkInfo = HypervisorHostHelper.prepareNetwork(_publicTrafficInfo.getVirtualSwitchName(), "cloud.public", + vmMo.getRunningHost(), vlanId, null, null, _ops_timeout, true, BroadcastDomainType.Vlan, null); + } } else { networkInfo = HypervisorHostHelper.prepareNetwork(_publicTrafficInfo.getVirtualSwitchName(), "cloud.public", - vmMo.getRunningHost(), vlanId, null, null, null, _ops_timeout, vSwitchType, _portsPerDvPortGroup, null, false, BroadcastDomainType.Vlan); + vmMo.getRunningHost(), vlanId, null, null, null, _ops_timeout, vSwitchType, _portsPerDvPortGroup, null, false, BroadcastDomainType.Vlan, _vsmCredentials); } int nicIndex = allocPublicNicIndex(vmMo); @@ -2132,36 +2181,36 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa protected Answer execute(final CreateIpAliasCommand cmd) { if (s_logger.isInfoEnabled()) { - s_logger.info("Executing createipAlias command: " + _gson.toJson(cmd)); + s_logger.info("Executing createIpAlias command: " + _gson.toJson(cmd)); } String routerIp = cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP); List ipAliasTOs = cmd.getIpAliasList(); - String args=routerIp+" "; + String args=""; for (IpAliasTO ipaliasto : ipAliasTOs) { args = args + ipaliasto.getAlias_count()+":"+ipaliasto.getRouterip()+":"+ipaliasto.getNetmask()+"-"; } if (s_logger.isDebugEnabled()) { - s_logger.debug("Run command on domR " + cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP) + ", /root/createipAlias " + args); + s_logger.debug("Run command on domR " + cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP) + ", /root/createIpAlias " + args); } try { VmwareManager mgr = getServiceContext().getStockObject(VmwareManager.CONTEXT_STOCK_NAME); String controlIp = getRouterSshControlIp(cmd); Pair result = SshHelper.sshExecute(controlIp, DEFAULT_DOMR_SSHPORT, "root", mgr.getSystemVMKeyFile(), null, - "/root/createipAlias.sh " + args); + "/root/createIpAlias.sh " + args); if (!result.first()) { - s_logger.error("ipAlias command on domr " + controlIp + " failed, message: " + result.second()); + s_logger.error("CreateIpAlias command on domr " + controlIp + " failed, message: " + result.second()); return new Answer(cmd, false, "createipAlias failed due to " + result.second()); } if (s_logger.isInfoEnabled()) { - s_logger.info("createipAlias command on domain router " + controlIp + " completed"); + s_logger.info("createIpAlias command on domain router " + controlIp + " completed"); } } catch (Throwable e) { - String msg = "createipAlias failed due to " + VmwareHelper.getExceptionMessage(e); + String msg = "createIpAlias failed due to " + VmwareHelper.getExceptionMessage(e); s_logger.error(msg, e); return new Answer(cmd, false, msg); } @@ -2174,9 +2223,9 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa List revokedIpAliasTOs = cmd.getDeleteIpAliasTos(); List activeIpAliasTOs = cmd.getCreateIpAliasTos(); if (s_logger.isInfoEnabled()) { - s_logger.info("Executing deleteipAlias command: " + _gson.toJson(cmd)); + s_logger.info("Executing deleteIpAlias command: " + _gson.toJson(cmd)); } - String args=routerIp+" "; + String args=""; for (IpAliasTO ipAliasTO : revokedIpAliasTOs) { args = args + ipAliasTO.getAlias_count()+":"+ipAliasTO.getRouterip()+":"+ipAliasTO.getNetmask()+"-"; } @@ -2185,27 +2234,27 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa args = args + ipAliasTO.getAlias_count()+":"+ipAliasTO.getRouterip()+":"+ipAliasTO.getNetmask()+"-"; } if (s_logger.isDebugEnabled()) { - s_logger.debug("Run command on domR " + cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP) + ", /root/deleteipAlias " + args); + s_logger.debug("Run command on domR " + cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP) + ", /root/deleteIpAlias " + args); } try { VmwareManager mgr = getServiceContext().getStockObject(VmwareManager.CONTEXT_STOCK_NAME); String controlIp = getRouterSshControlIp(cmd); Pair result = SshHelper.sshExecute(controlIp, DEFAULT_DOMR_SSHPORT, "root", mgr.getSystemVMKeyFile(), null, - "/root/deleteipAlias.sh " + args); + "/root/deleteIpAlias.sh " + args); if (!result.first()) { - s_logger.error("ipAlias command on domr " + controlIp + " failed, message: " + result.second()); + s_logger.error("deleteIpAlias command on domr " + controlIp + " failed, message: " + result.second()); - return new Answer(cmd, false, "deleteipAlias failed due to " + result.second()); + return new Answer(cmd, false, "deleteIpAlias failed due to " + result.second()); } if (s_logger.isInfoEnabled()) { - s_logger.info("deleteipAlias command on domain router " + controlIp + " completed"); + s_logger.info("deleteIpAlias command on domain router " + controlIp + " completed"); } } catch (Throwable e) { - String msg = "deleteipAlias failed due to " + VmwareHelper.getExceptionMessage(e); + String msg = "deleteIpAlias failed due to " + VmwareHelper.getExceptionMessage(e); s_logger.error(msg, e); return new Answer(cmd, false, msg); } @@ -2286,6 +2335,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } return new CheckS2SVpnConnectionsAnswer(cmd, true, result.second()); } + protected Answer execute(CheckRouterCommand cmd) { if (s_logger.isDebugEnabled()) { s_logger.debug("Executing resource CheckRouterCommand: " + _gson.toJson(cmd)); @@ -2514,6 +2564,15 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa return validatedDisks.toArray(new DiskTO[0]); } + + private static DiskTO getIsoDiskTO(DiskTO[] disks) { + for (DiskTO vol : disks) { + if (vol.getType() == Volume.Type.ISO) { + return vol; + } + } + return null; + } protected ScaleVmAnswer execute(ScaleVmCommand cmd) { @@ -2563,14 +2622,17 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } protected StartAnswer execute(StartCommand cmd) { - if (s_logger.isInfoEnabled()) { s_logger.info("Executing resource StartCommand: " + _gson.toJson(cmd)); } - + VirtualMachineTO vmSpec = cmd.getVirtualMachine(); - String vmName = vmSpec.getName(); - + + Pair names = composeVmNames(vmSpec); + String vmInternalCSName = names.first(); + String vmNameOnVcenter = names.second(); + + // Thus, vmInternalCSName always holds i-x-y, the cloudstack generated internal VM name. State state = State.Stopped; VmwareContext context = getServiceContext(); try { @@ -2578,13 +2640,9 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa // mark VM as starting state so that sync() can know not to report stopped too early synchronized (_vms) { - _vms.put(vmName, State.Starting); + _vms.put(vmInternalCSName, State.Starting); } - VirtualEthernetCardType nicDeviceType = VirtualEthernetCardType.valueOf(vmSpec.getDetails().get(VmDetailConstants.NIC_ADAPTER)); - if(s_logger.isDebugEnabled()) - s_logger.debug("VM " + vmName + " will be started with NIC device type: " + nicDeviceType); - VmwareHypervisorHost hyperHost = getHyperHost(context); DiskTO[] disks = validateDisks(vmSpec.getDisks()); assert (disks.length > 0); @@ -2596,32 +2654,45 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa s_logger.error(msg); throw new Exception(msg); } - - VirtualMachineMO vmMo = hyperHost.findVmOnHyperHost(vmName); + + DatastoreMO dsRootVolumeIsOn = getDatastoreThatRootDiskIsOn(dataStoresDetails, disks); + if(dsRootVolumeIsOn == null) { + String msg = "Unable to locate datastore details of root volume"; + s_logger.error(msg); + throw new Exception(msg); + } + + DatacenterMO dcMo = new DatacenterMO(hyperHost.getContext(), hyperHost.getHyperHostDatacenter()); + VirtualMachineDiskInfoBuilder diskInfoBuilder = null; + VirtualMachineMO vmMo = hyperHost.findVmOnHyperHost(vmInternalCSName); if (vmMo != null) { - s_logger.info("VM " + vmName + " already exists, tear down devices for reconfiguration"); + s_logger.info("VM " + vmInternalCSName + " already exists, tear down devices for reconfiguration"); if (getVmState(vmMo) != State.Stopped) vmMo.safePowerOff(_shutdown_waitMs); + + // retrieve disk information before we tear down + diskInfoBuilder = vmMo.getDiskInfoBuilder(); vmMo.tearDownDevices(new Class[] { VirtualDisk.class, VirtualEthernetCard.class }); vmMo.ensureScsiDeviceController(); } else { ManagedObjectReference morDc = hyperHost.getHyperHostDatacenter(); assert (morDc != null); - vmMo = hyperHost.findVmOnPeerHyperHost(vmName); + vmMo = hyperHost.findVmOnPeerHyperHost(vmInternalCSName); if (vmMo != null) { if (s_logger.isInfoEnabled()) { - s_logger.info("Found vm " + vmName + " at other host, relocate to " + hyperHost.getHyperHostName()); + s_logger.info("Found vm " + vmInternalCSName + " at other host, relocate to " + hyperHost.getHyperHostName()); } - takeVmFromOtherHyperHost(hyperHost, vmName); + takeVmFromOtherHyperHost(hyperHost, vmInternalCSName); if (getVmState(vmMo) != State.Stopped) vmMo.safePowerOff(_shutdown_waitMs); + + diskInfoBuilder = vmMo.getDiskInfoBuilder(); vmMo.tearDownDevices(new Class[] { VirtualDisk.class, VirtualEthernetCard.class }); vmMo.ensureScsiDeviceController(); } else { - int ramMb = (int) (vmSpec.getMinRam() / (1024 * 1024)); Pair rootDiskDataStoreDetails = null; for (DiskTO vol : disks) { if (vol.getType() == Volume.Type.ROOT) { @@ -2631,73 +2702,63 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } assert (vmSpec.getMinSpeed() != null) && (rootDiskDataStoreDetails != null); - if (!hyperHost.createBlankVm(vmName, vmSpec.getCpus(), vmSpec.getMaxSpeed().intValue(), - vmSpec.getMinSpeed(), vmSpec.getLimitCpuUse(),(int)(vmSpec.getMaxRam()/(1024*1024)), ramMb, + + if(rootDiskDataStoreDetails.second().folderExists(String.format("[%s]", rootDiskDataStoreDetails.second().getName()), vmNameOnVcenter)) { + s_logger.warn("WARN!!! Folder already exists on datastore for new VM " + vmNameOnVcenter + ", erase it"); + rootDiskDataStoreDetails.second().deleteFile(String.format("[%s] %s/", rootDiskDataStoreDetails.second().getName(), + vmNameOnVcenter), dcMo.getMor(), false); + } + + if (!hyperHost.createBlankVm(vmNameOnVcenter, vmInternalCSName, vmSpec.getCpus(), vmSpec.getMaxSpeed().intValue(), + getReservedCpuMHZ(vmSpec), vmSpec.getLimitCpuUse(),(int)(vmSpec.getMaxRam()/(1024*1024)), getReservedMemoryMb(vmSpec), translateGuestOsIdentifier(vmSpec.getArch(), vmSpec.getOs()).value(), rootDiskDataStoreDetails.first(), false)) { - throw new Exception("Failed to create VM. vmName: " + vmName); + throw new Exception("Failed to create VM. vmName: " + vmInternalCSName); } } - vmMo = hyperHost.findVmOnHyperHost(vmName); + vmMo = hyperHost.findVmOnHyperHost(vmInternalCSName); if (vmMo == null) { - throw new Exception("Failed to find the newly create or relocated VM. vmName: " + vmName); + throw new Exception("Failed to find the newly create or relocated VM. vmName: " + vmInternalCSName); } } - + int totalChangeDevices = disks.length + nics.length; + DiskTO volIso = null; if (vmSpec.getType() != VirtualMachine.Type.User) { // system VM needs a patch ISO totalChangeDevices++; } else { - for (DiskTO vol : disks) { - if (vol.getType() == Volume.Type.ISO) { - volIso = vol; - break; - } - } - + volIso = getIsoDiskTO(disks); if (volIso == null) totalChangeDevices++; } VirtualMachineConfigSpec vmConfigSpec = new VirtualMachineConfigSpec(); - int ramMb = (int) (vmSpec.getMinRam() / (1024 * 1024)); - VmwareHelper.setBasicVmConfig(vmConfigSpec, vmSpec.getCpus(), vmSpec.getMaxSpeed(), - vmSpec.getMinSpeed(),(int) (vmSpec.getMaxRam()/(1024*1024)), ramMb, - translateGuestOsIdentifier(vmSpec.getArch(), vmSpec.getOs()).value(), vmSpec.getLimitCpuUse()); String guestOsId = translateGuestOsIdentifier(vmSpec.getArch(), vmSpec.getOs()).value(); + + VmwareHelper.setBasicVmConfig(vmConfigSpec, vmSpec.getCpus(), vmSpec.getMaxSpeed(), + getReservedCpuMHZ(vmSpec),(int) (vmSpec.getMaxRam()/(1024*1024)), getReservedMemoryMb(vmSpec), + guestOsId, vmSpec.getLimitCpuUse()); + // Check for hotadd settings vmConfigSpec.setMemoryHotAddEnabled(vmMo.isMemoryHotAddSupported(guestOsId)); vmConfigSpec.setCpuHotAddEnabled(vmMo.isCpuHotAddSupported(guestOsId)); - - if ("true".equals(vmSpec.getDetails().get(VmDetailConstants.NESTED_VIRTUALIZATION_FLAG))) { - s_logger.debug("Nested Virtualization enabled in configuration, checking hypervisor capability"); - ManagedObjectReference hostMor = vmMo.getRunningHost().getMor(); - ManagedObjectReference computeMor = context.getVimClient().getMoRefProp(hostMor, "parent"); - ManagedObjectReference environmentBrowser = - context.getVimClient().getMoRefProp(computeMor, "environmentBrowser"); - HostCapability hostCapability = context.getService().queryTargetCapabilities(environmentBrowser, hostMor); - if (hostCapability.isNestedHVSupported()) { - s_logger.debug("Hypervisor supports nested virtualization, enabling for VM " + vmSpec.getName()); - vmConfigSpec.setNestedHVEnabled(true); - } - else { - s_logger.warn("Hypervisor doesn't support nested virtualization, unable to set config for VM " +vmSpec.getName()); - vmConfigSpec.setNestedHVEnabled(false); - } - } + configNestedHVSupport(vmMo, vmSpec, vmConfigSpec); VirtualDeviceConfigSpec[] deviceConfigSpecArray = new VirtualDeviceConfigSpec[totalChangeDevices]; int i = 0; int ideUnitNumber = 0; - int scsiUnitNumber =0; + int scsiUnitNumber = 0; int nicUnitNumber = 0; int ideControllerKey = vmMo.getIDEDeviceControllerKey(); int scsiControllerKey = vmMo.getScsiDeviceControllerKey(); int controllerKey; - String datastoreDiskPath; + // + // Setup ISO device + // + // prepare systemvm patch ISO if (vmSpec.getType() != VirtualMachine.Type.User) { // attach ISO (for patching of system VM) @@ -2729,8 +2790,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.EDIT); } } else { - // we will always plugin a CDROM device - + // Note: we will always plug a CDROM device if (volIso != null) { TemplateObjectTO iso = (TemplateObjectTO)volIso.getData(); @@ -2776,71 +2836,56 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } } } + i++; - for (DiskTO vol : sortVolumesByDeviceId(disks)) { + + // + // Setup ROOT/DATA disk devices + // + DiskTO[] sortedDisks = sortVolumesByDeviceId(disks); + for (DiskTO vol : sortedDisks) { deviceConfigSpecArray[i] = new VirtualDeviceConfigSpec(); - if (vol.getType() == Volume.Type.ISO) { - controllerKey = ideControllerKey; - } else { - if(vol.getType() == Volume.Type.ROOT) { - if(vmSpec.getDetails() != null && vmSpec.getDetails().get(VmDetailConstants.ROOK_DISK_CONTROLLER) != null) - { - if(vmSpec.getDetails().get(VmDetailConstants.ROOK_DISK_CONTROLLER).equalsIgnoreCase("scsi")) - controllerKey = scsiControllerKey; - else - controllerKey = ideControllerKey; - } else { - controllerKey = scsiControllerKey; - } - } else { - // DATA volume always use SCSI device - controllerKey = scsiControllerKey; - } - } + if (vol.getType() == Volume.Type.ISO) + continue; + + VirtualMachineDiskInfo matchingExistingDisk = getMatchingExistingDisk(diskInfoBuilder, vol); + controllerKey = getDiskController(matchingExistingDisk, vol, vmSpec, ideControllerKey, scsiControllerKey); - if (vol.getType() != Volume.Type.ISO) { - VolumeObjectTO volumeTO = (VolumeObjectTO)vol.getData(); - PrimaryDataStoreTO primaryStore = (PrimaryDataStoreTO)volumeTO.getDataStore(); - Pair volumeDsDetails = dataStoresDetails.get(primaryStore.getUuid()); - assert (volumeDsDetails != null); - VirtualDevice device; - datastoreDiskPath = String.format("[%s] %s.vmdk", volumeDsDetails.second().getName(), volumeTO.getPath()); - String chainInfo = volumeTO.getChainInfo(); + VolumeObjectTO volumeTO = (VolumeObjectTO)vol.getData(); + PrimaryDataStoreTO primaryStore = (PrimaryDataStoreTO)volumeTO.getDataStore(); + Pair volumeDsDetails = dataStoresDetails.get(primaryStore.getUuid()); + assert (volumeDsDetails != null); + VirtualDevice device; + + String[] diskChain = syncDiskChain(dcMo, vmMo, vmSpec, + vol, matchingExistingDisk, + dataStoresDetails); + + device = VmwareHelper.prepareDiskDevice(vmMo, null, controllerKey, + diskChain, + volumeDsDetails.first(), + (controllerKey == ideControllerKey) ? ideUnitNumber++ : scsiUnitNumber++, i + 1); + + deviceConfigSpecArray[i].setDevice(device); + deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.ADD); - if (chainInfo != null && !chainInfo.isEmpty()) { - String[] diskChain = _gson.fromJson(chainInfo, String[].class); - if (diskChain == null || diskChain.length < 1) { - s_logger.warn("Empty previously-saved chain info, fall back to the original"); - device = VmwareHelper.prepareDiskDevice(vmMo, controllerKey, new String[] { datastoreDiskPath }, volumeDsDetails.first(), - (controllerKey==ideControllerKey)?ideUnitNumber++:scsiUnitNumber++, i + 1); - } else { - s_logger.info("Attach the disk with stored chain info: " + chainInfo); - for (int j = 0; j < diskChain.length; j++) { - diskChain[j] = String.format("[%s] %s", volumeDsDetails.second().getName(), diskChain[j]); - } + if(s_logger.isDebugEnabled()) + s_logger.debug("Prepare volume at new device " + _gson.toJson(device)); - device = VmwareHelper.prepareDiskDevice(vmMo, controllerKey, diskChain, volumeDsDetails.first(), - (controllerKey==ideControllerKey)?ideUnitNumber++:scsiUnitNumber++, i + 1); - } - } else { - device = VmwareHelper.prepareDiskDevice(vmMo, controllerKey, new String[] { datastoreDiskPath }, volumeDsDetails.first(), - (controllerKey==ideControllerKey)?ideUnitNumber++:scsiUnitNumber++, i + 1); - } - deviceConfigSpecArray[i].setDevice(device); - deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.ADD); - - - if(s_logger.isDebugEnabled()) - s_logger.debug("Prepare volume at new device " + _gson.toJson(device)); - - i++; - } + i++; } + // + // Setup NIC devices + // VirtualDevice nic; int nicMask = 0; int nicCount = 0; + VirtualEthernetCardType nicDeviceType = VirtualEthernetCardType.valueOf(vmSpec.getDetails().get(VmDetailConstants.NIC_ADAPTER)); + if(s_logger.isDebugEnabled()) + s_logger.debug("VM " + vmInternalCSName + " will be started with NIC device type: " + nicDeviceType); + for (NicTO nicTo : sortNicsByDeviceId(nics)) { s_logger.info("Prepare NIC device based on NicTO: " + _gson.toJson(nicTo)); @@ -2877,172 +2922,49 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa vmConfigSpec.getDeviceChange().addAll(Arrays.asList(deviceConfigSpecArray)); + // + // Setup VM options + // + // pass boot arguments through machine.id & perform customized options to VMX - ArrayList extraOptions = new ArrayList(); - OptionValue newVal = new OptionValue(); - newVal.setKey("machine.id"); - newVal.setValue(vmSpec.getBootArgs()); - extraOptions.add(newVal); - - newVal = new OptionValue(); - newVal.setKey("devices.hotplug"); - newVal.setValue("true"); - extraOptions.add(newVal); - - /** - * Extra Config : nvp.vm-uuid = uuid - * - Required for Nicira NVP integration - */ - newVal = new OptionValue(); - newVal.setKey("nvp.vm-uuid"); - newVal.setValue(vmSpec.getUuid()); - extraOptions.add(newVal); - - /** - * Extra Config : nvp.iface-id. = uuid - * - Required for Nicira NVP integration - */ - int nicNum = 0; - for (NicTO nicTo : sortNicsByDeviceId(nics)) { - if (nicTo.getUuid() != null) { - newVal = new OptionValue(); - newVal.setKey("nvp.iface-id." + nicNum); - newVal.setValue(nicTo.getUuid()); - extraOptions.add(newVal); - } - nicNum++; - } - - for(Map.Entry entry : validateVmDetails(vmSpec.getDetails()).entrySet()) { - newVal = new OptionValue(); - newVal.setKey(entry.getKey()); - newVal.setValue(entry.getValue()); - extraOptions.add(newVal); - } + configBasicExtraOption(extraOptions, vmSpec); + configNvpExtraOption(extraOptions, vmSpec); + configCustomExtraOption(extraOptions, vmSpec); + // config VNC String keyboardLayout = null; if(vmSpec.getDetails() != null) keyboardLayout = vmSpec.getDetails().get(VmDetailConstants.KEYBOARD); - vmConfigSpec.getExtraConfig().addAll(Arrays.asList(configureVnc(extraOptions.toArray(new OptionValue[0]), hyperHost, vmName, vmSpec.getVncPassword(), keyboardLayout))); + vmConfigSpec.getExtraConfig().addAll( + Arrays.asList( + configureVnc( + extraOptions.toArray(new OptionValue[0]), + hyperHost, vmInternalCSName, vmSpec.getVncPassword(), keyboardLayout + ) + ) + ); + // + // Configure VM + // if (!vmMo.configureVm(vmConfigSpec)) { - throw new Exception("Failed to configure VM before start. vmName: " + vmName); + throw new Exception("Failed to configure VM before start. vmName: " + vmInternalCSName); } + // + // Post Configuration + // + vmMo.setCustomFieldValue(CustomFieldConstants.CLOUD_NIC_MASK, String.valueOf(nicMask)); - - /** - * We need to configure the port on the DV switch after the host is - * connected. So make this happen between the configure and start of - * the VM - */ - int nicIndex = 0; - for (NicTO nicTo : sortNicsByDeviceId(nics)) { - if (nicTo.getBroadcastType() == BroadcastDomainType.Lswitch) { - // We need to create a port with a unique vlan and pass the key to the nic device - s_logger.trace("Nic " + nicTo.toString() + " is connected to an NVP logicalswitch"); - VirtualDevice nicVirtualDevice = vmMo.getNicDeviceByIndex(nicIndex); - if (nicVirtualDevice == null) { - throw new Exception("Failed to find a VirtualDevice for nic " + nicIndex); //FIXME Generic exceptions are bad - } - VirtualDeviceBackingInfo backing = nicVirtualDevice.getBacking(); - if (backing instanceof VirtualEthernetCardDistributedVirtualPortBackingInfo) { - // This NIC is connected to a Distributed Virtual Switch - VirtualEthernetCardDistributedVirtualPortBackingInfo portInfo = (VirtualEthernetCardDistributedVirtualPortBackingInfo) backing; - DistributedVirtualSwitchPortConnection port = portInfo.getPort(); - String portKey = port.getPortKey(); - String portGroupKey = port.getPortgroupKey(); - String dvSwitchUuid = port.getSwitchUuid(); - - s_logger.debug("NIC " + nicTo.toString() + " is connected to dvSwitch " + dvSwitchUuid + " pg " + portGroupKey + " port " + portKey); - - ManagedObjectReference dvSwitchManager = vmMo.getContext().getVimClient().getServiceContent().getDvSwitchManager(); - ManagedObjectReference dvSwitch = vmMo.getContext().getVimClient().getService().queryDvsByUuid(dvSwitchManager, dvSwitchUuid); - - // Get all ports - DistributedVirtualSwitchPortCriteria criteria = new DistributedVirtualSwitchPortCriteria(); - criteria.setInside(true); - criteria.getPortgroupKey().add(portGroupKey); - List dvPorts = vmMo.getContext().getVimClient().getService().fetchDVPorts(dvSwitch, criteria); - - DistributedVirtualPort vmDvPort = null; - List usedVlans = new ArrayList(); - for (DistributedVirtualPort dvPort : dvPorts) { - // Find the port for this NIC by portkey - if (portKey.equals(dvPort.getKey())) { - vmDvPort = dvPort; - } - VMwareDVSPortSetting settings = (VMwareDVSPortSetting) dvPort - .getConfig().getSetting(); - VmwareDistributedVirtualSwitchVlanIdSpec vlanId = (VmwareDistributedVirtualSwitchVlanIdSpec) settings - .getVlan(); - s_logger.trace("Found port " + dvPort.getKey() - + " with vlan " + vlanId.getVlanId()); - if (vlanId.getVlanId() > 0 - && vlanId.getVlanId() < 4095) { - usedVlans.add(vlanId.getVlanId()); - } - } - - if (vmDvPort == null) { - throw new Exception("Empty port list from dvSwitch for nic " + nicTo.toString()); - } - - DVPortConfigInfo dvPortConfigInfo = vmDvPort - .getConfig(); - VMwareDVSPortSetting settings = (VMwareDVSPortSetting) dvPortConfigInfo.getSetting(); - - VmwareDistributedVirtualSwitchVlanIdSpec vlanId = (VmwareDistributedVirtualSwitchVlanIdSpec) settings.getVlan(); - BoolPolicy blocked = settings.getBlocked(); - if (blocked.isValue() == Boolean.TRUE) { - s_logger.trace("Port is blocked, set a vlanid and unblock"); - DVPortConfigSpec dvPortConfigSpec = new DVPortConfigSpec(); - VMwareDVSPortSetting edittedSettings = new VMwareDVSPortSetting(); - // Unblock - blocked.setValue(Boolean.FALSE); - blocked.setInherited(Boolean.FALSE); - edittedSettings.setBlocked(blocked); - // Set vlan - for (i = 1; i < 4095; i++) { - if (!usedVlans.contains(i)) - break; - } - vlanId.setVlanId(i); // FIXME should be a determined - // based on usage - vlanId.setInherited(false); - edittedSettings.setVlan(vlanId); - - dvPortConfigSpec.setSetting(edittedSettings); - dvPortConfigSpec.setOperation("edit"); - dvPortConfigSpec.setKey(portKey); - List dvPortConfigSpecs = new ArrayList(); - dvPortConfigSpecs.add(dvPortConfigSpec); - ManagedObjectReference task = vmMo.getContext().getVimClient().getService().reconfigureDVPortTask(dvSwitch, dvPortConfigSpecs); - if (!vmMo.getContext().getVimClient().waitForTask(task)) { - throw new Exception( - "Failed to configure the dvSwitch port for nic " - + nicTo.toString()); - } - s_logger.debug("NIC " + nicTo.toString() - + " connected to vlan " + i); - } else { - s_logger.trace("Port already configured and set to vlan " + vlanId.getVlanId()); - } - } else if (backing instanceof VirtualEthernetCardNetworkBackingInfo) { - // This NIC is connected to a Virtual Switch - // Nothing to do - } - else { - s_logger.error("nic device backing is of type " + backing.getClass().getName()); - throw new Exception("Incompatible backing for a VirtualDevice for nic " + nicIndex); //FIXME Generic exceptions are bad - } - } - nicIndex++; - } - + postNvpConfigBeforeStart(vmMo, vmSpec); + postDiskConfigBeforeStart(vmMo, vmSpec, sortedDisks, ideControllerKey, scsiControllerKey); + + // + // Power-on VM + // if (!vmMo.powerOn()) { - throw new Exception("Failed to start VM. vmName: " + vmName); + throw new Exception("Failed to start VM. vmName: " + vmInternalCSName + " with hostname " + vmNameOnVcenter); } state = State.Running; @@ -3059,14 +2981,397 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } finally { synchronized (_vms) { if (state != State.Stopped) { - _vms.put(vmName, state); + _vms.put(vmInternalCSName, state); } else { - _vms.remove(vmName); + _vms.remove(vmInternalCSName); } } } } + int getReservedMemoryMb(VirtualMachineTO vmSpec) { + if (vmSpec.getDetails().get(Config.VmwareReserveMem.key()).equalsIgnoreCase("true")) { + return (int) (vmSpec.getMinRam() / (1024 * 1024)); + } else if (vmSpec.getMinRam() != vmSpec.getMaxRam()) { + s_logger.warn("memory overprovisioning factor is set to "+ (vmSpec.getMaxRam()/vmSpec.getMinRam())+" ignoring the flag vmware.reserve.mem"); + return (int) (vmSpec.getMinRam() / (1024 * 1024)); + } + return 0; + } + + int getReservedCpuMHZ(VirtualMachineTO vmSpec) { + if (vmSpec.getDetails().get(Config.VmwareReserveCpu.key()).equalsIgnoreCase("true")) { + return vmSpec.getMinSpeed(); + }else if (vmSpec.getMinSpeed().intValue() != vmSpec.getMaxSpeed().intValue()) { + s_logger.warn("cpu overprovisioning factor is set to "+ (vmSpec.getMaxSpeed().intValue()/vmSpec.getMinSpeed().intValue())+" ignoring the flag vmware.reserve.cpu"); + return vmSpec.getMinSpeed(); + } + return 0; + } + + // return the finalized disk chain for startup, from top to bottom + private String[] syncDiskChain(DatacenterMO dcMo, VirtualMachineMO vmMo, VirtualMachineTO vmSpec, + DiskTO vol, VirtualMachineDiskInfo diskInfo, + HashMap> dataStoresDetails + ) throws Exception { + + VolumeObjectTO volumeTO = (VolumeObjectTO)vol.getData(); + PrimaryDataStoreTO primaryStore = (PrimaryDataStoreTO)volumeTO.getDataStore(); + + Pair volumeDsDetails = dataStoresDetails.get(primaryStore.getUuid()); + if(volumeDsDetails == null) + throw new Exception("Primary datastore " + primaryStore.getUuid() + " is not mounted on host."); + DatastoreMO dsMo = volumeDsDetails.second(); + + // we will honor vCenter's meta if it exists + if(diskInfo != null) { + // to deal with run-time upgrade to maintain the new datastore folder structure + String disks[] = diskInfo.getDiskChain(); + for(int i = 0; i < disks.length; i++) { + DatastoreFile file = new DatastoreFile(disks[i]); + if(file.getDir() != null && file.getDir().isEmpty()) { + s_logger.info("Perform run-time datastore folder upgrade. sync " + disks[i] + " to VM folder"); + disks[i] = VmwareStorageLayoutHelper.syncVolumeToVmDefaultFolder( + dcMo, vmMo.getName(), dsMo, file.getFileBaseName()); + } + } + return disks; + } + + String datastoreDiskPath = VmwareStorageLayoutHelper.syncVolumeToVmDefaultFolder( + dcMo, vmMo.getName(), dsMo, volumeTO.getPath()); + if(!dsMo.fileExists(datastoreDiskPath)) { + s_logger.warn("Volume " + volumeTO.getId() + " does not seem to exist on datastore, out of sync? path: " + datastoreDiskPath); + } + + return new String[] { datastoreDiskPath }; + } + + // Pair + private Pair composeVmNames(VirtualMachineTO vmSpec) { + String vmInternalCSName = null; + String vmNameOnVcenter = null; + if (vmSpec.getHostName() != null) { + vmInternalCSName = vmSpec.getName(); + if (_instanceNameFlag == true) + vmNameOnVcenter = vmSpec.getHostName(); + else + vmNameOnVcenter = vmSpec.getName(); + } else { + vmNameOnVcenter = vmInternalCSName = vmSpec.getName(); + } + + return new Pair(vmInternalCSName, vmNameOnVcenter); + } + + private static void configNestedHVSupport(VirtualMachineMO vmMo, + VirtualMachineTO vmSpec, VirtualMachineConfigSpec vmConfigSpec) throws Exception { + + VmwareContext context = vmMo.getContext(); + if ("true".equals(vmSpec.getDetails().get(VmDetailConstants.NESTED_VIRTUALIZATION_FLAG))) { + if(s_logger.isDebugEnabled()) + s_logger.debug("Nested Virtualization enabled in configuration, checking hypervisor capability"); + + ManagedObjectReference hostMor = vmMo.getRunningHost().getMor(); + ManagedObjectReference computeMor = context.getVimClient().getMoRefProp(hostMor, "parent"); + ManagedObjectReference environmentBrowser = context.getVimClient().getMoRefProp(computeMor, "environmentBrowser"); + HostCapability hostCapability = context.getService().queryTargetCapabilities(environmentBrowser, hostMor); + Boolean nestedHvSupported = hostCapability.isNestedHVSupported(); + if (nestedHvSupported == null) { + // nestedHvEnabled property is supported only since VMware 5.1. It's not defined for earlier versions. + s_logger.warn("Hypervisor doesn't support nested virtualization, unable to set config for VM " +vmSpec.getName()); + } else if (nestedHvSupported.booleanValue()) { + s_logger.debug("Hypervisor supports nested virtualization, enabling for VM " + vmSpec.getName()); + vmConfigSpec.setNestedHVEnabled(true); + } + else { + s_logger.warn("Hypervisor doesn't support nested virtualization, unable to set config for VM " +vmSpec.getName()); + vmConfigSpec.setNestedHVEnabled(false); + } + } + } + + private static void configBasicExtraOption(List extraOptions, VirtualMachineTO vmSpec) { + OptionValue newVal = new OptionValue(); + newVal.setKey("machine.id"); + newVal.setValue(vmSpec.getBootArgs()); + extraOptions.add(newVal); + + newVal = new OptionValue(); + newVal.setKey("devices.hotplug"); + newVal.setValue("true"); + extraOptions.add(newVal); + } + + private static void configNvpExtraOption(List extraOptions, VirtualMachineTO vmSpec) { + /** + * Extra Config : nvp.vm-uuid = uuid + * - Required for Nicira NVP integration + */ + OptionValue newVal = new OptionValue(); + newVal.setKey("nvp.vm-uuid"); + newVal.setValue(vmSpec.getUuid()); + extraOptions.add(newVal); + + /** + * Extra Config : nvp.iface-id. = uuid + * - Required for Nicira NVP integration + */ + int nicNum = 0; + for (NicTO nicTo : sortNicsByDeviceId(vmSpec.getNics())) { + if (nicTo.getUuid() != null) { + newVal = new OptionValue(); + newVal.setKey("nvp.iface-id." + nicNum); + newVal.setValue(nicTo.getUuid()); + extraOptions.add(newVal); + } + nicNum++; + } + } + + private static void configCustomExtraOption(List extraOptions, VirtualMachineTO vmSpec) { + // we no longer to validation anymore + for(Map.Entry entry : vmSpec.getDetails().entrySet()) { + OptionValue newVal = new OptionValue(); + newVal.setKey(entry.getKey()); + newVal.setValue(entry.getValue()); + extraOptions.add(newVal); + } + } + + private static void postNvpConfigBeforeStart(VirtualMachineMO vmMo, VirtualMachineTO vmSpec) throws Exception { + /** + * We need to configure the port on the DV switch after the host is + * connected. So make this happen between the configure and start of + * the VM + */ + int nicIndex = 0; + for (NicTO nicTo : sortNicsByDeviceId(vmSpec.getNics())) { + if (nicTo.getBroadcastType() == BroadcastDomainType.Lswitch) { + // We need to create a port with a unique vlan and pass the key to the nic device + s_logger.trace("Nic " + nicTo.toString() + " is connected to an NVP logicalswitch"); + VirtualDevice nicVirtualDevice = vmMo.getNicDeviceByIndex(nicIndex); + if (nicVirtualDevice == null) { + throw new Exception("Failed to find a VirtualDevice for nic " + nicIndex); //FIXME Generic exceptions are bad + } + VirtualDeviceBackingInfo backing = nicVirtualDevice.getBacking(); + if (backing instanceof VirtualEthernetCardDistributedVirtualPortBackingInfo) { + // This NIC is connected to a Distributed Virtual Switch + VirtualEthernetCardDistributedVirtualPortBackingInfo portInfo = (VirtualEthernetCardDistributedVirtualPortBackingInfo) backing; + DistributedVirtualSwitchPortConnection port = portInfo.getPort(); + String portKey = port.getPortKey(); + String portGroupKey = port.getPortgroupKey(); + String dvSwitchUuid = port.getSwitchUuid(); + + s_logger.debug("NIC " + nicTo.toString() + " is connected to dvSwitch " + dvSwitchUuid + " pg " + portGroupKey + " port " + portKey); + + ManagedObjectReference dvSwitchManager = vmMo.getContext().getVimClient().getServiceContent().getDvSwitchManager(); + ManagedObjectReference dvSwitch = vmMo.getContext().getVimClient().getService().queryDvsByUuid(dvSwitchManager, dvSwitchUuid); + + // Get all ports + DistributedVirtualSwitchPortCriteria criteria = new DistributedVirtualSwitchPortCriteria(); + criteria.setInside(true); + criteria.getPortgroupKey().add(portGroupKey); + List dvPorts = vmMo.getContext().getVimClient().getService().fetchDVPorts(dvSwitch, criteria); + + DistributedVirtualPort vmDvPort = null; + List usedVlans = new ArrayList(); + for (DistributedVirtualPort dvPort : dvPorts) { + // Find the port for this NIC by portkey + if (portKey.equals(dvPort.getKey())) { + vmDvPort = dvPort; + } + VMwareDVSPortSetting settings = (VMwareDVSPortSetting) dvPort + .getConfig().getSetting(); + VmwareDistributedVirtualSwitchVlanIdSpec vlanId = (VmwareDistributedVirtualSwitchVlanIdSpec) settings + .getVlan(); + s_logger.trace("Found port " + dvPort.getKey() + + " with vlan " + vlanId.getVlanId()); + if (vlanId.getVlanId() > 0 + && vlanId.getVlanId() < 4095) { + usedVlans.add(vlanId.getVlanId()); + } + } + + if (vmDvPort == null) { + throw new Exception("Empty port list from dvSwitch for nic " + nicTo.toString()); + } + + DVPortConfigInfo dvPortConfigInfo = vmDvPort + .getConfig(); + VMwareDVSPortSetting settings = (VMwareDVSPortSetting) dvPortConfigInfo.getSetting(); + + VmwareDistributedVirtualSwitchVlanIdSpec vlanId = (VmwareDistributedVirtualSwitchVlanIdSpec) settings.getVlan(); + BoolPolicy blocked = settings.getBlocked(); + if (blocked.isValue() == Boolean.TRUE) { + s_logger.trace("Port is blocked, set a vlanid and unblock"); + DVPortConfigSpec dvPortConfigSpec = new DVPortConfigSpec(); + VMwareDVSPortSetting edittedSettings = new VMwareDVSPortSetting(); + // Unblock + blocked.setValue(Boolean.FALSE); + blocked.setInherited(Boolean.FALSE); + edittedSettings.setBlocked(blocked); + // Set vlan + int i; + for (i = 1; i < 4095; i++) { + if (!usedVlans.contains(i)) + break; + } + vlanId.setVlanId(i); // FIXME should be a determined + // based on usage + vlanId.setInherited(false); + edittedSettings.setVlan(vlanId); + + dvPortConfigSpec.setSetting(edittedSettings); + dvPortConfigSpec.setOperation("edit"); + dvPortConfigSpec.setKey(portKey); + List dvPortConfigSpecs = new ArrayList(); + dvPortConfigSpecs.add(dvPortConfigSpec); + ManagedObjectReference task = vmMo.getContext().getVimClient().getService().reconfigureDVPortTask(dvSwitch, dvPortConfigSpecs); + if (!vmMo.getContext().getVimClient().waitForTask(task)) { + throw new Exception( + "Failed to configure the dvSwitch port for nic " + + nicTo.toString()); + } + s_logger.debug("NIC " + nicTo.toString() + + " connected to vlan " + i); + } else { + s_logger.trace("Port already configured and set to vlan " + vlanId.getVlanId()); + } + } else if (backing instanceof VirtualEthernetCardNetworkBackingInfo) { + // This NIC is connected to a Virtual Switch + // Nothing to do + } + else { + s_logger.error("nic device backing is of type " + backing.getClass().getName()); + throw new Exception("Incompatible backing for a VirtualDevice for nic " + nicIndex); //FIXME Generic exceptions are bad + } + } + nicIndex++; + } + } + + private VirtualMachineDiskInfo getMatchingExistingDisk(VirtualMachineDiskInfoBuilder diskInfoBuilder, DiskTO vol) { + if(diskInfoBuilder != null) { + VolumeObjectTO volume = (VolumeObjectTO)vol.getData(); + + VirtualMachineDiskInfo diskInfo = diskInfoBuilder.getDiskInfoByBackingFileBaseName(volume.getPath()); + if(diskInfo != null) { + s_logger.info("Found existing disk info from volume path: " + volume.getPath()); + return diskInfo; + } else { + String chainInfo = volume.getChainInfo(); + if(chainInfo != null) { + VirtualMachineDiskInfo infoInChain = _gson.fromJson(chainInfo, VirtualMachineDiskInfo.class); + if(infoInChain != null) { + String[] disks = infoInChain.getDiskChain(); + if(disks.length > 0) { + for(String diskPath : disks) { + DatastoreFile file = new DatastoreFile(diskPath); + diskInfo = diskInfoBuilder.getDiskInfoByBackingFileBaseName(file.getFileBaseName()); + if(diskInfo != null) { + s_logger.info("Found existing disk from chain info: " + diskPath); + return diskInfo; + } + } + } + + if(diskInfo == null) { + diskInfo = diskInfoBuilder.getDiskInfoByDeviceBusName(infoInChain.getDiskDeviceBusName()); + if(diskInfo != null) { + s_logger.info("Found existing disk from from chain device bus information: " + infoInChain.getDiskDeviceBusName()); + return diskInfo; + } + } + } + } + } + } + + return null; + } + + private int getDiskController(VirtualMachineDiskInfo matchingExistingDisk, DiskTO vol, + VirtualMachineTO vmSpec, int ideControllerKey, int scsiControllerKey) { + + int controllerKey; + if(matchingExistingDisk != null) { + s_logger.info("Chose disk controller based on existing information: " + matchingExistingDisk.getDiskDeviceBusName()); + if(matchingExistingDisk.getDiskDeviceBusName().startsWith("ide")) + return ideControllerKey; + else + return scsiControllerKey; + } + + if(vol.getType() == Volume.Type.ROOT) { + Map vmDetails = vmSpec.getDetails(); + if(vmDetails != null && vmDetails.get(VmDetailConstants.ROOK_DISK_CONTROLLER) != null) + { + if(vmDetails.get(VmDetailConstants.ROOK_DISK_CONTROLLER).equalsIgnoreCase("scsi")) { + s_logger.info("Chose disk controller for vol " + vol.getType() + " -> scsi, based on root disk controller settings: " + + vmDetails.get(VmDetailConstants.ROOK_DISK_CONTROLLER)); + controllerKey = scsiControllerKey; + } + else { + s_logger.info("Chose disk controller for vol " + vol.getType() + " -> ide, based on root disk controller settings: " + + vmDetails.get(VmDetailConstants.ROOK_DISK_CONTROLLER)); + controllerKey = ideControllerKey; + } + } else { + s_logger.info("Chose disk controller for vol " + vol.getType() + " -> scsi. due to null root disk controller setting"); + controllerKey = scsiControllerKey; + } + + } else { + // DATA volume always use SCSI device + s_logger.info("Chose disk controller for vol " + vol.getType() + " -> scsi"); + controllerKey = scsiControllerKey; + } + + return controllerKey; + } + + private void postDiskConfigBeforeStart(VirtualMachineMO vmMo, VirtualMachineTO vmSpec, DiskTO[] sortedDisks, + int ideControllerKey, int scsiControllerKey) throws Exception { + + VirtualMachineDiskInfoBuilder diskInfoBuilder = vmMo.getDiskInfoBuilder(); + + for(DiskTO vol: sortedDisks) { + if (vol.getType() == Volume.Type.ISO) + continue; + + VolumeObjectTO volumeTO = (VolumeObjectTO)vol.getData(); + + VirtualMachineDiskInfo diskInfo = getMatchingExistingDisk(diskInfoBuilder, vol); + assert(diskInfo != null); + + String[] diskChain = diskInfo.getDiskChain(); + assert(diskChain.length > 0); + + DatastoreFile file = new DatastoreFile(diskChain[0]); + if(!file.getFileBaseName().equalsIgnoreCase(volumeTO.getPath())) { + if(s_logger.isInfoEnabled()) + s_logger.info("Detected disk-chain top file change on volume: " + volumeTO.getId() + " " + + volumeTO.getPath() + " -> " + file.getFileBaseName()); + } + + VolumeObjectTO volInSpec = getVolumeInSpec(vmSpec, volumeTO); + volInSpec.setPath(file.getFileBaseName()); + volInSpec.setChainInfo(_gson.toJson(diskInfo)); + } + } + + private static VolumeObjectTO getVolumeInSpec(VirtualMachineTO vmSpec, VolumeObjectTO srcVol) { + for(DiskTO disk : vmSpec.getDisks()) { + VolumeObjectTO vol = (VolumeObjectTO)disk.getData(); + if(vol.getId() == srcVol.getId()) + return vol; + } + + return null; + } + + @Deprecated private Map validateVmDetails(Map vmDetails) { Map validatedDetails = new HashMap(); @@ -3092,9 +3397,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa return validatedDetails; } - - - private NicTO[] sortNicsByDeviceId(NicTO[] nics) { + private static NicTO[] sortNicsByDeviceId(NicTO[] nics) { List listForSort = new ArrayList(); for (NicTO nic : nics) { @@ -3117,7 +3420,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa return listForSort.toArray(new NicTO[0]); } - private DiskTO[] sortVolumesByDeviceId(DiskTO[] volumes) { + private static DiskTO[] sortVolumesByDeviceId(DiskTO[] volumes) { List listForSort = new ArrayList(); for (DiskTO vol : volumes) { @@ -3163,6 +3466,21 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa return poolMors; } + private DatastoreMO getDatastoreThatRootDiskIsOn(HashMap> dataStoresDetails, + DiskTO disks[]) { + + Pair rootDiskDataStoreDetails = null; + for (DiskTO vol : disks) { + if (vol.getType() == Volume.Type.ROOT) { + PrimaryDataStoreTO primaryStore = (PrimaryDataStoreTO)vol.getData().getDataStore(); + rootDiskDataStoreDetails = dataStoresDetails.get(primaryStore.getUuid()); + } + } + + if(rootDiskDataStoreDetails != null) + return rootDiskDataStoreDetails.second(); + return null; + } private String getPvlanInfo(NicTO nicTo) { if (nicTo.getBroadcastType() == BroadcastDomainType.Pvlan) { @@ -3220,9 +3538,11 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa s_logger.info("Prepare network on " + switchType + " " + switchName + " with name prefix: " + namePrefix); if (VirtualSwitchType.StandardVirtualSwitch == switchType) { - networkInfo = HypervisorHostHelper.prepareNetwork(switchName.first(), namePrefix, - hostMo, getVlanInfo(nicTo, switchName.second()), nicTo.getNetworkRateMbps(), nicTo.getNetworkRateMulticastMbps(), _ops_timeout, - !namePrefix.startsWith("cloud.private"), nicTo.getBroadcastType(), nicTo.getUuid()); + synchronized(hostMo.getMor().getValue().intern()) { + networkInfo = HypervisorHostHelper.prepareNetwork(switchName.first(), namePrefix, + hostMo, getVlanInfo(nicTo, switchName.second()), nicTo.getNetworkRateMbps(), nicTo.getNetworkRateMulticastMbps(), _ops_timeout, + !namePrefix.startsWith("cloud.private"), nicTo.getBroadcastType(), nicTo.getUuid()); + } } else { String vlanId = getVlanInfo(nicTo, switchName.second()); @@ -3237,7 +3557,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } networkInfo = HypervisorHostHelper.prepareNetwork(switchName.first(), namePrefix, hostMo, vlanId, svlanId, nicTo.getNetworkRateMbps(), nicTo.getNetworkRateMulticastMbps(), _ops_timeout, switchType, - _portsPerDvPortGroup, nicTo.getGateway(), configureVServiceInNexus, nicTo.getBroadcastType()); + _portsPerDvPortGroup, nicTo.getGateway(), configureVServiceInNexus, nicTo.getBroadcastType(), _vsmCredentials); } return networkInfo; @@ -3294,7 +3614,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } } - protected synchronized Answer execute(final RemoteAccessVpnCfgCommand cmd) { + protected Answer execute(final RemoteAccessVpnCfgCommand cmd) { String controlIp = getRouterSshControlIp(cmd); StringBuffer argsBuf = new StringBuffer(); if (cmd.isCreate()) { @@ -3338,7 +3658,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa return new Answer(cmd); } - protected synchronized Answer execute(final VpnUsersCfgCommand cmd) { + protected Answer execute(final VpnUsersCfgCommand cmd) { VmwareManager mgr = getServiceContext().getStockObject(VmwareManager.CONTEXT_STOCK_NAME); String controlIp = getRouterSshControlIp(cmd); @@ -3488,6 +3808,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa try { HashMap newStates = getVmStates(); + // getVmNames should return all i-x-y values. List requestedVmNames = cmd.getVmNames(); List vmNames = new ArrayList(); @@ -3519,6 +3840,10 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa return answer; } + protected Answer execute(GetVmDiskStatsCommand cmd) { + return new GetVmDiskStatsAnswer(cmd, null, null, null); + } + protected Answer execute(CheckHealthCommand cmd) { if (s_logger.isInfoEnabled()) { s_logger.info("Executing resource CheckHealthCommand: " + _gson.toJson(cmd)); @@ -3545,6 +3870,8 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa s_logger.info("Executing resource StopCommand: " + _gson.toJson(cmd)); } + // In the stop command, we're passed in the name of the VM as seen by cloudstack, + // i.e., i-x-y. This is the internal VM name. VmwareContext context = getServiceContext(); VmwareHypervisorHost hyperHost = getHyperHost(context); try { @@ -3721,9 +4048,15 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa // find VM through datacenter (VM is not at the target host yet) VirtualMachineMO vmMo = hyperHost.findVmOnPeerHyperHost(vmName); if (vmMo == null) { - String msg = "VM " + vmName + " does not exist in VMware datacenter"; - s_logger.error(msg); - throw new Exception(msg); + s_logger.info("VM " + vmName + " was not found in the cluster of host " + hyperHost.getHyperHostName() + ". Looking for the VM in datacenter."); + ManagedObjectReference dcMor = hyperHost.getHyperHostDatacenter(); + DatacenterMO dcMo = new DatacenterMO(hyperHost.getContext(), dcMor); + vmMo = dcMo.findVm(vmName); + if (vmMo == null) { + String msg = "VM " + vmName + " does not exist in VMware datacenter"; + s_logger.error(msg); + throw new Exception(msg); + } } NicTO[] nics = vm.getNics(); @@ -3911,10 +4244,13 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa relocateSpec.setDatastore(morDsAtSource); isFirstDs = false; } - srcDiskName = String.format("[%s] %s.vmdk", srcDsName, volume.getPath()); + srcDiskName = VmwareStorageLayoutHelper.getVmwareDatastorePathFromVmdkFileName( + new DatastoreMO(srcHyperHost.getContext(), morDsAtSource), + vmName, + volume.getPath() + ".vmdk"); diskLocator = new VirtualMachineRelocateSpecDiskLocator(); diskLocator.setDatastore(morDsAtSource); - diskLocator.setDiskId(getVirtualDiskInfo(vmMo, srcDiskName)); + diskLocator.setDiskId(getVirtualDiskInfo(vmMo, volume.getPath() + ".vmdk")); diskLocators.add(diskLocator); @@ -4016,8 +4352,9 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa srcDsName = volMgr.getStoragePoolOfVolume(cmd.getVolumeId()); tgtDsName = poolTo.getUuid().replace("-", ""); - // find VM through datacenter (VM is not at the target host yet) - vmMo = srcHyperHost.findVmOnPeerHyperHost(vmName); + // find VM in this datacenter not just in this cluster. + DatacenterMO dcMo = new DatacenterMO(getServiceContext(), morDc); + vmMo = dcMo.findVm(vmName); if (vmMo == null) { String msg = "VM " + vmName + " does not exist in VMware datacenter " + morDc.getValue(); s_logger.error(msg); @@ -4030,10 +4367,12 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa throw new Exception(msg); } - srcDiskName = String.format("[%s] %s.vmdk", srcDsName, volumePath); + srcDiskName = VmwareStorageLayoutHelper.getVmwareDatastorePathFromVmdkFileName( + new DatastoreMO(srcHyperHost.getContext(), morDs), vmName, + volumePath + ".vmdk"); diskLocator = new VirtualMachineRelocateSpecDiskLocator(); diskLocator.setDatastore(morDs); - diskLocator.setDiskId(getVirtualDiskInfo(vmMo, srcDiskName)); + diskLocator.setDiskId(getVirtualDiskInfo(vmMo, volumePath + ".vmdk")); diskLocators.add(diskLocator); relocateSpec.getDisk().add(diskLocator); @@ -4177,13 +4516,27 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa return false; } + private String trimIqn(String iqn) { + String[] tmp = iqn.split("/"); + + if (tmp.length != 3) { + String msg = "Wrong format for iScsi path: " + iqn + ". It should be formatted as '/targetIQN/LUN'."; + + s_logger.warn(msg); + + throw new CloudRuntimeException(msg); + } + + return tmp[1].trim(); + } + public ManagedObjectReference handleDatastoreAndVmdkAttach(Command cmd, String iqn, String storageHost, int storagePort, String initiatorUsername, String initiatorPassword, String targetUsername, String targetPassword) throws Exception { VmwareContext context = getServiceContext(); VmwareHypervisorHost hyperHost = getHyperHost(context); ManagedObjectReference morDs = createVmfsDatastore(hyperHost, getDatastoreName(iqn), - storageHost, storagePort, iqn, + storageHost, storagePort, trimIqn(iqn), initiatorUsername, initiatorPassword, targetUsername, targetPassword); @@ -4213,7 +4566,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa VmwareContext context = getServiceContext(); VmwareHypervisorHost hyperHost = getHyperHost(context); - deleteVmfsDatastore(hyperHost, getDatastoreName(iqn), storageHost, storagePort, iqn); + deleteVmfsDatastore(hyperHost, getDatastoreName(iqn), storageHost, storagePort, trimIqn(iqn)); } protected Answer execute(AttachVolumeCommand cmd) { @@ -4253,6 +4606,9 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } DatastoreMO dsMo = new DatastoreMO(getServiceContext(), morDs); + VmwareStorageLayoutHelper.syncVolumeToVmDefaultFolder(dsMo.getOwnerDatacenter().first(), cmd.getVmName(), + dsMo, cmd.getVolumePath()); + String datastoreVolumePath = dsMo.searchFileInSubFolders(cmd.getVolumePath() + ".vmdk", true); assert (datastoreVolumePath != null) : "Virtual disk file must exist in specified datastore for attach/detach operations."; @@ -4265,6 +4621,8 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa if (cmd.isManaged()) { handleDatastoreAndVmdkDetach(cmd.get_iScsiName(), cmd.getStorageHost(), cmd.getStoragePort()); + } else { + VmwareStorageLayoutHelper.syncVolumeToRootFolder(dsMo.getOwnerDatacenter().first(), dsMo, cmd.getVolumePath()); } } @@ -4423,10 +4781,14 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa auth.setChapAuthenticationType(strAuthType); auth.setChapName(chapName); auth.setChapSecret(chapSecret); - auth.setMutualChapInherited(false); - auth.setMutualChapAuthenticationType(strAuthType); - auth.setMutualChapName(mutualChapName); - auth.setMutualChapSecret(mutualChapSecret); + + if (StringUtils.isNotBlank(mutualChapName) && + StringUtils.isNotBlank(mutualChapSecret)) { + auth.setMutualChapInherited(false); + auth.setMutualChapAuthenticationType(strAuthType); + auth.setMutualChapName(mutualChapName); + auth.setMutualChapSecret(mutualChapSecret); + } target.setAuthenticationProperties(auth); @@ -5312,7 +5674,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa @Override - public synchronized CreateAnswer execute(CreateCommand cmd) { + public CreateAnswer execute(CreateCommand cmd) { if (s_logger.isInfoEnabled()) { s_logger.info("Executing resource CreateCommand: " + _gson.toJson(cmd)); } @@ -5346,12 +5708,11 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } String volumeDatastorePath = String.format("[%s] %s.vmdk", dsMo.getName(), vmdkName); - synchronized (this) { - s_logger.info("Delete file if exists in datastore to clear the way for creating the volume. file: " + volumeDatastorePath); - VmwareHelper.deleteVolumeVmdkFiles(dsMo, vmdkName, dcMo); - vmMo.createDisk(volumeDatastorePath, getMBsFromBytes(dskch.getSize()), morDatastore, -1); - vmMo.detachDisk(volumeDatastorePath, false); - } + + s_logger.info("Delete file if exists in datastore to clear the way for creating the volume. file: " + volumeDatastorePath); + VmwareStorageLayoutHelper.deleteVolumeVmdkFiles(dsMo, vmdkName, dcMo); + vmMo.createDisk(volumeDatastorePath, getMBsFromBytes(dskch.getSize()), morDatastore, -1); + vmMo.detachDisk(volumeDatastorePath, false); VolumeTO vol = new VolumeTO(cmd.getVolumeId(), dskch.getType(), pool.getType(), pool.getUuid(), dskch.getName(), pool.getPath(), vmdkName, dskch.getSize(), null); return new CreateAnswer(cmd, vol); @@ -5370,7 +5731,6 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa ManagedObjectReference morPool = hyperHost.getHyperHostOwnerResourcePool(); ManagedObjectReference morCluster = hyperHost.getHyperHostCluster(); - //createVMLinkedClone(vmTemplate, dcMo, dsMo, vmdkName, morDatastore, morPool); if (!_fullCloneFlag) { createVMLinkedClone(vmTemplate, dcMo, dsMo, vmdkName, morDatastore, morPool); } else { @@ -5404,13 +5764,11 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa throw new Exception("Unable to create a dummy VM for volume creation"); } - synchronized (this) { - // s_logger.info("Delete file if exists in datastore to clear the way for creating the volume. file: " + volumeDatastorePath); - VmwareHelper.deleteVolumeVmdkFiles(dsMo, volumeUuid.toString(), dcMo); + // s_logger.info("Delete file if exists in datastore to clear the way for creating the volume. file: " + volumeDatastorePath); + VmwareStorageLayoutHelper.deleteVolumeVmdkFiles(dsMo, volumeUuid.toString(), dcMo); - vmMo.createDisk(volumeDatastorePath, getMBsFromBytes(dskch.getSize()), morDatastore, vmMo.getScsiDeviceControllerKey()); - vmMo.detachDisk(volumeDatastorePath, false); - } + vmMo.createDisk(volumeDatastorePath, getMBsFromBytes(dskch.getSize()), morDatastore, vmMo.getScsiDeviceControllerKey()); + vmMo.detachDisk(volumeDatastorePath, false); VolumeTO vol = new VolumeTO(cmd.getVolumeId(), dskch.getType(), pool.getType(), pool.getUuid(), dskch.getName(), pool.getPath(), volumeUuid, dskch.getSize(), null); return new CreateAnswer(cmd, vol); @@ -5475,80 +5833,103 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa @Override public PingCommand getCurrentStatus(long id) { - HashMap newStates = sync(); - if (newStates == null) { - return null; - } - - try { - // take the chance to do left-over dummy VM cleanup from previous run - VmwareContext context = getServiceContext(); - VmwareHypervisorHost hyperHost = getHyperHost(context); - VmwareManager mgr = hyperHost.getContext().getStockObject(VmwareManager.CONTEXT_STOCK_NAME); - - if(hyperHost.isHyperHostConnected()) { - mgr.gcLeftOverVMs(context); - - if(_recycleHungWorker) { - s_logger.info("Scan hung worker VM to recycle"); - - // GC worker that has been running for too long - ObjectContent[] ocs = hyperHost.getVmPropertiesOnHyperHost( - new String[] {"name", "config.template", "runtime.powerState", "runtime.bootTime"}); - if(ocs != null) { - for(ObjectContent oc : ocs) { - List props = oc.getPropSet(); - if(props != null) { - String name = null; - boolean template = false; - VirtualMachinePowerState powerState = VirtualMachinePowerState.POWERED_OFF; - GregorianCalendar bootTime = null; - - for(DynamicProperty prop : props) { - if(prop.getName().equals("name")) - name = prop.getVal().toString(); - else if(prop.getName().equals("config.template")) - template = (Boolean)prop.getVal(); - else if(prop.getName().equals("runtime.powerState")) - powerState = (VirtualMachinePowerState)prop.getVal(); - else if(prop.getName().equals("runtime.bootTime")) - bootTime = (GregorianCalendar)prop.getVal(); - } - - if(!template && name.matches("[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}")) { - boolean recycle = false; - - // recycle stopped worker VM and VM that has been running for too long (hard-coded 10 hours for now) - if(powerState == VirtualMachinePowerState.POWERED_OFF) - recycle = true; - else if(bootTime != null && (new Date().getTime() - bootTime.getTimeInMillis() > 10*3600*1000)) - recycle = true; - - if(recycle) { - s_logger.info("Recycle pending worker VM: " + name); - - VirtualMachineMO vmMo = new VirtualMachineMO(hyperHost.getContext(), oc.getObj()); - vmMo.powerOff(); - vmMo.destroy(); - } - } - } - } - } - } - } else { - s_logger.error("Host is no longer connected."); - return null; - } - } catch (Throwable e) { - if (e instanceof RemoteException) { - s_logger.warn("Encounter remote exception to vCenter, invalidate VMware session context"); - invalidateServiceContext(); - return null; - } - } - - return new PingRoutingCommand(getType(), id, newStates); + try { + HashMap newStates = sync(); + if (newStates == null) { + return null; + } + + try { + // take the chance to do left-over dummy VM cleanup from previous run + VmwareContext context = getServiceContext(); + VmwareHypervisorHost hyperHost = getHyperHost(context); + VmwareManager mgr = hyperHost.getContext().getStockObject(VmwareManager.CONTEXT_STOCK_NAME); + + if(hyperHost.isHyperHostConnected()) { + mgr.gcLeftOverVMs(context); + + if(_recycleHungWorker) { + s_logger.info("Scan hung worker VM to recycle"); + + int key = ((HostMO)hyperHost).getCustomFieldKey("VirtualMachine", CustomFieldConstants.CLOUD_VM_INTERNAL_NAME); + if(key == 0) { + s_logger.warn("Custom field " + CustomFieldConstants.CLOUD_VM_INTERNAL_NAME + " is not registered ?!"); + } + String instanceNameCustomField = "value[" + key + "]"; + + // GC worker that has been running for too long + ObjectContent[] ocs = hyperHost.getVmPropertiesOnHyperHost( + new String[] {"name", "config.template", "runtime.powerState", "runtime.bootTime", instanceNameCustomField }); + if(ocs != null) { + for(ObjectContent oc : ocs) { + List props = oc.getPropSet(); + if(props != null) { + String vmName = null; + String internalName = null; + boolean template = false; + VirtualMachinePowerState powerState = VirtualMachinePowerState.POWERED_OFF; + GregorianCalendar bootTime = null; + + for(DynamicProperty prop : props) { + if (prop.getName().equals("name")) + vmName = prop.getVal().toString(); + else if(prop.getName().startsWith("value[")) { + if(prop.getVal() != null) + internalName = ((CustomFieldStringValue)prop.getVal()).getValue(); + } + else if(prop.getName().equals("config.template")) + template = (Boolean)prop.getVal(); + else if(prop.getName().equals("runtime.powerState")) + powerState = (VirtualMachinePowerState)prop.getVal(); + else if(prop.getName().equals("runtime.bootTime")) + bootTime = (GregorianCalendar)prop.getVal(); + } + + VirtualMachineMO vmMo = new VirtualMachineMO(hyperHost.getContext(), oc.getObj()); + String name = null; + if (internalName != null) { + name = internalName; + } else { + name = vmName; + } + + if(!template && name.matches("[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}")) { + boolean recycle = false; + + // recycle stopped worker VM and VM that has been running for too long (hard-coded 10 hours for now) + if(powerState == VirtualMachinePowerState.POWERED_OFF) + recycle = true; + else if(bootTime != null && (new Date().getTime() - bootTime.getTimeInMillis() > 10*3600*1000)) + recycle = true; + + if(recycle) { + s_logger.info("Recycle pending worker VM: " + name); + + vmMo.powerOff(); + vmMo.destroy(); + } + } + } + } + } + } + } else { + s_logger.error("Host is no longer connected."); + return null; + } + } catch (Throwable e) { + if (e instanceof RemoteException) { + s_logger.warn("Encounter remote exception to vCenter, invalidate VMware session context"); + invalidateServiceContext(); + return null; + } + } + + return new PingRoutingCommand(getType(), id, newStates); + + } finally { + recycleServiceContext(); + } } @Override @@ -5558,48 +5939,54 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa @Override public StartupCommand[] initialize() { - String hostApiVersion = "4.1"; - VmwareContext context = getServiceContext(); - try { - VmwareHypervisorHost hyperHost = getHyperHost(context); - assert(hyperHost instanceof HostMO); - if(!((HostMO)hyperHost).isHyperHostConnected()) { - s_logger.info("Host " + hyperHost.getHyperHostName() + " is not in connected state"); - return null; - } - - AboutInfo aboutInfo = ((HostMO)hyperHost).getHostAboutInfo(); - hostApiVersion = aboutInfo.getApiVersion(); - - } catch (Exception e) { - String msg = "VmwareResource intialize() failed due to : " + VmwareHelper.getExceptionMessage(e); - s_logger.error(msg); - invalidateServiceContext(); - return null; - } - - StartupRoutingCommand cmd = new StartupRoutingCommand(); - fillHostInfo(cmd); - - Map changes = null; - synchronized (_vms) { - _vms.clear(); - changes = sync(); - } - - cmd.setHypervisorType(HypervisorType.VMware); - cmd.setStateChanges(changes); - cmd.setCluster(_cluster); - cmd.setHypervisorVersion(hostApiVersion); - - List storageCmds = initializeLocalStorage(); - StartupCommand[] answerCmds = new StartupCommand[1 + storageCmds.size()]; - answerCmds[0] = cmd; - for (int i = 0; i < storageCmds.size(); i++) { - answerCmds[i + 1] = storageCmds.get(i); - } - - return answerCmds; + try { + String hostApiVersion = "4.1"; + VmwareContext context = getServiceContext(); + try { + VmwareHypervisorHost hyperHost = getHyperHost(context); + assert(hyperHost instanceof HostMO); + if(!((HostMO)hyperHost).isHyperHostConnected()) { + s_logger.info("Host " + hyperHost.getHyperHostName() + " is not in connected state"); + return null; + } + + ((HostMO)hyperHost).enableVncOnHostFirewall(); + + AboutInfo aboutInfo = ((HostMO)hyperHost).getHostAboutInfo(); + hostApiVersion = aboutInfo.getApiVersion(); + + } catch (Exception e) { + String msg = "VmwareResource intialize() failed due to : " + VmwareHelper.getExceptionMessage(e); + s_logger.error(msg); + invalidateServiceContext(); + return null; + } + + StartupRoutingCommand cmd = new StartupRoutingCommand(); + fillHostInfo(cmd); + + Map changes = null; + synchronized (_vms) { + _vms.clear(); + changes = sync(); + } + + cmd.setHypervisorType(HypervisorType.VMware); + cmd.setStateChanges(changes); + cmd.setCluster(_cluster); + cmd.setHypervisorVersion(hostApiVersion); + + List storageCmds = initializeLocalStorage(); + StartupCommand[] answerCmds = new StartupCommand[1 + storageCmds.size()]; + answerCmds[0] = cmd; + for (int i = 0; i < storageCmds.size(); i++) { + answerCmds[i + 1] = storageCmds.get(i); + } + + return answerCmds; + } finally { + recycleServiceContext(); + } } private List initializeLocalStorage() { @@ -5946,7 +6333,16 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa private HashMap getVmStates() throws Exception { VmwareHypervisorHost hyperHost = getHyperHost(getServiceContext()); - ObjectContent[] ocs = hyperHost.getVmPropertiesOnHyperHost(new String[] { "name", "runtime.powerState", "config.template" }); + + int key = ((HostMO)hyperHost).getCustomFieldKey("VirtualMachine", CustomFieldConstants.CLOUD_VM_INTERNAL_NAME); + if(key == 0) { + s_logger.warn("Custom field " + CustomFieldConstants.CLOUD_VM_INTERNAL_NAME + " is not registered ?!"); + } + String instanceNameCustomField = "value[" + key + "]"; + + // CLOUD_VM_INTERNAL_NAME stores the internal CS generated vm name. This was earlier stored in name. Now, name can be either the hostname or + // the internal CS name, but the custom field CLOUD_VM_INTERNAL_NAME always stores the internal CS name. + ObjectContent[] ocs = hyperHost.getVmPropertiesOnHyperHost(new String[] { "name", "runtime.powerState", "config.template", instanceNameCustomField }); HashMap newStates = new HashMap(); if (ocs != null && ocs.length > 0) { @@ -5956,20 +6352,28 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa boolean isTemplate = false; String name = null; + String VMInternalCSName = null; VirtualMachinePowerState powerState = VirtualMachinePowerState.POWERED_OFF; for (DynamicProperty objProp : objProps) { if (objProp.getName().equals("config.template")) { if (objProp.getVal().toString().equalsIgnoreCase("true")) { isTemplate = true; } - } else if (objProp.getName().equals("name")) { - name = (String) objProp.getVal(); } else if (objProp.getName().equals("runtime.powerState")) { powerState = (VirtualMachinePowerState) objProp.getVal(); - } else { + } else if (objProp.getName().equals("name")) { + name = (String) objProp.getVal(); + } else if(objProp.getName().contains(instanceNameCustomField)) { + if(objProp.getVal() != null) + VMInternalCSName = ((CustomFieldStringValue)objProp.getVal()).getValue(); + } + else { assert (false); } } + + if (VMInternalCSName != null) + name = VMInternalCSName; if (!isTemplate) { newStates.put(name, convertState(powerState)); @@ -5999,8 +6403,14 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } } } + + int key = ((HostMO)hyperHost).getCustomFieldKey("VirtualMachine", CustomFieldConstants.CLOUD_VM_INTERNAL_NAME); + if(key == 0) { + s_logger.warn("Custom field " + CustomFieldConstants.CLOUD_VM_INTERNAL_NAME + " is not registered ?!"); + } + String instanceNameCustomField = "value[" + key + "]"; - ObjectContent[] ocs = hyperHost.getVmPropertiesOnHyperHost(new String[] {"name", "summary.config.numCpu", "summary.quickStats.overallCpuUsage"}); + ObjectContent[] ocs = hyperHost.getVmPropertiesOnHyperHost(new String[] {"name", "summary.config.numCpu", "summary.quickStats.overallCpuUsage", instanceNameCustomField}); if (ocs != null && ocs.length > 0) { for (ObjectContent oc : ocs) { List objProps = oc.getPropSet(); @@ -6008,16 +6418,27 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa String name = null; String numberCPUs = null; String maxCpuUsage = null; - + String vmNameOnVcenter = null; + String vmInternalCSName = null; for (DynamicProperty objProp : objProps) { if (objProp.getName().equals("name")) { - name = objProp.getVal().toString(); - } else if (objProp.getName().equals("summary.config.numCpu")) { + vmNameOnVcenter = objProp.getVal().toString(); + } else if(objProp.getName().contains(instanceNameCustomField)) { + if(objProp.getVal() != null) + vmInternalCSName = ((CustomFieldStringValue)objProp.getVal()).getValue(); + } + else if (objProp.getName().equals("summary.config.numCpu")) { numberCPUs = objProp.getVal().toString(); } else if (objProp.getName().equals("summary.quickStats.overallCpuUsage")) { maxCpuUsage = objProp.getVal().toString(); } } + VirtualMachineMO vmMo = new VirtualMachineMO(hyperHost.getContext(), oc.getObj()); + if (vmInternalCSName != null) { + name = vmInternalCSName; + } else { + name = vmNameOnVcenter; + } if (!vmNames.contains(name)) { continue; @@ -6235,106 +6656,106 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa @Override public boolean configure(String name, Map params) throws ConfigurationException { - _name = name; + try { + _name = name; + + _url = (String) params.get("url"); + _username = (String) params.get("username"); + _password = (String) params.get("password"); + _dcId = (String) params.get("zone"); + _pod = (String) params.get("pod"); + _cluster = (String) params.get("cluster"); + + _guid = (String) params.get("guid"); + String[] tokens = _guid.split("@"); + _vCenterAddress = tokens[1]; + _morHyperHost = new ManagedObjectReference(); + String[] hostTokens = tokens[0].split(":"); + _morHyperHost.setType(hostTokens[0]); + _morHyperHost.setValue(hostTokens[1]); + + _guestTrafficInfo = (VmwareTrafficLabel) params.get("guestTrafficInfo"); + _publicTrafficInfo = (VmwareTrafficLabel) params.get("publicTrafficInfo"); + VmwareContext context = getServiceContext(); + volMgr = ComponentContext.inject(VolumeManagerImpl.class); + try { + VmwareManager mgr = context.getStockObject(VmwareManager.CONTEXT_STOCK_NAME); + mgr.setupResourceStartupParams(params); + + CustomFieldsManagerMO cfmMo = new CustomFieldsManagerMO(context, context.getServiceContent().getCustomFieldsManager()); + cfmMo.ensureCustomFieldDef("Datastore", CustomFieldConstants.CLOUD_UUID); + if (_publicTrafficInfo != null && _publicTrafficInfo.getVirtualSwitchType() != VirtualSwitchType.StandardVirtualSwitch || + _guestTrafficInfo != null && _guestTrafficInfo.getVirtualSwitchType() != VirtualSwitchType.StandardVirtualSwitch) { + cfmMo.ensureCustomFieldDef("DistributedVirtualPortgroup", CustomFieldConstants.CLOUD_GC_DVP); + } + cfmMo.ensureCustomFieldDef("Network", CustomFieldConstants.CLOUD_GC); + cfmMo.ensureCustomFieldDef("VirtualMachine", CustomFieldConstants.CLOUD_UUID); + cfmMo.ensureCustomFieldDef("VirtualMachine", CustomFieldConstants.CLOUD_NIC_MASK); + cfmMo.ensureCustomFieldDef("VirtualMachine", CustomFieldConstants.CLOUD_VM_INTERNAL_NAME); + + VmwareHypervisorHost hostMo = this.getHyperHost(context); + _hostName = hostMo.getHyperHostName(); + + if (_guestTrafficInfo.getVirtualSwitchType() == VirtualSwitchType.NexusDistributedVirtualSwitch || + _publicTrafficInfo.getVirtualSwitchType() == VirtualSwitchType.NexusDistributedVirtualSwitch) { + _vsmCredentials = mgr.getNexusVSMCredentialsByClusterId(Long.parseLong(_cluster)); + _privateNetworkVSwitchName = mgr.getPrivateVSwitchName(Long.parseLong(_dcId), HypervisorType.VMware); + } + + } catch (Exception e) { + s_logger.error("Unexpected Exception ", e); + } + + if(_privateNetworkVSwitchName == null) { + _privateNetworkVSwitchName = (String) params.get("private.network.vswitch.name"); + } + + String value = (String) params.get("vmware.recycle.hung.wokervm"); + if(value != null && value.equalsIgnoreCase("true")) + _recycleHungWorker = true; - _url = (String) params.get("url"); - _username = (String) params.get("username"); - _password = (String) params.get("password"); - _dcId = (String) params.get("zone"); - _pod = (String) params.get("pod"); - _cluster = (String) params.get("cluster"); - - _guid = (String) params.get("guid"); - String[] tokens = _guid.split("@"); - _vCenterAddress = tokens[1]; - _morHyperHost = new ManagedObjectReference(); - String[] hostTokens = tokens[0].split(":"); - _morHyperHost.setType(hostTokens[0]); - _morHyperHost.setValue(hostTokens[1]); - - _guestTrafficInfo = (VmwareTrafficLabel) params.get("guestTrafficInfo"); - _publicTrafficInfo = (VmwareTrafficLabel) params.get("publicTrafficInfo"); - VmwareContext context = getServiceContext(); - volMgr = ComponentContext.inject(VolumeManagerImpl.class); - try { - VmwareManager mgr = context.getStockObject(VmwareManager.CONTEXT_STOCK_NAME); - mgr.setupResourceStartupParams(params); - - CustomFieldsManagerMO cfmMo = new CustomFieldsManagerMO(context, context.getServiceContent().getCustomFieldsManager()); - cfmMo.ensureCustomFieldDef("Datastore", CustomFieldConstants.CLOUD_UUID); - if (_publicTrafficInfo != null && _publicTrafficInfo.getVirtualSwitchType() != VirtualSwitchType.StandardVirtualSwitch || - _guestTrafficInfo != null && _guestTrafficInfo.getVirtualSwitchType() != VirtualSwitchType.StandardVirtualSwitch) { - cfmMo.ensureCustomFieldDef("DistributedVirtualPortgroup", CustomFieldConstants.CLOUD_GC_DVP); - } - cfmMo.ensureCustomFieldDef("Network", CustomFieldConstants.CLOUD_GC); - cfmMo.ensureCustomFieldDef("VirtualMachine", CustomFieldConstants.CLOUD_UUID); - cfmMo.ensureCustomFieldDef("VirtualMachine", CustomFieldConstants.CLOUD_NIC_MASK); - - VmwareHypervisorHost hostMo = this.getHyperHost(context); - _hostName = hostMo.getHyperHostName(); - - Map vsmCredentials; - if (_guestTrafficInfo.getVirtualSwitchType() == VirtualSwitchType.NexusDistributedVirtualSwitch || - _publicTrafficInfo.getVirtualSwitchType() == VirtualSwitchType.NexusDistributedVirtualSwitch) { - vsmCredentials = mgr.getNexusVSMCredentialsByClusterId(Long.parseLong(_cluster)); - if (vsmCredentials != null) { - s_logger.info("Stocking credentials while configuring resource."); - context.registerStockObject("vsmcredentials", vsmCredentials); - } - _privateNetworkVSwitchName = mgr.getPrivateVSwitchName(Long.parseLong(_dcId), HypervisorType.VMware); - } - - } catch (Exception e) { - s_logger.error("Unexpected Exception ", e); - } - - if(_privateNetworkVSwitchName == null) { - _privateNetworkVSwitchName = (String) params.get("private.network.vswitch.name"); - } - - String value = (String) params.get("vmware.reserve.cpu"); - if(value != null && value.equalsIgnoreCase("true")) - _reserveCpu = true; - - value = (String) params.get("vmware.recycle.hung.wokervm"); - if(value != null && value.equalsIgnoreCase("true")) - _recycleHungWorker = true; - - value = (String) params.get("vmware.reserve.mem"); - if(value != null && value.equalsIgnoreCase("true")) - _reserveMem = true; - - value = (String)params.get("vmware.root.disk.controller"); - if(value != null && value.equalsIgnoreCase("scsi")) - _rootDiskController = DiskControllerType.scsi; - else - _rootDiskController = DiskControllerType.ide; - - Integer intObj = (Integer) params.get("ports.per.dvportgroup"); - if (intObj != null) - _portsPerDvPortGroup = intObj.intValue(); - - s_logger.info("VmwareResource network configuration info." + - " private traffic over vSwitch: " + _privateNetworkVSwitchName + ", public traffic over " + - _publicTrafficInfo.getVirtualSwitchType() + " : " + _publicTrafficInfo.getVirtualSwitchName() + - ", guest traffic over " + _guestTrafficInfo.getVirtualSwitchType() + " : " + - _guestTrafficInfo.getVirtualSwitchName()); - - value = params.get("vmware.create.full.clone").toString(); - if (value != null && value.equalsIgnoreCase("true")) { - _fullCloneFlag = true; - } else { - _fullCloneFlag = false; - } - value = (String)params.get("scripts.timeout"); - int timeout = NumbersUtil.parseInt(value, 1440) * 1000; - VmwareManager mgr = context.getStockObject(VmwareManager.CONTEXT_STOCK_NAME); - VmwareStorageProcessor storageProcessor = new VmwareStorageProcessor((VmwareHostService)this, _fullCloneFlag, (VmwareStorageMount)mgr, - timeout, this, _shutdown_waitMs - ); - storageHandler = new StorageSubsystemCommandHandlerBase(storageProcessor); - - return true; + value = (String)params.get("vmware.root.disk.controller"); + if(value != null && value.equalsIgnoreCase("scsi")) + _rootDiskController = DiskControllerType.scsi; + else + _rootDiskController = DiskControllerType.ide; + + Integer intObj = (Integer) params.get("ports.per.dvportgroup"); + if (intObj != null) + _portsPerDvPortGroup = intObj.intValue(); + + s_logger.info("VmwareResource network configuration info." + + " private traffic over vSwitch: " + _privateNetworkVSwitchName + ", public traffic over " + + _publicTrafficInfo.getVirtualSwitchType() + " : " + _publicTrafficInfo.getVirtualSwitchName() + + ", guest traffic over " + _guestTrafficInfo.getVirtualSwitchType() + " : " + + _guestTrafficInfo.getVirtualSwitchName()); + + value = params.get("vmware.create.full.clone").toString(); + if (value != null && value.equalsIgnoreCase("true")) { + _fullCloneFlag = true; + } else { + _fullCloneFlag = false; + } + + value = params.get("vm.instancename.flag").toString(); + if (value != null && value.equalsIgnoreCase("true")) { + _instanceNameFlag = true; + } else { + _instanceNameFlag = false; + } + + value = (String)params.get("scripts.timeout"); + int timeout = NumbersUtil.parseInt(value, 1440) * 1000; + VmwareManager mgr = context.getStockObject(VmwareManager.CONTEXT_STOCK_NAME); + VmwareStorageProcessor storageProcessor = new VmwareStorageProcessor((VmwareHostService)this, _fullCloneFlag, (VmwareStorageMount)mgr, + timeout, this, _shutdown_waitMs, null + ); + storageHandler = new VmwareStorageSubsystemCommandHandler(storageProcessor); + + return true; + } finally { + recycleServiceContext(); + } } @Override @@ -6365,48 +6786,40 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } @Override - public synchronized VmwareContext getServiceContext(Command cmd) { - if (_serviceContext == null) { - try { - _serviceContext = VmwareContextFactory.create(_vCenterAddress, _username, _password); - VmwareHypervisorHost hyperHost = getHyperHost(_serviceContext, cmd); - assert(hyperHost instanceof HostMO); - - HostFirewallSystemMO firewallMo = ((HostMO)hyperHost).getHostFirewallSystemMO(); - boolean bRefresh = false; - if(firewallMo != null) { - HostFirewallInfo firewallInfo = firewallMo.getFirewallInfo(); - if(firewallInfo != null && firewallInfo.getRuleset() != null) { - for(HostFirewallRuleset rule : firewallInfo.getRuleset()) { - if("vncServer".equalsIgnoreCase(rule.getKey())) { - bRefresh = true; - firewallMo.enableRuleset("vncServer"); - } else if("gdbserver".equalsIgnoreCase(rule.getKey())) { - bRefresh = true; - firewallMo.enableRuleset("gdbserver"); - } - } - } - - if(bRefresh) - firewallMo.refreshFirewall(); - } - } catch (Exception e) { - s_logger.error("Unable to connect to vSphere server: " + _vCenterAddress, e); - throw new CloudRuntimeException("Unable to connect to vSphere server: " + _vCenterAddress); - } + public VmwareContext getServiceContext(Command cmd) { + if(s_serviceContext.get() != null) + return s_serviceContext.get(); + + VmwareContext context = null; + try { + context = VmwareContextFactory.getContext(_vCenterAddress, _username, _password); + s_serviceContext.set(context); + } catch (Exception e) { + s_logger.error("Unable to connect to vSphere server: " + _vCenterAddress, e); + throw new CloudRuntimeException("Unable to connect to vSphere server: " + _vCenterAddress); } - return _serviceContext; + return context; } @Override - public synchronized void invalidateServiceContext(VmwareContext context) { - if (_serviceContext != null) { - _serviceContext.close(); - } - _serviceContext = null; + public void invalidateServiceContext(VmwareContext context) { + assert(s_serviceContext.get() == context); + + s_serviceContext.set(null); + if(context != null) + context.close(); } + private static void recycleServiceContext() { + VmwareContext context = s_serviceContext.get(); + s_serviceContext.set(null); + + if(context != null) { + assert(context.getPool() != null); + context.getPool().returnContext(context); + } + } + @Override public VmwareHypervisorHost getHyperHost(VmwareContext context, Command cmd) { if (_morHyperHost.getType().equalsIgnoreCase("HostSystem")) { @@ -6431,7 +6844,6 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa @Override public void setName(String name) { // TODO Auto-generated method stub - } @Override @@ -6455,14 +6867,55 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa @Override public void setRunLevel(int level) { // TODO Auto-generated method stub - } @Override public Answer execute(DestroyCommand cmd) { - // TODO Auto-generated method stub - return null; + if (s_logger.isInfoEnabled()) { + s_logger.info("Executing resource DestroyCommand to evict template from storage pool: " + _gson.toJson(cmd)); + } + + try { + VmwareContext context = getServiceContext(null); + VmwareHypervisorHost hyperHost = getHyperHost(context, null); + VolumeTO vol = cmd.getVolume(); + + ManagedObjectReference morDs = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, vol.getPoolUuid()); + if (morDs == null) { + String msg = "Unable to find datastore based on volume mount point " + vol.getMountPoint(); + s_logger.error(msg); + throw new Exception(msg); + } + + ManagedObjectReference morCluster = hyperHost.getHyperHostCluster(); + ClusterMO clusterMo = new ClusterMO(context, morCluster); + + VirtualMachineMO vmMo = clusterMo.findVmOnHyperHost(vol.getPath()); + if (vmMo != null) { + if (s_logger.isInfoEnabled()) { + s_logger.info("Destroy template volume " + vol.getPath()); + } + vmMo.destroy(); + } + else{ + if (s_logger.isInfoEnabled()) { + s_logger.info("Template volume " + vol.getPath() + " is not found, no need to delete."); + } + } + return new Answer(cmd, true, "Success"); + + } catch (Throwable e) { + if (e instanceof RemoteException) { + s_logger.warn("Encounter remote exception to vCenter, invalidate VMware session context"); + invalidateServiceContext(null); + } + + String msg = "DestroyCommand failed due to " + VmwareHelper.getExceptionMessage(e); + s_logger.error(msg, e); + return new Answer(cmd, false, msg); + } } + private boolean isVMWareToolsInstalled(VirtualMachineMO vmMo) throws Exception{ GuestInfo guestInfo = vmMo.getVmGuestInfo(); return (guestInfo != null && guestInfo.getGuestState() != null && guestInfo.getGuestState().equalsIgnoreCase("running")); diff --git a/plugins/hypervisors/vmware/src/com/cloud/network/VmwareTrafficLabel.java b/plugins/hypervisors/vmware/src/com/cloud/network/VmwareTrafficLabel.java index 8d2890a3c52..e92dc897e90 100644 --- a/plugins/hypervisors/vmware/src/com/cloud/network/VmwareTrafficLabel.java +++ b/plugins/hypervisors/vmware/src/com/cloud/network/VmwareTrafficLabel.java @@ -58,15 +58,15 @@ public class VmwareTrafficLabel implements TrafficLabel { } private void _parseLabel(String networkLabel, VirtualSwitchType defVswitchType) { + // Set defaults for label in case of distributed vSwitch + if (defVswitchType.equals(VirtualSwitchType.VMwareDistributedVirtualSwitch)) { + _vSwitchName = DEFAULT_DVSWITCH_NAME; + _vSwitchType = VirtualSwitchType.VMwareDistributedVirtualSwitch; + } else if (defVswitchType.equals(VirtualSwitchType.NexusDistributedVirtualSwitch)) { + _vSwitchName = DEFAULT_NDVSWITCH_NAME; + _vSwitchType = VirtualSwitchType.NexusDistributedVirtualSwitch; + } if (networkLabel == null || networkLabel.isEmpty()) { - // Set defaults for label in case of distributed vSwitch - if (defVswitchType.equals(VirtualSwitchType.VMwareDistributedVirtualSwitch)) { - _vSwitchName = DEFAULT_DVSWITCH_NAME; - _vSwitchType = VirtualSwitchType.VMwareDistributedVirtualSwitch; - } else if (defVswitchType.equals(VirtualSwitchType.NexusDistributedVirtualSwitch)) { - _vSwitchName = DEFAULT_NDVSWITCH_NAME; - _vSwitchType = VirtualSwitchType.NexusDistributedVirtualSwitch; - } return; } String[] tokens = networkLabel.split(","); diff --git a/plugins/hypervisors/vmware/src/com/cloud/network/element/CiscoNexusVSMElement.java b/plugins/hypervisors/vmware/src/com/cloud/network/element/CiscoNexusVSMElement.java index 4629b6c2d3e..9a866987e54 100644 --- a/plugins/hypervisors/vmware/src/com/cloud/network/element/CiscoNexusVSMElement.java +++ b/plugins/hypervisors/vmware/src/com/cloud/network/element/CiscoNexusVSMElement.java @@ -17,11 +17,9 @@ package com.cloud.network.element; -import java.lang.Class; -import java.lang.String; +import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.ArrayList; import java.util.Set; import javax.ejb.Local; @@ -30,8 +28,8 @@ import javax.inject.Inject; import org.apache.log4j.Logger; import com.cloud.api.commands.DeleteCiscoNexusVSMCmd; -import com.cloud.api.commands.EnableCiscoNexusVSMCmd; import com.cloud.api.commands.DisableCiscoNexusVSMCmd; +import com.cloud.api.commands.EnableCiscoNexusVSMCmd; import com.cloud.api.commands.ListCiscoNexusVSMsCmd; import com.cloud.api.response.CiscoNexusVSMResponse; import com.cloud.configuration.Config; @@ -44,29 +42,30 @@ import com.cloud.event.ActionEvent; import com.cloud.event.EventTypes; import com.cloud.exception.ConcurrentOperationException; import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.ResourceInUseException; import com.cloud.exception.ResourceUnavailableException; -import com.cloud.network.CiscoNexusVSMDeviceVO; import com.cloud.network.CiscoNexusVSMDevice; import com.cloud.network.CiscoNexusVSMDeviceManagerImpl; +import com.cloud.network.CiscoNexusVSMDeviceVO; import com.cloud.network.Network; -import com.cloud.network.PhysicalNetworkServiceProvider; import com.cloud.network.Network.Capability; import com.cloud.network.Network.Provider; import com.cloud.network.Network.Service; +import com.cloud.network.PhysicalNetworkServiceProvider; import com.cloud.network.dao.CiscoNexusVSMDeviceDao; -import com.cloud.vm.NicProfile; -import com.cloud.vm.ReservationContext; -import com.cloud.vm.VirtualMachine; -import com.cloud.vm.VirtualMachineProfile; import com.cloud.offering.NetworkOffering; import com.cloud.org.Cluster; +import com.cloud.server.ManagementService; +import com.cloud.utils.Pair; import com.cloud.utils.cisco.n1kv.vsm.NetconfHelper; import com.cloud.utils.component.Manager; import com.cloud.utils.db.DB; import com.cloud.utils.db.Transaction; -import com.cloud.exception.ResourceInUseException; import com.cloud.utils.exception.CloudRuntimeException; -import com.cloud.server.ManagementService; +import com.cloud.vm.NicProfile; +import com.cloud.vm.ReservationContext; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.VirtualMachineProfile; @Local(value = NetworkElement.class) public class CiscoNexusVSMElement extends CiscoNexusVSMDeviceManagerImpl implements CiscoNexusVSMElementService, NetworkElement, Manager { @@ -261,7 +260,10 @@ public class CiscoNexusVSMElement extends CiscoNexusVSMDeviceManagerImpl impleme } @DB - public boolean validateVsmCluster(String vsmIp, String vsmUser, String vsmPassword, long clusterId, String clusterName) throws ResourceInUseException { + public Pair validateAndAddVsm(String vsmIp, String vsmUser, String vsmPassword, long clusterId, String clusterName) throws ResourceInUseException { + CiscoNexusVSMDeviceVO vsm = null; + boolean vsmAdded = false; + Long vsmId = 0L; if(vsmIp != null && vsmUser != null && vsmPassword != null) { NetconfHelper netconfClient; try { @@ -277,7 +279,7 @@ public class CiscoNexusVSMElement extends CiscoNexusVSMDeviceManagerImpl impleme Transaction txn; // If VSM already exists and is mapped to a cluster, fail this operation. - CiscoNexusVSMDeviceVO vsm = _vsmDao.getVSMbyIpaddress(vsmIp); + vsm = _vsmDao.getVSMbyIpaddress(vsmIp); if(vsm != null) { List clusterList = _clusterVSMDao.listByVSMId(vsm.getId()); if (clusterList != null && !clusterList.isEmpty()) { @@ -343,6 +345,10 @@ public class CiscoNexusVSMElement extends CiscoNexusVSMDeviceManagerImpl impleme _clusterDao.remove(clusterId); throw new CloudRuntimeException(msg); } - return true; + if (vsm != null) { + vsmAdded = true; + vsmId = vsm.getId(); + } + return new Pair(vsmAdded, vsmId); } } diff --git a/plugins/hypervisors/vmware/src/com/cloud/network/element/CiscoNexusVSMElementService.java b/plugins/hypervisors/vmware/src/com/cloud/network/element/CiscoNexusVSMElementService.java index e90581ae56c..7d1618c964b 100644 --- a/plugins/hypervisors/vmware/src/com/cloud/network/element/CiscoNexusVSMElementService.java +++ b/plugins/hypervisors/vmware/src/com/cloud/network/element/CiscoNexusVSMElementService.java @@ -20,13 +20,14 @@ package com.cloud.network.element; import java.util.List; import com.cloud.api.commands.DeleteCiscoNexusVSMCmd; -import com.cloud.api.commands.EnableCiscoNexusVSMCmd; import com.cloud.api.commands.DisableCiscoNexusVSMCmd; +import com.cloud.api.commands.EnableCiscoNexusVSMCmd; import com.cloud.api.commands.ListCiscoNexusVSMsCmd; import com.cloud.api.response.CiscoNexusVSMResponse; import com.cloud.exception.ResourceInUseException; -import com.cloud.network.CiscoNexusVSMDeviceVO; import com.cloud.network.CiscoNexusVSMDevice; +import com.cloud.network.CiscoNexusVSMDeviceVO; +import com.cloud.utils.Pair; import com.cloud.utils.component.PluggableService; public interface CiscoNexusVSMElementService extends PluggableService { @@ -74,5 +75,5 @@ public interface CiscoNexusVSMElementService extends PluggableService { * Validate Cisco Nexus VSM before associating with cluster * */ - public boolean validateVsmCluster(String vsmIp, String vsmUser, String vsmPassword, long clusterId, String clusterName) throws ResourceInUseException; + public Pair validateAndAddVsm(String vsmIp, String vsmUser, String vsmPassword, long clusterId, String clusterName) throws ResourceInUseException; } diff --git a/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareSecondaryStorageContextFactory.java b/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareSecondaryStorageContextFactory.java index 646ef633fc7..5365e58e78e 100644 --- a/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareSecondaryStorageContextFactory.java +++ b/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareSecondaryStorageContextFactory.java @@ -16,20 +16,18 @@ // under the License. package com.cloud.storage.resource; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; - import com.cloud.hypervisor.vmware.util.VmwareClient; import com.cloud.hypervisor.vmware.util.VmwareContext; +import com.cloud.hypervisor.vmware.util.VmwareContextPool; public class VmwareSecondaryStorageContextFactory { private static volatile int s_seq = 1; - private static Map s_contextMap = new HashMap(); + private static VmwareContextPool s_pool; public static void initFactoryEnvironment() { System.setProperty("axis.socketSecureFactory", "org.apache.axis.components.net.SunFakeTrustSocketFactory"); + s_pool = new VmwareContextPool(); } public static VmwareContext create(String vCenterAddress, String vCenterUserName, String vCenterPassword) throws Exception { @@ -37,37 +35,33 @@ public class VmwareSecondaryStorageContextFactory { assert(vCenterUserName != null); assert(vCenterPassword != null); - VmwareContext context = null; - - synchronized(s_contextMap) { - context = s_contextMap.get(vCenterAddress); - if(context == null) { - String serviceUrl = "https://" + vCenterAddress + "/sdk/vimService"; - //String[] params = new String[] {"--url", serviceUrl, "--username", vCenterUserName, "--password", vCenterPassword }; - VmwareClient vimClient = new VmwareClient(vCenterAddress + "-" + s_seq++); - vimClient.connect(serviceUrl, vCenterUserName, vCenterPassword); - context = new VmwareContext(vimClient, vCenterAddress); - context.registerStockObject("username", vCenterUserName); - context.registerStockObject("password", vCenterPassword); - - s_contextMap.put(vCenterAddress, context); - } - } - + String serviceUrl = "https://" + vCenterAddress + "/sdk/vimService"; + VmwareClient vimClient = new VmwareClient(vCenterAddress + "-" + s_seq++); + vimClient.connect(serviceUrl, vCenterUserName, vCenterPassword); + VmwareContext context = new VmwareContext(vimClient, vCenterAddress); assert(context != null); + + context.setPoolInfo(s_pool, VmwareContextPool.composePoolKey(vCenterAddress, vCenterUserName)); + s_pool.registerOutstandingContext(context); + return context; } - - public static void invalidate(VmwareContext context) { - synchronized(s_contextMap) { - for(Iterator> entryIter = s_contextMap.entrySet().iterator(); entryIter.hasNext();) { - Map.Entry entry = entryIter.next(); - if(entry.getValue() == context) { - entryIter.remove(); - } - } + + public static VmwareContext getContext(String vCenterAddress, String vCenterUserName, String vCenterPassword) throws Exception { + VmwareContext context = s_pool.getContext(vCenterAddress, vCenterUserName); + if(context == null) { + context = create(vCenterAddress, vCenterUserName, vCenterPassword); } - + + if(context != null) { + context.registerStockObject("username", vCenterUserName); + context.registerStockObject("password", vCenterPassword); + } + + return context; + } + + public static void invalidate(VmwareContext context) { context.close(); } } diff --git a/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareSecondaryStorageResourceHandler.java b/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareSecondaryStorageResourceHandler.java index 3631e38080d..dcf71cb0e03 100644 --- a/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareSecondaryStorageResourceHandler.java +++ b/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareSecondaryStorageResourceHandler.java @@ -45,6 +45,7 @@ import com.cloud.hypervisor.vmware.util.VmwareContext; import com.cloud.hypervisor.vmware.util.VmwareHelper; import com.cloud.serializer.GsonHelper; import com.cloud.utils.Pair; +import com.cloud.utils.StringUtils; import com.google.gson.Gson; import com.vmware.vim25.ManagedObjectReference; @@ -57,6 +58,8 @@ public class VmwareSecondaryStorageResourceHandler implements SecondaryStorageRe private final Gson _gson; private final StorageSubsystemCommandHandler storageSubsystemHandler; + + private static ThreadLocal currentContext = new ThreadLocal(); /* * private Map _activeHosts = new HashMap(); @@ -68,48 +71,57 @@ public class VmwareSecondaryStorageResourceHandler implements SecondaryStorageRe _gson = GsonHelper.getGsonLogger(); VmwareStorageProcessor storageProcessor = new VmwareStorageProcessor(this, true, this, resource.getTimeout(), - null, null); - storageSubsystemHandler = new StorageSubsystemCommandHandlerBase(storageProcessor); + null, null, _resource); + VmwareStorageSubsystemCommandHandler vmwareStorageSubsystemCommandHandler = new VmwareStorageSubsystemCommandHandler(storageProcessor); + vmwareStorageSubsystemCommandHandler.setStorageResource(_resource); + vmwareStorageSubsystemCommandHandler.setStorageManager(_storageMgr); + storageSubsystemHandler = vmwareStorageSubsystemCommandHandler; } @Override public Answer executeRequest(Command cmd) { - Answer answer; - if (cmd instanceof PrimaryStorageDownloadCommand) { - answer = execute((PrimaryStorageDownloadCommand) cmd); - } else if (cmd instanceof BackupSnapshotCommand) { - answer = execute((BackupSnapshotCommand) cmd); - } else if (cmd instanceof CreatePrivateTemplateFromVolumeCommand) { - answer = execute((CreatePrivateTemplateFromVolumeCommand) cmd); - } else if (cmd instanceof CreatePrivateTemplateFromSnapshotCommand) { - answer = execute((CreatePrivateTemplateFromSnapshotCommand) cmd); - } else if (cmd instanceof CopyVolumeCommand) { - answer = execute((CopyVolumeCommand) cmd); - } else if (cmd instanceof CreateVolumeFromSnapshotCommand) { - answer = execute((CreateVolumeFromSnapshotCommand) cmd); - } else if (cmd instanceof StorageSubSystemCommand) { - answer = storageSubsystemHandler.handleStorageCommands((StorageSubSystemCommand) cmd); - } else if (cmd instanceof CreateEntityDownloadURLCommand) { - answer = execute((CreateEntityDownloadURLCommand)cmd); - } else { - answer = _resource.defaultAction(cmd); - } - - // special handling to pass-back context info for cleanups - if (cmd.getContextParam("execid") != null) { - answer.setContextParam("execid", cmd.getContextParam("execid")); - } - - if (cmd.getContextParam("checkpoint") != null) { - answer.setContextParam("checkpoint", cmd.getContextParam("checkpoint")); - } - - if (cmd.getContextParam("checkpoint2") != null) { - answer.setContextParam("checkpoint2", cmd.getContextParam("checkpoint2")); - } - - return answer; + + try { + Answer answer; + if (cmd instanceof PrimaryStorageDownloadCommand) { + answer = execute((PrimaryStorageDownloadCommand) cmd); + } else if (cmd instanceof BackupSnapshotCommand) { + answer = execute((BackupSnapshotCommand) cmd); + } else if (cmd instanceof CreatePrivateTemplateFromVolumeCommand) { + answer = execute((CreatePrivateTemplateFromVolumeCommand) cmd); + } else if (cmd instanceof CreatePrivateTemplateFromSnapshotCommand) { + answer = execute((CreatePrivateTemplateFromSnapshotCommand) cmd); + } else if (cmd instanceof CopyVolumeCommand) { + answer = execute((CopyVolumeCommand) cmd); + } else if (cmd instanceof CreateVolumeFromSnapshotCommand) { + answer = execute((CreateVolumeFromSnapshotCommand) cmd); + } else if (cmd instanceof StorageSubSystemCommand) { + answer = storageSubsystemHandler.handleStorageCommands((StorageSubSystemCommand) cmd); + } else if (cmd instanceof CreateEntityDownloadURLCommand) { + answer = execute((CreateEntityDownloadURLCommand)cmd); + } else { + answer = _resource.defaultAction(cmd); + } + + // special handling to pass-back context info for cleanups + if (cmd.getContextParam("execid") != null) { + answer.setContextParam("execid", cmd.getContextParam("execid")); + } + + if (cmd.getContextParam("checkpoint") != null) { + answer.setContextParam("checkpoint", cmd.getContextParam("checkpoint")); + } + + if (cmd.getContextParam("checkpoint2") != null) { + answer.setContextParam("checkpoint2", cmd.getContextParam("checkpoint2")); + } + + return answer; + } finally { + recycleServiceContext(); + } } + protected Answer execute(CreateEntityDownloadURLCommand cmd) { boolean result = _storageMgr.execute(this, cmd); return _resource.defaultAction(cmd); @@ -195,30 +207,38 @@ public class VmwareSecondaryStorageResourceHandler implements SecondaryStorageRe try { _resource.ensureOutgoingRuleForAddress(vCenterAddress); - VmwareContext context = null; - - // cached VmwareContext may be timed out in vCenter, give it a - // chance to reclaim a new context from factory - for (int i = 0; i < 2; i++) { - context = VmwareSecondaryStorageContextFactory.create(vCenterAddress, username, password); - if (!validateContext(context, cmd)) { - invalidateServiceContext(context); - } - } - + + VmwareContext context = currentContext.get(); + if(context == null) { + s_logger.info("Open new VmwareContext. vCenter: " + vCenterAddress + ", user: " + username + + ", password: " + StringUtils.getMaskedPasswordForDisplay(password)); + context = VmwareSecondaryStorageContextFactory.getContext(vCenterAddress, username, password); + } + if (context != null) { context.registerStockObject("serviceconsole", cmd.getContextParam("serviceconsole")); context.registerStockObject("manageportgroup", cmd.getContextParam("manageportgroup")); } + currentContext.set(context); return context; } catch (Exception e) { s_logger.error("Unexpected exception " + e.toString(), e); return null; } } + + public void recycleServiceContext() { + if(currentContext.get() != null) { + VmwareContext context = currentContext.get(); + currentContext.set(null); + assert(context.getPool() != null); + context.getPool().returnContext(context); + } + } @Override public void invalidateServiceContext(VmwareContext context) { + currentContext.set(null); VmwareSecondaryStorageContextFactory.invalidate(context); } diff --git a/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageLayoutHelper.java b/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageLayoutHelper.java new file mode 100644 index 00000000000..24b13c8b2ca --- /dev/null +++ b/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageLayoutHelper.java @@ -0,0 +1,299 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.storage.resource; + +import java.util.List; + +import org.apache.log4j.Logger; + +import com.cloud.hypervisor.vmware.mo.DatacenterMO; +import com.cloud.hypervisor.vmware.mo.DatastoreFile; +import com.cloud.hypervisor.vmware.mo.DatastoreMO; +import com.cloud.utils.Pair; + +/** + * + * To provide helper methods to handle storage layout in one place + * + */ +public class VmwareStorageLayoutHelper { + private static final Logger s_logger = Logger.getLogger(VmwareStorageLayoutHelper.class); + + public static String[] getVmdkFilePairDatastorePath(DatastoreMO dsMo, String vmName, String vmdkName, + VmwareStorageLayoutType layoutType, boolean linkedVmdk) throws Exception { + + String[] filePair = new String[2]; + switch(layoutType) { + case VMWARE : + assert(vmName != null && !vmName.isEmpty()); + filePair[0] = getVmwareDatastorePathFromVmdkFileName(dsMo, vmName, vmdkName + ".vmdk"); + + if(linkedVmdk) + filePair[1] = getVmwareDatastorePathFromVmdkFileName(dsMo, vmName, vmdkName + "-delta.vmdk"); + else + filePair[1] = getVmwareDatastorePathFromVmdkFileName(dsMo, vmName, vmdkName + "-flat.vmdk"); + return filePair; + + case CLOUDSTACK_LEGACY : + filePair[0] = getLegacyDatastorePathFromVmdkFileName(dsMo, vmdkName + ".vmdk"); + + if(linkedVmdk) + filePair[1] = getLegacyDatastorePathFromVmdkFileName(dsMo, vmdkName + "-delta.vmdk"); + else + filePair[1] = getLegacyDatastorePathFromVmdkFileName(dsMo, vmdkName + "-flat.vmdk"); + return filePair; + + default : + assert(false); + break; + } + + assert(false); + return null; + } + + public static String findVolumeDatastoreFullPath(DatastoreMO dsMo, String vmName, String vmdkFileName) throws Exception { + if(vmName != null) { + String path = getVmwareDatastorePathFromVmdkFileName(dsMo, vmName, vmdkFileName); + if(!dsMo.fileExists(path)) { + path = getLegacyDatastorePathFromVmdkFileName(dsMo, vmdkFileName); + + // to save one call to vCenter, we won't check file existence for this round, so the caller + // may still fail with exception, but if that's case, we will let it fail anyway + } + return path; + } else { + String path = getLegacyDatastorePathFromVmdkFileName(dsMo, vmdkFileName); + if(!dsMo.fileExists(path)) { + // Datastore file movement is not atomic operations, we need to sync and repair + path = dsMo.searchFileInSubFolders(vmdkFileName, false); + + // to save one call to vCenter, we won't check file existence for this round, so the caller + // may still fail with exception, but if that's case, we will let it fail anyway + } + return path; + } + } + + public static String syncVolumeToVmDefaultFolder(DatacenterMO dcMo, String vmName, + DatastoreMO ds, String vmdkName) throws Exception { + + assert(ds != null); + if(!ds.folderExists(String.format("[%s]", ds.getName()), vmName)) { + s_logger.info("VM folder does not exist on target datastore, we will create one. vm: " + vmName + ", datastore: " + ds.getName()); + + ds.makeDirectory(String.format("[%s] %s", ds.getName(), vmName), dcMo.getMor()); + } + + String[] vmdkLinkedCloneModeLegacyPair = getVmdkFilePairDatastorePath(ds, vmName, vmdkName, + VmwareStorageLayoutType.CLOUDSTACK_LEGACY, true); + String[] vmdkFullCloneModeLegacyPair = getVmdkFilePairDatastorePath(ds, vmName, vmdkName, + VmwareStorageLayoutType.CLOUDSTACK_LEGACY, false); + + String[] vmdkLinkedCloneModePair = getVmdkFilePairDatastorePath(ds, vmName, vmdkName, + VmwareStorageLayoutType.VMWARE, true); + String[] vmdkFullCloneModePair = getVmdkFilePairDatastorePath(ds, vmName, vmdkName, + VmwareStorageLayoutType.VMWARE, false); + + if(!ds.fileExists(vmdkLinkedCloneModeLegacyPair[0]) && !ds.fileExists(vmdkLinkedCloneModePair[0])) { + // To protect against inconsistency caused by non-atomic datastore file management, detached disk may + // be left over in its previous owner VM. We will do a fixup synchronization here by moving it to root + // again. + // + syncVolumeToRootFolder(dcMo, ds, vmdkName); + } + + if(ds.fileExists(vmdkFullCloneModeLegacyPair[1])) { + s_logger.info("sync " + vmdkFullCloneModeLegacyPair[1] + "->" + vmdkFullCloneModePair[1]); + + ds.moveDatastoreFile(vmdkFullCloneModeLegacyPair[1], dcMo.getMor(), ds.getMor(), + vmdkFullCloneModePair[1], dcMo.getMor(), true); + } + + if(ds.fileExists(vmdkLinkedCloneModeLegacyPair[1])) { + s_logger.info("sync " + vmdkLinkedCloneModeLegacyPair[1] + "->" + vmdkLinkedCloneModePair[1]); + + ds.moveDatastoreFile(vmdkLinkedCloneModeLegacyPair[1], dcMo.getMor(), ds.getMor(), + vmdkLinkedCloneModePair[1], dcMo.getMor(), true); + } + + if(ds.fileExists(vmdkLinkedCloneModeLegacyPair[0])) { + s_logger.info("sync " + vmdkLinkedCloneModeLegacyPair[0] + "->" + vmdkLinkedCloneModePair[0]); + ds.moveDatastoreFile(vmdkLinkedCloneModeLegacyPair[0], dcMo.getMor(), ds.getMor(), + vmdkLinkedCloneModePair[0], dcMo.getMor(), true); + } + + // Note: we will always return a path + return vmdkLinkedCloneModePair[0]; + } + + public static void syncVolumeToRootFolder(DatacenterMO dcMo, DatastoreMO ds, String vmdkName) throws Exception { + String fileDsFullPath = ds.searchFileInSubFolders(vmdkName + ".vmdk", false); + if(fileDsFullPath == null) + return; + + DatastoreFile srcDsFile = new DatastoreFile(fileDsFullPath); + String companionFilePath = srcDsFile.getCompanionPath(vmdkName + "-flat.vmdk"); + if(ds.fileExists(companionFilePath)) { + String targetPath = getLegacyDatastorePathFromVmdkFileName(ds, vmdkName + "-flat.vmdk"); + + s_logger.info("Fixup folder-synchronization. move " + companionFilePath + " -> " + targetPath); + ds.moveDatastoreFile(companionFilePath, dcMo.getMor(), ds.getMor(), targetPath, dcMo.getMor(), true); + } + + companionFilePath = srcDsFile.getCompanionPath(vmdkName + "-delta.vmdk"); + if(ds.fileExists(companionFilePath)) { + String targetPath = getLegacyDatastorePathFromVmdkFileName(ds, vmdkName + "-delta.vmdk"); + + s_logger.info("Fixup folder-synchronization. move " + companionFilePath + " -> " + targetPath); + ds.moveDatastoreFile(companionFilePath, dcMo.getMor(), ds.getMor(), targetPath, dcMo.getMor(), true); + } + + // move the identity VMDK file the last + String targetPath = getLegacyDatastorePathFromVmdkFileName(ds, vmdkName + ".vmdk"); + s_logger.info("Fixup folder-synchronization. move " + fileDsFullPath + " -> " + targetPath); + ds.moveDatastoreFile(fileDsFullPath, dcMo.getMor(), ds.getMor(), targetPath, dcMo.getMor(), true); + } + + public static void moveVolumeToRootFolder(DatacenterMO dcMo, List detachedDisks) throws Exception { + if(detachedDisks.size() > 0) { + for(String fileFullDsPath : detachedDisks) { + DatastoreFile file = new DatastoreFile(fileFullDsPath); + + s_logger.info("Check if we need to move " + fileFullDsPath + " to its root location"); + DatastoreMO dsMo = new DatastoreMO(dcMo.getContext(), dcMo.findDatastore(file.getDatastoreName())); + if(dsMo.getMor() != null) { + DatastoreFile targetFile = new DatastoreFile(file.getDatastoreName(), file.getFileName()); + if(!targetFile.getPath().equalsIgnoreCase(file.getPath())) { + s_logger.info("Move " + file.getPath() + " -> " + targetFile.getPath()); + dsMo.moveDatastoreFile(file.getPath(), dcMo.getMor(), dsMo.getMor(), targetFile.getPath(), dcMo.getMor(), true); + + String pairSrcFilePath = file.getCompanionPath(file.getFileBaseName() + "-flat.vmdk"); + String pairTargetFilePath = targetFile.getCompanionPath(file.getFileBaseName() + "-flat.vmdk"); + if(dsMo.fileExists(pairSrcFilePath)) { + s_logger.info("Move " + pairSrcFilePath + " -> " + pairTargetFilePath); + dsMo.moveDatastoreFile(pairSrcFilePath, dcMo.getMor(), dsMo.getMor(), pairTargetFilePath, dcMo.getMor(), true); + } + + pairSrcFilePath = file.getCompanionPath(file.getFileBaseName() + "-delta.vmdk"); + pairTargetFilePath = targetFile.getCompanionPath(file.getFileBaseName() + "-delta.vmdk"); + if(dsMo.fileExists(pairSrcFilePath)) { + s_logger.info("Move " + pairSrcFilePath + " -> " + pairTargetFilePath); + dsMo.moveDatastoreFile(pairSrcFilePath, dcMo.getMor(), dsMo.getMor(), pairTargetFilePath, dcMo.getMor(), true); + } + } + } else { + s_logger.warn("Datastore for " + fileFullDsPath + " no longer exists, we have to skip"); + } + } + } + } + + public static String getTemplateOnSecStorageFilePath(String secStorageMountPoint, String templateRelativeFolderPath, + String templateName, String fileExtension) { + + StringBuffer sb = new StringBuffer(); + sb.append(secStorageMountPoint); + if(!secStorageMountPoint.endsWith("/")) + sb.append("/"); + + sb.append(templateRelativeFolderPath); + if(!secStorageMountPoint.endsWith("/")) + sb.append("/"); + + sb.append(templateName); + if(!fileExtension.startsWith(".")) + sb.append("."); + sb.append(fileExtension); + + return sb.toString(); + } + + /* + * return Pair of