diff --git a/api/src/com/cloud/network/NetworkModel.java b/api/src/com/cloud/network/NetworkModel.java index 9bf5d27eea8..fcb6a2e58f4 100644 --- a/api/src/com/cloud/network/NetworkModel.java +++ b/api/src/com/cloud/network/NetworkModel.java @@ -269,4 +269,6 @@ public interface NetworkModel { boolean getExecuteInSeqNtwkElmtCmd(); boolean isNetworkReadyForGc(long networkId); + + boolean getNetworkEgressDefaultPolicy(Long networkId); } \ No newline at end of file diff --git a/api/src/com/cloud/network/OvsProvider.java b/api/src/com/cloud/network/OvsProvider.java index c9937282bef..e20491a3a2d 100644 --- a/api/src/com/cloud/network/OvsProvider.java +++ b/api/src/com/cloud/network/OvsProvider.java @@ -1,3 +1,19 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. package com.cloud.network; import org.apache.cloudstack.api.Identity; diff --git a/api/src/com/cloud/network/as/AutoScaleCounter.java b/api/src/com/cloud/network/as/AutoScaleCounter.java index 57539729934..f25d6699a00 100644 --- a/api/src/com/cloud/network/as/AutoScaleCounter.java +++ b/api/src/com/cloud/network/as/AutoScaleCounter.java @@ -27,6 +27,8 @@ public class AutoScaleCounter { public static final AutoScaleCounterType Snmp = new AutoScaleCounterType("snmp"); public static final AutoScaleCounterType Netscaler = new AutoScaleCounterType("netscaler"); + public static final AutoScaleCounterType Cpu = new AutoScaleCounterType("cpu"); + public static final AutoScaleCounterType Memory = new AutoScaleCounterType("memory"); public AutoScaleCounterType(String name) { _name = name; diff --git a/api/src/com/cloud/network/as/AutoScalePolicy.java b/api/src/com/cloud/network/as/AutoScalePolicy.java index 434220a4c8d..8452ba1492c 100644 --- a/api/src/com/cloud/network/as/AutoScalePolicy.java +++ b/api/src/com/cloud/network/as/AutoScalePolicy.java @@ -17,6 +17,8 @@ package com.cloud.network.as; +import java.util.Date; + import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.api.InternalIdentity; @@ -30,6 +32,8 @@ public interface AutoScalePolicy extends ControlledEntity, InternalIdentity { public int getDuration(); public int getQuietTime(); + + public Date getLastQuiteTime(); public String getAction(); diff --git a/api/src/com/cloud/network/as/AutoScaleVmGroup.java b/api/src/com/cloud/network/as/AutoScaleVmGroup.java index 3e07fe44fb0..15f7bea9a4d 100644 --- a/api/src/com/cloud/network/as/AutoScaleVmGroup.java +++ b/api/src/com/cloud/network/as/AutoScaleVmGroup.java @@ -17,6 +17,8 @@ package com.cloud.network.as; +import java.util.Date; + import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.api.InternalIdentity; @@ -45,6 +47,8 @@ public interface AutoScaleVmGroup extends ControlledEntity, InternalIdentity { int getInterval(); + Date getLastInterval(); + String getState(); String getUuid(); diff --git a/api/src/com/cloud/network/as/Counter.java b/api/src/com/cloud/network/as/Counter.java index 46585f39679..99b3364419a 100644 --- a/api/src/com/cloud/network/as/Counter.java +++ b/api/src/com/cloud/network/as/Counter.java @@ -23,7 +23,10 @@ import org.apache.cloudstack.api.InternalIdentity; public interface Counter extends InternalIdentity, Identity { public static enum Source { - netscaler, snmp + netscaler, + snmp, + cpu, + memory } String getName(); @@ -31,4 +34,5 @@ public interface Counter extends InternalIdentity, Identity { String getValue(); Source getSource(); + } diff --git a/api/src/com/cloud/region/ha/GlobalLoadBalancerRule.java b/api/src/com/cloud/region/ha/GlobalLoadBalancerRule.java index 946c2f93b32..c5fd9aed2b6 100644 --- a/api/src/com/cloud/region/ha/GlobalLoadBalancerRule.java +++ b/api/src/com/cloud/region/ha/GlobalLoadBalancerRule.java @@ -50,9 +50,11 @@ public interface GlobalLoadBalancerRule extends Identity, InternalIdentity, Cont } enum ServiceType { - tcp, udp; + tcp, udp, http; public static boolean isValidServiceType(String serviceType) { - if (tcp.name().equalsIgnoreCase(serviceType) || udp.name().equalsIgnoreCase(serviceType)) { + if (tcp.name().equalsIgnoreCase(serviceType) || + udp.name().equalsIgnoreCase(serviceType) || + http.name().equalsIgnoreCase(serviceType)) { return true; } return false; diff --git a/api/src/org/apache/cloudstack/api/ApiConstants.java b/api/src/org/apache/cloudstack/api/ApiConstants.java index 745a722bfa4..a132e548258 100755 --- a/api/src/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/org/apache/cloudstack/api/ApiConstants.java @@ -502,6 +502,7 @@ public class ApiConstants { public static final String BAREMETAL_DISCOVER_NAME = "baremetaldiscovername"; public static final String UCS_DN = "ucsdn"; public static final String GSLB_PROVIDER = "gslbprovider"; + public static final String EXCLUSIVE_GSLB_PROVIDER = "isexclusivegslbprovider"; public static final String GSLB_PROVIDER_PUBLIC_IP = "gslbproviderpublicip"; public static final String GSLB_PROVIDER_PRIVATE_IP = "gslbproviderprivateip"; public static final String VM_SNAPSHOT_DESCRIPTION = "description"; diff --git a/api/src/org/apache/cloudstack/api/command/admin/router/ConfigureOvsElementCmd.java b/api/src/org/apache/cloudstack/api/command/admin/router/ConfigureOvsElementCmd.java index a72dbefb32e..26cfb31a69d 100644 --- a/api/src/org/apache/cloudstack/api/command/admin/router/ConfigureOvsElementCmd.java +++ b/api/src/org/apache/cloudstack/api/command/admin/router/ConfigureOvsElementCmd.java @@ -1,3 +1,19 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. package org.apache.cloudstack.api.command.admin.router; import java.util.List; diff --git a/api/src/org/apache/cloudstack/api/command/admin/router/ListOvsElementsCmd.java b/api/src/org/apache/cloudstack/api/command/admin/router/ListOvsElementsCmd.java index 3e3ee635f48..9f5feeedc26 100644 --- a/api/src/org/apache/cloudstack/api/command/admin/router/ListOvsElementsCmd.java +++ b/api/src/org/apache/cloudstack/api/command/admin/router/ListOvsElementsCmd.java @@ -1,3 +1,19 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. package org.apache.cloudstack.api.command.admin.router; import java.util.ArrayList; diff --git a/api/src/org/apache/cloudstack/api/command/user/region/ha/gslb/CreateGlobalLoadBalancerRuleCmd.java b/api/src/org/apache/cloudstack/api/command/user/region/ha/gslb/CreateGlobalLoadBalancerRuleCmd.java index bf2d86f50c9..550d81755eb 100644 --- a/api/src/org/apache/cloudstack/api/command/user/region/ha/gslb/CreateGlobalLoadBalancerRuleCmd.java +++ b/api/src/org/apache/cloudstack/api/command/user/region/ha/gslb/CreateGlobalLoadBalancerRuleCmd.java @@ -87,7 +87,7 @@ public class CreateGlobalLoadBalancerRuleCmd extends BaseAsyncCreateCmd { @Parameter(name = ApiConstants.GSLB_SERVICE_DOMAIN_NAME, type = CommandType.STRING, required = true, description = "domain name for the GSLB service.") private String serviceDomainName; - @Parameter(name = ApiConstants.GSLB_SERVICE_TYPE, type = CommandType.STRING, required = true, description = "GSLB service type (tcp, udp)") + @Parameter(name = ApiConstants.GSLB_SERVICE_TYPE, type = CommandType.STRING, required = true, description = "GSLB service type (tcp, udp, http)") private String serviceType; ///////////////////////////////////////////////////// diff --git a/api/src/org/apache/cloudstack/api/response/OvsProviderResponse.java b/api/src/org/apache/cloudstack/api/response/OvsProviderResponse.java index 23e61341072..223576d001d 100644 --- a/api/src/org/apache/cloudstack/api/response/OvsProviderResponse.java +++ b/api/src/org/apache/cloudstack/api/response/OvsProviderResponse.java @@ -1,3 +1,19 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. package org.apache.cloudstack.api.response; import org.apache.cloudstack.api.ApiConstants; diff --git a/client/tomcatconf/commands.properties.in b/client/tomcatconf/commands.properties.in index 966b7c4227b..a05310bff77 100644 --- a/client/tomcatconf/commands.properties.in +++ b/client/tomcatconf/commands.properties.in @@ -549,10 +549,6 @@ listSrxFirewalls=1 listSrxFirewallNetworks=1 #### Palo Alto firewall commands -addExternalFirewall=1 -deleteExternalFirewall=1 -listExternalFirewalls=1 - addPaloAltoFirewall=1 deletePaloAltoFirewall=1 configurePaloAltoFirewall=1 diff --git a/core/src/com/cloud/agent/api/PerformanceMonitorAnswer.java b/core/src/com/cloud/agent/api/PerformanceMonitorAnswer.java new file mode 100644 index 00000000000..10f4185719d --- /dev/null +++ b/core/src/com/cloud/agent/api/PerformanceMonitorAnswer.java @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.api; + +public class PerformanceMonitorAnswer extends Answer { + public PerformanceMonitorAnswer() { + } + + public PerformanceMonitorAnswer(PerformanceMonitorCommand cmd, + boolean result, String details) { + super(cmd, result, details); + } +} diff --git a/core/src/com/cloud/agent/api/PerformanceMonitorCommand.java b/core/src/com/cloud/agent/api/PerformanceMonitorCommand.java new file mode 100644 index 00000000000..882d7b18763 --- /dev/null +++ b/core/src/com/cloud/agent/api/PerformanceMonitorCommand.java @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.api; + +import java.util.HashMap; +import java.util.Map; + +public class PerformanceMonitorCommand extends Command { + + Map params = new HashMap(); + + public PerformanceMonitorCommand() { + } + + public PerformanceMonitorCommand(Map params, int wait) { + setWait(wait); + this.params = params; + } + + @Override + public boolean executeInSequence() { + return false; + } + + public Map getParams() { + return params; + } + + public void setParams(Map params) { + this.params = params; + } +} diff --git a/engine/components-api/src/com/cloud/capacity/CapacityManager.java b/engine/components-api/src/com/cloud/capacity/CapacityManager.java index 13624e69453..bd1a61096c0 100755 --- a/engine/components-api/src/com/cloud/capacity/CapacityManager.java +++ b/engine/components-api/src/com/cloud/capacity/CapacityManager.java @@ -91,4 +91,15 @@ public interface CapacityManager { * @return true if the count of host's running VMs >= hypervisor limit */ boolean checkIfHostHasCpuCapability(long hostId, Integer cpuNum, Integer cpuSpeed); + + /** + * Check if cluster will cross threshold if the cpu/memory requested are accomodated + * @param clusterId the clusterId to check + * @param cpuRequested cpu requested + * @param ramRequested cpu requested + * @return true if the customer crosses threshold, false otherwise + */ + boolean checkIfClusterCrossesThreshold(Long clusterId, Integer cpuRequested, long ramRequested); + + float getClusterOverProvisioningFactor(Long clusterId, short capacityType); } diff --git a/engine/components-api/src/com/cloud/network/rules/FirewallManager.java b/engine/components-api/src/com/cloud/network/rules/FirewallManager.java index 6085b4b4341..3ba9861b1b2 100644 --- a/engine/components-api/src/com/cloud/network/rules/FirewallManager.java +++ b/engine/components-api/src/com/cloud/network/rules/FirewallManager.java @@ -86,5 +86,5 @@ public interface FirewallManager extends FirewallService { */ void removeRule(FirewallRule rule); - boolean applyDefaultEgressFirewallRule(Long networkId, boolean defaultPolicy) throws ResourceUnavailableException; + boolean applyDefaultEgressFirewallRule(Long networkId, boolean defaultPolicy, boolean add) throws ResourceUnavailableException; } diff --git a/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java b/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java index b5c1539ba9d..4ad0cd7af8f 100755 --- a/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java +++ b/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java @@ -1111,22 +1111,20 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra } List firewallEgressRulesToApply = _firewallDao.listByNetworkPurposeTrafficType(networkId, Purpose.Firewall, FirewallRule.TrafficType.Egress); - if (firewallEgressRulesToApply.size() == 0) { - NetworkOfferingVO offering = _networkOfferingDao.findById(network.getNetworkOfferingId()); - //there are no egress rules then apply the default egress rule - DataCenter zone = _dcDao.findById(network.getDataCenterId()); - if (offering.getEgressDefaultPolicy() && + NetworkOfferingVO offering = _networkOfferingDao.findById(network.getNetworkOfferingId()); + //there are no egress rules then apply the default egress rule + DataCenter zone = _dcDao.findById(network.getDataCenterId()); + if ( _networkModel.areServicesSupportedInNetwork(network.getId(), Service.Firewall) && _networkModel.areServicesSupportedInNetwork(network.getId(), Service.Firewall) && (network.getGuestType() == Network.GuestType.Isolated || (network.getGuestType() == Network.GuestType.Shared && zone.getNetworkType() == NetworkType.Advanced))) { - // add default egress rule to accept the traffic - _firewallMgr.applyDefaultEgressFirewallRule(network.getId(), true); - } - } else { - if (!_firewallMgr.applyFirewallRules(firewallEgressRulesToApply, false, caller)) { - s_logger.warn("Failed to reapply firewall Egress rule(s) as a part of network id=" + networkId + " restart"); - success = false; - } + // add default egress rule to accept the traffic + _firewallMgr.applyDefaultEgressFirewallRule(network.getId(), offering.getEgressDefaultPolicy(), true); } + if (!_firewallMgr.applyFirewallRules(firewallEgressRulesToApply, false, caller)) { + s_logger.warn("Failed to reapply firewall Egress rule(s) as a part of network id=" + networkId + " restart"); + success = false; + } + // apply port forwarding rules if (!_rulesMgr.applyPortForwardingRulesForNetwork(networkId, false, caller)) { @@ -2695,6 +2693,21 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra s_logger.debug("Releasing " + firewallEgressRules.size() + " firewall egress rules for network id=" + networkId + " as a part of shutdownNetworkRules"); } + try { + // delete default egress rule + DataCenter zone = _dcDao.findById(network.getDataCenterId()); + if ( _networkModel.areServicesSupportedInNetwork(network.getId(), Service.Firewall) && + (network.getGuestType() == Network.GuestType.Isolated || (network.getGuestType() == Network.GuestType.Shared && zone.getNetworkType() == NetworkType.Advanced))) { + // add default egress rule to accept the traffic + _firewallMgr.applyDefaultEgressFirewallRule(network.getId(), _networkModel.getNetworkEgressDefaultPolicy(networkId), false); + } + + } catch (ResourceUnavailableException ex) { + s_logger.warn("Failed to cleanup firewall default egress rule as a part of shutdownNetworkRules due to ", ex); + success = false; + } + + for (FirewallRuleVO firewallRule : firewallEgressRules) { s_logger.trace("Marking firewall egress rule " + firewallRule + " with Revoke state"); firewallRule.setState(FirewallRule.State.Revoke); diff --git a/engine/schema/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml b/engine/schema/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml index 4f7d7ea6562..e7caa203cc6 100644 --- a/engine/schema/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml +++ b/engine/schema/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml @@ -124,6 +124,7 @@ + diff --git a/engine/schema/src/com/cloud/capacity/dao/CapacityDao.java b/engine/schema/src/com/cloud/capacity/dao/CapacityDao.java index e32f96e953c..079d9a87649 100755 --- a/engine/schema/src/com/cloud/capacity/dao/CapacityDao.java +++ b/engine/schema/src/com/cloud/capacity/dao/CapacityDao.java @@ -45,11 +45,12 @@ public interface CapacityDao extends GenericDao { Pair, Map> orderPodsByAggregateCapacity(long zoneId, short capacityType); - List findCapacityBy(Integer capacityType, Long zoneId, Long podId, Long clusterId, String resourceState); + List findCapacityBy(Integer capacityType, Long zoneId, + Long podId, Long clusterId, String resourceState); + List listCapacitiesGroupedByLevelAndType(Integer capacityType, Long zoneId, Long podId, Long clusterId, int level, Long limit); + void updateCapacityState(Long dcId, Long podId, Long clusterId, + Long hostId, String capacityState); + List listClustersCrossingThreshold(short capacityType, Long zoneId, String ConfigName, long computeRequested); - List listCapacitiesGroupedByLevelAndType(Integer capacityType, Long zoneId, Long podId, Long clusterId, int level, Long limit); - - void updateCapacityState(Long dcId, Long podId, Long clusterId, Long hostId, String capacityState); - - List listClustersCrossingThreshold(short capacityType, Long zoneId, String ConfigName, long computeRequested); + float findClusterConsumption(Long clusterId, short capacityType, long computeRequested); } diff --git a/engine/schema/src/com/cloud/capacity/dao/CapacityDaoImpl.java b/engine/schema/src/com/cloud/capacity/dao/CapacityDaoImpl.java index cb916c37733..7919ed5bcd4 100755 --- a/engine/schema/src/com/cloud/capacity/dao/CapacityDaoImpl.java +++ b/engine/schema/src/com/cloud/capacity/dao/CapacityDaoImpl.java @@ -27,6 +27,9 @@ import java.util.Map; import javax.ejb.Local; import javax.inject.Inject; +import com.cloud.dc.ClusterDetailsDao; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -161,19 +164,24 @@ public class CapacityDaoImpl extends GenericDaoBase implements * query from the configuration table * * */ - private static final String LIST_CLUSTERS_CROSSING_THRESHOLD = - "SELECT clusterList.cluster_id " - + "FROM (SELECT cluster.cluster_id cluster_id, ( (sum(cluster.used) + sum(cluster.reserved) + ?)/sum(cluster.total) ) ratio, cluster.configValue value " - + "FROM (SELECT capacity.cluster_id cluster_id, capacity.used_capacity used, capacity.reserved_capacity reserved, capacity.total_capacity * overcommit.value total, " - + "CASE (SELECT count(*) FROM `cloud`.`cluster_details` details WHERE details.cluster_id = capacity.cluster_id AND details.name = ? ) " - + "WHEN 1 THEN (CASE WHEN (SELECT details.value FROM `cloud`.`cluster_details` details WHERE details.cluster_id = capacity.cluster_id AND details.name = ?) is NULL " - + "THEN (SELECT config.value FROM `cloud`.`configuration` config WHERE config.name = ?)" - + "ELSE (SELECT details.value FROM `cloud`.`cluster_details` details WHERE details.cluster_id = capacity.cluster_id AND details.name = ? ) END )" - + "ELSE ( SELECT config.value FROM `cloud`.`configuration` config WHERE config.name = ?) " + "END configValue " - + "FROM `cloud`.`op_host_capacity` capacity INNER JOIN `cloud`.`cluster_details` overcommit ON overcommit.cluster_id = capacity.cluster_id " - + "WHERE capacity.data_center_id = ? AND capacity.capacity_type = ? AND capacity.total_capacity > 0 AND overcommit.name = ?) cluster " + - "GROUP BY cluster.cluster_id) clusterList " + "WHERE clusterList.ratio > clusterList.value; "; + private static final String LIST_CLUSTERS_CROSSING_THRESHOLD = "SELECT clusterList.cluster_id " + + "FROM ( SELECT cluster.cluster_id cluster_id, ( (sum(cluster.used) + sum(cluster.reserved) + ?)/sum(cluster.total) ) ratio, cluster.configValue value " + + "FROM ( SELECT capacity.cluster_id cluster_id, capacity.used_capacity used, capacity.reserved_capacity reserved, capacity.total_capacity * overcommit.value total, " + + "CASE (SELECT count(*) FROM `cloud`.`cluster_details` details WHERE details.cluster_id = capacity.cluster_id AND details.name = ? ) " + + "WHEN 1 THEN ( CASE WHEN (SELECT details.value FROM `cloud`.`cluster_details` details WHERE details.cluster_id = capacity.cluster_id AND details.name = ?) is NULL " + + "THEN (SELECT config.value FROM `cloud`.`configuration` config WHERE config.name = ?)" + + "ELSE (SELECT details.value FROM `cloud`.`cluster_details` details WHERE details.cluster_id = capacity.cluster_id AND details.name = ? ) END )" + + "ELSE ( SELECT config.value FROM `cloud`.`configuration` config WHERE config.name = ?) " + + "END configValue " + + "FROM `cloud`.`op_host_capacity` capacity INNER JOIN `cloud`.`cluster_details` overcommit ON overcommit.cluster_id = capacity.cluster_id " + + "WHERE capacity.data_center_id = ? AND capacity.capacity_type = ? AND capacity.total_capacity > 0 AND overcommit.name = ?) cluster " + + + "GROUP BY cluster.cluster_id) clusterList " + + "WHERE clusterList.ratio > clusterList.value; "; + + private static final String FIND_CLUSTER_CONSUMPTION_RATIO = "select ( (sum(capacity.used_capacity) + sum(capacity.reserved_capacity) + ?)/sum(capacity.total_capacity) ) " + + "from op_host_capacity capacity where cluster_id = ? and capacity_type = ?;"; public CapacityDaoImpl() { _hostIdTypeSearch = createSearchBuilder(); @@ -883,4 +891,26 @@ public class CapacityDaoImpl extends GenericDaoBase implements s_logger.warn("Error updating CapacityVO", e); } } + + @Override + public float findClusterConsumption(Long clusterId, short capacityType, long computeRequested){ + TransactionLegacy txn = TransactionLegacy.currentTxn(); + StringBuilder sql = new StringBuilder(FIND_CLUSTER_CONSUMPTION_RATIO); + PreparedStatement pstmt = null; + try { + pstmt = txn.prepareAutoCloseStatement(sql.toString()); + + pstmt.setLong(1, computeRequested); + pstmt.setLong(2, clusterId); + pstmt.setShort(3, capacityType); + ResultSet rs = pstmt.executeQuery(); + while (rs.next()) { + return rs.getFloat(1); + } + } catch (Exception e) { + s_logger.warn("Error checking cluster threshold", e); + } + return 0; + } + } diff --git a/engine/schema/src/com/cloud/network/as/AutoScalePolicyVO.java b/engine/schema/src/com/cloud/network/as/AutoScalePolicyVO.java index fbaf8e94d03..6a104c2cb3e 100644 --- a/engine/schema/src/com/cloud/network/as/AutoScalePolicyVO.java +++ b/engine/schema/src/com/cloud/network/as/AutoScalePolicyVO.java @@ -27,6 +27,8 @@ import javax.persistence.Id; import javax.persistence.Inheritance; import javax.persistence.InheritanceType; import javax.persistence.Table; +import javax.persistence.Temporal; +import javax.persistence.TemporalType; import org.apache.cloudstack.api.InternalIdentity; @@ -57,6 +59,10 @@ public class AutoScalePolicyVO implements AutoScalePolicy, InternalIdentity { @Column(name = "quiet_time", updatable = true, nullable = false) private int quietTime; + @Column(name = "last_quiet_time", updatable = true) + @Temporal(value = TemporalType.TIMESTAMP) + private Date lastQuiteTime; + @Column(name = "action", updatable = false, nullable = false) private String action; @@ -69,12 +75,14 @@ public class AutoScalePolicyVO implements AutoScalePolicy, InternalIdentity { public AutoScalePolicyVO() { } - public AutoScalePolicyVO(long domainId, long accountId, int duration, int quietTime, String action) { + public AutoScalePolicyVO(long domainId, long accountId, int duration, + int quietTime, Date lastQuiteTime, String action) { this.uuid = UUID.randomUUID().toString(); this.domainId = domainId; this.accountId = accountId; this.duration = duration; this.quietTime = quietTime; + this.lastQuiteTime = lastQuiteTime; this.action = action; } @@ -113,6 +121,11 @@ public class AutoScalePolicyVO implements AutoScalePolicy, InternalIdentity { return quietTime; } + @Override + public Date getLastQuiteTime() { + return lastQuiteTime; + } + @Override public String getAction() { return action; @@ -133,4 +146,8 @@ public class AutoScalePolicyVO implements AutoScalePolicy, InternalIdentity { public void setQuietTime(Integer quietTime) { this.quietTime = quietTime; } + + public void setLastQuiteTime(Date lastQuiteTime) { + this.lastQuiteTime = lastQuiteTime; + } } diff --git a/engine/schema/src/com/cloud/network/as/AutoScaleVmGroupVO.java b/engine/schema/src/com/cloud/network/as/AutoScaleVmGroupVO.java index 4eed0863dd0..4d41583fd84 100644 --- a/engine/schema/src/com/cloud/network/as/AutoScaleVmGroupVO.java +++ b/engine/schema/src/com/cloud/network/as/AutoScaleVmGroupVO.java @@ -27,6 +27,8 @@ import javax.persistence.Id; import javax.persistence.Inheritance; import javax.persistence.InheritanceType; import javax.persistence.Table; +import javax.persistence.Temporal; +import javax.persistence.TemporalType; import org.apache.cloudstack.api.InternalIdentity; @@ -69,6 +71,10 @@ public class AutoScaleVmGroupVO implements AutoScaleVmGroup, InternalIdentity { @Column(name = "interval") private int interval; + @Column(name = "last_interval", updatable = true) + @Temporal(value = TemporalType.TIMESTAMP) + private Date lastInterval; + @Column(name = "profile_id") private long profileId; @@ -84,8 +90,10 @@ public class AutoScaleVmGroupVO implements AutoScaleVmGroup, InternalIdentity { public AutoScaleVmGroupVO() { } - public AutoScaleVmGroupVO(long lbRuleId, long zoneId, long domainId, long accountId, int minMembers, int maxMembers, int memberPort, int interval, long profileId, - String state) { + public AutoScaleVmGroupVO(long lbRuleId, long zoneId, long domainId, + long accountId, int minMembers, int maxMembers, int memberPort, + int interval, Date lastInterval, long profileId, String state) { + this.uuid = UUID.randomUUID().toString(); this.loadBalancerId = lbRuleId; this.minMembers = minMembers; @@ -97,6 +105,7 @@ public class AutoScaleVmGroupVO implements AutoScaleVmGroup, InternalIdentity { this.zoneId = zoneId; this.state = state; this.interval = interval; + this.lastInterval = lastInterval; } @Override @@ -148,6 +157,11 @@ public class AutoScaleVmGroupVO implements AutoScaleVmGroup, InternalIdentity { return interval; } + @Override + public Date getLastInterval() { + return lastInterval; + } + @Override public long getProfileId() { return profileId; @@ -182,6 +196,10 @@ public class AutoScaleVmGroupVO implements AutoScaleVmGroup, InternalIdentity { this.interval = interval; } + public void setLastInterval(Date lastInterval) { + this.lastInterval = lastInterval; + } + public void setLoadBalancerId(Long loadBalancerId) { this.loadBalancerId = loadBalancerId; } diff --git a/engine/schema/src/com/cloud/network/as/AutoScaleVmGroupVmMapVO.java b/engine/schema/src/com/cloud/network/as/AutoScaleVmGroupVmMapVO.java new file mode 100644 index 00000000000..d70f9bfba37 --- /dev/null +++ b/engine/schema/src/com/cloud/network/as/AutoScaleVmGroupVmMapVO.java @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.network.as; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; + +import org.apache.cloudstack.api.InternalIdentity; + +@Entity +@Table(name = ("autoscale_vmgroup_vm_map")) +public class AutoScaleVmGroupVmMapVO implements InternalIdentity { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + private long id; + + @Column(name = "vmgroup_id") + private long vmGroupId; + + @Column(name = "instance_id") + private long instanceId; + + public AutoScaleVmGroupVmMapVO() { + } + + public AutoScaleVmGroupVmMapVO(long vmGroupId, long instanceId) { + this.vmGroupId = vmGroupId; + this.instanceId = instanceId; + } + + public long getId() { + return id; + } + + public long getVmGroupId() { + return vmGroupId; + } + + public long getInstanceId() { + return instanceId; + } + +} diff --git a/engine/schema/src/com/cloud/network/as/dao/AutoScalePolicyConditionMapDao.java b/engine/schema/src/com/cloud/network/as/dao/AutoScalePolicyConditionMapDao.java index f0b2f5568c7..a9cfd188cc2 100644 --- a/engine/schema/src/com/cloud/network/as/dao/AutoScalePolicyConditionMapDao.java +++ b/engine/schema/src/com/cloud/network/as/dao/AutoScalePolicyConditionMapDao.java @@ -26,5 +26,7 @@ public interface AutoScalePolicyConditionMapDao extends GenericDao findByPolicyId(long id); } diff --git a/engine/schema/src/com/cloud/network/as/dao/AutoScalePolicyConditionMapDaoImpl.java b/engine/schema/src/com/cloud/network/as/dao/AutoScalePolicyConditionMapDaoImpl.java index 5933f36b8f7..557a6731af0 100644 --- a/engine/schema/src/com/cloud/network/as/dao/AutoScalePolicyConditionMapDaoImpl.java +++ b/engine/schema/src/com/cloud/network/as/dao/AutoScalePolicyConditionMapDaoImpl.java @@ -56,6 +56,14 @@ public class AutoScalePolicyConditionMapDaoImpl extends GenericDaoBase sc = createSearchCriteria(); sc.addAnd("policyId", SearchCriteria.Op.EQ, policyId); - return expunge(sc) > 0; - } + return expunge(sc) > 0; + } + + @Override + public List findByPolicyId(long policyId) { + SearchCriteria sc = createSearchCriteria(); + sc.addAnd("policyId", SearchCriteria.Op.EQ, policyId); + + return listBy(sc); + } } diff --git a/engine/schema/src/com/cloud/network/as/dao/AutoScaleVmGroupVmMapDao.java b/engine/schema/src/com/cloud/network/as/dao/AutoScaleVmGroupVmMapDao.java new file mode 100644 index 00000000000..b7234e7a64c --- /dev/null +++ b/engine/schema/src/com/cloud/network/as/dao/AutoScaleVmGroupVmMapDao.java @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.network.as.dao; + +import java.util.List; + +import com.cloud.network.as.AutoScaleVmGroupVmMapVO; +import com.cloud.utils.db.GenericDao; + +public interface AutoScaleVmGroupVmMapDao extends GenericDao { + public Integer countByGroup(long vmGroupId); + + public List listByGroup(long vmGroupId); + + public int remove(long vmGroupId, long vmId); + +} diff --git a/engine/schema/src/com/cloud/network/as/dao/AutoScaleVmGroupVmMapDaoImpl.java b/engine/schema/src/com/cloud/network/as/dao/AutoScaleVmGroupVmMapDaoImpl.java new file mode 100644 index 00000000000..494092d65d5 --- /dev/null +++ b/engine/schema/src/com/cloud/network/as/dao/AutoScaleVmGroupVmMapDaoImpl.java @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.network.as.dao; + +import java.util.List; + +import javax.ejb.Local; + +import org.springframework.stereotype.Component; + +import com.cloud.network.as.AutoScaleVmGroupVmMapVO; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchCriteria; + +@Component +@Local(value={AutoScaleVmGroupVmMapDao.class}) +public class AutoScaleVmGroupVmMapDaoImpl extends GenericDaoBase implements AutoScaleVmGroupVmMapDao { + + @Override + public Integer countByGroup(long vmGroupId) { + + SearchCriteria sc = createSearchCriteria(); + sc.addAnd("vmGroupId", SearchCriteria.Op.EQ, vmGroupId); + return getCount(sc); + } + + @Override + public List listByGroup(long vmGroupId) { + SearchCriteria sc = createSearchCriteria(); + sc.addAnd("vmGroupId", SearchCriteria.Op.EQ, vmGroupId); + return listBy(sc); + } + + @Override + public int remove(long vmGroupId, long vmId) { + SearchCriteria sc = createSearchCriteria(); + sc.addAnd("vmGroupId", SearchCriteria.Op.EQ, vmGroupId); + sc.addAnd("instanceId", SearchCriteria.Op.EQ, vmId); + return remove(sc); + } + +} diff --git a/engine/schema/src/com/cloud/network/dao/ExternalLoadBalancerDeviceDaoImpl.java b/engine/schema/src/com/cloud/network/dao/ExternalLoadBalancerDeviceDaoImpl.java index f12d06324bf..432b4a2eb9d 100644 --- a/engine/schema/src/com/cloud/network/dao/ExternalLoadBalancerDeviceDaoImpl.java +++ b/engine/schema/src/com/cloud/network/dao/ExternalLoadBalancerDeviceDaoImpl.java @@ -57,6 +57,7 @@ public class ExternalLoadBalancerDeviceDaoImpl extends GenericDaoBase params, + int wait) { + String result = null; + try { + result = callHostPluginAsync(conn, "vmopspremium", "asmonitor", 60, + params); + if (result != null) + return result; + } catch (Exception e) { + s_logger.error("Can not get performance monitor for AS due to ", e); + } + return null; + } + + protected String callHostPluginAsync(Connection conn, String plugin, + String cmd, int wait, Map params) { + int timeout = wait * 1000; + Map args = new HashMap(); + Task task = null; + try { + for (String key : params.keySet()) { + args.put(key, params.get(key)); + } + if (s_logger.isTraceEnabled()) { + s_logger.trace("callHostPlugin executing for command " + cmd + + " with " + getArgsString(args)); + } + Host host = Host.getByUuid(conn, _host.uuid); + task = host.callPluginAsync(conn, plugin, cmd, args); + // poll every 1 seconds + waitForTask(conn, task, 1000, timeout); + checkForSuccess(conn, task); + String result = task.getResult(conn); + if (s_logger.isTraceEnabled()) { + s_logger.trace("callHostPlugin Result: " + result); + } + return result.replace("", "").replace("", "") + .replace("\n", ""); + } catch (Types.HandleInvalid e) { + s_logger.warn("callHostPlugin failed for cmd: " + cmd + + " with args " + getArgsString(args) + + " due to HandleInvalid clazz:" + e.clazz + ", handle:" + + e.handle); + } catch (XenAPIException e) { + s_logger.warn( + "callHostPlugin failed for cmd: " + cmd + " with args " + + getArgsString(args) + " due to " + e.toString(), + e); + } catch (XmlRpcException e) { + s_logger.warn( + "callHostPlugin failed for cmd: " + cmd + " with args " + + getArgsString(args) + " due to " + e.getMessage(), + e); + } finally { + if (task != null) { + try { + task.destroy(conn); + } catch (Exception e1) { + s_logger.warn("unable to destroy task(" + task.toString() + + ") on host(" + _host.uuid + ") due to ", e1); + } + } + } + return null; + } + protected void scaleVM(Connection conn, VM vm, VirtualMachineTO vmSpec, Host host) throws XenAPIException, XmlRpcException { Long staticMemoryMax = vm.getMemoryStaticMax(conn); @@ -5300,7 +5379,7 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe com.trilead.ssh2.Session session = sshConnection.openSession(); - String cmd = "mkdir -p /opt/cloud/bin"; + String cmd = "mkdir -p /opt/cloud/bin /var/log/cloud"; if (!SSHCmdHelper.sshExecuteCmd(sshConnection, cmd)) { throw new CloudRuntimeException("Cannot create directory /opt/cloud/bin on XenServer hosts"); } diff --git a/plugins/network-elements/f5/src/com/cloud/network/element/F5ExternalLoadBalancerElement.java b/plugins/network-elements/f5/src/com/cloud/network/element/F5ExternalLoadBalancerElement.java index daabf369cf0..1a591e1da57 100644 --- a/plugins/network-elements/f5/src/com/cloud/network/element/F5ExternalLoadBalancerElement.java +++ b/plugins/network-elements/f5/src/com/cloud/network/element/F5ExternalLoadBalancerElement.java @@ -321,7 +321,8 @@ public class F5ExternalLoadBalancerElement extends ExternalLoadBalancerDeviceMan pNetwork = physicalNetworks.get(0); String deviceType = NetworkDevice.F5BigIpLoadBalancer.getName(); - lbDeviceVO = addExternalLoadBalancer(pNetwork.getId(), cmd.getUrl(), cmd.getUsername(), cmd.getPassword(), deviceType, new F5BigIpResource(), false, null, null); + lbDeviceVO = addExternalLoadBalancer(pNetwork.getId(), cmd.getUrl(), cmd.getUsername(), cmd.getPassword(), + deviceType, new F5BigIpResource(), false, false, null, null); if (lbDeviceVO != null) { lbHost = _hostDao.findById(lbDeviceVO.getHostId()); @@ -374,7 +375,8 @@ public class F5ExternalLoadBalancerElement extends ExternalLoadBalancerDeviceMan throw new InvalidParameterValueException("Invalid F5 load balancer device type"); } - return addExternalLoadBalancer(cmd.getPhysicalNetworkId(), cmd.getUrl(), cmd.getUsername(), cmd.getPassword(), deviceName, new F5BigIpResource(), false, null, + return addExternalLoadBalancer(cmd.getPhysicalNetworkId(), cmd.getUrl(), cmd.getUsername(), cmd.getPassword(), + deviceName, new F5BigIpResource(), false, false, null, null); } diff --git a/plugins/network-elements/juniper-srx/src/com/cloud/network/element/JuniperSRXExternalFirewallElement.java b/plugins/network-elements/juniper-srx/src/com/cloud/network/element/JuniperSRXExternalFirewallElement.java index 2ff90fb6d7b..82f701b4a06 100644 --- a/plugins/network-elements/juniper-srx/src/com/cloud/network/element/JuniperSRXExternalFirewallElement.java +++ b/plugins/network-elements/juniper-srx/src/com/cloud/network/element/JuniperSRXExternalFirewallElement.java @@ -218,6 +218,14 @@ public class JuniperSRXExternalFirewallElement extends ExternalFirewallDeviceMan return false; } + if (rules != null && rules.size() == 1 ) { + // for SRX no need to add default egress rule to DENY traffic + if (rules.get(0).getTrafficType() == FirewallRule.TrafficType.Egress && rules.get(0).getType() == FirewallRule.FirewallRuleType.System && + ! _networkManager.getNetworkEgressDefaultPolicy(config.getId())) + return true; + } + + return applyFirewallRules(config, rules); } diff --git a/plugins/network-elements/juniper-srx/src/com/cloud/network/resource/JuniperSrxResource.java b/plugins/network-elements/juniper-srx/src/com/cloud/network/resource/JuniperSrxResource.java index 338e0953249..b828ab66db6 100644 --- a/plugins/network-elements/juniper-srx/src/com/cloud/network/resource/JuniperSrxResource.java +++ b/plugins/network-elements/juniper-srx/src/com/cloud/network/resource/JuniperSrxResource.java @@ -835,6 +835,15 @@ public class JuniperSrxResource implements ServerResource { FirewallRule.FirewallRuleType type = rules[0].getType(); //getting String guestCidr = rules[0].getGuestCidr(); + List cidrs = new ArrayList(); + cidrs.add(guestCidr); + + List applications = new ArrayList(); + Object[] application = new Object[3]; + application[0] = Protocol.all; + application[1] = NetUtils.PORT_RANGE_MIN; + application[2] = NetUtils.PORT_RANGE_MAX; + applications.add(application); for (String guestVlan : guestVlans) { List activeRulesForGuestNw = activeRules.get(guestVlan); @@ -844,25 +853,24 @@ public class JuniperSrxResource implements ServerResource { if (activeRulesForGuestNw.size() > 0 && type == FirewallRule.FirewallRuleType.User) { addEgressSecurityPolicyAndApplications(SecurityPolicyType.SECURITYPOLICY_EGRESS, guestVlan, extractApplications(activeRulesForGuestNw), extractCidrs(activeRulesForGuestNw), defaultEgressPolicy); + + /* Adding default policy rules are required because the order of rules is important. + * Depending on the rules order the traffic accept/drop is performed + */ + removeEgressSecurityPolicyAndApplications(SecurityPolicyType.SECURITYPOLICY_EGRESS_DEFAULT, guestVlan, cidrs, defaultEgressPolicy); + addEgressSecurityPolicyAndApplications(SecurityPolicyType.SECURITYPOLICY_EGRESS_DEFAULT, guestVlan, applications, cidrs, defaultEgressPolicy); } - List applications = new ArrayList(); - Object[] application = new Object[3]; - application[0] = Protocol.all; - application[1] = NetUtils.PORT_RANGE_MIN; - application[2] = NetUtils.PORT_RANGE_MAX; - applications.add(application); - - List cidrs = new ArrayList(); - cidrs.add(guestCidr); //remove required with out comparing default policy because in upgrade network offering we may required to delete // the previously added rule removeEgressSecurityPolicyAndApplications(SecurityPolicyType.SECURITYPOLICY_EGRESS_DEFAULT, guestVlan, cidrs, false); - if (defaultEgressPolicy == true) { - //add default egress security policy - addEgressSecurityPolicyAndApplications(SecurityPolicyType.SECURITYPOLICY_EGRESS_DEFAULT, guestVlan, applications, cidrs, false); + if (defaultEgressPolicy == true && type == FirewallRule.FirewallRuleType.System) { + removeEgressSecurityPolicyAndApplications(SecurityPolicyType.SECURITYPOLICY_EGRESS_DEFAULT, guestVlan, cidrs, defaultEgressPolicy); + if (activeRulesForGuestNw.size() > 0) { + //add default egress security policy + addEgressSecurityPolicyAndApplications(SecurityPolicyType.SECURITYPOLICY_EGRESS_DEFAULT, guestVlan, applications, cidrs, defaultEgressPolicy); + } } - } commitConfiguration(); } else { @@ -2811,13 +2819,23 @@ public class JuniperSrxResource implements ServerResource { xml = replaceXmlValue(xml, "src-address", srcAddrs); dstAddrs = "any"; xml = replaceXmlValue(xml, "dst-address", dstAddrs); - if (defaultEgressAction == true) { - //configure block rules and default allow the traffic - action = ""; + if (type.equals(SecurityPolicyType.SECURITYPOLICY_EGRESS_DEFAULT)) { + if (defaultEgressAction == false) { + //for default policy is false add default deny rules + action = ""; + } else { + action = ""; + } } else { - action = ""; + if (defaultEgressAction == true) { + //configure egress rules to deny the traffic when default egress is allow + action = ""; + } else { + action = ""; + } + + xml = replaceXmlValue(xml, "action", action); } - xml = replaceXmlValue(xml, "action", action); } else { xml = replaceXmlValue(xml, "from-zone", fromZone); xml = replaceXmlValue(xml, "to-zone", toZone); diff --git a/plugins/network-elements/netscaler/src/com/cloud/api/commands/AddNetscalerLoadBalancerCmd.java b/plugins/network-elements/netscaler/src/com/cloud/api/commands/AddNetscalerLoadBalancerCmd.java index 9188f3abd90..1237e7f7104 100644 --- a/plugins/network-elements/netscaler/src/com/cloud/api/commands/AddNetscalerLoadBalancerCmd.java +++ b/plugins/network-elements/netscaler/src/com/cloud/api/commands/AddNetscalerLoadBalancerCmd.java @@ -77,7 +77,7 @@ public class AddNetscalerLoadBalancerCmd extends BaseAsyncCmd { type = CommandType.BOOLEAN, required = false, description = "true if NetScaler device being added is for providing GSLB service") - private boolean isGslbProvider; + private Boolean isGslbProvider; @Parameter(name = ApiConstants.GSLB_PROVIDER_PUBLIC_IP, type = CommandType.STRING, required = false, description = "public IP of the site") private String gslbSitePublicIp; @@ -85,6 +85,12 @@ public class AddNetscalerLoadBalancerCmd extends BaseAsyncCmd { @Parameter(name = ApiConstants.GSLB_PROVIDER_PRIVATE_IP, type = CommandType.STRING, required = false, description = "public IP of the site") private String gslbSitePrivateIp; + @Parameter(name = ApiConstants.EXCLUSIVE_GSLB_PROVIDER, + type = CommandType.BOOLEAN, + required = false, + description = "true if NetScaler device being added is for providing GSLB service exclusively and can not be used for LB") + private Boolean isExclusiveGslbProvider; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -110,7 +116,19 @@ public class AddNetscalerLoadBalancerCmd extends BaseAsyncCmd { } public boolean isGslbProvider() { - return isGslbProvider; + if (isGslbProvider != null) { + return isGslbProvider; + } else { + return false; + } + } + + public boolean isExclusiveGslbProvider() { + if (isExclusiveGslbProvider != null) { + return isExclusiveGslbProvider; + } else { + return false; + } } public String getSitePublicIp() { diff --git a/plugins/network-elements/netscaler/src/com/cloud/api/response/NetscalerLoadBalancerResponse.java b/plugins/network-elements/netscaler/src/com/cloud/api/response/NetscalerLoadBalancerResponse.java index 63de361cfc5..8b942f430b0 100644 --- a/plugins/network-elements/netscaler/src/com/cloud/api/response/NetscalerLoadBalancerResponse.java +++ b/plugins/network-elements/netscaler/src/com/cloud/api/response/NetscalerLoadBalancerResponse.java @@ -75,6 +75,10 @@ public class NetscalerLoadBalancerResponse extends BaseResponse { @Param(description = "true if NetScaler device is provisioned to be a GSLB service provider") private Boolean isGslbProvider; + @SerializedName(ApiConstants.EXCLUSIVE_GSLB_PROVIDER) + @Param(description = "true if NetScaler device is provisioned exclusively to be a GSLB service provider") + private Boolean isExclusiveGslbProvider; + @SerializedName(ApiConstants.GSLB_PROVIDER_PUBLIC_IP) @Param(description = "public IP of the NetScaler representing GSLB site") private String gslbSitePublicIp; @@ -137,6 +141,10 @@ public class NetscalerLoadBalancerResponse extends BaseResponse { this.isGslbProvider = isGslbProvider; } + public void setExclusiveGslbProvider(boolean isExclusiveGslbProvider) { + this.isExclusiveGslbProvider = isExclusiveGslbProvider; + } + public void setGslbSitePublicIp(String publicIP) { this.gslbSitePublicIp = publicIP; } diff --git a/plugins/network-elements/netscaler/src/com/cloud/network/element/NetscalerElement.java b/plugins/network-elements/netscaler/src/com/cloud/network/element/NetscalerElement.java index 7e68f4e941c..a2bda9042d9 100644 --- a/plugins/network-elements/netscaler/src/com/cloud/network/element/NetscalerElement.java +++ b/plugins/network-elements/netscaler/src/com/cloud/network/element/NetscalerElement.java @@ -381,9 +381,14 @@ public class NetscalerElement extends ExternalLoadBalancerDeviceManagerImpl impl } + if (cmd.isExclusiveGslbProvider() && !cmd.isGslbProvider()) { + throw new InvalidParameterValueException("NetScaler can be provisioned to be exclusive GSLB service provider" + + " only if its being configured as GSLB service provider also."); + } + ExternalLoadBalancerDeviceVO lbDeviceVO = addExternalLoadBalancer(cmd.getPhysicalNetworkId(), cmd.getUrl(), cmd.getUsername(), cmd.getPassword(), deviceName, new NetscalerResource(), - cmd.isGslbProvider(), cmd.getSitePublicIp(), cmd.getSitePrivateIp()); + cmd.isGslbProvider(), cmd.isExclusiveGslbProvider(), cmd.getSitePublicIp(), cmd.getSitePrivateIp()); return lbDeviceVO; } @@ -605,6 +610,7 @@ public class NetscalerElement extends ExternalLoadBalancerDeviceManagerImpl impl response.setObjectName("netscalerloadbalancer"); response.setGslbProvider(lbDeviceVO.getGslbProvider()); + response.setExclusiveGslbProvider(lbDeviceVO.getExclusiveGslbProvider()); response.setGslbSitePublicIp(lbDeviceVO.getGslbSitePublicIP()); response.setGslbSitePrivateIp(lbDeviceVO.getGslbSitePrivateIP()); diff --git a/plugins/network-elements/ovs/src/com/cloud/network/element/OvsElementService.java b/plugins/network-elements/ovs/src/com/cloud/network/element/OvsElementService.java index c382e627ec6..7ed7702acb5 100644 --- a/plugins/network-elements/ovs/src/com/cloud/network/element/OvsElementService.java +++ b/plugins/network-elements/ovs/src/com/cloud/network/element/OvsElementService.java @@ -1,3 +1,19 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. package com.cloud.network.element; import com.cloud.utils.component.PluggableService; diff --git a/plugins/network-elements/palo-alto/src/com/cloud/api/commands/AddExternalFirewallCmd.java b/plugins/network-elements/palo-alto/src/com/cloud/api/commands/AddExternalFirewallCmd.java deleted file mode 100644 index 2e4d6717181..00000000000 --- a/plugins/network-elements/palo-alto/src/com/cloud/api/commands/AddExternalFirewallCmd.java +++ /dev/null @@ -1,115 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package com.cloud.api.commands; - -import javax.inject.Inject; - -import org.apache.log4j.Logger; - -import org.apache.cloudstack.api.APICommand; -import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.ApiErrorCode; -import org.apache.cloudstack.api.BaseCmd; -import org.apache.cloudstack.api.Parameter; -import org.apache.cloudstack.api.ServerApiException; -import org.apache.cloudstack.api.response.ExternalFirewallResponse; -import org.apache.cloudstack.api.response.ZoneResponse; - -import com.cloud.exception.InvalidParameterValueException; -import com.cloud.host.Host; -import com.cloud.network.element.PaloAltoFirewallElementService; -import com.cloud.user.Account; -import com.cloud.utils.exception.CloudRuntimeException; - -@APICommand(name = "addExternalFirewall", description = "Adds an external firewall appliance", responseObject = ExternalFirewallResponse.class) -public class AddExternalFirewallCmd extends BaseCmd { - public static final Logger s_logger = Logger.getLogger(AddExternalFirewallCmd.class.getName()); - private static final String s_name = "addexternalfirewallresponse"; - - ///////////////////////////////////////////////////// - //////////////// API parameters ///////////////////// - ///////////////////////////////////////////////////// - - @Parameter(name = ApiConstants.ZONE_ID, - type = CommandType.UUID, - entityType = ZoneResponse.class, - required = true, - description = "Zone in which to add the external firewall appliance.") - private Long zoneId; - - @Parameter(name = ApiConstants.URL, type = CommandType.STRING, required = true, description = "URL of the external firewall appliance.") - private String url; - - @Parameter(name = ApiConstants.USERNAME, type = CommandType.STRING, required = true, description = "Username of the external firewall appliance.") - private String username; - - @Parameter(name = ApiConstants.PASSWORD, type = CommandType.STRING, required = true, description = "Password of the external firewall appliance.") - private String password; - - /////////////////////////////////////////////////// - /////////////////// Accessors /////////////////////// - ///////////////////////////////////////////////////// - - public Long getZoneId() { - return zoneId; - } - - public String getUrl() { - return url; - } - - public String getUsername() { - return username; - } - - public String getPassword() { - return password; - } - - ///////////////////////////////////////////////////// - /////////////// API Implementation/////////////////// - ///////////////////////////////////////////////////// - - @Inject - PaloAltoFirewallElementService _paElementService; - - @Override - public String getCommandName() { - return s_name; - } - - @Override - public long getEntityOwnerId() { - return Account.ACCOUNT_ID_SYSTEM; - } - - @SuppressWarnings("deprecation") - @Override - public void execute() { - try { - Host externalFirewall = _paElementService.addExternalFirewall(this); - ExternalFirewallResponse response = _paElementService.createExternalFirewallResponse(externalFirewall); - response.setObjectName("externalfirewall"); - response.setResponseName(getCommandName()); - this.setResponseObject(response); - } catch (InvalidParameterValueException ipve) { - throw new ServerApiException(ApiErrorCode.PARAM_ERROR, ipve.getMessage()); - } catch (CloudRuntimeException cre) { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, cre.getMessage()); - } - } -} diff --git a/plugins/network-elements/palo-alto/src/com/cloud/api/commands/DeleteExternalFirewallCmd.java b/plugins/network-elements/palo-alto/src/com/cloud/api/commands/DeleteExternalFirewallCmd.java deleted file mode 100644 index 7a3cec8824b..00000000000 --- a/plugins/network-elements/palo-alto/src/com/cloud/api/commands/DeleteExternalFirewallCmd.java +++ /dev/null @@ -1,89 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package com.cloud.api.commands; - -import javax.inject.Inject; - -import org.apache.log4j.Logger; - -import org.apache.cloudstack.api.APICommand; -import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.ApiErrorCode; -import org.apache.cloudstack.api.BaseCmd; -import org.apache.cloudstack.api.Parameter; -import org.apache.cloudstack.api.ServerApiException; -import org.apache.cloudstack.api.response.HostResponse; -import org.apache.cloudstack.api.response.SuccessResponse; - -import com.cloud.exception.InvalidParameterValueException; -import com.cloud.network.element.PaloAltoFirewallElementService; -import com.cloud.user.Account; - -@APICommand(name = "deleteExternalFirewall", description = "Deletes an external firewall appliance.", responseObject = SuccessResponse.class) -public class DeleteExternalFirewallCmd extends BaseCmd { - public static final Logger s_logger = Logger.getLogger(DeleteExternalFirewallCmd.class.getName()); - private static final String s_name = "deleteexternalfirewallresponse"; - - ///////////////////////////////////////////////////// - //////////////// API parameters ///////////////////// - ///////////////////////////////////////////////////// - - @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = HostResponse.class, required = true, description = "Id of the external firewall appliance.") - private Long id; - - /////////////////////////////////////////////////// - /////////////////// Accessors /////////////////////// - ///////////////////////////////////////////////////// - - public Long getId() { - return id; - } - - ///////////////////////////////////////////////////// - /////////////// API Implementation/////////////////// - ///////////////////////////////////////////////////// - - @Inject - PaloAltoFirewallElementService _paElementService; - - @Override - public String getCommandName() { - return s_name; - } - - @Override - public long getEntityOwnerId() { - return Account.ACCOUNT_ID_SYSTEM; - } - - @SuppressWarnings("deprecation") - @Override - public void execute() { - try { - boolean result = _paElementService.deleteExternalFirewall(this); - if (result) { - SuccessResponse response = new SuccessResponse(getCommandName()); - response.setResponseName(getCommandName()); - this.setResponseObject(response); - } else { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to delete external firewall."); - } - } catch (InvalidParameterValueException e) { - throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Failed to delete external firewall."); - } - } -} diff --git a/plugins/network-elements/palo-alto/src/com/cloud/api/commands/ListExternalFirewallsCmd.java b/plugins/network-elements/palo-alto/src/com/cloud/api/commands/ListExternalFirewallsCmd.java deleted file mode 100644 index 6c661def803..00000000000 --- a/plugins/network-elements/palo-alto/src/com/cloud/api/commands/ListExternalFirewallsCmd.java +++ /dev/null @@ -1,89 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package com.cloud.api.commands; - -import java.util.ArrayList; -import java.util.List; - -import javax.inject.Inject; - -import org.apache.log4j.Logger; - -import org.apache.cloudstack.api.APICommand; -import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.BaseListCmd; -import org.apache.cloudstack.api.Parameter; -import org.apache.cloudstack.api.command.user.offering.ListServiceOfferingsCmd; -import org.apache.cloudstack.api.response.ExternalFirewallResponse; -import org.apache.cloudstack.api.response.ListResponse; -import org.apache.cloudstack.api.response.ZoneResponse; - -import com.cloud.host.Host; -import com.cloud.network.element.PaloAltoFirewallElementService; - -@APICommand(name = "listExternalFirewalls", description = "List external firewall appliances.", responseObject = ExternalFirewallResponse.class) -public class ListExternalFirewallsCmd extends BaseListCmd { - public static final Logger s_logger = Logger.getLogger(ListServiceOfferingsCmd.class.getName()); - private static final String s_name = "listexternalfirewallsresponse"; - - ///////////////////////////////////////////////////// - //////////////// API parameters ///////////////////// - ///////////////////////////////////////////////////// - - @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.UUID, entityType = ZoneResponse.class, required = true, description = "zone Id") - private long zoneId; - - ///////////////////////////////////////////////////// - /////////////////// Accessors /////////////////////// - ///////////////////////////////////////////////////// - - public long getZoneId() { - return zoneId; - } - - ///////////////////////////////////////////////////// - /////////////// API Implementation/////////////////// - ///////////////////////////////////////////////////// - - @Inject - PaloAltoFirewallElementService _paElementService; - - @Override - public String getCommandName() { - return s_name; - } - - @SuppressWarnings("deprecation") - @Override - public void execute() { - - List externalFirewalls = _paElementService.listExternalFirewalls(this); - - ListResponse listResponse = new ListResponse(); - List responses = new ArrayList(); - for (Host externalFirewall : externalFirewalls) { - ExternalFirewallResponse response = _paElementService.createExternalFirewallResponse(externalFirewall); - response.setObjectName("externalfirewall"); - response.setResponseName(getCommandName()); - responses.add(response); - } - - listResponse.setResponses(responses); - listResponse.setResponseName(getCommandName()); - this.setResponseObject(listResponse); - } -} diff --git a/plugins/network-elements/palo-alto/src/com/cloud/network/element/PaloAltoExternalFirewallElement.java b/plugins/network-elements/palo-alto/src/com/cloud/network/element/PaloAltoExternalFirewallElement.java index a38ca713de3..be9a0b880fe 100644 --- a/plugins/network-elements/palo-alto/src/com/cloud/network/element/PaloAltoExternalFirewallElement.java +++ b/plugins/network-elements/palo-alto/src/com/cloud/network/element/PaloAltoExternalFirewallElement.java @@ -32,12 +32,9 @@ import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.network.ExternalNetworkDeviceManager.NetworkDevice; import com.cloud.api.ApiDBUtils; -import com.cloud.api.commands.AddExternalFirewallCmd; import com.cloud.api.commands.AddPaloAltoFirewallCmd; import com.cloud.api.commands.ConfigurePaloAltoFirewallCmd; -import com.cloud.api.commands.DeleteExternalFirewallCmd; import com.cloud.api.commands.DeletePaloAltoFirewallCmd; -import com.cloud.api.commands.ListExternalFirewallsCmd; import com.cloud.api.commands.ListPaloAltoFirewallNetworksCmd; import com.cloud.api.commands.ListPaloAltoFirewallsCmd; import com.cloud.api.response.PaloAltoFirewallResponse; @@ -293,85 +290,12 @@ public class PaloAltoExternalFirewallElement extends ExternalFirewallDeviceManag return true; } - @Override - @Deprecated - // should use more generic addNetworkDevice command to add firewall - public - Host addExternalFirewall(AddExternalFirewallCmd cmd) { - Long zoneId = cmd.getZoneId(); - DataCenterVO zone = null; - PhysicalNetworkVO pNetwork = null; - HostVO fwHost = null; - - zone = _dcDao.findById(zoneId); - if (zone == null) { - throw new InvalidParameterValueException("Could not find zone with ID: " + zoneId); - } - - List physicalNetworks = _physicalNetworkDao.listByZone(zoneId); - if ((physicalNetworks == null) || (physicalNetworks.size() > 1)) { - throw new InvalidParameterValueException("There are no physical networks or multiple physical networks configured in zone with ID: " + zoneId + - " to add this device."); - } - pNetwork = physicalNetworks.get(0); - - String deviceType = NetworkDevice.PaloAltoFirewall.getName(); - ExternalFirewallDeviceVO fwDeviceVO = - addExternalFirewall(pNetwork.getId(), cmd.getUrl(), cmd.getUsername(), cmd.getPassword(), deviceType, new PaloAltoResource()); - if (fwDeviceVO != null) { - fwHost = _hostDao.findById(fwDeviceVO.getHostId()); - } - - return fwHost; - } - - @Override - public boolean deleteExternalFirewall(DeleteExternalFirewallCmd cmd) { - return deleteExternalFirewall(cmd.getId()); - } - - @Override - @Deprecated - // should use more generic listNetworkDevice command - public - List listExternalFirewalls(ListExternalFirewallsCmd cmd) { - List firewallHosts = new ArrayList(); - Long zoneId = cmd.getZoneId(); - DataCenterVO zone = null; - PhysicalNetworkVO pNetwork = null; - - if (zoneId != null) { - zone = _dcDao.findById(zoneId); - if (zone == null) { - throw new InvalidParameterValueException("Could not find zone with ID: " + zoneId); - } - - List physicalNetworks = _physicalNetworkDao.listByZone(zoneId); - if ((physicalNetworks == null) || (physicalNetworks.size() > 1)) { - throw new InvalidParameterValueException("There are no physical networks or multiple physical networks configured in zone with ID: " + zoneId + - " to add this device."); - } - pNetwork = physicalNetworks.get(0); - } - - firewallHosts.addAll(listExternalFirewalls(pNetwork.getId(), NetworkDevice.PaloAltoFirewall.getName())); - return firewallHosts; - } - - @Override - public ExternalFirewallResponse createExternalFirewallResponse(Host externalFirewall) { - return super.createExternalFirewallResponse(externalFirewall); - } - @Override public List> getCommands() { List> cmdList = new ArrayList>(); - cmdList.add(AddExternalFirewallCmd.class); cmdList.add(AddPaloAltoFirewallCmd.class); cmdList.add(ConfigurePaloAltoFirewallCmd.class); - cmdList.add(DeleteExternalFirewallCmd.class); cmdList.add(DeletePaloAltoFirewallCmd.class); - cmdList.add(ListExternalFirewallsCmd.class); cmdList.add(ListPaloAltoFirewallNetworksCmd.class); cmdList.add(ListPaloAltoFirewallsCmd.class); return cmdList; diff --git a/plugins/network-elements/palo-alto/src/com/cloud/network/element/PaloAltoFirewallElementService.java b/plugins/network-elements/palo-alto/src/com/cloud/network/element/PaloAltoFirewallElementService.java index 17b3133b391..12f04077215 100644 --- a/plugins/network-elements/palo-alto/src/com/cloud/network/element/PaloAltoFirewallElementService.java +++ b/plugins/network-elements/palo-alto/src/com/cloud/network/element/PaloAltoFirewallElementService.java @@ -20,12 +20,9 @@ import java.util.List; import org.apache.cloudstack.api.response.ExternalFirewallResponse; -import com.cloud.api.commands.AddExternalFirewallCmd; import com.cloud.api.commands.AddPaloAltoFirewallCmd; import com.cloud.api.commands.ConfigurePaloAltoFirewallCmd; -import com.cloud.api.commands.DeleteExternalFirewallCmd; import com.cloud.api.commands.DeletePaloAltoFirewallCmd; -import com.cloud.api.commands.ListExternalFirewallsCmd; import com.cloud.api.commands.ListPaloAltoFirewallNetworksCmd; import com.cloud.api.commands.ListPaloAltoFirewallsCmd; import com.cloud.api.response.PaloAltoFirewallResponse; @@ -72,24 +69,4 @@ public interface PaloAltoFirewallElementService extends PluggableService { public List listNetworks(ListPaloAltoFirewallNetworksCmd cmd); public PaloAltoFirewallResponse createPaloAltoFirewallResponse(ExternalFirewallDeviceVO fwDeviceVO); - - @Deprecated - // API helper function supported for backward compatibility - public - Host addExternalFirewall(AddExternalFirewallCmd cmd); - - @Deprecated - // API helper function supported for backward compatibility - public - boolean deleteExternalFirewall(DeleteExternalFirewallCmd cmd); - - @Deprecated - // API helper function supported for backward compatibility - public - List listExternalFirewalls(ListExternalFirewallsCmd cmd); - - @Deprecated - // API helper function supported for backward compatibility - public - ExternalFirewallResponse createExternalFirewallResponse(Host externalFirewall); } diff --git a/plugins/network-elements/palo-alto/src/com/cloud/network/resource/PaloAltoResource.java b/plugins/network-elements/palo-alto/src/com/cloud/network/resource/PaloAltoResource.java index 2fac31efac5..1b05a116f33 100644 --- a/plugins/network-elements/palo-alto/src/com/cloud/network/resource/PaloAltoResource.java +++ b/plugins/network-elements/palo-alto/src/com/cloud/network/resource/PaloAltoResource.java @@ -29,6 +29,7 @@ import java.util.Map; import javax.naming.ConfigurationException; import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.transform.dom.DOMSource; import javax.xml.transform.OutputKeys; import javax.xml.transform.Source; import javax.xml.transform.Transformer; @@ -55,6 +56,7 @@ import org.apache.http.message.BasicNameValuePair; import org.apache.http.protocol.HTTP; import org.apache.log4j.Logger; import org.w3c.dom.Document; +import org.w3c.dom.Node; import org.w3c.dom.NodeList; import org.xml.sax.InputSource; @@ -462,11 +464,14 @@ public class PaloAltoResource implements ServerResource { String guestVlanSubnet = NetUtils.getCidrSubNet(guestVlanGateway, cidrSize); Long publicVlanTag = null; - if (ip.getBroadcastUri() != null && !ip.getBroadcastUri().equals("untagged")) { - try { - publicVlanTag = Long.parseLong(ip.getBroadcastUri()); - } catch (Exception e) { - throw new ExecutionException("Could not parse public VLAN tag: " + ip.getBroadcastUri()); + if (ip.getBroadcastUri() != null) { + String parsedVlanTag = parsePublicVlanTag(ip.getBroadcastUri()); + if (!parsedVlanTag.equals("untagged")) { + try { + publicVlanTag = Long.parseLong(parsedVlanTag); + } catch (Exception e) { + throw new ExecutionException("Could not parse public VLAN tag: " + parsedVlanTag); + } } } @@ -520,6 +525,9 @@ public class PaloAltoResource implements ServerResource { String privateGateway, String privateSubnet, long privateCidrSize) throws ExecutionException { privateSubnet = privateSubnet + "/" + privateCidrSize; + // remove any orphaned egress rules if they exist... + removeOrphanedFirewallRules(cmdList, privateVlanTag); + if (type.equals(GuestNetworkType.SOURCE_NAT)) { manageNetworkIsolation(cmdList, PaloAltoPrimative.DELETE, privateVlanTag, privateSubnet, privateGateway); manageSrcNatRule(cmdList, PaloAltoPrimative.DELETE, type, publicVlanTag, sourceNatIpAddress + "/32", privateVlanTag, privateGateway + "/" + privateCidrSize); @@ -943,11 +951,16 @@ public class PaloAltoResource implements ServerResource { String dstNatName = genDstNatRuleName(publicIp, rule.getId()); String publicInterfaceName; - String publicVlanTag = rule.getSrcVlanTag(); - if (publicVlanTag == null || publicVlanTag.equals("untagged")) { + String publicVlanTag; + if (rule.getSrcVlanTag() == null) { publicInterfaceName = genPublicInterfaceName(new Long("9999")); } else { - publicInterfaceName = genPublicInterfaceName(new Long(publicVlanTag)); + publicVlanTag = parsePublicVlanTag(rule.getSrcVlanTag()); + if (publicVlanTag.equals("untagged")) { + publicInterfaceName = genPublicInterfaceName(new Long("9999")); + } else { + publicInterfaceName = genPublicInterfaceName(new Long(publicVlanTag)); + } } switch (prim) { @@ -1085,11 +1098,16 @@ public class PaloAltoResource implements ServerResource { String stcNatName = genStcNatRuleName(publicIp, rule.getId()); String publicInterfaceName; - String publicVlanTag = rule.getSrcVlanTag(); - if (publicVlanTag == null || publicVlanTag.equals("untagged")) { + String publicVlanTag; + if (rule.getSrcVlanTag() == null) { publicInterfaceName = genPublicInterfaceName(new Long("9999")); } else { - publicInterfaceName = genPublicInterfaceName(new Long(publicVlanTag)); + publicVlanTag = parsePublicVlanTag(rule.getSrcVlanTag()); + if (publicVlanTag.equals("untagged")) { + publicInterfaceName = genPublicInterfaceName(new Long("9999")); + } else { + publicInterfaceName = genPublicInterfaceName(new Long(publicVlanTag)); + } } switch (prim) { @@ -1170,13 +1188,20 @@ public class PaloAltoResource implements ServerResource { /* * Firewall rule implementation */ - - private String genFirewallRuleName(long id) { + private String genFirewallRuleName(long id) { // ingress return "policy_" + Long.toString(id); } + private String genFirewallRuleName(long id, String vlan) { // egress + return "policy_"+Long.toString(id)+"_"+vlan; + } public boolean manageFirewallRule(ArrayList cmdList, PaloAltoPrimative prim, FirewallRuleTO rule) throws ExecutionException { - String ruleName = genFirewallRuleName(rule.getId()); + String ruleName; + if (rule.getTrafficType() == FirewallRule.TrafficType.Egress) { + ruleName = genFirewallRuleName(rule.getId(), rule.getSrcVlanTag()); + } else { + ruleName = genFirewallRuleName(rule.getId()); + } switch (prim) { @@ -1203,6 +1228,7 @@ public class PaloAltoResource implements ServerResource { String serviceXML; String protocol = rule.getProtocol(); + String action = "allow"; // Only ICMP will use an Application, so others will be any. if (protocol.equals(Protocol.ICMP.toString())) { @@ -1232,11 +1258,23 @@ public class PaloAltoResource implements ServerResource { serviceXML = "any"; } - if (rule.getTrafficType() == FirewallRule.TrafficType.Egress) { // Network egress rule + // handle different types of fire wall rules (egress | ingress) + if (rule.getTrafficType() == FirewallRule.TrafficType.Egress) { // Egress Rule srcZone = _privateZone; dstZone = _publicZone; dstAddressXML = "any"; - } else { + + // defaults to 'allow', the deny rules are as follows + if (rule.getType() == FirewallRule.FirewallRuleType.System) { + if (!rule.isDefaultEgressPolicy()) { // default of deny && system rule, so deny + action = "deny"; + } + } else { + if (rule.isDefaultEgressPolicy()) { // default is allow && user rule, so deny + action = "deny"; + } + } + } else { // Ingress Rule srcZone = _publicZone; dstZone = _privateZone; dstAddressXML = "" + rule.getSrcIp() + ""; @@ -1265,6 +1303,7 @@ public class PaloAltoResource implements ServerResource { } } + // build new rule xml String xml = ""; xml += "" + srcZone + ""; xml += "" + dstZone + ""; @@ -1272,16 +1311,52 @@ public class PaloAltoResource implements ServerResource { xml += "" + dstAddressXML + ""; xml += "" + appXML + ""; xml += "" + serviceXML + ""; - xml += "allow"; + xml += ""+action+""; xml += "no"; xml += "no"; - if (_threatProfile != null) { // add the threat profile if it exists + if (_threatProfile != null && action.equals("allow")) { // add the threat profile if it exists xml += "" + _threatProfile + ""; } - if (_logProfile != null) { // add the log profile if it exists + if (_logProfile != null && action.equals("allow")) { // add the log profile if it exists xml += "" + _logProfile + ""; } + boolean has_default = false; + String defaultEgressRule = ""; + if (rule.getTrafficType() == FirewallRule.TrafficType.Egress) { + // check if a default egress rule exists because it always has to be after the other rules. + Map e_params = new HashMap(); + e_params.put("type", "config"); + e_params.put("action", "get"); + e_params.put("xpath", "/config/devices/entry/vsys/entry[@name='vsys1']/rulebase/security/rules/entry[@name='policy_0_"+rule.getSrcVlanTag()+"']"); + String e_response = request(PaloAltoMethod.GET, e_params); + has_default = (validResponse(e_response) && responseNotEmpty(e_response)); + + // there is an existing default rule, so we need to remove it and add it back after the new rule is added. + if (has_default) { + s_logger.debug("Moving the default egress rule after the new rule: "+ruleName); + NodeList response_body; + Document doc = getDocument(e_response); + XPath xpath = XPathFactory.newInstance().newXPath(); + try { + XPathExpression expr = xpath.compile("/response[@status='success']/result/entry/node()"); + response_body = (NodeList) expr.evaluate(doc, XPathConstants.NODESET); + } catch (XPathExpressionException e) { + throw new ExecutionException(e.getCause().getMessage()); + } + for (int i=0; i dd_params = new HashMap(); + dd_params.put("type", "config"); + dd_params.put("action", "delete"); + dd_params.put("xpath", "/config/devices/entry/vsys/entry[@name='vsys1']/rulebase/security/rules/entry[@name='policy_0_"+rule.getSrcVlanTag()+"']"); + cmdList.add(new DefaultPaloAltoCommand(PaloAltoMethod.POST, dd_params)); + } + } + + // add the new rule... Map a_params = new HashMap(); a_params.put("type", "config"); a_params.put("action", "set"); @@ -1289,6 +1364,17 @@ public class PaloAltoResource implements ServerResource { a_params.put("element", xml); cmdList.add(new DefaultPaloAltoCommand(PaloAltoMethod.POST, a_params)); + // add back the default rule + if (rule.getTrafficType() == FirewallRule.TrafficType.Egress && has_default) { + Map da_params = new HashMap(); + da_params.put("type", "config"); + da_params.put("action", "set"); + da_params.put("xpath", "/config/devices/entry/vsys/entry[@name='vsys1']/rulebase/security/rules/entry[@name='policy_0_"+rule.getSrcVlanTag()+"']"); + da_params.put("element", defaultEgressRule); + cmdList.add(new DefaultPaloAltoCommand(PaloAltoMethod.POST, da_params)); + s_logger.debug("Completed move of the default egress rule after rule: "+ruleName); + } + return true; case DELETE: @@ -1310,6 +1396,25 @@ public class PaloAltoResource implements ServerResource { } } + // remove orphaned rules if they exist... + public void removeOrphanedFirewallRules(ArrayList cmdList, long vlan) throws ExecutionException { + Map params = new HashMap(); + params.put("type", "config"); + params.put("action", "get"); + params.put("xpath", "/config/devices/entry/vsys/entry[@name='vsys1']/rulebase/security/rules/entry[contains(@name, 'policy') and contains(@name, '"+Long.toString(vlan)+"')]"); + String response = request(PaloAltoMethod.GET, params); + boolean has_orphans = (validResponse(response) && responseNotEmpty(response)); + + if (has_orphans) { + Map d_params = new HashMap(); + d_params.put("type", "config"); + d_params.put("action", "delete"); + d_params.put("xpath", "/config/devices/entry/vsys/entry[@name='vsys1']/rulebase/security/rules/entry[contains(@name, 'policy') and contains(@name, '"+Long.toString(vlan)+"')]"); + cmdList.add(new DefaultPaloAltoCommand(PaloAltoMethod.POST, d_params)); + } + } + + /* * Usage */ @@ -1935,6 +2040,10 @@ public class PaloAltoResource implements ServerResource { return ip.replace('.', '-').replace('/', '-'); } + private String parsePublicVlanTag(String uri) { + return uri.replace("vlan://", ""); + } + private Protocol getProtocol(String protocolName) throws ExecutionException { protocolName = protocolName.toLowerCase(); @@ -1964,6 +2073,20 @@ public class PaloAltoResource implements ServerResource { } } + // return an xml node as a string + private String nodeToString(Node node) throws ExecutionException { + StringWriter sw = new StringWriter(); + try { + Transformer t = TransformerFactory.newInstance().newTransformer(); + t.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes"); + t.transform(new DOMSource(node), new StreamResult(sw)); + } catch (Throwable t) { + throw new ExecutionException("XML convert error when modifying PA config: "+t.getMessage()); + } + return sw.toString(); + } + + // pretty printing of xml strings private String prettyFormat(String input) { int indent = 4; try { @@ -1999,14 +2122,12 @@ public class PaloAltoResource implements ServerResource { @Override public void setName(String name) { // TODO Auto-generated method stub - } //@Override @Override public void setConfigParams(Map params) { // TODO Auto-generated method stub - } //@Override @@ -2027,7 +2148,5 @@ public class PaloAltoResource implements ServerResource { @Override public void setRunLevel(int level) { // TODO Auto-generated method stub - } - } diff --git a/plugins/network-elements/palo-alto/test/com/cloud/network/resource/MockablePaloAltoResource.java b/plugins/network-elements/palo-alto/test/com/cloud/network/resource/MockablePaloAltoResource.java index 3b70bd61776..44cc65bfcd8 100755 --- a/plugins/network-elements/palo-alto/test/com/cloud/network/resource/MockablePaloAltoResource.java +++ b/plugins/network-elements/palo-alto/test/com/cloud/network/resource/MockablePaloAltoResource.java @@ -161,12 +161,12 @@ public class MockablePaloAltoResource extends PaloAltoResource { } } - // get egress firewall rule | has_egress_fw_rule | policy_0 - if (params.get("xpath").equals("/config/devices/entry/vsys/entry[@name='vsys1']/rulebase/security/rules/entry[@name='policy_0']")) { + // get egress firewall rule | has_egress_fw_rule | policy_0_3954 + if (params.get("xpath").equals("/config/devices/entry/vsys/entry[@name='vsys1']/rulebase/security/rules/entry[@name='policy_0_3954']")) { if (context.containsKey("has_egress_fw_rule") && context.get("has_egress_fw_rule").equals("true")) { - response = + response = "" - + "" + + "" + "trustuntrust10.3.96.1/20" + "anyanycs_tcp_80" + "allownono"; @@ -190,6 +190,11 @@ public class MockablePaloAltoResource extends PaloAltoResource { } } + // get default egress rule | policy_0_3954 + if (params.get("xpath").equals("/config/devices/entry/vsys/entry[@name='vsys1']/rulebase/security/rules/entry[contains(@name, 'policy') and contains(@name, '3954')]")) { + response = ""; + } + // get destination nat rule (port forwarding) | has_dst_nat_rule if (params.get("xpath").equals("/config/devices/entry/vsys/entry[@name='vsys1']/rulebase/nat/rules/entry[@name='dst_nat.192-168-80-103_9']")) { if (context.containsKey("has_dst_nat_rule") && context.get("has_dst_nat_rule").equals("true")) { @@ -292,7 +297,7 @@ public class MockablePaloAltoResource extends PaloAltoResource { } // add egress firewall rule - if (params.get("xpath").equals("/config/devices/entry/vsys/entry[@name='vsys1']/rulebase/security/rules/entry[@name='policy_0']")) { + if (params.get("xpath").equals("/config/devices/entry/vsys/entry[@name='vsys1']/rulebase/security/rules/entry[@name='policy_0_3954']")) { response = "command succeeded"; context.put("has_egress_fw_rule", "true"); } @@ -325,7 +330,7 @@ public class MockablePaloAltoResource extends PaloAltoResource { // action = 'delete' if (params.get("action").equals("delete")) { // remove egress firewall rule - if (params.get("xpath").equals("/config/devices/entry/vsys/entry[@name='vsys1']/rulebase/security/rules/entry[@name='policy_0']")) { + if (params.get("xpath").equals("/config/devices/entry/vsys/entry[@name='vsys1']/rulebase/security/rules/entry[@name='policy_0_3954']")) { response = "command succeeded"; context.remove("has_egress_fw_rule"); } diff --git a/scripts/vm/hypervisor/xenserver/cloudlog b/scripts/vm/hypervisor/xenserver/cloudlog new file mode 100644 index 00000000000..19560ef50e2 --- /dev/null +++ b/scripts/vm/hypervisor/xenserver/cloudlog @@ -0,0 +1,12 @@ +/var/log/cloud/vmops.log { + daily + size 1M + rotate 20 +} + +/var/log/cloud/ovstunnel.log /var/log/cloud/ovs-pvlan.log /var/log/cloud/swiftxen.log /var/log/cloud/s3xen /var/log/cloud/storageplugin { + daily + size 1M + rotate 2 +} + diff --git a/scripts/vm/hypervisor/xenserver/ovs-pvlan b/scripts/vm/hypervisor/xenserver/ovs-pvlan index 8579b551d9d..c9280a57af7 100755 --- a/scripts/vm/hypervisor/xenserver/ovs-pvlan +++ b/scripts/vm/hypervisor/xenserver/ovs-pvlan @@ -31,7 +31,7 @@ import util from time import localtime as _localtime, asctime as _asctime xePath = "/opt/xensource/bin/xe" -lib.setup_logging("/var/log/ovs-pvlan.log") +lib.setup_logging("/var/log/cloud/ovs-pvlan.log") dhcpSetupPath = "/opt/cloud/bin/ovs-pvlan-dhcp-host.sh" vmSetupPath = "/opt/cloud/bin/ovs-pvlan-vm.sh" getDhcpIfacePath = "/opt/cloud/bin/ovs-get-dhcp-iface.sh" @@ -41,9 +41,9 @@ getBridgePath = "/opt/cloud/bin/ovs-get-bridge.sh" def echo(fn): def wrapped(*v, **k): name = fn.__name__ - util.SMlog("#### VMOPS enter %s ####" % name) + logging.debug("#### VMOPS enter %s ####" % name) res = fn(*v, **k) - util.SMlog("#### VMOPS exit %s ####" % name) + logging.debug("#### VMOPS exit %s ####" % name) return res return wrapped diff --git a/scripts/vm/hypervisor/xenserver/ovstunnel b/scripts/vm/hypervisor/xenserver/ovstunnel index a38a0665522..6ecd8ea7757 100755 --- a/scripts/vm/hypervisor/xenserver/ovstunnel +++ b/scripts/vm/hypervisor/xenserver/ovstunnel @@ -35,7 +35,7 @@ import util from time import localtime as _localtime, asctime as _asctime xePath = "/opt/xensource/bin/xe" -lib.setup_logging("/var/log/ovstunnel.log") +lib.setup_logging("/var/log/cloud/ovstunnel.log") def block_ipv6_v5(bridge): @@ -54,9 +54,9 @@ block_ipv6_handlers = { def echo(fn): def wrapped(*v, **k): name = fn.__name__ - util.SMlog("#### VMOPS enter %s ####" % name) + logging.debug("#### VMOPS enter %s ####" % name) res = fn(*v, **k) - util.SMlog("#### VMOPS exit %s ####" % name) + logging.debug("#### VMOPS exit %s ####" % name) return res return wrapped diff --git a/scripts/vm/hypervisor/xenserver/perfmon.py b/scripts/vm/hypervisor/xenserver/perfmon.py new file mode 100755 index 00000000000..eb14e32aa54 --- /dev/null +++ b/scripts/vm/hypervisor/xenserver/perfmon.py @@ -0,0 +1,261 @@ +#!/usr/bin/python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pprint +import XenAPI +import urllib +from xml.dom import minidom +import time +import commands + +# Per VM dictionary (used by RRDUpdates to look up column numbers by variable names) +class VMReport(dict): + """Used internally by RRDUpdates""" + def __init__(self, uuid): + self.uuid = uuid + super(dict, self).__init__() + + +# Per Host dictionary (used by RRDUpdates to look up column numbers by variable names) +class HostReport(dict): + """Used internally by RRDUpdates""" + def __init__(self, uuid): + self.uuid = uuid + super(dict, self).__init__() + + +class PerfMonException(Exception): + pass + + +class XmlConfigException(PerfMonException): + pass + + +class UsageException(Exception): + pass + + +class RRDUpdates: + """ Object used to get and parse the output the http://localhost/rrd_udpates?... + """ + def __init__(self): + # params are what get passed to the CGI executable in the URL + self.params = dict() + self.params['start'] = int(time.time()) - 1000 # For demo purposes! + self.params['host'] = 'false' # include data for host (as well as for VMs) + self.params['cf'] = 'AVERAGE' # consolidation function, each sample averages 12 from the 5 second RRD + self.params['interval'] = '60' + + def get_nrows(self): + return self.rows + + def get_vm_list(self): + return self.vm_reports.keys() + + def get_vm_param_list(self, uuid): + report = self.vm_reports[uuid] + if not report: + return [] + return report.keys() + + def get_total_cpu_core(self, uuid): + report = self.vm_reports[uuid] + if not report: + return 0 + else: + param_keys = report.keys() + result = 0 + for param in param_keys: + if "cpu" in param: + result += 1 + return result + + def get_vm_data(self, uuid, param, row): + #pp = pprint.PrettyPrinter(indent=4) + #pp.pprint(self.vm_reports) + report = self.vm_reports[uuid] + col = report[param] + return self.__lookup_data(col, row) + + def get_host_uuid(self): + report = self.host_report + if not report: + return None + return report.uuid + + def get_host_param_list(self): + report = self.host_report + if not report: + return [] + return report.keys() + + def get_host_data(self, param, row): + report = self.host_report + col = report[param] + return self.__lookup_data(col, row) + + def get_row_time(self, row): + return self.__lookup_timestamp(row) + + # extract float from value () node by col,row + def __lookup_data(self, col, row): + # Note: the nodes are in reverse chronological order, and comprise + # a timestamp node, followed by self.columns data nodes + node = self.data_node.childNodes[self.rows - 1 - row].childNodes[col + 1] + return float(node.firstChild.toxml()) # node.firstChild should have nodeType TEXT_NODE + + # extract int from value () node by row + def __lookup_timestamp(self, row): + # Note: the nodes are in reverse chronological order, and comprise + # a timestamp node, followed by self.columns data nodes + node = self.data_node.childNodes[self.rows - 1 - row].childNodes[0] + return int(node.firstChild.toxml()) # node.firstChild should have nodeType TEXT_NODE + + def refresh(self, login, starttime, session, override_params): + self.params['start'] = starttime + params = override_params + params['session_id'] = session + params.update(self.params) + paramstr = "&".join(["%s=%s" % (k, params[k]) for k in params]) + # this is better than urllib.urlopen() as it raises an Exception on http 401 'Unauthorised' error + # rather than drop into interactive mode + for host in login.host.get_all(): + #print "http://" + str(login.host.get_address(host)) + "/rrd_updates?%s" % paramstr + sock = urllib.URLopener().open("http://" + str(login.host.get_address(host)) + "/rrd_updates?%s" % paramstr) + xmlsource = sock.read() + sock.close() + xmldoc = minidom.parseString(xmlsource) + self.__parse_xmldoc(xmldoc) + # Update the time used on the next run + self.params['start'] = self.end_time + 1 # avoid retrieving same data twice + + def __parse_xmldoc(self, xmldoc): + # The 1st node contains meta data (description of the data) + # The 2nd node contains the data + self.meta_node = xmldoc.firstChild.childNodes[0] + self.data_node = xmldoc.firstChild.childNodes[1] + + def lookup_metadata_bytag(name): + return int(self.meta_node.getElementsByTagName(name)[0].firstChild.toxml()) + # rows = number of samples per variable + # columns = number of variables + self.rows = lookup_metadata_bytag('rows') + self.columns = lookup_metadata_bytag('columns') + # These indicate the period covered by the data + self.start_time = lookup_metadata_bytag('start') + self.step_time = lookup_metadata_bytag('step') + self.end_time = lookup_metadata_bytag('end') + # the Node describes the variables + self.legend = self.meta_node.getElementsByTagName('legend')[0] + # vm_reports matches uuid to per VM report + if not hasattr(self,'vm_reports'): + self.vm_reports = {} + # There is just one host_report and its uuid should not change! + self.host_report = None + # Handle each column. (I.e. each variable) + for col in range(self.columns): + self.__handle_col(col) + + def __handle_col(self, col): + # work out how to interpret col from the legend + col_meta_data = self.legend.childNodes[col].firstChild.toxml() + # vm_or_host will be 'vm' or 'host'. Note that the Control domain counts as a VM! + (cf, vm_or_host, uuid, param) = col_meta_data.split(':') + if vm_or_host == 'vm': + # Create a report for this VM if it doesn't exist + if not uuid in self.vm_reports: + self.vm_reports[uuid] = VMReport(uuid) + # Update the VMReport with the col data and meta data + vm_report = self.vm_reports[uuid] + vm_report[param] = col + elif vm_or_host == 'host': + # Create a report for the host if it doesn't exist + if not self.host_report: + self.host_report = HostReport(uuid) + elif self.host_report.uuid != uuid: + raise PerfMonException("Host UUID changed: (was %s, is %s)" % (self.host_report.uuid, uuid)) + # Update the HostReport with the col data and meta data + self.host_report[param] = col + else: + raise PerfMonException("Invalid string in : %s" % col_meta_data) + +def getuuid(vm_name): + status, output = commands.getstatusoutput("xe vm-list | grep "+vm_name+" -B 1 | head -n 1 | awk -F':' '{print $2}' | tr -d ' '") + if (status != 0): + raise PerfMonException("Invalid vm name: %s" % vm_name) + return output + +def get_vm_group_perfmon(args={}): + login = XenAPI.xapi_local() + login.login_with_password("","") + result = "" + + total_vm = int(args['total_vm']) + total_counter = int(args['total_counter']) + now = int(time.time()) / 60 + + # Get pool's info of this host + #pool = login.xenapi.pool.get_all()[0] + # Get master node's address of pool + #master = login.xenapi.pool.get_master(pool) + #master_address = login.xenapi.host.get_address(master) + session = login._session + + max_duration = 0 + for counter_count in xrange(1, total_counter + 1): + duration = int(args['duration' + str(counter_count)]) + if duration > max_duration: + max_duration = duration + + rrd_updates = RRDUpdates() + rrd_updates.refresh(login.xenapi, now * 60 - max_duration, session, {}) + + #for uuid in rrd_updates.get_vm_list(): + for vm_count in xrange(1, total_vm + 1): + vm_name = args['vmname' + str(vm_count)] + vm_uuid = getuuid(vm_name) + #print "Got values for VM: " + str(vm_count) + " " + vm_uuid + for counter_count in xrange(1, total_counter + 1): + #refresh average + average_cpu = 0 + average_memory = 0 + counter = args['counter' + str(counter_count)] + total_row = rrd_updates.get_nrows() + duration = int(args['duration' + str(counter_count)]) / 60 + duration_diff = total_row - duration + if counter == "cpu": + total_cpu = rrd_updates.get_total_cpu_core(vm_uuid) + for row in xrange(duration_diff, total_row): + for cpu in xrange(0, total_cpu): + average_cpu += rrd_updates.get_vm_data(vm_uuid, "cpu" + str(cpu), row) + average_cpu /= (duration * total_cpu) + if result == "": + result += str(vm_count) + '.' + str(counter_count) + ':' + str(average_cpu) + else: + result += ',' + str(vm_count) + '.' + str(counter_count) + ':' + str(average_cpu) + elif counter == "memory": + for row in xrange(duration_diff, total_row): + average_memory += rrd_updates.get_vm_data(vm_uuid, "memory_target", row) / 1048576 - rrd_updates.get_vm_data(vm_uuid, "memory_internal_free", row) / 1024 + average_memory /= duration + if result == "": + result += str(vm_count) + '.' + str(counter_count) + ':' + str(average_memory) + else: + result += ',' + str(vm_count) + '.' + str(counter_count) + ':' + str(average_memory) + return result + diff --git a/scripts/vm/hypervisor/xenserver/s3xen b/scripts/vm/hypervisor/xenserver/s3xen index bf81bbd34a6..ccd7ce51f27 100644 --- a/scripts/vm/hypervisor/xenserver/s3xen +++ b/scripts/vm/hypervisor/xenserver/s3xen @@ -39,6 +39,10 @@ from xml.dom.minidom import parseString import XenAPIPlugin sys.path.extend(["/opt/xensource/sm/"]) import util +import cloudstack_pluginlib as lib +import logging + +lib.setup_logging("/var/log/cloud/s3xen.log") NULL = 'null' @@ -102,7 +106,7 @@ def get_optional_key(map, key, default=''): def log(message): - util.SMlog('#### VMOPS %s ####' % message) + logging.debug('#### VMOPS %s ####' % message) def echo(fn): diff --git a/scripts/vm/hypervisor/xenserver/storagePlugin b/scripts/vm/hypervisor/xenserver/storagePlugin index bb033797620..ff00fa273c3 100755 --- a/scripts/vm/hypervisor/xenserver/storagePlugin +++ b/scripts/vm/hypervisor/xenserver/storagePlugin @@ -32,13 +32,17 @@ import subprocess import zlib import urllib2 import traceback +import cloudstack_pluginlib as lib +import logging + +lib.setup_logging("/var/log/cloud/storageplugin.log") def echo(fn): def wrapped(*v, **k): name = fn.__name__ - util.SMlog("#### xen plugin enter %s ####" % name ) + logging.debug("#### xen plugin enter %s ####" % name ) res = fn(*v, **k) - util.SMlog("#### xen plugin exit %s ####" % name ) + logging.debug("#### xen plugin exit %s ####" % name ) return res return wrapped @@ -53,7 +57,7 @@ def downloadTemplateFromUrl(session, args): destFile.close() return "success" except: - util.SMlog("exception: " + str(sys.exc_info())) + logging.debug("exception: " + str(sys.exc_info())) return "" @echo diff --git a/scripts/vm/hypervisor/xenserver/swiftxen b/scripts/vm/hypervisor/xenserver/swiftxen index 46229e37262..8342238ad82 100644 --- a/scripts/vm/hypervisor/xenserver/swiftxen +++ b/scripts/vm/hypervisor/xenserver/swiftxen @@ -24,13 +24,17 @@ import os, sys, time import XenAPIPlugin sys.path.extend(["/opt/xensource/sm/"]) import util +import cloudstack_pluginlib as lib +import logging + +lib.setup_logging("/var/log/cloud/swiftxen.log") def echo(fn): def wrapped(*v, **k): name = fn.__name__ - util.SMlog("#### VMOPS enter %s ####" % name ) + logging.debug("#### VMOPS enter %s ####" % name ) res = fn(*v, **k) - util.SMlog("#### VMOPS exit %s ####" % name ) + logging.debug("#### VMOPS exit %s ####" % name ) return res return wrapped @@ -48,7 +52,7 @@ def upload(args): lfilename = args['lfilename'] isISCSI = args['isISCSI'] segment = 0 - util.SMlog("#### VMOPS upload %s to swift ####", lfilename) + logging.debug("#### VMOPS upload %s to swift ####", lfilename) savedpath = os.getcwd() os.chdir(ldir) try : @@ -85,7 +89,7 @@ def swift(session, args): elif op == 'delete' : cmd = ["st", "-A https://" + hostname + ":8080/auth/v1.0 -U " + account + ":" + username + " -K " + token + " delete " + rfilename] else : - util.SMlog("doesn't support swift operation %s " % op ) + logging.debug("doesn't support swift operation %s " % op ) return 'false' try: util.pread2(cmd) diff --git a/scripts/vm/hypervisor/xenserver/vmops b/scripts/vm/hypervisor/xenserver/vmops index e4b3caf38a8..2e14fad5b9c 100755 --- a/scripts/vm/hypervisor/xenserver/vmops +++ b/scripts/vm/hypervisor/xenserver/vmops @@ -34,14 +34,18 @@ import tempfile import util import subprocess import zlib +import cloudstack_pluginlib as lib +import logging from util import CommandException +lib.setup_logging("/var/log/cloud/vmops.log") + def echo(fn): def wrapped(*v, **k): name = fn.__name__ - util.SMlog("#### VMOPS enter %s ####" % name ) + logging.debug("#### VMOPS enter %s ####" % name ) res = fn(*v, **k) - util.SMlog("#### VMOPS exit %s ####" % name ) + logging.debug("#### VMOPS exit %s ####" % name ) return res return wrapped @@ -96,7 +100,7 @@ def preparemigration(session, args): util.pread2(cmd) txt = 'success' except: - util.SMlog("Catch prepare migration exception" ) + logging.debug("Catch prepare migration exception" ) txt = '' return txt @@ -108,7 +112,7 @@ def setIptables(session, args): txt = util.pread2(cmd) txt = 'success' except: - util.SMlog(" setIptables execution failed " ) + logging.debug(" setIptables execution failed " ) txt = '' return txt @@ -153,7 +157,7 @@ def pingtest(session, args): txt = util.pread2(cmd) txt = 'success' except: - util.SMlog(" pingtest failed " ) + logging.debug(" pingtest failed " ) txt = '' return txt @@ -168,7 +172,7 @@ def savePassword(session, args): txt = util.pread2(cmd) txt = 'success' except: - util.SMlog(" save password to domr failed " ) + logging.debug(" save password to domr failed " ) txt = '' return txt @@ -183,7 +187,7 @@ def saveDhcpEntry(session, args): txt = util.pread2(cmd) txt = 'success' except: - util.SMlog(" save dhcp entry failed " ) + logging.debug(" save dhcp entry failed " ) txt = '' return txt @@ -245,7 +249,7 @@ def setFirewallRule(session, args): txt = util.pread2(cmd) txt = 'success' except: - util.SMlog(" set firewall rule failed " ) + logging.debug(" set firewall rule failed " ) txt = '' return txt @@ -261,7 +265,7 @@ def routerProxy(session, args): if txt is None or len(txt) == 0 : txt = 'success' except: - util.SMlog("routerProxy command " + sargs + " failed " ) + logging.debug("routerProxy command " + sargs + " failed " ) txt = '' return txt @@ -278,7 +282,7 @@ def setLoadBalancerRule(session, args): txt = util.pread2(cmd) txt = 'success' except: - util.SMlog(" set loadbalancer rule failed " ) + logging.debug(" set loadbalancer rule failed " ) txt = '' return txt @@ -291,7 +295,7 @@ def configdnsmasq(session, args): util.pread2(['ssh','-p','3922','-q','-o','StrictHostKeyChecking=no','-i','/root/.ssh/id_rsa.cloud',target,'/root/dnsmasq.sh',args]) txt='success' except: - util.SMlog("failed to config dnsmasq server") + logging.debug("failed to config dnsmasq server") txt='' return txt @@ -305,7 +309,7 @@ def createipAlias(session, args): txt=util.pread2(cmd) txt='success' except: - util.SMlog("failed to create ip alias on router vm") + logging.debug("failed to create ip alias on router vm") txt='' return txt @@ -319,7 +323,7 @@ def deleteipAlias(session, args): txt=util.pread2(cmd) txt='success' except: - util.SMlog("failed to create ip alias on router vm") + logging.debug("failed to create ip alias on router vm") txt='' return txt @@ -334,7 +338,7 @@ def createFile(session, args): f.close() txt = 'success' except: - util.SMlog(" failed to create HA proxy cfg file ") + logging.debug(" failed to create HA proxy cfg file ") txt = '' return txt @@ -354,7 +358,7 @@ def createFileInDomr(session, args): util.pread2(['rm',tmpfile]) txt = 'success' except: - util.SMlog(" failed to create HA proxy cfg file ") + logging.debug(" failed to create HA proxy cfg file ") txt = '' return txt @@ -368,7 +372,7 @@ def deleteFile(session, args): os.remove(file_path) txt = 'success' except: - util.SMlog(" failed to remove HA proxy cfg file ") + logging.debug(" failed to remove HA proxy cfg file ") txt = '' return txt @@ -409,7 +413,7 @@ def can_bridge_firewall(session, args): util.pread2(['iptables', '-A', 'BRIDGE-FIREWALL', '-m', 'physdev', '--physdev-is-bridged', '-p', 'udp', '--dport', '68', '--sport', '67', '-j', 'ACCEPT']) util.pread2(['iptables', '-D', 'FORWARD', '-j', 'RH-Firewall-1-INPUT']) except: - util.SMlog('Chain BRIDGE-FIREWALL already exists') + logging.debug('Chain BRIDGE-FIREWALL already exists') try: util.pread2(['iptables', '-N', 'BRIDGE-DEFAULT-FIREWALL']) @@ -421,7 +425,7 @@ def can_bridge_firewall(session, args): util.pread2(['iptables', '-D', 'BRIDGE-FIREWALL', '-m', 'physdev', '--physdev-is-bridged', '-p', 'udp', '--dport', '67', '--sport', '68', '-j', 'ACCEPT']) util.pread2(['iptables', '-D', 'BRIDGE-FIREWALL', '-m', 'physdev', '--physdev-is-bridged', '-p', 'udp', '--dport', '68', '--sport', '67', '-j', 'ACCEPT']) except: - util.SMlog('Chain BRIDGE-DEFAULT-FIREWALL already exists') + logging.debug('Chain BRIDGE-DEFAULT-FIREWALL already exists') result = 'true' try: @@ -468,7 +472,7 @@ def default_ebtables_rules(): # deny all others (e.g., 802.1d, CDP) util.pread2(['ebtables', '-A', 'DEFAULT_EBTABLES', '-j', 'DROP']) except: - util.SMlog('Chain DEFAULT_EBTABLES already exists') + logging.debug('Chain DEFAULT_EBTABLES already exists') @echo @@ -485,7 +489,7 @@ def allow_egress_traffic(session): try: util.pread2(['iptables', '-I', 'FORWARD', '2', '-m', 'physdev', '--physdev-is-bridged', '--physdev-out', d, '-j', 'ACCEPT']) except: - util.SMlog("Failed to add FORWARD rule through to %s" % d) + logging.debug("Failed to add FORWARD rule through to %s" % d) return 'false' return 'true' @@ -494,7 +498,7 @@ def ipset(ipsetname, proto, start, end, ips): try: util.pread2(['ipset', '-N', ipsetname, 'iptreemap']) except: - util.SMlog("ipset chain already exists" + ipsetname) + logging.debug("ipset chain already exists" + ipsetname) result = True ipsettmp = ''.join(''.join(ipsetname.split('-')).split('_')) + str(int(time.time()) % 1000) @@ -502,11 +506,11 @@ def ipset(ipsetname, proto, start, end, ips): try: util.pread2(['ipset', '-N', ipsettmp, 'iptreemap']) except: - util.SMlog("Failed to create temp ipset, reusing old name= " + ipsettmp) + logging.debug("Failed to create temp ipset, reusing old name= " + ipsettmp) try: util.pread2(['ipset', '-F', ipsettmp]) except: - util.SMlog("Failed to clear old temp ipset name=" + ipsettmp) + logging.debug("Failed to clear old temp ipset name=" + ipsettmp) return False try: @@ -517,7 +521,7 @@ def ipset(ipsetname, proto, start, end, ips): if cex.reason.rfind('already in set') == -1: raise except: - util.SMlog("Failed to program ipset " + ipsetname) + logging.debug("Failed to program ipset " + ipsetname) util.pread2(['ipset', '-F', ipsettmp]) util.pread2(['ipset', '-X', ipsettmp]) return False @@ -525,7 +529,7 @@ def ipset(ipsetname, proto, start, end, ips): try: util.pread2(['ipset', '-W', ipsettmp, ipsetname]) except: - util.SMlog("Failed to swap ipset " + ipsetname) + logging.debug("Failed to swap ipset " + ipsetname) result = False try: @@ -533,7 +537,7 @@ def ipset(ipsetname, proto, start, end, ips): util.pread2(['ipset', '-X', ipsettmp]) except: # if the temporary name clashes next time we'll just reuse it - util.SMlog("Failed to delete temp ipset " + ipsettmp) + logging.debug("Failed to delete temp ipset " + ipsettmp) return result @@ -550,7 +554,7 @@ def destroy_network_rules_for_vm(session, args): util.pread2(['iptables', '-F', vmchain_default]) util.pread2(['iptables', '-X', vmchain_default]) except: - util.SMlog("Ignoring failure to delete chain " + vmchain_default) + logging.debug("Ignoring failure to delete chain " + vmchain_default) destroy_ebtables_rules(vmchain) destroy_arptables_rules(vmchain) @@ -559,14 +563,14 @@ def destroy_network_rules_for_vm(session, args): util.pread2(['iptables', '-F', vmchain]) util.pread2(['iptables', '-X', vmchain]) except: - util.SMlog("Ignoring failure to delete ingress chain " + vmchain) + logging.debug("Ignoring failure to delete ingress chain " + vmchain) try: util.pread2(['iptables', '-F', vmchain_egress]) util.pread2(['iptables', '-X', vmchain_egress]) except: - util.SMlog("Ignoring failure to delete egress chain " + vmchain_egress) + logging.debug("Ignoring failure to delete egress chain " + vmchain_egress) remove_rule_log_for_vm(vm_name) remove_secip_log_for_vm(vm_name) @@ -582,7 +586,7 @@ def destroy_network_rules_for_vm(session, args): util.pread2(['ipset', '-F', set]) util.pread2(['ipset', '-X', set]) except: - util.SMlog("Failed to destroy ipsets for %" % vm_name) + logging.debug("Failed to destroy ipsets for %" % vm_name) return 'true' @@ -599,12 +603,12 @@ def destroy_ebtables_rules(vm_chain): dc.insert(0, 'ebtables') util.pread2(dc) except: - util.SMlog("Ignoring failure to delete ebtables rules for vm " + vm_chain) + logging.debug("Ignoring failure to delete ebtables rules for vm " + vm_chain) try: util.pread2(['ebtables', '-F', vm_chain]) util.pread2(['ebtables', '-X', vm_chain]) except: - util.SMlog("Ignoring failure to delete ebtables chain for vm " + vm_chain) + logging.debug("Ignoring failure to delete ebtables chain for vm " + vm_chain) @echo def destroy_arptables_rules(vm_chain): @@ -619,18 +623,18 @@ def destroy_arptables_rules(vm_chain): dc.insert(2, 'FORWARD') util.pread2(dc) except: - util.SMlog("Ignoring failure to delete arptables rules for vm " + vm_chain) + logging.debug("Ignoring failure to delete arptables rules for vm " + vm_chain) try: util.pread2(['arptables', '-F', vm_chain]) util.pread2(['arptables', '-X', vm_chain]) except: - util.SMlog("Ignoring failure to delete arptables chain for vm " + vm_chain) + logging.debug("Ignoring failure to delete arptables chain for vm " + vm_chain) @echo def default_ebtables_antispoof_rules(vm_chain, vifs, vm_ip, vm_mac): if vm_mac == 'ff:ff:ff:ff:ff:ff': - util.SMlog("Ignoring since mac address is not valid") + logging.debug("Ignoring since mac address is not valid") return 'true' try: @@ -639,7 +643,7 @@ def default_ebtables_antispoof_rules(vm_chain, vifs, vm_ip, vm_mac): try: util.pread2(['ebtables', '-F', vm_chain]) except: - util.SMlog("Failed to create ebtables antispoof chain, skipping") + logging.debug("Failed to create ebtables antispoof chain, skipping") return 'true' # note all rules for packets into the bridge (-i) precede all output rules (-o) @@ -649,7 +653,7 @@ def default_ebtables_antispoof_rules(vm_chain, vifs, vm_ip, vm_mac): util.pread2(['ebtables', '-I', 'FORWARD', '2', '-i', vif, '-j', vm_chain]) util.pread2(['ebtables', '-A', 'FORWARD', '-o', vif, '-j', vm_chain]) except: - util.SMlog("Failed to program default ebtables FORWARD rules for %s" % vm_chain) + logging.debug("Failed to program default ebtables FORWARD rules for %s" % vm_chain) return 'false' try: @@ -665,7 +669,7 @@ def default_ebtables_antispoof_rules(vm_chain, vifs, vm_ip, vm_mac): # do not allow snooping of dhcp requests util.pread2(['ebtables', '-A', vm_chain, '-o', vif, '-p', 'IPv4', '--ip-proto', 'udp', '--ip-dport', '67', '-j', 'DROP']) except: - util.SMlog("Failed to program default ebtables antispoof rules for %s" % vm_chain) + logging.debug("Failed to program default ebtables antispoof rules for %s" % vm_chain) return 'false' return 'true' @@ -673,7 +677,7 @@ def default_ebtables_antispoof_rules(vm_chain, vifs, vm_ip, vm_mac): @echo def default_arp_antispoof(vm_chain, vifs, vm_ip, vm_mac): if vm_mac == 'ff:ff:ff:ff:ff:ff': - util.SMlog("Ignoring since mac address is not valid") + logging.debug("Ignoring since mac address is not valid") return 'true' try: @@ -682,7 +686,7 @@ def default_arp_antispoof(vm_chain, vifs, vm_ip, vm_mac): try: util.pread2(['arptables', '-F', vm_chain]) except: - util.SMlog("Failed to create arptables rule, skipping") + logging.debug("Failed to create arptables rule, skipping") return 'true' # note all rules for packets into the bridge (-i) precede all output rules (-o) @@ -691,7 +695,7 @@ def default_arp_antispoof(vm_chain, vifs, vm_ip, vm_mac): util.pread2(['arptables', '-I', 'FORWARD', '-i', vif, '-j', vm_chain]) util.pread2(['arptables', '-A', 'FORWARD', '-o', vif, '-j', vm_chain]) except: - util.SMlog("Failed to program default arptables rules in FORWARD chain vm=" + vm_chain) + logging.debug("Failed to program default arptables rules in FORWARD chain vm=" + vm_chain) return 'false' try: @@ -708,7 +712,7 @@ def default_arp_antispoof(vm_chain, vifs, vm_ip, vm_mac): util.pread2(['arptables', '-A', vm_chain, '-j', 'DROP']) except: - util.SMlog("Failed to program default arptables rules") + logging.debug("Failed to program default arptables rules") return 'false' return 'true' @@ -720,10 +724,10 @@ def network_rules_vmSecondaryIp(session, args): vm_mac = args.pop('vmMac') ip_secondary = args.pop('vmSecIp') action = args.pop('action') - util.SMlog("vmMac = "+ vm_mac) - util.SMlog("vmName = "+ vm_name) + logging.debug("vmMac = "+ vm_mac) + logging.debug("vmName = "+ vm_name) #action = "-A" - util.SMlog("action = "+ action) + logging.debug("action = "+ action) try: vm = session.xenapi.VM.get_by_name_label(vm_name) if len(vm) != 1: @@ -733,11 +737,11 @@ def network_rules_vmSecondaryIp(session, args): vifnums = [session.xenapi.VIF.get_record(vif).get('device') for vif in vm_vifs] domid = vm_rec.get('domid') except: - util.SMlog("### Failed to get domid or vif list for vm ##" + vm_name) + logging.debug("### Failed to get domid or vif list for vm ##" + vm_name) return 'false' if domid == '-1': - util.SMlog("### Failed to get domid for vm (-1): " + vm_name) + logging.debug("### Failed to get domid for vm (-1): " + vm_name) return 'false' vifs = ["vif" + domid + "." + v for v in vifnums] @@ -767,11 +771,11 @@ def default_network_rules_systemvm(session, args): vifnums = [session.xenapi.VIF.get_record(vif).get('device') for vif in vm_vifs] domid = vm_rec.get('domid') except: - util.SMlog("### Failed to get domid or vif list for vm ##" + vm_name) + logging.debug("### Failed to get domid or vif list for vm ##" + vm_name) return 'false' if domid == '-1': - util.SMlog("### Failed to get domid for vm (-1): " + vm_name) + logging.debug("### Failed to get domid for vm (-1): " + vm_name) return 'false' vifs = ["vif" + domid + "." + v for v in vifnums] @@ -792,26 +796,26 @@ def default_network_rules_systemvm(session, args): util.pread2(['iptables', '-I', 'BRIDGE-FIREWALL', '2', '-m', 'physdev', '--physdev-is-bridged', '--physdev-in', vif, '-j', vmchain]) util.pread2(['iptables', '-I', vmchain, '-m', 'physdev', '--physdev-is-bridged', '--physdev-in', vif, '-j', 'RETURN']) except: - util.SMlog("Failed to program default rules") + logging.debug("Failed to program default rules") return 'false' util.pread2(['iptables', '-A', vmchain, '-j', 'ACCEPT']) if write_rule_log_for_vm(vm_name, '-1', '_ignore_', domid, '_initial_', '-1') == False: - util.SMlog("Failed to log default network rules for systemvm, ignoring") + logging.debug("Failed to log default network rules for systemvm, ignoring") return 'true' @echo def create_ipset_forvm (ipsetname): result = True try: - util.SMlog("Creating ipset chain .... " + ipsetname) + logging.debug("Creating ipset chain .... " + ipsetname) util.pread2(['ipset', '-F', ipsetname]) util.pread2(['ipset', '-X', ipsetname]) util.pread2(['ipset', '-N', ipsetname, 'iphash']) except: - util.SMlog("ipset chain not exists creating.... " + ipsetname) + logging.debug("ipset chain not exists creating.... " + ipsetname) util.pread2(['ipset', '-N', ipsetname, 'iphash']) return result @@ -821,10 +825,10 @@ def add_to_ipset(ipsetname, ips, action): result = True for ip in ips: try: - util.SMlog("vm ip " + ip) + logging.debug("vm ip " + ip) util.pread2(['ipset', action, ipsetname, ip]) except: - util.SMlog("vm ip alreday in ip set" + ip) + logging.debug("vm ip alreday in ip set" + ip) continue return result @@ -846,7 +850,7 @@ def arp_rules_vmip (vm_chain, vifs, ips, vm_mac, action): #also important to restrict source ip and src mac in these requests as they can be used to update arp tables on destination util.pread2(['arptables', action, vm_chain, '-i', vif, '--opcode', 'Request', '--source-mac', vm_mac, '--source-ip', vm_ip, '-j', 'RETURN']) except: - util.SMlog("Failed to program arptables rules for ip") + logging.debug("Failed to program arptables rules for ip") return 'false' return 'true' @@ -864,15 +868,15 @@ def default_network_rules(session, args): try: vm = session.xenapi.VM.get_by_name_label(vm_name) if len(vm) != 1: - util.SMlog("### Failed to get record for vm " + vm_name) + logging.debug("### Failed to get record for vm " + vm_name) return 'false' vm_rec = session.xenapi.VM.get_record(vm[0]) domid = vm_rec.get('domid') except: - util.SMlog("### Failed to get domid for vm " + vm_name) + logging.debug("### Failed to get domid for vm " + vm_name) return 'false' if domid == '-1': - util.SMlog("### Failed to get domid for vm (-1): " + vm_name) + logging.debug("### Failed to get domid for vm (-1): " + vm_name) return 'false' vif = "vif" + domid + ".0" @@ -912,12 +916,12 @@ def default_network_rules(session, args): vmipset = vm_name #create ipset and add vm ips to that ip set if create_ipset_forvm(vmipset) == False: - util.SMlog(" failed to create ipset for rule " + str(tokens)) + logging.debug(" failed to create ipset for rule " + str(tokens)) return 'false' #add primary nic ip to ipset if add_to_ipset(vmipset, [vm_ip], action ) == False: - util.SMlog(" failed to add vm " + vm_ip + " ip to set ") + logging.debug(" failed to add vm " + vm_ip + " ip to set ") return 'false' #add secodnary nic ips to ipset @@ -928,10 +932,10 @@ def default_network_rules(session, args): secIpSet = "0"; if secIpSet == "1": - util.SMlog("Adding ipset for secondary ips") + logging.debug("Adding ipset for secondary ips") add_to_ipset(vmipset, ips, action) if write_secip_log_for_vm(vm_name, sec_ips, vm_id) == False: - util.SMlog("Failed to log default network rules, ignoring") + logging.debug("Failed to log default network rules, ignoring") keyword = '--' + get_ipset_keyword() @@ -949,21 +953,21 @@ def default_network_rules(session, args): util.pread2(['iptables', '-A', vmchain_default, '-m', 'physdev', '--physdev-is-bridged', '--physdev-in', v, '-m', 'set', keyword, vmipset, 'src', '-j', vmchain_egress]) util.pread2(['iptables', '-A', vmchain_default, '-m', 'physdev', '--physdev-is-bridged', '--physdev-out', v, '-j', vmchain]) except: - util.SMlog("Failed to program default rules for vm " + vm_name) + logging.debug("Failed to program default rules for vm " + vm_name) return 'false' default_arp_antispoof(vmchain, vifs, vm_ip, vm_mac) #add default arp rules for secondary ips; if secIpSet == "1": - util.SMlog("Adding arp rules for sec ip") + logging.debug("Adding arp rules for sec ip") arp_rules_vmip(vmchain, vifs, ips, vm_mac, action) default_ebtables_antispoof_rules(vmchain, vifs, vm_ip, vm_mac) if write_rule_log_for_vm(vm_name, vm_id, vm_ip, domid, '_initial_', '-1', vm_mac) == False: - util.SMlog("Failed to log default network rules, ignoring") + logging.debug("Failed to log default network rules, ignoring") - util.SMlog("Programmed default rules for vm " + vm_name) + logging.debug("Programmed default rules for vm " + vm_name) return 'true' @echo @@ -972,12 +976,12 @@ def check_domid_changed(session, vmName): try: vm = session.xenapi.VM.get_by_name_label(vmName) if len(vm) != 1: - util.SMlog("### Could not get record for vm ## " + vmName) + logging.debug("### Could not get record for vm ## " + vmName) else: vm_rec = session.xenapi.VM.get_record(vm[0]) curr_domid = vm_rec.get('domid') except: - util.SMlog("### Failed to get domid for vm ## " + vmName) + logging.debug("### Failed to get domid for vm ## " + vmName) logfilename = "/var/run/cloud/" + vmName +".log" @@ -1011,7 +1015,7 @@ def delete_rules_for_vm_in_bridge_firewall_chain(vmName): dc.pop() util.pread2(filter(None, dc)) except: - util.SMlog("Ignoring failure to delete rules for vm " + vmName) + logging.debug("Ignoring failure to delete rules for vm " + vmName) @echo @@ -1028,7 +1032,7 @@ def network_rules_for_rebooted_vm(session, vmName): if curr_domid == '-1': return True - util.SMlog("Found a rebooted VM -- reprogramming rules for " + vm_name) + logging.debug("Found a rebooted VM -- reprogramming rules for " + vm_name) delete_rules_for_vm_in_bridge_firewall_chain(vm_name) if 1 in [ vm_name.startswith(c) for c in ['r-', 's-', 'v-', 'l-'] ]: @@ -1073,9 +1077,9 @@ def network_rules_for_rebooted_vm(session, vmName): try: util.pread2(filter(None,ipt)) except: - util.SMlog("Failed to rewrite antispoofing rules for vm " + vm_name) + logging.debug("Failed to rewrite antispoofing rules for vm " + vm_name) except: - util.SMlog("No rules found for vm " + vm_name) + logging.debug("No rules found for vm " + vm_name) destroy_ebtables_rules(vmchain) destroy_arptables_rules(vmchain) @@ -1179,7 +1183,7 @@ def get_rule_logs_for_vms(session, args): hostrec = session.xenapi.host.get_record(thishost) vms = hostrec.get('resident_VMs') except: - util.SMlog("Failed to get host from uuid " + host_uuid) + logging.debug("Failed to get host from uuid " + host_uuid) return ' ' result = [] @@ -1192,7 +1196,7 @@ def get_rule_logs_for_vms(session, args): log = get_rule_log_for_vm(session, name) result.append(log) except: - util.SMlog("Failed to get rule logs, better luck next time!") + logging.debug("Failed to get rule logs, better luck next time!") return ";".join(result) @@ -1209,13 +1213,13 @@ def cleanup_rules_for_dead_vms(session): vm_rec = session.xenapi.VM.get_record(vm[0]) state = vm_rec.get('power_state') if state != 'Running' and state != 'Paused': - util.SMlog("vm " + vm_name + " is not running, cleaning up") + logging.debug("vm " + vm_name + " is not running, cleaning up") destroy_network_rules_for_vm(session, {'vmName':vm_name}) cleaned = cleaned+1 - util.SMlog("Cleaned up rules for " + str(cleaned) + " vms") + logging.debug("Cleaned up rules for " + str(cleaned) + " vms") except: - util.SMlog("Failed to cleanup rules for dead vms!") + logging.debug("Failed to cleanup rules for dead vms!") @echo @@ -1234,12 +1238,12 @@ def cleanup_rules(session, args): hostrec = session.xenapi.host.get_record(thishost[0]) vms = hostrec.get('resident_VMs') resident_vms = [session.xenapi.VM.get_name_label(x) for x in vms] - util.SMlog('cleanup_rules: found %s resident vms on this host %s' % (len(resident_vms)-1, hostname[0])) + logging.debug('cleanup_rules: found %s resident vms on this host %s' % (len(resident_vms)-1, hostname[0])) chainscmd = "iptables-save | grep '^:' | awk '{print $1}' | cut -d':' -f2 | sed 's/-def/-%s/'| sed 's/-eg//' | sort|uniq" % instance chains = util.pread2(['/bin/bash', '-c', chainscmd]).split('\n') vmchains = [ch for ch in chains if 1 in [ ch.startswith(c) for c in ['r-', 'i-', 's-', 'v-', 'l-']]] - util.SMlog('cleanup_rules: found %s iptables chains for vms on this host %s' % (len(vmchains), hostname[0])) + logging.debug('cleanup_rules: found %s iptables chains for vms on this host %s' % (len(vmchains), hostname[0])) cleaned = 0 cleanup = [] for chain in vmchains: @@ -1247,16 +1251,16 @@ def cleanup_rules(session, args): if vmname not in resident_vms: vmname = chain + "-untagged" if vmname not in resident_vms: - util.SMlog("vm " + chain + " is not running on this host, cleaning up") + logging.debug("vm " + chain + " is not running on this host, cleaning up") cleanup.append(chain) for vm_name in cleanup: destroy_network_rules_for_vm(session, {'vmName':vm_name}) - util.SMlog("Cleaned up rules for " + str(len(cleanup)) + " chains") + logging.debug("Cleaned up rules for " + str(len(cleanup)) + " chains") return str(len(cleanup)) except Exception, ex: - util.SMlog("Failed to cleanup rules, reason= " + str(ex)) + logging.debug("Failed to cleanup rules, reason= " + str(ex)) return '-1'; @echo @@ -1264,7 +1268,7 @@ def check_rule_log_for_vm(vmName, vmID, vmIP, domID, signature, seqno): vm_name = vmName; logfilename = "/var/run/cloud/" + vm_name +".log" if not os.path.exists(logfilename): - util.SMlog("Failed to find logfile %s" %logfilename) + logging.debug("Failed to find logfile %s" %logfilename) return [True, True, True] lines = (line.rstrip() for line in open(logfilename)) @@ -1278,39 +1282,39 @@ def check_rule_log_for_vm(vmName, vmID, vmIP, domID, signature, seqno): [_vmName,_vmID,_vmIP,_domID,_signature,_seqno] = line.split(',') break except: - util.SMlog("Failed to parse log file for vm " + vmName) + logging.debug("Failed to parse log file for vm " + vmName) remove_rule_log_for_vm(vmName) return [True, True, True] reprogramDefault = False if (domID != _domID) or (vmID != _vmID) or (vmIP != _vmIP): - util.SMlog("Change in default info set of vm %s" % vmName) + logging.debug("Change in default info set of vm %s" % vmName) return [True, True, True] else: - util.SMlog("No change in default info set of vm %s" % vmName) + logging.debug("No change in default info set of vm %s" % vmName) reprogramChain = False rewriteLog = True if (int(seqno) > int(_seqno)): if (_signature != signature): reprogramChain = True - util.SMlog("Seqno increased from %s to %s: reprogamming "\ + logging.debug("Seqno increased from %s to %s: reprogamming "\ "ingress rules for vm %s" % (_seqno, seqno, vmName)) else: - util.SMlog("Seqno increased from %s to %s: but no change "\ + logging.debug("Seqno increased from %s to %s: but no change "\ "in signature for vm: skip programming ingress "\ "rules %s" % (_seqno, seqno, vmName)) elif (int(seqno) < int(_seqno)): - util.SMlog("Seqno decreased from %s to %s: ignoring these "\ + logging.debug("Seqno decreased from %s to %s: ignoring these "\ "ingress rules for vm %s" % (_seqno, seqno, vmName)) rewriteLog = False elif (signature != _signature): - util.SMlog("Seqno %s stayed the same but signature changed from "\ + logging.debug("Seqno %s stayed the same but signature changed from "\ "%s to %s for vm %s" % (seqno, _signature, signature, vmName)) rewriteLog = True reprogramChain = True else: - util.SMlog("Seqno and signature stayed the same: %s : ignoring these "\ + logging.debug("Seqno and signature stayed the same: %s : ignoring these "\ "ingress rules for vm %s" % (seqno, vmName)) rewriteLog = False @@ -1320,7 +1324,7 @@ def check_rule_log_for_vm(vmName, vmID, vmIP, domID, signature, seqno): def write_secip_log_for_vm (vmName, secIps, vmId): vm_name = vmName logfilename = "/var/run/cloud/"+vm_name+".ip" - util.SMlog("Writing log to " + logfilename) + logging.debug("Writing log to " + logfilename) logf = open(logfilename, 'w') output = ','.join([vmName, secIps, vmId]) result = True @@ -1329,7 +1333,7 @@ def write_secip_log_for_vm (vmName, secIps, vmId): logf.write(output) logf.write('\n') except: - util.SMlog("Failed to write to rule log file " + logfilename) + logging.debug("Failed to write to rule log file " + logfilename) result = False logf.close() @@ -1345,7 +1349,7 @@ def remove_secip_log_for_vm(vmName): try: os.remove(logfilename) except: - util.SMlog("Failed to delete rule log file " + logfilename) + logging.debug("Failed to delete rule log file " + logfilename) result = False return result @@ -1354,7 +1358,7 @@ def remove_secip_log_for_vm(vmName): def write_rule_log_for_vm(vmName, vmID, vmIP, domID, signature, seqno, vmMac='ff:ff:ff:ff:ff:ff'): vm_name = vmName logfilename = "/var/run/cloud/" + vm_name +".log" - util.SMlog("Writing log to " + logfilename) + logging.debug("Writing log to " + logfilename) logf = open(logfilename, 'w') output = ','.join([vmName, vmID, vmIP, domID, signature, seqno, vmMac]) result = True @@ -1362,7 +1366,7 @@ def write_rule_log_for_vm(vmName, vmID, vmIP, domID, signature, seqno, vmMac='ff logf.write(output) logf.write('\n') except: - util.SMlog("Failed to write to rule log file " + logfilename) + logging.debug("Failed to write to rule log file " + logfilename) result = False logf.close() @@ -1378,7 +1382,7 @@ def remove_rule_log_for_vm(vmName): try: os.remove(logfilename) except: - util.SMlog("Failed to delete rule log file " + logfilename) + logging.debug("Failed to delete rule log file " + logfilename) result = False return result @@ -1408,13 +1412,13 @@ def cache_ipset_keyword(): pass cachefile = "/var/cache/cloud/ipset.keyword" - util.SMlog("Writing ipset keyword to " + cachefile) + logging.debug("Writing ipset keyword to " + cachefile) cachef = open(cachefile, 'w') try: cachef.write(keyword) cachef.write('\n') except: - util.SMlog("Failed to write to cache file " + cachef) + logging.debug("Failed to write to cache file " + cachef) cachef.close() return keyword @@ -1425,7 +1429,7 @@ def get_ipset_keyword(): keyword = 'match-set' if not os.path.exists(cachefile): - util.SMlog("Failed to find ipset keyword cachefile %s" %cachefile) + logging.debug("Failed to find ipset keyword cachefile %s" %cachefile) keyword = cache_ipset_keyword() else: lines = (line.rstrip() for line in open(cachefile)) @@ -1458,15 +1462,15 @@ def network_rules(session, args): try: vm = session.xenapi.VM.get_by_name_label(vm_name) if len(vm) != 1: - util.SMlog("### Could not get record for vm ## " + vm_name) + logging.debug("### Could not get record for vm ## " + vm_name) return 'false' vm_rec = session.xenapi.VM.get_record(vm[0]) domid = vm_rec.get('domid') except: - util.SMlog("### Failed to get domid for vm ## " + vm_name) + logging.debug("### Failed to get domid for vm ## " + vm_name) return 'false' if domid == '-1': - util.SMlog("### Failed to get domid for vm (-1): " + vm_name) + logging.debug("### Failed to get domid for vm (-1): " + vm_name) return 'false' vif = "vif" + domid + ".0" @@ -1484,21 +1488,21 @@ def network_rules(session, args): check_rule_log_for_vm (vm_name, vm_id, vm_ip, domid, signature, seqno) if not reprogramDefault and not reprogramChain: - util.SMlog("No changes detected between current state and received state") + logging.debug("No changes detected between current state and received state") reason = 'seqno_same_sig_same' if rewriteLog: reason = 'seqno_increased_sig_same' write_rule_log_for_vm(vm_name, vm_id, vm_ip, domid, signature, seqno, vm_mac) - util.SMlog("Programming network rules for vm %s seqno=%s signature=%s guestIp=%s,"\ + logging.debug("Programming network rules for vm %s seqno=%s signature=%s guestIp=%s,"\ " do nothing, reason=%s" % (vm_name, seqno, signature, vm_ip, reason)) return 'true' if not reprogramChain: - util.SMlog("###Not programming any ingress rules since no changes detected?") + logging.debug("###Not programming any ingress rules since no changes detected?") return 'true' if reprogramDefault: - util.SMlog("Change detected in vmId or vmIp or domId, resetting default rules") + logging.debug("Change detected in vmId or vmIp or domId, resetting default rules") default_network_rules(session, args) reason = 'domid_change' @@ -1508,7 +1512,7 @@ def network_rules(session, args): keyword = '--' + get_ipset_keyword() lines = rules.split(' ') - util.SMlog("Programming network rules for vm %s seqno=%s numrules=%s signature=%s guestIp=%s,"\ + logging.debug("Programming network rules for vm %s seqno=%s numrules=%s signature=%s guestIp=%s,"\ " update iptables, reason=%s" % (vm_name, seqno, len(lines), signature, vm_ip, reason)) cmds = [] @@ -1546,7 +1550,7 @@ def network_rules(session, args): ipsetname = vmchain + "_" + protocol + "_any" if ipset(ipsetname, protocol, start, end, ips) == False: - util.SMlog(" failed to create ipset for rule " + str(tokens)) + logging.debug(" failed to create ipset for rule " + str(tokens)) if protocol == 'all': iptables = ['iptables', '-I', vmchain, '-m', 'state', '--state', 'NEW', '-m', 'set', keyword, ipsetname, direction, '-j', action] @@ -1559,7 +1563,7 @@ def network_rules(session, args): iptables = ['iptables', '-I', vmchain, '-p', 'icmp', '--icmp-type', range, '-m', 'set', keyword, ipsetname, direction, '-j', action] cmds.append(iptables) - util.SMlog(iptables) + logging.debug(iptables) if allow_any and protocol != 'all': if protocol != 'icmp': @@ -1570,20 +1574,20 @@ def network_rules(session, args): range = "any" iptables = ['iptables', '-I', vmchain, '-p', 'icmp', '--icmp-type', range, '-j', action] cmds.append(iptables) - util.SMlog(iptables) + logging.debug(iptables) vmchain = chain_name(vm_name) try: util.pread2(['iptables', '-F', vmchain]) except: - util.SMlog("Ignoring failure to delete chain " + vmchain) + logging.debug("Ignoring failure to delete chain " + vmchain) util.pread2(['iptables', '-N', vmchain]) egress_vmchain = egress_chain_name(vm_name) try: util.pread2(['iptables', '-F', egress_vmchain]) except: - util.SMlog("Ignoring failure to delete chain " + egress_vmchain) + logging.debug("Ignoring failure to delete chain " + egress_vmchain) util.pread2(['iptables', '-N', egress_vmchain]) @@ -1602,7 +1606,7 @@ def network_rules(session, args): return 'true' except: - util.SMlog("Failed to network rule !") + logging.debug("Failed to network rule !") @echo def bumpUpPriority(session, args): @@ -1614,7 +1618,7 @@ def bumpUpPriority(session, args): txt = util.pread2(cmd) txt = 'success' except: - util.SMlog("bump up priority fail! ") + logging.debug("bump up priority fail! ") txt = '' return txt diff --git a/scripts/vm/hypervisor/xenserver/vmopsSnapshot b/scripts/vm/hypervisor/xenserver/vmopsSnapshot index 00ed93af337..a9a5a72ea62 100755 --- a/scripts/vm/hypervisor/xenserver/vmopsSnapshot +++ b/scripts/vm/hypervisor/xenserver/vmopsSnapshot @@ -38,6 +38,10 @@ import xs_errors import cleanup import stat import random +import cloudstack_pluginlib as lib +import logging + +lib.setup_logging("/var/log/cloud/vmops.log") VHDUTIL = "vhd-util" VHD_PREFIX = 'VHD-' @@ -46,9 +50,9 @@ CLOUD_DIR = '/var/run/cloud_mount' def echo(fn): def wrapped(*v, **k): name = fn.__name__ - util.SMlog("#### VMOPS enter %s ####" % name ) + logging.debug("#### VMOPS enter %s ####" % name ) res = fn(*v, **k) - util.SMlog("#### VMOPS exit %s ####" % name ) + logging.debug("#### VMOPS exit %s ####" % name ) return res return wrapped @@ -57,7 +61,7 @@ def echo(fn): def create_secondary_storage_folder(session, args): local_mount_path = None - util.SMlog("create_secondary_storage_folder, args: " + str(args)) + logging.debug("create_secondary_storage_folder, args: " + str(args)) try: try: @@ -74,11 +78,11 @@ def create_secondary_storage_folder(session, args): os.umask(current_umask) except OSError, (errno, strerror): errMsg = "create_secondary_storage_folder failed: errno: " + str(errno) + ", strerr: " + strerror - util.SMlog(errMsg) + logging.debug(errMsg) raise xs_errors.XenError(errMsg) except: errMsg = "create_secondary_storage_folder failed." - util.SMlog(errMsg) + logging.debug(errMsg) raise xs_errors.XenError(errMsg) finally: if local_mount_path != None: @@ -93,7 +97,7 @@ def create_secondary_storage_folder(session, args): def delete_secondary_storage_folder(session, args): local_mount_path = None - util.SMlog("delete_secondary_storage_folder, args: " + str(args)) + logging.debug("delete_secondary_storage_folder, args: " + str(args)) try: try: @@ -109,11 +113,11 @@ def delete_secondary_storage_folder(session, args): os.system("rmdir " + folder) except OSError, (errno, strerror): errMsg = "delete_secondary_storage_folder failed: errno: " + str(errno) + ", strerr: " + strerror - util.SMlog(errMsg) + logging.debug(errMsg) raise xs_errors.XenError(errMsg) except: errMsg = "delete_secondary_storage_folder failed." - util.SMlog(errMsg) + logging.debug(errMsg) raise xs_errors.XenError(errMsg) finally: if local_mount_path != None: @@ -159,16 +163,16 @@ def post_create_private_template(session, args): f.write("vhd.size=" + str(file_size) + "\n") f.write("size=" + str(file_size) + "\n") f.close() - util.SMlog("Created template.properties file") + logging.debug("Created template.properties file") # Set permissions permissions = stat.S_IREAD | stat.S_IWRITE | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH | stat.S_IWOTH os.chmod(template_properties_install_path, permissions) - util.SMlog("Set permissions on template and template.properties") + logging.debug("Set permissions on template and template.properties") except: errMsg = "post_create_private_template failed." - util.SMlog(errMsg) + logging.debug(errMsg) raise xs_errors.XenError(errMsg) finally: @@ -189,12 +193,12 @@ def isfile(path, isISCSI): if not exists: errMsg = "File " + path + " does not exist." - util.SMlog(errMsg) + logging.debug(errMsg) raise xs_errors.XenError(errMsg) return errMsg def copyfile(fromFile, toFile, isISCSI): - util.SMlog("Starting to copy " + fromFile + " to " + toFile) + logging.debug("Starting to copy " + fromFile + " to " + toFile) errMsg = '' try: cmd = ['dd', 'if=' + fromFile, 'of=' + toFile, 'bs=4M'] @@ -206,10 +210,10 @@ def copyfile(fromFile, toFile, isISCSI): txt = '' txt = '' errMsg = "Error while copying " + fromFile + " to " + toFile + " in secondary storage" - util.SMlog(errMsg) + logging.debug(errMsg) raise xs_errors.XenError(errMsg) - util.SMlog("Successfully copied " + fromFile + " to " + toFile) + logging.debug("Successfully copied " + fromFile + " to " + toFile) return errMsg def chdir(path): @@ -217,9 +221,9 @@ def chdir(path): os.chdir(path) except OSError, (errno, strerror): errMsg = "Unable to chdir to " + path + " because of OSError with errno: " + str(errno) + " and strerr: " + strerror - util.SMlog(errMsg) + logging.debug(errMsg) raise xs_errors.XenError(errMsg) - util.SMlog("Chdired to " + path) + logging.debug("Chdired to " + path) return def scanParent(path): @@ -234,7 +238,7 @@ def scanParent(path): parentUUID = vhdInfo.parentUuid except: errMsg = "Could not get vhd parent of " + path - util.SMlog(errMsg) + logging.debug(errMsg) raise xs_errors.XenError(errMsg) return parentUUID @@ -247,7 +251,7 @@ def getParent(path, isISCSI): parentUUID = vhdutil.getParent(path, cleanup.FileVDI.extractUuid) except: errMsg = "Could not get vhd parent of " + path - util.SMlog(errMsg) + logging.debug(errMsg) raise xs_errors.XenError(errMsg) return parentUUID @@ -262,7 +266,7 @@ def getParentOfSnapshot(snapshotUuid, primarySRPath, isISCSI): else: baseCopyUuid = getParent(snapshotPath, isISCSI) - util.SMlog("Base copy of snapshotUuid: " + snapshotUuid + " is " + baseCopyUuid) + logging.debug("Base copy of snapshotUuid: " + snapshotUuid + " is " + baseCopyUuid) return baseCopyUuid def setParent(parent, child): @@ -271,9 +275,9 @@ def setParent(parent, child): txt = util.pread2(cmd) except: errMsg = "Unexpected error while trying to set parent of " + child + " to " + parent - util.SMlog(errMsg) + logging.debug(errMsg) raise xs_errors.XenError(errMsg) - util.SMlog("Successfully set parent of " + child + " to " + parent) + logging.debug("Successfully set parent of " + child + " to " + parent) return def rename(originalVHD, newVHD): @@ -281,7 +285,7 @@ def rename(originalVHD, newVHD): os.rename(originalVHD, newVHD) except OSError, (errno, strerror): errMsg = "OSError while renaming " + origiinalVHD + " to " + newVHD + "with errno: " + str(errno) + " and strerr: " + strerror - util.SMlog(errMsg) + logging.debug(errMsg) raise xs_errors.XenError(errMsg) return @@ -294,7 +298,7 @@ def makedirs(path): if os.path.isdir(path): return errMsg = "OSError while creating " + path + " with errno: " + str(errno) + " and strerr: " + strerror - util.SMlog(errMsg) + logging.debug(errMsg) raise xs_errors.XenError(errMsg) return @@ -307,9 +311,9 @@ def mount(remoteDir, localDir): except: txt = '' errMsg = "Unexpected error while trying to mount " + remoteDir + " to " + localDir - util.SMlog(errMsg) + logging.debug(errMsg) raise xs_errors.XenError(errMsg) - util.SMlog("Successfully mounted " + remoteDir + " to " + localDir) + logging.debug("Successfully mounted " + remoteDir + " to " + localDir) return @@ -319,10 +323,10 @@ def umount(localDir): util.pread2(cmd) except CommandException: errMsg = "CommandException raised while trying to umount " + localDir - util.SMlog(errMsg) + logging.debug(errMsg) raise xs_errors.XenError(errMsg) - util.SMlog("Successfully unmounted " + localDir) + logging.debug("Successfully unmounted " + localDir) return def mountSnapshotsDir(secondaryStorageMountPath, localMountPointPath, path): @@ -344,7 +348,7 @@ def mountSnapshotsDir(secondaryStorageMountPath, localMountPointPath, path): # There is more than one secondary storage per zone. # And we are mounting each sec storage under a zone-specific directory # So two secondary storage snapshot dirs will never get mounted on the same point on the same XenServer. - util.SMlog("The remote snapshots directory has already been mounted on " + localMountPointPath) + logging.debug("The remote snapshots directory has already been mounted on " + localMountPointPath) else: mount(snapshotsDir, localMountPointPath) @@ -357,11 +361,11 @@ def unmountAll(path): try: for dir in os.listdir(path): if dir.isdigit(): - util.SMlog("Unmounting Sub-Directory: " + dir) + logging.debug("Unmounting Sub-Directory: " + dir) localMountPointPath = os.path.join(path, dir) umount(localMountPointPath) except: - util.SMlog("Ignoring the error while trying to unmount the snapshots dir") + logging.debug("Ignoring the error while trying to unmount the snapshots dir") @echo def unmountSnapshotsDir(session, args): @@ -372,7 +376,7 @@ def unmountSnapshotsDir(session, args): try: umount(localMountPointPath) except: - util.SMlog("Ignoring the error while trying to unmount the snapshots dir.") + logging.debug("Ignoring the error while trying to unmount the snapshots dir.") return "1" @@ -409,14 +413,14 @@ def makeUnavailable(uuid, primarySRPath, isISCSI): def manageAvailability(path, value): if path.__contains__("/var/run/sr-mount"): return - util.SMlog("Setting availability of " + path + " to " + value) + logging.debug("Setting availability of " + path + " to " + value) try: cmd = ['/usr/sbin/lvchange', value, path] util.pread2(cmd) except: #CommandException, (rc, cmdListStr, stderr): #errMsg = "CommandException thrown while executing: " + cmdListStr + " with return code: " + str(rc) + " and stderr: " + stderr errMsg = "Unexpected exception thrown by lvchange" - util.SMlog(errMsg) + logging.debug(errMsg) if value == "-ay": # Raise an error only if we are trying to make it available. # Just warn if we are trying to make it unavailable after the @@ -434,7 +438,7 @@ def checkVolumeAvailablility(path): manageAvailability(path, '-ay') except: errMsg = "Could not determine status of ISCSI path: " + path - util.SMlog(errMsg) + logging.debug(errMsg) raise xs_errors.XenError(errMsg) success = False @@ -445,7 +449,7 @@ def checkVolumeAvailablility(path): # set isISCSI to true success = isVolumeAvailable(path) if success: - util.SMlog("Made vhd: " + path + " available and confirmed that it is visible") + logging.debug("Made vhd: " + path + " available and confirmed that it is visible") break # Sleep for 10 seconds before checking again. @@ -453,7 +457,7 @@ def checkVolumeAvailablility(path): # If not visible within 1 min fail if not success: - util.SMlog("Could not make vhd: " + path + " available despite waiting for 1 minute. Does it exist?") + logging.debug("Could not make vhd: " + path + " available despite waiting for 1 minute. Does it exist?") return success @@ -465,19 +469,19 @@ def isVolumeAvailable(path): status = p.communicate()[0].strip("\n") except: errMsg = "Could not determine status of ISCSI path: " + path - util.SMlog(errMsg) + logging.debug(errMsg) raise xs_errors.XenError(errMsg) return (status == "1") def getVhdParent(session, args): - util.SMlog("getParent with " + str(args)) + logging.debug("getParent with " + str(args)) primaryStorageSRUuid = args['primaryStorageSRUuid'] snapshotUuid = args['snapshotUuid'] isISCSI = getIsTrueString(args['isISCSI']) primarySRPath = getPrimarySRPath(primaryStorageSRUuid, isISCSI) - util.SMlog("primarySRPath: " + primarySRPath) + logging.debug("primarySRPath: " + primarySRPath) baseCopyUuid = getParentOfSnapshot(snapshotUuid, primarySRPath, isISCSI) @@ -485,7 +489,7 @@ def getVhdParent(session, args): def backupSnapshot(session, args): - util.SMlog("Called backupSnapshot with " + str(args)) + logging.debug("Called backupSnapshot with " + str(args)) primaryStorageSRUuid = args['primaryStorageSRUuid'] secondaryStorageMountPath = args['secondaryStorageMountPath'] snapshotUuid = args['snapshotUuid'] @@ -495,19 +499,19 @@ def backupSnapshot(session, args): path = args['path'] localMountPoint = args['localMountPoint'] primarySRPath = getPrimarySRPath(primaryStorageSRUuid, isISCSI) - util.SMlog("primarySRPath: " + primarySRPath) + logging.debug("primarySRPath: " + primarySRPath) baseCopyUuid = getParentOfSnapshot(snapshotUuid, primarySRPath, isISCSI) baseCopyVHD = getVHD(baseCopyUuid, isISCSI) baseCopyPath = os.path.join(primarySRPath, baseCopyVHD) - util.SMlog("Base copy path: " + baseCopyPath) + logging.debug("Base copy path: " + baseCopyPath) # Mount secondary storage mount path on XenServer along the path # /var/run/sr-mount//snapshots/ and create / dir # on it. backupsDir = mountSnapshotsDir(secondaryStorageMountPath, localMountPoint, path) - util.SMlog("Backups dir " + backupsDir) + logging.debug("Backups dir " + backupsDir) prevBackupUuid = prevBackupUuid.split("/")[-1] # Check existence of snapshot on primary storage isfile(baseCopyPath, isISCSI) @@ -520,7 +524,7 @@ def backupSnapshot(session, args): # copy baseCopyPath to backupsDir with new uuid backupVHD = getBackupVHD(backupUuid) backupFile = os.path.join(backupsDir, backupVHD) - util.SMlog("Back up " + baseCopyUuid + " to Secondary Storage as " + backupUuid) + logging.debug("Back up " + baseCopyUuid + " to Secondary Storage as " + backupUuid) copyfile(baseCopyPath, backupFile, isISCSI) vhdutil.setHidden(backupFile, False) @@ -536,7 +540,7 @@ def backupSnapshot(session, args): @echo def deleteSnapshotBackup(session, args): - util.SMlog("Calling deleteSnapshotBackup with " + str(args)) + logging.debug("Calling deleteSnapshotBackup with " + str(args)) secondaryStorageMountPath = args['secondaryStorageMountPath'] backupUUID = args['backupUUID'] path = args['path'] @@ -547,27 +551,27 @@ def deleteSnapshotBackup(session, args): chdir(backupsDir) backupVHD = getBackupVHD(backupUUID) - util.SMlog("checking existence of " + backupVHD) + logging.debug("checking existence of " + backupVHD) # The backupVHD is on secondary which is NFS and not ISCSI. if not os.path.isfile(backupVHD): - util.SMlog("backupVHD " + backupVHD + "does not exist. Not trying to delete it") + logging.debug("backupVHD " + backupVHD + "does not exist. Not trying to delete it") return "1" - util.SMlog("backupVHD " + backupVHD + " exists.") + logging.debug("backupVHD " + backupVHD + " exists.") # Just delete the backupVHD try: os.remove(backupVHD) except OSError, (errno, strerror): errMsg = "OSError while removing " + backupVHD + " with errno: " + str(errno) + " and strerr: " + strerror - util.SMlog(errMsg) + logging.debug(errMsg) raise xs_errors.XenError(errMsg) return "1" @echo def revert_memory_snapshot(session, args): - util.SMlog("Calling revert_memory_snapshot with " + str(args)) + logging.debug("Calling revert_memory_snapshot with " + str(args)) vmName = args['vmName'] snapshotUUID = args['snapshotUUID'] oldVmUuid = args['oldVmUuid'] @@ -587,7 +591,7 @@ def revert_memory_snapshot(session, args): os.system("xe vdi-destroy uuid=" + vdiUuid) except OSError, (errno, strerror): errMsg = "OSError while reverting vm " + vmName + " to snapshot " + snapshotUUID + " with errno: " + str(errno) + " and strerr: " + strerror - util.SMlog(errMsg) + logging.debug(errMsg) raise xs_errors.XenError(errMsg) return "0" diff --git a/scripts/vm/hypervisor/xenserver/vmopspremium b/scripts/vm/hypervisor/xenserver/vmopspremium index d7d0c6f60bf..143aa922a8a 100755 --- a/scripts/vm/hypervisor/xenserver/vmopspremium +++ b/scripts/vm/hypervisor/xenserver/vmopspremium @@ -28,13 +28,18 @@ if os.path.exists("/usr/lib/xcp/sm"): sys.path.extend(["/usr/lib/xcp/sm/", "/usr/local/sbin/", "/sbin/"]) import util import socket +import perfmon +import cloudstack_pluginlib as lib +import logging + +lib.setup_logging("/var/log/cloud/vmops.log") def echo(fn): def wrapped(*v, **k): name = fn.__name__ - util.SMlog("#### VMOPS enter %s ####" % name ) + logging.debug("#### VMOPS enter %s ####" % name ) res = fn(*v, **k) - util.SMlog("#### VMOPS exit %s ####" % name ) + logging.debug("#### VMOPS exit %s ####" % name ) return res return wrapped @@ -142,6 +147,11 @@ def heartbeat(session, args): txt='fail' return txt -if __name__ == "__main__": - XenAPIPlugin.dispatch({"forceShutdownVM":forceShutdownVM, "upgrade_snapshot":upgrade_snapshot, "create_privatetemplate_from_snapshot":create_privatetemplate_from_snapshot, "copy_vhd_to_secondarystorage":copy_vhd_to_secondarystorage, "copy_vhd_from_secondarystorage":copy_vhd_from_secondarystorage, "setup_heartbeat_sr":setup_heartbeat_sr, "setup_heartbeat_file":setup_heartbeat_file, "check_heartbeat":check_heartbeat, "heartbeat": heartbeat}) +@echo +def asmonitor(session, args): + result = perfmon.get_vm_group_perfmon(args) + return result + +if __name__ == "__main__": + XenAPIPlugin.dispatch({"forceShutdownVM":forceShutdownVM, "upgrade_snapshot":upgrade_snapshot, "create_privatetemplate_from_snapshot":create_privatetemplate_from_snapshot, "copy_vhd_to_secondarystorage":copy_vhd_to_secondarystorage, "copy_vhd_from_secondarystorage":copy_vhd_from_secondarystorage, "setup_heartbeat_sr":setup_heartbeat_sr, "setup_heartbeat_file":setup_heartbeat_file, "check_heartbeat":check_heartbeat, "heartbeat": heartbeat, "asmonitor": asmonitor}) diff --git a/scripts/vm/hypervisor/xenserver/xcposs/patch b/scripts/vm/hypervisor/xenserver/xcposs/patch index efb4a1814ec..2f902c23b44 100644 --- a/scripts/vm/hypervisor/xenserver/xcposs/patch +++ b/scripts/vm/hypervisor/xenserver/xcposs/patch @@ -70,3 +70,4 @@ deleteipAlias.sh=..,0755,/opt/cloud/bin ###add cloudstack plugin script for XCP cloudstack_plugins.conf=..,0644,/etc/xensource cloudstack_pluginlib.py=..,0755,/etc/xapi.d/plugins +cloudlog=..,0644,/etc/logrotate.d diff --git a/scripts/vm/hypervisor/xenserver/xcpserver/patch b/scripts/vm/hypervisor/xenserver/xcpserver/patch index 45ff59d0618..01cd73b605c 100644 --- a/scripts/vm/hypervisor/xenserver/xcpserver/patch +++ b/scripts/vm/hypervisor/xenserver/xcpserver/patch @@ -68,3 +68,4 @@ add_to_vcpus_params_live.sh=..,0755,/opt/cloud/bin ###add cloudstack plugin script for XCP cloudstack_plugins.conf=..,0644,/etc/xensource cloudstack_pluginlib.py=..,0755,/etc/xapi.d/plugins +cloudlog=..,0644,/etc/logrotate.d diff --git a/scripts/vm/hypervisor/xenserver/xenserver56/patch b/scripts/vm/hypervisor/xenserver/xenserver56/patch index 602cb99a224..50116935789 100644 --- a/scripts/vm/hypervisor/xenserver/xenserver56/patch +++ b/scripts/vm/hypervisor/xenserver/xenserver56/patch @@ -69,3 +69,4 @@ add_to_vcpus_params_live.sh=..,0755,/opt/cloud/bin ###add cloudstack plugin script for XCP cloudstack_plugins.conf=..,0644,/etc/xensource cloudstack_pluginlib.py=..,0755,/etc/xapi.d/plugins +cloudlog=..,0644,/etc/logrotate.d diff --git a/scripts/vm/hypervisor/xenserver/xenserver56fp1/patch b/scripts/vm/hypervisor/xenserver/xenserver56fp1/patch index 8e72c0088e8..55d538edf4a 100644 --- a/scripts/vm/hypervisor/xenserver/xenserver56fp1/patch +++ b/scripts/vm/hypervisor/xenserver/xenserver56fp1/patch @@ -68,4 +68,4 @@ add_to_vcpus_params_live.sh=..,0755,/opt/cloud/bin ###add cloudstack plugin script for XCP cloudstack_plugins.conf=..,0644,/etc/xensource cloudstack_pluginlib.py=..,0755,/etc/xapi.d/plugins - +cloudlog=..,0644,/etc/logrotate.d diff --git a/scripts/vm/hypervisor/xenserver/xenserver60/patch b/scripts/vm/hypervisor/xenserver/xenserver60/patch index 8a0448a2009..5a648e0990f 100644 --- a/scripts/vm/hypervisor/xenserver/xenserver60/patch +++ b/scripts/vm/hypervisor/xenserver/xenserver60/patch @@ -75,3 +75,4 @@ ovs-pvlan-vm.sh=../../../network,0755,/opt/cloud/bin ovs-pvlan-cleanup.sh=../../../network,0755,/opt/cloud/bin ovs-get-dhcp-iface.sh=..,0755,/opt/cloud/bin ovs-get-bridge.sh=..,0755,/opt/cloud/bin +cloudlog=..,0644,/etc/logrotate.d diff --git a/server/src/com/cloud/capacity/CapacityManagerImpl.java b/server/src/com/cloud/capacity/CapacityManagerImpl.java index 2358f92ade4..e42879cbbd2 100755 --- a/server/src/com/cloud/capacity/CapacityManagerImpl.java +++ b/server/src/com/cloud/capacity/CapacityManagerImpl.java @@ -27,7 +27,10 @@ import javax.ejb.Local; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.deploy.DeploymentClusterPlanner; +import com.cloud.deploy.DeploymentPlanner; import com.cloud.event.UsageEventVO; +import com.cloud.utils.exception.CloudRuntimeException; import org.apache.log4j.Logger; import org.apache.cloudstack.framework.config.ConfigDepot; @@ -94,6 +97,7 @@ import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.UserVmDetailsDao; import com.cloud.vm.dao.VMInstanceDao; import com.cloud.vm.snapshot.dao.VMSnapshotDao; +import org.springframework.instrument.classloading.glassfish.GlassFishLoadTimeWeaver; @Local(value = CapacityManager.class) public class CapacityManagerImpl extends ManagerBase implements CapacityManager, StateListener, Listener, ResourceListener, @@ -852,6 +856,50 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager, } + @Override + public float getClusterOverProvisioningFactor(Long clusterId, short capacityType){ + + String capacityOverProvisioningName = ""; + if(capacityType == Capacity.CAPACITY_TYPE_CPU){ + capacityOverProvisioningName = "cpuOvercommitRatio"; + }else if(capacityType == Capacity.CAPACITY_TYPE_MEMORY){ + capacityOverProvisioningName = "memoryOvercommitRatio"; + }else{ + throw new CloudRuntimeException("Invalid capacityType - " + capacityType); + } + + ClusterDetailsVO clusterDetailCpu = _clusterDetailsDao.findDetail(clusterId, capacityOverProvisioningName); + Float clusterOverProvisioningRatio = Float.parseFloat(clusterDetailCpu.getValue()); + return clusterOverProvisioningRatio; + + } + + @Override + public boolean checkIfClusterCrossesThreshold(Long clusterId, Integer cpuRequested, long ramRequested){ + + Float clusterCpuOverProvisioning = getClusterOverProvisioningFactor(clusterId, Capacity.CAPACITY_TYPE_CPU); + Float clusterMemoryOverProvisioning = getClusterOverProvisioningFactor(clusterId, Capacity.CAPACITY_TYPE_MEMORY); + Float clusterCpuCapacityDisableThreshold = DeploymentClusterPlanner.ClusterCPUCapacityDisableThreshold.valueIn(clusterId); + Float clusterMemoryCapacityDisableThreshold = DeploymentClusterPlanner.ClusterMemoryCapacityDisableThreshold.valueIn(clusterId); + + float cpuConsumption = _capacityDao.findClusterConsumption(clusterId, Capacity.CAPACITY_TYPE_CPU, cpuRequested); + if(cpuConsumption/clusterCpuOverProvisioning > clusterCpuCapacityDisableThreshold){ + s_logger.debug("Cluster: " +clusterId + " cpu consumption " + cpuConsumption/clusterCpuOverProvisioning + + " crosses disable threshold " + clusterCpuCapacityDisableThreshold); + return true; + } + + float memoryConsumption = _capacityDao.findClusterConsumption(clusterId, Capacity.CAPACITY_TYPE_MEMORY, ramRequested); + if(memoryConsumption/clusterMemoryOverProvisioning > clusterMemoryCapacityDisableThreshold){ + s_logger.debug("Cluster: " +clusterId + " memory consumption " + memoryConsumption/clusterMemoryOverProvisioning + + " crosses disable threshold " + clusterMemoryCapacityDisableThreshold); + return true; + } + + return false; + + } + @Override public boolean processAnswers(long agentId, long seq, Answer[] answers) { // TODO Auto-generated method stub diff --git a/server/src/com/cloud/configuration/Config.java b/server/src/com/cloud/configuration/Config.java index afc09d60167..f08ca2445ae 100755 --- a/server/src/com/cloud/configuration/Config.java +++ b/server/src/com/cloud/configuration/Config.java @@ -2009,6 +2009,8 @@ public enum Config { return "VpcManager"; } else if (_componentClass == SnapshotManager.class) { return "SnapshotManager"; + } else if (_componentClass == VMSnapshotManager.class) { + return "VMSnapshotManager"; } else { return "none"; } diff --git a/server/src/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/com/cloud/configuration/ConfigurationManagerImpl.java index a2c0204438c..c345795fe37 100755 --- a/server/src/com/cloud/configuration/ConfigurationManagerImpl.java +++ b/server/src/com/cloud/configuration/ConfigurationManagerImpl.java @@ -3718,9 +3718,9 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati Set firewallProviderSet = new HashSet(); firewallProviderSet.add(firewallProvider); serviceProviderMap.put(Service.Firewall, firewallProviderSet); - if (!(firewallProvider.getName().equals(Provider.JuniperSRX.getName()) || firewallProvider.getName().equals(Provider.VirtualRouter.getName())) && + if (!(firewallProvider.getName().equals(Provider.JuniperSRX.getName()) || firewallProvider.getName().equals(Provider.PaloAlto.getName()) || firewallProvider.getName().equals(Provider.VirtualRouter.getName())) && egressDefaultPolicy == false) { - throw new InvalidParameterValueException("Firewall egress with default policy " + egressDefaultPolicy + "is not supported by the provider " + + throw new InvalidParameterValueException("Firewall egress with default policy " + egressDefaultPolicy + " is not supported by the provider " + firewallProvider.getName()); } } diff --git a/server/src/com/cloud/deploy/DeploymentPlanningManagerImpl.java b/server/src/com/cloud/deploy/DeploymentPlanningManagerImpl.java index 8b7ed877877..69230e08050 100644 --- a/server/src/com/cloud/deploy/DeploymentPlanningManagerImpl.java +++ b/server/src/com/cloud/deploy/DeploymentPlanningManagerImpl.java @@ -19,8 +19,10 @@ package com.cloud.deploy; import java.util.ArrayList; import java.util.Comparator; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.Timer; import java.util.TreeSet; @@ -1119,6 +1121,12 @@ public class DeploymentPlanningManagerImpl extends ManagerBase implements Deploy // for each volume find list of suitable storage pools by calling the // allocators + Set originalAvoidPoolSet = avoid.getPoolsToAvoid(); + if (originalAvoidPoolSet == null) { + originalAvoidPoolSet = new HashSet(); + } + Set poolsToAvoidOutput = new HashSet(originalAvoidPoolSet); + for (VolumeVO toBeCreated : volumesTobeCreated) { s_logger.debug("Checking suitable pools for volume (Id, Type): (" + toBeCreated.getId() + "," + toBeCreated.getVolumeType().name() + ")"); @@ -1228,6 +1236,11 @@ public class DeploymentPlanningManagerImpl extends ManagerBase implements Deploy } } + if (avoid.getPoolsToAvoid() != null) { + poolsToAvoidOutput.addAll(avoid.getPoolsToAvoid()); + avoid.getPoolsToAvoid().retainAll(originalAvoidPoolSet); + } + if (!foundPotentialPools) { s_logger.debug("No suitable pools found for volume: " + toBeCreated + " under cluster: " + plan.getClusterId()); // No suitable storage pools found under this cluster for this @@ -1239,6 +1252,13 @@ public class DeploymentPlanningManagerImpl extends ManagerBase implements Deploy } } + if (suitableVolumeStoragePools.values() != null) { + poolsToAvoidOutput.removeAll(suitableVolumeStoragePools.values()); + } + if (avoid.getPoolsToAvoid() != null) { + avoid.getPoolsToAvoid().addAll(poolsToAvoidOutput); + } + if (suitableVolumeStoragePools.isEmpty()) { s_logger.debug("No suitable pools found"); } @@ -1346,4 +1366,4 @@ public class DeploymentPlanningManagerImpl extends ManagerBase implements Deploy } return true; } -} +} \ No newline at end of file diff --git a/server/src/com/cloud/network/ExternalLoadBalancerDeviceManager.java b/server/src/com/cloud/network/ExternalLoadBalancerDeviceManager.java index 9f13c2a8f9f..850d0d7cb03 100644 --- a/server/src/com/cloud/network/ExternalLoadBalancerDeviceManager.java +++ b/server/src/com/cloud/network/ExternalLoadBalancerDeviceManager.java @@ -47,7 +47,7 @@ public interface ExternalLoadBalancerDeviceManager extends Manager { * @return Host object for the device added */ public ExternalLoadBalancerDeviceVO addExternalLoadBalancer(long physicalNetworkId, String url, String username, String password, String deviceName, - ServerResource resource, boolean gslbProvider, String gslbSitePublicIp, String gslbSitePrivateIp); + ServerResource resource, boolean gslbProvider, boolean exclusiveGslbProvider, String gslbSitePublicIp, String gslbSitePrivateIp); /** * deletes load balancer device added in to a physical network diff --git a/server/src/com/cloud/network/ExternalLoadBalancerDeviceManagerImpl.java b/server/src/com/cloud/network/ExternalLoadBalancerDeviceManagerImpl.java index ea23b8a9de2..2a22ac27491 100644 --- a/server/src/com/cloud/network/ExternalLoadBalancerDeviceManagerImpl.java +++ b/server/src/com/cloud/network/ExternalLoadBalancerDeviceManagerImpl.java @@ -198,7 +198,8 @@ public abstract class ExternalLoadBalancerDeviceManagerImpl extends AdapterBase @Override @DB public ExternalLoadBalancerDeviceVO addExternalLoadBalancer(long physicalNetworkId, String url, String username, String password, final String deviceName, - ServerResource resource, final boolean gslbProvider, final String gslbSitePublicIp, final String gslbSitePrivateIp) { + ServerResource resource, final boolean gslbProvider, final boolean exclusiveGslbProivider, + final String gslbSitePublicIp, final String gslbSitePrivateIp) { PhysicalNetworkVO pNetwork = null; final NetworkDevice ntwkDevice = NetworkDevice.getNetworkDevice(deviceName); @@ -285,6 +286,7 @@ public abstract class ExternalLoadBalancerDeviceManagerImpl extends AdapterBase if (gslbProvider) { lbDeviceVO.setGslbSitePublicIP(gslbSitePublicIp); lbDeviceVO.setGslbSitePrivateIP(gslbSitePrivateIp); + lbDeviceVO.setExclusiveGslbProvider(exclusiveGslbProivider); } _externalLoadBalancerDeviceDao.persist(lbDeviceVO); DetailVO hostDetail = new DetailVO(host.getId(), ApiConstants.LOAD_BALANCER_DEVICE_ID, String.valueOf(lbDeviceVO.getId())); @@ -528,7 +530,7 @@ public abstract class ExternalLoadBalancerDeviceManagerImpl extends AdapterBase try { lbAppliance = addExternalLoadBalancer(physicalNetworkId, url, username, password, createLbAnswer.getDeviceName(), - createLbAnswer.getServerResource(), false, null, null); + createLbAnswer.getServerResource(), false, false, null, null); } catch (Exception e) { s_logger.error("Failed to add load balancer appliance in to cloudstack due to " + e.getMessage() + ". So provisioned load balancer appliance will be destroyed."); diff --git a/server/src/com/cloud/network/NetworkModelImpl.java b/server/src/com/cloud/network/NetworkModelImpl.java index 7568da07702..71fbbf5c149 100755 --- a/server/src/com/cloud/network/NetworkModelImpl.java +++ b/server/src/com/cloud/network/NetworkModelImpl.java @@ -2183,4 +2183,17 @@ public class NetworkModelImpl extends ManagerBase implements NetworkModel { return true; } + + @Override + public boolean getNetworkEgressDefaultPolicy (Long networkId) { + NetworkVO network = _networksDao.findById(networkId); + + if (network != null) { + NetworkOfferingVO offering = _networkOfferingDao.findById(network.getNetworkOfferingId()); + return offering.getEgressDefaultPolicy(); + } else { + InvalidParameterValueException ex = new InvalidParameterValueException("network with network id does not exist"); + throw ex; + } + } } diff --git a/server/src/com/cloud/network/as/AutoScaleManager.java b/server/src/com/cloud/network/as/AutoScaleManager.java index 5b84b30d744..9c841d451be 100644 --- a/server/src/com/cloud/network/as/AutoScaleManager.java +++ b/server/src/com/cloud/network/as/AutoScaleManager.java @@ -19,4 +19,8 @@ package com.cloud.network.as; public interface AutoScaleManager extends AutoScaleService { void cleanUpAutoScaleResources(Long accountId); + + void doScaleUp(long groupId, Integer numVm); + + void doScaleDown(long groupId); } diff --git a/server/src/com/cloud/network/as/AutoScaleManagerImpl.java b/server/src/com/cloud/network/as/AutoScaleManagerImpl.java index fb7c922dcf8..fd415b1fe90 100644 --- a/server/src/com/cloud/network/as/AutoScaleManagerImpl.java +++ b/server/src/com/cloud/network/as/AutoScaleManagerImpl.java @@ -17,22 +17,25 @@ package com.cloud.network.as; import java.security.InvalidParameterException; +import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import javax.ejb.Local; import javax.inject.Inject; -import org.apache.log4j.Logger; - -import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; - import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd.HTTPMethod; import org.apache.cloudstack.api.BaseListAccountResourcesCmd; +import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.command.admin.autoscale.CreateCounterCmd; import org.apache.cloudstack.api.command.user.autoscale.CreateAutoScalePolicyCmd; import org.apache.cloudstack.api.command.user.autoscale.CreateAutoScaleVmGroupCmd; @@ -49,42 +52,55 @@ import org.apache.cloudstack.api.command.user.autoscale.UpdateAutoScaleVmProfile import org.apache.cloudstack.api.command.user.vm.DeployVMCmd; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.log4j.Logger; import com.cloud.api.ApiDBUtils; import com.cloud.api.ApiDispatcher; import com.cloud.configuration.Config; import com.cloud.configuration.ConfigurationManager; import com.cloud.dc.DataCenter; +import com.cloud.dc.DataCenter.NetworkType; import com.cloud.dc.dao.DataCenterDao; import com.cloud.event.ActionEvent; import com.cloud.event.EventTypes; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.InsufficientServerCapacityException; import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceInUseException; import com.cloud.exception.ResourceUnavailableException; +import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.network.Network.Capability; +import com.cloud.network.Network.IpAddresses; import com.cloud.network.as.AutoScaleCounter.AutoScaleCounterParam; import com.cloud.network.as.dao.AutoScalePolicyConditionMapDao; import com.cloud.network.as.dao.AutoScalePolicyDao; import com.cloud.network.as.dao.AutoScaleVmGroupDao; import com.cloud.network.as.dao.AutoScaleVmGroupPolicyMapDao; +import com.cloud.network.as.dao.AutoScaleVmGroupVmMapDao; import com.cloud.network.as.dao.AutoScaleVmProfileDao; import com.cloud.network.as.dao.ConditionDao; import com.cloud.network.as.dao.CounterDao; import com.cloud.network.dao.IPAddressDao; import com.cloud.network.dao.LoadBalancerDao; import com.cloud.network.dao.LoadBalancerVMMapDao; +import com.cloud.network.dao.LoadBalancerVMMapVO; import com.cloud.network.dao.LoadBalancerVO; import com.cloud.network.dao.NetworkDao; import com.cloud.network.lb.LoadBalancingRulesManager; +import com.cloud.network.lb.LoadBalancingRulesService; import com.cloud.offering.ServiceOffering; import com.cloud.projects.Project.ListProjectResourcesCriteria; import com.cloud.template.TemplateManager; import com.cloud.template.VirtualMachineTemplate; import com.cloud.user.Account; import com.cloud.user.AccountManager; +import com.cloud.user.AccountService; import com.cloud.user.User; import com.cloud.user.dao.AccountDao; import com.cloud.user.dao.UserDao; +import com.cloud.uservm.UserVm; import com.cloud.utils.Pair; import com.cloud.utils.Ternary; import com.cloud.utils.component.ManagerBase; @@ -100,10 +116,15 @@ import com.cloud.utils.db.Transaction; import com.cloud.utils.db.TransactionCallback; import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.net.NetUtils; +import com.cloud.vm.UserVmManager; +import com.cloud.vm.UserVmService; +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; @Local(value = {AutoScaleService.class, AutoScaleManager.class}) public class AutoScaleManagerImpl extends ManagerBase implements AutoScaleManager, AutoScaleService { private static final Logger s_logger = Logger.getLogger(AutoScaleManagerImpl.class); + private ScheduledExecutorService _executor = Executors.newScheduledThreadPool(1); @Inject EntityManager _entityMgr; @@ -138,6 +159,8 @@ public class AutoScaleManagerImpl extends ManagerBase implements AutoScale @Inject AutoScaleVmGroupPolicyMapDao _autoScaleVmGroupPolicyMapDao; @Inject + AutoScaleVmGroupVmMapDao _autoScaleVmGroupVmMapDao; + @Inject DataCenterDao _dcDao = null; @Inject UserDao _userDao; @@ -145,6 +168,16 @@ public class AutoScaleManagerImpl extends ManagerBase implements AutoScale ConfigurationDao _configDao; @Inject IPAddressDao _ipAddressDao; + @Inject + AccountService _accountService; + @Inject + UserVmService _userVmService; + @Inject + UserVmManager _userVmManager; + @Inject + LoadBalancerVMMapDao _LbVmMapDao; + @Inject + LoadBalancingRulesService _LoadBalancingRulesService; public List getSupportedAutoScaleCounters(long networkid) { String capability = _lbRulesMgr.getLBCapability(networkid, Capability.AutoScaleCounters.getName()); @@ -501,7 +534,7 @@ public class AutoScaleManagerImpl extends ManagerBase implements AutoScale throw new InvalidParameterValueException("action is invalid, only 'scaleup' and 'scaledown' is supported"); } - AutoScalePolicyVO policyVO = new AutoScalePolicyVO(cmd.getDomainId(), cmd.getAccountId(), duration, quietTime, action); + AutoScalePolicyVO policyVO = new AutoScalePolicyVO(cmd.getDomainId(), cmd.getAccountId(), duration, quietTime, null, action); policyVO = checkValidityAndPersist(policyVO, cmd.getConditionIds()); s_logger.info("Successfully created AutoScale Policy with Id: " + policyVO.getId()); @@ -705,9 +738,8 @@ public class AutoScaleManagerImpl extends ManagerBase implements AutoScale "there are Vms already bound to the specified LoadBalancing Rule. User bound Vms and AutoScaled Vm Group cannot co-exist on a Load Balancing Rule"); } - AutoScaleVmGroupVO vmGroupVO = - new AutoScaleVmGroupVO(cmd.getLbRuleId(), zoneId, loadBalancer.getDomainId(), loadBalancer.getAccountId(), minMembers, maxMembers, - loadBalancer.getDefaultPortStart(), interval, cmd.getProfileId(), AutoScaleVmGroup.State_New); + AutoScaleVmGroupVO vmGroupVO = new AutoScaleVmGroupVO(cmd.getLbRuleId(), zoneId, loadBalancer.getDomainId(), loadBalancer.getAccountId(), minMembers, maxMembers, + loadBalancer.getDefaultPortStart(), interval, null, cmd.getProfileId(), AutoScaleVmGroup.State_New); vmGroupVO = checkValidityAndPersist(vmGroupVO, cmd.getScaleUpPolicyIds(), cmd.getScaleDownPolicyIds()); s_logger.info("Successfully created Autoscale Vm Group with Id: " + vmGroupVO.getId()); @@ -1169,4 +1201,266 @@ public class AutoScaleManagerImpl extends ManagerBase implements AutoScale s_logger.debug("Deleted " + count + " Conditions for account Id: " + accountId); } } + + private boolean checkConditionUp(AutoScaleVmGroupVO asGroup, Integer numVm) { + // check maximum + Integer currentVM = _autoScaleVmGroupVmMapDao.countByGroup(asGroup.getId()); + Integer maxVm = asGroup.getMaxMembers(); + if (currentVM + numVm > maxVm) { + s_logger.warn("number of VM will greater than the maximum in this group if scaling up, so do nothing more"); + return false; + } + return true; + } + + private boolean checkConditionDown(AutoScaleVmGroupVO asGroup) { + Integer currentVM = _autoScaleVmGroupVmMapDao.countByGroup(asGroup.getId()); + Integer minVm = asGroup.getMinMembers(); + if (currentVM - 1 < minVm) { + s_logger.warn("number of VM will less than the minimum in this group if scaling down, so do nothing more"); + return false; + } + return true; + } + + private long createNewVM(AutoScaleVmGroupVO asGroup) { + AutoScaleVmProfileVO profileVo = _autoScaleVmProfileDao.findById(asGroup.getProfileId()); + long templateId = profileVo.getTemplateId(); + long serviceOfferingId = profileVo.getServiceOfferingId(); + if (templateId == -1) { + return -1; + } + // create new VM into DB + try { + //Verify that all objects exist before passing them to the service + Account owner = _accountService.getActiveAccountById(profileVo.getAccountId()); + + DataCenter zone = _entityMgr.findById(DataCenter.class, profileVo.getZoneId()); + if (zone == null) { + throw new InvalidParameterValueException("Unable to find zone by id=" + profileVo.getZoneId()); + } + + ServiceOffering serviceOffering = _entityMgr.findById(ServiceOffering.class, serviceOfferingId); + if (serviceOffering == null) { + throw new InvalidParameterValueException("Unable to find service offering: " + serviceOfferingId); + } + + VirtualMachineTemplate template = _entityMgr.findById(VirtualMachineTemplate.class, templateId); + // Make sure a valid template ID was specified + if (template == null) { + throw new InvalidParameterValueException("Unable to use template " + templateId); + } + + if (!zone.isLocalStorageEnabled()) { + if (serviceOffering.getUseLocalStorage()) { + throw new InvalidParameterValueException("Zone is not configured to use local storage but service offering " + serviceOffering.getName() + " uses it"); + } + } + + UserVm vm = null; + IpAddresses addrs = new IpAddresses(null, null); + if (zone.getNetworkType() == NetworkType.Basic) { + vm = _userVmService.createBasicSecurityGroupVirtualMachine(zone, serviceOffering, template, null, owner, "autoScaleVm-" + asGroup.getId() + "-" + getCurrentTimeStampString(), + "autoScaleVm-" + asGroup.getId() + "-" + getCurrentTimeStampString(), null, null, null, HypervisorType.XenServer, HTTPMethod.GET, null, null, null, null, true, null, null, null, null); + } else { + if (zone.isSecurityGroupEnabled()) { + vm = _userVmService.createAdvancedSecurityGroupVirtualMachine(zone, serviceOffering, template, null, null, + owner, "autoScaleVm-" + asGroup.getId() + "-" + getCurrentTimeStampString(), "autoScaleVm-" + asGroup.getId() + "-" + getCurrentTimeStampString(), null, null, null, HypervisorType.XenServer, HTTPMethod.GET, null, null, null, null, true, null, null, null, null); + + } else { + vm = _userVmService.createAdvancedVirtualMachine(zone, serviceOffering, template, null, owner, "autoScaleVm-" + asGroup.getId() + "-" + getCurrentTimeStampString(), "autoScaleVm-" + asGroup.getId() + "-" + getCurrentTimeStampString(), + null, null, null, HypervisorType.XenServer, HTTPMethod.GET, null, null, null, addrs, true, null, null, null, null); + + } + } + + if (vm != null) { + return vm.getId(); + } else { + return -1; + } + } catch (InsufficientCapacityException ex) { + s_logger.info(ex); + s_logger.trace(ex.getMessage(), ex); + throw new ServerApiException(ApiErrorCode.INSUFFICIENT_CAPACITY_ERROR, ex.getMessage()); + } catch (ResourceUnavailableException ex) { + s_logger.warn("Exception: ", ex); + throw new ServerApiException(ApiErrorCode.RESOURCE_UNAVAILABLE_ERROR, ex.getMessage()); + } catch (ConcurrentOperationException ex) { + s_logger.warn("Exception: ", ex); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage()); + } catch (ResourceAllocationException ex) { + s_logger.warn("Exception: ", ex); + throw new ServerApiException(ApiErrorCode.RESOURCE_ALLOCATION_ERROR, ex.getMessage()); + } + } + + private String getCurrentTimeStampString(){ + Date current = new Date(); + SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmss"); + + return sdf.format(current); + } + + private boolean startNewVM(long vmId) { + try { + CallContext.current().setEventDetails("Vm Id: "+vmId); + _userVmManager.startVirtualMachine(vmId, null, null); + } catch (ResourceUnavailableException ex) { + s_logger.warn("Exception: ", ex); + throw new ServerApiException(ApiErrorCode.RESOURCE_UNAVAILABLE_ERROR, ex.getMessage()); + } catch (ConcurrentOperationException ex) { + s_logger.warn("Exception: ", ex); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage()); + } catch (InsufficientCapacityException ex) { + StringBuilder message = new StringBuilder(ex.getMessage()); + if (ex instanceof InsufficientServerCapacityException) { + if(((InsufficientServerCapacityException)ex).isAffinityApplied()){ + message.append(", Please check the affinity groups provided, there may not be sufficient capacity to follow them"); + } + } + s_logger.info(ex); + s_logger.info(message.toString(), ex); + throw new ServerApiException(ApiErrorCode.INSUFFICIENT_CAPACITY_ERROR, message.toString()); + } + return true; + } + + private boolean assignLBruleToNewVm(long vmId, AutoScaleVmGroupVO asGroup) { + List lstVmId = new ArrayList(); + long lbId = asGroup.getLoadBalancerId(); + + List LbVmMapVos = _LbVmMapDao.listByLoadBalancerId(lbId); + if ((LbVmMapVos != null) && (LbVmMapVos.size() > 0)) { + for (LoadBalancerVMMapVO LbVmMapVo : LbVmMapVos) { + long instanceId = LbVmMapVo.getInstanceId(); + if (instanceId == vmId) { + s_logger.warn("the new VM is already mapped to LB rule. What's wrong?"); + return true; + } + } + } + lstVmId.add(new Long(vmId)); + return _LoadBalancingRulesService.assignToLoadBalancer(lbId, lstVmId); + + } + + private long removeLBrule(AutoScaleVmGroupVO asGroup) { + long lbId = asGroup.getLoadBalancerId(); + long instanceId = -1; + List LbVmMapVos = _LbVmMapDao.listByLoadBalancerId(lbId); + if ((LbVmMapVos != null) && (LbVmMapVos.size() > 0)) { + for (LoadBalancerVMMapVO LbVmMapVo : LbVmMapVos) { + instanceId = LbVmMapVo.getInstanceId(); + } + } + // take last VM out of the list + List lstVmId = new ArrayList(); + if (instanceId != -1) + lstVmId.add(instanceId); + if (_LoadBalancingRulesService.removeFromLoadBalancer(lbId, lstVmId)) + return instanceId; + else + return -1; + } + + @Override + public void doScaleUp(long groupId, Integer numVm) { + AutoScaleVmGroupVO asGroup = _autoScaleVmGroupDao.findById(groupId); + if (asGroup == null) { + s_logger.error("Can not find the groupid " + groupId + " for scaling up"); + return; + } + if (!checkConditionUp(asGroup, numVm)) { + return; + } + for (int i = 0; i < numVm; i++) { + long vmId = createNewVM(asGroup); + if (vmId == -1) { + s_logger.error("Can not deploy new VM for scaling up in the group " + + asGroup.getId() + ". Waiting for next round"); + break; + } + if (startNewVM(vmId)) { + if (assignLBruleToNewVm(vmId, asGroup)) { + // persist to DB + AutoScaleVmGroupVmMapVO GroupVmVO = new AutoScaleVmGroupVmMapVO( + asGroup.getId(), vmId); + _autoScaleVmGroupVmMapDao.persist(GroupVmVO); + // update last_quiettime + List GroupPolicyVOs = _autoScaleVmGroupPolicyMapDao + .listByVmGroupId(groupId); + for (AutoScaleVmGroupPolicyMapVO GroupPolicyVO : GroupPolicyVOs) { + AutoScalePolicyVO vo = _autoScalePolicyDao + .findById(GroupPolicyVO.getPolicyId()); + if (vo.getAction().equals("scaleup")) { + vo.setLastQuiteTime(new Date()); + _autoScalePolicyDao.persist(vo); + break; + } + } + } else { + s_logger.error("Can not assign LB rule for this new VM"); + break; + } + } else { + s_logger.error("Can not deploy new VM for scaling up in the group " + + asGroup.getId() + ". Waiting for next round"); + break; + } + } + } + + @Override + public void doScaleDown(final long groupId) { + AutoScaleVmGroupVO asGroup = _autoScaleVmGroupDao.findById(groupId); + if (asGroup == null) { + s_logger.error("Can not find the groupid " + groupId + " for scaling up"); + return; + } + if (!checkConditionDown(asGroup)) { + return; + } + final long vmId = removeLBrule(asGroup); + if (vmId != -1) { + long profileId = asGroup.getProfileId(); + + // update group-vm mapping + _autoScaleVmGroupVmMapDao.remove(groupId, vmId); + // update last_quiettime + List GroupPolicyVOs = _autoScaleVmGroupPolicyMapDao.listByVmGroupId(groupId); + for (AutoScaleVmGroupPolicyMapVO GroupPolicyVO : GroupPolicyVOs) { + AutoScalePolicyVO vo = _autoScalePolicyDao.findById(GroupPolicyVO.getPolicyId()); + if (vo.getAction().equals("scaledown")) { + vo.setLastQuiteTime(new Date()); + _autoScalePolicyDao.persist(vo); + break; + } + } + + // get destroyvmgrace param + AutoScaleVmProfileVO asProfile = _autoScaleVmProfileDao.findById(profileId); + Integer destroyVmGracePeriod = asProfile.getDestroyVmGraceperiod(); + if (destroyVmGracePeriod >= 0) { + _executor.schedule(new Runnable() { + @Override + public void run() { + try { + + _userVmManager.destroyVm(vmId); + + } catch (ResourceUnavailableException e) { + e.printStackTrace(); + } catch (ConcurrentOperationException e) { + e.printStackTrace(); + } + } + }, destroyVmGracePeriod, TimeUnit.SECONDS); + } + } else { + s_logger.error("Can not remove LB rule for the VM being destroyed. Do nothing more."); + } + } + + } diff --git a/server/src/com/cloud/network/element/VirtualRouterElement.java b/server/src/com/cloud/network/element/VirtualRouterElement.java index 37d14f2a776..7833d241d6d 100755 --- a/server/src/com/cloud/network/element/VirtualRouterElement.java +++ b/server/src/com/cloud/network/element/VirtualRouterElement.java @@ -30,11 +30,13 @@ import org.apache.log4j.Logger; import com.google.gson.Gson; import org.apache.cloudstack.api.command.admin.router.ConfigureOvsElementCmd; + import org.apache.cloudstack.api.command.admin.router.ConfigureVirtualRouterElementCmd; import org.apache.cloudstack.api.command.admin.router.CreateVirtualRouterElementCmd; import org.apache.cloudstack.api.command.admin.router.ListOvsElementsCmd; import org.apache.cloudstack.api.command.admin.router.ListVirtualRouterElementsCmd; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.log4j.Logger; import com.cloud.agent.api.to.LoadBalancerTO; import com.cloud.configuration.ConfigurationManager; @@ -62,6 +64,8 @@ import com.cloud.network.RemoteAccessVpn; import com.cloud.network.VirtualRouterProvider; import com.cloud.network.VirtualRouterProvider.Type; import com.cloud.network.VpnUser; +import com.cloud.network.as.AutoScaleCounter; +import com.cloud.network.as.AutoScaleCounter.AutoScaleCounterType; import com.cloud.network.dao.IPAddressDao; import com.cloud.network.dao.LoadBalancerDao; import com.cloud.network.dao.NetworkDao; @@ -99,6 +103,7 @@ import com.cloud.vm.VirtualMachine.State; import com.cloud.vm.VirtualMachineProfile; import com.cloud.vm.dao.DomainRouterDao; import com.cloud.vm.dao.UserVmDao; +import com.google.gson.Gson; import com.google.gson.Gson; @@ -112,8 +117,9 @@ public class VirtualRouterElement extends AdapterBase implements VirtualRouterEl LoadBalancingServiceProvider, PortForwardingServiceProvider, RemoteAccessVPNServiceProvider, IpDeployer, NetworkMigrationResponder { private static final Logger s_logger = Logger.getLogger(VirtualRouterElement.class); - - protected static final Map> capabilities = setCapabilities(); + public static final AutoScaleCounterType AutoScaleCounterCpu = new AutoScaleCounterType("cpu"); + public static final AutoScaleCounterType AutoScaleCounterMemory = new AutoScaleCounterType("memory"); + protected static final Map> capabilities = setCapabilities(); @Inject NetworkDao _networksDao; @@ -240,6 +246,13 @@ public class VirtualRouterElement extends AdapterBase implements VirtualRouterEl return true; } + if (rules != null && rules.size() == 1 ) { + // for VR no need to add default egress rule to DENY traffic + if (rules.get(0).getTrafficType() == FirewallRule.TrafficType.Egress && rules.get(0).getType() == FirewallRule.FirewallRuleType.System && + ! _networkMgr.getNetworkEgressDefaultPolicy(config.getId())) + return true; + } + if (!_routerMgr.applyFirewallRules(config, rules, routers)) { throw new CloudRuntimeException("Failed to apply firewall rules in network " + config.getId()); } else { @@ -555,7 +568,17 @@ public class VirtualRouterElement extends AdapterBase implements VirtualRouterEl lbCapabilities.put(Capability.SupportedProtocols, "tcp, udp"); lbCapabilities.put(Capability.SupportedStickinessMethods, getHAProxyStickinessCapability()); lbCapabilities.put(Capability.LbSchemes, LoadBalancerContainer.Scheme.Public.toString()); - + + //specifies that LB rules can support autoscaling and the list of counters it supports + AutoScaleCounter counter; + List counterList = new ArrayList(); + counter = new AutoScaleCounter(AutoScaleCounterCpu); + counterList.add(counter); + counter = new AutoScaleCounter(AutoScaleCounterMemory); + counterList.add(counter); + Gson gson = new Gson(); + String autoScaleCounterList = gson.toJson(counterList); + lbCapabilities.put(Capability.AutoScaleCounters, autoScaleCounterList); capabilities.put(Service.Lb, lbCapabilities); // Set capabilities for Firewall service diff --git a/server/src/com/cloud/network/firewall/FirewallManagerImpl.java b/server/src/com/cloud/network/firewall/FirewallManagerImpl.java index 848ef1f70e3..448abe317b6 100644 --- a/server/src/com/cloud/network/firewall/FirewallManagerImpl.java +++ b/server/src/com/cloud/network/firewall/FirewallManagerImpl.java @@ -612,7 +612,6 @@ public class FirewallManagerImpl extends ManagerBase implements FirewallService, @Override public boolean applyEgressFirewallRules(FirewallRule rule, Account caller) throws ResourceUnavailableException { List rules = _firewallDao.listByNetworkPurposeTrafficType(rule.getNetworkId(), Purpose.Firewall, FirewallRule.TrafficType.Egress); - applyDefaultEgressFirewallRule(rule.getNetworkId(), true); return applyFirewallRules(rules, false, caller); } @@ -646,12 +645,8 @@ public class FirewallManagerImpl extends ManagerBase implements FirewallService, } @Override - public boolean applyDefaultEgressFirewallRule(Long networkId, boolean defaultPolicy) throws ResourceUnavailableException { + public boolean applyDefaultEgressFirewallRule(Long networkId, boolean defaultPolicy, boolean add) throws ResourceUnavailableException { - if (defaultPolicy == false) { - //If default policy is false no need apply rules on backend because firewall provider blocks by default - return true; - } s_logger.debug("applying default firewall egress rules "); NetworkVO network = _networkDao.findById(networkId); @@ -661,6 +656,7 @@ public class FirewallManagerImpl extends ManagerBase implements FirewallService, FirewallRuleVO ruleVO = new FirewallRuleVO(null, null, null, null, "all", networkId, network.getAccountId(), network.getDomainId(), Purpose.Firewall, sourceCidr, null, null, null, FirewallRule.TrafficType.Egress, FirewallRuleType.System); + ruleVO.setState(add ? State.Add : State.Revoke); List rules = new ArrayList(); rules.add(ruleVO); diff --git a/server/src/com/cloud/network/lb/LoadBalancingRulesManagerImpl.java b/server/src/com/cloud/network/lb/LoadBalancingRulesManagerImpl.java index 501bb14c193..aab4c6ba273 100755 --- a/server/src/com/cloud/network/lb/LoadBalancingRulesManagerImpl.java +++ b/server/src/com/cloud/network/lb/LoadBalancingRulesManagerImpl.java @@ -915,7 +915,8 @@ public class LoadBalancingRulesManagerImpl extends ManagerBase implements if (provider == null || provider.size() == 0) { return false; } - if (provider.get(0) == Provider.Netscaler || provider.get(0) == Provider.F5BigIp) { + if (provider.get(0) == Provider.Netscaler || provider.get(0) == Provider.F5BigIp || + provider.get(0) == Provider.VirtualRouter) { return true; } return false; diff --git a/server/src/com/cloud/server/StatsCollector.java b/server/src/com/cloud/server/StatsCollector.java index 53001058a91..82f774fcbe6 100755 --- a/server/src/com/cloud/server/StatsCollector.java +++ b/server/src/com/cloud/server/StatsCollector.java @@ -48,6 +48,7 @@ import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; import com.cloud.agent.api.GetStorageStatsCommand; import com.cloud.agent.api.HostStatsEntry; +import com.cloud.agent.api.PerformanceMonitorCommand; import com.cloud.agent.api.VmDiskStatsEntry; import com.cloud.agent.api.VmStatsEntry; import com.cloud.cluster.ManagementServerHostVO; @@ -59,8 +60,29 @@ import com.cloud.host.HostVO; import com.cloud.host.Status; import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.network.as.AutoScaleManager; +import com.cloud.network.as.AutoScalePolicyConditionMapVO; +import com.cloud.network.as.AutoScalePolicyVO; +import com.cloud.network.as.AutoScaleVmGroupPolicyMapVO; +import com.cloud.network.as.AutoScaleVmGroupVO; +import com.cloud.network.as.AutoScaleVmGroupVmMapVO; +import com.cloud.network.as.AutoScaleVmProfileVO; +import com.cloud.network.as.Condition.Operator; +import com.cloud.network.as.ConditionVO; +import com.cloud.network.as.Counter; +import com.cloud.network.as.CounterVO; +import com.cloud.network.as.dao.AutoScalePolicyConditionMapDao; +import com.cloud.network.as.dao.AutoScalePolicyDao; +import com.cloud.network.as.dao.AutoScaleVmGroupDao; +import com.cloud.network.as.dao.AutoScaleVmGroupPolicyMapDao; +import com.cloud.network.as.dao.AutoScaleVmGroupVmMapDao; +import com.cloud.network.as.dao.AutoScaleVmProfileDao; +import com.cloud.network.as.dao.ConditionDao; +import com.cloud.network.as.dao.CounterDao; import com.cloud.resource.ResourceManager; import com.cloud.resource.ResourceState; +import com.cloud.service.ServiceOfferingVO; +import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.storage.StorageManager; import com.cloud.storage.StorageStats; import com.cloud.storage.VolumeStats; @@ -70,6 +92,7 @@ import com.cloud.storage.dao.VolumeDao; import com.cloud.user.VmDiskStatisticsVO; import com.cloud.user.dao.VmDiskStatisticsDao; import com.cloud.utils.NumbersUtil; +import com.cloud.utils.Pair; import com.cloud.utils.component.ComponentMethodInterceptable; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.concurrency.NamedThreadFactory; @@ -82,9 +105,11 @@ import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.net.MacAddress; import com.cloud.vm.UserVmManager; import com.cloud.vm.UserVmVO; +import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VmStats; import com.cloud.vm.dao.UserVmDao; +import com.cloud.vm.dao.VMInstanceDao; /** * Provides real time stats for various agent resources up to x seconds @@ -92,55 +117,53 @@ import com.cloud.vm.dao.UserVmDao; */ @Component public class StatsCollector extends ManagerBase implements ComponentMethodInterceptable { - public static final Logger s_logger = Logger.getLogger(StatsCollector.class.getName()); - private static StatsCollector s_instance = null; + public static final Logger s_logger = Logger.getLogger(StatsCollector.class.getName()); - private ScheduledExecutorService _executor = null; - @Inject - private AgentManager _agentMgr; - @Inject - private UserVmManager _userVmMgr; - @Inject - private HostDao _hostDao; - @Inject - private UserVmDao _userVmDao; - @Inject - private VolumeDao _volsDao; - @Inject - private PrimaryDataStoreDao _storagePoolDao; - @Inject - private ImageStoreDao _imageStoreDao; - @Inject - private StorageManager _storageManager; - @Inject - private StoragePoolHostDao _storagePoolHostDao; - @Inject - private DataStoreManager _dataStoreMgr; - @Inject - private ResourceManager _resourceMgr; - @Inject - private ConfigurationDao _configDao; - @Inject - private EndPointSelector _epSelector; - @Inject - private VmDiskStatisticsDao _vmDiskStatsDao; - @Inject - private ManagementServerHostDao _msHostDao; + private static StatsCollector s_instance = null; - private ConcurrentHashMap _hostStats = new ConcurrentHashMap(); - private final ConcurrentHashMap _VmStats = new ConcurrentHashMap(); - private ConcurrentHashMap _volumeStats = new ConcurrentHashMap(); - private ConcurrentHashMap _storageStats = new ConcurrentHashMap(); - private ConcurrentHashMap _storagePoolStats = new ConcurrentHashMap(); + private ScheduledExecutorService _executor = null; + @Inject private AgentManager _agentMgr; + @Inject private UserVmManager _userVmMgr; + @Inject private HostDao _hostDao; + @Inject private UserVmDao _userVmDao; + @Inject private VolumeDao _volsDao; + @Inject private PrimaryDataStoreDao _storagePoolDao; + @Inject private ImageStoreDao _imageStoreDao; + @Inject private StorageManager _storageManager; + @Inject private StoragePoolHostDao _storagePoolHostDao; + @Inject private DataStoreManager _dataStoreMgr; + @Inject private ResourceManager _resourceMgr; + @Inject private ConfigurationDao _configDao; + @Inject private EndPointSelector _epSelector; + @Inject private VmDiskStatisticsDao _vmDiskStatsDao; + @Inject private ManagementServerHostDao _msHostDao; + @Inject private AutoScaleVmGroupDao _asGroupDao; + @Inject private AutoScaleVmGroupVmMapDao _asGroupVmDao; + @Inject private AutoScaleManager _asManager; + @Inject private VMInstanceDao _vmInstance; + @Inject private AutoScaleVmGroupPolicyMapDao _asGroupPolicyDao; + @Inject private AutoScalePolicyDao _asPolicyDao; + @Inject private AutoScalePolicyConditionMapDao _asConditionMapDao; + @Inject private ConditionDao _asConditionDao; + @Inject private CounterDao _asCounterDao; + @Inject private AutoScaleVmProfileDao _asProfileDao; + @Inject private ServiceOfferingDao _serviceOfferingDao; - long hostStatsInterval = -1L; - long hostAndVmStatsInterval = -1L; - long storageStatsInterval = -1L; - long volumeStatsInterval = -1L; - int vmDiskStatsInterval = 0; + private ConcurrentHashMap _hostStats = new ConcurrentHashMap(); + private final ConcurrentHashMap _VmStats = new ConcurrentHashMap(); + private ConcurrentHashMap _volumeStats = new ConcurrentHashMap(); + private ConcurrentHashMap _storageStats = new ConcurrentHashMap(); + private ConcurrentHashMap _storagePoolStats = new ConcurrentHashMap(); - private ScheduledExecutorService _diskStatsUpdateExecutor; + long hostStatsInterval = -1L; + long hostAndVmStatsInterval = -1L; + long storageStatsInterval = -1L; + long volumeStatsInterval = -1L; + long autoScaleStatsInterval = -1L; + int vmDiskStatsInterval = 0; + + private ScheduledExecutorService _diskStatsUpdateExecutor; private int _usageAggregationRange = 1440; private String _usageTimeZone = "GMT"; private final long mgmtSrvrId = MacAddress.getMacAddress().toLong(); @@ -169,14 +192,15 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc return true; } - private void init(Map configs) { - _executor = Executors.newScheduledThreadPool(3, new NamedThreadFactory("StatsCollector")); + private void init(Map configs) { + _executor = Executors.newScheduledThreadPool(4, new NamedThreadFactory("StatsCollector")); - hostStatsInterval = NumbersUtil.parseLong(configs.get("host.stats.interval"), 60000L); - hostAndVmStatsInterval = NumbersUtil.parseLong(configs.get("vm.stats.interval"), 60000L); - storageStatsInterval = NumbersUtil.parseLong(configs.get("storage.stats.interval"), 60000L); - volumeStatsInterval = NumbersUtil.parseLong(configs.get("volume.stats.interval"), -1L); - vmDiskStatsInterval = NumbersUtil.parseInt(configs.get("vm.disk.stats.interval"), 0); + hostStatsInterval = NumbersUtil.parseLong(configs.get("host.stats.interval"), 60000L); + hostAndVmStatsInterval = NumbersUtil.parseLong(configs.get("vm.stats.interval"), 60000L); + storageStatsInterval = NumbersUtil.parseLong(configs.get("storage.stats.interval"), 60000L); + volumeStatsInterval = NumbersUtil.parseLong(configs.get("volume.stats.interval"), -1L); + autoScaleStatsInterval = NumbersUtil.parseLong(configs.get("autoscale.stats.interval"), 60000L); + vmDiskStatsInterval = NumbersUtil.parseInt(configs.get("vm.disk.stats.interval"), 0); if (hostStatsInterval > 0) { _executor.scheduleWithFixedDelay(new HostCollector(), 15000L, hostStatsInterval, TimeUnit.MILLISECONDS); @@ -186,9 +210,13 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc _executor.scheduleWithFixedDelay(new VmStatsCollector(), 15000L, hostAndVmStatsInterval, TimeUnit.MILLISECONDS); } - if (storageStatsInterval > 0) { - _executor.scheduleWithFixedDelay(new StorageCollector(), 15000L, storageStatsInterval, TimeUnit.MILLISECONDS); - } + if (storageStatsInterval > 0) { + _executor.scheduleWithFixedDelay(new StorageCollector(), 15000L, storageStatsInterval, TimeUnit.MILLISECONDS); + } + + if (autoScaleStatsInterval > 0) { + _executor.scheduleWithFixedDelay(new AutoScaleMonitor(), 15000L, autoScaleStatsInterval, TimeUnit.MILLISECONDS); + } if (vmDiskStatsInterval > 0) { if (vmDiskStatsInterval < 300) @@ -578,21 +606,300 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc } } _storagePoolStats = storagePoolStats; - } catch (Throwable t) { - s_logger.error("Error trying to retrieve storage stats", t); - } - } - } + } catch (Throwable t) { + s_logger.error("Error trying to retrieve storage stats", t); + } + } + } - public StorageStats getStorageStats(long id) { - return _storageStats.get(id); - } + class AutoScaleMonitor extends ManagedContextRunnable { + @Override + protected void runInContext() { + try { + if (s_logger.isDebugEnabled()) { + s_logger.debug("AutoScaling Monitor is running..."); + } + // list all AS VMGroups + List asGroups = _asGroupDao.listAll(); + for (AutoScaleVmGroupVO asGroup : asGroups) { + // check group state + if ((asGroup.getState().equals("enabled")) && (is_native(asGroup.getId()))) { + // check minimum vm of group + Integer currentVM = _asGroupVmDao.countByGroup(asGroup.getId()); + if (currentVM < asGroup.getMinMembers()) { + _asManager.doScaleUp(asGroup.getId(), asGroup.getMinMembers() - currentVM); + continue; + } + + //check interval + long now = (new Date()).getTime(); + if (asGroup.getLastInterval() != null) + if ((now - asGroup.getLastInterval().getTime()) < (long) asGroup + .getInterval()) { + continue; + } - public HostStats getHostStats(long hostId) { - return _hostStats.get(hostId); - } + // update last_interval + asGroup.setLastInterval(new Date()); + _asGroupDao.persist(asGroup); - public StorageStats getStoragePoolStats(long id) { - return _storagePoolStats.get(id); - } + // collect RRDs data for this group + if (s_logger.isDebugEnabled()) { + s_logger.debug("[AutoScale] Collecting RRDs data..."); + } + Map params = new HashMap(); + List asGroupVmVOs = _asGroupVmDao.listByGroup(asGroup.getId()); + params.put("total_vm", String.valueOf(asGroupVmVOs.size())); + for (int i = 0; i < asGroupVmVOs.size(); i++) { + long vmId = asGroupVmVOs.get(i).getInstanceId(); + VMInstanceVO vmVO = _vmInstance.findById(vmId); + //xe vm-list | grep vmname -B 1 | head -n 1 | awk -F':' '{print $2}' + params.put("vmname" + String.valueOf(i + 1), vmVO.getInstanceName()); + params.put("vmid" + String.valueOf(i + 1), String.valueOf(vmVO.getId())); + + } + // get random hostid because all vms are in a cluster + long vmId = asGroupVmVOs.get(0).getInstanceId(); + VMInstanceVO vmVO = _vmInstance.findById(vmId); + Long receiveHost = vmVO.getHostId(); + + // setup parameters phase: duration and counter + // list pair [counter, duration] + List> lstPair = getPairofCounternameAndDuration(asGroup.getId()); + int total_counter = 0; + String[] lstCounter = new String[lstPair.size()]; + for (int i = 0; i < lstPair.size(); i++) { + Pair pair = lstPair.get(i); + String strCounterNames = pair.first(); + Integer duration = pair.second(); + + lstCounter[i] = strCounterNames.split(",")[0]; + total_counter++; + params.put("duration" + String.valueOf(total_counter), duration.toString()); + params.put("counter" + String.valueOf(total_counter), lstCounter[i]); + params.put("con" + String.valueOf(total_counter), strCounterNames.split(",")[1]); + } + params.put("total_counter", String.valueOf(total_counter)); + + PerformanceMonitorCommand perfMon = new PerformanceMonitorCommand(params, 20); + + try { + Answer answer = _agentMgr.send(receiveHost, perfMon); + if (answer == null || !answer.getResult()) { + s_logger.debug("Failed to send data to node !"); + } else { + String result = answer.getDetails(); + s_logger.debug("[AutoScale] RRDs collection answer: " + result); + HashMap avgCounter = new HashMap(); + + // extract data + String[] counterElements = result.split(","); + if ((counterElements != null) && (counterElements.length > 0)) { + for (String string : counterElements) { + try { + String[] counterVals = string.split(":"); + String[] counter_vm = counterVals[0].split("\\."); + + Long counterId = Long.parseLong(counter_vm[1]); + Long conditionId = Long.parseLong(params.get("con" + counter_vm[1])); + Double coVal = Double.parseDouble(counterVals[1]); + + // Summary of all counter by counterId key + if (avgCounter.get(counterId) == null){ + /* initialize if data is not set */ + avgCounter.put(counterId, new Double(0)); + } + + String counterName = getCounternamebyCondition(conditionId.longValue()); + if (counterName == Counter.Source.memory.toString()) { + // calculate memory in percent + Long profileId = asGroup.getProfileId(); + AutoScaleVmProfileVO profileVo = _asProfileDao.findById(profileId); + ServiceOfferingVO serviceOff = _serviceOfferingDao.findById(profileVo.getServiceOfferingId()); + int maxRAM = serviceOff.getRamSize(); + + // get current RAM percent + coVal = coVal / maxRAM; + } else { + // cpu + coVal = coVal * 100; + } + + // update data entry + avgCounter.put(counterId, avgCounter.get(counterId)+ coVal); + + } catch (Exception e) { + e.printStackTrace(); + } + } + + String scaleAction = getAutoscaleAction(avgCounter, asGroup.getId(), currentVM, params); + if (scaleAction != null) { + s_logger.debug("[AutoScale] Doing scale action: " + scaleAction + " for group " + asGroup.getId()); + if (scaleAction.equals("scaleup")) { + _asManager.doScaleUp(asGroup.getId(), 1); + } else { + _asManager.doScaleDown(asGroup.getId()); + } + } + } + } + + } catch (Exception e) { + e.printStackTrace(); + } + + } + } + + } catch (Throwable t) { + s_logger.error("Error trying to monitor autoscaling", t); + } + + } + + private boolean is_native(long groupId) { + List vos = _asGroupPolicyDao.listByVmGroupId(groupId); + for (AutoScaleVmGroupPolicyMapVO vo : vos) { + List ConditionPolicies = _asConditionMapDao.findByPolicyId(vo.getPolicyId()); + for (AutoScalePolicyConditionMapVO ConditionPolicy : ConditionPolicies) { + ConditionVO condition = _asConditionDao.findById(ConditionPolicy.getConditionId()); + CounterVO counter = _asCounterDao.findById(condition.getCounterid()); + if (counter.getSource() == Counter.Source.cpu || counter.getSource() == Counter.Source.memory) + return true; + } + } + return false; + } + + private String getAutoscaleAction(HashMap avgCounter, long groupId, long currentVM, Map params) { + + List listMap = _asGroupPolicyDao.listByVmGroupId(groupId); + if ( (listMap == null) || (listMap.size() == 0) ) + return null; + for (AutoScaleVmGroupPolicyMapVO asVmgPmap : listMap) { + AutoScalePolicyVO policyVO = _asPolicyDao.findById(asVmgPmap.getPolicyId()); + if (policyVO != null) { + Integer quitetime = policyVO.getQuietTime(); + Date quitetimeDate = policyVO.getLastQuiteTime(); + long last_quitetime = 0L; + if (quitetimeDate != null) { + last_quitetime = policyVO.getLastQuiteTime().getTime(); + } + long current_time = (new Date()).getTime(); + + // check quite time for this policy + if ((current_time - last_quitetime) >= (long) quitetime) { + + // list all condition of this policy + boolean bValid = true; + List lstConditions = getConditionsbyPolicyId(policyVO.getId()); + if ((lstConditions != null) && (lstConditions.size() > 0)) { + // check whole conditions of this policy + for (ConditionVO conditionVO : lstConditions) { + long thresholdValue = conditionVO.getThreshold(); + Double thresholdPercent = (double)thresholdValue / 100; + CounterVO counterVO = _asCounterDao.findById(conditionVO.getCounterid()); +// Double sum = avgCounter.get(conditionVO.getCounterid()); + long counter_count=1; + do { + String counter_param = params.get("counter" + String.valueOf(counter_count)); + Counter.Source counter_source = counterVO.getSource(); + if (counter_param.equals(counter_source.toString())) + break; + counter_count++; + } while (1==1); + + Double sum = avgCounter.get(counter_count); + Double avg = sum / currentVM; + Operator op = conditionVO.getRelationalOperator(); + boolean bConditionCheck = ((op == com.cloud.network.as.Condition.Operator.EQ) && (thresholdPercent == avg)) + || ((op == com.cloud.network.as.Condition.Operator.GE) && (avg >= thresholdPercent)) + || ((op == com.cloud.network.as.Condition.Operator.GT) && (avg > thresholdPercent)) + || ((op == com.cloud.network.as.Condition.Operator.LE) && (avg <= thresholdPercent)) + || ((op == com.cloud.network.as.Condition.Operator.LT) && (avg < thresholdPercent)); + + if (!bConditionCheck) { + bValid = false; + break; + } + } + if (bValid) { + return policyVO.getAction(); + } + } + } + } + } + return null; + } + + private List getConditionsbyPolicyId(long policyId){ + List conditionMap = _asConditionMapDao.findByPolicyId(policyId); + if ( (conditionMap == null) || (conditionMap.size() == 0)) + return null; + + List lstResult = new ArrayList(); + for (AutoScalePolicyConditionMapVO asPCmap : conditionMap) { + lstResult.add(_asConditionDao.findById(asPCmap.getConditionId())); + } + + return lstResult; + } + + public List> getPairofCounternameAndDuration( + long groupId) { + AutoScaleVmGroupVO groupVo = _asGroupDao.findById(groupId); + if (groupVo == null) + return null; + List> result = new ArrayList>(); + //list policy map + List groupPolicymap = _asGroupPolicyDao.listByVmGroupId(groupVo.getId()); + if (groupPolicymap == null) + return null; + for (AutoScaleVmGroupPolicyMapVO gpMap : groupPolicymap) { + //get duration + AutoScalePolicyVO policyVo = _asPolicyDao.findById(gpMap.getPolicyId()); + Integer duration = policyVo.getDuration(); + //get collection of counter name + String counterNames = ""; + List lstPCmap = _asConditionMapDao.findByPolicyId(policyVo.getId()); + for (AutoScalePolicyConditionMapVO pcMap : lstPCmap) { + String counterName = getCounternamebyCondition(pcMap.getConditionId()); + + counterNames += counterName + "," + pcMap.getConditionId(); + } + // add to result + Pair pair = new Pair(counterNames, duration); + result.add(pair); + } + + return result; + } + + public String getCounternamebyCondition(long conditionId){ + + ConditionVO condition = _asConditionDao.findById(conditionId); + if (condition == null) + return ""; + + long counterId = condition.getCounterid(); + CounterVO counter = _asCounterDao.findById(counterId); + if (counter == null) + return ""; + + return counter.getSource().toString(); + } + } + public StorageStats getStorageStats(long id) { + return _storageStats.get(id); + } + + public HostStats getHostStats(long hostId){ + return _hostStats.get(hostId); + } + + public StorageStats getStoragePoolStats(long id) { + return _storagePoolStats.get(id); + } } diff --git a/server/src/com/cloud/vm/UserVmManagerImpl.java b/server/src/com/cloud/vm/UserVmManagerImpl.java index 1752c22a1e3..fa45cc8ed90 100755 --- a/server/src/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/com/cloud/vm/UserVmManagerImpl.java @@ -36,6 +36,8 @@ import javax.naming.ConfigurationException; import com.cloud.event.UsageEventVO; import com.cloud.uuididentity.UUIDManager; +import com.cloud.capacity.Capacity; +import com.cloud.exception.InsufficientServerCapacityException; import org.apache.commons.codec.binary.Base64; import org.apache.log4j.Logger; @@ -84,6 +86,8 @@ import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.to.TemplateObjectTO; +import org.apache.commons.codec.binary.Base64; +import org.apache.log4j.Logger; import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; @@ -1306,6 +1310,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir int currentCpu = currentServiceOffering.getCpu(); int currentMemory = currentServiceOffering.getRamSize(); int currentSpeed = currentServiceOffering.getSpeed(); + int memoryDiff = newMemory - currentMemory; + int cpuDiff = newCpu*newSpeed - currentCpu*currentSpeed; // Don't allow to scale when (Any of the new values less than current values) OR (All current and new values are same) if ((newSpeed < currentSpeed || newMemory < currentMemory || newCpu < currentCpu) || @@ -1328,14 +1334,24 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir if (vmInstance.getState().equals(State.Running)) { int retry = _scaleRetry; ExcludeList excludes = new ExcludeList(); + + // Check zone wide flag boolean enableDynamicallyScaleVm = EnableDynamicallyScaleVm.valueIn(vmInstance.getDataCenterId()); if (!enableDynamicallyScaleVm) { throw new PermissionDeniedException("Dynamically scaling virtual machines is disabled for this zone, please contact your admin"); } + + // Check vm flag if (!vmInstance.isDynamicallyScalable()) { throw new CloudRuntimeException("Unable to Scale the vm: " + vmInstance.getUuid() + " as vm does not have tools to support dynamic scaling"); } + // Check disable threshold for cluster is not crossed + HostVO host = _hostDao.findById(vmInstance.getHostId()); + if(_capacityMgr.checkIfClusterCrossesThreshold(host.getClusterId(), cpuDiff, memoryDiff)){ + throw new CloudRuntimeException("Unable to scale vm: " + vmInstance.getUuid() + " due to insufficient resources"); + } + while (retry-- != 0) { // It's != so that it can match -1. try { boolean existingHostHasCapacity = false; @@ -1344,15 +1360,17 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir if (newCpu > currentCpu) { _resourceLimitMgr.incrementResourceCount(caller.getAccountId(), ResourceType.cpu, new Long(newCpu - currentCpu)); } - if (newMemory > currentMemory) { - _resourceLimitMgr.incrementResourceCount(caller.getAccountId(), ResourceType.memory, new Long(newMemory - currentMemory)); + + if (memoryDiff > 0) { + _resourceLimitMgr.incrementResourceCount(caller.getAccountId(), ResourceType.memory, new Long (memoryDiff)); } // #1 Check existing host has capacity if( !excludes.shouldAvoid(ApiDBUtils.findHostById(vmInstance.getHostId())) ){ existingHostHasCapacity = _capacityMgr.checkIfHostHasCpuCapability(vmInstance.getHostId(), newCpu, newSpeed) - && _capacityMgr.checkIfHostHasCapacity(vmInstance.getHostId(), newServiceOffering.getSpeed() - currentServiceOffering.getSpeed(), - (newServiceOffering.getRamSize() - currentServiceOffering.getRamSize()) * 1024L * 1024L, false, ApiDBUtils.getCpuOverprovisioningFactor(), 1f, false); // TO DO fill it with mem. + && _capacityMgr.checkIfHostHasCapacity(vmInstance.getHostId(), cpuDiff, + (memoryDiff) * 1024L * 1024L, false, _capacityMgr.getClusterOverProvisioningFactor(host.getClusterId(), Capacity.CAPACITY_TYPE_CPU), + _capacityMgr.getClusterOverProvisioningFactor(host.getClusterId(), Capacity.CAPACITY_TYPE_MEMORY), false); excludes.addHost(vmInstance.getHostId()); } @@ -1392,8 +1410,9 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir if (newCpu > currentCpu) { _resourceLimitMgr.decrementResourceCount(caller.getAccountId(), ResourceType.cpu, new Long(newCpu - currentCpu)); } - if (newMemory > currentMemory) { - _resourceLimitMgr.decrementResourceCount(caller.getAccountId(), ResourceType.memory, new Long(newMemory - currentMemory)); + + if (memoryDiff > 0) { + _resourceLimitMgr.decrementResourceCount(caller.getAccountId(), ResourceType.memory, new Long (memoryDiff)); } } } @@ -3345,9 +3364,12 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir } @Override - public UserVm destroyVm(long vmId) throws ResourceUnavailableException, ConcurrentOperationException { - Account caller = CallContext.current().getCallingAccount(); - Long userId = CallContext.current().getCallingUserId(); + + public UserVm destroyVm(long vmId) throws ResourceUnavailableException, + ConcurrentOperationException { + // Account caller = CallContext.current().getCallingAccount(); + // Long userId = CallContext.current().getCallingUserId(); + Long userId = 2L; // Verify input parameters UserVmVO vm = _vmDao.findById(vmId); @@ -3361,8 +3383,9 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir return vm; } - _accountMgr.checkAccess(caller, null, true, vm); - _userDao.findById(userId); + + // _accountMgr.checkAccess(caller, null, true, vm); + User userCaller = _userDao.findById(userId); boolean status; State vmState = vm.getState(); diff --git a/server/test/com/cloud/network/MockFirewallManagerImpl.java b/server/test/com/cloud/network/MockFirewallManagerImpl.java index 5d74c851144..d3d11f1ab4f 100644 --- a/server/test/com/cloud/network/MockFirewallManagerImpl.java +++ b/server/test/com/cloud/network/MockFirewallManagerImpl.java @@ -149,7 +149,7 @@ public class MockFirewallManagerImpl extends ManagerBase implements FirewallMana } @Override - public boolean applyDefaultEgressFirewallRule(Long networkId, boolean defaultPolicy) throws ResourceUnavailableException { + public boolean applyDefaultEgressFirewallRule(Long networkId, boolean defaultPolicy, boolean add) throws ResourceUnavailableException { return false; //To change body of implemented methods use File | Settings | File Templates. } diff --git a/server/test/com/cloud/network/MockNetworkModelImpl.java b/server/test/com/cloud/network/MockNetworkModelImpl.java index 78f3d51370b..6ae6ae8beef 100644 --- a/server/test/com/cloud/network/MockNetworkModelImpl.java +++ b/server/test/com/cloud/network/MockNetworkModelImpl.java @@ -864,4 +864,9 @@ public class MockNetworkModelImpl extends ManagerBase implements NetworkModel { public boolean isNetworkReadyForGc(long networkId) { return true; } + + @Override + public boolean getNetworkEgressDefaultPolicy(Long networkId) { + return false; //To change body of implemented methods use File | Settings | File Templates. + } } diff --git a/server/test/com/cloud/vpc/MockNetworkModelImpl.java b/server/test/com/cloud/vpc/MockNetworkModelImpl.java index 05093c7a14a..e0583d8d317 100644 --- a/server/test/com/cloud/vpc/MockNetworkModelImpl.java +++ b/server/test/com/cloud/vpc/MockNetworkModelImpl.java @@ -878,4 +878,9 @@ public class MockNetworkModelImpl extends ManagerBase implements NetworkModel { public boolean isNetworkReadyForGc(long networkId) { return true; } + + @Override + public boolean getNetworkEgressDefaultPolicy(Long networkId) { + return false; //To change body of implemented methods use File | Settings | File Templates. + } } diff --git a/setup/db/create-schema.sql b/setup/db/create-schema.sql index 55cb4cce7b9..2a53a712823 100755 --- a/setup/db/create-schema.sql +++ b/setup/db/create-schema.sql @@ -152,6 +152,7 @@ DROP TABLE IF EXISTS `cloud`.`template_s3_ref`; DROP TABLE IF EXISTS `cloud`.`nicira_nvp_router_map`; DROP TABLE IF EXISTS `cloud`.`external_bigswitch_vns_devices`; DROP TABLE IF EXISTS `cloud`.`autoscale_vmgroup_policy_map`; +DROP TABLE IF EXISTS `cloud`.`autoscale_vmgroup_vm_map`; DROP TABLE IF EXISTS `cloud`.`autoscale_policy_condition_map`; DROP TABLE IF EXISTS `cloud`.`autoscale_vmgroups`; DROP TABLE IF EXISTS `cloud`.`autoscale_vmprofiles`; diff --git a/setup/db/db/schema-40to410.sql b/setup/db/db/schema-40to410.sql index 24f6d4c5e19..65fb350bf19 100644 --- a/setup/db/db/schema-40to410.sql +++ b/setup/db/db/schema-40to410.sql @@ -345,6 +345,7 @@ CREATE TABLE `cloud`.`autoscale_policies` ( `account_id` bigint unsigned NOT NULL, `duration` int unsigned NOT NULL, `quiet_time` int unsigned NOT NULL, + `last_quiet_time` datetime DEFAULT NULL, `action` varchar(15), `created` datetime NOT NULL COMMENT 'date created', `removed` datetime COMMENT 'date removed if not null', @@ -366,6 +367,7 @@ CREATE TABLE `cloud`.`autoscale_vmgroups` ( `max_members` int unsigned NOT NULL, `member_port` int unsigned NOT NULL, `interval` int unsigned NOT NULL, + `last_interval` datetime DEFAULT NULL, `profile_id` bigint unsigned NOT NULL, `state` varchar(255) NOT NULL COMMENT 'enabled or disabled, a vmgroup is disabled to stop autoscaling activity', `created` datetime NOT NULL COMMENT 'date created', @@ -401,10 +403,22 @@ CREATE TABLE `cloud`.`autoscale_vmgroup_policy_map` ( INDEX `i_autoscale_vmgroup_policy_map__vmgroup_id`(`vmgroup_id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; +CREATE TABLE `cloud`.`autoscale_vmgroup_vm_map` ( + `id` bigint unsigned NOT NULL auto_increment, + `vmgroup_id` bigint unsigned NOT NULL, + `instance_id` bigint unsigned NOT NULL, + PRIMARY KEY (`id`), + CONSTRAINT `fk_autoscale_vmgroup_vm_map__vmgroup_id` FOREIGN KEY `fk_autoscale_vmgroup_vm_map__vmgroup_id` (`vmgroup_id`) REFERENCES `autoscale_vmgroups` (`id`) ON DELETE CASCADE, + CONSTRAINT `fk_autoscale_vmgroup_vm_map__instance_id` FOREIGN KEY `fk_autoscale_vmgroup_vm_map__instance_id` (`instance_id`) REFERENCES `vm_instance` (`id`), + INDEX `i_autoscale_vmgroup_vm_map__vmgroup_id`(`vmgroup_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + INSERT INTO `cloud`.`counter` (id, uuid, source, name, value,created) VALUES (1, UUID(), 'snmp','Linux User CPU - percentage', '1.3.6.1.4.1.2021.11.9.0', now()); INSERT INTO `cloud`.`counter` (id, uuid, source, name, value,created) VALUES (2, UUID(), 'snmp','Linux System CPU - percentage', '1.3.6.1.4.1.2021.11.10.0', now()); INSERT INTO `cloud`.`counter` (id, uuid, source, name, value,created) VALUES (3, UUID(), 'snmp','Linux CPU Idle - percentage', '1.3.6.1.4.1.2021.11.11.0', now()); INSERT INTO `cloud`.`counter` (id, uuid, source, name, value,created) VALUES (100, UUID(), 'netscaler','Response Time - microseconds', 'RESPTIME', now()); +INSERT INTO `cloud`.`counter` (id, uuid, source, name, value,created) VALUES (4, UUID(), 'cpu','Linux User CPU - percentage - native', '1.3.6.1.4.1.2021.11.9.1', now()); +INSERT INTO `cloud`.`counter` (id, uuid, source, name, value,created) VALUES (5, UUID(), 'memory','Linux User RAM - percentage - native', '1.3.6.1.4.1.2021.11.10.1', now()); CREATE TABLE `cloud`.`user_ipv6_address` ( `id` bigint unsigned NOT NULL UNIQUE auto_increment, diff --git a/setup/db/db/schema-421to430.sql b/setup/db/db/schema-421to430.sql index cf6f5963c5c..1ec4e94474f 100644 --- a/setup/db/db/schema-421to430.sql +++ b/setup/db/db/schema-421to430.sql @@ -804,3 +804,12 @@ CREATE TABLE `cloud`.`network_acl_item_details` ( ALTER TABLE `cloud`.`alert` ADD COLUMN `name` varchar(255) DEFAULT NULL COMMENT 'name of the alert'; UPDATE `cloud`.`hypervisor_capabilities` SET `max_data_volumes_limit`=13 WHERE `hypervisor_type`='Vmware'; + +ALTER TABLE `cloud`.`external_load_balancer_devices` ADD COLUMN `is_exclusive_gslb_provider` int(1) unsigned NOT NULL DEFAULT 0 COMMENT '1 if load balancer appliance is acting exclusively as gslb service provider in the zone and can not be used for LB'; + +DELETE FROM `cloud`.`configuration` WHERE `name` IN ("xen.update.url", "update.check.interval", "baremetal_dhcp_devices", "host.updates.enable"); + +INSERT IGNORE INTO `cloud`.`configuration` VALUES ("Advanced", 'DEFAULT', 'VMSnapshotManager', "vmsnapshot.create.wait", "1800", "In second, timeout for create vm snapshot", NULL, NULL,NULL,0); +INSERT IGNORE INTO `cloud`.`configuration` VALUES ("Advanced", 'DEFAULT', 'VMSnapshotManager', "vmsnapshot.max", "10", "Maximum vm snapshots for a vm", NULL, NULL,NULL,0); + +UPDATE `cloud`.`configuration` SET `component` = 'VMSnapshotManager' WHERE `name` IN ("vmsnapshot.create.wait", "vmsnapshot.max"); diff --git a/systemvm/patches/debian/vpn/opt/cloud/bin/vpn_l2tp.sh b/systemvm/patches/debian/vpn/opt/cloud/bin/vpn_l2tp.sh index 5afe0096d9a..83d5272f3e1 100755 --- a/systemvm/patches/debian/vpn/opt/cloud/bin/vpn_l2tp.sh +++ b/systemvm/patches/debian/vpn/opt/cloud/bin/vpn_l2tp.sh @@ -35,21 +35,41 @@ get_intf_ip() { iptables_() { local op=$1 local public_ip=$2 + local is_vpc=false + local forward_action="ACCEPT" + if grep "vpcrouter" /var/cache/cloud/cmdline &> /dev/null + then + is_vpc=true + fi sudo iptables $op INPUT -i $dev --dst $public_ip -p udp -m udp --dport 500 -j ACCEPT sudo iptables $op INPUT -i $dev --dst $public_ip -p udp -m udp --dport 4500 -j ACCEPT sudo iptables $op INPUT -i $dev --dst $public_ip -p udp -m udp --dport 1701 -j ACCEPT sudo iptables $op INPUT -i $dev -p ah -j ACCEPT sudo iptables $op INPUT -i $dev -p esp -j ACCEPT - sudo iptables $op FORWARD -i ppp+ -d $cidr -j ACCEPT - sudo iptables $op FORWARD -s $cidr -o ppp+ -j ACCEPT - sudo iptables $op FORWARD -i ppp+ -o ppp+ -j ACCEPT + if $is_vpc + then + # Need to apply the following ACL rules as well. + if sudo iptables -N VPN_FORWARD &> /dev/null + then + sudo iptables -I FORWARD -i ppp+ -j VPN_FORWARD + sudo iptables -I FORWARD -o ppp+ -j VPN_FORWARD + sudo iptables -A VPN_FORWARD -j DROP + fi + sudo iptables $op VPN_FORWARD -i ppp+ -o ppp+ -j RETURN + sudo iptables $op VPN_FORWARD -i ppp+ -d $cidr -j RETURN + sudo iptables $op VPN_FORWARD -s $cidr -o ppp+ -j RETURN + else + sudo iptables $op FORWARD -i ppp+ -d $cidr -j ACCEPT + sudo iptables $op FORWARD -s $cidr -o ppp+ -j ACCEPT + sudo iptables $op FORWARD -i ppp+ -o ppp+ -j ACCEPT + fi sudo iptables $op INPUT -i ppp+ -m udp -p udp --dport 53 -j ACCEPT sudo iptables $op INPUT -i ppp+ -m tcp -p tcp --dport 53 -j ACCEPT sudo iptables -t nat $op PREROUTING -i ppp+ -p tcp -m tcp --dport 53 -j DNAT --to-destination $local_ip sudo iptables -t nat $op PREROUTING -i ppp+ -p udp -m udp --dport 53 -j DNAT --to-destination $local_ip - if grep "vpcrouter" /var/cache/cloud/cmdline &> /dev/null + if $is_vpc then return fi diff --git a/test/integration/component/test_vpc_network.py b/test/integration/component/test_vpc_network.py index c918f54043d..ec36deb1fe5 100644 --- a/test/integration/component/test_vpc_network.py +++ b/test/integration/component/test_vpc_network.py @@ -2332,11 +2332,6 @@ class TestVPCNetworkGc(cloudstackTestCase): wait_for_cleanup(self.apiclient, ["network.gc.interval", "network.gc.wait"]) - #Bug???: Network Acls are not cleared - netacls = NetworkACL.list(self.apiclient, networkid=self.network_1.id) - self.debug("List of NetACLS %s" % netacls) - self.assertEqual(netacls, None, "Netacls were not cleared after network GC thread is run") - lbrules = LoadBalancerRule.list(self.apiclient, networkid=self.network_1.id) self.debug("List of LB Rules %s" % lbrules) self.assertEqual(lbrules, None, "LBrules were not cleared after network GC thread is run") diff --git a/tools/marvin/marvin/configGenerator.py b/tools/marvin/marvin/configGenerator.py index 3f98aca9ff1..6d5b70d3108 100644 --- a/tools/marvin/marvin/configGenerator.py +++ b/tools/marvin/marvin/configGenerator.py @@ -93,11 +93,6 @@ class trafficType(object): if 'vmware' in labeldict.keys() else None self.simulator = labeldict['simulator']\ if 'simulator' in labeldict.keys() else None - #{ - # 'xen' : 'cloud-xen', - # 'kvm' : 'cloud-kvm', - # 'vmware' : 'cloud-vmware' - #} class pod(object): diff --git a/tools/marvin/marvin/deployAndRun.py b/tools/marvin/marvin/deployAndRun.py index 8a758a10e36..56747a7cb2d 100644 --- a/tools/marvin/marvin/deployAndRun.py +++ b/tools/marvin/marvin/deployAndRun.py @@ -15,82 +15,153 @@ # specific language governing permissions and limitations # under the License. -import deployDataCenter -import TestCaseExecuteEngine +from tcExecuteEngine import TestCaseExecuteEngine import sys +import os +import traceback +import time from argparse import ArgumentParser -if __name__ == "__main__": +from marvinInit import MarvinInit +from marvin.codes import (SUCCESS, + FAILED, + EXCEPTION, + UNKNOWN_ERROR + ) + +parser = None + + +def printAndExit(): + ''' + Prints pretty message for parser and exit + ''' + global parser + if parser is not None: + parser.print_usage() + exit(1) + + +def parseAndCheck(): + ''' + Parses,reads the options and verifies for the config file + ''' + global parser parser = ArgumentParser() - parser.add_argument("-d", "--directory", dest="testCaseFolder", - help="the test case directory") - parser.add_argument("-f", "--file", dest="module", - help="run tests in the given file") - parser.add_argument("-r", "--result", dest="result", - help="test result log file", default='/tmp/t.log') - parser.add_argument("-t", "--client", dest="testcaselog", - help="test case log file", default='/tmp/r.log') + parser.add_argument("-d", "--tcpath", dest="tcpath", + help="the test case directory or file path") parser.add_argument("-c", "--config", action="store", default="./datacenterCfg", dest="config", help="the path where the json config file generated,\ - by default is ./datacenterCfg") + by default is ./datacenterCfg") parser.add_argument("-l", "--load", dest="load", action="store_true", help="only load config, do not deploy,\ - it will only run testcase") + it will only run testcase") parser.add_argument("-n", "--num", dest="number", - help="how many times you want run the test case") + help="how many times you want to run the tests") options = parser.parse_args() + cfg_file = options.config + tc_path = options.tcpath + load_flag = options.load + num_iter = 1 if options.number is None else int(options.number) - testResultLogFile = None - if options.result is not None: - testResultLogFile = options.result + ''' + Check if the config file is None or not and exit accordingly + ''' + if cfg_file is None: + printAndExit() + return {"cfg_file": cfg_file, + "load_flag": load_flag, + "tc_path": tc_path, + "num_iter": num_iter} - testCaseLogFile = None - if options.testcaselog is not None: - testCaseLogFile = options.testcaselog - deploy = deployDataCenter.deployDataCenters(options.config) - if options.load: - deploy.loadCfg() - else: - deploy.deploy() - iterates = 1 - if options.number is not None: - if options.number == "loop": - iterates = sys.maxint + +def startMarvin(cfg_file, load_flag): + ''' + Initialize the Marvin + ''' + try: + obj_marvininit = MarvinInit(cfg_file, load_flag) + if obj_marvininit.init() == SUCCESS: + testClient = obj_marvininit.getTestClient() + tcRunLogger = obj_marvininit.getLogger() + parsedConfig = obj_marvininit.getParsedConfig() + debugStream = obj_marvininit.getDebugFile() + return {"tc_client": testClient, + "tc_runlogger": tcRunLogger, + "tc_parsedcfg": parsedConfig, + "tc_debugstream": debugStream} else: - try: - iterates = int(options.number) - except: - iterates = 1 - - if options.testCaseFolder is None: - if options.module is None: - parser.print_usage() + print "\nMarvin Initialization Failed" exit(1) + except Exception, e: + print "\n Exception occurred while starting Marvin %s" % str(e) + exit(1) + + +def runTCs(num_iter, inp1, inp2): + ''' + Run Test Cases based upon number of iterations + ''' + n = 0 + while(n < num_iter): + engine = TestCaseExecuteEngine(inp2["tc_client"], + inp2["tc_parsedcfg"], + inp2["tc_runlogger"], + inp2["tc_debugstream"]) + if inp1["tc_file"] is not None: + engine.loadTestsFromFile(inp1["tc_file"]) else: - n = 0 - while(n < iterates): - engine = \ - TestCaseExecuteEngine.TestCaseExecuteEngine( - deploy.testClient, - deploy.getCfg( - ), - testCaseLogFile, - testResultLogFile) - engine.loadTestsFromFile(options.module) - engine.run() - n = n + 1 + engine.loadTestsFromDir(inp1["tc_dir"]) + engine.run() + n = n + 1 + + +def checkTCPath(tc_path): + ''' + Verifies if the tc_path is a folder or file and its existence + ''' + ret = {"tc_file": None, "tc_dir": None} + check = True + if tc_path is None: + printAndExit() else: - n = 0 - while(n < iterates): - engine = TestCaseExecuteEngine.TestCaseExecuteEngine( - deploy.testClient, - deploy.getCfg( - ), - testCaseLogFile, - testResultLogFile) - engine.loadTestsFromDir(options.testCaseFolder) - engine.run() - n = n + 1 + if os.path.isfile(tc_path): + ret["tc_file"] = tc_path + elif os.path.isdir(tc_path): + ret["tc_dir"] = tc_path + else: + check = False + if check is False: + print"\nTC Path is Invalid.So Exiting" + exit(1) + + return ret + +if __name__ == "__main__": + + ''' + 1. Parse and Check + ''' + out1 = parseAndCheck() + print "\nStep1 :Parsing Options And Check Went Fine" + + ''' + 2. Start Marvin + ''' + out2 = startMarvin(out1["cfg_file"], out1["load_flag"]) + print "\nStep2: Marvin Initialization Went Fine" + + ''' + 3. Check TC folder or Module and Path existence + ''' + out3 = checkTCPath(out1["tc_path"]) + print "\nStep3: TC Path Check Went Fine" + + ''' + 4. Run TCs + ''' + runTCs(out1["num_iter"], out3, out2) + print "\nStep4: TC Running Finished" diff --git a/tools/marvin/marvin/integration/lib/base.py b/tools/marvin/marvin/integration/lib/base.py index 86f962ad9ad..b43af33ed9c 100755 --- a/tools/marvin/marvin/integration/lib/base.py +++ b/tools/marvin/marvin/integration/lib/base.py @@ -246,13 +246,17 @@ class VirtualMachine: if zone.securitygroupsenabled: list_security_groups = SecurityGroup.list( apiclient, + account=cmd.account, + domainid=cmd.domainid, securitygroupname="basic_sec_grp" ) if not isinstance(list_security_groups, list): basic_mode_security_group = SecurityGroup.create( apiclient, - {"name": "basic_sec_grp"} + {"name": "basic_sec_grp"}, + cmd.account, + cmd.domainid, ) sec_grp_services = { "protocol": "TCP", diff --git a/tools/marvin/marvin/marvinInit.py b/tools/marvin/marvin/marvinInit.py index 17ee9bd7481..2bbf11903a0 100644 --- a/tools/marvin/marvin/marvinInit.py +++ b/tools/marvin/marvin/marvinInit.py @@ -39,14 +39,16 @@ import sys import time import os import logging +import string +import random class MarvinInit: - def __init__(self, config_file, load_flag): + def __init__(self, config_file, load_flag, log_folder_path=None): self.__configFile = config_file self.__loadFlag = load_flag self.__parsedConfig = None - self.__logFolderPath = None + self.__logFolderPath = log_folder_path self.__tcRunLogger = None self.__testClient = None self.__tcRunDebugFile = None @@ -111,15 +113,21 @@ class MarvinInit: for a given test run are available under a given timestamped folder ''' - log_config = self.__parsedConfig.logger temp_path = "".join(str(time.time()).split(".")) - if log_config is not None: - if log_config.LogFolderPath is not None: - self.logFolderPath = log_config.LogFolderPath + temp_path + if self.__logFolderPath is None: + log_config = self.__parsedConfig.logger + if log_config is not None: + if log_config.LogFolderPath is not None: + self.logFolderPath = log_config.LogFolderPath + '/' + temp_path + else: + self.logFolderPath = temp_path else: self.logFolderPath = temp_path else: - self.logFolderPath = temp_path + self.logFolderPath = self.__logFolderPath + '/' + temp_path + if os.path.exists(self.logFolderPath): + self.logFolderPath = self.logFolderPath \ + + ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(3)) os.makedirs(self.logFolderPath) ''' Log File Paths diff --git a/tools/marvin/marvin/marvinPlugin.py b/tools/marvin/marvin/marvinPlugin.py index 540ede96d63..1353352bbcc 100644 --- a/tools/marvin/marvin/marvinPlugin.py +++ b/tools/marvin/marvin/marvinPlugin.py @@ -68,6 +68,7 @@ class MarvinPlugin(Plugin): self.enabled = True self.configFile = options.config_file self.loadFlag = options.load + self.logFolderPath = options.log_folder_path self.conf = conf ''' Initializes the marvin with required settings @@ -87,6 +88,9 @@ class MarvinPlugin(Plugin): parser.add_option("--load", action="store_true", default=False, dest="load", help="Only load the deployment configuration given") + parser.add_option("--log-folder-path", action="store", default=None, + dest="log_folder_path", + help="Path to the folder where log files will be stored") Plugin.options(self, parser, env) def wantClass(self, cls): @@ -165,7 +169,7 @@ class MarvinPlugin(Plugin): Creates a debugstream for tc debug log ''' try: - obj_marvininit = MarvinInit(self.configFile, self.loadFlag) + obj_marvininit = MarvinInit(self.configFile, self.loadFlag, self.logFolderPath) if obj_marvininit.init() == SUCCESS: self.testClient = obj_marvininit.getTestClient() self.tcRunLogger = obj_marvininit.getLogger() diff --git a/tools/marvin/marvin/TestCaseExecuteEngine.py b/tools/marvin/marvin/tcExecuteEngine.py similarity index 63% rename from tools/marvin/marvin/TestCaseExecuteEngine.py rename to tools/marvin/marvin/tcExecuteEngine.py index f5af1fe2c1e..f959e7e935f 100644 --- a/tools/marvin/marvin/TestCaseExecuteEngine.py +++ b/tools/marvin/marvin/tcExecuteEngine.py @@ -22,14 +22,8 @@ import logging from functools import partial -def testCaseLogger(message, logger=None): - if logger is not None: - logger.debug(message) - - class TestCaseExecuteEngine(object): - def __init__(self, testclient, config, testcaseLogFile=None, - testResultLogFile=None): + def __init__(self, testclient, config, tc_logger=None, debug_stream=None): """ Initialize the testcase execution engine, just the basics here @var testcaseLogFile: client log file @@ -37,29 +31,11 @@ class TestCaseExecuteEngine(object): """ self.testclient = testclient self.config = config - self.logformat =\ - logging.Formatter( - "%(asctime)s - %(levelname)s - %(name)s - %(message)s") + self.tcRunLogger = tc_logger + self.debugStream = debug_stream self.loader = unittest.loader.TestLoader() self.suite = None - if testcaseLogFile is not None: - self.logfile = testcaseLogFile - self.logger = logging.getLogger("TestCaseExecuteEngine") - fh = logging.FileHandler(self.logfile) - fh.setFormatter(self.logformat) - self.logger.addHandler(fh) - self.logger.setLevel(logging.DEBUG) - if testResultLogFile is not None: - ch = logging.StreamHandler() - ch.setLevel(logging.ERROR) - ch.setFormatter(self.logformat) - self.logger.addHandler(ch) - fp = open(testResultLogFile, "w") - self.testResultLogFile = fp - else: - self.testResultLogFile = sys.stdout - def loadTestsFromDir(self, testDirectory): """ Load the test suites from a package with multiple test files """ self.suite = self.loader.discover(testDirectory) @@ -77,18 +53,11 @@ class TestCaseExecuteEngine(object): if isinstance(test, unittest.BaseTestSuite): self.injectTestCase(test) else: - #logger bears the name of the test class - testcaselogger = logging.getLogger("%s" % (test)) - fh = logging.FileHandler(self.logfile) - fh.setFormatter(self.logformat) - testcaselogger.addHandler(fh) - testcaselogger.setLevel(logging.DEBUG) - #inject testclient and logger into each unittest + self.tcRunLogger.name = test.__str__() setattr(test, "testClient", self.testclient) setattr(test, "config", self.config) - setattr(test, "debug", partial(testCaseLogger, - logger=testcaselogger)) + setattr(test, "debug", self.tcRunLogger.debug) setattr(test.__class__, "clstestclient", self.testclient) if hasattr(test, "user"): # attribute when test is entirely executed as user @@ -98,5 +67,5 @@ class TestCaseExecuteEngine(object): def run(self): if self.suite: - unittest.TextTestRunner(stream=self.testResultLogFile, + unittest.TextTestRunner(stream=self.debugStream, verbosity=2).run(self.suite) diff --git a/ui/scripts/cloudStack.js b/ui/scripts/cloudStack.js index 7846ae879ec..5195a7b817f 100644 --- a/ui/scripts/cloudStack.js +++ b/ui/scripts/cloudStack.js @@ -168,8 +168,8 @@ expires: 1 }); - g_KVMsnapshotenabled = json.listcapabilitiesresponse.capability.KVMsnapshotenabled; //boolean - $.cookie('KVMsnapshotenabled', g_KVMsnapshotenabled, { + g_kvmsnapshotenabled = json.listcapabilitiesresponse.capability.kvmsnapshotenabled; //boolean + $.cookie('kvmsnapshotenabled', g_kvmsnapshotenabled, { expires: 1 }); @@ -309,8 +309,8 @@ expires: 1 }); - g_KVMsnapshotenabled = json.listcapabilitiesresponse.capability.KVMsnapshotenabled; //boolean - $.cookie('KVMsnapshotenabled', g_KVMsnapshotenabled, { + g_kvmsnapshotenabled = json.listcapabilitiesresponse.capability.kvmsnapshotenabled; //boolean + $.cookie('kvmsnapshotenabled', g_kvmsnapshotenabled, { expires: 1 }); @@ -386,7 +386,7 @@ g_timezoneoffset = null; g_timezone = null; g_supportELB = null; - g_KVMsnapshotenabled = null; + g_kvmsnapshotenabled = null; g_regionsecondaryenabled = null; g_loginCmdText = null; diff --git a/ui/scripts/instances.js b/ui/scripts/instances.js index 0074335d2d4..2b265292915 100644 --- a/ui/scripts/instances.js +++ b/ui/scripts/instances.js @@ -2184,7 +2184,7 @@ } else if (jsonObj.state == 'Running') { allowedActions.push("stop"); allowedActions.push("restart"); - if (jsonObj.hypervisor != 'KVM' || g_KVMsnapshotenabled == true) + if (jsonObj.hypervisor != 'KVM' || g_kvmsnapshotenabled == true) allowedActions.push("snapshot"); allowedActions.push("destroy"); allowedActions.push("reset"); @@ -2214,7 +2214,7 @@ allowedActions.push("start"); allowedActions.push("destroy"); allowedActions.push("reset"); - if (jsonObj.hypervisor != 'KVM' || g_KVMsnapshotenabled == true) + if (jsonObj.hypervisor != 'KVM' || g_kvmsnapshotenabled == true) allowedActions.push("snapshot"); allowedActions.push("scaleUp"); //when vm is stopped, scaleUp is supported for all hypervisors allowedActions.push("changeAffinity"); diff --git a/ui/scripts/sharedFunctions.js b/ui/scripts/sharedFunctions.js index c9c4a4113cb..1bb408ba9f9 100644 --- a/ui/scripts/sharedFunctions.js +++ b/ui/scripts/sharedFunctions.js @@ -25,7 +25,7 @@ var g_enableLogging = false; var g_timezoneoffset = null; var g_timezone = null; var g_supportELB = null; -var g_KVMsnapshotenabled = null; +var g_kvmsnapshotenabled = null; var g_regionsecondaryenabled = null; var g_userPublicTemplateEnabled = "true"; var g_cloudstackversion = null; diff --git a/ui/scripts/storage.js b/ui/scripts/storage.js index e58f5fdf5ef..86f7aa0ca06 100644 --- a/ui/scripts/storage.js +++ b/ui/scripts/storage.js @@ -70,6 +70,10 @@ add: { label: 'label.add.volume', + preFilter: function(args) { + return !args.context.instances; + }, + messages: { confirm: function(args) { return 'message.add.volume'; @@ -252,6 +256,9 @@ uploadVolume: { isHeader: true, label: 'label.upload.volume', + preFilter: function(args) { + return !args.context.instances; + }, messages: { notification: function() { return 'label.upload.volume'; @@ -1920,7 +1927,7 @@ if (jsonObj.hypervisor != "Ovm" && jsonObj.state == "Ready") { if (jsonObj.hypervisor == 'KVM') { if (jsonObj.vmstate == 'Running') { - if (g_KVMsnapshotenabled == true) { //"kvm.snapshot.enabled" flag should be taken to account only when snapshot is being created for Running vm (CLOUDSTACK-4428) + if (g_kvmsnapshotenabled == true) { //"kvm.snapshot.enabled" flag should be taken to account only when snapshot is being created for Running vm (CLOUDSTACK-4428) allowedActions.push("takeSnapshot"); allowedActions.push("recurringSnapshot"); } diff --git a/ui/scripts/system.js b/ui/scripts/system.js index 3444fa9036f..4a59b182be0 100644 --- a/ui/scripts/system.js +++ b/ui/scripts/system.js @@ -8805,19 +8805,21 @@ var zoneObjs = json.listzonesresponse.zone; if (zoneObjs != null) { for (var i = 0; i < zoneObjs.length; i++) { + var currentPage = 1; $.ajax({ url: createURL('listRouters'), data: { - zoneid: zoneObjs[i].id + zoneid: zoneObjs[i].id, + listAll: true, + page: currentPage, + pagesize: pageSize //global variable }, async: false, success: function(json) { if (json.listroutersresponse.count != undefined) { - zoneObjs[i].routerCount = json.listroutersresponse.count; - - var routerCountFromAllPages = zoneObjs[i].routerCount; - var currentPage = 1; - var routerCountFromFirstPageToCurrentPage = 0; + zoneObjs[i].routerCount = json.listroutersresponse.count; + var routerCountFromAllPages = json.listroutersresponse.count; + var routerCountFromFirstPageToCurrentPage = json.listroutersresponse.router.length; var routerRequiresUpgrade = 0; var callListApiWithPage = function() { $.ajax({ @@ -8825,6 +8827,7 @@ async: false, data: { zoneid: zoneObjs[i].id, + listAll: true, page: currentPage, pagesize: pageSize //global variable }, @@ -8843,7 +8846,10 @@ } }); } - callListApiWithPage(); + if (routerCountFromFirstPageToCurrentPage < routerCountFromAllPages) { + currentPage++; + callListApiWithPage(); + } zoneObjs[i].routerRequiresUpgrade = routerRequiresUpgrade; } else { @@ -9024,26 +9030,29 @@ var podObjs = json.listpodsresponse.pod; if (podObjs != null) { for (var i = 0; i < podObjs.length; i++) { - $.ajax({ + var currentPage = 1; + $.ajax({ url: createURL('listRouters'), data: { - podid: podObjs[i].id + podid: podObjs[i].id, + listAll: true, + page: currentPage, + pagesize: pageSize //global variable }, async: false, success: function (json) { if (json.listroutersresponse.count != undefined) { podObjs[i].routerCount = json.listroutersresponse.count; - - var routerCountFromAllPages = podObjs[i].routerCount; - var currentPage = 1; - var routerCountFromFirstPageToCurrentPage = 0; - var routerRequiresUpgrade = 0; + var routerCountFromAllPages = json.listroutersresponse.count; + var routerCountFromFirstPageToCurrentPage = json.listroutersresponse.router.length; + var routerRequiresUpgrade = 0; var callListApiWithPage = function () { $.ajax({ url: createURL('listRouters'), async: false, data: { podid: podObjs[i].id, + listAll: true, page: currentPage, pagesize: pageSize //global variable }, @@ -9061,8 +9070,11 @@ } } }); - } - callListApiWithPage(); + } + if (routerCountFromFirstPageToCurrentPage < routerCountFromAllPages) { + currentPage++; + callListApiWithPage(); + } podObjs[i].routerRequiresUpgrade = routerRequiresUpgrade; } else { @@ -9245,19 +9257,21 @@ var clusterObjs = json.listclustersresponse.cluster; if (clusterObjs != null) { for (var i = 0; i < clusterObjs.length; i++) { - $.ajax({ + var currentPage = 1; + $.ajax({ url: createURL('listRouters'), data: { - clusterid: clusterObjs[i].id + clusterid: clusterObjs[i].id, + listAll: true, + page: currentPage, + pagesize: pageSize //global variable }, async: false, success: function (json) { if (json.listroutersresponse.count != undefined) { clusterObjs[i].routerCount = json.listroutersresponse.count; - - var routerCountFromAllPages = clusterObjs[i].routerCount; - var currentPage = 1; - var routerCountFromFirstPageToCurrentPage = 0; + var routerCountFromAllPages = json.listroutersresponse.count; + var routerCountFromFirstPageToCurrentPage = json.listroutersresponse.router.length; var routerRequiresUpgrade = 0; var callListApiWithPage = function () { $.ajax({ @@ -9265,6 +9279,7 @@ async: false, data: { clusterid: clusterObjs[i].id, + listAll: true, page: currentPage, pagesize: pageSize //global variable }, @@ -9283,7 +9298,10 @@ } }); } - callListApiWithPage(); + if (routerCountFromFirstPageToCurrentPage < routerCountFromAllPages) { + currentPage++; + callListApiWithPage(); + } clusterObjs[i].routerRequiresUpgrade = routerRequiresUpgrade; } else { @@ -14787,7 +14805,7 @@ var items = []; items.push({ id: "SMB", - description: "SMB/cifs" + description: "SMB/CIFS" }); args.response.success({ data: items @@ -16255,7 +16273,7 @@ description: 'NFS' }, { id: 'SMB', - description: 'SMB/cifs' + description: 'SMB/CIFS' }, { id: 'S3', description: 'S3' @@ -16356,7 +16374,7 @@ $form.find('.form-item[rel=sockettimeout]').css('display', 'inline-block'); $form.find('.form-item[rel=createNfsCache]').find('input').attr('checked', 'checked'); - $form.find('.form-item[rel=createNfsCache]').find('input').attr('disabled', 'disabled'); //Create NFS staging is required for S3 at this moment. So, disallow user to uncheck "Create NFS Secondary Staging" checkbox + //$form.find('.form-item[rel=createNfsCache]').find('input').attr('disabled', 'disabled'); //This checkbox should not be disabled any more because NFS staging (of a zone) might already exist (from "NFS secondary storage => Prepare Object Store Migration => NFS staging") $form.find('.form-item[rel=createNfsCache]').css('display', 'inline-block'); $form.find('.form-item[rel=nfsCacheZoneid]').css('display', 'inline-block'); $form.find('.form-item[rel=nfsCacheNfsServer]').css('display', 'inline-block'); diff --git a/ui/scripts/ui/widgets/listView.js b/ui/scripts/ui/widgets/listView.js index 35aab03292f..79196d02d5f 100644 --- a/ui/scripts/ui/widgets/listView.js +++ b/ui/scripts/ui/widgets/listView.js @@ -1685,7 +1685,7 @@ $.each(listViewData.actions, function(actionName, action) { if (!action.isHeader || ( action.preFilter && !action.preFilter({ - context: listViewData.context ? listViewData.context : cloudStack.context + context: $listView.data('view-args').context ? $listView.data('view-args').context : cloudStack.context }) )) return true; diff --git a/ui/scripts/zoneWizard.js b/ui/scripts/zoneWizard.js index 6c1c814929a..ec9431ddb5d 100755 --- a/ui/scripts/zoneWizard.js +++ b/ui/scripts/zoneWizard.js @@ -1523,7 +1523,7 @@ var items = []; items.push({ id: "SMB", - description: "SMB/cifs" + description: "SMB/CIFS" }); args.response.success({ data: items @@ -1845,7 +1845,7 @@ (2) Provider "SMB" which is handled by UI is not returned from "listStorageProviders&type=image" */ storageproviders.push({ id: 'NFS', description: 'NFS'}); - storageproviders.push({ id: 'SMB', description: 'SMB/cifs'}); + storageproviders.push({ id: 'SMB', description: 'SMB/CIFS'}); storageproviders.push({ id: 'S3', description: 'S3'}); storageproviders.push({ id: 'Swift', description: 'Swift'}); }